refactor daily_plot.py
This commit is contained in:
frederik
+126
-199
@@ -1,217 +1,144 @@
|
|||||||
import sqlite3
|
import argparse
|
||||||
import os
|
import os
|
||||||
|
import sqlite3
|
||||||
|
import textwrap
|
||||||
|
from datetime import datetime
|
||||||
|
from time import sleep
|
||||||
|
|
||||||
|
import matplotlib.font_manager as font_manager
|
||||||
|
import matplotlib.pyplot as plt
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
import seaborn as sns
|
import seaborn as sns
|
||||||
import matplotlib.pyplot as plt
|
|
||||||
from matplotlib.colors import LogNorm
|
|
||||||
from datetime import datetime
|
|
||||||
import textwrap
|
|
||||||
import matplotlib.font_manager as font_manager
|
|
||||||
from matplotlib import rcParams
|
from matplotlib import rcParams
|
||||||
|
from matplotlib.colors import LogNorm
|
||||||
|
|
||||||
userDir = os.path.expanduser('~')
|
from utils.helpers import DB_PATH
|
||||||
conn = sqlite3.connect(userDir + '/BirdNET-Pi/scripts/birds.db')
|
|
||||||
df = pd.read_sql_query("SELECT * from detections", conn)
|
|
||||||
cursor = conn.cursor()
|
|
||||||
cursor.execute('SELECT * FROM detections WHERE Date = DATE(\'now\', \'localtime\')')
|
|
||||||
|
|
||||||
table_rows = cursor.fetchall()
|
|
||||||
|
|
||||||
# df=pd.DataFrame(table_rows)
|
|
||||||
|
|
||||||
# Convert Date and Time Fields to Panda's format
|
|
||||||
df['Date'] = pd.to_datetime(df['Date'])
|
|
||||||
df['Time'] = pd.to_datetime(df['Time'], unit='ns')
|
|
||||||
|
|
||||||
|
|
||||||
# Add round hours to dataframe
|
def get_data():
|
||||||
df['Hour of Day'] = [r.hour for r in df.Time]
|
conn = sqlite3.connect(DB_PATH)
|
||||||
|
now = datetime.now()
|
||||||
|
df = pd.read_sql_query("SELECT * from detections WHERE Date = DATE('now', 'localtime')", conn)
|
||||||
|
|
||||||
# Create separate dataframes for separate locations
|
# Convert Date and Time Fields to Panda's format
|
||||||
df_plt = df # Default to use the whole Dbase
|
df['Date'] = pd.to_datetime(df['Date'])
|
||||||
|
df['Time'] = pd.to_datetime(df['Time'], unit='ns')
|
||||||
|
|
||||||
# Add every font at the specified location
|
# Add round hours to dataframe
|
||||||
font_dir = [userDir + '/BirdNET-Pi/homepage/static']
|
df['Hour of Day'] = [r.hour for r in df.Time]
|
||||||
for font in font_manager.findSystemFonts(font_dir):
|
|
||||||
font_manager.fontManager.addfont(font)
|
|
||||||
|
|
||||||
# Set font family globally
|
return df, now
|
||||||
rcParams['font.family'] = 'Roboto Flex'
|
|
||||||
|
|
||||||
# Get todays readings
|
|
||||||
now = datetime.now()
|
|
||||||
df_plt_today = df_plt[df_plt['Date'] == now.strftime("%Y-%m-%d")]
|
|
||||||
|
|
||||||
# Set number of species to report
|
|
||||||
readings = 10
|
|
||||||
|
|
||||||
plt_top10_today = (df_plt_today['Com_Name'].value_counts()[:readings])
|
|
||||||
df_plt_top10_today = df_plt_today[df_plt_today.Com_Name.isin(plt_top10_today.index)]
|
|
||||||
|
|
||||||
if df_plt_top10_today.empty:
|
|
||||||
exit(0)
|
|
||||||
|
|
||||||
# Set Palette for graphics
|
|
||||||
pal = "Greens"
|
|
||||||
|
|
||||||
# Set up plot axes and titles
|
|
||||||
f, axs = plt.subplots(1, 2, figsize=(10, 4), gridspec_kw=dict(width_ratios=[3, 6]), facecolor='#77C487')
|
|
||||||
plt.subplots_adjust(left=None, bottom=None, right=None, top=None, wspace=0, hspace=0)
|
|
||||||
|
|
||||||
# generate y-axis order for all figures based on frequency
|
|
||||||
freq_order = pd.value_counts(df_plt_top10_today['Com_Name']).iloc[:readings].index
|
|
||||||
|
|
||||||
# make color for max confidence --> this groups by name and calculates max conf
|
|
||||||
confmax = df_plt_top10_today.groupby('Com_Name')['Confidence'].max()
|
|
||||||
# reorder confmax to detection frequency order
|
|
||||||
confmax = confmax.reindex(freq_order)
|
|
||||||
|
|
||||||
# norm values for color palette
|
|
||||||
norm = plt.Normalize(confmax.values.min(), confmax.values.max())
|
|
||||||
colors = plt.cm.Greens(norm(confmax))
|
|
||||||
|
|
||||||
# Generate frequency plot
|
|
||||||
plot = sns.countplot(y='Com_Name', data=df_plt_top10_today, palette=colors, order=freq_order, ax=axs[0])
|
|
||||||
|
|
||||||
|
|
||||||
# Try plot grid lines between bars - problem at the moment plots grid lines on bars - want between bars
|
def create_plot(df_plt_today, now, is_top):
|
||||||
z = plot.get_ymajorticklabels()
|
readings = 10
|
||||||
plot.set_yticklabels(['\n'.join(textwrap.wrap(ticklabel.get_text(), 15)) for ticklabel in plot.get_yticklabels()], fontsize=10)
|
if is_top:
|
||||||
plot.set(ylabel=None)
|
plt_selection_today = (df_plt_today['Com_Name'].value_counts()[:readings])
|
||||||
plot.set(xlabel="Detections")
|
else:
|
||||||
|
plt_selection_today = (df_plt_today['Com_Name'].value_counts()[-readings:])
|
||||||
|
|
||||||
|
df_plt_selection_today = df_plt_today[df_plt_today.Com_Name.isin(plt_selection_today.index)]
|
||||||
|
|
||||||
|
# Set up plot axes and titles
|
||||||
|
f, axs = plt.subplots(1, 2, figsize=(10, 4), gridspec_kw=dict(width_ratios=[3, 6]), facecolor='#77C487')
|
||||||
|
plt.subplots_adjust(left=None, bottom=None, right=None, top=None, wspace=0, hspace=0)
|
||||||
|
|
||||||
|
# generate y-axis order for all figures based on frequency
|
||||||
|
freq_order = pd.value_counts(df_plt_selection_today['Com_Name']).index
|
||||||
|
|
||||||
|
# make color for max confidence --> this groups by name and calculates max conf
|
||||||
|
confmax = df_plt_selection_today.groupby('Com_Name')['Confidence'].max()
|
||||||
|
# reorder confmax to detection frequency order
|
||||||
|
confmax = confmax.reindex(freq_order)
|
||||||
|
|
||||||
|
# norm values for color palette
|
||||||
|
norm = plt.Normalize(confmax.values.min(), confmax.values.max())
|
||||||
|
if is_top:
|
||||||
|
# Set Palette for graphics
|
||||||
|
pal = "Greens"
|
||||||
|
colors = plt.cm.Greens(norm(confmax))
|
||||||
|
plot_type = "Top"
|
||||||
|
name = "Combo"
|
||||||
|
else:
|
||||||
|
# Set Palette for graphics
|
||||||
|
pal = "Reds"
|
||||||
|
colors = plt.cm.Reds(norm(confmax))
|
||||||
|
plot_type = "Bottom"
|
||||||
|
name = "Combo2"
|
||||||
|
|
||||||
|
# Generate frequency plot
|
||||||
|
plot = sns.countplot(y='Com_Name', data=df_plt_selection_today, palette=colors, order=freq_order, ax=axs[0])
|
||||||
|
|
||||||
|
# Try plot grid lines between bars - problem at the moment plots grid lines on bars - want between bars
|
||||||
|
yticklabels = ['\n'.join(textwrap.wrap(ticklabel.get_text(), 15)) for ticklabel in plot.get_yticklabels()]
|
||||||
|
plot.set_yticklabels(yticklabels, fontsize=10)
|
||||||
|
plot.set(ylabel=None)
|
||||||
|
plot.set(xlabel="Detections")
|
||||||
|
|
||||||
|
# Generate crosstab matrix for heatmap plot
|
||||||
|
heat = pd.crosstab(df_plt_selection_today['Com_Name'], df_plt_selection_today['Hour of Day'])
|
||||||
|
|
||||||
|
# Order heatmap Birds by frequency of occurrance
|
||||||
|
heat.index = pd.CategoricalIndex(heat.index, categories=freq_order)
|
||||||
|
heat.sort_index(level=0, inplace=True)
|
||||||
|
|
||||||
|
hours_in_day = pd.Series(data=range(0, 24))
|
||||||
|
heat_frame = pd.DataFrame(data=0, index=heat.index, columns=hours_in_day)
|
||||||
|
heat = (heat+heat_frame).fillna(0)
|
||||||
|
|
||||||
|
# Generatie heatmap plot
|
||||||
|
plot = sns.heatmap(heat, norm=LogNorm(), annot=True, annot_kws={"fontsize": 7}, fmt="g", cmap=pal, square=False,
|
||||||
|
cbar=False, linewidths=0.5, linecolor="Grey", ax=axs[1], yticklabels=False)
|
||||||
|
|
||||||
|
# Set color and weight of tick label for current hour
|
||||||
|
for label in plot.get_xticklabels():
|
||||||
|
if int(label.get_text()) == now.hour:
|
||||||
|
label.set_color('yellow')
|
||||||
|
|
||||||
|
plot.set_xticklabels(plot.get_xticklabels(), rotation=0, size=7)
|
||||||
|
|
||||||
|
# Set heatmap border
|
||||||
|
for _, spine in plot.spines.items():
|
||||||
|
spine.set_visible(True)
|
||||||
|
|
||||||
|
plot.set(ylabel=None)
|
||||||
|
plot.set(xlabel="Hour of Day")
|
||||||
|
# Set combined plot layout and titles
|
||||||
|
f.subplots_adjust(top=0.9)
|
||||||
|
plt.suptitle(f"{plot_type} {readings} Last Updated: {now.strftime('%Y-%m-%d %H:%M')}")
|
||||||
|
|
||||||
|
# Save combined plot
|
||||||
|
save_name = os.path.expanduser(f"~/BirdSongs/Extracted/Charts/{name}-{now.strftime('%Y-%m-%d')}.png")
|
||||||
|
plt.savefig(save_name)
|
||||||
|
plt.show()
|
||||||
|
plt.close()
|
||||||
|
|
||||||
|
|
||||||
# Generate crosstab matrix for heatmap plot
|
def load_fonts():
|
||||||
heat = pd.crosstab(df_plt_top10_today['Com_Name'], df_plt_top10_today['Hour of Day'])
|
# Add every font at the specified location
|
||||||
|
font_dir = [os.path.expanduser('~/BirdNET-Pi/homepage/static')]
|
||||||
# Order heatmap Birds by frequency of occurrance
|
for font in font_manager.findSystemFonts(font_dir, fontext='ttf'):
|
||||||
heat.index = pd.CategoricalIndex(heat.index, categories=freq_order)
|
font_manager.fontManager.addfont(font)
|
||||||
heat.sort_index(level=0, inplace=True)
|
# Set font family globally
|
||||||
|
rcParams['font.family'] = 'Roboto Flex'
|
||||||
hours_in_day = pd.Series(data=range(0, 24))
|
|
||||||
heat_frame = pd.DataFrame(data=0, index=heat.index, columns=hours_in_day)
|
|
||||||
heat = (heat + heat_frame).fillna(0)
|
|
||||||
|
|
||||||
# Get current hour
|
|
||||||
current_hour = now.hour
|
|
||||||
|
|
||||||
# Generate heatmap plot
|
|
||||||
plot = sns.heatmap(
|
|
||||||
heat,
|
|
||||||
norm=LogNorm(),
|
|
||||||
annot=True,
|
|
||||||
annot_kws={"fontsize": 7},
|
|
||||||
fmt="g",
|
|
||||||
cmap=pal,
|
|
||||||
square=False,
|
|
||||||
cbar=False,
|
|
||||||
linewidths=0.5,
|
|
||||||
linecolor="Grey",
|
|
||||||
ax=axs[1],
|
|
||||||
yticklabels=False
|
|
||||||
)
|
|
||||||
|
|
||||||
# Set color and weight of tick label for current hour
|
|
||||||
for label in plot.get_xticklabels():
|
|
||||||
if int(label.get_text()) == current_hour:
|
|
||||||
label.set_color('yellow')
|
|
||||||
|
|
||||||
plot.set_xticklabels(plot.get_xticklabels(), rotation=0, size=7)
|
|
||||||
|
|
||||||
|
|
||||||
|
def main(daemon, sleep_m):
|
||||||
# Set heatmap border
|
load_fonts()
|
||||||
for _, spine in plot.spines.items():
|
while True:
|
||||||
spine.set_visible(True)
|
data, time = get_data()
|
||||||
|
if not data.empty:
|
||||||
plot.set(ylabel=None)
|
create_plot(data, time, is_top=True)
|
||||||
plot.set(xlabel="Hour of Day")
|
create_plot(data, time, is_top=False)
|
||||||
# Set combined plot layout and titles
|
if daemon:
|
||||||
f.subplots_adjust(top=0.9)
|
sleep(60 * sleep_m)
|
||||||
plt.suptitle("Top 10 Last Updated: " + str(now.strftime("%Y-%m-%d %H:%M")))
|
else:
|
||||||
|
break
|
||||||
# Save combined plot
|
|
||||||
userDir = os.path.expanduser('~')
|
|
||||||
savename = userDir + '/BirdSongs/Extracted/Charts/Combo-' + str(now.strftime("%Y-%m-%d")) + '.png'
|
|
||||||
plt.savefig(savename)
|
|
||||||
plt.show()
|
|
||||||
plt.close()
|
|
||||||
|
|
||||||
|
|
||||||
# Get Bottom detection frequency
|
if __name__ == '__main__':
|
||||||
plt_Bot10_today = (df_plt_today['Com_Name'].value_counts()[-readings:])
|
parser = argparse.ArgumentParser()
|
||||||
df_plt_Bot10_today = df_plt_today[df_plt_today.Com_Name.isin(plt_Bot10_today.index)]
|
parser.add_argument('--daemon', action='store_true')
|
||||||
|
parser.add_argument('--sleep', default=2, type=int, help='Time between runs (minutes)')
|
||||||
# Set Palette for graphics
|
args = parser.parse_args()
|
||||||
pal = "Reds"
|
main(args.daemon, args.sleep)
|
||||||
|
|
||||||
# Set up plot axes and titles
|
|
||||||
|
|
||||||
f, axs = plt.subplots(1, 2, figsize=(10, 4), gridspec_kw=dict(width_ratios=[3, 6]), facecolor='#77C487')
|
|
||||||
plt.subplots_adjust(left=None, bottom=None, right=None, top=None, wspace=0, hspace=0)
|
|
||||||
|
|
||||||
# generate y-axis order for all figures based on frequency
|
|
||||||
freq_order = pd.value_counts(df_plt_Bot10_today['Com_Name']).iloc[-readings:].index
|
|
||||||
|
|
||||||
# make color for max confidence --> this groups by name and calculates max conf
|
|
||||||
confmax = df_plt_Bot10_today.groupby('Com_Name')['Confidence'].max()
|
|
||||||
confmax = confmax.reindex(freq_order)
|
|
||||||
# probably wrong order . . . how to sort by no. of detections ?
|
|
||||||
# norm values for color palette
|
|
||||||
norm = plt.Normalize(confmax.values.min(), confmax.values.max())
|
|
||||||
colors = plt.cm.Reds(norm(confmax))
|
|
||||||
|
|
||||||
# Generate frequency plot
|
|
||||||
plot = sns.countplot(y='Com_Name', data=df_plt_Bot10_today, palette=colors, order=freq_order, ax=axs[0])
|
|
||||||
|
|
||||||
|
|
||||||
# Try plot grid lines between bars - problem at the moment plots grid lines on bars - want between bars
|
|
||||||
z = plot.get_ymajorticklabels()
|
|
||||||
plot.set_yticklabels(['\n'.join(textwrap.wrap(ticklabel.get_text(), 15)) for ticklabel in plot.get_yticklabels()], fontsize=10)
|
|
||||||
plot.set(ylabel=None)
|
|
||||||
plot.set(xlabel="Detections")
|
|
||||||
|
|
||||||
# Generate crosstab matrix for heatmap plot
|
|
||||||
|
|
||||||
heat = pd.crosstab(df_plt_Bot10_today['Com_Name'], df_plt_Bot10_today['Hour of Day'])
|
|
||||||
# Order heatmap Birds by frequency of occurrance
|
|
||||||
heat.index = pd.CategoricalIndex(heat.index, categories=freq_order)
|
|
||||||
heat.sort_index(level=0, inplace=True)
|
|
||||||
|
|
||||||
|
|
||||||
hours_in_day = pd.Series(data=range(0, 24))
|
|
||||||
heat_frame = pd.DataFrame(data=0, index=heat.index, columns=hours_in_day)
|
|
||||||
heat = (heat + heat_frame).fillna(0)
|
|
||||||
|
|
||||||
# Generatie heatmap plot
|
|
||||||
plot = sns.heatmap(
|
|
||||||
heat,
|
|
||||||
norm=LogNorm(),
|
|
||||||
annot=True,
|
|
||||||
fmt="g",
|
|
||||||
annot_kws={
|
|
||||||
"fontsize": 7},
|
|
||||||
cmap=pal,
|
|
||||||
square=False,
|
|
||||||
cbar=False,
|
|
||||||
linewidths=0.5,
|
|
||||||
linecolor="Grey",
|
|
||||||
ax=axs[1],
|
|
||||||
yticklabels=False)
|
|
||||||
plot.set_xticklabels(plot.get_xticklabels(), rotation=0, size=7)
|
|
||||||
|
|
||||||
# Set heatmap border
|
|
||||||
for _, spine in plot.spines.items():
|
|
||||||
spine.set_visible(True)
|
|
||||||
|
|
||||||
plot.set(ylabel=None)
|
|
||||||
plot.set(xlabel="Hour of Day")
|
|
||||||
# Set combined plot layout and titles
|
|
||||||
f.subplots_adjust(top=0.9)
|
|
||||||
plt.suptitle("Bottom 10 Last Updated: " + str(now.strftime("%Y-%m-%d %H:%M")))
|
|
||||||
|
|
||||||
# Save combined plot
|
|
||||||
savename = userDir + '/BirdSongs/Extracted/Charts/Combo2-' + str(now.strftime("%Y-%m-%d")) + '.png'
|
|
||||||
plt.savefig(savename)
|
|
||||||
plt.show()
|
|
||||||
plt.close()
|
|
||||||
|
|||||||
Reference in New Issue
Block a user