refactor daily_plot.py

scripts/daily_plot.py

@@ -1,203 +1,100 @@
-import sqlite3
+import argparse
 import os
+import sqlite3
+import textwrap
+from datetime import datetime
+from time import sleep
+
+import matplotlib.font_manager as font_manager
+import matplotlib.pyplot as plt
 import pandas as pd
 import seaborn as sns
-import matplotlib.pyplot as plt
-from matplotlib.colors import LogNorm
-from datetime import datetime
-import textwrap
-import matplotlib.font_manager as font_manager
 from matplotlib import rcParams
+from matplotlib.colors import LogNorm
 
-userDir = os.path.expanduser('~')
+from utils.helpers import DB_PATH
-conn = sqlite3.connect(userDir + '/BirdNET-Pi/scripts/birds.db')
-df = pd.read_sql_query("SELECT * from detections", conn)
-cursor = conn.cursor()
-cursor.execute('SELECT * FROM detections WHERE Date = DATE(\'now\', \'localtime\')')
 
-table_rows = cursor.fetchall()
 
-# df=pd.DataFrame(table_rows)
+def get_data():
+    conn = sqlite3.connect(DB_PATH)
+    now = datetime.now()
+    df = pd.read_sql_query("SELECT * from detections WHERE Date = DATE('now', 'localtime')", conn)
 
     # Convert Date and Time Fields to Panda's format
     df['Date'] = pd.to_datetime(df['Date'])
     df['Time'] = pd.to_datetime(df['Time'], unit='ns')
 
-
     # Add round hours to dataframe
     df['Hour of Day'] = [r.hour for r in df.Time]
 
-# Create separate dataframes for separate locations
+    return df, now
-df_plt = df  # Default to use the whole Dbase
 
-# Add every font at the specified location
-font_dir = [userDir + '/BirdNET-Pi/homepage/static']
-for font in font_manager.findSystemFonts(font_dir):
-    font_manager.fontManager.addfont(font)
 
-# Set font family globally
+def create_plot(df_plt_today, now, is_top):
-rcParams['font.family'] = 'Roboto Flex'
-
-# Get todays readings
-now = datetime.now()
-df_plt_today = df_plt[df_plt['Date'] == now.strftime("%Y-%m-%d")]
-
-# Set number of species to report
     readings = 10
+    if is_top:
+        plt_selection_today = (df_plt_today['Com_Name'].value_counts()[:readings])
+    else:
+        plt_selection_today = (df_plt_today['Com_Name'].value_counts()[-readings:])
 
-plt_top10_today = (df_plt_today['Com_Name'].value_counts()[:readings])
+    df_plt_selection_today = df_plt_today[df_plt_today.Com_Name.isin(plt_selection_today.index)]
-df_plt_top10_today = df_plt_today[df_plt_today.Com_Name.isin(plt_top10_today.index)]
-
-if df_plt_top10_today.empty:
-    exit(0)
-
-# Set Palette for graphics
-pal = "Greens"
 
     # Set up plot axes and titles
     f, axs = plt.subplots(1, 2, figsize=(10, 4), gridspec_kw=dict(width_ratios=[3, 6]), facecolor='#77C487')
     plt.subplots_adjust(left=None, bottom=None, right=None, top=None, wspace=0, hspace=0)
 
     # generate y-axis order for all figures based on frequency
-freq_order = pd.value_counts(df_plt_top10_today['Com_Name']).iloc[:readings].index
+    freq_order = pd.value_counts(df_plt_selection_today['Com_Name']).index
 
     # make color for max confidence --> this groups by name and calculates max conf
-confmax = df_plt_top10_today.groupby('Com_Name')['Confidence'].max()
+    confmax = df_plt_selection_today.groupby('Com_Name')['Confidence'].max()
     # reorder confmax to detection frequency order
     confmax = confmax.reindex(freq_order)
 
     # norm values for color palette
     norm = plt.Normalize(confmax.values.min(), confmax.values.max())
+    if is_top:
+        # Set Palette for graphics
+        pal = "Greens"
         colors = plt.cm.Greens(norm(confmax))
-
+        plot_type = "Top"
-# Generate frequency plot
+        name = "Combo"
-plot = sns.countplot(y='Com_Name', data=df_plt_top10_today, palette=colors, order=freq_order, ax=axs[0])
+    else:
-
-
-# Try plot grid lines between bars - problem at the moment plots grid lines on bars - want between bars
-z = plot.get_ymajorticklabels()
-plot.set_yticklabels(['\n'.join(textwrap.wrap(ticklabel.get_text(), 15)) for ticklabel in plot.get_yticklabels()], fontsize=10)
-plot.set(ylabel=None)
-plot.set(xlabel="Detections")
-
-
-# Generate crosstab matrix for heatmap plot
-heat = pd.crosstab(df_plt_top10_today['Com_Name'], df_plt_top10_today['Hour of Day'])
-
-# Order heatmap Birds by frequency of occurrance
-heat.index = pd.CategoricalIndex(heat.index, categories=freq_order)
-heat.sort_index(level=0, inplace=True)
-
-hours_in_day = pd.Series(data=range(0, 24))
-heat_frame = pd.DataFrame(data=0, index=heat.index, columns=hours_in_day)
-heat = (heat + heat_frame).fillna(0)
-
-# Get current hour
-current_hour = now.hour
-
-# Generate heatmap plot
-plot = sns.heatmap(
-    heat,
-    norm=LogNorm(),
-    annot=True,
-    annot_kws={"fontsize": 7},
-    fmt="g",
-    cmap=pal,
-    square=False,
-    cbar=False,
-    linewidths=0.5,
-    linecolor="Grey",
-    ax=axs[1],
-    yticklabels=False
-)
-
-# Set color and weight of tick label for current hour
-for label in plot.get_xticklabels():
-    if int(label.get_text()) == current_hour:
-        label.set_color('yellow')
-
-plot.set_xticklabels(plot.get_xticklabels(), rotation=0, size=7)
-
-
-
-# Set heatmap border
-for _, spine in plot.spines.items():
-    spine.set_visible(True)
-
-plot.set(ylabel=None)
-plot.set(xlabel="Hour of Day")
-# Set combined plot layout and titles
-f.subplots_adjust(top=0.9)
-plt.suptitle("Top 10 Last Updated: " + str(now.strftime("%Y-%m-%d %H:%M")))
-
-# Save combined plot
-userDir = os.path.expanduser('~')
-savename = userDir + '/BirdSongs/Extracted/Charts/Combo-' + str(now.strftime("%Y-%m-%d")) + '.png'
-plt.savefig(savename)
-plt.show()
-plt.close()
-
-
-# Get Bottom detection frequency
-plt_Bot10_today = (df_plt_today['Com_Name'].value_counts()[-readings:])
-df_plt_Bot10_today = df_plt_today[df_plt_today.Com_Name.isin(plt_Bot10_today.index)]
-
         # Set Palette for graphics
         pal = "Reds"
-
-# Set up plot axes and titles
-
-f, axs = plt.subplots(1, 2, figsize=(10, 4), gridspec_kw=dict(width_ratios=[3, 6]), facecolor='#77C487')
-plt.subplots_adjust(left=None, bottom=None, right=None, top=None, wspace=0, hspace=0)
-
-# generate y-axis order for all figures based on frequency
-freq_order = pd.value_counts(df_plt_Bot10_today['Com_Name']).iloc[-readings:].index
-
-# make color for max confidence --> this groups by name and calculates max conf
-confmax = df_plt_Bot10_today.groupby('Com_Name')['Confidence'].max()
-confmax = confmax.reindex(freq_order)
-# probably wrong order . . . how to sort by no. of detections ?
-# norm values for color palette
-norm = plt.Normalize(confmax.values.min(), confmax.values.max())
         colors = plt.cm.Reds(norm(confmax))
+        plot_type = "Bottom"
+        name = "Combo2"
 
     # Generate frequency plot
-plot = sns.countplot(y='Com_Name', data=df_plt_Bot10_today, palette=colors, order=freq_order, ax=axs[0])
+    plot = sns.countplot(y='Com_Name', data=df_plt_selection_today, palette=colors,  order=freq_order, ax=axs[0])
-
 
     # Try plot grid lines between bars - problem at the moment plots grid lines on bars - want between bars
-z = plot.get_ymajorticklabels()
+    yticklabels = ['\n'.join(textwrap.wrap(ticklabel.get_text(), 15)) for ticklabel in plot.get_yticklabels()]
-plot.set_yticklabels(['\n'.join(textwrap.wrap(ticklabel.get_text(), 15)) for ticklabel in plot.get_yticklabels()], fontsize=10)
+    plot.set_yticklabels(yticklabels, fontsize=10)
     plot.set(ylabel=None)
     plot.set(xlabel="Detections")
 
     # Generate crosstab matrix for heatmap plot
+    heat = pd.crosstab(df_plt_selection_today['Com_Name'], df_plt_selection_today['Hour of Day'])
 
-heat = pd.crosstab(df_plt_Bot10_today['Com_Name'], df_plt_Bot10_today['Hour of Day'])
     # Order heatmap Birds by frequency of occurrance
     heat.index = pd.CategoricalIndex(heat.index, categories=freq_order)
     heat.sort_index(level=0, inplace=True)
 
-
     hours_in_day = pd.Series(data=range(0, 24))
     heat_frame = pd.DataFrame(data=0, index=heat.index, columns=hours_in_day)
     heat = (heat+heat_frame).fillna(0)
 
     # Generatie heatmap plot
-plot = sns.heatmap(
+    plot = sns.heatmap(heat, norm=LogNorm(),  annot=True,  annot_kws={"fontsize": 7}, fmt="g", cmap=pal, square=False,
-    heat,
+                       cbar=False, linewidths=0.5, linecolor="Grey", ax=axs[1], yticklabels=False)
-    norm=LogNorm(),
+
-    annot=True,
+    # Set color and weight of tick label for current hour
-    fmt="g",
+    for label in plot.get_xticklabels():
-    annot_kws={
+        if int(label.get_text()) == now.hour:
-        "fontsize": 7},
+            label.set_color('yellow')
-    cmap=pal,
+
-    square=False,
-    cbar=False,
-    linewidths=0.5,
-    linecolor="Grey",
-    ax=axs[1],
-    yticklabels=False)
     plot.set_xticklabels(plot.get_xticklabels(), rotation=0, size=7)
 
     # Set heatmap border
@@ -208,10 +105,40 @@ plot.set(ylabel=None)
     plot.set(xlabel="Hour of Day")
     # Set combined plot layout and titles
     f.subplots_adjust(top=0.9)
-plt.suptitle("Bottom 10 Last Updated: " + str(now.strftime("%Y-%m-%d %H:%M")))
+    plt.suptitle(f"{plot_type} {readings} Last Updated: {now.strftime('%Y-%m-%d %H:%M')}")
 
     # Save combined plot
-savename = userDir + '/BirdSongs/Extracted/Charts/Combo2-' + str(now.strftime("%Y-%m-%d")) + '.png'
+    save_name = os.path.expanduser(f"~/BirdSongs/Extracted/Charts/{name}-{now.strftime('%Y-%m-%d')}.png")
-plt.savefig(savename)
+    plt.savefig(save_name)
     plt.show()
     plt.close()
+
+
+def load_fonts():
+    # Add every font at the specified location
+    font_dir = [os.path.expanduser('~/BirdNET-Pi/homepage/static')]
+    for font in font_manager.findSystemFonts(font_dir, fontext='ttf'):
+        font_manager.fontManager.addfont(font)
+    # Set font family globally
+    rcParams['font.family'] = 'Roboto Flex'
+
+
+def main(daemon, sleep_m):
+    load_fonts()
+    while True:
+        data, time = get_data()
+        if not data.empty:
+            create_plot(data, time, is_top=True)
+            create_plot(data, time, is_top=False)
+        if daemon:
+            sleep(60 * sleep_m)
+        else:
+            break
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--daemon', action='store_true')
+    parser.add_argument('--sleep', default=2, type=int, help='Time between runs (minutes)')
+    args = parser.parse_args()
+    main(args.daemon, args.sleep)