diff --git a/scripts/daily_plot.py b/scripts/daily_plot.py index 675ba0a..8369eb1 100755 --- a/scripts/daily_plot.py +++ b/scripts/daily_plot.py @@ -1,4 +1,8 @@ #!/home/pi/BirdNET-Pi/birdnet/bin/python3 + +import mysql.connector as sql +import os + import pandas as pd import seaborn as sns # import numpy as np @@ -7,82 +11,73 @@ from matplotlib.colors import LogNorm from datetime import datetime import textwrap +BIRD_DB_PWD=os.getenv('DB_PWD') +print(BIRD_DB_PWD) -#Read database into Pandas dataframe -df = pd.read_csv('~/BirdNET-Pi/BirdDB.txt', sep=';') +db_connection = sql.connect(host='localhost', + database='birds', + user='birder', + password='forms') + #password = BIRD_DB_PASSWORD) + +db_cursor=db_connection.cursor(dictionary=True) + +db_cursor.execute('SELECT * FROM detections') + +table_rows = db_cursor.fetchall() + +df=pd.DataFrame(table_rows) #Convert Date and Time Fields to Panda's format df['Date']=pd.to_datetime(df['Date']) -df['Time']=pd.to_datetime(df['Time']) +df['Time']=pd.to_datetime(df['Time'], unit='ns') + #Add round hours to dataframe df['Hour of Day'] = [r.hour for r in df.Time] #Create separate dataframes for separate locations -df_jhb=df[df.Lat > -32] -df_ec = df[df.Lat < -32] +df_plt=df #Default to use the whole Dbase -#Get todays readings for Joburg +#Get todays readings now = datetime.now() -df_jhb_today = df_jhb[df_jhb['Date']==now.strftime("%Y-%m-%d")] +df_plt_today = df_plt[df_plt['Date']==now.strftime("%Y-%m-%d")] -# Definition to start getting top N detections - work in process -def filter_by_freq(df: pd.DataFrame, column: str, min_freq: int) -> pd.DataFrame: - """Filters the DataFrame based on the value frequency in the specified column. +#Set number of species to report +readings=10 - :param df: DataFrame to be filtered. - :param column: Column name that should be frequency filtered. - :param min_freq: Minimal value frequency for the row to be accepted. - :return: Frequency filtered DataFrame. - """ - # Frequencies of each value in the column. - freq = df[column].value_counts() - # Select frequent values. Value is in the index. - frequent_values = freq[freq >= min_freq].index - # Return only rows with value frequency above threshold. - return df[df[column].isin(frequent_values)] - -#Get top readings today -min_valuecounts = 2 - - -jhb_gt_min = filter_by_freq (df_jhb_today,'Com_Name', min_valuecounts) - -jhb_gt_min_counts = jhb_gt_min['Com_Name'].value_counts() -print(jhb_gt_min_counts) - - -jhb_top10_today = (df_jhb_today['Com_Name'].value_counts()[:10]) -df_jhb_top10_today = df_jhb_today[df_jhb_today.Com_Name.isin(jhb_top10_today.index)] - -#Get bottom 10 today -jhb_bot10_today=(df_jhb_today['Com_Name'].value_counts()[-10:]) -df_jhb_bot10_today = df_jhb_today[df_jhb_today.Com_Name.isin(jhb_bot10_today.index)] +plt_top10_today = (df_plt_today['Com_Name'].value_counts()[:readings]) +df_plt_top10_today = df_plt_today[df_plt_today.Com_Name.isin(plt_top10_today.index)] #Set Palette for graphics pal = "Greens" #Set up plot axes and titles -f, axs = plt.subplots(1, 2, figsize = (10, 4), gridspec_kw=dict(width_ratios=[3, 5])) -plt.subplots_adjust(left=None, bottom=None, right=None, top=None, wspace=0, hspace=None) +f, axs = plt.subplots(1, 3, figsize = (10, 4), gridspec_kw=dict(width_ratios=[3, 2, 5])) +plt.subplots_adjust(left=None, bottom=None, right=None, top=None, wspace=0, hspace=0) #Generate frequency plot -plot=sns.countplot(y='Com_Name', data = df_jhb_top10_today, palette = pal+"_r", order=pd.value_counts(df_jhb_top10_today['Com_Name']).iloc[:20].index, ax=axs[0]) +plot=sns.countplot(y='Com_Name', data = df_plt_top10_today, palette = pal+"_r", order=pd.value_counts(df_plt_top10_today['Com_Name']).iloc[:readings].index, ax=axs[0]) #Try plot grid lines between bars - problem at the moment plots grid lines on bars - want between bars # plot.grid(True, axis='y') - -plot.set_yticklabels(['\n'.join(textwrap.wrap(ticklabel.get_text(),15)) for ticklabel in plot.get_yticklabels()]) +z=plot.get_ymajorticklabels() +plot.set_yticklabels(['\n'.join(textwrap.wrap(ticklabel.get_text(),15)) for ticklabel in plot.get_yticklabels()], fontsize = 10) plot.set(ylabel=None) plot.set(xlabel="Detections") +huw=df_plt_top10_today.groupby('Com_Name')['Confidence'].mean() +plot = sns.boxenplot(x=df_plt_top10_today['Confidence']*100,color='Green', y=df_plt_top10_today['Com_Name'], ax=axs[1],order=pd.value_counts(df_plt_top10_today['Com_Name']).iloc[:readings].index) +plot.set(xlabel="Confidence", ylabel=None,yticklabels=[]) + + #Generate crosstab matrix for heatmap plot -heat = pd.crosstab(df_jhb_top10_today['Com_Name'],df_jhb_top10_today['Hour of Day']) +heat = pd.crosstab(df_plt_top10_today['Com_Name'],df_plt_top10_today['Hour of Day']) #Order heatmap Birds by frequency of occurrance -heat.index = pd.CategoricalIndex(heat.index, categories = pd.value_counts(df_jhb_top10_today['Com_Name']).iloc[:10].index) +heat.index = pd.CategoricalIndex(heat.index, categories = pd.value_counts(df_plt_top10_today['Com_Name']).iloc[:readings].index) heat.sort_index(level=0, inplace=True) @@ -91,7 +86,7 @@ heat_frame = pd.DataFrame(data=0, index=heat.index, columns = hours_in_day) heat=(heat+heat_frame).fillna(0) #Generatie heatmap plot -plot = sns.heatmap(heat, norm=LogNorm(), annot=True, fmt="g", annot_kws={"fontsize":7}, cmap = pal , square = False, cbar=False, linewidths = 0.5, linecolor = "Grey", ax=axs[1], yticklabels = False) +plot = sns.heatmap(heat, norm=LogNorm(), annot=True, annot_kws={"fontsize":7}, cmap = pal , square = False, cbar=False, linewidths = 0.5, linecolor = "Grey", ax=axs[2], yticklabels = False) # Set heatmap border for _, spine in plot.spines.items(): @@ -100,55 +95,71 @@ for _, spine in plot.spines.items(): plot.set(ylabel=None) plot.set(xlabel="Hour of Day") #Set combined plot layout and titles -plt.tight_layout() +# plt.tight_layout() f.subplots_adjust(top=0.9) -plt.suptitle("Last Updated: "+ str(now.strftime("%B, %d at %T"))) +plt.suptitle("Last Updated: "+ str(now.strftime("%d %m %Y %H:%M"))) #Save combined plot savename='/home/pi/BirdSongs/Extracted/Charts/Combo-'+str(now.strftime("%d-%m-%Y"))+'.png' plt.savefig(savename) +#plt.show() plt.close() - #Get bottom 10 today -jhb_bot10_today=(df_jhb_today['Com_Name'].value_counts()[-10:]) -df_jhb_bot10_today = df_jhb_today[df_jhb_today.Com_Name.isin(jhb_bot10_today.index)] +plt_bot10_today = (df_plt_today['Com_Name'].value_counts()[-readings:]) +df_plt_bot10_today = df_plt_today[df_plt_today.Com_Name.isin(plt_bot10_today.index)] #Set Palette for graphics pal = "Reds" #Set up plot axes and titles -f, axs = plt.subplots(1, 2, figsize = (8, 4), gridspec_kw=dict(width_ratios=[3, 5])) +f, axs = plt.subplots(1, 3, figsize = (10, 4), gridspec_kw=dict(width_ratios=[3, 2, 5])) +plt.subplots_adjust(left=None, bottom=None, right=None, top=None, wspace=0, hspace=0) + #Generate frequency plot -plot=sns.countplot(y='Com_Name', data = df_jhb_bot10_today, palette = pal+"_r", order=pd.value_counts(df_jhb_bot10_today['Com_Name']).iloc[:10].index, ax=axs[0]) -plot.set_yticklabels(['\n'.join(textwrap.wrap(ticklabel.get_text(),17)) for ticklabel in plot.get_yticklabels()]) -plot.set(ylabel=None) -plot.set(xlabel="no. of detections") -#Generate crosstab matrix for heatmap plot -heat = pd.crosstab(df_jhb_bot10_today['Com_Name'],df_jhb_bot10_today['Hour of Day']) +plot=sns.countplot(y='Com_Name', data = df_plt_bot10_today, palette = pal+"_r", order=pd.value_counts(df_plt_bot10_today['Com_Name']).iloc[-readings:].index, ax=axs[0]) +#Try plot grid lines between bars - problem at the moment plots grid lines on bars - want between bars +# plot.grid(True, axis='y') +z=plot.get_ymajorticklabels() +plot.set_yticklabels(['\n'.join(textwrap.wrap(ticklabel.get_text(),15)) for ticklabel in plot.get_yticklabels()], fontsize = 10) +plot.set(ylabel=None) +plot.set(xlabel="Detections") + +huw=df_plt_bot10_today.groupby('Com_Name')['Confidence'].mean() +plot = sns.boxenplot(x=df_plt_bot10_today['Confidence']*100,color='Red', y=df_plt_bot10_today['Com_Name'], ax=axs[1],order=pd.value_counts(df_plt_bot10_today['Com_Name']).iloc[-readings:].index) +plot.set(xlabel="Confidence", ylabel=None,yticklabels=[]) + + +#Generate crosstab matrix for heatmap plot + +heat = pd.crosstab(df_plt_bot10_today['Com_Name'],df_plt_bot10_today['Hour of Day']) #Order heatmap Birds by frequency of occurrance -heat.index = pd.CategoricalIndex(heat.index, categories = pd.value_counts(df_jhb_bot10_today['Com_Name']).iloc[:10].index) +heat.index = pd.CategoricalIndex(heat.index, categories = pd.value_counts(df_plt_bot10_today['Com_Name']).iloc[-readings:].index) heat.sort_index(level=0, inplace=True) + + +hours_in_day = pd.Series(data = range(0,24)) heat_frame = pd.DataFrame(data=0, index=heat.index, columns = hours_in_day) heat=(heat+heat_frame).fillna(0) -#Generate heatmap plot -plot = sns.heatmap(heat, norm=LogNorm(), annot=True, annot_kws={"fontsize":7}, cmap = pal , square = False, cbar=False, linewidths = 0.5, linecolor = "Grey", ax=axs[1], yticklabels = False) +#Generatie heatmap plot +plot = sns.heatmap(heat, norm=LogNorm(), annot=True, annot_kws={"fontsize":7}, cmap = pal , square = False, cbar=False, linewidths = 0.5, linecolor = "Grey", ax=axs[2], yticklabels = False) # Set heatmap border for _, spine in plot.spines.items(): spine.set_visible(True) -plot.set(ylabel=None) -#Set combined plot layout and titles -plt.tight_layout() -f.subplots_adjust(top=0.9) -plt.suptitle("Bottom 10 Detected: "+ str(now.strftime("%d-%h-%Y %H:%M"))) +plot.set(ylabel=None) plot.set(xlabel="Hour of Day") +#Set combined plot layout and titles +# plt.tight_layout() +f.subplots_adjust(top=0.9) +plt.suptitle("Last Updated: "+ str(now.strftime("%d %m %Y %H:%M"))) + #Save combined plot savename='/home/pi/BirdSongs/Extracted/Charts/Combo2-'+str(now.strftime("%d-%m-%Y"))+'.png' plt.savefig(savename) - +#plt.show() plt.close()