import os import streamlit as st import pandas as pd import numpy as np from numpy import ma import plotly.graph_objects as go from plotly.subplots import make_subplots import plotly.io as pio from datetime import timedelta import sqlite3 from sqlite3 import Connection import plotly.express as px from sklearn.preprocessing import normalize from suntime import Sun profile = False if profile: try: from pyinstrument import Profiler except ImportError as e: print(e) profile = False else: profiler = Profiler() profiler.start() pio.templates.default = "plotly_white" userDir = os.path.expanduser('~') URI_SQLITE_DB = userDir + '/BirdNET-Pi/scripts/birds.db' st.set_page_config(layout='wide') # Remove whitespace from the top of the page st.markdown(""" """, unsafe_allow_html=True) @st.cache_resource() def get_connection(path: str): return sqlite3.connect(path, check_same_thread=False) def get_data(_conn: Connection): df1 = pd.read_sql("SELECT * FROM detections", con=conn) return df1 conn = get_connection(URI_SQLITE_DB) df2 = get_data(conn) df2['DateTime'] = pd.to_datetime(df2['Date'] + " " + df2['Time']) df2 = df2.set_index('DateTime') daily = st.sidebar.checkbox('Single Day View', help='Select if you want single day view, unselect for multi-day views') if daily: # Date as slider Start_Date = pd.to_datetime(df2.index.min()).date() End_Date = pd.to_datetime(df2.index.max()).date() # cols1, cols2 = st.columns((1, 1)) end_date = st.sidebar.date_input('Date to View', min_value=Start_Date, max_value=End_Date, value=(End_Date), help='Select date for single day view') start_date = end_date else: Start_Date = pd.to_datetime(df2.index.min()).date() End_Date = pd.to_datetime(df2.index.max()).date() # cols1, cols2 = st.columns((1, 1)) start_date, end_date = st.sidebar.slider('Date Range', min_value=Start_Date-timedelta(days=1), max_value=End_Date, value=(Start_Date, End_Date), help='Select start and end date, if same date get a clockplot for a single day') # start_date, end_date = cols1.date_input( # "Date Input for Analysis - select Range for single specie analysis, select single date for daily view", # value=(Start_Date, End_Date), # min_value=Start_Date, # max_value=End_Date) # start_date = datetime(2022 ,5 ,17).date() # end_date = datetime(2022 ,5 ,17).date() @st.cache_data() def date_filter(df, start_date, end_date): filt = (df2.index >= pd.Timestamp(start_date)) & (df2.index <= pd.Timestamp(end_date + timedelta(days=1))) df = df[filt] return (df) df2 = date_filter(df2, start_date, end_date) st.write('', unsafe_allow_html=True) st.write('', unsafe_allow_html=True) # Select time period buttons # Disallow "Daily time period" for "Daily Chart" if start_date == end_date: resample_sel = st.sidebar.radio( "Resample Resolution", ('Raw', '15 minutes', 'Hourly'), index=1, help='Select resolution for single day - larger times run faster') resample_times = {'Raw': 'Raw', '1 minute': '1min', '15 minutes': '15min', 'Hourly': '1H' } resample_time = resample_times[resample_sel] else: resample_sel = st.sidebar.radio( "Resample Resolution", ('Raw', '15 minutes', 'Hourly', 'DAILY'), index=1, help='Select resolution for species - DAILY provides time series') resample_times = {'Raw': 'Raw', '1 minute': '1min', '15 minutes': '15min', 'Hourly': '1H', 'DAILY': '1D' } resample_time = resample_times[resample_sel] @st.cache_data() def time_resample(df, resample_time): if resample_time == 'Raw': df_resample = df['Com_Name'] else: df_resample = df.resample(resample_time)['Com_Name'].aggregate('unique').explode() return (df_resample) top_bird = df2['Com_Name'].mode()[0] df5 = time_resample(df2, resample_time) # Create species count for selected date range Specie_Count = df5.value_counts() # Create Hourly Crosstab hourly = pd.crosstab(df5, df5.index.hour, dropna=True, margins=True) # Filter on species species = list(hourly.sort_values("All", ascending=False).index) # cols1, cols2 = st.columns((1, 1)) top_N = st.sidebar.slider( 'Select Number of Birds to Show', min_value=1, max_value=len(Specie_Count), value=min(10, len(Specie_Count)) ) top_N_species = (df5.value_counts()[:top_N]) font_size = 15 def sunrise_sunset_scatter(date_range): latitude = df2['Lat'][0] longitude = df2['Lon'][0] sun = Sun(latitude, longitude) sunrise_list = [] sunset_list = [] sunrise_text_list = [] sunset_text_list = [] daysback_range = [] current_date = start_date for current_date in date_range: # current_date = datetime.fromisocalendar(2022, week + 1, 5) # time_zone = datetime.now() sun_rise = sun.get_local_sunrise_time(current_date) sun_dusk = sun.get_local_sunset_time(current_date) sun_rise_time = float(sun_rise.hour) + float(sun_rise.minute) / 60.0 sun_dusk_time = float(sun_dusk.hour) + float(sun_dusk.minute) / 60.0 temp_time = str(sun_rise)[-14:-9] + " Sunrise" sunrise_text_list.append(temp_time) temp_time = str(sun_dusk)[-14:-9] + " Sunset" sunset_text_list.append(temp_time) sunrise_list.append(sun_rise_time) sunset_list.append(sun_dusk_time) daysback_range.append(current_date.strftime('%d-%m-%Y')) sunrise_list.append(None) sunrise_text_list.append(None) sunrise_list.extend(sunset_list) sunrise_text_list.extend(sunset_text_list) daysback_range.append(None) daysback_range.extend(daysback_range) return daysback_range, sunrise_list, sunrise_text_list def hms_to_dec(t): # (h, m, s) = t.split(':') h = t.hour m = t.minute / 60 s = t.second / 3600 result = h + m + s return result def hms_to_str(t): # (h, m, s) = t.split(':') h = t.hour m = t.minute # s = t.second / 3600 # result = h + m + s return "%02d:%02d" % (h, m) if daily is False: if resample_time != '1D': specie = st.selectbox( 'Which bird would you like to explore for the dates ' + str(start_date) + ' to ' + str(end_date) + '?', species, index=0) # filt = df2['Com_Name'] == specie if specie == 'All': df_counts = int(hourly[hourly.index == specie]['All'].iloc[0]) fig = make_subplots( rows=3, cols=2, specs=[[{"type": "xy", "rowspan": 3}, {"type": "polar", "rowspan": 2}], [{"rowspan": 1}, {"rowspan": 1}], [None, {"type": "xy", "rowspan": 1}]], subplot_titles=('Top ' + str(top_N) + ' Species in Date Range ' + str(start_date) + ' to ' + str( end_date) + '
for ' + str(resample_sel) + ' sampling interval.' + '
', 'Total Detect:' + str('{:,}'.format(df_counts)) # + ' Confidence Max:' + str( # '{:.2f}%'.format(max(df2[df2['Com_Name'] == specie]['Confidence']) * 100)) + # ' ' + ' Median:' + str( # '{:.2f}%'.format(np.median(df2[df2['Com_Name'] == specie]['Confidence']) * 100)) ) ) fig.layout.annotations[1].update(x=0.7, y=0.25, font_size=15) # Plot seen species for selected date range and number of species fig.add_trace(go.Bar(y=top_N_species.index, x=top_N_species, orientation='h', marker_color='seagreen'), row=1, col=1) fig.update_layout( margin=dict(l=0, r=0, t=50, b=0), yaxis={'categoryorder': 'total ascending'}) # Set 360 degrees, 24 hours for polar plot theta = np.linspace(0.0, 360, 24, endpoint=False) specie_filt = df5 == specie df3 = df5[specie_filt] detections2 = pd.crosstab(df3, df3.index.hour) d = pd.DataFrame(np.zeros((23, 1))).squeeze() detections = hourly.loc[specie] detections = (d + detections).fillna(0) fig.add_trace(go.Barpolar(r=detections, theta=theta, marker_color='seagreen'), row=1, col=2) fig.update_layout( autosize=False, width=1000, height=500, showlegend=False, polar=dict( radialaxis=dict( tickfont_size=font_size, showticklabels=False, hoverformat="#%{theta}:
Popularity: %{percent}
%{r}" ), angularaxis=dict( tickfont_size=font_size, rotation=-90, direction='clockwise', tickmode='array', tickvals=[0, 15, 35, 45, 60, 75, 90, 105, 120, 135, 150, 165, 180, 195, 210, 225, 240, 255, 270, 285, 300, 315, 330, 345], ticktext=['12am', '1am', '2am', '3am', '4am', '5am', '6am', '7am', '8am', '9am', '10am', '11am', '12pm', '1pm', '2pm', '3pm', '4pm', '5pm', '6pm', '7pm', '8pm', '9pm', '10pm', '11pm'], hoverformat="#%{theta}:
Popularity: %{percent}
%{r}" ), ), ) daily = pd.crosstab(df5, df5.index.date, dropna=True, margins=True) fig.add_trace(go.Bar(x=daily.columns[:-1], y=daily.loc[specie][:-1], marker_color='seagreen'), row=3, col=2) st.plotly_chart(fig, use_container_width=True) # , config=config) else: col1, col2 = st.columns(2) with col1: fig = make_subplots( rows=3, cols=1, specs=[[{"type": "polar", "rowspan": 2}], [{"rowspan": 1}], [{"type": "xy", "rowspan": 1}]] ) # Set 360 degrees, 24 hours for polar plot theta = np.linspace(0.0, 360, 24, endpoint=False) specie_filt = df5 == specie df3 = df5[specie_filt] detections2 = pd.crosstab(df3, df3.index.hour) d = pd.DataFrame(np.zeros((23, 1))).squeeze() detections = hourly.loc[specie] detections = (d + detections).fillna(0) fig.add_trace(go.Barpolar(r=detections, theta=theta, marker_color='seagreen'), row=1, col=1) fig.update_layout( autosize=False, width=1000, height=500, showlegend=False, polar=dict( radialaxis=dict( tickfont_size=font_size, showticklabels=False, hoverformat="#%{theta}:
Popularity: %{percent}
%{r}" ), angularaxis=dict( tickfont_size=font_size, rotation=-90, direction='clockwise', tickmode='array', tickvals=[0, 15, 35, 45, 60, 75, 90, 105, 120, 135, 150, 165, 180, 195, 210, 225, 240, 255, 270, 285, 300, 315, 330, 345], ticktext=['12am', '1am', '2am', '3am', '4am', '5am', '6am', '7am', '8am', '9am', '10am', '11am', '12pm', '1pm', '2pm', '3pm', '4pm', '5pm', '6pm', '7pm', '8pm', '9pm', '10pm', '11pm'], hoverformat="#%{theta}:
Popularity: %{percent}
%{r}" ), ), ) daily = pd.crosstab(df5, df5.index.date, dropna=True, margins=True) fig.add_trace(go.Bar(x=daily.columns[:-1], y=daily.loc[specie][:-1], marker_color='seagreen'), row=3, col=1) st.plotly_chart(fig, use_container_width=True) # , config=config) df_counts = int(hourly[hourly.index == specie]['All'].iloc[0]) st.subheader('Total Detect:' + str('{:,}'.format(df_counts)) + ' Confidence Max:' + str('{:.2f}%'.format(max(df2[df2['Com_Name'] == specie]['Confidence']) * 100)) + ' ' + ' Median:' + str('{:.2f}%'.format(np.median(df2[df2['Com_Name'] == specie]['Confidence']) * 100))) recordings = df2[df2['Com_Name'] == specie]['File_Name'] with col2: try: recording = st.selectbox('Available recordings', recordings.sort_index(ascending=False)) date_specie = df2.loc[df2['File_Name'] == recording, ['Date', 'Com_Name']] date_dir = date_specie['Date'].values[0] specie_dir = date_specie['Com_Name'].values[0].replace(" ", "_") st.image(userDir + '/BirdSongs/Extracted/By_Date/' + date_dir + '/' + specie_dir + '/' + recording + '.png') st.audio(userDir + '/BirdSongs/Extracted/By_Date/' + date_dir + '/' + specie_dir + '/' + recording) except Exception: st.title('RECORDING NOT AVAILABLE :(') # try: # con = sqlite3.connect(userDir + '/BirdNET-Pi/scripts/birds.db') # cur = con.cursor() cola, colb, colc, cold = st.columns((3, 1, 1, 1)) with colb: seen = st.checkbox('Reviewed') if seen: with colc: verified = st.radio("Verification", ['True Positive', 'False Positive']) if verified == "False Positive": df_names = pd.read_csv(userDir + '/BirdNET-Pi/model/labels.txt', delimiter='_', names=['Sci_Name', 'Com_Name']) df_unknown = pd.DataFrame({"Sci_Name": ["UNKNOWN"], "Com_Name": ["UNKNOWN"]}) df_names = pd.concat([df_unknown, df_names], ignore_index=True) with cold: corrected = st.selectbox('What species?', df_names['Com_Name']) # cur.execute("UPDATE detections SET Seen = seen WHERE File_Name = recording") # con.commit() # con.close() # except BaseException: # print("Database busy") # time.sleep(2) else: specie = st.selectbox('Which bird would you like to explore for the dates ' + str(start_date) + ' to ' + str(end_date) + '?', species[1:], index=0) # filt = df2[df2['Com_Name'] == specie] df_counts = int(hourly[hourly.index == specie]['All']) fig = st.container() fig = make_subplots(rows=1, cols=1) # specs= [[{"type":"xy","rowspan":1},{"type":"heatmap","rowspan":1}]], # subplot_titles=('Daily Top '+ str(top_N) + ' Species in Date Range '+ str(start_date) +' to '+ str(end_date) +'', # 'Daily ' + specie+ ' Detections on 15 minute intervals '), # # 'Total Detect:'+str('{:,}'.format(df_counts))+ # # ' Confidence Max:'+str('{:.2f}%'.format(max(df2[df2['Com_Name']==specie]['Confidence'])*100))+ # # ' '+' Median:'+str('{:.2f}%'.format(np.median(df2[df2['Com_Name']==specie]['Confidence'])*100)) # # ) # ) # fig.add_trace(go.Bar(y=top_N_species.index, x=top_N_species, orientation='h'), row=1,col=1) df4 = df2['Com_Name'][df2['Com_Name'] == specie].resample('15min').count() df4.index = [df4.index.date, df4.index.time] day_hour_freq = df4.unstack().fillna(0) saved_time_labels = [hms_to_str(h) for h in day_hour_freq.columns.tolist()] fig_dec_y = [hms_to_dec(h) for h in day_hour_freq.columns.tolist()] fig_x = [d.strftime('%d-%m-%Y') for d in day_hour_freq.index.tolist()] fig_y = [h.strftime('%H:%M') for h in day_hour_freq.columns.tolist()] day_hour_freq.columns = fig_dec_y fig_z = day_hour_freq.values.transpose() # fig_heatmap = go.Figure(data=go.Heatmap(x=fig_x,y=fig_y,z=fig_z)) # fig.update_layout( # margin=dict(l=0, r=0, t=50, b=0), # yaxis={'categoryorder':'total ascending'}) color_pals = px.colors.named_colorscales() selected_pal = st.sidebar.selectbox('Select Color Pallet for Daily Detections', color_pals) heatmap = go.Heatmap( # x=fig_x, y=fig_y, x=fig_x, y=day_hour_freq.columns, z=fig_z, # heat.values, showscale=False, # text=labels, texttemplate="%{text}", autocolorscale=False, colorscale=selected_pal ) daysback_range, sunrise_list, sunrise_text_list = sunrise_sunset_scatter(day_hour_freq.index.tolist()) sunrise_sunset = go.Scatter(x=daysback_range, y=sunrise_list, mode='lines', hoverinfo='text', text=sunrise_text_list, line_color='orange', line_width=1, name=' ') fig = go.Figure(data=[heatmap, sunrise_sunset]) number_of_y_ticks = 12 y_downscale_factor = int(len(saved_time_labels) / number_of_y_ticks) fig.update_layout( yaxis=dict( tickmode='array', tickvals=day_hour_freq.columns[::y_downscale_factor], ticktext=saved_time_labels[::y_downscale_factor], nticks=6 ) ) st.plotly_chart(fig, use_container_width=True) # , config=config) else: fig = make_subplots( rows=1, cols=2, specs=[[{"type": "xy", "rowspan": 1}, {"type": "xy", "rowspan": 1}]], subplot_titles=('Top ' + str(top_N) + ' Species For ' + str(start_date) + '', 'Daily ' + str(start_date) + ' Detections on ' + resample_sel + ' interval'), shared_yaxes='all', horizontal_spacing=0 ) df6 = df5.to_frame(name='Com_Name') readings = top_N plt_topN_today = (df6['Com_Name'].value_counts()[:readings]) freq_order = pd.value_counts(df6['Com_Name']).iloc[:readings].index # confmax = df6.groupby('Com_Name')['Confidence'].max() # reorder confmax to detection frequency order # confmax = confmax.reindex(freq_order) # norm = plt.Normalize(confmax.values.min(), confmax.values.max()) # # colors = plt.cm.Greens(norm(confmax)) fig.add_trace(go.Bar(y=plt_topN_today.index, x=plt_topN_today, marker_color='seagreen', orientation='h'), row=1, col=1) # plot=sns.countplot(y='Com_Name', data = df_plt_topN_today, palette = colors, order=freq_order, ax=axs[0]) df6['Hour of Day'] = [r.hour for r in df6.index.time] heat = pd.crosstab(df6['Com_Name'], df6['Hour of Day']) # Order heatmap Birds by frequency of occurrance heat.index = pd.CategoricalIndex(heat.index, categories=freq_order) heat.sort_index(level=0, inplace=True) heat_plot_values = ma.log(heat.values).filled(0) hours_in_day = pd.Series(data=range(0, 24)) heat_frame = pd.DataFrame(data=0, index=heat.index, columns=hours_in_day) heat = (heat + heat_frame).fillna(0) heat_values_normalized = normalize(heat.values, axis=1, norm='l1') labels = heat.values.astype(int).astype('str') labels[labels == '0'] = "" fig.add_trace(go.Heatmap(x=heat.columns, y=heat.index, z=heat_values_normalized, # heat.values, showscale=False, text=labels, texttemplate="%{text}", colorscale='Blugrn' ), row=1, col=2) fig.update_yaxes(visible=True, autorange="reversed", ticks="inside", tickson="boundaries", ticklen=10000, showgrid=True) fig.update_layout(xaxis_ticks="inside", margin=dict(l=0, r=0, t=50, b=0)) # container=st.container() # config={'displayModelBar': False} st.plotly_chart(fig, use_container_width=True) # , config=config) # cols3,cols4=st.columns((1,1)) # extract_date=Date_Slider # audio_file = open('/home/*/BirdSongs/Extracted/By_Date/2022-03-22/Yellow-streaked_Greenbul/Yellow-streaked_Greenbul-77-2022-03-22-birdnet-15:04:28.mp3', 'rb') # audio_bytes = audio_file.read() # cols4.audio(audio_bytes, format='audio/mp3') if profile: profiler.stop() profiler.print() print('**profiler done**', flush=True)