Updated Species stats with start of Species verification

scripts/plotly_streamlit.py

@@ -6,11 +6,13 @@ from numpy import ma
 import plotly.graph_objects as go
 from plotly.subplots import make_subplots
 import plotly.io as pio
-from datetime import timedelta
+from datetime import timedelta, datetime
+from pathlib import Path
 import sqlite3
 from sqlite3 import Connection
 import plotly.express as px
 from sklearn.preprocessing import normalize
+import time
 
 pio.templates.default = "plotly_white"
 
@@ -18,7 +20,7 @@ userDir = os.path.expanduser('~')
 URI_SQLITE_DB = userDir + '/BirdNET-Pi/scripts/birds.db'
 
 st.set_page_config(layout='wide')
-#
+
 # Remove whitespace from the top of the page
 st.markdown("""
         <style>
@@ -38,8 +40,9 @@ st.markdown("""
         """, unsafe_allow_html=True)
 
 
-# @st.cache(hash_funcs={Connection: id})
+
-@st.cache(allow_output_mutation=True)
+@st.cache(hash_funcs={Connection: id})
+#@st.cache(allow_output_mutation=True)
 def get_connection(path: str):
     return sqlite3.connect(path, check_same_thread=False)
 
@@ -62,7 +65,7 @@ if daily:
     Start_Date = pd.to_datetime(df2.index.min()).date()
     End_Date = pd.to_datetime(df2.index.max()).date()
 #     cols1, cols2 = st.columns((1, 1))
-    end_date = st.sidebar.slider('Date to View',
+    end_date = st.sidebar.date_input('Date to View',
                             min_value = Start_Date,
                             max_value = End_Date,
                             value=(End_Date),
@@ -90,14 +93,12 @@ else:
 # start_date = datetime(2022 ,5 ,17).date()
 # end_date = datetime(2022 ,5 ,17).date()
 
-
 @st.cache()
 def date_filter(df, start_date, end_date):
     filt = (df2.index >= pd.Timestamp(start_date)) & (df2.index <= pd.Timestamp(end_date + timedelta(days=1)))
     df = df[filt]
     return(df)
 
-
 df2 = date_filter(df2, start_date, end_date)
 
 st.write('<style>div.row-widget.stRadio > div{flex-direction:row;justify-content: left;} </style>',
@@ -133,7 +134,6 @@ else:
                       }
     resample_time = resample_times[resample_sel]
 
-
 @st.cache()
 def time_resample(df, resample_time):
     if resample_time == 'Raw':
@@ -143,8 +143,6 @@ def time_resample(df, resample_time):
         df_resample = df.resample(resample_time)['Com_Name'].aggregate('unique').explode()
 
     return(df_resample)
-
-
 top_bird = df2['Com_Name'].mode()[0]
 df5 = time_resample(df2, resample_time)
 
@@ -153,10 +151,10 @@ df5 = time_resample(df2, resample_time)
 Specie_Count = df5.value_counts()
 
 # Create Hourly Crosstab
-hourly = pd.crosstab(df5, df5.index.hour, dropna=False)
+hourly = pd.crosstab(df5, df5.index.hour, dropna=True, margins= True)
 
 # Filter on species
-species = list(hourly.index)
+species = list(hourly.sort_values("All", ascending= False).index)
 
 #cols1, cols2 = st.columns((1, 1))
 top_N = st.sidebar.slider(
@@ -170,35 +168,36 @@ top_N_species = (df5.value_counts()[:top_N])
 
 font_size = 15
 
-if daily is False:
+if daily == False:
+    
+    if resample_time != '1D':
         specie = st.selectbox(
         'Which bird would you like to explore for the dates ' 
         + str(start_date) + ' to ' + str(end_date) + '?', 
         species,
-        index=species.index(top_bird))
+        index = 0)
 
-    filt = df2['Com_Name'] == specie
-
-    df_counts = sum(df5 == specie)
-
-    if resample_time != '1D':
 
+    # filt = df2['Com_Name'] == specie
+        if specie == 'All':
+            df_counts = int(hourly[hourly.index==specie]['All'])
             fig = make_subplots(
                 rows=3, cols=2,
                 specs=[[{"type": "xy", "rowspan": 3}, {"type": "polar", "rowspan": 2}], [{"rowspan": 1}, {"rowspan": 1}],
                     [None, {"type": "xy", "rowspan": 1}]],
                 subplot_titles=('<b>Top ' + str(top_N) + ' Species in Date Range ' + str(start_date) + ' to ' + str(
-                end_date) + '<br>for ' + str(resample_sel) + ' sampling interval.' + '</b>',
+                    end_date) + ' for ' + str(resample_sel) + ' sampling interval.' + '</b>',
-                            'Total Detect:' + str('{:,}'.format(df_counts)) +
+                                'Total Detect:' + str('{:,}'.format(df_counts)) 
-                            '   Confidence Max:' + str(
+                                # +   '   Confidence Max:' + str(
-                                '{:.2f}%'.format(max(df2[df2['Com_Name'] == specie]['Confidence']) * 100)) +
+                                #     '{:.2f}%'.format(max(df2[df2['Com_Name'] == specie]['Confidence']) * 100)) +
-                            '   ' + '   Median:' + str(
+                                # '   ' + '   Median:' + str(
-                                '{:.2f}%'.format(np.median(df2[df2['Com_Name'] == specie]['Confidence']) * 100))
+                                #     '{:.2f}%'.format(np.median(df2[df2['Com_Name'] == specie]['Confidence']) * 100))
                                 )
             )
             fig.layout.annotations[1].update(x=0.7, y=0.25, font_size=15)
 
             # Plot seen species for selected date range and number of species
+            
             fig.add_trace(go.Bar(y=top_N_species.index, x=top_N_species, orientation='h', marker_color='seagreen'), row=1, col=1)
             
             fig.update_layout(
@@ -242,21 +241,128 @@ if daily is False:
                 ),
             )
 
-        daily = pd.crosstab(df5, df5.index.date, dropna=False)
+            daily = pd.crosstab(df5, df5.index.date, dropna=True, margins = True)
+            fig.add_trace(go.Bar(x=daily.columns[:-1], y=daily.loc[specie][:-1], marker_color='seagreen'), row=3, col=2)
+            st.plotly_chart(fig, use_container_width=True)  # , config=config)
 
-        fig.add_trace(go.Bar(x=daily.columns, y=daily.loc[specie], marker_color='seagreen'), row=3, col=2)
         else:
+            col1, col2 = st.columns(2)
+            with col1:
+                fig = make_subplots(
+                    rows=3, cols=1,
+                    specs=[[{"type": "polar", "rowspan": 2}],[{"rowspan": 1}], [{"type": "xy", "rowspan": 1}]]
+                            )
+                # Set 360 degrees, 24 hours for polar plot
+                theta = np.linspace(0.0, 360, 24, endpoint=False)
+
+                specie_filt = df5 == specie
+                df3 = df5[specie_filt]
+
+                detections2 = pd.crosstab(df3, df3.index.hour)
+
+                d = pd.DataFrame(np.zeros((23, 1))).squeeze()
+                detections = hourly.loc[specie]
+                detections = (d + detections).fillna(0)
+                fig.add_trace(go.Barpolar(r=detections, theta=theta, marker_color='seagreen'), row=1, col=1)
+                fig.update_layout(
+                    autosize=False,
+                    width=1000,
+                    height=500,
+                    showlegend=False,
+                    polar=dict(
+                        radialaxis=dict(
+                            tickfont_size=font_size,
+                            showticklabels=False,
+                            hoverformat="#%{theta}: <br>Popularity: %{percent} </br> %{r}"
+                        ),
+                        angularaxis=dict(
+                            tickfont_size=font_size,
+                            rotation=-90,
+                            direction='clockwise',
+                            tickmode='array',
+                            tickvals=[0, 15, 35, 45, 60, 75, 90, 105, 120, 135, 150, 165, 180, 195, 210, 225, 240, 255, 270,
+                                    285, 300, 315, 330, 345],
+                            ticktext=['12am', '1am', '2am', '3am', '4am', '5am', '6am', '7am', '8am', '9am', '10am', '11am',
+                                    '12pm', '1pm', '2pm', '3pm', '4pm', '5pm', '6pm', '7pm', '8pm', '9pm', '10pm', '11pm'],
+                            hoverformat="#%{theta}: <br>Popularity: %{percent} </br> %{r}"
+                        ),
+                    ),
+                )
+
+                daily = pd.crosstab(df5, df5.index.date, dropna=True, margins = True)
+                fig.add_trace(go.Bar(x=daily.columns[:-1], y=daily.loc[specie][:-1], marker_color='seagreen'), row=3, col=1)
+                st.plotly_chart(fig, use_container_width=True)  # , config=config)
+                df_counts = int(hourly[hourly.index==specie]['All'])
+                st.subheader('Total Detect:' + str('{:,}'.format(df_counts)) 
+                                +   '   Confidence Max:' + str(
+                                    '{:.2f}%'.format(max(df2[df2['Com_Name'] == specie]['Confidence']) * 100)) +
+                                '   ' + '   Median:' + str(
+                                    '{:.2f}%'.format(np.median(df2[df2['Com_Name'] == specie]['Confidence']) * 100)))
+
+
+            recordings=df2[df2['Com_Name']==specie]['File_Name']
+            
+        
+            with col2:
+                try:
+                    recording = st.selectbox('Available recordings', recordings.sort_index(ascending=False))
+                    date_specie = df2.loc[df2['File_Name']==recording,['Date','Com_Name']]
+                    date_dir = date_specie['Date'].values[0]
+                    specie_dir = date_specie['Com_Name'].values[0].replace(" ","_")
+                    st.image(userDir + '/BirdSongs/Extracted/By_Date/'+ date_dir + '/'+ specie_dir + '/' + recording + '.png')
+                    st.audio(userDir +'/BirdSongs/Extracted/By_Date/'+ date_dir + '/'+ specie_dir + '/' + recording)
+                except:
+                    st.title('RECORDING NOT AVAILABLE :(')    
+            # try:
+            #     con = sqlite3.connect(userDir + '/BirdNET-Pi/scripts/birds.db')
+            #     cur = con.cursor()
+            cola, colb, colc, cold = st.columns((3,1,1,1))
+            with colb:
+                seen = st.checkbox('Reviewed')
+            if seen:
+                with colc:
+                    verified = st.radio("Verification",['True Positive','False Positive'])
+            
+                    if verified == "False Positive":
+                        df_names = pd.read_csv(userDir+'/BirdNET-Pi/model/labels.txt', delimiter= '_', names=['Sci_Name', 'Com_Name']) 
+                        df_unknown= pd.DataFrame({"Sci_Name":["UNKNOWN"],"Com_Name":["UNKNOWN"]})
+                        df_names = pd.concat([df_unknown,df_names], ignore_index=True)
+                        with cold:
+                            corrected = st.selectbox('What specie?', df_names['Com_Name'])
+            #     cur.execute("UPDATE detections SET  Seen = seen WHERE File_Name = recording")
+            #     con.commit()
+            #     con.close()
+            
+            # except BaseException:
+            #     print("Database busy")
+            #     time.sleep(2)
+            
+            
+    else:
+
+        specie = st.selectbox(
+        'Which bird would you like to explore for the dates ' 
+        + str(start_date) + ' to ' + str(end_date) + '?', 
+        species[1:],
+        index = 0)
+
+
+    # filt = df2[df2['Com_Name'] == specie]
+
+        df_counts = int(hourly[hourly.index==specie]['All'])
         fig = st.container()
         fig = make_subplots(
                         rows=1, cols =1)
     #                     specs= [[{"type":"xy","rowspan":1},{"type":"heatmap","rowspan":1}]],
                             
+
     #                     subplot_titles=('<b>Daily Top '+ str(top_N) + ' Species in Date Range '+ str(start_date) +' to '+ str(end_date) +'</b>',
     #                                     '<b>Daily ' + specie+ ' Detections on 15 minute intervals </b>'),
     # #                                     'Total Detect:'+str('{:,}'.format(df_counts))+
     # #                                     '   Confidence Max:'+str('{:.2f}%'.format(max(df2[df2['Com_Name']==specie]['Confidence'])*100))+
     # #                                     '   '+'   Median:'+str('{:.2f}%'.format(np.median(df2[df2['Com_Name']==specie]['Confidence'])*100))
     # #                                     )
+    #                     )
 
 #        fig.add_trace(go.Bar(y=top_N_species.index, x=top_N_species, orientation='h'), row=1,col=1)
         df4=df2['Com_Name'][df2['Com_Name']==specie].resample('15min').count()
@@ -274,7 +380,7 @@ if daily is False:
         color_pals= px.colors.named_colorscales()
         selected_pal = st.sidebar.selectbox('Select Color Pallet for Daily Detections', color_pals)
         fig.add_trace(go.Heatmap(x=fig_x,y=fig_y,z=fig_z, autocolorscale = False, colorscale = selected_pal), row=1, col=1)
-
+        st.plotly_chart(fig, use_container_width=True)  # , config=config)
 else:
     fig = make_subplots(
         rows=1, cols=2,