From 6ef05c2bc3e960045a87f7cefe61737100c5673e Mon Sep 17 00:00:00 2001
From: CaiusX <dennis_cowen@mac.com>
Date: Tue, 31 May 2022 17:39:25 +0200
Subject: [PATCH] Updated Species stats with start of Species verification

---
 scripts/plotly_streamlit.py | 308 ++++++++++++++++++++++++------------
 1 file changed, 207 insertions(+), 101 deletions(-)

diff --git a/scripts/plotly_streamlit.py b/scripts/plotly_streamlit.py
index 5417c83..355db04 100755
--- a/scripts/plotly_streamlit.py
+++ b/scripts/plotly_streamlit.py
@@ -6,11 +6,13 @@ from numpy import ma
 import plotly.graph_objects as go
 from plotly.subplots import make_subplots
 import plotly.io as pio
-from datetime import timedelta
+from datetime import timedelta, datetime
+from pathlib import Path
 import sqlite3
 from sqlite3 import Connection
 import plotly.express as px
 from sklearn.preprocessing import normalize
+import time
 
 pio.templates.default = "plotly_white"
 
@@ -18,7 +20,7 @@ userDir = os.path.expanduser('~')
 URI_SQLITE_DB = userDir + '/BirdNET-Pi/scripts/birds.db'
 
 st.set_page_config(layout='wide')
-#
+
 # Remove whitespace from the top of the page
 st.markdown("""
         <style>
@@ -38,8 +40,9 @@ st.markdown("""
         """, unsafe_allow_html=True)
 
 
-# @st.cache(hash_funcs={Connection: id})
-@st.cache(allow_output_mutation=True)
+
+@st.cache(hash_funcs={Connection: id})
+#@st.cache(allow_output_mutation=True)
 def get_connection(path: str):
     return sqlite3.connect(path, check_same_thread=False)
 
@@ -55,19 +58,19 @@ df2 = df.copy()
 df2['DateTime'] = pd.to_datetime(df2['Date'] + " " + df2['Time'])
 df2 = df2.set_index('DateTime')
 
-daily = st.sidebar.checkbox('Single Day View', help='Select if you want single day view, unselect for multi-day views')
+daily = st.sidebar.checkbox('Single Day View', help= 'Select if you want single day view, unselect for multi-day views')
 
 if daily:
-    # Date as slider
+# Date as slider
     Start_Date = pd.to_datetime(df2.index.min()).date()
     End_Date = pd.to_datetime(df2.index.max()).date()
 #     cols1, cols2 = st.columns((1, 1))
-    end_date = st.sidebar.slider('Date to View',
-                                 min_value=Start_Date,
-                                 max_value=End_Date,
-                                 value=(End_Date),
-                                 help='Select date for single day view'
-                                 )
+    end_date = st.sidebar.date_input('Date to View',
+                            min_value = Start_Date,
+                            max_value = End_Date,
+                            value=(End_Date),
+                            help= 'Select date for single day view'                                     
+                            )
     start_date = end_date
 else:
     Start_Date = pd.to_datetime(df2.index.min()).date()
@@ -75,11 +78,11 @@ else:
 
 #    cols1, cols2 = st.columns((1, 1))
     start_date, end_date = st.sidebar.slider('Date Range',
-                                             min_value=Start_Date - timedelta(days=1),
-                                             max_value=End_Date,
-                                             value=(Start_Date, End_Date),
-                                             help='Select start and end date, if same date get a clockplot for a single day'
-                                             )
+                                    min_value = Start_Date-timedelta(days=1),
+                                    max_value = End_Date,
+                                    value=(Start_Date, End_Date),
+                                    help= 'Select start and end date, if same date get a clockplot for a single day'                                        
+                                    )
 
 # start_date, end_date = cols1.date_input(
 #     "Date Input for Analysis - select Range for single specie analysis, select single date for daily view",
@@ -90,14 +93,12 @@ else:
 # start_date = datetime(2022 ,5 ,17).date()
 # end_date = datetime(2022 ,5 ,17).date()
 
-
 @st.cache()
 def date_filter(df, start_date, end_date):
     filt = (df2.index >= pd.Timestamp(start_date)) & (df2.index <= pd.Timestamp(end_date + timedelta(days=1)))
     df = df[filt]
     return(df)
 
-
 df2 = date_filter(df2, start_date, end_date)
 
 st.write('<style>div.row-widget.stRadio > div{flex-direction:row;justify-content: left;} </style>',
@@ -111,7 +112,7 @@ st.write('<style>div.st-bf{flex-direction:column;} div.st-ag{font-weight:bold;pa
 if start_date == end_date:
     resample_sel = st.sidebar.radio(
         "Resample Resolution",
-        ('Raw', '15 minutes', 'Hourly'), index=1, help='Select resolution for single day - larger times run faster')
+        ('Raw', '15 minutes', 'Hourly'), index=1, help= 'Select resolution for single day - larger times run faster' )
 
     resample_times = {'Raw': 'Raw',
                       '1 minute': '1min',
@@ -123,7 +124,7 @@ if start_date == end_date:
 else:
     resample_sel = st.sidebar.radio(
         "Resample Resolution",
-        ('Raw', '15 minutes', 'Hourly', 'DAILY'), index=1, help='Select resolution for species - DAILY provides time series')
+        ('Raw', '15 minutes', 'Hourly', 'DAILY'), index=1, help= 'Select resolution for species - DAILY provides time series')
 
     resample_times = {'Raw': 'Raw',
                       '1 minute': '1min',
@@ -133,7 +134,6 @@ else:
                       }
     resample_time = resample_times[resample_sel]
 
-
 @st.cache()
 def time_resample(df, resample_time):
     if resample_time == 'Raw':
@@ -143,8 +143,6 @@ def time_resample(df, resample_time):
         df_resample = df.resample(resample_time)['Com_Name'].aggregate('unique').explode()
 
     return(df_resample)
-
-
 top_bird = df2['Com_Name'].mode()[0]
 df5 = time_resample(df2, resample_time)
 
@@ -153,12 +151,12 @@ df5 = time_resample(df2, resample_time)
 Specie_Count = df5.value_counts()
 
 # Create Hourly Crosstab
-hourly = pd.crosstab(df5, df5.index.hour, dropna=False)
+hourly = pd.crosstab(df5, df5.index.hour, dropna=True, margins= True)
 
 # Filter on species
-species = list(hourly.index)
+species = list(hourly.sort_values("All", ascending= False).index)
 
-# cols1, cols2 = st.columns((1, 1))
+#cols1, cols2 = st.columns((1, 1))
 top_N = st.sidebar.slider(
     'Select Number of Birds to Show',
     min_value=1,
@@ -170,86 +168,193 @@ top_N_species = (df5.value_counts()[:top_N])
 
 font_size = 15
 
-if daily is False:
-    specie = st.selectbox(
-        'Which bird would you like to explore for the dates '
-        + str(start_date) + ' to ' + str(end_date) + '?',
-        species,
-        index=species.index(top_bird))
-
-    filt = df2['Com_Name'] == specie
-
-    df_counts = sum(df5 == specie)
-
+if daily == False:
+    
     if resample_time != '1D':
+        specie = st.selectbox(
+        'Which bird would you like to explore for the dates ' 
+        + str(start_date) + ' to ' + str(end_date) + '?', 
+        species,
+        index = 0)
 
-        fig = make_subplots(
-            rows=3, cols=2,
-            specs=[[{"type": "xy", "rowspan": 3}, {"type": "polar", "rowspan": 2}], [{"rowspan": 1}, {"rowspan": 1}],
-                   [None, {"type": "xy", "rowspan": 1}]],
-            subplot_titles=('<b>Top ' + str(top_N) + ' Species in Date Range ' + str(start_date) + ' to ' + str(
-                end_date) + '<br>for ' + str(resample_sel) + ' sampling interval.' + '</b>',
-                            'Total Detect:' + str('{:,}'.format(df_counts)) +
-                            '   Confidence Max:' + str(
-                                '{:.2f}%'.format(max(df2[df2['Com_Name'] == specie]['Confidence']) * 100)) +
-                            '   ' + '   Median:' + str(
-                                '{:.2f}%'.format(np.median(df2[df2['Com_Name'] == specie]['Confidence']) * 100))
+
+    # filt = df2['Com_Name'] == specie
+        if specie == 'All':
+            df_counts = int(hourly[hourly.index==specie]['All'])
+            fig = make_subplots(
+                rows=3, cols=2,
+                specs=[[{"type": "xy", "rowspan": 3}, {"type": "polar", "rowspan": 2}], [{"rowspan": 1}, {"rowspan": 1}],
+                    [None, {"type": "xy", "rowspan": 1}]],
+                subplot_titles=('<b>Top ' + str(top_N) + ' Species in Date Range ' + str(start_date) + ' to ' + str(
+                    end_date) + ' for ' + str(resample_sel) + ' sampling interval.' + '</b>',
+                                'Total Detect:' + str('{:,}'.format(df_counts)) 
+                                # +   '   Confidence Max:' + str(
+                                #     '{:.2f}%'.format(max(df2[df2['Com_Name'] == specie]['Confidence']) * 100)) +
+                                # '   ' + '   Median:' + str(
+                                #     '{:.2f}%'.format(np.median(df2[df2['Com_Name'] == specie]['Confidence']) * 100))
+                                )
             )
-        )
-        fig.layout.annotations[1].update(x=0.7, y=0.25, font_size=15)
+            fig.layout.annotations[1].update(x=0.7, y=0.25, font_size=15)
 
-        # Plot seen species for selected date range and number of species
-        fig.add_trace(go.Bar(y=top_N_species.index, x=top_N_species, orientation='h', marker_color='seagreen'), row=1, col=1)
+            # Plot seen species for selected date range and number of species
+            
+            fig.add_trace(go.Bar(y=top_N_species.index, x=top_N_species, orientation='h', marker_color='seagreen'), row=1, col=1)
+            
+            fig.update_layout(
+                margin=dict(l=0, r=0, t=50, b=0),
+                yaxis={'categoryorder': 'total ascending'})
 
-        fig.update_layout(
-            margin=dict(l=0, r=0, t=50, b=0),
-            yaxis={'categoryorder': 'total ascending'})
+            # Set 360 degrees, 24 hours for polar plot
+            theta = np.linspace(0.0, 360, 24, endpoint=False)
 
-        # Set 360 degrees, 24 hours for polar plot
-        theta = np.linspace(0.0, 360, 24, endpoint=False)
+            specie_filt = df5 == specie
+            df3 = df5[specie_filt]
 
-        specie_filt = df5 == specie
-        df3 = df5[specie_filt]
+            detections2 = pd.crosstab(df3, df3.index.hour)
 
-        detections2 = pd.crosstab(df3, df3.index.hour)
-
-        d = pd.DataFrame(np.zeros((23, 1))).squeeze()
-        detections = hourly.loc[specie]
-        detections = (d + detections).fillna(0)
-        fig.add_trace(go.Barpolar(r=detections, theta=theta, marker_color='seagreen'), row=1, col=2)
-        fig.update_layout(
-            autosize=False,
-            width=1000,
-            height=500,
-            showlegend=False,
-            polar=dict(
-                radialaxis=dict(
-                    tickfont_size=font_size,
-                    showticklabels=False,
-                    hoverformat="#%{theta}: <br>Popularity: %{percent} </br> %{r}"
+            d = pd.DataFrame(np.zeros((23, 1))).squeeze()
+            detections = hourly.loc[specie]
+            detections = (d + detections).fillna(0)
+            fig.add_trace(go.Barpolar(r=detections, theta=theta, marker_color='seagreen'), row=1, col=2)
+            fig.update_layout(
+                autosize=False,
+                width=1000,
+                height=500,
+                showlegend=False,
+                polar=dict(
+                    radialaxis=dict(
+                        tickfont_size=font_size,
+                        showticklabels=False,
+                        hoverformat="#%{theta}: <br>Popularity: %{percent} </br> %{r}"
+                    ),
+                    angularaxis=dict(
+                        tickfont_size=font_size,
+                        rotation=-90,
+                        direction='clockwise',
+                        tickmode='array',
+                        tickvals=[0, 15, 35, 45, 60, 75, 90, 105, 120, 135, 150, 165, 180, 195, 210, 225, 240, 255, 270,
+                                285, 300, 315, 330, 345],
+                        ticktext=['12am', '1am', '2am', '3am', '4am', '5am', '6am', '7am', '8am', '9am', '10am', '11am',
+                                '12pm', '1pm', '2pm', '3pm', '4pm', '5pm', '6pm', '7pm', '8pm', '9pm', '10pm', '11pm'],
+                        hoverformat="#%{theta}: <br>Popularity: %{percent} </br> %{r}"
+                    ),
                 ),
-                angularaxis=dict(
-                    tickfont_size=font_size,
-                    rotation=-90,
-                    direction='clockwise',
-                    tickmode='array',
-                    tickvals=[0, 15, 35, 45, 60, 75, 90, 105, 120, 135, 150, 165, 180, 195, 210, 225, 240, 255, 270,
-                              285, 300, 315, 330, 345],
-                    ticktext=['12am', '1am', '2am', '3am', '4am', '5am', '6am', '7am', '8am', '9am', '10am', '11am',
-                              '12pm', '1pm', '2pm', '3pm', '4pm', '5pm', '6pm', '7pm', '8pm', '9pm', '10pm', '11pm'],
-                    hoverformat="#%{theta}: <br>Popularity: %{percent} </br> %{r}"
-                ),
-            ),
-        )
+            )
 
-        daily = pd.crosstab(df5, df5.index.date, dropna=False)
+            daily = pd.crosstab(df5, df5.index.date, dropna=True, margins = True)
+            fig.add_trace(go.Bar(x=daily.columns[:-1], y=daily.loc[specie][:-1], marker_color='seagreen'), row=3, col=2)
+            st.plotly_chart(fig, use_container_width=True)  # , config=config)
 
-        fig.add_trace(go.Bar(x=daily.columns, y=daily.loc[specie], marker_color='seagreen'), row=3, col=2)
+        else:
+            col1, col2 = st.columns(2)
+            with col1:
+                fig = make_subplots(
+                    rows=3, cols=1,
+                    specs=[[{"type": "polar", "rowspan": 2}],[{"rowspan": 1}], [{"type": "xy", "rowspan": 1}]]
+                            )
+                # Set 360 degrees, 24 hours for polar plot
+                theta = np.linspace(0.0, 360, 24, endpoint=False)
+
+                specie_filt = df5 == specie
+                df3 = df5[specie_filt]
+
+                detections2 = pd.crosstab(df3, df3.index.hour)
+
+                d = pd.DataFrame(np.zeros((23, 1))).squeeze()
+                detections = hourly.loc[specie]
+                detections = (d + detections).fillna(0)
+                fig.add_trace(go.Barpolar(r=detections, theta=theta, marker_color='seagreen'), row=1, col=1)
+                fig.update_layout(
+                    autosize=False,
+                    width=1000,
+                    height=500,
+                    showlegend=False,
+                    polar=dict(
+                        radialaxis=dict(
+                            tickfont_size=font_size,
+                            showticklabels=False,
+                            hoverformat="#%{theta}: <br>Popularity: %{percent} </br> %{r}"
+                        ),
+                        angularaxis=dict(
+                            tickfont_size=font_size,
+                            rotation=-90,
+                            direction='clockwise',
+                            tickmode='array',
+                            tickvals=[0, 15, 35, 45, 60, 75, 90, 105, 120, 135, 150, 165, 180, 195, 210, 225, 240, 255, 270,
+                                    285, 300, 315, 330, 345],
+                            ticktext=['12am', '1am', '2am', '3am', '4am', '5am', '6am', '7am', '8am', '9am', '10am', '11am',
+                                    '12pm', '1pm', '2pm', '3pm', '4pm', '5pm', '6pm', '7pm', '8pm', '9pm', '10pm', '11pm'],
+                            hoverformat="#%{theta}: <br>Popularity: %{percent} </br> %{r}"
+                        ),
+                    ),
+                )
+
+                daily = pd.crosstab(df5, df5.index.date, dropna=True, margins = True)
+                fig.add_trace(go.Bar(x=daily.columns[:-1], y=daily.loc[specie][:-1], marker_color='seagreen'), row=3, col=1)
+                st.plotly_chart(fig, use_container_width=True)  # , config=config)
+                df_counts = int(hourly[hourly.index==specie]['All'])
+                st.subheader('Total Detect:' + str('{:,}'.format(df_counts)) 
+                                +   '   Confidence Max:' + str(
+                                    '{:.2f}%'.format(max(df2[df2['Com_Name'] == specie]['Confidence']) * 100)) +
+                                '   ' + '   Median:' + str(
+                                    '{:.2f}%'.format(np.median(df2[df2['Com_Name'] == specie]['Confidence']) * 100)))
+
+
+            recordings=df2[df2['Com_Name']==specie]['File_Name']
+            
+        
+            with col2:
+                try:
+                    recording = st.selectbox('Available recordings', recordings.sort_index(ascending=False))
+                    date_specie = df2.loc[df2['File_Name']==recording,['Date','Com_Name']]
+                    date_dir = date_specie['Date'].values[0]
+                    specie_dir = date_specie['Com_Name'].values[0].replace(" ","_")
+                    st.image(userDir + '/BirdSongs/Extracted/By_Date/'+ date_dir + '/'+ specie_dir + '/' + recording + '.png')
+                    st.audio(userDir +'/BirdSongs/Extracted/By_Date/'+ date_dir + '/'+ specie_dir + '/' + recording)
+                except:
+                    st.title('RECORDING NOT AVAILABLE :(')    
+            # try:
+            #     con = sqlite3.connect(userDir + '/BirdNET-Pi/scripts/birds.db')
+            #     cur = con.cursor()
+            cola, colb, colc, cold = st.columns((3,1,1,1))
+            with colb:
+                seen = st.checkbox('Reviewed')
+            if seen:
+                with colc:
+                    verified = st.radio("Verification",['True Positive','False Positive'])
+            
+                    if verified == "False Positive":
+                        df_names = pd.read_csv(userDir+'/BirdNET-Pi/model/labels.txt', delimiter= '_', names=['Sci_Name', 'Com_Name']) 
+                        df_unknown= pd.DataFrame({"Sci_Name":["UNKNOWN"],"Com_Name":["UNKNOWN"]})
+                        df_names = pd.concat([df_unknown,df_names], ignore_index=True)
+                        with cold:
+                            corrected = st.selectbox('What specie?', df_names['Com_Name'])
+            #     cur.execute("UPDATE detections SET  Seen = seen WHERE File_Name = recording")
+            #     con.commit()
+            #     con.close()
+            
+            # except BaseException:
+            #     print("Database busy")
+            #     time.sleep(2)
+            
+            
     else:
+
+        specie = st.selectbox(
+        'Which bird would you like to explore for the dates ' 
+        + str(start_date) + ' to ' + str(end_date) + '?', 
+        species[1:],
+        index = 0)
+
+
+    # filt = df2[df2['Com_Name'] == specie]
+
+        df_counts = int(hourly[hourly.index==specie]['All'])
         fig = st.container()
         fig = make_subplots(
-            rows=1, cols=1)
+                        rows=1, cols =1)
     #                     specs= [[{"type":"xy","rowspan":1},{"type":"heatmap","rowspan":1}]],
+                            
 
     #                     subplot_titles=('<b>Daily Top '+ str(top_N) + ' Species in Date Range '+ str(start_date) +' to '+ str(end_date) +'</b>',
     #                                     '<b>Daily ' + specie+ ' Detections on 15 minute intervals </b>'),
@@ -257,11 +362,12 @@ if daily is False:
     # #                                     '   Confidence Max:'+str('{:.2f}%'.format(max(df2[df2['Com_Name']==specie]['Confidence'])*100))+
     # #                                     '   '+'   Median:'+str('{:.2f}%'.format(np.median(df2[df2['Com_Name']==specie]['Confidence'])*100))
     # #                                     )
+    #                     )
 
 #        fig.add_trace(go.Bar(y=top_N_species.index, x=top_N_species, orientation='h'), row=1,col=1)
-        df4 = df2['Com_Name'][df2['Com_Name'] == specie].resample('15min').count()
-        df4.index = [df4.index.date, df4.index.time]
-        day_hour_freq = df4.unstack().fillna(0)
+        df4=df2['Com_Name'][df2['Com_Name']==specie].resample('15min').count()
+        df4.index=[df4.index.date, df4.index.time]
+        day_hour_freq=df4.unstack().fillna(0)
 
         fig_x = [d.strftime('%d-%m-%Y') for d in day_hour_freq.index.tolist()]
         fig_y = [h.strftime('%H:%M') for h in day_hour_freq.columns.tolist()]
@@ -271,10 +377,10 @@ if daily is False:
         # fig.update_layout(
         # margin=dict(l=0, r=0, t=50, b=0),
         # yaxis={'categoryorder':'total ascending'})
-        color_pals = px.colors.named_colorscales()
+        color_pals= px.colors.named_colorscales()
         selected_pal = st.sidebar.selectbox('Select Color Pallet for Daily Detections', color_pals)
-        fig.add_trace(go.Heatmap(x=fig_x, y=fig_y, z=fig_z, autocolorscale=False, colorscale=selected_pal), row=1, col=1)
-
+        fig.add_trace(go.Heatmap(x=fig_x,y=fig_y,z=fig_z, autocolorscale = False, colorscale = selected_pal), row=1, col=1)
+        st.plotly_chart(fig, use_container_width=True)  # , config=config)
 else:
     fig = make_subplots(
         rows=1, cols=2,
@@ -327,12 +433,12 @@ else:
                       margin=dict(l=0, r=0, t=50, b=0))
 # container=st.container()
 # config={'displayModelBar': False}
-st.plotly_chart(fig, use_container_width=True)  # , config=config)
+    st.plotly_chart(fig, use_container_width=True)  # , config=config)
 
 # cols3,cols4=st.columns((1,1))
-#
+# 
 # extract_date=Date_Slider
-#
+# 
 # audio_file = open('/home/*/BirdSongs/Extracted/By_Date/2022-03-22/Yellow-streaked_Greenbul/Yellow-streaked_Greenbul-77-2022-03-22-birdnet-15:04:28.mp3', 'rb')
 # audio_bytes = audio_file.read()
 # cols4.audio(audio_bytes, format='audio/mp3')