claudecode/AvianVisitors
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Updated Species stats with start of Species verification

Browse Source
This commit is contained in:
CaiusX
2022-05-31 17:39:25 +02:00
parent d5a2dbae58
commit 6ef05c2bc3
1 changed files with 207 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/plotly_streamlit.py
+207 -101
View File
@@ -6,11 +6,13 @@ from numpy import ma
import plotly.graph_objects as go import plotly.graph_objects as go
from plotly.subplots import make_subplots from plotly.subplots import make_subplots
import plotly.io as pio import plotly.io as pio
from datetime import timedelta from datetime import timedelta, datetime
from pathlib import Path
import sqlite3 import sqlite3
from sqlite3 import Connection from sqlite3 import Connection
import plotly.express as px import plotly.express as px
from sklearn.preprocessing import normalize from sklearn.preprocessing import normalize
import time
pio.templates.default = "plotly_white" pio.templates.default = "plotly_white"
@@ -18,7 +20,7 @@ userDir = os.path.expanduser('~')
URI_SQLITE_DB = userDir + '/BirdNET-Pi/scripts/birds.db' URI_SQLITE_DB = userDir + '/BirdNET-Pi/scripts/birds.db'
st.set_page_config(layout='wide') st.set_page_config(layout='wide')
#
# Remove whitespace from the top of the page # Remove whitespace from the top of the page
st.markdown(""" st.markdown("""
<style> <style>
@@ -38,8 +40,9 @@ st.markdown("""
""", unsafe_allow_html=True) """, unsafe_allow_html=True)
# @st.cache(hash_funcs={Connection: id})
@st.cache(allow_output_mutation=True) @st.cache(hash_funcs={Connection: id})
#@st.cache(allow_output_mutation=True)
def get_connection(path: str): def get_connection(path: str):
return sqlite3.connect(path, check_same_thread=False) return sqlite3.connect(path, check_same_thread=False)
@@ -55,19 +58,19 @@ df2 = df.copy()
df2['DateTime'] = pd.to_datetime(df2['Date'] + " " + df2['Time']) df2['DateTime'] = pd.to_datetime(df2['Date'] + " " + df2['Time'])
df2 = df2.set_index('DateTime') df2 = df2.set_index('DateTime')
daily = st.sidebar.checkbox('Single Day View', help='Select if you want single day view, unselect for multi-day views') daily = st.sidebar.checkbox('Single Day View', help= 'Select if you want single day view, unselect for multi-day views')
if daily: if daily:
# Date as slider # Date as slider
Start_Date = pd.to_datetime(df2.index.min()).date() Start_Date = pd.to_datetime(df2.index.min()).date()
End_Date = pd.to_datetime(df2.index.max()).date() End_Date = pd.to_datetime(df2.index.max()).date()
# cols1, cols2 = st.columns((1, 1)) # cols1, cols2 = st.columns((1, 1))
end_date = st.sidebar.slider('Date to View', end_date = st.sidebar.date_input('Date to View',
min_value=Start_Date, min_value = Start_Date,
max_value=End_Date, max_value = End_Date,
value=(End_Date), value=(End_Date),
help='Select date for single day view' help= 'Select date for single day view'
) )
start_date = end_date start_date = end_date
else: else:
Start_Date = pd.to_datetime(df2.index.min()).date() Start_Date = pd.to_datetime(df2.index.min()).date()
@@ -75,11 +78,11 @@ else:
# cols1, cols2 = st.columns((1, 1)) # cols1, cols2 = st.columns((1, 1))
start_date, end_date = st.sidebar.slider('Date Range', start_date, end_date = st.sidebar.slider('Date Range',
min_value=Start_Date - timedelta(days=1), min_value = Start_Date-timedelta(days=1),
max_value=End_Date, max_value = End_Date,
value=(Start_Date, End_Date), value=(Start_Date, End_Date),
help='Select start and end date, if same date get a clockplot for a single day' help= 'Select start and end date, if same date get a clockplot for a single day'
) )
# start_date, end_date = cols1.date_input( # start_date, end_date = cols1.date_input(
# "Date Input for Analysis - select Range for single specie analysis, select single date for daily view", # "Date Input for Analysis - select Range for single specie analysis, select single date for daily view",
@@ -90,14 +93,12 @@ else:
# start_date = datetime(2022 ,5 ,17).date() # start_date = datetime(2022 ,5 ,17).date()
# end_date = datetime(2022 ,5 ,17).date() # end_date = datetime(2022 ,5 ,17).date()
@st.cache() @st.cache()
def date_filter(df, start_date, end_date): def date_filter(df, start_date, end_date):
filt = (df2.index >= pd.Timestamp(start_date)) & (df2.index <= pd.Timestamp(end_date + timedelta(days=1))) filt = (df2.index >= pd.Timestamp(start_date)) & (df2.index <= pd.Timestamp(end_date + timedelta(days=1)))
df = df[filt] df = df[filt]
return(df) return(df)
df2 = date_filter(df2, start_date, end_date) df2 = date_filter(df2, start_date, end_date)
st.write('<style>div.row-widget.stRadio > div{flex-direction:row;justify-content: left;} </style>', st.write('<style>div.row-widget.stRadio > div{flex-direction:row;justify-content: left;} </style>',
@@ -111,7 +112,7 @@ st.write('<style>div.st-bf{flex-direction:column;} div.st-ag{font-weight:bold;pa
if start_date == end_date: if start_date == end_date:
resample_sel = st.sidebar.radio( resample_sel = st.sidebar.radio(
"Resample Resolution", "Resample Resolution",
('Raw', '15 minutes', 'Hourly'), index=1, help='Select resolution for single day - larger times run faster') ('Raw', '15 minutes', 'Hourly'), index=1, help= 'Select resolution for single day - larger times run faster' )
resample_times = {'Raw': 'Raw', resample_times = {'Raw': 'Raw',
'1 minute': '1min', '1 minute': '1min',
@@ -123,7 +124,7 @@ if start_date == end_date:
else: else:
resample_sel = st.sidebar.radio( resample_sel = st.sidebar.radio(
"Resample Resolution", "Resample Resolution",
('Raw', '15 minutes', 'Hourly', 'DAILY'), index=1, help='Select resolution for species - DAILY provides time series') ('Raw', '15 minutes', 'Hourly', 'DAILY'), index=1, help= 'Select resolution for species - DAILY provides time series')
resample_times = {'Raw': 'Raw', resample_times = {'Raw': 'Raw',
'1 minute': '1min', '1 minute': '1min',
@@ -133,7 +134,6 @@ else:
} }
resample_time = resample_times[resample_sel] resample_time = resample_times[resample_sel]
@st.cache() @st.cache()
def time_resample(df, resample_time): def time_resample(df, resample_time):
if resample_time == 'Raw': if resample_time == 'Raw':
@@ -143,8 +143,6 @@ def time_resample(df, resample_time):
df_resample = df.resample(resample_time)['Com_Name'].aggregate('unique').explode() df_resample = df.resample(resample_time)['Com_Name'].aggregate('unique').explode()
return(df_resample) return(df_resample)
top_bird = df2['Com_Name'].mode()[0] top_bird = df2['Com_Name'].mode()[0]
df5 = time_resample(df2, resample_time) df5 = time_resample(df2, resample_time)
@@ -153,12 +151,12 @@ df5 = time_resample(df2, resample_time)
Specie_Count = df5.value_counts() Specie_Count = df5.value_counts()
# Create Hourly Crosstab # Create Hourly Crosstab
hourly = pd.crosstab(df5, df5.index.hour, dropna=False) hourly = pd.crosstab(df5, df5.index.hour, dropna=True, margins= True)
# Filter on species # Filter on species
species = list(hourly.index) species = list(hourly.sort_values("All", ascending= False).index)
# cols1, cols2 = st.columns((1, 1)) #cols1, cols2 = st.columns((1, 1))
top_N = st.sidebar.slider( top_N = st.sidebar.slider(
'Select Number of Birds to Show', 'Select Number of Birds to Show',
min_value=1, min_value=1,
@@ -170,86 +168,193 @@ top_N_species = (df5.value_counts()[:top_N])
font_size = 15 font_size = 15
if daily is False: if daily == False:
specie = st.selectbox(
'Which bird would you like to explore for the dates '
+ str(start_date) + ' to ' + str(end_date) + '?',
species,
index=species.index(top_bird))
filt = df2['Com_Name'] == specie
df_counts = sum(df5 == specie)
if resample_time != '1D': if resample_time != '1D':
specie = st.selectbox(
'Which bird would you like to explore for the dates '
+ str(start_date) + ' to ' + str(end_date) + '?',
species,
index = 0)
fig = make_subplots(
rows=3, cols=2, # filt = df2['Com_Name'] == specie
specs=[[{"type": "xy", "rowspan": 3}, {"type": "polar", "rowspan": 2}], [{"rowspan": 1}, {"rowspan": 1}], if specie == 'All':
[None, {"type": "xy", "rowspan": 1}]], df_counts = int(hourly[hourly.index==specie]['All'])
subplot_titles=('<b>Top ' + str(top_N) + ' Species in Date Range ' + str(start_date) + ' to ' + str( fig = make_subplots(
end_date) + '<br>for ' + str(resample_sel) + ' sampling interval.' + '</b>', rows=3, cols=2,
'Total Detect:' + str('{:,}'.format(df_counts)) + specs=[[{"type": "xy", "rowspan": 3}, {"type": "polar", "rowspan": 2}], [{"rowspan": 1}, {"rowspan": 1}],
' Confidence Max:' + str( [None, {"type": "xy", "rowspan": 1}]],
'{:.2f}%'.format(max(df2[df2['Com_Name'] == specie]['Confidence']) * 100)) + subplot_titles=('<b>Top ' + str(top_N) + ' Species in Date Range ' + str(start_date) + ' to ' + str(
' ' + ' Median:' + str( end_date) + ' for ' + str(resample_sel) + ' sampling interval.' + '</b>',
'{:.2f}%'.format(np.median(df2[df2['Com_Name'] == specie]['Confidence']) * 100)) 'Total Detect:' + str('{:,}'.format(df_counts))
# + ' Confidence Max:' + str(
# '{:.2f}%'.format(max(df2[df2['Com_Name'] == specie]['Confidence']) * 100)) +
# ' ' + ' Median:' + str(
# '{:.2f}%'.format(np.median(df2[df2['Com_Name'] == specie]['Confidence']) * 100))
)
) )
) fig.layout.annotations[1].update(x=0.7, y=0.25, font_size=15)
fig.layout.annotations[1].update(x=0.7, y=0.25, font_size=15)
# Plot seen species for selected date range and number of species # Plot seen species for selected date range and number of species
fig.add_trace(go.Bar(y=top_N_species.index, x=top_N_species, orientation='h', marker_color='seagreen'), row=1, col=1)
fig.add_trace(go.Bar(y=top_N_species.index, x=top_N_species, orientation='h', marker_color='seagreen'), row=1, col=1)
fig.update_layout(
margin=dict(l=0, r=0, t=50, b=0),
yaxis={'categoryorder': 'total ascending'})
fig.update_layout( # Set 360 degrees, 24 hours for polar plot
margin=dict(l=0, r=0, t=50, b=0), theta = np.linspace(0.0, 360, 24, endpoint=False)
yaxis={'categoryorder': 'total ascending'})
# Set 360 degrees, 24 hours for polar plot specie_filt = df5 == specie
theta = np.linspace(0.0, 360, 24, endpoint=False) df3 = df5[specie_filt]
specie_filt = df5 == specie detections2 = pd.crosstab(df3, df3.index.hour)
df3 = df5[specie_filt]
detections2 = pd.crosstab(df3, df3.index.hour) d = pd.DataFrame(np.zeros((23, 1))).squeeze()
detections = hourly.loc[specie]
d = pd.DataFrame(np.zeros((23, 1))).squeeze() detections = (d + detections).fillna(0)
detections = hourly.loc[specie] fig.add_trace(go.Barpolar(r=detections, theta=theta, marker_color='seagreen'), row=1, col=2)
detections = (d + detections).fillna(0) fig.update_layout(
fig.add_trace(go.Barpolar(r=detections, theta=theta, marker_color='seagreen'), row=1, col=2) autosize=False,
fig.update_layout( width=1000,
autosize=False, height=500,
width=1000, showlegend=False,
height=500, polar=dict(
showlegend=False, radialaxis=dict(
polar=dict( tickfont_size=font_size,
radialaxis=dict( showticklabels=False,
tickfont_size=font_size, hoverformat="#%{theta}: <br>Popularity: %{percent} </br> %{r}"
showticklabels=False, ),
hoverformat="#%{theta}: <br>Popularity: %{percent} </br> %{r}" angularaxis=dict(
tickfont_size=font_size,
rotation=-90,
direction='clockwise',
tickmode='array',
tickvals=[0, 15, 35, 45, 60, 75, 90, 105, 120, 135, 150, 165, 180, 195, 210, 225, 240, 255, 270,
285, 300, 315, 330, 345],
ticktext=['12am', '1am', '2am', '3am', '4am', '5am', '6am', '7am', '8am', '9am', '10am', '11am',
'12pm', '1pm', '2pm', '3pm', '4pm', '5pm', '6pm', '7pm', '8pm', '9pm', '10pm', '11pm'],
hoverformat="#%{theta}: <br>Popularity: %{percent} </br> %{r}"
),
), ),
angularaxis=dict( )
tickfont_size=font_size,
rotation=-90,
direction='clockwise',
tickmode='array',
tickvals=[0, 15, 35, 45, 60, 75, 90, 105, 120, 135, 150, 165, 180, 195, 210, 225, 240, 255, 270,
285, 300, 315, 330, 345],
ticktext=['12am', '1am', '2am', '3am', '4am', '5am', '6am', '7am', '8am', '9am', '10am', '11am',
'12pm', '1pm', '2pm', '3pm', '4pm', '5pm', '6pm', '7pm', '8pm', '9pm', '10pm', '11pm'],
hoverformat="#%{theta}: <br>Popularity: %{percent} </br> %{r}"
),
),
)
daily = pd.crosstab(df5, df5.index.date, dropna=False) daily = pd.crosstab(df5, df5.index.date, dropna=True, margins = True)
fig.add_trace(go.Bar(x=daily.columns[:-1], y=daily.loc[specie][:-1], marker_color='seagreen'), row=3, col=2)
st.plotly_chart(fig, use_container_width=True) # , config=config)
fig.add_trace(go.Bar(x=daily.columns, y=daily.loc[specie], marker_color='seagreen'), row=3, col=2) else:
col1, col2 = st.columns(2)
with col1:
fig = make_subplots(
rows=3, cols=1,
specs=[[{"type": "polar", "rowspan": 2}],[{"rowspan": 1}], [{"type": "xy", "rowspan": 1}]]
)
# Set 360 degrees, 24 hours for polar plot
theta = np.linspace(0.0, 360, 24, endpoint=False)
specie_filt = df5 == specie
df3 = df5[specie_filt]
detections2 = pd.crosstab(df3, df3.index.hour)
d = pd.DataFrame(np.zeros((23, 1))).squeeze()
detections = hourly.loc[specie]
detections = (d + detections).fillna(0)
fig.add_trace(go.Barpolar(r=detections, theta=theta, marker_color='seagreen'), row=1, col=1)
fig.update_layout(
autosize=False,
width=1000,
height=500,
showlegend=False,
polar=dict(
radialaxis=dict(
tickfont_size=font_size,
showticklabels=False,
hoverformat="#%{theta}: <br>Popularity: %{percent} </br> %{r}"
),
angularaxis=dict(
tickfont_size=font_size,
rotation=-90,
direction='clockwise',
tickmode='array',
tickvals=[0, 15, 35, 45, 60, 75, 90, 105, 120, 135, 150, 165, 180, 195, 210, 225, 240, 255, 270,
285, 300, 315, 330, 345],
ticktext=['12am', '1am', '2am', '3am', '4am', '5am', '6am', '7am', '8am', '9am', '10am', '11am',
'12pm', '1pm', '2pm', '3pm', '4pm', '5pm', '6pm', '7pm', '8pm', '9pm', '10pm', '11pm'],
hoverformat="#%{theta}: <br>Popularity: %{percent} </br> %{r}"
),
),
)
daily = pd.crosstab(df5, df5.index.date, dropna=True, margins = True)
fig.add_trace(go.Bar(x=daily.columns[:-1], y=daily.loc[specie][:-1], marker_color='seagreen'), row=3, col=1)
st.plotly_chart(fig, use_container_width=True) # , config=config)
df_counts = int(hourly[hourly.index==specie]['All'])
st.subheader('Total Detect:' + str('{:,}'.format(df_counts))
+ ' Confidence Max:' + str(
'{:.2f}%'.format(max(df2[df2['Com_Name'] == specie]['Confidence']) * 100)) +
' ' + ' Median:' + str(
'{:.2f}%'.format(np.median(df2[df2['Com_Name'] == specie]['Confidence']) * 100)))
recordings=df2[df2['Com_Name']==specie]['File_Name']
with col2:
try:
recording = st.selectbox('Available recordings', recordings.sort_index(ascending=False))
date_specie = df2.loc[df2['File_Name']==recording,['Date','Com_Name']]
date_dir = date_specie['Date'].values[0]
specie_dir = date_specie['Com_Name'].values[0].replace(" ","_")
st.image(userDir + '/BirdSongs/Extracted/By_Date/'+ date_dir + '/'+ specie_dir + '/' + recording + '.png')
st.audio(userDir +'/BirdSongs/Extracted/By_Date/'+ date_dir + '/'+ specie_dir + '/' + recording)
except:
st.title('RECORDING NOT AVAILABLE :(')
# try:
# con = sqlite3.connect(userDir + '/BirdNET-Pi/scripts/birds.db')
# cur = con.cursor()
cola, colb, colc, cold = st.columns((3,1,1,1))
with colb:
seen = st.checkbox('Reviewed')
if seen:
with colc:
verified = st.radio("Verification",['True Positive','False Positive'])
if verified == "False Positive":
df_names = pd.read_csv(userDir+'/BirdNET-Pi/model/labels.txt', delimiter= '_', names=['Sci_Name', 'Com_Name'])
df_unknown= pd.DataFrame({"Sci_Name":["UNKNOWN"],"Com_Name":["UNKNOWN"]})
df_names = pd.concat([df_unknown,df_names], ignore_index=True)
with cold:
corrected = st.selectbox('What specie?', df_names['Com_Name'])
# cur.execute("UPDATE detections SET Seen = seen WHERE File_Name = recording")
# con.commit()
# con.close()
# except BaseException:
# print("Database busy")
# time.sleep(2)
else: else:
specie = st.selectbox(
'Which bird would you like to explore for the dates '
+ str(start_date) + ' to ' + str(end_date) + '?',
species[1:],
index = 0)
# filt = df2[df2['Com_Name'] == specie]
df_counts = int(hourly[hourly.index==specie]['All'])
fig = st.container() fig = st.container()
fig = make_subplots( fig = make_subplots(
rows=1, cols=1) rows=1, cols =1)
# specs= [[{"type":"xy","rowspan":1},{"type":"heatmap","rowspan":1}]], # specs= [[{"type":"xy","rowspan":1},{"type":"heatmap","rowspan":1}]],
# subplot_titles=('<b>Daily Top '+ str(top_N) + ' Species in Date Range '+ str(start_date) +' to '+ str(end_date) +'</b>', # subplot_titles=('<b>Daily Top '+ str(top_N) + ' Species in Date Range '+ str(start_date) +' to '+ str(end_date) +'</b>',
# '<b>Daily ' + specie+ ' Detections on 15 minute intervals </b>'), # '<b>Daily ' + specie+ ' Detections on 15 minute intervals </b>'),
@@ -257,11 +362,12 @@ if daily is False:
# # ' Confidence Max:'+str('{:.2f}%'.format(max(df2[df2['Com_Name']==specie]['Confidence'])*100))+ # # ' Confidence Max:'+str('{:.2f}%'.format(max(df2[df2['Com_Name']==specie]['Confidence'])*100))+
# # ' '+' Median:'+str('{:.2f}%'.format(np.median(df2[df2['Com_Name']==specie]['Confidence'])*100)) # # ' '+' Median:'+str('{:.2f}%'.format(np.median(df2[df2['Com_Name']==specie]['Confidence'])*100))
# # ) # # )
# )
# fig.add_trace(go.Bar(y=top_N_species.index, x=top_N_species, orientation='h'), row=1,col=1) # fig.add_trace(go.Bar(y=top_N_species.index, x=top_N_species, orientation='h'), row=1,col=1)
df4 = df2['Com_Name'][df2['Com_Name'] == specie].resample('15min').count() df4=df2['Com_Name'][df2['Com_Name']==specie].resample('15min').count()
df4.index = [df4.index.date, df4.index.time] df4.index=[df4.index.date, df4.index.time]
day_hour_freq = df4.unstack().fillna(0) day_hour_freq=df4.unstack().fillna(0)
fig_x = [d.strftime('%d-%m-%Y') for d in day_hour_freq.index.tolist()] fig_x = [d.strftime('%d-%m-%Y') for d in day_hour_freq.index.tolist()]
fig_y = [h.strftime('%H:%M') for h in day_hour_freq.columns.tolist()] fig_y = [h.strftime('%H:%M') for h in day_hour_freq.columns.tolist()]
@@ -271,10 +377,10 @@ if daily is False:
# fig.update_layout( # fig.update_layout(
# margin=dict(l=0, r=0, t=50, b=0), # margin=dict(l=0, r=0, t=50, b=0),
# yaxis={'categoryorder':'total ascending'}) # yaxis={'categoryorder':'total ascending'})
color_pals = px.colors.named_colorscales() color_pals= px.colors.named_colorscales()
selected_pal = st.sidebar.selectbox('Select Color Pallet for Daily Detections', color_pals) selected_pal = st.sidebar.selectbox('Select Color Pallet for Daily Detections', color_pals)
fig.add_trace(go.Heatmap(x=fig_x, y=fig_y, z=fig_z, autocolorscale=False, colorscale=selected_pal), row=1, col=1) fig.add_trace(go.Heatmap(x=fig_x,y=fig_y,z=fig_z, autocolorscale = False, colorscale = selected_pal), row=1, col=1)
st.plotly_chart(fig, use_container_width=True) # , config=config)
else: else:
fig = make_subplots( fig = make_subplots(
rows=1, cols=2, rows=1, cols=2,
@@ -327,12 +433,12 @@ else:
margin=dict(l=0, r=0, t=50, b=0)) margin=dict(l=0, r=0, t=50, b=0))
# container=st.container() # container=st.container()
# config={'displayModelBar': False} # config={'displayModelBar': False}
st.plotly_chart(fig, use_container_width=True) # , config=config) st.plotly_chart(fig, use_container_width=True) # , config=config)
# cols3,cols4=st.columns((1,1)) # cols3,cols4=st.columns((1,1))
# #
# extract_date=Date_Slider # extract_date=Date_Slider
# #
# audio_file = open('/home/*/BirdSongs/Extracted/By_Date/2022-03-22/Yellow-streaked_Greenbul/Yellow-streaked_Greenbul-77-2022-03-22-birdnet-15:04:28.mp3', 'rb') # audio_file = open('/home/*/BirdSongs/Extracted/By_Date/2022-03-22/Yellow-streaked_Greenbul/Yellow-streaked_Greenbul-77-2022-03-22-birdnet-15:04:28.mp3', 'rb')
# audio_bytes = audio_file.read() # audio_bytes = audio_file.read()
# cols4.audio(audio_bytes, format='audio/mp3') # cols4.audio(audio_bytes, format='audio/mp3')