claudecode/AvianVisitors
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Updated Species stats with start of Species verification

Browse Source
This commit is contained in:
CaiusX
2022-05-31 17:39:25 +02:00
parent d5a2dbae58
commit 6ef05c2bc3
1 changed files with 207 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/plotly_streamlit.py
+207 -101
View File
@@ -6,11 +6,13 @@ from numpy import ma
import plotly.graph_objects as go
from plotly.subplots import make_subplots
import plotly.io as pio
from datetime import timedelta
from datetime import timedelta, datetime
from pathlib import Path
import sqlite3
from sqlite3 import Connection
import plotly.express as px
from sklearn.preprocessing import normalize
import time
pio.templates.default = "plotly_white"
@@ -18,7 +20,7 @@ userDir = os.path.expanduser('~')
URI_SQLITE_DB = userDir + '/BirdNET-Pi/scripts/birds.db'
st.set_page_config(layout='wide')
#
# Remove whitespace from the top of the page
st.markdown("""
<style>
@@ -38,8 +40,9 @@ st.markdown("""
""", unsafe_allow_html=True)
# @st.cache(hash_funcs={Connection: id})
@st.cache(allow_output_mutation=True)
@st.cache(hash_funcs={Connection: id})
#@st.cache(allow_output_mutation=True)
def get_connection(path: str):
return sqlite3.connect(path, check_same_thread=False)
@@ -55,19 +58,19 @@ df2 = df.copy()
df2['DateTime'] = pd.to_datetime(df2['Date'] + " " + df2['Time'])
df2 = df2.set_index('DateTime')
daily = st.sidebar.checkbox('Single Day View', help='Select if you want single day view, unselect for multi-day views')
daily = st.sidebar.checkbox('Single Day View', help= 'Select if you want single day view, unselect for multi-day views')
if daily:
# Date as slider
# Date as slider
Start_Date = pd.to_datetime(df2.index.min()).date()
End_Date = pd.to_datetime(df2.index.max()).date()
# cols1, cols2 = st.columns((1, 1))
end_date = st.sidebar.slider('Date to View',
min_value=Start_Date,
max_value=End_Date,
value=(End_Date),
help='Select date for single day view'
)
end_date = st.sidebar.date_input('Date to View',
min_value = Start_Date,
max_value = End_Date,
value=(End_Date),
help= 'Select date for single day view'
)
start_date = end_date
else:
Start_Date = pd.to_datetime(df2.index.min()).date()
@@ -75,11 +78,11 @@ else:
# cols1, cols2 = st.columns((1, 1))
start_date, end_date = st.sidebar.slider('Date Range',
min_value=Start_Date - timedelta(days=1),
max_value=End_Date,
value=(Start_Date, End_Date),
help='Select start and end date, if same date get a clockplot for a single day'
)
min_value = Start_Date-timedelta(days=1),
max_value = End_Date,
value=(Start_Date, End_Date),
help= 'Select start and end date, if same date get a clockplot for a single day'
)
# start_date, end_date = cols1.date_input(
# "Date Input for Analysis - select Range for single specie analysis, select single date for daily view",
@@ -90,14 +93,12 @@ else:
# start_date = datetime(2022 ,5 ,17).date()
# end_date = datetime(2022 ,5 ,17).date()
@st.cache()
def date_filter(df, start_date, end_date):
filt = (df2.index >= pd.Timestamp(start_date)) & (df2.index <= pd.Timestamp(end_date + timedelta(days=1)))
df = df[filt]
return(df)
df2 = date_filter(df2, start_date, end_date)
st.write('<style>div.row-widget.stRadio > div{flex-direction:row;justify-content: left;} </style>',
@@ -111,7 +112,7 @@ st.write('<style>div.st-bf{flex-direction:column;} div.st-ag{font-weight:bold;pa
if start_date == end_date:
resample_sel = st.sidebar.radio(
"Resample Resolution",
('Raw', '15 minutes', 'Hourly'), index=1, help='Select resolution for single day - larger times run faster')
('Raw', '15 minutes', 'Hourly'), index=1, help= 'Select resolution for single day - larger times run faster' )
resample_times = {'Raw': 'Raw',
'1 minute': '1min',
@@ -123,7 +124,7 @@ if start_date == end_date:
else:
resample_sel = st.sidebar.radio(
"Resample Resolution",
('Raw', '15 minutes', 'Hourly', 'DAILY'), index=1, help='Select resolution for species - DAILY provides time series')
('Raw', '15 minutes', 'Hourly', 'DAILY'), index=1, help= 'Select resolution for species - DAILY provides time series')
resample_times = {'Raw': 'Raw',
'1 minute': '1min',
@@ -133,7 +134,6 @@ else:
}
resample_time = resample_times[resample_sel]
@st.cache()
def time_resample(df, resample_time):
if resample_time == 'Raw':
@@ -143,8 +143,6 @@ def time_resample(df, resample_time):
df_resample = df.resample(resample_time)['Com_Name'].aggregate('unique').explode()
return(df_resample)
top_bird = df2['Com_Name'].mode()[0]
df5 = time_resample(df2, resample_time)
@@ -153,12 +151,12 @@ df5 = time_resample(df2, resample_time)
Specie_Count = df5.value_counts()
# Create Hourly Crosstab
hourly = pd.crosstab(df5, df5.index.hour, dropna=False)
hourly = pd.crosstab(df5, df5.index.hour, dropna=True, margins= True)
# Filter on species
species = list(hourly.index)
species = list(hourly.sort_values("All", ascending= False).index)
# cols1, cols2 = st.columns((1, 1))
#cols1, cols2 = st.columns((1, 1))
top_N = st.sidebar.slider(
'Select Number of Birds to Show',
min_value=1,
@@ -170,86 +168,193 @@ top_N_species = (df5.value_counts()[:top_N])
font_size = 15
if daily is False:
specie = st.selectbox(
'Which bird would you like to explore for the dates '
+ str(start_date) + ' to ' + str(end_date) + '?',
species,
index=species.index(top_bird))
filt = df2['Com_Name'] == specie
df_counts = sum(df5 == specie)
if daily == False:
if resample_time != '1D':
specie = st.selectbox(
'Which bird would you like to explore for the dates '
+ str(start_date) + ' to ' + str(end_date) + '?',
species,
index = 0)
fig = make_subplots(
rows=3, cols=2,
specs=[[{"type": "xy", "rowspan": 3}, {"type": "polar", "rowspan": 2}], [{"rowspan": 1}, {"rowspan": 1}],
[None, {"type": "xy", "rowspan": 1}]],
subplot_titles=('<b>Top ' + str(top_N) + ' Species in Date Range ' + str(start_date) + ' to ' + str(
end_date) + '<br>for ' + str(resample_sel) + ' sampling interval.' + '</b>',
'Total Detect:' + str('{:,}'.format(df_counts)) +
' Confidence Max:' + str(
'{:.2f}%'.format(max(df2[df2['Com_Name'] == specie]['Confidence']) * 100)) +
' ' + ' Median:' + str(
'{:.2f}%'.format(np.median(df2[df2['Com_Name'] == specie]['Confidence']) * 100))
# filt = df2['Com_Name'] == specie
if specie == 'All':
df_counts = int(hourly[hourly.index==specie]['All'])
fig = make_subplots(
rows=3, cols=2,
specs=[[{"type": "xy", "rowspan": 3}, {"type": "polar", "rowspan": 2}], [{"rowspan": 1}, {"rowspan": 1}],
[None, {"type": "xy", "rowspan": 1}]],
subplot_titles=('<b>Top ' + str(top_N) + ' Species in Date Range ' + str(start_date) + ' to ' + str(
end_date) + ' for ' + str(resample_sel) + ' sampling interval.' + '</b>',
'Total Detect:' + str('{:,}'.format(df_counts))
# + ' Confidence Max:' + str(
# '{:.2f}%'.format(max(df2[df2['Com_Name'] == specie]['Confidence']) * 100)) +
# ' ' + ' Median:' + str(
# '{:.2f}%'.format(np.median(df2[df2['Com_Name'] == specie]['Confidence']) * 100))
)
)
)
fig.layout.annotations[1].update(x=0.7, y=0.25, font_size=15)
fig.layout.annotations[1].update(x=0.7, y=0.25, font_size=15)
# Plot seen species for selected date range and number of species
fig.add_trace(go.Bar(y=top_N_species.index, x=top_N_species, orientation='h', marker_color='seagreen'), row=1, col=1)
# Plot seen species for selected date range and number of species
fig.add_trace(go.Bar(y=top_N_species.index, x=top_N_species, orientation='h', marker_color='seagreen'), row=1, col=1)
fig.update_layout(
margin=dict(l=0, r=0, t=50, b=0),
yaxis={'categoryorder': 'total ascending'})
fig.update_layout(
margin=dict(l=0, r=0, t=50, b=0),
yaxis={'categoryorder': 'total ascending'})
# Set 360 degrees, 24 hours for polar plot
theta = np.linspace(0.0, 360, 24, endpoint=False)
# Set 360 degrees, 24 hours for polar plot
theta = np.linspace(0.0, 360, 24, endpoint=False)
specie_filt = df5 == specie
df3 = df5[specie_filt]
specie_filt = df5 == specie
df3 = df5[specie_filt]
detections2 = pd.crosstab(df3, df3.index.hour)
detections2 = pd.crosstab(df3, df3.index.hour)
d = pd.DataFrame(np.zeros((23, 1))).squeeze()
detections = hourly.loc[specie]
detections = (d + detections).fillna(0)
fig.add_trace(go.Barpolar(r=detections, theta=theta, marker_color='seagreen'), row=1, col=2)
fig.update_layout(
autosize=False,
width=1000,
height=500,
showlegend=False,
polar=dict(
radialaxis=dict(
tickfont_size=font_size,
showticklabels=False,
hoverformat="#%{theta}: <br>Popularity: %{percent} </br> %{r}"
d = pd.DataFrame(np.zeros((23, 1))).squeeze()
detections = hourly.loc[specie]
detections = (d + detections).fillna(0)
fig.add_trace(go.Barpolar(r=detections, theta=theta, marker_color='seagreen'), row=1, col=2)
fig.update_layout(
autosize=False,
width=1000,
height=500,
showlegend=False,
polar=dict(
radialaxis=dict(
tickfont_size=font_size,
showticklabels=False,
hoverformat="#%{theta}: <br>Popularity: %{percent} </br> %{r}"
),
angularaxis=dict(
tickfont_size=font_size,
rotation=-90,
direction='clockwise',
tickmode='array',
tickvals=[0, 15, 35, 45, 60, 75, 90, 105, 120, 135, 150, 165, 180, 195, 210, 225, 240, 255, 270,
285, 300, 315, 330, 345],
ticktext=['12am', '1am', '2am', '3am', '4am', '5am', '6am', '7am', '8am', '9am', '10am', '11am',
'12pm', '1pm', '2pm', '3pm', '4pm', '5pm', '6pm', '7pm', '8pm', '9pm', '10pm', '11pm'],
hoverformat="#%{theta}: <br>Popularity: %{percent} </br> %{r}"
),
),
angularaxis=dict(
tickfont_size=font_size,
rotation=-90,
direction='clockwise',
tickmode='array',
tickvals=[0, 15, 35, 45, 60, 75, 90, 105, 120, 135, 150, 165, 180, 195, 210, 225, 240, 255, 270,
285, 300, 315, 330, 345],
ticktext=['12am', '1am', '2am', '3am', '4am', '5am', '6am', '7am', '8am', '9am', '10am', '11am',
'12pm', '1pm', '2pm', '3pm', '4pm', '5pm', '6pm', '7pm', '8pm', '9pm', '10pm', '11pm'],
hoverformat="#%{theta}: <br>Popularity: %{percent} </br> %{r}"
),
),
)
)
daily = pd.crosstab(df5, df5.index.date, dropna=False)
daily = pd.crosstab(df5, df5.index.date, dropna=True, margins = True)
fig.add_trace(go.Bar(x=daily.columns[:-1], y=daily.loc[specie][:-1], marker_color='seagreen'), row=3, col=2)
st.plotly_chart(fig, use_container_width=True) # , config=config)
fig.add_trace(go.Bar(x=daily.columns, y=daily.loc[specie], marker_color='seagreen'), row=3, col=2)
else:
col1, col2 = st.columns(2)
with col1:
fig = make_subplots(
rows=3, cols=1,
specs=[[{"type": "polar", "rowspan": 2}],[{"rowspan": 1}], [{"type": "xy", "rowspan": 1}]]
)
# Set 360 degrees, 24 hours for polar plot
theta = np.linspace(0.0, 360, 24, endpoint=False)
specie_filt = df5 == specie
df3 = df5[specie_filt]
detections2 = pd.crosstab(df3, df3.index.hour)
d = pd.DataFrame(np.zeros((23, 1))).squeeze()
detections = hourly.loc[specie]
detections = (d + detections).fillna(0)
fig.add_trace(go.Barpolar(r=detections, theta=theta, marker_color='seagreen'), row=1, col=1)
fig.update_layout(
autosize=False,
width=1000,
height=500,
showlegend=False,
polar=dict(
radialaxis=dict(
tickfont_size=font_size,
showticklabels=False,
hoverformat="#%{theta}: <br>Popularity: %{percent} </br> %{r}"
),
angularaxis=dict(
tickfont_size=font_size,
rotation=-90,
direction='clockwise',
tickmode='array',
tickvals=[0, 15, 35, 45, 60, 75, 90, 105, 120, 135, 150, 165, 180, 195, 210, 225, 240, 255, 270,
285, 300, 315, 330, 345],
ticktext=['12am', '1am', '2am', '3am', '4am', '5am', '6am', '7am', '8am', '9am', '10am', '11am',
'12pm', '1pm', '2pm', '3pm', '4pm', '5pm', '6pm', '7pm', '8pm', '9pm', '10pm', '11pm'],
hoverformat="#%{theta}: <br>Popularity: %{percent} </br> %{r}"
),
),
)
daily = pd.crosstab(df5, df5.index.date, dropna=True, margins = True)
fig.add_trace(go.Bar(x=daily.columns[:-1], y=daily.loc[specie][:-1], marker_color='seagreen'), row=3, col=1)
st.plotly_chart(fig, use_container_width=True) # , config=config)
df_counts = int(hourly[hourly.index==specie]['All'])
st.subheader('Total Detect:' + str('{:,}'.format(df_counts))
+ ' Confidence Max:' + str(
'{:.2f}%'.format(max(df2[df2['Com_Name'] == specie]['Confidence']) * 100)) +
' ' + ' Median:' + str(
'{:.2f}%'.format(np.median(df2[df2['Com_Name'] == specie]['Confidence']) * 100)))
recordings=df2[df2['Com_Name']==specie]['File_Name']
with col2:
try:
recording = st.selectbox('Available recordings', recordings.sort_index(ascending=False))
date_specie = df2.loc[df2['File_Name']==recording,['Date','Com_Name']]
date_dir = date_specie['Date'].values[0]
specie_dir = date_specie['Com_Name'].values[0].replace(" ","_")
st.image(userDir + '/BirdSongs/Extracted/By_Date/'+ date_dir + '/'+ specie_dir + '/' + recording + '.png')
st.audio(userDir +'/BirdSongs/Extracted/By_Date/'+ date_dir + '/'+ specie_dir + '/' + recording)
except:
st.title('RECORDING NOT AVAILABLE :(')
# try:
# con = sqlite3.connect(userDir + '/BirdNET-Pi/scripts/birds.db')
# cur = con.cursor()
cola, colb, colc, cold = st.columns((3,1,1,1))
with colb:
seen = st.checkbox('Reviewed')
if seen:
with colc:
verified = st.radio("Verification",['True Positive','False Positive'])
if verified == "False Positive":
df_names = pd.read_csv(userDir+'/BirdNET-Pi/model/labels.txt', delimiter= '_', names=['Sci_Name', 'Com_Name'])
df_unknown= pd.DataFrame({"Sci_Name":["UNKNOWN"],"Com_Name":["UNKNOWN"]})
df_names = pd.concat([df_unknown,df_names], ignore_index=True)
with cold:
corrected = st.selectbox('What specie?', df_names['Com_Name'])
# cur.execute("UPDATE detections SET Seen = seen WHERE File_Name = recording")
# con.commit()
# con.close()
# except BaseException:
# print("Database busy")
# time.sleep(2)
else:
specie = st.selectbox(
'Which bird would you like to explore for the dates '
+ str(start_date) + ' to ' + str(end_date) + '?',
species[1:],
index = 0)
# filt = df2[df2['Com_Name'] == specie]
df_counts = int(hourly[hourly.index==specie]['All'])
fig = st.container()
fig = make_subplots(
rows=1, cols=1)
rows=1, cols =1)
# specs= [[{"type":"xy","rowspan":1},{"type":"heatmap","rowspan":1}]],
# subplot_titles=('<b>Daily Top '+ str(top_N) + ' Species in Date Range '+ str(start_date) +' to '+ str(end_date) +'</b>',
# '<b>Daily ' + specie+ ' Detections on 15 minute intervals </b>'),
@@ -257,11 +362,12 @@ if daily is False:
# # ' Confidence Max:'+str('{:.2f}%'.format(max(df2[df2['Com_Name']==specie]['Confidence'])*100))+
# # ' '+' Median:'+str('{:.2f}%'.format(np.median(df2[df2['Com_Name']==specie]['Confidence'])*100))
# # )
# )
# fig.add_trace(go.Bar(y=top_N_species.index, x=top_N_species, orientation='h'), row=1,col=1)
df4 = df2['Com_Name'][df2['Com_Name'] == specie].resample('15min').count()
df4.index = [df4.index.date, df4.index.time]
day_hour_freq = df4.unstack().fillna(0)
df4=df2['Com_Name'][df2['Com_Name']==specie].resample('15min').count()
df4.index=[df4.index.date, df4.index.time]
day_hour_freq=df4.unstack().fillna(0)
fig_x = [d.strftime('%d-%m-%Y') for d in day_hour_freq.index.tolist()]
fig_y = [h.strftime('%H:%M') for h in day_hour_freq.columns.tolist()]
@@ -271,10 +377,10 @@ if daily is False:
# fig.update_layout(
# margin=dict(l=0, r=0, t=50, b=0),
# yaxis={'categoryorder':'total ascending'})
color_pals = px.colors.named_colorscales()
color_pals= px.colors.named_colorscales()
selected_pal = st.sidebar.selectbox('Select Color Pallet for Daily Detections', color_pals)
fig.add_trace(go.Heatmap(x=fig_x, y=fig_y, z=fig_z, autocolorscale=False, colorscale=selected_pal), row=1, col=1)
fig.add_trace(go.Heatmap(x=fig_x,y=fig_y,z=fig_z, autocolorscale = False, colorscale = selected_pal), row=1, col=1)
st.plotly_chart(fig, use_container_width=True) # , config=config)
else:
fig = make_subplots(
rows=1, cols=2,
@@ -327,12 +433,12 @@ else:
margin=dict(l=0, r=0, t=50, b=0))
# container=st.container()
# config={'displayModelBar': False}
st.plotly_chart(fig, use_container_width=True) # , config=config)
st.plotly_chart(fig, use_container_width=True) # , config=config)
# cols3,cols4=st.columns((1,1))
#
#
# extract_date=Date_Slider
#
#
# audio_file = open('/home/*/BirdSongs/Extracted/By_Date/2022-03-22/Yellow-streaked_Greenbul/Yellow-streaked_Greenbul-77-2022-03-22-birdnet-15:04:28.mp3', 'rb')
# audio_bytes = audio_file.read()
# cols4.audio(audio_bytes, format='audio/mp3')