claudecode/AvianVisitors
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #405 from mcguirepr89/daily_new_species_notifier

Browse Source 
Daily new species notifier
This commit is contained in:
Patrick McGuire
2022-06-25 16:11:13 -04:00
committed by GitHub
parent a0ec7c2d3c e19e22f32e
commit a601de3705
15 changed files with 499 additions and 164 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/__init__.py
View File
scripts/config.php
+57 -8
View File
@@ -41,6 +41,11 @@ if(isset($_GET["latitude"])){
} else {
$apprise_notify_new_species = 0;
}
if(isset($_GET['apprise_notify_new_species_each_day'])) {
$apprise_notify_new_species_each_day = 1;
} else {
$apprise_notify_new_species_each_day = 0;
}
if(isset($timezone)) {
shell_exec("sudo timedatectl set-timezone ".$timezone);
@@ -92,9 +97,10 @@ if(isset($_GET["latitude"])){
$contents = preg_replace("/LONGITUDE=.*/", "LONGITUDE=$longitude", $contents);
$contents = preg_replace("/BIRDWEATHER_ID=.*/", "BIRDWEATHER_ID=$birdweather_id", $contents);
$contents = preg_replace("/APPRISE_NOTIFICATION_TITLE=.*/", "APPRISE_NOTIFICATION_TITLE=\"$apprise_notification_title\"", $contents);
$contents = preg_replace("/APPRISE_NOTIFICATION_BODY=.*/", "APPRISE_NOTIFICATION_BODY=\"$apprise_notification_body\"", $contents);
$contents = preg_replace("/APPRISE_NOTIFICATION_BODY=.*/", "APPRISE_NOTIFICATION_BODY='$apprise_notification_body'", $contents);
$contents = preg_replace("/APPRISE_NOTIFY_EACH_DETECTION=.*/", "APPRISE_NOTIFY_EACH_DETECTION=$apprise_notify_each_detection", $contents);
$contents = preg_replace("/APPRISE_NOTIFY_NEW_SPECIES=.*/", "APPRISE_NOTIFY_NEW_SPECIES=$apprise_notify_new_species", $contents);
$contents = preg_replace("/APPRISE_NOTIFY_NEW_SPECIES_EACH_DAY=.*/", "APPRISE_NOTIFY_NEW_SPECIES_EACH_DAY=$apprise_notify_new_species_each_day", $contents);
$contents = preg_replace("/FLICKR_API_KEY=.*/", "FLICKR_API_KEY=$flickr_api_key", $contents);
$contents = preg_replace("/DATABASE_LANG=.*/", "DATABASE_LANG=$language", $contents);
$contents = preg_replace("/FLICKR_FILTER_EMAIL=.*/", "FLICKR_FILTER_EMAIL=$flickr_filter_email", $contents);
@@ -104,9 +110,10 @@ if(isset($_GET["latitude"])){
$contents2 = preg_replace("/LONGITUDE=.*/", "LONGITUDE=$longitude", $contents2);
$contents2 = preg_replace("/BIRDWEATHER_ID=.*/", "BIRDWEATHER_ID=$birdweather_id", $contents2);
$contents2 = preg_replace("/APPRISE_NOTIFICATION_TITLE=.*/", "APPRISE_NOTIFICATION_TITLE=\"$apprise_notification_title\"", $contents2);
$contents2 = preg_replace("/APPRISE_NOTIFICATION_BODY=.*/", "APPRISE_NOTIFICATION_BODY=\"$apprise_notification_body\"", $contents2);
$contents2 = preg_replace("/APPRISE_NOTIFICATION_BODY=.*/", "APPRISE_NOTIFICATION_BODY='$apprise_notification_body'", $contents2);
$contents2 = preg_replace("/APPRISE_NOTIFY_EACH_DETECTION=.*/", "APPRISE_NOTIFY_EACH_DETECTION=$apprise_notify_each_detection", $contents2);
$contents2 = preg_replace("/APPRISE_NOTIFY_NEW_SPECIES=.*/", "APPRISE_NOTIFY_NEW_SPECIES=$apprise_notify_new_species", $contents2);
$contents2 = preg_replace("/APPRISE_NOTIFY_NEW_SPECIES_EACH_DAY=.*/", "APPRISE_NOTIFY_NEW_SPECIES_EACH_DAY=$apprise_notify_new_species_each_day", $contents2);
$contents2 = preg_replace("/FLICKR_API_KEY=.*/", "FLICKR_API_KEY=$flickr_api_key", $contents2);
$contents2 = preg_replace("/DATABASE_LANG=.*/", "DATABASE_LANG=$language", $contents2);
$contents2 = preg_replace("/FLICKR_FILTER_EMAIL=.*/", "FLICKR_FILTER_EMAIL=$flickr_filter_email", $contents2);
@@ -150,10 +157,18 @@ if(isset($_GET['sendtest']) && $_GET['sendtest'] == "true") {
$result0 = $statement0->execute();
while($todaytable=$result0->fetchArray(SQLITE3_ASSOC))
{
$comname = $todaytable['Com_Name'];
$filename = $todaytable['File_Name'];
$sciname = $todaytable['Sci_Name'];
$confidence = $todaytable["Confidence"];
$comname = $todaytable['Com_Name'];
$confidence = $todaytable['Confidence'];
$filename = $todaytable['File_Name'];
$date = $todaytable['Date'];
$time = $todaytable['Time'];
$week = $todaytable['Week'];
$latitude = $todaytable['Lat'];
$longitude = $todaytable['Lon'];
$cutoff = $todaytable['Cutoff'];
$sens = $todaytable['Sens'];
$overlap = $todaytable['Overlap'];
}
$title = $_GET['apprise_notification_title'];
@@ -165,15 +180,31 @@ if(isset($_GET['sendtest']) && $_GET['sendtest'] == "true") {
$filename = "http://birdnetpi.local/"."?filename=".$filename;
}
$title = str_replace("\$comname", $comname, $title);
$title = str_replace("\$sciname", $sciname, $title);
$title = str_replace("\$comname", $comname, $title);
$title = str_replace("\$confidence", $confidence, $title);
$title = str_replace("\$listenurl", $filename, $title);
$title = str_replace("\$date", $date, $title);
$title = str_replace("\$time", $time, $title);
$title = str_replace("\$week", $week, $title);
$title = str_replace("\$latitude", $latitude, $title);
$title = str_replace("\$longitude", $longitude, $title);
$title = str_replace("\$cutoff", $cutoff, $title);
$title = str_replace("\$sens", $sens, $title);
$title = str_replace("\$overlap", $overlap, $title);
$body = str_replace("\$comname", $comname, $body);
$body = str_replace("\$sciname", $sciname, $body);
$body = str_replace("\$comname", $comname, $body);
$body = str_replace("\$confidence", $confidence, $body);
$body = str_replace("\$listenurl", $filename, $body);
$body = str_replace("\$date", $date, $body);
$body = str_replace("\$time", $time, $body);
$body = str_replace("\$week", $week, $body);
$body = str_replace("\$latitude", $latitude, $body);
$body = str_replace("\$longitude", $longitude, $body);
$body = str_replace("\$cutoff", $cutoff, $body);
$body = str_replace("\$sens", $sens, $body);
$body = str_replace("\$overlap", $overlap, $body);
echo "<pre class=\"bash\">".shell_exec($home."/BirdNET-Pi/birdnet/bin/apprise -vv -t '".$title."' -b '".$body."' ".$cf." ")."</pre>";
@@ -273,14 +304,32 @@ https://discordapp.com/api/webhooks/{WebhookID}/{WebhookToken}
<dd>Confidence Score</dd>
<dt>$listenurl</dt>
<dd>A link to the detection</dd>
<dt>$date</dt>
<dd>Date</dd>
<dt>$time</dt>
<dd>Time</dd>
<dt>$week</dt>
<dd>Week</dd>
<dt>$latitude</dt>
<dd>Latitude</dd>
<dt>$longitude</dt>
<dd>Longitude</dd>
<dt>$cutoff</dt>
<dd>Minimum Confidence set in "Advanced Settings"</dd>
<dt>$sens</dt>
<dd>Sigmoid Sensitivity set in "Advanced Settings"</dd>
<dt>$overlap</dt>
<dd>Overlap set in "Advanced Settings"</dd>
</dl>
<p>Use the variables defined above to customize your notification title and body.</p>
<label for="apprise_notification_title">Notification Title: </label>
<input name="apprise_notification_title" type="text" value="<?php print($config['APPRISE_NOTIFICATION_TITLE']);?>" /><br>
<label for="apprise_notification_body">Notification Body: </label>
<input name="apprise_notification_body" type="text" value="<?php print($config['APPRISE_NOTIFICATION_BODY']);?>" /><br>
<input name="apprise_notification_body" type="text" value='<?php print($config['APPRISE_NOTIFICATION_BODY']);?>' /><br>
<input type="checkbox" name="apprise_notify_new_species" <?php if($config['APPRISE_NOTIFY_NEW_SPECIES'] == 1 && filesize($home."/BirdNET-Pi/apprise.txt") != 0) { echo "checked"; };?> >
<label for="apprise_notify_new_species">Notify each new infrequent species detection (<5 visits per week)</label><br>
<input type="checkbox" name="apprise_notify_new_species_each_day" <?php if($config['APPRISE_NOTIFY_NEW_SPECIES_EACH_DAY'] == 1 && filesize($home."/BirdNET-Pi/apprise.txt") != 0) { echo "checked"; };?> >
<label for="apprise_notify_new_species_each_day">Notify each new species detection of the day</label><br>
<input type="checkbox" name="apprise_notify_each_detection" <?php if($config['APPRISE_NOTIFY_EACH_DETECTION'] == 1 && filesize($home."/BirdNET-Pi/apprise.txt") != 0) { echo "checked"; };?> >
<label for="apprise_notify_each_detection">Notify each new detection</label><br><br>
scripts/plotly_streamlit.py
+80 -87
View File
@@ -6,13 +6,11 @@ from numpy import ma
import plotly.graph_objects as go
from plotly.subplots import make_subplots
import plotly.io as pio
from datetime import timedelta, datetime
from pathlib import Path
from datetime import timedelta
import sqlite3
from sqlite3 import Connection
import plotly.express as px
from sklearn.preprocessing import normalize
import time
pio.templates.default = "plotly_white"
@@ -40,9 +38,8 @@ st.markdown("""
""", unsafe_allow_html=True)
@st.cache(hash_funcs={Connection: id})
#@st.cache(allow_output_mutation=True)
# @st.cache(allow_output_mutation=True)
def get_connection(path: str):
return sqlite3.connect(path, check_same_thread=False)
@@ -58,19 +55,18 @@ df2 = df.copy()
df2['DateTime'] = pd.to_datetime(df2['Date'] + " " + df2['Time'])
df2 = df2.set_index('DateTime')
daily = st.sidebar.checkbox('Single Day View', help= 'Select if you want single day view, unselect for multi-day views')
daily = st.sidebar.checkbox('Single Day View', help='Select if you want single day view, unselect for multi-day views')
if daily:
# Date as slider
# Date as slider
Start_Date = pd.to_datetime(df2.index.min()).date()
End_Date = pd.to_datetime(df2.index.max()).date()
# cols1, cols2 = st.columns((1, 1))
end_date = st.sidebar.date_input('Date to View',
min_value = Start_Date,
max_value = End_Date,
value=(End_Date),
help= 'Select date for single day view'
)
min_value=Start_Date,
max_value=End_Date,
value=(End_Date),
help='Select date for single day view')
start_date = end_date
else:
Start_Date = pd.to_datetime(df2.index.min()).date()
@@ -78,11 +74,10 @@ else:
# cols1, cols2 = st.columns((1, 1))
start_date, end_date = st.sidebar.slider('Date Range',
min_value = Start_Date-timedelta(days=1),
max_value = End_Date,
value=(Start_Date, End_Date),
help= 'Select start and end date, if same date get a clockplot for a single day'
)
min_value=Start_Date-timedelta(days=1),
max_value=End_Date,
value=(Start_Date, End_Date),
help='Select start and end date, if same date get a clockplot for a single day')
# start_date, end_date = cols1.date_input(
# "Date Input for Analysis - select Range for single specie analysis, select single date for daily view",
@@ -93,12 +88,14 @@ else:
# start_date = datetime(2022 ,5 ,17).date()
# end_date = datetime(2022 ,5 ,17).date()
@st.cache()
def date_filter(df, start_date, end_date):
filt = (df2.index >= pd.Timestamp(start_date)) & (df2.index <= pd.Timestamp(end_date + timedelta(days=1)))
df = df[filt]
return(df)
df2 = date_filter(df2, start_date, end_date)
st.write('<style>div.row-widget.stRadio > div{flex-direction:row;justify-content: left;} </style>',
@@ -112,7 +109,7 @@ st.write('<style>div.st-bf{flex-direction:column;} div.st-ag{font-weight:bold;pa
if start_date == end_date:
resample_sel = st.sidebar.radio(
"Resample Resolution",
('Raw', '15 minutes', 'Hourly'), index=1, help= 'Select resolution for single day - larger times run faster' )
('Raw', '15 minutes', 'Hourly'), index=1, help='Select resolution for single day - larger times run faster')
resample_times = {'Raw': 'Raw',
'1 minute': '1min',
@@ -124,7 +121,7 @@ if start_date == end_date:
else:
resample_sel = st.sidebar.radio(
"Resample Resolution",
('Raw', '15 minutes', 'Hourly', 'DAILY'), index=1, help= 'Select resolution for species - DAILY provides time series')
('Raw', '15 minutes', 'Hourly', 'DAILY'), index=1, help='Select resolution for species - DAILY provides time series')
resample_times = {'Raw': 'Raw',
'1 minute': '1min',
@@ -134,6 +131,7 @@ else:
}
resample_time = resample_times[resample_sel]
@st.cache()
def time_resample(df, resample_time):
if resample_time == 'Raw':
@@ -143,6 +141,8 @@ def time_resample(df, resample_time):
df_resample = df.resample(resample_time)['Com_Name'].aggregate('unique').explode()
return(df_resample)
top_bird = df2['Com_Name'].mode()[0]
df5 = time_resample(df2, resample_time)
@@ -151,12 +151,12 @@ df5 = time_resample(df2, resample_time)
Specie_Count = df5.value_counts()
# Create Hourly Crosstab
hourly = pd.crosstab(df5, df5.index.hour, dropna=True, margins= True)
hourly = pd.crosstab(df5, df5.index.hour, dropna=True, margins=True)
# Filter on species
species = list(hourly.sort_values("All", ascending= False).index)
species = list(hourly.sort_values("All", ascending=False).index)
#cols1, cols2 = st.columns((1, 1))
# cols1, cols2 = st.columns((1, 1))
top_N = st.sidebar.slider(
'Select Number of Birds to Show',
min_value=1,
@@ -168,26 +168,26 @@ top_N_species = (df5.value_counts()[:top_N])
font_size = 15
if daily == False:
if daily is False:
if resample_time != '1D':
specie = st.selectbox(
'Which bird would you like to explore for the dates '
+ str(start_date) + ' to ' + str(end_date) + '?',
species,
index = 0)
'Which bird would you like to explore for the dates '
+ str(start_date) + ' to ' + str(end_date) + '?',
species,
index=0)
# filt = df2['Com_Name'] == specie
if specie == 'All':
df_counts = int(hourly[hourly.index==specie]['All'])
df_counts = int(hourly[hourly.index == specie]['All'])
fig = make_subplots(
rows=3, cols=2,
specs=[[{"type": "xy", "rowspan": 3}, {"type": "polar", "rowspan": 2}], [{"rowspan": 1}, {"rowspan": 1}],
[None, {"type": "xy", "rowspan": 1}]],
specs=[[{"type": "xy", "rowspan": 3}, {"type": "polar", "rowspan": 2}],
[{"rowspan": 1}, {"rowspan": 1}],
[None, {"type": "xy", "rowspan": 1}]],
subplot_titles=('<b>Top ' + str(top_N) + ' Species in Date Range ' + str(start_date) + ' to ' + str(
end_date) + '<br>for ' + str(resample_sel) + ' sampling interval.' + '</b>',
'Total Detect:' + str('{:,}'.format(df_counts))
'Total Detect:' + str('{:,}'.format(df_counts))
# + ' Confidence Max:' + str(
# '{:.2f}%'.format(max(df2[df2['Com_Name'] == specie]['Confidence']) * 100)) +
# ' ' + ' Median:' + str(
@@ -197,9 +197,9 @@ if daily == False:
fig.layout.annotations[1].update(x=0.7, y=0.25, font_size=15)
# Plot seen species for selected date range and number of species
fig.add_trace(go.Bar(y=top_N_species.index, x=top_N_species, orientation='h', marker_color='seagreen'), row=1, col=1)
fig.update_layout(
margin=dict(l=0, r=0, t=50, b=0),
yaxis={'categoryorder': 'total ascending'})
@@ -232,16 +232,17 @@ if daily == False:
rotation=-90,
direction='clockwise',
tickmode='array',
tickvals=[0, 15, 35, 45, 60, 75, 90, 105, 120, 135, 150, 165, 180, 195, 210, 225, 240, 255, 270,
285, 300, 315, 330, 345],
ticktext=['12am', '1am', '2am', '3am', '4am', '5am', '6am', '7am', '8am', '9am', '10am', '11am',
'12pm', '1pm', '2pm', '3pm', '4pm', '5pm', '6pm', '7pm', '8pm', '9pm', '10pm', '11pm'],
tickvals=[0, 15, 35, 45, 60, 75, 90, 105, 120, 135, 150, 165, 180, 195, 210,
225, 240, 255, 270, 285, 300, 315, 330, 345],
ticktext=['12am', '1am', '2am', '3am', '4am', '5am', '6am', '7am', '8am', '9am',
'10am', '11am', '12pm', '1pm', '2pm', '3pm', '4pm', '5pm', '6pm',
'7pm', '8pm', '9pm', '10pm', '11pm'],
hoverformat="#%{theta}: <br>Popularity: %{percent} </br> %{r}"
),
),
)
daily = pd.crosstab(df5, df5.index.date, dropna=True, margins = True)
daily = pd.crosstab(df5, df5.index.date, dropna=True, margins=True)
fig.add_trace(go.Bar(x=daily.columns[:-1], y=daily.loc[specie][:-1], marker_color='seagreen'), row=3, col=2)
st.plotly_chart(fig, use_container_width=True) # , config=config)
@@ -250,7 +251,7 @@ if daily == False:
with col1:
fig = make_subplots(
rows=3, cols=1,
specs=[[{"type": "polar", "rowspan": 2}],[{"rowspan": 1}], [{"type": "xy", "rowspan": 1}]]
specs=[[{"type": "polar", "rowspan": 2}], [{"rowspan": 1}], [{"type": "xy", "rowspan": 1}]]
)
# Set 360 degrees, 24 hours for polar plot
theta = np.linspace(0.0, 360, 24, endpoint=False)
@@ -280,81 +281,75 @@ if daily == False:
rotation=-90,
direction='clockwise',
tickmode='array',
tickvals=[0, 15, 35, 45, 60, 75, 90, 105, 120, 135, 150, 165, 180, 195, 210, 225, 240, 255, 270,
285, 300, 315, 330, 345],
ticktext=['12am', '1am', '2am', '3am', '4am', '5am', '6am', '7am', '8am', '9am', '10am', '11am',
'12pm', '1pm', '2pm', '3pm', '4pm', '5pm', '6pm', '7pm', '8pm', '9pm', '10pm', '11pm'],
tickvals=[0, 15, 35, 45, 60, 75, 90, 105, 120, 135, 150, 165, 180, 195,
210, 225, 240, 255, 270, 285, 300, 315, 330, 345],
ticktext=['12am', '1am', '2am', '3am', '4am', '5am', '6am', '7am', '8am',
'9am', '10am', '11am', '12pm', '1pm', '2pm', '3pm', '4pm', '5pm',
'6pm', '7pm', '8pm', '9pm', '10pm', '11pm'],
hoverformat="#%{theta}: <br>Popularity: %{percent} </br> %{r}"
),
),
)
daily = pd.crosstab(df5, df5.index.date, dropna=True, margins = True)
daily = pd.crosstab(df5, df5.index.date, dropna=True, margins=True)
fig.add_trace(go.Bar(x=daily.columns[:-1], y=daily.loc[specie][:-1], marker_color='seagreen'), row=3, col=1)
st.plotly_chart(fig, use_container_width=True) # , config=config)
df_counts = int(hourly[hourly.index==specie]['All'])
st.subheader('Total Detect:' + str('{:,}'.format(df_counts))
+ ' Confidence Max:' + str(
'{:.2f}%'.format(max(df2[df2['Com_Name'] == specie]['Confidence']) * 100)) +
' ' + ' Median:' + str(
'{:.2f}%'.format(np.median(df2[df2['Com_Name'] == specie]['Confidence']) * 100)))
df_counts = int(hourly[hourly.index == specie]['All'])
st.subheader('Total Detect:' + str('{:,}'.format(df_counts))
+ ' Confidence Max:' +
str('{:.2f}%'.format(max(df2[df2['Com_Name'] == specie]['Confidence']) * 100))
+ ' ' + ' Median:' +
str('{:.2f}%'.format(np.median(df2[df2['Com_Name'] == specie]['Confidence']) * 100)))
recordings = df2[df2['Com_Name'] == specie]['File_Name']
recordings=df2[df2['Com_Name']==specie]['File_Name']
with col2:
try:
recording = st.selectbox('Available recordings', recordings.sort_index(ascending=False))
date_specie = df2.loc[df2['File_Name']==recording,['Date','Com_Name']]
date_specie = df2.loc[df2['File_Name'] == recording, ['Date', 'Com_Name']]
date_dir = date_specie['Date'].values[0]
specie_dir = date_specie['Com_Name'].values[0].replace(" ","_")
st.image(userDir + '/BirdSongs/Extracted/By_Date/'+ date_dir + '/'+ specie_dir + '/' + recording + '.png')
st.audio(userDir +'/BirdSongs/Extracted/By_Date/'+ date_dir + '/'+ specie_dir + '/' + recording)
except:
st.title('RECORDING NOT AVAILABLE :(')
specie_dir = date_specie['Com_Name'].values[0].replace(" ", "_")
st.image(userDir + '/BirdSongs/Extracted/By_Date/' + date_dir + '/' + specie_dir + '/' + recording + '.png')
st.audio(userDir + '/BirdSongs/Extracted/By_Date/' + date_dir + '/' + specie_dir + '/' + recording)
except Exception:
st.title('RECORDING NOT AVAILABLE :(')
# try:
# con = sqlite3.connect(userDir + '/BirdNET-Pi/scripts/birds.db')
# cur = con.cursor()
cola, colb, colc, cold = st.columns((3,1,1,1))
cola, colb, colc, cold = st.columns((3, 1, 1, 1))
with colb:
seen = st.checkbox('Reviewed')
if seen:
with colc:
verified = st.radio("Verification",['True Positive','False Positive'])
verified = st.radio("Verification", ['True Positive', 'False Positive'])
if verified == "False Positive":
df_names = pd.read_csv(userDir+'/BirdNET-Pi/model/labels.txt', delimiter= '_', names=['Sci_Name', 'Com_Name'])
df_unknown= pd.DataFrame({"Sci_Name":["UNKNOWN"],"Com_Name":["UNKNOWN"]})
df_names = pd.concat([df_unknown,df_names], ignore_index=True)
df_names = pd.read_csv(userDir + '/BirdNET-Pi/model/labels.txt', delimiter='_', names=['Sci_Name', 'Com_Name'])
df_unknown = pd.DataFrame({"Sci_Name": ["UNKNOWN"], "Com_Name": ["UNKNOWN"]})
df_names = pd.concat([df_unknown, df_names], ignore_index=True)
with cold:
corrected = st.selectbox('What species?', df_names['Com_Name'])
# cur.execute("UPDATE detections SET Seen = seen WHERE File_Name = recording")
# con.commit()
# con.close()
# except BaseException:
# print("Database busy")
# time.sleep(2)
else:
specie = st.selectbox(
'Which bird would you like to explore for the dates '
+ str(start_date) + ' to ' + str(end_date) + '?',
species[1:],
index = 0)
specie = st.selectbox('Which bird would you like to explore for the dates '
+ str(start_date) + ' to ' + str(end_date) + '?',
species[1:],
index=0)
# filt = df2[df2['Com_Name'] == specie]
df_counts = int(hourly[hourly.index==specie]['All'])
df_counts = int(hourly[hourly.index == specie]['All'])
fig = st.container()
fig = make_subplots(
rows=1, cols =1)
fig = make_subplots(rows=1, cols=1)
# specs= [[{"type":"xy","rowspan":1},{"type":"heatmap","rowspan":1}]],
# subplot_titles=('<b>Daily Top '+ str(top_N) + ' Species in Date Range '+ str(start_date) +' to '+ str(end_date) +'</b>',
# '<b>Daily ' + specie+ ' Detections on 15 minute intervals </b>'),
@@ -365,9 +360,9 @@ if daily == False:
# )
# fig.add_trace(go.Bar(y=top_N_species.index, x=top_N_species, orientation='h'), row=1,col=1)
df4=df2['Com_Name'][df2['Com_Name']==specie].resample('15min').count()
df4.index=[df4.index.date, df4.index.time]
day_hour_freq=df4.unstack().fillna(0)
df4 = df2['Com_Name'][df2['Com_Name'] == specie].resample('15min').count()
df4.index = [df4.index.date, df4.index.time]
day_hour_freq = df4.unstack().fillna(0)
fig_x = [d.strftime('%d-%m-%Y') for d in day_hour_freq.index.tolist()]
fig_y = [h.strftime('%H:%M') for h in day_hour_freq.columns.tolist()]
@@ -377,9 +372,9 @@ if daily == False:
# fig.update_layout(
# margin=dict(l=0, r=0, t=50, b=0),
# yaxis={'categoryorder':'total ascending'})
color_pals= px.colors.named_colorscales()
color_pals = px.colors.named_colorscales()
selected_pal = st.sidebar.selectbox('Select Color Pallet for Daily Detections', color_pals)
fig.add_trace(go.Heatmap(x=fig_x,y=fig_y,z=fig_z, autocolorscale = False, colorscale = selected_pal), row=1, col=1)
fig.add_trace(go.Heatmap(x=fig_x, y=fig_y, z=fig_z, autocolorscale=False, colorscale=selected_pal), row=1, col=1)
st.plotly_chart(fig, use_container_width=True) # , config=config)
else:
fig = make_subplots(
@@ -436,9 +431,7 @@ else:
st.plotly_chart(fig, use_container_width=True) # , config=config)
# cols3,cols4=st.columns((1,1))
#
# extract_date=Date_Slider
#
# audio_file = open('/home/*/BirdSongs/Extracted/By_Date/2022-03-22/Yellow-streaked_Greenbul/Yellow-streaked_Greenbul-77-2022-03-22-birdnet-15:04:28.mp3', 'rb')
# audio_bytes = audio_file.read()
# cols4.audio(audio_bytes, format='audio/mp3')
scripts/server.py
+32 -66
View File
@@ -1,4 +1,3 @@
import apprise
from pathlib import Path
from tzlocal import get_localzone
import datetime
@@ -13,6 +12,10 @@ import operator
import socket
import threading
import os
from utils.notifications import sendAppriseNotifications
from utils.parse_settings import config_to_settings
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
os.environ['CUDA_VISIBLE_DEVICES'] = ''
@@ -29,6 +32,10 @@ ADDR = (SERVER, PORT)
FORMAT = 'utf-8'
DISCONNECT_MESSAGE = "!DISCONNECT"
userDir = os.path.expanduser('~')
DB_PATH = userDir + '/BirdNET-Pi/scripts/birds.db'
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
@@ -40,7 +47,6 @@ except BaseException:
# Open most recent Configuration and grab DB_PWD as a python variable
userDir = os.path.expanduser('~')
with open(userDir + '/BirdNET-Pi/scripts/thisrun.txt', 'r') as f:
this_run = f.readlines()
audiofmt = "." + str(str(str([i for i in this_run if i.startswith('AUDIOFMT')]).split('=')[1]).split('\\')[0])
@@ -228,59 +234,6 @@ def analyzeAudioData(chunks, lat, lon, week, sensitivity, overlap,):
return detections
def sendAppriseNotifications(species, confidence, path):
if os.path.exists(userDir + '/BirdNET-Pi/apprise.txt') and os.path.getsize(userDir + '/BirdNET-Pi/apprise.txt') > 0:
with open(userDir + '/BirdNET-Pi/scripts/thisrun.txt', 'r') as f:
this_run = f.readlines()
title = str(str(str([i for i in this_run if i.startswith('APPRISE_NOTIFICATION_TITLE')]).split('=')[1]).split('\\')[0]).replace('"', '')
body = str(str(str([i for i in this_run if i.startswith('APPRISE_NOTIFICATION_BODY')]).split('=')[1]).split('\\')[0]).replace('"', '')
try:
websiteurl = str(str(str([i for i in this_run if i.startswith('BIRDNETPI_URL')]).split('=')[1]).split('\\')[0]).replace('"', '')
if len(websiteurl) == 0:
raise ValueError('Blank URL')
except Exception as e:
websiteurl = "http://"+socket.gethostname()+".local"
listenurl = websiteurl+"?filename="+path
if str(str(str([i for i in this_run if i.startswith('APPRISE_NOTIFY_EACH_DETECTION')]).split('=')[1]).split('\\')[0]) == "1":
apobj = apprise.Apprise()
config = apprise.AppriseConfig()
config.add(userDir + '/BirdNET-Pi/apprise.txt')
apobj.add(config)
apobj.notify(
body=body.replace("$sciname", species.split("_")[0]).replace("$comname", species.split("_")[1]).replace("$confidence", confidence).replace("$listenurl", listenurl),
title=title.replace("$sciname", species.split("_")[0]).replace("$comname", species.split("_")[1]).replace("$confidence", confidence).replace("$listenurl", listenurl),
)
if str(str(str([i for i in this_run if i.startswith('APPRISE_NOTIFY_NEW_SPECIES')]).split('=')[1]).split('\\')[0]) == "1":
try:
con = sqlite3.connect(userDir + '/BirdNET-Pi/scripts/birds.db')
cur = con.cursor()
cur.execute("SELECT DISTINCT(Com_Name), count(Com_Name) FROM detections WHERE date > (SELECT DATETIME('now', '-7 day')) GROUP BY Com_Name")
known_species = cur.fetchall()
sciName, comName = species.split("_")
numberDetections = [d[1] for d in known_species if d[0] == comName.replace("'","")][0]
if numberDetections <= 5:
apobj = apprise.Apprise()
config = apprise.AppriseConfig()
config.add(userDir + '/BirdNET-Pi/apprise.txt')
apobj.add(config)
apobj.notify(
body=body.replace("$sciname", species.split("_")[0]).replace("$comname", species.split("_")[1]).replace("$confidence", confidence).replace("$listenurl", listenurl) + " (only seen "+str(int(numberDetections))+" times in last 7d)",
title=title.replace("$sciname", species.split("_")[0]).replace("$comname", species.split("_")[1]).replace("$confidence", confidence).replace("$listenurl", listenurl) + " (only seen "+str(int(numberDetections))+" times in last 7d)",
)
con.close()
except BaseException:
print("Database busy")
time.sleep(2)
def writeResultsToFile(detections, min_conf, path):
print('WRITING RESULTS TO', path, '...', end=' ')
@@ -408,13 +361,16 @@ def handle_client(conn, addr):
with open(userDir + '/BirdNET-Pi/BirdDB.txt', 'a') as rfile:
for d in detections:
species_apprised_this_run = []
for entry in detections[d]:
if entry[1] >= min_conf and ((entry[0] in INCLUDE_LIST or len(INCLUDE_LIST) == 0)
and (entry[0] not in EXCLUDE_LIST or len(EXCLUDE_LIST) == 0)):
# Write to text file.
rfile.write(str(current_date) + ';' + str(current_time) + ';' + entry[0].replace('_', ';') + ';'
+ str(entry[1]) + ";" + str(args.lat) + ';' + str(args.lon) + ';' + str(min_conf) + ';' + str(week) + ';'
+ str(args.sensitivity) + ';' + str(args.overlap) + '\n')
# Write to database
Date = str(current_date)
Time = str(current_time)
species = entry[0]
@@ -434,7 +390,7 @@ def handle_client(conn, addr):
# Connect to SQLite Database
for attempt_number in range(3):
try:
con = sqlite3.connect(userDir + '/BirdNET-Pi/scripts/birds.db')
con = sqlite3.connect(DB_PATH)
cur = con.cursor()
cur.execute("INSERT INTO detections VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (Date, Time,
Sci_Name, Com_Name, str(score), Lat, Lon, Cutoff, Week, Sens, Overlap, File_Name))
@@ -445,8 +401,24 @@ def handle_client(conn, addr):
except BaseException:
print("Database busy")
time.sleep(2)
sendAppriseNotifications(str(entry[0]), str(entry[1]), File_Name)
# Apprise of detection if not already alerted this run.
if not entry[0] in species_apprised_this_run:
settings_dict = config_to_settings(userDir + '/BirdNET-Pi/scripts/thisrun.txt')
sendAppriseNotifications(species,
str(score),
File_Name,
Date,
Time,
Week,
Lat,
Lon,
Cutoff,
Sens,
Overlap,
settings_dict,
DB_PATH)
species_apprised_this_run.append(entry[0])
print(str(current_date) +
';' +
@@ -467,14 +439,8 @@ def handle_client(conn, addr):
str(args.sensitivity) +
';' +
str(args.overlap) +
Com_Name.replace(" ", "_") +
'-' +
str(score) +
'-' +
str(current_date) +
'-birdnet-' +
str(current_time) +
audiofmt +
';' +
File_Name +
'\n')
if birdweather_id != "99999":
scripts/update_birdnet_snippets.sh
+12 -2
View File
@@ -40,6 +40,9 @@ fi
if ! grep APPRISE_NOTIFY_NEW_SPECIES /etc/birdnet/birdnet.conf &>/dev/null;then
sudo -u$USER echo "APPRISE_NOTIFY_NEW_SPECIES=0 " >> /etc/birdnet/birdnet.conf
fi
if ! grep APPRISE_NOTIFY_NEW_SPECIES_EACH_DAY /etc/birdnet/birdnet.conf &>/dev/null;then
sudo -u$USER echo "APPRISE_NOTIFY_NEW_SPECIES_EACH_DAY=0 " >> /etc/birdnet/birdnet.conf
fi
# If the config does not contain the DATABASE_LANG setting, we'll want to add it.
# Defaults to not-selected, which config.php will know to render as a language option.
@@ -50,8 +53,8 @@ fi
apprise_installation_status=$(~/BirdNET-Pi/birdnet/bin/python3 -c 'import pkgutil; print("installed" if pkgutil.find_loader("apprise") else "not installed")')
if [[ "$apprise_installation_status" = "not installed" ]];then
~/BirdNET-Pi/birdnet/bin/pip3 install -U pip
~/BirdNET-Pi/birdnet/bin/pip3 install apprise
$HOME/BirdNET-Pi/birdnet/bin/pip3 install -U pip
$HOME/BirdNET-Pi/birdnet/bin/pip3 install apprise
fi
[ -f $HOME/BirdNET-Pi/apprise.txt ] || sudo -E -ucaddy touch $HOME/BirdNET-Pi/apprise.txt
if ! which lsof &>/dev/null;then
@@ -85,5 +88,12 @@ if ! grep FLICKR_FILTER_EMAIL /etc/birdnet/birdnet.conf &>/dev/null;then
sudo -u$USER echo "FLICKR_FILTER_EMAIL=" >> /etc/birdnet/birdnet.conf
fi
pytest_installation_status=$(~/BirdNET-Pi/birdnet/bin/python3 -c 'import pkgutil; print("installed" if pkgutil.find_loader("pytest") else "not installed")')
if [[ "$pytest_installation_status" = "not installed" ]];then
$HOME/BirdNET-Pi/birdnet/bin/pip3 install -U pip
$HOME/BirdNET-Pi/birdnet/bin/pip3 install pytest==7.1.2 pytest-mock==3.7.0
fi
sudo systemctl daemon-reload
restart_services.sh
scripts/utils/__init__.py
View File
scripts/utils/notifications.py
+161
View File
@@ -0,0 +1,161 @@
import apprise
import os
import socket
import sqlite3
from datetime import datetime
userDir = os.path.expanduser('~')
APPRISE_CONFIG = userDir + '/BirdNET-Pi/apprise.txt'
DB_PATH = userDir + '/BirdNET-Pi/scripts/birds.db'
def notify(body, title):
apobj = apprise.Apprise()
config = apprise.AppriseConfig()
config.add(APPRISE_CONFIG)
apobj.add(config)
apobj.notify(
body=body,
title=title,
)
def sendAppriseNotifications(species, confidence, path, date, time, week, latitude, longitude, cutoff, sens, overlap, settings_dict, db_path=DB_PATH):
# print(sendAppriseNotifications)
# print(settings_dict)
if os.path.exists(APPRISE_CONFIG) and os.path.getsize(APPRISE_CONFIG) > 0:
title = settings_dict.get('APPRISE_NOTIFICATION_TITLE')
body = settings_dict.get('APPRISE_NOTIFICATION_BODY')
sciName, comName = species.split("_")
try:
websiteurl = settings_dict.get('BIRDNETPI_URL')
if len(websiteurl) == 0:
raise ValueError('Blank URL')
except Exception:
websiteurl = "http://"+socket.gethostname()+".local"
listenurl = websiteurl+"?filename="+path
if settings_dict.get('APPRISE_NOTIFY_EACH_DETECTION') == "1":
notify_body = body.replace("$sciname", sciName)\
.replace("$comname", comName)\
.replace("$confidence", confidence)\
.replace("$listenurl", listenurl)\
.replace("$date", date)\
.replace("$time", time)\
.replace("$week", week)\
.replace("$latitude", latitude)\
.replace("$longitude", longitude)\
.replace("$cutoff", cutoff)\
.replace("$sens", sens)\
.replace("$overlap", overlap)
notify_title = title.replace("$sciname", sciName)\
.replace("$comname", comName)\
.replace("$confidence", confidence)\
.replace("$listenurl", listenurl)\
.replace("$date", date)\
.replace("$time", time)\
.replace("$week", week)\
.replace("$latitude", latitude)\
.replace("$longitude", longitude)\
.replace("$cutoff", cutoff)\
.replace("$sens", sens)\
.replace("$overlap", overlap)
notify(notify_body, notify_title)
APPRISE_NOTIFICATION_NEW_SPECIES_DAILY_COUNT_LIMIT = 1 # Notifies the first N per day.
if settings_dict.get('APPRISE_NOTIFY_NEW_SPECIES_EACH_DAY') == "1":
try:
con = sqlite3.connect(db_path)
cur = con.cursor()
today = datetime.now().strftime("%Y-%m-%d")
cur.execute(f"SELECT DISTINCT(Com_Name), COUNT(Com_Name) FROM detections WHERE Date = DATE('{today}') GROUP BY Com_Name")
known_species = cur.fetchall()
detections = [d[1] for d in known_species if d[0] == comName.replace("'", "")]
numberDetections = 0
if len(detections):
numberDetections = detections[0]
if numberDetections > 0 and numberDetections <= APPRISE_NOTIFICATION_NEW_SPECIES_DAILY_COUNT_LIMIT:
print("send the notification")
notify_body = body.replace("$sciname", sciName)\
.replace("$comname", comName)\
.replace("$confidence", confidence)\
.replace("$listenurl", listenurl)\
.replace("$date", date)\
.replace("$time", time)\
.replace("$week", week)\
.replace("$latitude", latitude)\
.replace("$longitude", longitude)\
.replace("$cutoff", cutoff)\
.replace("$sens", sens)\
.replace("$overlap", overlap)\
+ " (first time today)"
notify_title = title.replace("$sciname", sciName)\
.replace("$comname", comName)\
.replace("$confidence", confidence)\
.replace("$listenurl", listenurl)\
.replace("$date", date)\
.replace("$time", time)\
.replace("$week", week)\
.replace("$latitude", latitude)\
.replace("$longitude", longitude)\
.replace("$cutoff", cutoff)\
.replace("$sens", sens)\
.replace("$overlap", overlap)\
+ " (first time today)"
notify(notify_body, notify_title)
con.close()
except sqlite3.Error as e:
print(e)
print("Database busy")
time.sleep(2)
if settings_dict.get('APPRISE_NOTIFY_NEW_SPECIES') == "1":
try:
con = sqlite3.connect(db_path)
cur = con.cursor()
today = datetime.now().strftime("%Y-%m-%d")
cur.execute(f"SELECT DISTINCT(Com_Name), COUNT(Com_Name) FROM detections WHERE Date >= DATE('{today}', '-7 day') GROUP BY Com_Name")
known_species = cur.fetchall()
detections = [d[1] for d in known_species if d[0] == comName.replace("'", "")]
numberDetections = 0
if len(detections):
numberDetections = detections[0]
if numberDetections > 0 and numberDetections <= 5:
notify_body = body.replace("$sciname", sciName)\
.replace("$comname", comName)\
.replace("$confidence", confidence)\
.replace("$listenurl", listenurl)\
.replace("$date", date)\
.replace("$time", time)\
.replace("$week", week)\
.replace("$latitude", latitude)\
.replace("$longitude", longitude)\
.replace("$cutoff", cutoff)\
.replace("$sens", sens)\
.replace("$overlap", overlap)\
+ " (only seen " + str(int(numberDetections)) + " times in last 7d)"
notify_title = title.replace("$sciname", sciName)\
.replace("$comname", comName)\
.replace("$confidence", confidence)\
.replace("$listenurl", listenurl)\
.replace("$date", date)\
.replace("$time", time)\
.replace("$week", week)\
.replace("$latitude", latitude)\
.replace("$longitude", longitude)\
.replace("$cutoff", cutoff)\
.replace("$sens", sens)\
.replace("$overlap", overlap)\
+ " (only seen " + str(int(numberDetections)) + " times in last 7d)"
notify(notify_body, notify_title)
con.close()
except sqlite3.Error:
print("Database busy")
time.sleep(2)
if __name__ == "__main__":
print("notfications")
scripts/utils/parse_settings.py
+16
View File
@@ -0,0 +1,16 @@
def config_to_settings(path):
# Returns settings dict from BirdNET-pi text format.
# Consider refactoring to use ConfigParser files, or another standardized format.
settings = {}
with open(path, 'r') as f:
this_run = f.readlines()
for i in this_run:
key = i.split("=")[0]
value = i.split("=")[1][:-1]
# Trim for strings, not ideal
if value.startswith('"') and value.endswith('"'):
value = value[1:-1]
if value.startswith("'") and value.endswith("'"):
value = value[1:-1]
settings[key] = value
return settings