Merge pull request #405 from mcguirepr89/daily_new_species_notifier

Daily new species notifier
This commit is contained in:
Patrick McGuire
2022-06-25 16:11:13 -04:00
committed by GitHub
15 changed files with 499 additions and 164 deletions
+11
View File
@@ -7,3 +7,14 @@ birdnet.conf
IdentifiedSoFar.txt
IdentifiedSoFar.txt.bak
Birders_Guide_Installer_Configuration.txt
birdnet/
templates/*.service
scripts/*.txt
scripts/birds.db
analyzing_now.txt
apprise.txt
BirdDB.txt
exclude_species_list.txt
firstrun.ini
HUMAN.txt
include_species_list.txt
View File
+1 -1
View File
@@ -116,7 +116,7 @@ if(isset($_GET['view'])){
<button type=\"submit\" name=\"view\" value=\"System Controls\" form=\"views\">System Controls".$updatediv."</button>
<button type=\"submit\" name=\"view\" value=\"Services\" form=\"views\">Services</button>
<button type=\"submit\" name=\"view\" value=\"File\" form=\"views\">File Manager</button>
<a href=\"scripts/adminer.php\" target=\"_top\"><button type=\"submit\" form=\"\">Database Maintenance</button></a>
<a href=\"scripts/adminer.php\" target=\"_blank\"><button type=\"submit\" form=\"\">Database Maintenance</button></a>
<button type=\"submit\" name=\"view\" value=\"Webterm\" form=\"views\">Web Terminal</button>
<button type=\"submit\" name=\"view\" value=\"Included\" form=\"views\">Custom Species List</button>
<button type=\"submit\" name=\"view\" value=\"Excluded\" form=\"views\">Excluded Species List</button>
+2
View File
@@ -12,3 +12,5 @@ streamlit
plotly
apprise
paho-mqtt
pytest==7.1.2
pytest-mock==3.7.0
View File
+57 -8
View File
@@ -41,6 +41,11 @@ if(isset($_GET["latitude"])){
} else {
$apprise_notify_new_species = 0;
}
if(isset($_GET['apprise_notify_new_species_each_day'])) {
$apprise_notify_new_species_each_day = 1;
} else {
$apprise_notify_new_species_each_day = 0;
}
if(isset($timezone)) {
shell_exec("sudo timedatectl set-timezone ".$timezone);
@@ -92,9 +97,10 @@ if(isset($_GET["latitude"])){
$contents = preg_replace("/LONGITUDE=.*/", "LONGITUDE=$longitude", $contents);
$contents = preg_replace("/BIRDWEATHER_ID=.*/", "BIRDWEATHER_ID=$birdweather_id", $contents);
$contents = preg_replace("/APPRISE_NOTIFICATION_TITLE=.*/", "APPRISE_NOTIFICATION_TITLE=\"$apprise_notification_title\"", $contents);
$contents = preg_replace("/APPRISE_NOTIFICATION_BODY=.*/", "APPRISE_NOTIFICATION_BODY=\"$apprise_notification_body\"", $contents);
$contents = preg_replace("/APPRISE_NOTIFICATION_BODY=.*/", "APPRISE_NOTIFICATION_BODY='$apprise_notification_body'", $contents);
$contents = preg_replace("/APPRISE_NOTIFY_EACH_DETECTION=.*/", "APPRISE_NOTIFY_EACH_DETECTION=$apprise_notify_each_detection", $contents);
$contents = preg_replace("/APPRISE_NOTIFY_NEW_SPECIES=.*/", "APPRISE_NOTIFY_NEW_SPECIES=$apprise_notify_new_species", $contents);
$contents = preg_replace("/APPRISE_NOTIFY_NEW_SPECIES_EACH_DAY=.*/", "APPRISE_NOTIFY_NEW_SPECIES_EACH_DAY=$apprise_notify_new_species_each_day", $contents);
$contents = preg_replace("/FLICKR_API_KEY=.*/", "FLICKR_API_KEY=$flickr_api_key", $contents);
$contents = preg_replace("/DATABASE_LANG=.*/", "DATABASE_LANG=$language", $contents);
$contents = preg_replace("/FLICKR_FILTER_EMAIL=.*/", "FLICKR_FILTER_EMAIL=$flickr_filter_email", $contents);
@@ -104,9 +110,10 @@ if(isset($_GET["latitude"])){
$contents2 = preg_replace("/LONGITUDE=.*/", "LONGITUDE=$longitude", $contents2);
$contents2 = preg_replace("/BIRDWEATHER_ID=.*/", "BIRDWEATHER_ID=$birdweather_id", $contents2);
$contents2 = preg_replace("/APPRISE_NOTIFICATION_TITLE=.*/", "APPRISE_NOTIFICATION_TITLE=\"$apprise_notification_title\"", $contents2);
$contents2 = preg_replace("/APPRISE_NOTIFICATION_BODY=.*/", "APPRISE_NOTIFICATION_BODY=\"$apprise_notification_body\"", $contents2);
$contents2 = preg_replace("/APPRISE_NOTIFICATION_BODY=.*/", "APPRISE_NOTIFICATION_BODY='$apprise_notification_body'", $contents2);
$contents2 = preg_replace("/APPRISE_NOTIFY_EACH_DETECTION=.*/", "APPRISE_NOTIFY_EACH_DETECTION=$apprise_notify_each_detection", $contents2);
$contents2 = preg_replace("/APPRISE_NOTIFY_NEW_SPECIES=.*/", "APPRISE_NOTIFY_NEW_SPECIES=$apprise_notify_new_species", $contents2);
$contents2 = preg_replace("/APPRISE_NOTIFY_NEW_SPECIES_EACH_DAY=.*/", "APPRISE_NOTIFY_NEW_SPECIES_EACH_DAY=$apprise_notify_new_species_each_day", $contents2);
$contents2 = preg_replace("/FLICKR_API_KEY=.*/", "FLICKR_API_KEY=$flickr_api_key", $contents2);
$contents2 = preg_replace("/DATABASE_LANG=.*/", "DATABASE_LANG=$language", $contents2);
$contents2 = preg_replace("/FLICKR_FILTER_EMAIL=.*/", "FLICKR_FILTER_EMAIL=$flickr_filter_email", $contents2);
@@ -150,10 +157,18 @@ if(isset($_GET['sendtest']) && $_GET['sendtest'] == "true") {
$result0 = $statement0->execute();
while($todaytable=$result0->fetchArray(SQLITE3_ASSOC))
{
$comname = $todaytable['Com_Name'];
$filename = $todaytable['File_Name'];
$sciname = $todaytable['Sci_Name'];
$confidence = $todaytable["Confidence"];
$comname = $todaytable['Com_Name'];
$confidence = $todaytable['Confidence'];
$filename = $todaytable['File_Name'];
$date = $todaytable['Date'];
$time = $todaytable['Time'];
$week = $todaytable['Week'];
$latitude = $todaytable['Lat'];
$longitude = $todaytable['Lon'];
$cutoff = $todaytable['Cutoff'];
$sens = $todaytable['Sens'];
$overlap = $todaytable['Overlap'];
}
$title = $_GET['apprise_notification_title'];
@@ -165,15 +180,31 @@ if(isset($_GET['sendtest']) && $_GET['sendtest'] == "true") {
$filename = "http://birdnetpi.local/"."?filename=".$filename;
}
$title = str_replace("\$comname", $comname, $title);
$title = str_replace("\$sciname", $sciname, $title);
$title = str_replace("\$comname", $comname, $title);
$title = str_replace("\$confidence", $confidence, $title);
$title = str_replace("\$listenurl", $filename, $title);
$title = str_replace("\$date", $date, $title);
$title = str_replace("\$time", $time, $title);
$title = str_replace("\$week", $week, $title);
$title = str_replace("\$latitude", $latitude, $title);
$title = str_replace("\$longitude", $longitude, $title);
$title = str_replace("\$cutoff", $cutoff, $title);
$title = str_replace("\$sens", $sens, $title);
$title = str_replace("\$overlap", $overlap, $title);
$body = str_replace("\$comname", $comname, $body);
$body = str_replace("\$sciname", $sciname, $body);
$body = str_replace("\$comname", $comname, $body);
$body = str_replace("\$confidence", $confidence, $body);
$body = str_replace("\$listenurl", $filename, $body);
$body = str_replace("\$date", $date, $body);
$body = str_replace("\$time", $time, $body);
$body = str_replace("\$week", $week, $body);
$body = str_replace("\$latitude", $latitude, $body);
$body = str_replace("\$longitude", $longitude, $body);
$body = str_replace("\$cutoff", $cutoff, $body);
$body = str_replace("\$sens", $sens, $body);
$body = str_replace("\$overlap", $overlap, $body);
echo "<pre class=\"bash\">".shell_exec($home."/BirdNET-Pi/birdnet/bin/apprise -vv -t '".$title."' -b '".$body."' ".$cf." ")."</pre>";
@@ -273,14 +304,32 @@ https://discordapp.com/api/webhooks/{WebhookID}/{WebhookToken}
<dd>Confidence Score</dd>
<dt>$listenurl</dt>
<dd>A link to the detection</dd>
<dt>$date</dt>
<dd>Date</dd>
<dt>$time</dt>
<dd>Time</dd>
<dt>$week</dt>
<dd>Week</dd>
<dt>$latitude</dt>
<dd>Latitude</dd>
<dt>$longitude</dt>
<dd>Longitude</dd>
<dt>$cutoff</dt>
<dd>Minimum Confidence set in "Advanced Settings"</dd>
<dt>$sens</dt>
<dd>Sigmoid Sensitivity set in "Advanced Settings"</dd>
<dt>$overlap</dt>
<dd>Overlap set in "Advanced Settings"</dd>
</dl>
<p>Use the variables defined above to customize your notification title and body.</p>
<label for="apprise_notification_title">Notification Title: </label>
<input name="apprise_notification_title" type="text" value="<?php print($config['APPRISE_NOTIFICATION_TITLE']);?>" /><br>
<label for="apprise_notification_body">Notification Body: </label>
<input name="apprise_notification_body" type="text" value="<?php print($config['APPRISE_NOTIFICATION_BODY']);?>" /><br>
<input name="apprise_notification_body" type="text" value='<?php print($config['APPRISE_NOTIFICATION_BODY']);?>' /><br>
<input type="checkbox" name="apprise_notify_new_species" <?php if($config['APPRISE_NOTIFY_NEW_SPECIES'] == 1 && filesize($home."/BirdNET-Pi/apprise.txt") != 0) { echo "checked"; };?> >
<label for="apprise_notify_new_species">Notify each new infrequent species detection (<5 visits per week)</label><br>
<input type="checkbox" name="apprise_notify_new_species_each_day" <?php if($config['APPRISE_NOTIFY_NEW_SPECIES_EACH_DAY'] == 1 && filesize($home."/BirdNET-Pi/apprise.txt") != 0) { echo "checked"; };?> >
<label for="apprise_notify_new_species_each_day">Notify each new species detection of the day</label><br>
<input type="checkbox" name="apprise_notify_each_detection" <?php if($config['APPRISE_NOTIFY_EACH_DETECTION'] == 1 && filesize($home."/BirdNET-Pi/apprise.txt") != 0) { echo "checked"; };?> >
<label for="apprise_notify_each_detection">Notify each new detection</label><br><br>
+80 -87
View File
@@ -6,13 +6,11 @@ from numpy import ma
import plotly.graph_objects as go
from plotly.subplots import make_subplots
import plotly.io as pio
from datetime import timedelta, datetime
from pathlib import Path
from datetime import timedelta
import sqlite3
from sqlite3 import Connection
import plotly.express as px
from sklearn.preprocessing import normalize
import time
pio.templates.default = "plotly_white"
@@ -40,9 +38,8 @@ st.markdown("""
""", unsafe_allow_html=True)
@st.cache(hash_funcs={Connection: id})
#@st.cache(allow_output_mutation=True)
# @st.cache(allow_output_mutation=True)
def get_connection(path: str):
return sqlite3.connect(path, check_same_thread=False)
@@ -58,19 +55,18 @@ df2 = df.copy()
df2['DateTime'] = pd.to_datetime(df2['Date'] + " " + df2['Time'])
df2 = df2.set_index('DateTime')
daily = st.sidebar.checkbox('Single Day View', help= 'Select if you want single day view, unselect for multi-day views')
daily = st.sidebar.checkbox('Single Day View', help='Select if you want single day view, unselect for multi-day views')
if daily:
# Date as slider
# Date as slider
Start_Date = pd.to_datetime(df2.index.min()).date()
End_Date = pd.to_datetime(df2.index.max()).date()
# cols1, cols2 = st.columns((1, 1))
end_date = st.sidebar.date_input('Date to View',
min_value = Start_Date,
max_value = End_Date,
value=(End_Date),
help= 'Select date for single day view'
)
min_value=Start_Date,
max_value=End_Date,
value=(End_Date),
help='Select date for single day view')
start_date = end_date
else:
Start_Date = pd.to_datetime(df2.index.min()).date()
@@ -78,11 +74,10 @@ else:
# cols1, cols2 = st.columns((1, 1))
start_date, end_date = st.sidebar.slider('Date Range',
min_value = Start_Date-timedelta(days=1),
max_value = End_Date,
value=(Start_Date, End_Date),
help= 'Select start and end date, if same date get a clockplot for a single day'
)
min_value=Start_Date-timedelta(days=1),
max_value=End_Date,
value=(Start_Date, End_Date),
help='Select start and end date, if same date get a clockplot for a single day')
# start_date, end_date = cols1.date_input(
# "Date Input for Analysis - select Range for single specie analysis, select single date for daily view",
@@ -93,12 +88,14 @@ else:
# start_date = datetime(2022 ,5 ,17).date()
# end_date = datetime(2022 ,5 ,17).date()
@st.cache()
def date_filter(df, start_date, end_date):
filt = (df2.index >= pd.Timestamp(start_date)) & (df2.index <= pd.Timestamp(end_date + timedelta(days=1)))
df = df[filt]
return(df)
df2 = date_filter(df2, start_date, end_date)
st.write('<style>div.row-widget.stRadio > div{flex-direction:row;justify-content: left;} </style>',
@@ -112,7 +109,7 @@ st.write('<style>div.st-bf{flex-direction:column;} div.st-ag{font-weight:bold;pa
if start_date == end_date:
resample_sel = st.sidebar.radio(
"Resample Resolution",
('Raw', '15 minutes', 'Hourly'), index=1, help= 'Select resolution for single day - larger times run faster' )
('Raw', '15 minutes', 'Hourly'), index=1, help='Select resolution for single day - larger times run faster')
resample_times = {'Raw': 'Raw',
'1 minute': '1min',
@@ -124,7 +121,7 @@ if start_date == end_date:
else:
resample_sel = st.sidebar.radio(
"Resample Resolution",
('Raw', '15 minutes', 'Hourly', 'DAILY'), index=1, help= 'Select resolution for species - DAILY provides time series')
('Raw', '15 minutes', 'Hourly', 'DAILY'), index=1, help='Select resolution for species - DAILY provides time series')
resample_times = {'Raw': 'Raw',
'1 minute': '1min',
@@ -134,6 +131,7 @@ else:
}
resample_time = resample_times[resample_sel]
@st.cache()
def time_resample(df, resample_time):
if resample_time == 'Raw':
@@ -143,6 +141,8 @@ def time_resample(df, resample_time):
df_resample = df.resample(resample_time)['Com_Name'].aggregate('unique').explode()
return(df_resample)
top_bird = df2['Com_Name'].mode()[0]
df5 = time_resample(df2, resample_time)
@@ -151,12 +151,12 @@ df5 = time_resample(df2, resample_time)
Specie_Count = df5.value_counts()
# Create Hourly Crosstab
hourly = pd.crosstab(df5, df5.index.hour, dropna=True, margins= True)
hourly = pd.crosstab(df5, df5.index.hour, dropna=True, margins=True)
# Filter on species
species = list(hourly.sort_values("All", ascending= False).index)
species = list(hourly.sort_values("All", ascending=False).index)
#cols1, cols2 = st.columns((1, 1))
# cols1, cols2 = st.columns((1, 1))
top_N = st.sidebar.slider(
'Select Number of Birds to Show',
min_value=1,
@@ -168,26 +168,26 @@ top_N_species = (df5.value_counts()[:top_N])
font_size = 15
if daily == False:
if daily is False:
if resample_time != '1D':
specie = st.selectbox(
'Which bird would you like to explore for the dates '
+ str(start_date) + ' to ' + str(end_date) + '?',
species,
index = 0)
'Which bird would you like to explore for the dates '
+ str(start_date) + ' to ' + str(end_date) + '?',
species,
index=0)
# filt = df2['Com_Name'] == specie
if specie == 'All':
df_counts = int(hourly[hourly.index==specie]['All'])
df_counts = int(hourly[hourly.index == specie]['All'])
fig = make_subplots(
rows=3, cols=2,
specs=[[{"type": "xy", "rowspan": 3}, {"type": "polar", "rowspan": 2}], [{"rowspan": 1}, {"rowspan": 1}],
[None, {"type": "xy", "rowspan": 1}]],
specs=[[{"type": "xy", "rowspan": 3}, {"type": "polar", "rowspan": 2}],
[{"rowspan": 1}, {"rowspan": 1}],
[None, {"type": "xy", "rowspan": 1}]],
subplot_titles=('<b>Top ' + str(top_N) + ' Species in Date Range ' + str(start_date) + ' to ' + str(
end_date) + '<br>for ' + str(resample_sel) + ' sampling interval.' + '</b>',
'Total Detect:' + str('{:,}'.format(df_counts))
'Total Detect:' + str('{:,}'.format(df_counts))
# + ' Confidence Max:' + str(
# '{:.2f}%'.format(max(df2[df2['Com_Name'] == specie]['Confidence']) * 100)) +
# ' ' + ' Median:' + str(
@@ -197,9 +197,9 @@ if daily == False:
fig.layout.annotations[1].update(x=0.7, y=0.25, font_size=15)
# Plot seen species for selected date range and number of species
fig.add_trace(go.Bar(y=top_N_species.index, x=top_N_species, orientation='h', marker_color='seagreen'), row=1, col=1)
fig.update_layout(
margin=dict(l=0, r=0, t=50, b=0),
yaxis={'categoryorder': 'total ascending'})
@@ -232,16 +232,17 @@ if daily == False:
rotation=-90,
direction='clockwise',
tickmode='array',
tickvals=[0, 15, 35, 45, 60, 75, 90, 105, 120, 135, 150, 165, 180, 195, 210, 225, 240, 255, 270,
285, 300, 315, 330, 345],
ticktext=['12am', '1am', '2am', '3am', '4am', '5am', '6am', '7am', '8am', '9am', '10am', '11am',
'12pm', '1pm', '2pm', '3pm', '4pm', '5pm', '6pm', '7pm', '8pm', '9pm', '10pm', '11pm'],
tickvals=[0, 15, 35, 45, 60, 75, 90, 105, 120, 135, 150, 165, 180, 195, 210,
225, 240, 255, 270, 285, 300, 315, 330, 345],
ticktext=['12am', '1am', '2am', '3am', '4am', '5am', '6am', '7am', '8am', '9am',
'10am', '11am', '12pm', '1pm', '2pm', '3pm', '4pm', '5pm', '6pm',
'7pm', '8pm', '9pm', '10pm', '11pm'],
hoverformat="#%{theta}: <br>Popularity: %{percent} </br> %{r}"
),
),
)
daily = pd.crosstab(df5, df5.index.date, dropna=True, margins = True)
daily = pd.crosstab(df5, df5.index.date, dropna=True, margins=True)
fig.add_trace(go.Bar(x=daily.columns[:-1], y=daily.loc[specie][:-1], marker_color='seagreen'), row=3, col=2)
st.plotly_chart(fig, use_container_width=True) # , config=config)
@@ -250,7 +251,7 @@ if daily == False:
with col1:
fig = make_subplots(
rows=3, cols=1,
specs=[[{"type": "polar", "rowspan": 2}],[{"rowspan": 1}], [{"type": "xy", "rowspan": 1}]]
specs=[[{"type": "polar", "rowspan": 2}], [{"rowspan": 1}], [{"type": "xy", "rowspan": 1}]]
)
# Set 360 degrees, 24 hours for polar plot
theta = np.linspace(0.0, 360, 24, endpoint=False)
@@ -280,81 +281,75 @@ if daily == False:
rotation=-90,
direction='clockwise',
tickmode='array',
tickvals=[0, 15, 35, 45, 60, 75, 90, 105, 120, 135, 150, 165, 180, 195, 210, 225, 240, 255, 270,
285, 300, 315, 330, 345],
ticktext=['12am', '1am', '2am', '3am', '4am', '5am', '6am', '7am', '8am', '9am', '10am', '11am',
'12pm', '1pm', '2pm', '3pm', '4pm', '5pm', '6pm', '7pm', '8pm', '9pm', '10pm', '11pm'],
tickvals=[0, 15, 35, 45, 60, 75, 90, 105, 120, 135, 150, 165, 180, 195,
210, 225, 240, 255, 270, 285, 300, 315, 330, 345],
ticktext=['12am', '1am', '2am', '3am', '4am', '5am', '6am', '7am', '8am',
'9am', '10am', '11am', '12pm', '1pm', '2pm', '3pm', '4pm', '5pm',
'6pm', '7pm', '8pm', '9pm', '10pm', '11pm'],
hoverformat="#%{theta}: <br>Popularity: %{percent} </br> %{r}"
),
),
)
daily = pd.crosstab(df5, df5.index.date, dropna=True, margins = True)
daily = pd.crosstab(df5, df5.index.date, dropna=True, margins=True)
fig.add_trace(go.Bar(x=daily.columns[:-1], y=daily.loc[specie][:-1], marker_color='seagreen'), row=3, col=1)
st.plotly_chart(fig, use_container_width=True) # , config=config)
df_counts = int(hourly[hourly.index==specie]['All'])
st.subheader('Total Detect:' + str('{:,}'.format(df_counts))
+ ' Confidence Max:' + str(
'{:.2f}%'.format(max(df2[df2['Com_Name'] == specie]['Confidence']) * 100)) +
' ' + ' Median:' + str(
'{:.2f}%'.format(np.median(df2[df2['Com_Name'] == specie]['Confidence']) * 100)))
df_counts = int(hourly[hourly.index == specie]['All'])
st.subheader('Total Detect:' + str('{:,}'.format(df_counts))
+ ' Confidence Max:' +
str('{:.2f}%'.format(max(df2[df2['Com_Name'] == specie]['Confidence']) * 100))
+ ' ' + ' Median:' +
str('{:.2f}%'.format(np.median(df2[df2['Com_Name'] == specie]['Confidence']) * 100)))
recordings = df2[df2['Com_Name'] == specie]['File_Name']
recordings=df2[df2['Com_Name']==specie]['File_Name']
with col2:
try:
recording = st.selectbox('Available recordings', recordings.sort_index(ascending=False))
date_specie = df2.loc[df2['File_Name']==recording,['Date','Com_Name']]
date_specie = df2.loc[df2['File_Name'] == recording, ['Date', 'Com_Name']]
date_dir = date_specie['Date'].values[0]
specie_dir = date_specie['Com_Name'].values[0].replace(" ","_")
st.image(userDir + '/BirdSongs/Extracted/By_Date/'+ date_dir + '/'+ specie_dir + '/' + recording + '.png')
st.audio(userDir +'/BirdSongs/Extracted/By_Date/'+ date_dir + '/'+ specie_dir + '/' + recording)
except:
st.title('RECORDING NOT AVAILABLE :(')
specie_dir = date_specie['Com_Name'].values[0].replace(" ", "_")
st.image(userDir + '/BirdSongs/Extracted/By_Date/' + date_dir + '/' + specie_dir + '/' + recording + '.png')
st.audio(userDir + '/BirdSongs/Extracted/By_Date/' + date_dir + '/' + specie_dir + '/' + recording)
except Exception:
st.title('RECORDING NOT AVAILABLE :(')
# try:
# con = sqlite3.connect(userDir + '/BirdNET-Pi/scripts/birds.db')
# cur = con.cursor()
cola, colb, colc, cold = st.columns((3,1,1,1))
cola, colb, colc, cold = st.columns((3, 1, 1, 1))
with colb:
seen = st.checkbox('Reviewed')
if seen:
with colc:
verified = st.radio("Verification",['True Positive','False Positive'])
verified = st.radio("Verification", ['True Positive', 'False Positive'])
if verified == "False Positive":
df_names = pd.read_csv(userDir+'/BirdNET-Pi/model/labels.txt', delimiter= '_', names=['Sci_Name', 'Com_Name'])
df_unknown= pd.DataFrame({"Sci_Name":["UNKNOWN"],"Com_Name":["UNKNOWN"]})
df_names = pd.concat([df_unknown,df_names], ignore_index=True)
df_names = pd.read_csv(userDir + '/BirdNET-Pi/model/labels.txt', delimiter='_', names=['Sci_Name', 'Com_Name'])
df_unknown = pd.DataFrame({"Sci_Name": ["UNKNOWN"], "Com_Name": ["UNKNOWN"]})
df_names = pd.concat([df_unknown, df_names], ignore_index=True)
with cold:
corrected = st.selectbox('What species?', df_names['Com_Name'])
# cur.execute("UPDATE detections SET Seen = seen WHERE File_Name = recording")
# con.commit()
# con.close()
# except BaseException:
# print("Database busy")
# time.sleep(2)
else:
specie = st.selectbox(
'Which bird would you like to explore for the dates '
+ str(start_date) + ' to ' + str(end_date) + '?',
species[1:],
index = 0)
specie = st.selectbox('Which bird would you like to explore for the dates '
+ str(start_date) + ' to ' + str(end_date) + '?',
species[1:],
index=0)
# filt = df2[df2['Com_Name'] == specie]
df_counts = int(hourly[hourly.index==specie]['All'])
df_counts = int(hourly[hourly.index == specie]['All'])
fig = st.container()
fig = make_subplots(
rows=1, cols =1)
fig = make_subplots(rows=1, cols=1)
# specs= [[{"type":"xy","rowspan":1},{"type":"heatmap","rowspan":1}]],
# subplot_titles=('<b>Daily Top '+ str(top_N) + ' Species in Date Range '+ str(start_date) +' to '+ str(end_date) +'</b>',
# '<b>Daily ' + specie+ ' Detections on 15 minute intervals </b>'),
@@ -365,9 +360,9 @@ if daily == False:
# )
# fig.add_trace(go.Bar(y=top_N_species.index, x=top_N_species, orientation='h'), row=1,col=1)
df4=df2['Com_Name'][df2['Com_Name']==specie].resample('15min').count()
df4.index=[df4.index.date, df4.index.time]
day_hour_freq=df4.unstack().fillna(0)
df4 = df2['Com_Name'][df2['Com_Name'] == specie].resample('15min').count()
df4.index = [df4.index.date, df4.index.time]
day_hour_freq = df4.unstack().fillna(0)
fig_x = [d.strftime('%d-%m-%Y') for d in day_hour_freq.index.tolist()]
fig_y = [h.strftime('%H:%M') for h in day_hour_freq.columns.tolist()]
@@ -377,9 +372,9 @@ if daily == False:
# fig.update_layout(
# margin=dict(l=0, r=0, t=50, b=0),
# yaxis={'categoryorder':'total ascending'})
color_pals= px.colors.named_colorscales()
color_pals = px.colors.named_colorscales()
selected_pal = st.sidebar.selectbox('Select Color Pallet for Daily Detections', color_pals)
fig.add_trace(go.Heatmap(x=fig_x,y=fig_y,z=fig_z, autocolorscale = False, colorscale = selected_pal), row=1, col=1)
fig.add_trace(go.Heatmap(x=fig_x, y=fig_y, z=fig_z, autocolorscale=False, colorscale=selected_pal), row=1, col=1)
st.plotly_chart(fig, use_container_width=True) # , config=config)
else:
fig = make_subplots(
@@ -436,9 +431,7 @@ else:
st.plotly_chart(fig, use_container_width=True) # , config=config)
# cols3,cols4=st.columns((1,1))
#
# extract_date=Date_Slider
#
# audio_file = open('/home/*/BirdSongs/Extracted/By_Date/2022-03-22/Yellow-streaked_Greenbul/Yellow-streaked_Greenbul-77-2022-03-22-birdnet-15:04:28.mp3', 'rb')
# audio_bytes = audio_file.read()
# cols4.audio(audio_bytes, format='audio/mp3')
+32 -66
View File
@@ -1,4 +1,3 @@
import apprise
from pathlib import Path
from tzlocal import get_localzone
import datetime
@@ -13,6 +12,10 @@ import operator
import socket
import threading
import os
from utils.notifications import sendAppriseNotifications
from utils.parse_settings import config_to_settings
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
os.environ['CUDA_VISIBLE_DEVICES'] = ''
@@ -29,6 +32,10 @@ ADDR = (SERVER, PORT)
FORMAT = 'utf-8'
DISCONNECT_MESSAGE = "!DISCONNECT"
userDir = os.path.expanduser('~')
DB_PATH = userDir + '/BirdNET-Pi/scripts/birds.db'
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
@@ -40,7 +47,6 @@ except BaseException:
# Open most recent Configuration and grab DB_PWD as a python variable
userDir = os.path.expanduser('~')
with open(userDir + '/BirdNET-Pi/scripts/thisrun.txt', 'r') as f:
this_run = f.readlines()
audiofmt = "." + str(str(str([i for i in this_run if i.startswith('AUDIOFMT')]).split('=')[1]).split('\\')[0])
@@ -228,59 +234,6 @@ def analyzeAudioData(chunks, lat, lon, week, sensitivity, overlap,):
return detections
def sendAppriseNotifications(species, confidence, path):
if os.path.exists(userDir + '/BirdNET-Pi/apprise.txt') and os.path.getsize(userDir + '/BirdNET-Pi/apprise.txt') > 0:
with open(userDir + '/BirdNET-Pi/scripts/thisrun.txt', 'r') as f:
this_run = f.readlines()
title = str(str(str([i for i in this_run if i.startswith('APPRISE_NOTIFICATION_TITLE')]).split('=')[1]).split('\\')[0]).replace('"', '')
body = str(str(str([i for i in this_run if i.startswith('APPRISE_NOTIFICATION_BODY')]).split('=')[1]).split('\\')[0]).replace('"', '')
try:
websiteurl = str(str(str([i for i in this_run if i.startswith('BIRDNETPI_URL')]).split('=')[1]).split('\\')[0]).replace('"', '')
if len(websiteurl) == 0:
raise ValueError('Blank URL')
except Exception as e:
websiteurl = "http://"+socket.gethostname()+".local"
listenurl = websiteurl+"?filename="+path
if str(str(str([i for i in this_run if i.startswith('APPRISE_NOTIFY_EACH_DETECTION')]).split('=')[1]).split('\\')[0]) == "1":
apobj = apprise.Apprise()
config = apprise.AppriseConfig()
config.add(userDir + '/BirdNET-Pi/apprise.txt')
apobj.add(config)
apobj.notify(
body=body.replace("$sciname", species.split("_")[0]).replace("$comname", species.split("_")[1]).replace("$confidence", confidence).replace("$listenurl", listenurl),
title=title.replace("$sciname", species.split("_")[0]).replace("$comname", species.split("_")[1]).replace("$confidence", confidence).replace("$listenurl", listenurl),
)
if str(str(str([i for i in this_run if i.startswith('APPRISE_NOTIFY_NEW_SPECIES')]).split('=')[1]).split('\\')[0]) == "1":
try:
con = sqlite3.connect(userDir + '/BirdNET-Pi/scripts/birds.db')
cur = con.cursor()
cur.execute("SELECT DISTINCT(Com_Name), count(Com_Name) FROM detections WHERE date > (SELECT DATETIME('now', '-7 day')) GROUP BY Com_Name")
known_species = cur.fetchall()
sciName, comName = species.split("_")
numberDetections = [d[1] for d in known_species if d[0] == comName.replace("'","")][0]
if numberDetections <= 5:
apobj = apprise.Apprise()
config = apprise.AppriseConfig()
config.add(userDir + '/BirdNET-Pi/apprise.txt')
apobj.add(config)
apobj.notify(
body=body.replace("$sciname", species.split("_")[0]).replace("$comname", species.split("_")[1]).replace("$confidence", confidence).replace("$listenurl", listenurl) + " (only seen "+str(int(numberDetections))+" times in last 7d)",
title=title.replace("$sciname", species.split("_")[0]).replace("$comname", species.split("_")[1]).replace("$confidence", confidence).replace("$listenurl", listenurl) + " (only seen "+str(int(numberDetections))+" times in last 7d)",
)
con.close()
except BaseException:
print("Database busy")
time.sleep(2)
def writeResultsToFile(detections, min_conf, path):
print('WRITING RESULTS TO', path, '...', end=' ')
@@ -408,13 +361,16 @@ def handle_client(conn, addr):
with open(userDir + '/BirdNET-Pi/BirdDB.txt', 'a') as rfile:
for d in detections:
species_apprised_this_run = []
for entry in detections[d]:
if entry[1] >= min_conf and ((entry[0] in INCLUDE_LIST or len(INCLUDE_LIST) == 0)
and (entry[0] not in EXCLUDE_LIST or len(EXCLUDE_LIST) == 0)):
# Write to text file.
rfile.write(str(current_date) + ';' + str(current_time) + ';' + entry[0].replace('_', ';') + ';'
+ str(entry[1]) + ";" + str(args.lat) + ';' + str(args.lon) + ';' + str(min_conf) + ';' + str(week) + ';'
+ str(args.sensitivity) + ';' + str(args.overlap) + '\n')
# Write to database
Date = str(current_date)
Time = str(current_time)
species = entry[0]
@@ -434,7 +390,7 @@ def handle_client(conn, addr):
# Connect to SQLite Database
for attempt_number in range(3):
try:
con = sqlite3.connect(userDir + '/BirdNET-Pi/scripts/birds.db')
con = sqlite3.connect(DB_PATH)
cur = con.cursor()
cur.execute("INSERT INTO detections VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (Date, Time,
Sci_Name, Com_Name, str(score), Lat, Lon, Cutoff, Week, Sens, Overlap, File_Name))
@@ -445,8 +401,24 @@ def handle_client(conn, addr):
except BaseException:
print("Database busy")
time.sleep(2)
sendAppriseNotifications(str(entry[0]), str(entry[1]), File_Name)
# Apprise of detection if not already alerted this run.
if not entry[0] in species_apprised_this_run:
settings_dict = config_to_settings(userDir + '/BirdNET-Pi/scripts/thisrun.txt')
sendAppriseNotifications(species,
str(score),
File_Name,
Date,
Time,
Week,
Lat,
Lon,
Cutoff,
Sens,
Overlap,
settings_dict,
DB_PATH)
species_apprised_this_run.append(entry[0])
print(str(current_date) +
';' +
@@ -467,14 +439,8 @@ def handle_client(conn, addr):
str(args.sensitivity) +
';' +
str(args.overlap) +
Com_Name.replace(" ", "_") +
'-' +
str(score) +
'-' +
str(current_date) +
'-birdnet-' +
str(current_time) +
audiofmt +
';' +
File_Name +
'\n')
if birdweather_id != "99999":
+12 -2
View File
@@ -40,6 +40,9 @@ fi
if ! grep APPRISE_NOTIFY_NEW_SPECIES /etc/birdnet/birdnet.conf &>/dev/null;then
sudo -u$USER echo "APPRISE_NOTIFY_NEW_SPECIES=0 " >> /etc/birdnet/birdnet.conf
fi
if ! grep APPRISE_NOTIFY_NEW_SPECIES_EACH_DAY /etc/birdnet/birdnet.conf &>/dev/null;then
sudo -u$USER echo "APPRISE_NOTIFY_NEW_SPECIES_EACH_DAY=0 " >> /etc/birdnet/birdnet.conf
fi
# If the config does not contain the DATABASE_LANG setting, we'll want to add it.
# Defaults to not-selected, which config.php will know to render as a language option.
@@ -50,8 +53,8 @@ fi
apprise_installation_status=$(~/BirdNET-Pi/birdnet/bin/python3 -c 'import pkgutil; print("installed" if pkgutil.find_loader("apprise") else "not installed")')
if [[ "$apprise_installation_status" = "not installed" ]];then
~/BirdNET-Pi/birdnet/bin/pip3 install -U pip
~/BirdNET-Pi/birdnet/bin/pip3 install apprise
$HOME/BirdNET-Pi/birdnet/bin/pip3 install -U pip
$HOME/BirdNET-Pi/birdnet/bin/pip3 install apprise
fi
[ -f $HOME/BirdNET-Pi/apprise.txt ] || sudo -E -ucaddy touch $HOME/BirdNET-Pi/apprise.txt
if ! which lsof &>/dev/null;then
@@ -85,5 +88,12 @@ if ! grep FLICKR_FILTER_EMAIL /etc/birdnet/birdnet.conf &>/dev/null;then
sudo -u$USER echo "FLICKR_FILTER_EMAIL=" >> /etc/birdnet/birdnet.conf
fi
pytest_installation_status=$(~/BirdNET-Pi/birdnet/bin/python3 -c 'import pkgutil; print("installed" if pkgutil.find_loader("pytest") else "not installed")')
if [[ "$pytest_installation_status" = "not installed" ]];then
$HOME/BirdNET-Pi/birdnet/bin/pip3 install -U pip
$HOME/BirdNET-Pi/birdnet/bin/pip3 install pytest==7.1.2 pytest-mock==3.7.0
fi
sudo systemctl daemon-reload
restart_services.sh
View File
+161
View File
@@ -0,0 +1,161 @@
import apprise
import os
import socket
import sqlite3
from datetime import datetime
userDir = os.path.expanduser('~')
APPRISE_CONFIG = userDir + '/BirdNET-Pi/apprise.txt'
DB_PATH = userDir + '/BirdNET-Pi/scripts/birds.db'
def notify(body, title):
apobj = apprise.Apprise()
config = apprise.AppriseConfig()
config.add(APPRISE_CONFIG)
apobj.add(config)
apobj.notify(
body=body,
title=title,
)
def sendAppriseNotifications(species, confidence, path, date, time, week, latitude, longitude, cutoff, sens, overlap, settings_dict, db_path=DB_PATH):
# print(sendAppriseNotifications)
# print(settings_dict)
if os.path.exists(APPRISE_CONFIG) and os.path.getsize(APPRISE_CONFIG) > 0:
title = settings_dict.get('APPRISE_NOTIFICATION_TITLE')
body = settings_dict.get('APPRISE_NOTIFICATION_BODY')
sciName, comName = species.split("_")
try:
websiteurl = settings_dict.get('BIRDNETPI_URL')
if len(websiteurl) == 0:
raise ValueError('Blank URL')
except Exception:
websiteurl = "http://"+socket.gethostname()+".local"
listenurl = websiteurl+"?filename="+path
if settings_dict.get('APPRISE_NOTIFY_EACH_DETECTION') == "1":
notify_body = body.replace("$sciname", sciName)\
.replace("$comname", comName)\
.replace("$confidence", confidence)\
.replace("$listenurl", listenurl)\
.replace("$date", date)\
.replace("$time", time)\
.replace("$week", week)\
.replace("$latitude", latitude)\
.replace("$longitude", longitude)\
.replace("$cutoff", cutoff)\
.replace("$sens", sens)\
.replace("$overlap", overlap)
notify_title = title.replace("$sciname", sciName)\
.replace("$comname", comName)\
.replace("$confidence", confidence)\
.replace("$listenurl", listenurl)\
.replace("$date", date)\
.replace("$time", time)\
.replace("$week", week)\
.replace("$latitude", latitude)\
.replace("$longitude", longitude)\
.replace("$cutoff", cutoff)\
.replace("$sens", sens)\
.replace("$overlap", overlap)
notify(notify_body, notify_title)
APPRISE_NOTIFICATION_NEW_SPECIES_DAILY_COUNT_LIMIT = 1 # Notifies the first N per day.
if settings_dict.get('APPRISE_NOTIFY_NEW_SPECIES_EACH_DAY') == "1":
try:
con = sqlite3.connect(db_path)
cur = con.cursor()
today = datetime.now().strftime("%Y-%m-%d")
cur.execute(f"SELECT DISTINCT(Com_Name), COUNT(Com_Name) FROM detections WHERE Date = DATE('{today}') GROUP BY Com_Name")
known_species = cur.fetchall()
detections = [d[1] for d in known_species if d[0] == comName.replace("'", "")]
numberDetections = 0
if len(detections):
numberDetections = detections[0]
if numberDetections > 0 and numberDetections <= APPRISE_NOTIFICATION_NEW_SPECIES_DAILY_COUNT_LIMIT:
print("send the notification")
notify_body = body.replace("$sciname", sciName)\
.replace("$comname", comName)\
.replace("$confidence", confidence)\
.replace("$listenurl", listenurl)\
.replace("$date", date)\
.replace("$time", time)\
.replace("$week", week)\
.replace("$latitude", latitude)\
.replace("$longitude", longitude)\
.replace("$cutoff", cutoff)\
.replace("$sens", sens)\
.replace("$overlap", overlap)\
+ " (first time today)"
notify_title = title.replace("$sciname", sciName)\
.replace("$comname", comName)\
.replace("$confidence", confidence)\
.replace("$listenurl", listenurl)\
.replace("$date", date)\
.replace("$time", time)\
.replace("$week", week)\
.replace("$latitude", latitude)\
.replace("$longitude", longitude)\
.replace("$cutoff", cutoff)\
.replace("$sens", sens)\
.replace("$overlap", overlap)\
+ " (first time today)"
notify(notify_body, notify_title)
con.close()
except sqlite3.Error as e:
print(e)
print("Database busy")
time.sleep(2)
if settings_dict.get('APPRISE_NOTIFY_NEW_SPECIES') == "1":
try:
con = sqlite3.connect(db_path)
cur = con.cursor()
today = datetime.now().strftime("%Y-%m-%d")
cur.execute(f"SELECT DISTINCT(Com_Name), COUNT(Com_Name) FROM detections WHERE Date >= DATE('{today}', '-7 day') GROUP BY Com_Name")
known_species = cur.fetchall()
detections = [d[1] for d in known_species if d[0] == comName.replace("'", "")]
numberDetections = 0
if len(detections):
numberDetections = detections[0]
if numberDetections > 0 and numberDetections <= 5:
notify_body = body.replace("$sciname", sciName)\
.replace("$comname", comName)\
.replace("$confidence", confidence)\
.replace("$listenurl", listenurl)\
.replace("$date", date)\
.replace("$time", time)\
.replace("$week", week)\
.replace("$latitude", latitude)\
.replace("$longitude", longitude)\
.replace("$cutoff", cutoff)\
.replace("$sens", sens)\
.replace("$overlap", overlap)\
+ " (only seen " + str(int(numberDetections)) + " times in last 7d)"
notify_title = title.replace("$sciname", sciName)\
.replace("$comname", comName)\
.replace("$confidence", confidence)\
.replace("$listenurl", listenurl)\
.replace("$date", date)\
.replace("$time", time)\
.replace("$week", week)\
.replace("$latitude", latitude)\
.replace("$longitude", longitude)\
.replace("$cutoff", cutoff)\
.replace("$sens", sens)\
.replace("$overlap", overlap)\
+ " (only seen " + str(int(numberDetections)) + " times in last 7d)"
notify(notify_body, notify_title)
con.close()
except sqlite3.Error:
print("Database busy")
time.sleep(2)
if __name__ == "__main__":
print("notfications")
+16
View File
@@ -0,0 +1,16 @@
def config_to_settings(path):
# Returns settings dict from BirdNET-pi text format.
# Consider refactoring to use ConfigParser files, or another standardized format.
settings = {}
with open(path, 'r') as f:
this_run = f.readlines()
for i in this_run:
key = i.split("=")[0]
value = i.split("=")[1][:-1]
# Trim for strings, not ideal
if value.startswith('"') and value.endswith('"'):
value = value[1:-1]
if value.startswith("'") and value.endswith("'"):
value = value[1:-1]
settings[key] = value
return settings
View File
+83
View File
@@ -0,0 +1,83 @@
import os
import sqlite3
import pytest
from scripts.utils.notifications import sendAppriseNotifications
from datetime import datetime
def create_test_db(db_file):
""" create a database connection to a SQLite database """
conn = None
try:
conn = sqlite3.connect(db_file)
sql_create_detections_table = """ CREATE TABLE IF NOT EXISTS detections (
id integer PRIMARY KEY,
Com_Name text NOT NULL,
Date date NULL,
Time time NULL
); """
cur = conn.cursor()
cur.execute(sql_create_detections_table)
sql = ''' INSERT INTO detections(Com_Name, Date)
VALUES(?,?) '''
cur = conn.cursor()
today = datetime.now().strftime("%Y-%m-%d") # SQLite stores date as YYYY-MM-DD
cur.execute(sql, ["Great Crested Flycatcher", today])
conn.commit()
except Exception as e:
print(e)
finally:
if conn:
conn.close()
@pytest.fixture(autouse=True)
def clean_up_after_each_test():
yield
os.remove("test.db")
def test_notifications(mocker):
notify_call = mocker.patch('scripts.utils.notifications.notify')
create_test_db("test.db")
settings_dict = {
"APPRISE_NOTIFICATION_TITLE": "New backyard bird!",
"APPRISE_NOTIFICATION_BODY": "A $comname ($sciname) was just detected with a confidence of $confidence",
"APPRISE_NOTIFY_EACH_DETECTION": "0",
"APPRISE_NOTIFY_NEW_SPECIES": "0",
"APPRISE_NOTIFY_NEW_SPECIES_EACH_DAY": "0"
}
# No active apprise notifcations configured. Confirm no notifications.
sendAppriseNotifications("Myiarchus crinitus_Great Crested Flycatcher", "91", "filename", settings_dict, "test.db")
assert(notify_call.call_count == 0) # No notification should be sent.
# Add daily notification.
notify_call.reset_mock()
settings_dict["APPRISE_NOTIFY_NEW_SPECIES_EACH_DAY"] = "1"
sendAppriseNotifications("Myiarchus crinitus_Great Crested Flycatcher", "91", "filename", settings_dict, "test.db")
assert(notify_call.call_count == 1)
assert(
notify_call.call_args_list[0][0][0] == "A Great Crested Flycatcher (Myiarchus crinitus) was just detected with a confidence of 91 (first time today)"
)
# Add new species notification.
notify_call.reset_mock()
settings_dict["APPRISE_NOTIFY_NEW_SPECIES"] = "1"
sendAppriseNotifications("Myiarchus crinitus_Great Crested Flycatcher", "91", "filename", settings_dict, "test.db")
assert(notify_call.call_count == 2)
assert(
notify_call.call_args_list[0][0][0] == "A Great Crested Flycatcher (Myiarchus crinitus) was just detected with a confidence of 91 (first time today)"
)
assert(
notify_call.call_args_list[1][0][0] == "A Great Crested Flycatcher (Myiarchus crinitus) \
was just detected with a confidence of 91 (only seen 1 times in last 7d)"
)
# Add each species notification.
notify_call.reset_mock()
settings_dict["APPRISE_NOTIFY_EACH_DETECTION"] = "1"
sendAppriseNotifications("Myiarchus crinitus_Great Crested Flycatcher", "91", "filename", settings_dict, "test.db")
assert(notify_call.call_count == 3)
+44
View File
@@ -0,0 +1,44 @@
from scripts.utils.parse_settings import config_to_settings
import tempfile
def test():
text = """LATITUDE=32.0
LONGITUDE=-73.0
BIRDWEATHER_ID=
CADDY_PWD="nonsuchpass"
ICE_PWD=birdnetpi
BIRDNETPI_URL=
RTSP_STREAM=
APPRISE_NOTIFICATION_TITLE="Bird!"
APPRISE_NOTIFICATION_BODY="A $comname ($sciname) was just detected with a confidence of $confidence"
APPRISE_NOTIFY_EACH_DETECTION=0
APPRISE_NOTIFY_NEW_SPECIES=1
FLICKR_API_KEY=
FLICKR_FILTER_EMAIL=
RECS_DIR=/home/pi/BirdSongs
REC_CARD=default
PROCESSED=/home/pi/BirdSongs/Processed
EXTRACTED=/home/pi/BirdSongs/Extracted
OVERLAP=0.0
CONFIDENCE=0.7
SENSITIVITY=1.25
CHANNELS=2
FULL_DISK=purge
PRIVACY_THRESHOLD=0
RECORDING_LENGTH=15
EXTRACTION_LENGTH=
AUDIOFMT=mp3
DATABASE_LANG=en
LAST_RUN=
THIS_RUN=
IDFILE=/home/pi/BirdNET-Pi/IdentifiedSoFar.txt"""
filename = tempfile.NamedTemporaryFile(suffix='.txt', delete=False)
with open(filename.name, 'w', encoding='utf8', newline='') as f:
f.write(text)
settings = config_to_settings(filename.name)
assert(settings["APPRISE_NOTIFICATION_TITLE"] == "Bird!")
assert(settings["FULL_DISK"] == "purge")
assert(settings["OVERLAP"] == "0.0") # Yes, it's a string at this point.