From 528802a0e6dd7b9b24e1a8a5b068ac3666d05413 Mon Sep 17 00:00:00 2001
From: Patrick McGuire <60325264+mcguirepr89@users.noreply.github.com>
Date: Wed, 11 May 2022 09:05:59 -0400
Subject: [PATCH] Revert "Adding Flake8 Github Action for Python Linting "
---
.flake8 | 3 -
.github/workflows/ci.yml | 23 ----
scripts/analyze.py | 96 +++-----------
scripts/daily_plot.py | 210 +++++++++++-------------------
scripts/plotly_streamlit.py | 148 ++++++++++-----------
scripts/server.py | 249 ++++++++++++++----------------------
6 files changed, 256 insertions(+), 473 deletions(-)
delete mode 100644 .flake8
delete mode 100644 .github/workflows/ci.yml
diff --git a/.flake8 b/.flake8
deleted file mode 100644
index 467c30e..0000000
--- a/.flake8
+++ /dev/null
@@ -1,3 +0,0 @@
-[flake8]
-max-line-length = 128
-
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
deleted file mode 100644
index d2f684a..0000000
--- a/.github/workflows/ci.yml
+++ /dev/null
@@ -1,23 +0,0 @@
-name: CI Jobs
-
-on: pull_request
-
-jobs:
- python-lint:
- runs-on: ubuntu-latest
- steps:
- - name: Checkout
- uses: actions/checkout@v3
-
- - name: Setup Python
- uses: actions/setup-python@v3
- with:
- python-version: '3.9.x'
- cache: 'pip'
- architecture: 'x64'
-
- - name: Install flake8
- run: pip install flake8
-
- - name: Run Flake8 Lint
- uses: py-actions/flake8@v2
diff --git a/scripts/analyze.py b/scripts/analyze.py
index 90edf55..4f36e80 100755
--- a/scripts/analyze.py
+++ b/scripts/analyze.py
@@ -1,16 +1,3 @@
-#!/usr/bin/env python3
-
-"""
-# Example calls
-python3 analyze.py --i 'example/XC558716 - Soundscape.mp3' \
- --lat 35.4244 --lon -120.7463 --week 18
-
-python3 analyze.py --i 'example/XC563936 - Soundscape.mp3' \
- --lat 47.6766 --lon -122.294 --week 11 \
- --overlap 1.5 --min_conf 0.25 --sensitivity 1.25 \
- --custom_list 'example/custom_species_list.txt'
-"""
-
import argparse
import socket
@@ -24,7 +11,6 @@ ADDR = (SERVER, PORT)
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client.connect(ADDR)
-
def send(msg):
message = msg.encode(FORMAT)
msg_length = len(message)
@@ -34,7 +20,6 @@ def send(msg):
client.send(message)
print(client.recv(2048).decode(FORMAT))
-
def main():
global INCLUDE_LIST
@@ -42,62 +27,17 @@ def main():
# Parse passed arguments
parser = argparse.ArgumentParser()
- parser.add_argument(
- '--i',
- help='Path to input file.')
- parser.add_argument(
- '--o',
- default='result.csv',
- help='Path to output file. Defaults to result.csv.')
- parser.add_argument(
- '--lat',
- type=float,
- default=-1,
- help='Recording location latitude. Set -1 to ignore.')
- parser.add_argument(
- '--lon',
- type=float,
- default=-1,
- help='Recording location longitude. Set -1 to ignore.')
- parser.add_argument(
- '--week',
- type=int,
- default=-1,
- help='''Week of the year when the recording was made.
- Values in [1, 48] (4 weeks per month). Set -1 to ignore.''')
- parser.add_argument(
- '--overlap',
- type=float,
- default=0.0,
- help='''Overlap in seconds between extracted spectrograms.
- Values in [0.0, 2.9]. Defaults tp 0.0.''')
- parser.add_argument(
- '--sensitivity',
- type=float,
- default=1.0,
- help='''Detection sensitivity;
- Higher values result in higher sensitivity.
- Values in [0.5, 1.5]. Defaults to 1.0.''')
- parser.add_argument(
- '--min_conf',
- type=float,
- default=0.1,
- help='''Minimum confidence threshold.
- Values in [0.01, 0.99]. Defaults to 0.1.''')
- parser.add_argument(
- '--include_list',
- default='null',
- help='''Path to text file containing a list of included species.
- Not used if not provided.''')
- parser.add_argument(
- '--exclude_list',
- default='null',
- help='''Path to text file containing a list of excluded species.
- Not used if not provided.''')
- parser.add_argument(
- '--birdweather_id',
- default='99999',
- help='Private Station ID for BirdWeather.')
+ parser.add_argument('--i', help='Path to input file.')
+ parser.add_argument('--o', default='result.csv', help='Path to output file. Defaults to result.csv.')
+ parser.add_argument('--lat', type=float, default=-1, help='Recording location latitude. Set -1 to ignore.')
+ parser.add_argument('--lon', type=float, default=-1, help='Recording location longitude. Set -1 to ignore.')
+ parser.add_argument('--week', type=int, default=-1, help='Week of the year when the recording was made. Values in [1, 48] (4 weeks per month). Set -1 to ignore.')
+ parser.add_argument('--overlap', type=float, default=0.0, help='Overlap in seconds between extracted spectrograms. Values in [0.0, 2.9]. Defaults tp 0.0.')
+ parser.add_argument('--sensitivity', type=float, default=1.0, help='Detection sensitivity; Higher values result in higher sensitivity. Values in [0.5, 1.5]. Defaults to 1.0.')
+ parser.add_argument('--min_conf', type=float, default=0.1, help='Minimum confidence threshold. Values in [0.01, 0.99]. Defaults to 0.1.')
+ parser.add_argument('--include_list', default='null', help='Path to text file containing a list of included species. Not used if not provided.')
+ parser.add_argument('--exclude_list', default='null', help='Path to text file containing a list of excluded species. Not used if not provided.')
+ parser.add_argument('--birdweather_id', default='99999', help='Private Station ID for BirdWeather.')
args = parser.parse_args()
@@ -124,15 +64,19 @@ def main():
sockParams += 'lat=' + str(args.lat) + '||'
if args.lon:
sockParams += 'lon=' + str(args.lon) + '||'
-
+
send(sockParams)
send(DISCONNECT_MESSAGE)
- # time.sleep(3)
-
-###############################################################################
-###############################################################################
+ #time.sleep(3)
+###############################################################################
+###############################################################################
if __name__ == '__main__':
+
main()
+
+ # Example calls
+ # python3 analyze.py --i 'example/XC558716 - Soundscape.mp3' --lat 35.4244 --lon -120.7463 --week 18
+ # python3 analyze.py --i 'example/XC563936 - Soundscape.mp3' --lat 47.6766 --lon -122.294 --week 11 --overlap 1.5 --min_conf 0.25 --sensitivity 1.25 --custom_list 'example/custom_species_list.txt'
diff --git a/scripts/daily_plot.py b/scripts/daily_plot.py
index be7d853..e2147b0 100755
--- a/scripts/daily_plot.py
+++ b/scripts/daily_plot.py
@@ -1,5 +1,6 @@
import sqlite3
import os
+import configparser
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
@@ -11,113 +12,80 @@ userDir = os.path.expanduser('~')
conn = sqlite3.connect(userDir + '/BirdNET-Pi/scripts/birds.db')
df = pd.read_sql_query("SELECT * from detections", conn)
cursor = conn.cursor()
-cursor.execute(
- 'SELECT * FROM detections WHERE Date = DATE(\'now\', \'localtime\')')
+cursor.execute('SELECT * FROM detections WHERE Date = DATE(\'now\', \'localtime\')')
table_rows = cursor.fetchall()
-# df=pd.DataFrame(table_rows)
+#df=pd.DataFrame(table_rows)
-# Convert Date and Time Fields to Panda's format
-df['Date'] = pd.to_datetime(df['Date'])
-df['Time'] = pd.to_datetime(df['Time'], unit='ns')
+#Convert Date and Time Fields to Panda's format
+df['Date']=pd.to_datetime(df['Date'])
+df['Time']=pd.to_datetime(df['Time'], unit='ns')
-# Add round hours to dataframe
+#Add round hours to dataframe
df['Hour of Day'] = [r.hour for r in df.Time]
-# Create separate dataframes for separate locations
-df_plt = df # Default to use the whole Dbase
+#Create separate dataframes for separate locations
+df_plt=df #Default to use the whole Dbase
-# Get todays readings
+#Get todays readings
now = datetime.now()
-df_plt_today = df_plt[df_plt['Date'] == now.strftime("%Y-%m-%d")]
+df_plt_today = df_plt[df_plt['Date']==now.strftime("%Y-%m-%d")]
-# Set number of species to report
-readings = 10
+#Set number of species to report
+readings=10
plt_top10_today = (df_plt_today['Com_Name'].value_counts()[:readings])
-df_plt_top10_today = df_plt_today[df_plt_today.Com_Name.isin(
- plt_top10_today.index)]
+df_plt_top10_today = df_plt_today[df_plt_today.Com_Name.isin(plt_top10_today.index)]
-# Set Palette for graphics
+#Set Palette for graphics
pal = "Greens"
-# Set up plot axes and titles
-f, axs = plt.subplots(
- 1, 2, figsize=(
- 10, 4), gridspec_kw=dict(
- width_ratios=[
- 3, 6]), facecolor='#77C487')
-plt.subplots_adjust(
- left=None,
- bottom=None,
- right=None,
- top=None,
- wspace=0,
- hspace=0)
+#Set up plot axes and titles
+f, axs = plt.subplots(1, 2, figsize = (10, 4), gridspec_kw=dict(width_ratios=[3, 6]), facecolor='#77C487')
+plt.subplots_adjust(left=None, bottom=None, right=None, top=None, wspace=0, hspace=0)
-# generate y-axis order for all figures based on frequency
-freq_order = pd.value_counts(
- df_plt_top10_today['Com_Name']).iloc[:readings].index
+#generate y-axis order for all figures based on frequency
+freq_order = pd.value_counts(df_plt_top10_today['Com_Name']).iloc[:readings].index
-# make color for max confidence --> this groups by name and calculates max conf
+#make color for max confidence --> this groups by name and calculates max conf
confmax = df_plt_top10_today.groupby('Com_Name')['Confidence'].max()
-# reorder confmax to detection frequency order
+#reorder confmax to detection frequency order
confmax = confmax.reindex(freq_order)
# norm values for color palette
norm = plt.Normalize(confmax.values.min(), confmax.values.max())
colors = plt.cm.Greens(norm(confmax))
-# Generate frequency plot
-plot = sns.countplot(
- y='Com_Name',
- data=df_plt_top10_today,
- palette=colors,
- order=freq_order,
- ax=axs[0])
+#Generate frequency plot
+plot=sns.countplot(y='Com_Name', data = df_plt_top10_today, palette = colors, order=freq_order, ax=axs[0])
-# Try plot grid lines between bars - problem at the moment plots grid
-# lines on bars - want between bars
-z = plot.get_ymajorticklabels()
-plot.set_yticklabels(['\n'.join(textwrap.wrap(ticklabel.get_text(), 15))
- for ticklabel in plot.get_yticklabels()], fontsize=10)
+
+
+#Try plot grid lines between bars - problem at the moment plots grid lines on bars - want between bars
+z=plot.get_ymajorticklabels()
+plot.set_yticklabels(['\n'.join(textwrap.wrap(ticklabel.get_text(),15)) for ticklabel in plot.get_yticklabels()], fontsize = 10)
plot.set(ylabel=None)
plot.set(xlabel="Detections")
-# Generate crosstab matrix for heatmap plot
+#Generate crosstab matrix for heatmap plot
-heat = pd.crosstab(
- df_plt_top10_today['Com_Name'],
- df_plt_top10_today['Hour of Day'])
-# Order heatmap Birds by frequency of occurrance
-heat.index = pd.CategoricalIndex(heat.index, categories=freq_order)
+heat = pd.crosstab(df_plt_top10_today['Com_Name'],df_plt_top10_today['Hour of Day'])
+#Order heatmap Birds by frequency of occurrance
+heat.index = pd.CategoricalIndex(heat.index, categories = freq_order)
heat.sort_index(level=0, inplace=True)
-hours_in_day = pd.Series(data=range(0, 24))
-heat_frame = pd.DataFrame(data=0, index=heat.index, columns=hours_in_day)
-heat = (heat + heat_frame).fillna(0)
+hours_in_day = pd.Series(data = range(0,24))
+heat_frame = pd.DataFrame(data=0, index=heat.index, columns = hours_in_day)
+heat=(heat+heat_frame).fillna(0)
-# Generatie heatmap plot
-plot = sns.heatmap(
- heat,
- norm=LogNorm(),
- annot=True,
- annot_kws={
- "fontsize": 7},
- fmt="g",
- cmap=pal,
- square=False,
- cbar=False,
- linewidths=0.5,
- linecolor="Grey",
- ax=axs[1],
- yticklabels=False)
-plot.set_xticklabels(plot.get_xticklabels(), rotation=0, size=7)
+#Generatie heatmap plot
+plot = sns.heatmap(heat, norm=LogNorm(), annot=True, annot_kws={"fontsize":7}, fmt="g", cmap = pal , square = False, cbar=False, linewidths = 0.5, linecolor = "Grey", ax=axs[1], yticklabels = False)
+plot.set_xticklabels(plot.get_xticklabels(), rotation = 0, size = 7)
# Set heatmap border
for _, spine in plot.spines.items():
@@ -125,14 +93,13 @@ for _, spine in plot.spines.items():
plot.set(ylabel=None)
plot.set(xlabel="Hour of Day")
-# Set combined plot layout and titles
+#Set combined plot layout and titles
f.subplots_adjust(top=0.9)
-plt.suptitle("Top 10 Last Updated: " + str(now.strftime("%Y-%m-%d %H:%M")))
+plt.suptitle("Top 10 Last Updated: "+ str(now.strftime("%Y-%m-%d %H:%M")))
-# Save combined plot
+#Save combined plot
userDir = os.path.expanduser('~')
-savename = userDir + '/BirdSongs/Extracted/Charts/Combo-' + \
- str(now.strftime("%Y-%m-%d")) + '.png'
+savename=userDir + '/BirdSongs/Extracted/Charts/Combo-'+str(now.strftime("%Y-%m-%d"))+'.png'
plt.savefig(savename)
plt.show()
plt.close()
@@ -140,32 +107,20 @@ plt.close()
# Get Bottom detection frequency
plt_Bot10_today = (df_plt_today['Com_Name'].value_counts()[-readings:])
-df_plt_Bot10_today = df_plt_today[df_plt_today.Com_Name.isin(
- plt_Bot10_today.index)]
+df_plt_Bot10_today = df_plt_today[df_plt_today.Com_Name.isin(plt_Bot10_today.index)]
-# Set Palette for graphics
+#Set Palette for graphics
pal = "Reds"
-# Set up plot axes and titles
+#Set up plot axes and titles
-f, axs = plt.subplots(
- 1, 2, figsize=(
- 10, 4), gridspec_kw=dict(
- width_ratios=[
- 3, 6]), facecolor='#77C487')
-plt.subplots_adjust(
- left=None,
- bottom=None,
- right=None,
- top=None,
- wspace=0,
- hspace=0)
+f, axs = plt.subplots(1, 2, figsize = (10, 4), gridspec_kw=dict(width_ratios=[3, 6]), facecolor='#77C487')
+plt.subplots_adjust(left=None, bottom=None, right=None, top=None, wspace=0, hspace=0)
-# generate y-axis order for all figures based on frequency
-freq_order = pd.value_counts(
- df_plt_Bot10_today['Com_Name']).iloc[-readings:].index
+#generate y-axis order for all figures based on frequency
+freq_order = pd.value_counts(df_plt_Bot10_today['Com_Name']).iloc[-readings:].index
-# make color for max confidence --> this groups by name and calculates max conf
+#make color for max confidence --> this groups by name and calculates max conf
confmax = df_plt_Bot10_today.groupby('Com_Name')['Confidence'].max()
confmax = confmax.reindex(freq_order)
# probably wrong order . . . how to sort by no. of detections ?
@@ -173,53 +128,33 @@ confmax = confmax.reindex(freq_order)
norm = plt.Normalize(confmax.values.min(), confmax.values.max())
colors = plt.cm.Reds(norm(confmax))
-# Generate frequency plot
-plot = sns.countplot(
- y='Com_Name',
- data=df_plt_Bot10_today,
- palette=colors,
- order=freq_order,
- ax=axs[0])
+#Generate frequency plot
+plot=sns.countplot(y='Com_Name', data = df_plt_Bot10_today, palette = colors, order=freq_order, ax=axs[0])
-# Try plot grid lines between bars - problem at the moment plots grid
-# lines on bars - want between bars
-z = plot.get_ymajorticklabels()
-plot.set_yticklabels(['\n'.join(textwrap.wrap(ticklabel.get_text(), 15))
- for ticklabel in plot.get_yticklabels()], fontsize=10)
+
+
+#Try plot grid lines between bars - problem at the moment plots grid lines on bars - want between bars
+z=plot.get_ymajorticklabels()
+plot.set_yticklabels(['\n'.join(textwrap.wrap(ticklabel.get_text(),15)) for ticklabel in plot.get_yticklabels()], fontsize = 10)
plot.set(ylabel=None)
plot.set(xlabel="Detections")
-# Generate crosstab matrix for heatmap plot
+#Generate crosstab matrix for heatmap plot
-heat = pd.crosstab(
- df_plt_Bot10_today['Com_Name'],
- df_plt_Bot10_today['Hour of Day'])
-# Order heatmap Birds by frequency of occurrance
-heat.index = pd.CategoricalIndex(heat.index, categories=freq_order)
+heat = pd.crosstab(df_plt_Bot10_today['Com_Name'],df_plt_Bot10_today['Hour of Day'])
+#Order heatmap Birds by frequency of occurrance
+heat.index = pd.CategoricalIndex(heat.index, categories = freq_order)
heat.sort_index(level=0, inplace=True)
-hours_in_day = pd.Series(data=range(0, 24))
-heat_frame = pd.DataFrame(data=0, index=heat.index, columns=hours_in_day)
-heat = (heat + heat_frame).fillna(0)
+hours_in_day = pd.Series(data = range(0,24))
+heat_frame = pd.DataFrame(data=0, index=heat.index, columns = hours_in_day)
+heat=(heat+heat_frame).fillna(0)
-# Generatie heatmap plot
-plot = sns.heatmap(
- heat,
- norm=LogNorm(),
- annot=True,
- fmt="g",
- annot_kws={
- "fontsize": 7},
- cmap=pal,
- square=False,
- cbar=False,
- linewidths=0.5,
- linecolor="Grey",
- ax=axs[1],
- yticklabels=False)
-plot.set_xticklabels(plot.get_xticklabels(), rotation=0, size=7)
+#Generatie heatmap plot
+plot = sns.heatmap(heat, norm=LogNorm(), annot=True, fmt="g", annot_kws={"fontsize":7}, cmap = pal , square = False, cbar=False, linewidths = 0.5, linecolor = "Grey", ax=axs[1], yticklabels = False)
+plot.set_xticklabels(plot.get_xticklabels(), rotation = 0, size = 7)
# Set heatmap border
for _, spine in plot.spines.items():
@@ -227,13 +162,12 @@ for _, spine in plot.spines.items():
plot.set(ylabel=None)
plot.set(xlabel="Hour of Day")
-# Set combined plot layout and titles
+#Set combined plot layout and titles
f.subplots_adjust(top=0.9)
-plt.suptitle("Bottom 10 Last Updated: " + str(now.strftime("%Y-%m-%d %H:%M")))
+plt.suptitle("Bottom 10 Last Updated: "+ str(now.strftime("%Y-%m-%d %H:%M")))
-# Save combined plot
-savename = userDir + '/BirdSongs/Extracted/Charts/Combo2-' + \
- str(now.strftime("%Y-%m-%d")) + '.png'
+#Save combined plot
+savename=userDir + '/BirdSongs/Extracted/Charts/Combo2-'+str(now.strftime("%Y-%m-%d"))+'.png'
plt.savefig(savename)
plt.show()
plt.close()
diff --git a/scripts/plotly_streamlit.py b/scripts/plotly_streamlit.py
index 1adea4f..b42906d 100755
--- a/scripts/plotly_streamlit.py
+++ b/scripts/plotly_streamlit.py
@@ -4,7 +4,8 @@ import pandas as pd
import numpy as np
import plotly.graph_objects as go
from plotly.subplots import make_subplots
-from datetime import timedelta
+from datetime import timedelta, datetime
+from pathlib import Path
import sqlite3
from sqlite3 import Connection
@@ -33,22 +34,22 @@ st.markdown("""
@st.cache(hash_funcs={Connection: id})
-def get_connection(path: str):
- return sqlite3.connect(path, check_same_thread=False)
+def get_connection(path:str):
+ return sqlite3.connect(path,check_same_thread=False)
def get_data(conn: Connection):
- df1 = pd.read_sql("SELECT * FROM detections", con=conn)
+ df1=pd.read_sql("SELECT * FROM detections", con=conn)
return df1
-
conn = get_connection(URI_SQLITE_DB)
# Read in the cereal data
# df = load_data()
-df = get_data(conn)
-df2 = df.copy()
-df2['DateTime'] = pd.to_datetime(df2['Date'] + " " + df2['Time'])
-df2 = df2.set_index('DateTime')
+df=get_data(conn)
+df2=df.copy()
+df2['DateTime']=pd.to_datetime(df2['Date'] + " " + df2['Time'])
+df2=df2.set_index('DateTime')
+
# Filter on date range
@@ -58,124 +59,115 @@ df2 = df2.set_index('DateTime')
# Date as slider
Start_Date = pd.to_datetime(df2.index.min()).date()
-End_Date = pd.to_datetime(df2.index.max()).date()
+End_Date = pd.to_datetime(df2.index.max()).date()
Date_Slider = st.slider('Date Range',
- min_value=Start_Date - timedelta(days=1),
- max_value=End_Date,
- value=(Start_Date,
- End_Date)
- )
+ min_value = Start_Date-timedelta(days=1),
+ max_value = End_Date,
+ value=(Start_Date,
+ End_Date)
+ )
-filt = (df2.index >= pd.Timestamp(Date_Slider[0])) & (df2.index <= pd.Timestamp(Date_Slider[1] + timedelta(days=1)))
+
+filt = (df2.index >= pd.Timestamp(Date_Slider[0])) & (df2.index <= pd.Timestamp(Date_Slider[1]+timedelta(days=1)))
df2 = df2[filt]
-# Create species count for selected date range
+#Create species count for selected date range
-Specie_Count = df2['Com_Name'].value_counts()
+Specie_Count=df2['Com_Name'].value_counts()
-# Create species treemap
+#Create species treemap
# Create Hourly Crosstab
-hourly = pd.crosstab(df2['Com_Name'], df2.index.hour, dropna=False)
+hourly=pd.crosstab(df2['Com_Name'],df2.index.hour, dropna=False)
# Filter on species
species = list(hourly.index)
-cols1, cols2 = st.columns((1, 1))
+cols1,cols2= st.columns((1,1))
top_N = cols1.slider(
'Select Number of Birds to Show',
- min_value=1,
- value=min(10, len(Specie_Count))
-)
+ min_value = 1,
+ value=min(10,len(Specie_Count))
+ )
top_N_species = (df2['Com_Name'].value_counts()[:top_N])
-specie = cols2.selectbox(
- 'Which bird would you like to explore for the dates ' +
- str(Date_Slider[0]) + ' to ' + str(Date_Slider[1]) + '?',
- species,
- index=species.index(list(top_N_species.index)[0]))
+specie = cols2.selectbox('Which bird would you like to explore for the dates '+str(Date_Slider[0])+' to '+str(Date_Slider[1])+'?', species,
+ index=species.index(list(top_N_species.index)[0]))
-font_size = 15
+font_size=15
-# specie filter
-filt = df2['Com_Name'] == specie
+#specie filter
+filt=df2['Com_Name']==specie
-df_counts = df2[filt].resample('D').count()
+df_counts=df2[filt].resample('D').count()
fig = make_subplots(
- rows=3, cols=2,
- specs=[[{"type": "xy", "rowspan": 3}, {"type": "polar", "rowspan": 2}], [
- {"rowspan": 1}, {"rowspan": 1}], [None, {"type": "xy", "rowspan": 1}]],
- subplot_titles=(
- 'Top ' + str(top_N) +
- ' Species in Date Range ' + str(Date_Slider[0]) +
- ' to ' + str(Date_Slider[1]) +
- '',
- 'Total Detect:' + str('{:,}'.format(sum(df_counts.Time))) +
- ' Confidence Max:' + str('{:.2f}%'.format(max(df2[df2['Com_Name'] == specie]['Confidence']) * 100)) +
- ' ' + ' Median:' +
- str('{:.2f}%'.format(np.median(df2[df2['Com_Name'] == specie]['Confidence']) * 100))
- )
-)
-fig.layout.annotations[1].update(x=0.7, y=0.25, font_size=15)
+ rows=3, cols =2,
+ specs= [[{"type":"xy","rowspan":3}, {"type":"polar","rowspan":2}], [{"rowspan":1}, {"rowspan":1} ], [None, {"type":"xy","rowspan":1}]],
+ subplot_titles=('Top '+ str(top_N) + ' Species in Date Range '+str(Date_Slider[0])+' to '+str(Date_Slider[1])+'',
+ 'Total Detect:'+str('{:,}'.format(sum(df_counts.Time)))+
+ ' Confidence Max:'+str('{:.2f}%'.format(max(df2[df2['Com_Name']==specie]['Confidence'])*100))+
+ ' '+' Median:'+str('{:.2f}%'.format(np.median(df2[df2['Com_Name']==specie]['Confidence'])*100))
+ )
+ )
+fig.layout.annotations[1].update(x=0.7,y=0.25, font_size=15)
-# Plot seen species for selected date range and number of species
-fig.add_trace(go.Bar(y=top_N_species.index, x=top_N_species, orientation='h'), row=1, col=1)
+#Plot seen species for selected date range and number of species
+fig.add_trace(go.Bar(y=top_N_species.index, x=top_N_species, orientation='h'), row=1,col=1)
fig.update_layout(
margin=dict(l=0, r=0, t=50, b=0),
- yaxis={'categoryorder': 'total ascending'})
+ yaxis={'categoryorder':'total ascending'})
# Set 360 degrees, 24 hours for polar plot
theta = np.linspace(0.0, 360, 24, endpoint=False)
-d = pd.DataFrame(np.zeros((23, 1))).squeeze()
+d=pd.DataFrame(np.zeros((23,1))).squeeze()
detections = hourly.loc[specie]
-detections = (d + detections).fillna(0)
-fig.add_trace(go.Barpolar(r=detections, theta=theta), row=1, col=2)
+detections=(d+detections).fillna(0)
+fig.add_trace(go.Barpolar(r = detections, theta=theta), row=1, col=2)
fig.update_layout(
autosize=False,
- width=1000,
- height=500,
+ width = 1000,
+ height = 500,
showlegend=False,
- polar=dict(
- radialaxis=dict(
- tickfont_size=font_size,
- showticklabels=True,
- hoverformat="#%{theta}:
Popularity: %{percent} %{r}"
- ),
- angularaxis=dict(
- tickfont_size=font_size,
- rotation=-90,
- direction='clockwise',
+ polar = dict(
+ radialaxis = dict(
+ tickfont_size = font_size,
+ showticklabels = True,
+ hoverformat = "#%{theta}:
Popularity: %{percent} %{r}"
+ ),
+ angularaxis = dict(
+ tickfont_size= font_size,
+ rotation = -90,
+ direction = 'clockwise',
tickmode='array',
- tickvals=[0, 15, 35, 45, 60, 75, 90, 105, 120, 135, 150, 165,
- 180, 195, 210, 225, 240, 255, 270, 285, 300, 315, 330, 345],
- ticktext=['12am', '1am', '2am', '3am', '4am', '5am', '6am', '7am', '8am', '9am', '10am', '11am',
- '12pm', '1pm', '2pm', '3pm', '4pm', '5pm', '6pm', '7pm', '8pm', '9pm', '10pm', '11pm'],
- hoverformat="#%{theta}:
Popularity: %{percent} %{r}"
+ tickvals=[0,15,35,45,60,75,90,105,120,135,150,165,180,195,210,225,240,255,270,285,300,315,330,345],
+ ticktext=['12am','1am','2am','3am','4am','5am', '6am','7am','8am','9am','10am','11am','12pm','1pm','2pm','3pm','4pm','5pm', '6pm','7pm','8pm','9pm','10pm','11pm'],
+ hoverformat = "#%{theta}:
Popularity: %{percent} %{r}"
),
- ),
-)
+ ),
+ )
-daily = pd.crosstab(df2['Com_Name'], df2.index.date, dropna=False)
+
+daily=pd.crosstab(df2['Com_Name'],df2.index.date, dropna=False)
fig.add_trace(go.Bar(x=daily.columns, y=daily.loc[specie]), row=3, col=2)
# container=st.container()
# config={'displayModelBar': False}
-st.plotly_chart(fig, use_container_width=True) # , config=config)
+st.plotly_chart(fig, use_container_width=True) #, config=config)
# cols3,cols4=st.columns((1,1))
-#
+#
# extract_date=Date_Slider
-#
-# audio_file = open('/home/*/BirdSongs/Extracted/By_Date/2022-03-22/Yellow-streaked_Greenbul/Yellow-streaked_Greenbul-77-2022-03-22-birdnet-15:04:28.mp3', 'rb') # noqa: E501
+#
+# audio_file = open('/home/*/BirdSongs/Extracted/By_Date/2022-03-22/Yellow-streaked_Greenbul/Yellow-streaked_Greenbul-77-2022-03-22-birdnet-15:04:28.mp3', 'rb')
# audio_bytes = audio_file.read()
# cols4.audio(audio_bytes, format='audio/mp3')
diff --git a/scripts/server.py b/scripts/server.py
index 42c7868..6c96e12 100755
--- a/scripts/server.py
+++ b/scripts/server.py
@@ -1,27 +1,32 @@
-import os
-import socket
+import socket
import threading
-import operator
-import librosa
-import numpy as np
-import math
-import time
-import json
-import requests
-import sqlite3
-import datetime
-from tzlocal import get_localzone
-from pathlib import Path
-import apprise
-
+import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
os.environ['CUDA_VISIBLE_DEVICES'] = ''
try:
import tflite_runtime.interpreter as tflite
-except BaseException:
+except:
from tensorflow import lite as tflite
+import argparse
+import operator
+import librosa
+import numpy as np
+import math
+import time
+from decimal import Decimal
+import json
+import requests
+import sqlite3
+import datetime
+from time import sleep
+import pytz
+from tzlocal import get_localzone
+from pathlib import Path
+import apprise
+
+
HEADER = 64
PORT = 5050
SERVER = socket.gethostbyname(socket.gethostname())
@@ -32,9 +37,10 @@ DISCONNECT_MESSAGE = "!DISCONNECT"
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
server.bind(ADDR)
-except BaseException:
+except:
print("Waiting on socket")
time.sleep(5)
+
# Open most recent Configuration and grab DB_PWD as a python variable
@@ -42,8 +48,7 @@ userDir = os.path.expanduser('~')
with open(userDir + '/BirdNET-Pi/scripts/thisrun.txt', 'r') as f:
this_run = f.readlines()
audiofmt = "." + str(str(str([i for i in this_run if i.startswith('AUDIOFMT')]).split('=')[1]).split('\\')[0])
- priv_thresh = float(
- "." + str(str(str([i for i in this_run if i.startswith('PRIVACY_THRESHOLD')]).split('=')[1]).split('\\')[0])) / 10
+ priv_thresh = float("." + str(str(str([i for i in this_run if i.startswith('PRIVACY_THRESHOLD')]).split('=')[1]).split('\\')[0]))/10
def loadModel():
@@ -57,7 +62,7 @@ def loadModel():
# Load TFLite model and allocate tensors.
modelpath = userDir + '/BirdNET-Pi/model/BirdNET_6K_GLOBAL_MODEL.tflite'
- myinterpreter = tflite.Interpreter(model_path=modelpath, num_threads=2)
+ myinterpreter = tflite.Interpreter(model_path=modelpath,num_threads=2)
myinterpreter.allocate_tensors()
# Get input and output tensors.
@@ -80,7 +85,6 @@ def loadModel():
return myinterpreter
-
def loadCustomSpeciesList(path):
slist = []
@@ -91,7 +95,6 @@ def loadCustomSpeciesList(path):
return slist
-
def splitSignal(sig, rate, overlap, seconds=3.0, minlen=1.5):
# Split signal with overlap
@@ -102,18 +105,17 @@ def splitSignal(sig, rate, overlap, seconds=3.0, minlen=1.5):
# End of signal?
if len(split) < int(minlen * rate):
break
-
+
# Signal chunk too short? Fill with zeros.
if len(split) < int(rate * seconds):
temp = np.zeros((int(rate * seconds)))
temp[:len(split)] = split
split = temp
-
+
sig_splits.append(split)
return sig_splits
-
def readAudioData(path, overlap, sample_rate=48000):
print('READING AUDIO DATA...', end=' ', flush=True)
@@ -128,12 +130,11 @@ def readAudioData(path, overlap, sample_rate=48000):
return chunks
-
def convertMetadata(m):
# Convert week to cosine
if m[2] >= 1 and m[2] <= 48:
- m[2] = math.cos(math.radians(m[2] * 7.5)) + 1
+ m[2] = math.cos(math.radians(m[2] * 7.5)) + 1
else:
m[2] = -1
@@ -146,11 +147,9 @@ def convertMetadata(m):
return np.concatenate([m, mask])
-
def custom_sigmoid(x, sensitivity=1.0):
return 1 / (1.0 + np.exp(-sensitivity * x))
-
def predict(sample, sensitivity):
global INTERPRETER
# Make a prediction
@@ -167,22 +166,21 @@ def predict(sample, sensitivity):
# Sort by score
p_sorted = sorted(p_labels.items(), key=operator.itemgetter(1), reverse=True)
-
+
# #print("DATABASE SIZE:", len(p_sorted))
# #print("HUMAN-CUTOFF AT:", int(len(p_sorted)*priv_thresh)/10)
-#
+#
# # Remove species that are on blacklist
- human_cutoff = max(10, int(len(p_sorted) * priv_thresh))
+ human_cutoff = max(10,int(len(p_sorted)*priv_thresh))
for i in range(min(10, len(p_sorted))):
- if p_sorted[i][0] == 'Human_Human':
+ if p_sorted[i][0]=='Human_Human':
with open(userDir + '/BirdNET-Pi/HUMAN.txt', 'a') as rfile:
- rfile.write(str(datetime.datetime.now()) + str(p_sorted[i]) + ' ' + str(human_cutoff) + '\n')
+ rfile.write(str(datetime.datetime.now())+str(p_sorted[i])+ ' ' + str(human_cutoff)+ '\n')
return p_sorted[:human_cutoff]
-
def analyzeAudioData(chunks, lat, lon, week, sensitivity, overlap,):
global INTERPRETER
@@ -204,57 +202,47 @@ def analyzeAudioData(chunks, lat, lon, week, sensitivity, overlap,):
# Make prediction
p = predict([sig, mdata], sensitivity)
# print("PPPPP",p)
- HUMAN_DETECTED = False
-
- # Catch if Human is recognized
+ HUMAN_DETECTED=False
+
+ #Catch if Human is recognized
for x in range(len(p)):
if "Human" in p[x][0]:
- HUMAN_DETECTED = True
-
+ HUMAN_DETECTED=True
+
# Save result and timestamp
pred_end = pred_start + 3.0
-
- # If human detected set all detections to human to make sure voices are not saved
- if HUMAN_DETECTED is True:
- p = [('Human_Human', 0.0)] * 10
+
+ #If human detected set all detections to human to make sure voices are not saved
+ if HUMAN_DETECTED == True:
+ p=[('Human_Human',0.0)]*10
detections[str(pred_start) + ';' + str(pred_end)] = p
-
+
pred_start = pred_end - overlap
print('DONE! Time', int((time.time() - start) * 10) / 10.0, 'SECONDS')
# print('DETECTIONS:::::',detections)
return detections
-
-def sendAppriseNotifications(species, confidence):
+def sendAppriseNotifications(species,confidence):
if os.path.exists(userDir + '/BirdNET-Pi/apprise.txt') and os.path.getsize(userDir + '/BirdNET-Pi/apprise.txt') > 0:
with open(userDir + '/BirdNET-Pi/scripts/thisrun.txt', 'r') as f:
this_run = f.readlines()
- title = str(str(str([i for i in this_run if i.startswith('APPRISE_NOTIFICATION_TITLE')]
- ).split('=')[1]).split('\\')[0]).replace('"', '')
- body = str(str(str([i for i in this_run if i.startswith('APPRISE_NOTIFICATION_BODY')]
- ).split('=')[1]).split('\\')[0]).replace('"', '')
+ title = str(str(str([i for i in this_run if i.startswith('APPRISE_NOTIFICATION_TITLE')]).split('=')[1]).split('\\')[0]).replace('"', '')
+ body = str(str(str([i for i in this_run if i.startswith('APPRISE_NOTIFICATION_BODY')]).split('=')[1]).split('\\')[0]).replace('"', '')
- if str(str(str([i for i in this_run if i.startswith('APPRISE_NOTIFY_EACH_DETECTION')]).split('=')[1]).split('\\')[0]) == "1": # noqa E501
+ if str(str(str([i for i in this_run if i.startswith('APPRISE_NOTIFY_EACH_DETECTION')]).split('=')[1]).split('\\')[0]) == "1":
apobj = apprise.Apprise()
config = apprise.AppriseConfig()
config.add(userDir + '/BirdNET-Pi/apprise.txt')
apobj.add(config)
-
+
apobj.notify(
- body=body.replace(
- "$sciname",
- species.split("_")[0]).replace(
- "$comname",
- species.split("_")[1]).replace(
- "$confidence",
- confidence),
+ body=body.replace("$sciname",species.split("_")[0]).replace("$comname",species.split("_")[1]).replace("$confidence",confidence),
title=title,
)
-
def writeResultsToFile(detections, min_conf, path):
print('WRITING RESULTS TO', path, '...', end=' ')
@@ -263,15 +251,13 @@ def writeResultsToFile(detections, min_conf, path):
rfile.write('Start (s);End (s);Scientific name;Common name;Confidence\n')
for d in detections:
for entry in detections[d]:
- if entry[1] >= min_conf and ((entry[0] in INCLUDE_LIST or len(INCLUDE_LIST) == 0)
- and (entry[0] not in EXCLUDE_LIST or len(EXCLUDE_LIST) == 0)):
- sendAppriseNotifications(str(entry[0]), str(entry[1]))
+ if entry[1] >= min_conf and ((entry[0] in INCLUDE_LIST or len(INCLUDE_LIST) == 0) and (entry[0] not in EXCLUDE_LIST or len(EXCLUDE_LIST) == 0) ):
+ sendAppriseNotifications(str(entry[0]),str(entry[1]));
rfile.write(d + ';' + entry[0].replace('_', ';') + ';' + str(entry[1]) + '\n')
rcnt += 1
print('DONE! WROTE', rcnt, 'RESULTS.')
return
-
def handle_client(conn, addr):
global INCLUDE_LIST
global EXCLUDE_LIST
@@ -286,10 +272,10 @@ def handle_client(conn, addr):
if msg == DISCONNECT_MESSAGE:
connected = False
else:
- # print(f"[{addr}] {msg}")
-
+ #print(f"[{addr}] {msg}")
+
args = type('', (), {})()
-
+
args.i = ''
args.o = ''
args.birdweather_id = '99999'
@@ -300,7 +286,8 @@ def handle_client(conn, addr):
args.sensitivity = 1.25
args.min_conf = 0.70
args.lat = -1
- args.lon = -1
+ args.lon = -1
+
for line in msg.split('||'):
inputvars = line.split('=')
@@ -327,12 +314,14 @@ def handle_client(conn, addr):
elif inputvars[0] == 'lon':
args.lon = float(inputvars[1])
+
+
# Load custom species lists - INCLUDED and EXCLUDED
if not args.include_list == 'null':
INCLUDE_LIST = loadCustomSpeciesList(args.include_list)
else:
INCLUDE_LIST = []
-
+
if not args.exclude_list == 'null':
EXCLUDE_LIST = loadCustomSpeciesList(args.exclude_list)
else:
@@ -344,7 +333,7 @@ def handle_client(conn, addr):
audioData = readAudioData(args.i, args.overlap)
# Get Date/Time from filename in case Pi gets behind
- # now = datetime.now()
+ #now = datetime.now()
full_file_name = args.i
print('FULL FILENAME: -' + full_file_name + '-')
file_name = Path(full_file_name).stem
@@ -352,14 +341,14 @@ def handle_client(conn, addr):
file_time = file_name.split('-birdnet-')[1]
date_time_str = file_date + ' ' + file_time
date_time_obj = datetime.datetime.strptime(date_time_str, '%Y-%m-%d %H:%M:%S')
- # print('Date:', date_time_obj.date())
- # print('Time:', date_time_obj.time())
+ #print('Date:', date_time_obj.date())
+ #print('Time:', date_time_obj.time())
print('Date-time:', date_time_obj)
now = date_time_obj
current_date = now.strftime("%Y-%m-%d")
current_time = now.strftime("%H:%M:%S")
current_iso8601 = now.astimezone(get_localzone()).isoformat()
-
+
week_number = int(now.strftime("%V"))
week = max(1, min(week_number, 48))
@@ -371,33 +360,32 @@ def handle_client(conn, addr):
# Write detections to output file
min_conf = max(0.01, min(args.min_conf, 0.99))
writeResultsToFile(detections, min_conf, args.o)
-
- ###############################################################################
- ###############################################################################
-
+
+ ###############################################################################
+ ###############################################################################
+
soundscape_uploaded = False
# Write detections to Database
myReturn = ''
for i in detections:
- myReturn += str(i) + '-' + str(detections[i][0]) + '\n'
-
+ myReturn += str(i) + '-' + str(detections[i][0]) + '\n'
+
+
with open(userDir + '/BirdNET-Pi/BirdDB.txt', 'a') as rfile:
for d in detections:
for entry in detections[d]:
- if entry[1] >= min_conf and ((entry[0] in INCLUDE_LIST or len(INCLUDE_LIST) == 0) and (
- entry[0] not in EXCLUDE_LIST or len(EXCLUDE_LIST) == 0)):
- rfile.write(str(current_date) + ';' + str(current_time) + ';' + entry[0].replace('_', ';') + ';'
- + str(entry[1]) + ";" + str(args.lat) + ';' + str(args.lon) +
- ';' + str(min_conf) + ';' + str(week) + ';'
- + str(args.sensitivity) + ';' + str(args.overlap) + '\n')
-
+ if entry[1] >= min_conf and ((entry[0] in INCLUDE_LIST or len(INCLUDE_LIST) == 0) and (entry[0] not in EXCLUDE_LIST or len(EXCLUDE_LIST) == 0) ):
+ rfile.write(str(current_date) + ';' + str(current_time) + ';' + entry[0].replace('_', ';') + ';' \
+ + str(entry[1]) +";" + str(args.lat) + ';' + str(args.lon) + ';' + str(min_conf) + ';' + str(week) + ';' \
+ + str(args.sensitivity) +';' + str(args.overlap) + '\n')
+
Date = str(current_date)
Time = str(current_time)
species = entry[0]
- Sci_Name, Com_Name = species.split('_')
+ Sci_Name,Com_Name = species.split('_')
score = entry[1]
- Confidence = str(round(score * 100))
+ Confidence = str(round(score*100))
Lat = str(args.lat)
Lon = str(args.lon)
Cutoff = str(args.min_conf)
@@ -406,91 +394,47 @@ def handle_client(conn, addr):
Overlap = str(args.overlap)
Com_Name = Com_Name.replace("'", "")
File_Name = Com_Name.replace(" ", "_") + '-' + Confidence + '-' + \
- Date.replace("/", "-") + '-birdnet-' + Time + audiofmt
+ Date.replace("/", "-") + '-birdnet-' + Time + audiofmt
- # Connect to SQLite Database
+ #Connect to SQLite Database
for attempt_number in range(3):
- try:
+ try:
con = sqlite3.connect(userDir + '/BirdNET-Pi/scripts/birds.db')
cur = con.cursor()
- cur.execute(
- "INSERT INTO detections VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
- (Date,
- Time,
- Sci_Name,
- Com_Name,
- str(score),
- Lat,
- Lon,
- Cutoff,
- Week,
- Sens,
- Overlap,
- File_Name))
+ cur.execute("INSERT INTO detections VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (Date, Time, Sci_Name, Com_Name, str(score), Lat, Lon, Cutoff, Week, Sens, Overlap, File_Name))
con.commit()
con.close()
break
- except BaseException:
+ except:
print("Database busy")
time.sleep(2)
- print(str(current_date) +
- ';' +
- str(current_time) +
- ';' +
- entry[0].replace('_', ';') +
- ';' +
- str(entry[1]) +
- ';' +
- str(args.lat) +
- ';' +
- str(args.lon) +
- ';' +
- str(min_conf) +
- ';' +
- str(week) +
- ';' +
- str(args.sensitivity) +
- ';' +
- str(args.overlap) +
- Com_Name.replace(" ", "_") +
- '-' +
- str(score) +
- '-' +
- str(current_date) +
- '-birdnet-' +
- str(current_time) +
- audiofmt +
- '\n')
+ print(str(current_date) + ';' + str(current_time) + ';' + entry[0].replace('_', ';') + ';' + str(entry[1]) + ';' + str(args.lat) + ';' + str(args.lon) + ';' + str(min_conf) + ';' + str(week) + ';' + str(args.sensitivity) +';' + str(args.overlap) + Com_Name.replace(" ", "_") + '-' + str(score) + '-' + str(current_date) + '-birdnet-' + str(current_time) + audiofmt + '\n')
if birdweather_id != "99999":
try:
if soundscape_uploaded is False:
# POST soundscape to server
- soundscape_url = "https://app.birdweather.com/api/v1/stations/" + \
- birdweather_id + "/soundscapes" + "?timestamp=" + current_iso8601
-
+ soundscape_url = "https://app.birdweather.com/api/v1/stations/" + birdweather_id + "/soundscapes" + "?timestamp=" + current_iso8601
+
with open(args.i, 'rb') as f:
wav_data = f.read()
- response = requests.post(
- url=soundscape_url, data=wav_data, headers={
- 'Content-Type': 'application/octet-stream'})
+ response = requests.post(url=soundscape_url, data=wav_data, headers={'Content-Type': 'application/octet-stream'})
print("Soundscape POST Response Status - ", response.status_code)
sdata = response.json()
soundscape_id = sdata['soundscape']['id']
soundscape_uploaded = True
-
+
# POST detection to server
- detection_url = "https://app.birdweather.com/api/v1/stations/" + \
- birdweather_id + "/detections"
+ detection_url = "https://app.birdweather.com/api/v1/stations/" + birdweather_id + "/detections"
start_time = d.split(';')[0]
end_time = d.split(';')[1]
post_begin = "{ "
now_p_start = now + datetime.timedelta(seconds=float(start_time))
current_iso8601 = now_p_start.astimezone(get_localzone()).isoformat()
- post_timestamp = "\"timestamp\": \"" + current_iso8601 + "\","
+ post_timestamp = "\"timestamp\": \"" + current_iso8601 + "\","
post_lat = "\"lat\": " + str(args.lat) + ","
post_lon = "\"lon\": " + str(args.lon) + ","
post_soundscape_id = "\"soundscapeId\": " + str(soundscape_id) + ","
@@ -501,23 +445,18 @@ def handle_client(conn, addr):
post_algorithm = "\"algorithm\": " + "\"alpha\"" + ","
post_confidence = "\"confidence\": " + str(entry[1])
post_end = " }"
-
- post_json = post_begin + \
- post_timestamp + post_lat + post_lon + \
- post_soundscape_id + post_soundscape_start_time + \
- post_soundscape_end_time + post_commonName + post_scientificName + \
- post_algorithm + post_confidence + post_end
+
+ post_json = post_begin + post_timestamp + post_lat + post_lon + post_soundscape_id + post_soundscape_start_time + post_soundscape_end_time + post_commonName + post_scientificName + post_algorithm + post_confidence + post_end
print(post_json)
response = requests.post(detection_url, json=json.loads(post_json))
print("Detection POST Response Status - ", response.status_code)
- except BaseException:
+ except:
print("Cannot POST right now")
conn.send(myReturn.encode(FORMAT))
- # time.sleep(3)
-
- conn.close()
+ #time.sleep(3)
+ conn.close()
def start():
# Load model