claudecode/AvianVisitors
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #308 from jmherbst/python-lint

Browse Source 
Python linting
This commit is contained in:
Patrick McGuire
2022-05-24 12:51:59 -04:00
committed by GitHub
parent 3a95504f04 166ca7fcf8
commit f9cc6b6aa5
6 changed files with 399 additions and 295 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.flake8
+2
View File
@@ -0,0 +1,2 @@
[flake8]
max-line-length = 160
.github/workflows/python-ci.yml
+1 -5
View File
@@ -13,9 +13,6 @@ jobs:
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v3 uses: actions/checkout@v3
with:
# This is a bit heavy handed, but we need `origin/main` to get a ref to diff against
fetch-depth: 0
- name: Setup Python - name: Setup Python
uses: actions/setup-python@v3 uses: actions/setup-python@v3
@@ -29,5 +26,4 @@ jobs:
- name: Run Flake8 Lint - name: Run Flake8 Lint
run: | run: |
DIFF="$(git --no-pager diff -u $(git merge-base HEAD origin/main) -- '**/*.py')" flake8
echo "$DIFF" | flake8 --diff
scripts/analyze.py
+53 -12
View File
@@ -11,6 +11,7 @@ ADDR = (SERVER, PORT)
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM) client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client.connect(ADDR) client.connect(ADDR)
def send(msg): def send(msg):
message = msg.encode(FORMAT) message = msg.encode(FORMAT)
msg_length = len(message) msg_length = len(message)
@@ -20,6 +21,7 @@ def send(msg):
client.send(message) client.send(message)
print(client.recv(2048).decode(FORMAT)) print(client.recv(2048).decode(FORMAT))
def main(): def main():
global INCLUDE_LIST global INCLUDE_LIST
@@ -28,16 +30,52 @@ def main():
# Parse passed arguments # Parse passed arguments
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument('--i', help='Path to input file.') parser.add_argument('--i', help='Path to input file.')
parser.add_argument('--o', default='result.csv', help='Path to output file. Defaults to result.csv.') parser.add_argument(
parser.add_argument('--lat', type=float, default=-1, help='Recording location latitude. Set -1 to ignore.') '--o',
parser.add_argument('--lon', type=float, default=-1, help='Recording location longitude. Set -1 to ignore.') default='result.csv',
parser.add_argument('--week', type=int, default=-1, help='Week of the year when the recording was made. Values in [1, 48] (4 weeks per month). Set -1 to ignore.') help='Path to output file. Defaults to result.csv.')
parser.add_argument('--overlap', type=float, default=0.0, help='Overlap in seconds between extracted spectrograms. Values in [0.0, 2.9]. Defaults tp 0.0.') parser.add_argument(
parser.add_argument('--sensitivity', type=float, default=1.0, help='Detection sensitivity; Higher values result in higher sensitivity. Values in [0.5, 1.5]. Defaults to 1.0.') '--lat',
parser.add_argument('--min_conf', type=float, default=0.1, help='Minimum confidence threshold. Values in [0.01, 0.99]. Defaults to 0.1.') type=float,
parser.add_argument('--include_list', default='null', help='Path to text file containing a list of included species. Not used if not provided.') default=-1,
parser.add_argument('--exclude_list', default='null', help='Path to text file containing a list of excluded species. Not used if not provided.') help='Recording location latitude. Set -1 to ignore.')
parser.add_argument('--birdweather_id', default='99999', help='Private Station ID for BirdWeather.') parser.add_argument(
'--lon',
type=float,
default=-1,
help='Recording location longitude. Set -1 to ignore.')
parser.add_argument(
'--week',
type=int,
default=-1,
help='Week of the year when the recording was made. Values in [1, 48] (4 weeks per month). Set -1 to ignore.')
parser.add_argument(
'--overlap',
type=float,
default=0.0,
help='Overlap in seconds between extracted spectrograms. Values in [0.0, 2.9]. Defaults tp 0.0.')
parser.add_argument(
'--sensitivity',
type=float,
default=1.0,
help='Detection sensitivity; Higher values result in higher sensitivity. Values in [0.5, 1.5]. Defaults to 1.0.')
parser.add_argument(
'--min_conf',
type=float,
default=0.1,
help='Minimum confidence threshold. Values in [0.01, 0.99]. Defaults to 0.1.')
parser.add_argument(
'--include_list',
default='null',
help='Path to text file containing a list of included species. Not used if not provided.')
parser.add_argument(
'--exclude_list',
default='null',
help='Path to text file containing a list of excluded species. Not used if not provided.')
parser.add_argument(
'--birdweather_id',
default='99999',
help='Private Station ID for BirdWeather.')
args = parser.parse_args() args = parser.parse_args()
@@ -68,15 +106,18 @@ def main():
send(sockParams) send(sockParams)
send(DISCONNECT_MESSAGE) send(DISCONNECT_MESSAGE)
#time.sleep(3) # time.sleep(3)
############################################################################### ###############################################################################
############################################################################### ###############################################################################
if __name__ == '__main__': if __name__ == '__main__':
main() main()
# Example calls # Example calls
# python3 analyze.py --i 'example/XC558716 - Soundscape.mp3' --lat 35.4244 --lon -120.7463 --week 18 # python3 analyze.py --i 'example/XC558716 - Soundscape.mp3' --lat 35.4244 --lon -120.7463 --week 18
# python3 analyze.py --i 'example/XC563936 - Soundscape.mp3' --lat 47.6766 --lon -122.294 --week 11 --overlap 1.5 --min_conf 0.25 --sensitivity 1.25 --custom_list 'example/custom_species_list.txt' # python3 analyze.py --i 'example/XC563936 - Soundscape.mp3' --lat 47.6766
# --lon -122.294 --week 11 --overlap 1.5 --min_conf 0.25 --sensitivity
# 1.25 --custom_list 'example/custom_species_list.txt'
scripts/daily_plot.py
+87 -66
View File
@@ -1,6 +1,5 @@
import sqlite3 import sqlite3
import os import os
import configparser
import pandas as pd import pandas as pd
import seaborn as sns import seaborn as sns
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
@@ -18,84 +17,95 @@ cursor.execute('SELECT * FROM detections WHERE Date = DATE(\'now\', \'localtime\
table_rows = cursor.fetchall() table_rows = cursor.fetchall()
#df=pd.DataFrame(table_rows) # df=pd.DataFrame(table_rows)
#Convert Date and Time Fields to Panda's format # Convert Date and Time Fields to Panda's format
df['Date']=pd.to_datetime(df['Date']) df['Date'] = pd.to_datetime(df['Date'])
df['Time']=pd.to_datetime(df['Time'], unit='ns') df['Time'] = pd.to_datetime(df['Time'], unit='ns')
#Add round hours to dataframe # Add round hours to dataframe
df['Hour of Day'] = [r.hour for r in df.Time] df['Hour of Day'] = [r.hour for r in df.Time]
#Create separate dataframes for separate locations # Create separate dataframes for separate locations
df_plt=df #Default to use the whole Dbase df_plt = df # Default to use the whole Dbase
# Add every font at the specified location # Add every font at the specified location
font_dir = [userDir+'/BirdNET-Pi/homepage/static'] font_dir = [userDir + '/BirdNET-Pi/homepage/static']
for font in font_manager.findSystemFonts(font_dir): for font in font_manager.findSystemFonts(font_dir):
font_manager.fontManager.addfont(font) font_manager.fontManager.addfont(font)
# Set font family globally # Set font family globally
rcParams['font.family'] = 'Roboto Flex' rcParams['font.family'] = 'Roboto Flex'
#Get todays readings # Get todays readings
now = datetime.now() now = datetime.now()
df_plt_today = df_plt[df_plt['Date']==now.strftime("%Y-%m-%d")] df_plt_today = df_plt[df_plt['Date'] == now.strftime("%Y-%m-%d")]
#Set number of species to report # Set number of species to report
readings=10 readings = 10
plt_top10_today = (df_plt_today['Com_Name'].value_counts()[:readings]) plt_top10_today = (df_plt_today['Com_Name'].value_counts()[:readings])
df_plt_top10_today = df_plt_today[df_plt_today.Com_Name.isin(plt_top10_today.index)] df_plt_top10_today = df_plt_today[df_plt_today.Com_Name.isin(plt_top10_today.index)]
#Set Palette for graphics # Set Palette for graphics
pal = "Greens" pal = "Greens"
#Set up plot axes and titles # Set up plot axes and titles
f, axs = plt.subplots(1, 2, figsize = (10, 4), gridspec_kw=dict(width_ratios=[3, 6]), facecolor='#77C487') f, axs = plt.subplots(1, 2, figsize=(10, 4), gridspec_kw=dict(width_ratios=[3, 6]), facecolor='#77C487')
plt.subplots_adjust(left=None, bottom=None, right=None, top=None, wspace=0, hspace=0) plt.subplots_adjust(left=None, bottom=None, right=None, top=None, wspace=0, hspace=0)
#generate y-axis order for all figures based on frequency # generate y-axis order for all figures based on frequency
freq_order = pd.value_counts(df_plt_top10_today['Com_Name']).iloc[:readings].index freq_order = pd.value_counts(df_plt_top10_today['Com_Name']).iloc[:readings].index
#make color for max confidence --> this groups by name and calculates max conf # make color for max confidence --> this groups by name and calculates max conf
confmax = df_plt_top10_today.groupby('Com_Name')['Confidence'].max() confmax = df_plt_top10_today.groupby('Com_Name')['Confidence'].max()
#reorder confmax to detection frequency order # reorder confmax to detection frequency order
confmax = confmax.reindex(freq_order) confmax = confmax.reindex(freq_order)
# norm values for color palette # norm values for color palette
norm = plt.Normalize(confmax.values.min(), confmax.values.max()) norm = plt.Normalize(confmax.values.min(), confmax.values.max())
colors = plt.cm.Greens(norm(confmax)) colors = plt.cm.Greens(norm(confmax))
#Generate frequency plot # Generate frequency plot
plot=sns.countplot(y='Com_Name', data = df_plt_top10_today, palette = colors, order=freq_order, ax=axs[0]) plot = sns.countplot(y='Com_Name', data=df_plt_top10_today, palette=colors, order=freq_order, ax=axs[0])
# Try plot grid lines between bars - problem at the moment plots grid lines on bars - want between bars
z = plot.get_ymajorticklabels()
#Try plot grid lines between bars - problem at the moment plots grid lines on bars - want between bars plot.set_yticklabels(['\n'.join(textwrap.wrap(ticklabel.get_text(), 15)) for ticklabel in plot.get_yticklabels()], fontsize=10)
z=plot.get_ymajorticklabels()
plot.set_yticklabels(['\n'.join(textwrap.wrap(ticklabel.get_text(),15)) for ticklabel in plot.get_yticklabels()], fontsize = 10)
plot.set(ylabel=None) plot.set(ylabel=None)
plot.set(xlabel="Detections") plot.set(xlabel="Detections")
#Generate crosstab matrix for heatmap plot # Generate crosstab matrix for heatmap plot
heat = pd.crosstab(df_plt_top10_today['Com_Name'],df_plt_top10_today['Hour of Day']) heat = pd.crosstab(df_plt_top10_today['Com_Name'], df_plt_top10_today['Hour of Day'])
#Order heatmap Birds by frequency of occurrance # Order heatmap Birds by frequency of occurrance
heat.index = pd.CategoricalIndex(heat.index, categories = freq_order) heat.index = pd.CategoricalIndex(heat.index, categories=freq_order)
heat.sort_index(level=0, inplace=True) heat.sort_index(level=0, inplace=True)
hours_in_day = pd.Series(data = range(0,24)) hours_in_day = pd.Series(data=range(0, 24))
heat_frame = pd.DataFrame(data=0, index=heat.index, columns = hours_in_day) heat_frame = pd.DataFrame(data=0, index=heat.index, columns=hours_in_day)
heat=(heat+heat_frame).fillna(0) heat = (heat + heat_frame).fillna(0)
#Generatie heatmap plot # Generatie heatmap plot
plot = sns.heatmap(heat, norm=LogNorm(), annot=True, annot_kws={"fontsize":7}, fmt="g", cmap = pal , square = False, cbar=False, linewidths = 0.5, linecolor = "Grey", ax=axs[1], yticklabels = False) plot = sns.heatmap(
plot.set_xticklabels(plot.get_xticklabels(), rotation = 0, size = 7) heat,
norm=LogNorm(),
annot=True,
annot_kws={
"fontsize": 7},
fmt="g",
cmap=pal,
square=False,
cbar=False,
linewidths=0.5,
linecolor="Grey",
ax=axs[1],
yticklabels=False)
plot.set_xticklabels(plot.get_xticklabels(), rotation=0, size=7)
# Set heatmap border # Set heatmap border
for _, spine in plot.spines.items(): for _, spine in plot.spines.items():
@@ -103,13 +113,13 @@ for _, spine in plot.spines.items():
plot.set(ylabel=None) plot.set(ylabel=None)
plot.set(xlabel="Hour of Day") plot.set(xlabel="Hour of Day")
#Set combined plot layout and titles # Set combined plot layout and titles
f.subplots_adjust(top=0.9) f.subplots_adjust(top=0.9)
plt.suptitle("Top 10 Last Updated: "+ str(now.strftime("%Y-%m-%d %H:%M"))) plt.suptitle("Top 10 Last Updated: " + str(now.strftime("%Y-%m-%d %H:%M")))
#Save combined plot # Save combined plot
userDir = os.path.expanduser('~') userDir = os.path.expanduser('~')
savename=userDir + '/BirdSongs/Extracted/Charts/Combo-'+str(now.strftime("%Y-%m-%d"))+'.png' savename = userDir + '/BirdSongs/Extracted/Charts/Combo-' + str(now.strftime("%Y-%m-%d")) + '.png'
plt.savefig(savename) plt.savefig(savename)
plt.show() plt.show()
plt.close() plt.close()
@@ -119,18 +129,18 @@ plt.close()
plt_Bot10_today = (df_plt_today['Com_Name'].value_counts()[-readings:]) plt_Bot10_today = (df_plt_today['Com_Name'].value_counts()[-readings:])
df_plt_Bot10_today = df_plt_today[df_plt_today.Com_Name.isin(plt_Bot10_today.index)] df_plt_Bot10_today = df_plt_today[df_plt_today.Com_Name.isin(plt_Bot10_today.index)]
#Set Palette for graphics # Set Palette for graphics
pal = "Reds" pal = "Reds"
#Set up plot axes and titles # Set up plot axes and titles
f, axs = plt.subplots(1, 2, figsize = (10, 4), gridspec_kw=dict(width_ratios=[3, 6]), facecolor='#77C487') f, axs = plt.subplots(1, 2, figsize=(10, 4), gridspec_kw=dict(width_ratios=[3, 6]), facecolor='#77C487')
plt.subplots_adjust(left=None, bottom=None, right=None, top=None, wspace=0, hspace=0) plt.subplots_adjust(left=None, bottom=None, right=None, top=None, wspace=0, hspace=0)
#generate y-axis order for all figures based on frequency # generate y-axis order for all figures based on frequency
freq_order = pd.value_counts(df_plt_Bot10_today['Com_Name']).iloc[-readings:].index freq_order = pd.value_counts(df_plt_Bot10_today['Com_Name']).iloc[-readings:].index
#make color for max confidence --> this groups by name and calculates max conf # make color for max confidence --> this groups by name and calculates max conf
confmax = df_plt_Bot10_today.groupby('Com_Name')['Confidence'].max() confmax = df_plt_Bot10_today.groupby('Com_Name')['Confidence'].max()
confmax = confmax.reindex(freq_order) confmax = confmax.reindex(freq_order)
# probably wrong order . . . how to sort by no. of detections ? # probably wrong order . . . how to sort by no. of detections ?
@@ -138,33 +148,44 @@ confmax = confmax.reindex(freq_order)
norm = plt.Normalize(confmax.values.min(), confmax.values.max()) norm = plt.Normalize(confmax.values.min(), confmax.values.max())
colors = plt.cm.Reds(norm(confmax)) colors = plt.cm.Reds(norm(confmax))
#Generate frequency plot # Generate frequency plot
plot=sns.countplot(y='Com_Name', data = df_plt_Bot10_today, palette = colors, order=freq_order, ax=axs[0]) plot = sns.countplot(y='Com_Name', data=df_plt_Bot10_today, palette=colors, order=freq_order, ax=axs[0])
# Try plot grid lines between bars - problem at the moment plots grid lines on bars - want between bars
z = plot.get_ymajorticklabels()
#Try plot grid lines between bars - problem at the moment plots grid lines on bars - want between bars plot.set_yticklabels(['\n'.join(textwrap.wrap(ticklabel.get_text(), 15)) for ticklabel in plot.get_yticklabels()], fontsize=10)
z=plot.get_ymajorticklabels()
plot.set_yticklabels(['\n'.join(textwrap.wrap(ticklabel.get_text(),15)) for ticklabel in plot.get_yticklabels()], fontsize = 10)
plot.set(ylabel=None) plot.set(ylabel=None)
plot.set(xlabel="Detections") plot.set(xlabel="Detections")
#Generate crosstab matrix for heatmap plot # Generate crosstab matrix for heatmap plot
heat = pd.crosstab(df_plt_Bot10_today['Com_Name'],df_plt_Bot10_today['Hour of Day']) heat = pd.crosstab(df_plt_Bot10_today['Com_Name'], df_plt_Bot10_today['Hour of Day'])
#Order heatmap Birds by frequency of occurrance # Order heatmap Birds by frequency of occurrance
heat.index = pd.CategoricalIndex(heat.index, categories = freq_order) heat.index = pd.CategoricalIndex(heat.index, categories=freq_order)
heat.sort_index(level=0, inplace=True) heat.sort_index(level=0, inplace=True)
hours_in_day = pd.Series(data = range(0,24)) hours_in_day = pd.Series(data=range(0, 24))
heat_frame = pd.DataFrame(data=0, index=heat.index, columns = hours_in_day) heat_frame = pd.DataFrame(data=0, index=heat.index, columns=hours_in_day)
heat=(heat+heat_frame).fillna(0) heat = (heat + heat_frame).fillna(0)
#Generatie heatmap plot # Generatie heatmap plot
plot = sns.heatmap(heat, norm=LogNorm(), annot=True, fmt="g", annot_kws={"fontsize":7}, cmap = pal , square = False, cbar=False, linewidths = 0.5, linecolor = "Grey", ax=axs[1], yticklabels = False) plot = sns.heatmap(
plot.set_xticklabels(plot.get_xticklabels(), rotation = 0, size = 7) heat,
norm=LogNorm(),
annot=True,
fmt="g",
annot_kws={
"fontsize": 7},
cmap=pal,
square=False,
cbar=False,
linewidths=0.5,
linecolor="Grey",
ax=axs[1],
yticklabels=False)
plot.set_xticklabels(plot.get_xticklabels(), rotation=0, size=7)
# Set heatmap border # Set heatmap border
for _, spine in plot.spines.items(): for _, spine in plot.spines.items():
@@ -172,12 +193,12 @@ for _, spine in plot.spines.items():
plot.set(ylabel=None) plot.set(ylabel=None)
plot.set(xlabel="Hour of Day") plot.set(xlabel="Hour of Day")
#Set combined plot layout and titles # Set combined plot layout and titles
f.subplots_adjust(top=0.9) f.subplots_adjust(top=0.9)
plt.suptitle("Bottom 10 Last Updated: "+ str(now.strftime("%Y-%m-%d %H:%M"))) plt.suptitle("Bottom 10 Last Updated: " + str(now.strftime("%Y-%m-%d %H:%M")))
#Save combined plot # Save combined plot
savename=userDir + '/BirdSongs/Extracted/Charts/Combo2-'+str(now.strftime("%Y-%m-%d"))+'.png' savename = userDir + '/BirdSongs/Extracted/Charts/Combo2-' + str(now.strftime("%Y-%m-%d")) + '.png'
plt.savefig(savename) plt.savefig(savename)
plt.show() plt.show()
plt.close() plt.close()
scripts/plotly_streamlit.py
+111 -102
View File
@@ -4,8 +4,7 @@ import pandas as pd
import numpy as np import numpy as np
import plotly.graph_objects as go import plotly.graph_objects as go
from plotly.subplots import make_subplots from plotly.subplots import make_subplots
from datetime import timedelta, datetime from datetime import timedelta
from pathlib import Path
import sqlite3 import sqlite3
from sqlite3 import Connection from sqlite3 import Connection
import plotly.express as px import plotly.express as px
@@ -35,22 +34,22 @@ st.markdown("""
@st.cache(hash_funcs={Connection: id}) @st.cache(hash_funcs={Connection: id})
def get_connection(path:str): def get_connection(path: str):
return sqlite3.connect(path,check_same_thread=False) return sqlite3.connect(path, check_same_thread=False)
def get_data(conn: Connection): def get_data(conn: Connection):
df1=pd.read_sql("SELECT * FROM detections", con=conn) df1 = pd.read_sql("SELECT * FROM detections", con=conn)
return df1 return df1
conn = get_connection(URI_SQLITE_DB) conn = get_connection(URI_SQLITE_DB)
# Read in the cereal data # Read in the cereal data
# df = load_data() # df = load_data()
df=get_data(conn) df = get_data(conn)
df2=df.copy() df2 = df.copy()
df2['DateTime']=pd.to_datetime(df2['Date'] + " " + df2['Time']) df2['DateTime'] = pd.to_datetime(df2['Date'] + " " + df2['Time'])
df2=df2.set_index('DateTime') df2 = df2.set_index('DateTime')
# Filter on date range # Filter on date range
@@ -60,169 +59,179 @@ df2=df2.set_index('DateTime')
# Date as slider # Date as slider
Start_Date = pd.to_datetime(df2.index.min()).date() Start_Date = pd.to_datetime(df2.index.min()).date()
End_Date = pd.to_datetime(df2.index.max()).date() End_Date = pd.to_datetime(df2.index.max()).date()
cols1,cols2= st.columns((1,1)) cols1, cols2 = st.columns((1, 1))
Date_Slider = cols1.slider('Date Range', Date_Slider = cols1.slider('Date Range',
min_value = Start_Date-timedelta(days=1), min_value=Start_Date - timedelta(days=1),
max_value = End_Date, max_value=End_Date,
value=(Start_Date, value=(Start_Date,
End_Date) End_Date)
) )
filt = (df2.index >= pd.Timestamp(Date_Slider[0])) & (df2.index <= pd.Timestamp(Date_Slider[1] + timedelta(days=1)))
filt = (df2.index >= pd.Timestamp(Date_Slider[0])) & (df2.index <= pd.Timestamp(Date_Slider[1]+timedelta(days=1)))
df2 = df2[filt] df2 = df2[filt]
st.write('<style>div.row-widget.stRadio > div{flex-direction:row;justify-content: left;} </style>', unsafe_allow_html=True) st.write('<style>div.row-widget.stRadio > div{flex-direction:row;justify-content: left;} </style>', unsafe_allow_html=True)
st.write('<style>div.st-bf{flex-direction:column;} div.st-ag{font-weight:bold;padding-left:2px;}</style>', unsafe_allow_html=True) st.write('<style>div.st-bf{flex-direction:column;} div.st-ag{font-weight:bold;padding-left:2px;}</style>', unsafe_allow_html=True)
resample_sel=cols2.radio("Select Resample Resolution - To downsample and make run faster select longer period, Daily provides a view on detections at 15 min intervals through the day", ('1 minute', '5 minutes', '10 minutes', 'Hourly', 'Daily')) resample_sel = cols2.radio(
'''
Select Resample Resolution - To downsample and make run faster select longer period,
Daily provides a view on detections at 15 min intervals through the day
''',
('1 minute',
'5 minutes',
'10 minutes',
'Hourly',
'Daily'))
resample_times = {'1 minute':'1min', resample_times = {'1 minute': '1min',
'5 minutes':'5min', '5 minutes': '5min',
'10 minutes':'10min', '10 minutes': '10min',
'Hourly':'1H', 'Hourly': '1H',
'Daily':'1D' 'Daily': '1D'
} }
resample_time = resample_times[resample_sel] resample_time = resample_times[resample_sel]
df5=df2.resample(resample_time)['Com_Name'].aggregate('unique').explode() df5 = df2.resample(resample_time)['Com_Name'].aggregate('unique').explode()
#Create species count for selected date range # Create species count for selected date range
Specie_Count=df5.value_counts() Specie_Count = df5.value_counts()
#Create species treemap # Create species treemap
# Create Hourly Crosstab # Create Hourly Crosstab
hourly=pd.crosstab(df5,df5.index.hour, dropna=False) hourly = pd.crosstab(df5, df5.index.hour, dropna=False)
# Filter on species # Filter on species
species = list(hourly.index) species = list(hourly.index)
cols1,cols2= st.columns((1,1)) cols1, cols2 = st.columns((1, 1))
top_N = cols1.slider( top_N = cols1.slider(
'Select Number of Birds to Show', 'Select Number of Birds to Show',
min_value = 1, min_value=1,
value=min(10,len(Specie_Count)) value=min(10, len(Specie_Count))
) )
top_N_species = (df5.value_counts()[:top_N]) top_N_species = (df5.value_counts()[:top_N])
specie = cols2.selectbox('Which bird would you like to explore for the dates '+str(Date_Slider[0])+' to '+str(Date_Slider[1])+'?', species, specie = cols2.selectbox('Which bird would you like to explore for the dates ' + str(Date_Slider[0]) + ' to ' + str(Date_Slider[1]) + '?', species,
index=species.index(list(top_N_species.index)[0])) index=species.index(list(top_N_species.index)[0]))
font_size=15 font_size = 15
#specie filter # specie filter
filt=df2['Com_Name']==specie filt = df2['Com_Name'] == specie
df_counts=sum(df5==specie)
df_counts = sum(df5 == specie)
if resample_time != '1D': if resample_time != '1D':
fig = make_subplots( fig = make_subplots(
rows=3, cols =2, rows=3, cols=2,
specs= [[{"type":"xy","rowspan":3}, {"type":"polar","rowspan":2}], [{"rowspan":1}, {"rowspan":1} ], [None, {"type":"xy","rowspan":1}]], specs=[[{"type": "xy", "rowspan": 3}, {"type": "polar", "rowspan": 2}], [{"rowspan": 1}, {"rowspan": 1}], [None, {"type": "xy", "rowspan": 1}]],
subplot_titles=('<b>Top '+ str(top_N) + ' Species in Date Range '+str(Date_Slider[0])+' to '+str(Date_Slider[1])+' for '+str(resample_sel)+' sampling interval.'+'</b>', subplot_titles=('<b>Top ' +
'Total Detect:'+str('{:,}'.format(df_counts))+ str(top_N) +
' Confidence Max:'+str('{:.2f}%'.format(max(df2[df2['Com_Name']==specie]['Confidence'])*100))+ ' Species in Date Range ' +
' '+' Median:'+str('{:.2f}%'.format(np.median(df2[df2['Com_Name']==specie]['Confidence'])*100)) str(Date_Slider[0]) +
) ' to ' +
str(Date_Slider[1]) +
' for ' +
str(resample_sel) +
' sampling interval.' +
'</b>',
'Total Detect:' + str('{:,}'.format(df_counts)) +
' Confidence Max:' + str('{:.2f}%'.format(max(df2[df2['Com_Name'] == specie]['Confidence']) * 100)) +
' ' + ' Median:' + str('{:.2f}%'.format(np.median(df2[df2['Com_Name'] == specie]['Confidence']) * 100))
) )
fig.layout.annotations[1].update(x=0.7,y=0.25, font_size=15) )
fig.layout.annotations[1].update(x=0.7, y=0.25, font_size=15)
#Plot seen species for selected date range and number of species # Plot seen species for selected date range and number of species
fig.add_trace(go.Bar(y=top_N_species.index, x=top_N_species, orientation='h'), row=1,col=1) fig.add_trace(go.Bar(y=top_N_species.index, x=top_N_species, orientation='h'), row=1, col=1)
fig.update_layout( fig.update_layout(
margin=dict(l=0, r=0, t=50, b=0), margin=dict(l=0, r=0, t=50, b=0),
yaxis={'categoryorder':'total ascending'}) yaxis={'categoryorder': 'total ascending'})
# Set 360 degrees, 24 hours for polar plot # Set 360 degrees, 24 hours for polar plot
theta = np.linspace(0.0, 360, 24, endpoint=False) theta = np.linspace(0.0, 360, 24, endpoint=False)
specie_filt= df5==specie specie_filt = df5 == specie
df3=df5[specie_filt] df3 = df5[specie_filt]
detections2= pd.crosstab(df3, df3.index.hour) detections2 = pd.crosstab(df3, df3.index.hour)
d = pd.DataFrame(np.zeros((23, 1))).squeeze()
d=pd.DataFrame(np.zeros((23,1))).squeeze()
detections = hourly.loc[specie] detections = hourly.loc[specie]
detections=(d+detections).fillna(0) detections = (d + detections).fillna(0)
fig.add_trace(go.Barpolar(r = detections, theta=theta), row=1, col=2) fig.add_trace(go.Barpolar(r=detections, theta=theta), row=1, col=2)
fig.update_layout( fig.update_layout(
autosize=False, autosize=False,
width = 1000, width=1000,
height = 500, height=500,
showlegend=False, showlegend=False,
polar = dict( polar=dict(
radialaxis = dict( radialaxis=dict(
tickfont_size = font_size, tickfont_size=font_size,
showticklabels = False, showticklabels=False,
hoverformat = "#%{theta}: <br>Popularity: %{percent} </br> %{r}" hoverformat="#%{theta}: <br>Popularity: %{percent} </br> %{r}"
), ),
angularaxis = dict( angularaxis=dict(
tickfont_size= font_size, tickfont_size=font_size,
rotation = -90, rotation=-90,
direction = 'clockwise', direction='clockwise',
tickmode='array', tickmode='array',
tickvals=[0,15,35,45,60,75,90,105,120,135,150,165,180,195,210,225,240,255,270,285,300,315,330,345], tickvals=[0, 15, 35, 45, 60, 75, 90, 105, 120, 135, 150, 165, 180, 195, 210, 225, 240, 255, 270, 285, 300, 315, 330, 345],
ticktext=['12am','1am','2am','3am','4am','5am', '6am','7am','8am','9am','10am','11am','12pm','1pm','2pm','3pm','4pm','5pm', '6pm','7pm','8pm','9pm','10pm','11pm'], ticktext=['12am', '1am', '2am', '3am', '4am', '5am', '6am', '7am', '8am', '9am', '10am', '11am',
hoverformat = "#%{theta}: <br>Popularity: %{percent} </br> %{r}" '12pm', '1pm', '2pm', '3pm', '4pm', '5pm', '6pm', '7pm', '8pm', '9pm', '10pm', '11pm'],
hoverformat="#%{theta}: <br>Popularity: %{percent} </br> %{r}"
), ),
), ),
) )
daily = pd.crosstab(df5, df5.index.date, dropna=False)
daily=pd.crosstab(df5,df5.index.date, dropna=False)
fig.add_trace(go.Bar(x=daily.columns, y=daily.loc[specie]), row=3, col=2) fig.add_trace(go.Bar(x=daily.columns, y=daily.loc[specie]), row=3, col=2)
else: else:
fig = make_subplots( fig = make_subplots(
rows=1, cols =2, rows=1, cols=2,
specs= [[{"type":"xy","rowspan":1},{"type":"xy","rowspan":1}]], specs=[[{"type": "xy", "rowspan": 1}, {"type": "xy", "rowspan": 1}]],
subplot_titles=('<b>Daily Top '+ str(top_N) + ' Species in Date Range '+str(Date_Slider[0])+' to '+str(Date_Slider[1])+'</b>', subplot_titles=('<b>Daily Top ' + str(top_N) + ' Species in Date Range ' + str(Date_Slider[0]) + ' to ' + str(Date_Slider[1]) + '</b>',
'<b>Daily ' + specie+ ' Detections on 15 minute intervals </b>'), '<b>Daily ' + specie + ' Detections on 15 minute intervals </b>'),
# 'Total Detect:'+str('{:,}'.format(df_counts))+ # 'Total Detect:'+str('{:,}'.format(df_counts))+
# ' Confidence Max:'+str('{:.2f}%'.format(max(df2[df2['Com_Name']==specie]['Confidence'])*100))+ # ' Confidence Max:'+str('{:.2f}%'.format(max(df2[df2['Com_Name']==specie]['Confidence'])*100))+
# ' '+' Median:'+str('{:.2f}%'.format(np.median(df2[df2['Com_Name']==specie]['Confidence'])*100)) # ' '+' Median:'+str('{:.2f}%'.format(np.median(df2[df2['Com_Name']==specie]['Confidence'])*100))
# ) # )
) )
fig.add_trace(go.Bar(y=top_N_species.index, x=top_N_species, orientation='h'), row=1,col=1) fig.add_trace(go.Bar(y=top_N_species.index, x=top_N_species, orientation='h'), row=1, col=1)
df4=df2['Com_Name'][df2['Com_Name']==specie].resample('15min').count() df4 = df2['Com_Name'][df2['Com_Name'] == specie].resample('15min').count()
df4.index=[df4.index.date, df4.index.time] df4.index = [df4.index.date, df4.index.time]
day_hour_freq=df4.unstack().fillna(0) day_hour_freq = df4.unstack().fillna(0)
fig_x = [d.strftime('%d-%m-%Y') for d in day_hour_freq.index.tolist()] fig_x = [d.strftime('%d-%m-%Y') for d in day_hour_freq.index.tolist()]
fig_y = [h.strftime('%H:%M') for h in day_hour_freq.columns.tolist()] fig_y = [h.strftime('%H:%M') for h in day_hour_freq.columns.tolist()]
fig_z = day_hour_freq.values.transpose() fig_z = day_hour_freq.values.transpose()
fig_heatmap = go.Figure(data=go.Heatmap(x=fig_x,y=fig_y,z=fig_z)) fig_heatmap = go.Figure(data=go.Heatmap(x=fig_x, y=fig_y, z=fig_z))
fig.update_layout( fig.update_layout(
margin=dict(l=0, r=0, t=50, b=0), margin=dict(l=0, r=0, t=50, b=0),
yaxis={'categoryorder':'total ascending'}) yaxis={'categoryorder': 'total ascending'})
color_pals= px.colors.named_colorscales() color_pals = px.colors.named_colorscales()
selected_pal = cols2.selectbox('Select Color Pallet for Daily Detections', color_pals) selected_pal = cols2.selectbox('Select Color Pallet for Daily Detections', color_pals)
fig.add_trace(go.Heatmap(x=fig_x,y=fig_y,z=fig_z, autocolorscale = False, colorscale = selected_pal), row=1, col=2) fig.add_trace(go.Heatmap(x=fig_x, y=fig_y, z=fig_z, autocolorscale=False, colorscale=selected_pal), row=1, col=2)
# container=st.container() # container=st.container()
# config={'displayModelBar': False} # config={'displayModelBar': False}
st.plotly_chart(fig, use_container_width=True) #, config=config) st.plotly_chart(fig, use_container_width=True) # , config=config)
# cols3,cols4=st.columns((1,1)) # cols3,cols4=st.columns((1,1))
# #
scripts/server.py
+109 -74
View File
@@ -1,3 +1,15 @@
import apprise
from pathlib import Path
from tzlocal import get_localzone
import datetime
import sqlite3
import requests
import json
import time
import math
import numpy as np
import librosa
import operator
import socket import socket
import threading import threading
import os import os
@@ -6,26 +18,9 @@ os.environ['CUDA_VISIBLE_DEVICES'] = ''
try: try:
import tflite_runtime.interpreter as tflite import tflite_runtime.interpreter as tflite
except: except BaseException:
from tensorflow import lite as tflite from tensorflow import lite as tflite
import argparse
import operator
import librosa
import numpy as np
import math
import time
from decimal import Decimal
import json
import requests
import sqlite3
import datetime
from time import sleep
import pytz
from tzlocal import get_localzone
from pathlib import Path
import apprise
HEADER = 64 HEADER = 64
PORT = 5050 PORT = 5050
@@ -37,18 +32,17 @@ DISCONNECT_MESSAGE = "!DISCONNECT"
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try: try:
server.bind(ADDR) server.bind(ADDR)
except: except BaseException:
print("Waiting on socket") print("Waiting on socket")
time.sleep(5) time.sleep(5)
# Open most recent Configuration and grab DB_PWD as a python variable # Open most recent Configuration and grab DB_PWD as a python variable
userDir = os.path.expanduser('~') userDir = os.path.expanduser('~')
with open(userDir + '/BirdNET-Pi/scripts/thisrun.txt', 'r') as f: with open(userDir + '/BirdNET-Pi/scripts/thisrun.txt', 'r') as f:
this_run = f.readlines() this_run = f.readlines()
audiofmt = "." + str(str(str([i for i in this_run if i.startswith('AUDIOFMT')]).split('=')[1]).split('\\')[0]) audiofmt = "." + str(str(str([i for i in this_run if i.startswith('AUDIOFMT')]).split('=')[1]).split('\\')[0])
priv_thresh = float("." + str(str(str([i for i in this_run if i.startswith('PRIVACY_THRESHOLD')]).split('=')[1]).split('\\')[0]))/10 priv_thresh = float("." + str(str(str([i for i in this_run if i.startswith('PRIVACY_THRESHOLD')]).split('=')[1]).split('\\')[0])) / 10
def loadModel(): def loadModel():
@@ -62,7 +56,7 @@ def loadModel():
# Load TFLite model and allocate tensors. # Load TFLite model and allocate tensors.
modelpath = userDir + '/BirdNET-Pi/model/BirdNET_6K_GLOBAL_MODEL.tflite' modelpath = userDir + '/BirdNET-Pi/model/BirdNET_6K_GLOBAL_MODEL.tflite'
myinterpreter = tflite.Interpreter(model_path=modelpath,num_threads=2) myinterpreter = tflite.Interpreter(model_path=modelpath, num_threads=2)
myinterpreter.allocate_tensors() myinterpreter.allocate_tensors()
# Get input and output tensors. # Get input and output tensors.
@@ -85,6 +79,7 @@ def loadModel():
return myinterpreter return myinterpreter
def loadCustomSpeciesList(path): def loadCustomSpeciesList(path):
slist = [] slist = []
@@ -95,6 +90,7 @@ def loadCustomSpeciesList(path):
return slist return slist
def splitSignal(sig, rate, overlap, seconds=3.0, minlen=1.5): def splitSignal(sig, rate, overlap, seconds=3.0, minlen=1.5):
# Split signal with overlap # Split signal with overlap
@@ -116,6 +112,7 @@ def splitSignal(sig, rate, overlap, seconds=3.0, minlen=1.5):
return sig_splits return sig_splits
def readAudioData(path, overlap, sample_rate=48000): def readAudioData(path, overlap, sample_rate=48000):
print('READING AUDIO DATA...', end=' ', flush=True) print('READING AUDIO DATA...', end=' ', flush=True)
@@ -130,6 +127,7 @@ def readAudioData(path, overlap, sample_rate=48000):
return chunks return chunks
def convertMetadata(m): def convertMetadata(m):
# Convert week to cosine # Convert week to cosine
@@ -147,9 +145,11 @@ def convertMetadata(m):
return np.concatenate([m, mask]) return np.concatenate([m, mask])
def custom_sigmoid(x, sensitivity=1.0): def custom_sigmoid(x, sensitivity=1.0):
return 1 / (1.0 + np.exp(-sensitivity * x)) return 1 / (1.0 + np.exp(-sensitivity * x))
def predict(sample, sensitivity): def predict(sample, sensitivity):
global INTERPRETER global INTERPRETER
# Make a prediction # Make a prediction
@@ -167,20 +167,21 @@ def predict(sample, sensitivity):
# Sort by score # Sort by score
p_sorted = sorted(p_labels.items(), key=operator.itemgetter(1), reverse=True) p_sorted = sorted(p_labels.items(), key=operator.itemgetter(1), reverse=True)
# #print("DATABASE SIZE:", len(p_sorted)) # # print("DATABASE SIZE:", len(p_sorted))
# #print("HUMAN-CUTOFF AT:", int(len(p_sorted)*priv_thresh)/10) # # print("HUMAN-CUTOFF AT:", int(len(p_sorted)*priv_thresh)/10)
# #
# # Remove species that are on blacklist # # Remove species that are on blacklist
human_cutoff = max(10,int(len(p_sorted)*priv_thresh)) human_cutoff = max(10, int(len(p_sorted) * priv_thresh))
for i in range(min(10, len(p_sorted))): for i in range(min(10, len(p_sorted))):
if p_sorted[i][0]=='Human_Human': if p_sorted[i][0] == 'Human_Human':
with open(userDir + '/BirdNET-Pi/HUMAN.txt', 'a') as rfile: with open(userDir + '/BirdNET-Pi/HUMAN.txt', 'a') as rfile:
rfile.write(str(datetime.datetime.now())+str(p_sorted[i])+ ' ' + str(human_cutoff)+ '\n') rfile.write(str(datetime.datetime.now()) + str(p_sorted[i]) + ' ' + str(human_cutoff) + '\n')
return p_sorted[:human_cutoff] return p_sorted[:human_cutoff]
def analyzeAudioData(chunks, lat, lon, week, sensitivity, overlap,): def analyzeAudioData(chunks, lat, lon, week, sensitivity, overlap,):
global INTERPRETER global INTERPRETER
@@ -202,19 +203,19 @@ def analyzeAudioData(chunks, lat, lon, week, sensitivity, overlap,):
# Make prediction # Make prediction
p = predict([sig, mdata], sensitivity) p = predict([sig, mdata], sensitivity)
# print("PPPPP",p) # print("PPPPP",p)
HUMAN_DETECTED=False HUMAN_DETECTED = False
#Catch if Human is recognized # Catch if Human is recognized
for x in range(len(p)): for x in range(len(p)):
if "Human" in p[x][0]: if "Human" in p[x][0]:
HUMAN_DETECTED=True HUMAN_DETECTED = True
# Save result and timestamp # Save result and timestamp
pred_end = pred_start + 3.0 pred_end = pred_start + 3.0
#If human detected set all detections to human to make sure voices are not saved # If human detected set all detections to human to make sure voices are not saved
if HUMAN_DETECTED == True: if HUMAN_DETECTED is True:
p=[('Human_Human',0.0)]*10 p = [('Human_Human', 0.0)] * 10
detections[str(pred_start) + ';' + str(pred_end)] = p detections[str(pred_start) + ';' + str(pred_end)] = p
@@ -224,7 +225,8 @@ def analyzeAudioData(chunks, lat, lon, week, sensitivity, overlap,):
# print('DETECTIONS:::::',detections) # print('DETECTIONS:::::',detections)
return detections return detections
def sendAppriseNotifications(species,confidence):
def sendAppriseNotifications(species, confidence):
if os.path.exists(userDir + '/BirdNET-Pi/apprise.txt') and os.path.getsize(userDir + '/BirdNET-Pi/apprise.txt') > 0: if os.path.exists(userDir + '/BirdNET-Pi/apprise.txt') and os.path.getsize(userDir + '/BirdNET-Pi/apprise.txt') > 0:
with open(userDir + '/BirdNET-Pi/scripts/thisrun.txt', 'r') as f: with open(userDir + '/BirdNET-Pi/scripts/thisrun.txt', 'r') as f:
this_run = f.readlines() this_run = f.readlines()
@@ -239,7 +241,7 @@ def sendAppriseNotifications(species,confidence):
apobj.add(config) apobj.add(config)
apobj.notify( apobj.notify(
body=body.replace("$sciname",species.split("_")[0]).replace("$comname",species.split("_")[1]).replace("$confidence",confidence), body=body.replace("$sciname", species.split("_")[0]).replace("$comname", species.split("_")[1]).replace("$confidence", confidence),
title=title, title=title,
) )
@@ -250,10 +252,10 @@ def sendAppriseNotifications(species,confidence):
cur = con.cursor() cur = con.cursor()
cur.execute("SELECT DISTINCT(Com_Name) FROM detections") cur.execute("SELECT DISTINCT(Com_Name) FROM detections")
known_species = cur.fetchall() known_species = cur.fetchall()
sciName,comName = species.split("_") sciName, comName = species.split("_")
print("\ncomName: ",comName) print("\ncomName: ", comName)
print("\nknown_species: ",known_species) print("\nknown_species: ", known_species)
if comName not in known_species: if comName not in known_species:
apobj = apprise.Apprise() apobj = apprise.Apprise()
config = apprise.AppriseConfig() config = apprise.AppriseConfig()
@@ -261,15 +263,16 @@ def sendAppriseNotifications(species,confidence):
apobj.add(config) apobj.add(config)
apobj.notify( apobj.notify(
body=body.replace("$sciname",species.split("_")[0]).replace("$comname",species.split("_")[1]).replace("$confidence",confidence), body=body.replace("$sciname", species.split("_")[0]).replace("$comname", species.split("_")[1]).replace("$confidence", confidence),
title=title, title=title,
) )
con.close() con.close()
except: except BaseException:
print("Database busy") print("Database busy")
time.sleep(2) time.sleep(2)
def writeResultsToFile(detections, min_conf, path): def writeResultsToFile(detections, min_conf, path):
print('WRITING RESULTS TO', path, '...', end=' ') print('WRITING RESULTS TO', path, '...', end=' ')
@@ -278,17 +281,18 @@ def writeResultsToFile(detections, min_conf, path):
rfile.write('Start (s);End (s);Scientific name;Common name;Confidence\n') rfile.write('Start (s);End (s);Scientific name;Common name;Confidence\n')
for d in detections: for d in detections:
for entry in detections[d]: for entry in detections[d]:
if entry[1] >= min_conf and ((entry[0] in INCLUDE_LIST or len(INCLUDE_LIST) == 0) and (entry[0] not in EXCLUDE_LIST or len(EXCLUDE_LIST) == 0) ): if entry[1] >= min_conf and ((entry[0] in INCLUDE_LIST or len(INCLUDE_LIST) == 0) and (entry[0] not in EXCLUDE_LIST or len(EXCLUDE_LIST) == 0)):
sendAppriseNotifications(str(entry[0]),str(entry[1])); sendAppriseNotifications(str(entry[0]), str(entry[1]))
rfile.write(d + ';' + entry[0].replace('_', ';') + ';' + str(entry[1]) + '\n') rfile.write(d + ';' + entry[0].replace('_', ';') + ';' + str(entry[1]) + '\n')
rcnt += 1 rcnt += 1
print('DONE! WROTE', rcnt, 'RESULTS.') print('DONE! WROTE', rcnt, 'RESULTS.')
return return
def handle_client(conn, addr): def handle_client(conn, addr):
global INCLUDE_LIST global INCLUDE_LIST
global EXCLUDE_LIST global EXCLUDE_LIST
#print(f"[NEW CONNECTION] {addr} connected.") # print(f"[NEW CONNECTION] {addr} connected.")
connected = True connected = True
while connected: while connected:
@@ -299,7 +303,7 @@ def handle_client(conn, addr):
if msg == DISCONNECT_MESSAGE: if msg == DISCONNECT_MESSAGE:
connected = False connected = False
else: else:
#print(f"[{addr}] {msg}") # print(f"[{addr}] {msg}")
args = type('', (), {})() args = type('', (), {})()
@@ -313,8 +317,7 @@ def handle_client(conn, addr):
args.sensitivity = 1.25 args.sensitivity = 1.25
args.min_conf = 0.70 args.min_conf = 0.70
args.lat = -1 args.lat = -1
args.lon = -1 args.lon = -1
for line in msg.split('||'): for line in msg.split('||'):
inputvars = line.split('=') inputvars = line.split('=')
@@ -341,8 +344,6 @@ def handle_client(conn, addr):
elif inputvars[0] == 'lon': elif inputvars[0] == 'lon':
args.lon = float(inputvars[1]) args.lon = float(inputvars[1])
# Load custom species lists - INCLUDED and EXCLUDED # Load custom species lists - INCLUDED and EXCLUDED
if not args.include_list == 'null': if not args.include_list == 'null':
INCLUDE_LIST = loadCustomSpeciesList(args.include_list) INCLUDE_LIST = loadCustomSpeciesList(args.include_list)
@@ -360,16 +361,16 @@ def handle_client(conn, addr):
audioData = readAudioData(args.i, args.overlap) audioData = readAudioData(args.i, args.overlap)
# Get Date/Time from filename in case Pi gets behind # Get Date/Time from filename in case Pi gets behind
#now = datetime.now() # now = datetime.now()
full_file_name = args.i full_file_name = args.i
#print('FULL FILENAME: -' + full_file_name + '-') # print('FULL FILENAME: -' + full_file_name + '-')
file_name = Path(full_file_name).stem file_name = Path(full_file_name).stem
file_date = file_name.split('-birdnet-')[0] file_date = file_name.split('-birdnet-')[0]
file_time = file_name.split('-birdnet-')[1] file_time = file_name.split('-birdnet-')[1]
date_time_str = file_date + ' ' + file_time date_time_str = file_date + ' ' + file_time
date_time_obj = datetime.datetime.strptime(date_time_str, '%Y-%m-%d %H:%M:%S') date_time_obj = datetime.datetime.strptime(date_time_str, '%Y-%m-%d %H:%M:%S')
#print('Date:', date_time_obj.date()) # print('Date:', date_time_obj.date())
#print('Time:', date_time_obj.time()) # print('Time:', date_time_obj.time())
print('Date-time:', date_time_obj) print('Date-time:', date_time_obj)
now = date_time_obj now = date_time_obj
current_date = now.strftime("%Y-%m-%d") current_date = now.strftime("%Y-%m-%d")
@@ -396,23 +397,23 @@ def handle_client(conn, addr):
# Write detections to Database # Write detections to Database
myReturn = '' myReturn = ''
for i in detections: for i in detections:
myReturn += str(i) + '-' + str(detections[i][0]) + '\n' myReturn += str(i) + '-' + str(detections[i][0]) + '\n'
with open(userDir + '/BirdNET-Pi/BirdDB.txt', 'a') as rfile: with open(userDir + '/BirdNET-Pi/BirdDB.txt', 'a') as rfile:
for d in detections: for d in detections:
for entry in detections[d]: for entry in detections[d]:
if entry[1] >= min_conf and ((entry[0] in INCLUDE_LIST or len(INCLUDE_LIST) == 0) and (entry[0] not in EXCLUDE_LIST or len(EXCLUDE_LIST) == 0) ): if entry[1] >= min_conf and ((entry[0] in INCLUDE_LIST or len(INCLUDE_LIST) == 0)
rfile.write(str(current_date) + ';' + str(current_time) + ';' + entry[0].replace('_', ';') + ';' \ and (entry[0] not in EXCLUDE_LIST or len(EXCLUDE_LIST) == 0)):
+ str(entry[1]) +";" + str(args.lat) + ';' + str(args.lon) + ';' + str(min_conf) + ';' + str(week) + ';' \ rfile.write(str(current_date) + ';' + str(current_time) + ';' + entry[0].replace('_', ';') + ';'
+ str(args.sensitivity) +';' + str(args.overlap) + '\n') + str(entry[1]) + ";" + str(args.lat) + ';' + str(args.lon) + ';' + str(min_conf) + ';' + str(week) + ';'
+ str(args.sensitivity) + ';' + str(args.overlap) + '\n')
Date = str(current_date) Date = str(current_date)
Time = str(current_time) Time = str(current_time)
species = entry[0] species = entry[0]
Sci_Name,Com_Name = species.split('_') Sci_Name, Com_Name = species.split('_')
score = entry[1] score = entry[1]
Confidence = str(round(score*100)) Confidence = str(round(score * 100))
Lat = str(args.lat) Lat = str(args.lat)
Lon = str(args.lon) Lon = str(args.lon)
Cutoff = str(args.min_conf) Cutoff = str(args.min_conf)
@@ -421,30 +422,62 @@ def handle_client(conn, addr):
Overlap = str(args.overlap) Overlap = str(args.overlap)
Com_Name = Com_Name.replace("'", "") Com_Name = Com_Name.replace("'", "")
File_Name = Com_Name.replace(" ", "_") + '-' + Confidence + '-' + \ File_Name = Com_Name.replace(" ", "_") + '-' + Confidence + '-' + \
Date.replace("/", "-") + '-birdnet-' + Time + audiofmt Date.replace("/", "-") + '-birdnet-' + Time + audiofmt
#Connect to SQLite Database # Connect to SQLite Database
for attempt_number in range(3): for attempt_number in range(3):
try: try:
con = sqlite3.connect(userDir + '/BirdNET-Pi/scripts/birds.db') con = sqlite3.connect(userDir + '/BirdNET-Pi/scripts/birds.db')
cur = con.cursor() cur = con.cursor()
cur.execute("INSERT INTO detections VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (Date, Time, Sci_Name, Com_Name, str(score), Lat, Lon, Cutoff, Week, Sens, Overlap, File_Name)) cur.execute("INSERT INTO detections VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (Date, Time,
Sci_Name, Com_Name, str(score), Lat, Lon, Cutoff, Week, Sens, Overlap, File_Name))
con.commit() con.commit()
con.close() con.close()
break break
except: except BaseException:
print("Database busy") print("Database busy")
time.sleep(2) time.sleep(2)
print(str(current_date) + ';' + str(current_time) + ';' + entry[0].replace('_', ';') + ';' + str(entry[1]) + ';' + str(args.lat) + ';' + str(args.lon) + ';' + str(min_conf) + ';' + str(week) + ';' + str(args.sensitivity) +';' + str(args.overlap) + Com_Name.replace(" ", "_") + '-' + str(score) + '-' + str(current_date) + '-birdnet-' + str(current_time) + audiofmt + '\n') print(str(current_date) +
';' +
str(current_time) +
';' +
entry[0].replace('_', ';') +
';' +
str(entry[1]) +
';' +
str(args.lat) +
';' +
str(args.lon) +
';' +
str(min_conf) +
';' +
str(week) +
';' +
str(args.sensitivity) +
';' +
str(args.overlap) +
Com_Name.replace(" ", "_") +
'-' +
str(score) +
'-' +
str(current_date) +
'-birdnet-' +
str(current_time) +
audiofmt +
'\n')
if birdweather_id != "99999": if birdweather_id != "99999":
try: try:
if soundscape_uploaded is False: if soundscape_uploaded is False:
# POST soundscape to server # POST soundscape to server
soundscape_url = "https://app.birdweather.com/api/v1/stations/" + birdweather_id + "/soundscapes" + "?timestamp=" + current_iso8601 soundscape_url = 'https://app.birdweather.com/api/v1/stations/' + \
birdweather_id + \
'/soundscapes' + \
'?timestamp=' + \
current_iso8601
with open(args.i, 'rb') as f: with open(args.i, 'rb') as f:
wav_data = f.read() wav_data = f.read()
@@ -461,7 +494,7 @@ def handle_client(conn, addr):
post_begin = "{ " post_begin = "{ "
now_p_start = now + datetime.timedelta(seconds=float(start_time)) now_p_start = now + datetime.timedelta(seconds=float(start_time))
current_iso8601 = now_p_start.astimezone(get_localzone()).isoformat() current_iso8601 = now_p_start.astimezone(get_localzone()).isoformat()
post_timestamp = "\"timestamp\": \"" + current_iso8601 + "\"," post_timestamp = "\"timestamp\": \"" + current_iso8601 + "\","
post_lat = "\"lat\": " + str(args.lat) + "," post_lat = "\"lat\": " + str(args.lat) + ","
post_lon = "\"lon\": " + str(args.lon) + "," post_lon = "\"lon\": " + str(args.lon) + ","
post_soundscape_id = "\"soundscapeId\": " + str(soundscape_id) + "," post_soundscape_id = "\"soundscapeId\": " + str(soundscape_id) + ","
@@ -473,30 +506,32 @@ def handle_client(conn, addr):
post_confidence = "\"confidence\": " + str(entry[1]) post_confidence = "\"confidence\": " + str(entry[1])
post_end = " }" post_end = " }"
post_json = post_begin + post_timestamp + post_lat + post_lon + post_soundscape_id + post_soundscape_start_time + post_soundscape_end_time + post_commonName + post_scientificName + post_algorithm + post_confidence + post_end post_json = post_begin + post_timestamp + post_lat + post_lon + post_soundscape_id + post_soundscape_start_time + \
post_soundscape_end_time + post_commonName + post_scientificName + post_algorithm + post_confidence + post_end
print(post_json) print(post_json)
response = requests.post(detection_url, json=json.loads(post_json)) response = requests.post(detection_url, json=json.loads(post_json))
print("Detection POST Response Status - ", response.status_code) print("Detection POST Response Status - ", response.status_code)
except: except BaseException:
print("Cannot POST right now") print("Cannot POST right now")
conn.send(myReturn.encode(FORMAT)) conn.send(myReturn.encode(FORMAT))
#time.sleep(3) # time.sleep(3)
conn.close() conn.close()
def start(): def start():
# Load model # Load model
global INTERPRETER, INCLUDE_LIST, EXCLUDE_LIST global INTERPRETER, INCLUDE_LIST, EXCLUDE_LIST
INTERPRETER = loadModel() INTERPRETER = loadModel()
server.listen() server.listen()
#print(f"[LISTENING] Server is listening on {SERVER}") # print(f"[LISTENING] Server is listening on {SERVER}")
while True: while True:
conn, addr = server.accept() conn, addr = server.accept()
thread = threading.Thread(target=handle_client, args=(conn, addr)) thread = threading.Thread(target=handle_client, args=(conn, addr))
thread.start() thread.start()
#print(f"[ACTIVE CONNECTIONS] {threading.activeCount() - 1}") # print(f"[ACTIVE CONNECTIONS] {threading.activeCount() - 1}")
#print("[STARTING] server is starting...") # print("[STARTING] server is starting...")
start() start()