From dbb14cae69b25c1d5bff116618c70c0169173af1 Mon Sep 17 00:00:00 2001 From: Jake Herbst Date: Tue, 10 May 2022 20:15:09 -0400 Subject: [PATCH 01/17] Updating script/analyze.py to satisfy pep8 style guide and pass flake8 linting. Used 'autopep8 --in-place --aggressive scripts/analyze.py' as initial style fixes. Manually adjusted line lengths for parser help options. Moved example comment to top below shebang. --- scripts/analyze.py | 96 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 76 insertions(+), 20 deletions(-) diff --git a/scripts/analyze.py b/scripts/analyze.py index 4f36e80..90edf55 100755 --- a/scripts/analyze.py +++ b/scripts/analyze.py @@ -1,3 +1,16 @@ +#!/usr/bin/env python3 + +""" +# Example calls +python3 analyze.py --i 'example/XC558716 - Soundscape.mp3' \ + --lat 35.4244 --lon -120.7463 --week 18 + +python3 analyze.py --i 'example/XC563936 - Soundscape.mp3' \ + --lat 47.6766 --lon -122.294 --week 11 \ + --overlap 1.5 --min_conf 0.25 --sensitivity 1.25 \ + --custom_list 'example/custom_species_list.txt' +""" + import argparse import socket @@ -11,6 +24,7 @@ ADDR = (SERVER, PORT) client = socket.socket(socket.AF_INET, socket.SOCK_STREAM) client.connect(ADDR) + def send(msg): message = msg.encode(FORMAT) msg_length = len(message) @@ -20,6 +34,7 @@ def send(msg): client.send(message) print(client.recv(2048).decode(FORMAT)) + def main(): global INCLUDE_LIST @@ -27,17 +42,62 @@ def main(): # Parse passed arguments parser = argparse.ArgumentParser() - parser.add_argument('--i', help='Path to input file.') - parser.add_argument('--o', default='result.csv', help='Path to output file. Defaults to result.csv.') - parser.add_argument('--lat', type=float, default=-1, help='Recording location latitude. Set -1 to ignore.') - parser.add_argument('--lon', type=float, default=-1, help='Recording location longitude. Set -1 to ignore.') - parser.add_argument('--week', type=int, default=-1, help='Week of the year when the recording was made. Values in [1, 48] (4 weeks per month). Set -1 to ignore.') - parser.add_argument('--overlap', type=float, default=0.0, help='Overlap in seconds between extracted spectrograms. Values in [0.0, 2.9]. Defaults tp 0.0.') - parser.add_argument('--sensitivity', type=float, default=1.0, help='Detection sensitivity; Higher values result in higher sensitivity. Values in [0.5, 1.5]. Defaults to 1.0.') - parser.add_argument('--min_conf', type=float, default=0.1, help='Minimum confidence threshold. Values in [0.01, 0.99]. Defaults to 0.1.') - parser.add_argument('--include_list', default='null', help='Path to text file containing a list of included species. Not used if not provided.') - parser.add_argument('--exclude_list', default='null', help='Path to text file containing a list of excluded species. Not used if not provided.') - parser.add_argument('--birdweather_id', default='99999', help='Private Station ID for BirdWeather.') + parser.add_argument( + '--i', + help='Path to input file.') + parser.add_argument( + '--o', + default='result.csv', + help='Path to output file. Defaults to result.csv.') + parser.add_argument( + '--lat', + type=float, + default=-1, + help='Recording location latitude. Set -1 to ignore.') + parser.add_argument( + '--lon', + type=float, + default=-1, + help='Recording location longitude. Set -1 to ignore.') + parser.add_argument( + '--week', + type=int, + default=-1, + help='''Week of the year when the recording was made. + Values in [1, 48] (4 weeks per month). Set -1 to ignore.''') + parser.add_argument( + '--overlap', + type=float, + default=0.0, + help='''Overlap in seconds between extracted spectrograms. + Values in [0.0, 2.9]. Defaults tp 0.0.''') + parser.add_argument( + '--sensitivity', + type=float, + default=1.0, + help='''Detection sensitivity; + Higher values result in higher sensitivity. + Values in [0.5, 1.5]. Defaults to 1.0.''') + parser.add_argument( + '--min_conf', + type=float, + default=0.1, + help='''Minimum confidence threshold. + Values in [0.01, 0.99]. Defaults to 0.1.''') + parser.add_argument( + '--include_list', + default='null', + help='''Path to text file containing a list of included species. + Not used if not provided.''') + parser.add_argument( + '--exclude_list', + default='null', + help='''Path to text file containing a list of excluded species. + Not used if not provided.''') + parser.add_argument( + '--birdweather_id', + default='99999', + help='Private Station ID for BirdWeather.') args = parser.parse_args() @@ -64,19 +124,15 @@ def main(): sockParams += 'lat=' + str(args.lat) + '||' if args.lon: sockParams += 'lon=' + str(args.lon) + '||' - + send(sockParams) send(DISCONNECT_MESSAGE) - #time.sleep(3) + # time.sleep(3) + +############################################################################### +############################################################################### -############################################################################### -############################################################################### if __name__ == '__main__': - main() - - # Example calls - # python3 analyze.py --i 'example/XC558716 - Soundscape.mp3' --lat 35.4244 --lon -120.7463 --week 18 - # python3 analyze.py --i 'example/XC563936 - Soundscape.mp3' --lat 47.6766 --lon -122.294 --week 11 --overlap 1.5 --min_conf 0.25 --sensitivity 1.25 --custom_list 'example/custom_species_list.txt' From 6067415a8ca7cc4e2d1e01843b14142aa645e4c7 Mon Sep 17 00:00:00 2001 From: Jake Herbst Date: Tue, 10 May 2022 20:41:22 -0400 Subject: [PATCH 02/17] Adding flake8 for python linting --- .github/workflows/ci.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..09c1f68 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,25 @@ +name: CI Jobs + +on: pull_request + +jobs: + python-lint: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Setup Python + uses: actions/setup-python@v3 + with: + python-version: '3.7.x' + cache: 'pip' + architecture: 'x64' + + - name: Install flake8 + run: pip install flake8 + + - name: Run Flake8 Lint + uses: py-actions/flake8@v2 + with: + exclude: "./scripts/daily_plot.py,./scripts/server.py,./scripts/plotly_streamlit.py,./scripts/privacy_server.py" From 26079e5696a2b459d3a1ba986f322016f655ec38 Mon Sep 17 00:00:00 2001 From: Jake Herbst Date: Wed, 11 May 2022 07:47:33 -0400 Subject: [PATCH 03/17] Bumping python linter to use v3.9.x to match Debian Bullseye default --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 09c1f68..5764bb8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v3 with: - python-version: '3.7.x' + python-version: '3.9.x' cache: 'pip' architecture: 'x64' From 3e20ee732e8a91225feb9cd78e58689b5410d857 Mon Sep 17 00:00:00 2001 From: Jake Herbst Date: Wed, 11 May 2022 07:56:41 -0400 Subject: [PATCH 04/17] Updating script/daily_plot.py to satisfy pep8 style guide Used 'autopep8 --in-place --aggressive scripts/daily_plot.py' as initial style fixes. --- scripts/daily_plot.py | 209 ++++++++++++++++++++++++++++-------------- 1 file changed, 138 insertions(+), 71 deletions(-) diff --git a/scripts/daily_plot.py b/scripts/daily_plot.py index e2147b0..a3e8a42 100755 --- a/scripts/daily_plot.py +++ b/scripts/daily_plot.py @@ -12,80 +12,113 @@ userDir = os.path.expanduser('~') conn = sqlite3.connect(userDir + '/BirdNET-Pi/scripts/birds.db') df = pd.read_sql_query("SELECT * from detections", conn) cursor = conn.cursor() -cursor.execute('SELECT * FROM detections WHERE Date = DATE(\'now\', \'localtime\')') +cursor.execute( + 'SELECT * FROM detections WHERE Date = DATE(\'now\', \'localtime\')') table_rows = cursor.fetchall() -#df=pd.DataFrame(table_rows) +# df=pd.DataFrame(table_rows) -#Convert Date and Time Fields to Panda's format -df['Date']=pd.to_datetime(df['Date']) -df['Time']=pd.to_datetime(df['Time'], unit='ns') +# Convert Date and Time Fields to Panda's format +df['Date'] = pd.to_datetime(df['Date']) +df['Time'] = pd.to_datetime(df['Time'], unit='ns') -#Add round hours to dataframe +# Add round hours to dataframe df['Hour of Day'] = [r.hour for r in df.Time] -#Create separate dataframes for separate locations -df_plt=df #Default to use the whole Dbase +# Create separate dataframes for separate locations +df_plt = df # Default to use the whole Dbase -#Get todays readings +# Get todays readings now = datetime.now() -df_plt_today = df_plt[df_plt['Date']==now.strftime("%Y-%m-%d")] +df_plt_today = df_plt[df_plt['Date'] == now.strftime("%Y-%m-%d")] -#Set number of species to report -readings=10 +# Set number of species to report +readings = 10 plt_top10_today = (df_plt_today['Com_Name'].value_counts()[:readings]) -df_plt_top10_today = df_plt_today[df_plt_today.Com_Name.isin(plt_top10_today.index)] +df_plt_top10_today = df_plt_today[df_plt_today.Com_Name.isin( + plt_top10_today.index)] -#Set Palette for graphics +# Set Palette for graphics pal = "Greens" -#Set up plot axes and titles -f, axs = plt.subplots(1, 2, figsize = (10, 4), gridspec_kw=dict(width_ratios=[3, 6]), facecolor='#77C487') -plt.subplots_adjust(left=None, bottom=None, right=None, top=None, wspace=0, hspace=0) +# Set up plot axes and titles +f, axs = plt.subplots( + 1, 2, figsize=( + 10, 4), gridspec_kw=dict( + width_ratios=[ + 3, 6]), facecolor='#77C487') +plt.subplots_adjust( + left=None, + bottom=None, + right=None, + top=None, + wspace=0, + hspace=0) -#generate y-axis order for all figures based on frequency -freq_order = pd.value_counts(df_plt_top10_today['Com_Name']).iloc[:readings].index +# generate y-axis order for all figures based on frequency +freq_order = pd.value_counts( + df_plt_top10_today['Com_Name']).iloc[:readings].index -#make color for max confidence --> this groups by name and calculates max conf +# make color for max confidence --> this groups by name and calculates max conf confmax = df_plt_top10_today.groupby('Com_Name')['Confidence'].max() -#reorder confmax to detection frequency order +# reorder confmax to detection frequency order confmax = confmax.reindex(freq_order) # norm values for color palette norm = plt.Normalize(confmax.values.min(), confmax.values.max()) colors = plt.cm.Greens(norm(confmax)) -#Generate frequency plot -plot=sns.countplot(y='Com_Name', data = df_plt_top10_today, palette = colors, order=freq_order, ax=axs[0]) +# Generate frequency plot +plot = sns.countplot( + y='Com_Name', + data=df_plt_top10_today, + palette=colors, + order=freq_order, + ax=axs[0]) - - -#Try plot grid lines between bars - problem at the moment plots grid lines on bars - want between bars -z=plot.get_ymajorticklabels() -plot.set_yticklabels(['\n'.join(textwrap.wrap(ticklabel.get_text(),15)) for ticklabel in plot.get_yticklabels()], fontsize = 10) +# Try plot grid lines between bars - problem at the moment plots grid +# lines on bars - want between bars +z = plot.get_ymajorticklabels() +plot.set_yticklabels(['\n'.join(textwrap.wrap(ticklabel.get_text(), 15)) + for ticklabel in plot.get_yticklabels()], fontsize=10) plot.set(ylabel=None) plot.set(xlabel="Detections") -#Generate crosstab matrix for heatmap plot +# Generate crosstab matrix for heatmap plot -heat = pd.crosstab(df_plt_top10_today['Com_Name'],df_plt_top10_today['Hour of Day']) -#Order heatmap Birds by frequency of occurrance -heat.index = pd.CategoricalIndex(heat.index, categories = freq_order) +heat = pd.crosstab( + df_plt_top10_today['Com_Name'], + df_plt_top10_today['Hour of Day']) +# Order heatmap Birds by frequency of occurrance +heat.index = pd.CategoricalIndex(heat.index, categories=freq_order) heat.sort_index(level=0, inplace=True) -hours_in_day = pd.Series(data = range(0,24)) -heat_frame = pd.DataFrame(data=0, index=heat.index, columns = hours_in_day) -heat=(heat+heat_frame).fillna(0) +hours_in_day = pd.Series(data=range(0, 24)) +heat_frame = pd.DataFrame(data=0, index=heat.index, columns=hours_in_day) +heat = (heat + heat_frame).fillna(0) -#Generatie heatmap plot -plot = sns.heatmap(heat, norm=LogNorm(), annot=True, annot_kws={"fontsize":7}, fmt="g", cmap = pal , square = False, cbar=False, linewidths = 0.5, linecolor = "Grey", ax=axs[1], yticklabels = False) -plot.set_xticklabels(plot.get_xticklabels(), rotation = 0, size = 7) +# Generatie heatmap plot +plot = sns.heatmap( + heat, + norm=LogNorm(), + annot=True, + annot_kws={ + "fontsize": 7}, + fmt="g", + cmap=pal, + square=False, + cbar=False, + linewidths=0.5, + linecolor="Grey", + ax=axs[1], + yticklabels=False) +plot.set_xticklabels(plot.get_xticklabels(), rotation=0, size=7) # Set heatmap border for _, spine in plot.spines.items(): @@ -93,13 +126,14 @@ for _, spine in plot.spines.items(): plot.set(ylabel=None) plot.set(xlabel="Hour of Day") -#Set combined plot layout and titles +# Set combined plot layout and titles f.subplots_adjust(top=0.9) -plt.suptitle("Top 10 Last Updated: "+ str(now.strftime("%Y-%m-%d %H:%M"))) +plt.suptitle("Top 10 Last Updated: " + str(now.strftime("%Y-%m-%d %H:%M"))) -#Save combined plot +# Save combined plot userDir = os.path.expanduser('~') -savename=userDir + '/BirdSongs/Extracted/Charts/Combo-'+str(now.strftime("%Y-%m-%d"))+'.png' +savename = userDir + '/BirdSongs/Extracted/Charts/Combo-' + \ + str(now.strftime("%Y-%m-%d")) + '.png' plt.savefig(savename) plt.show() plt.close() @@ -107,20 +141,32 @@ plt.close() # Get Bottom detection frequency plt_Bot10_today = (df_plt_today['Com_Name'].value_counts()[-readings:]) -df_plt_Bot10_today = df_plt_today[df_plt_today.Com_Name.isin(plt_Bot10_today.index)] +df_plt_Bot10_today = df_plt_today[df_plt_today.Com_Name.isin( + plt_Bot10_today.index)] -#Set Palette for graphics +# Set Palette for graphics pal = "Reds" -#Set up plot axes and titles +# Set up plot axes and titles -f, axs = plt.subplots(1, 2, figsize = (10, 4), gridspec_kw=dict(width_ratios=[3, 6]), facecolor='#77C487') -plt.subplots_adjust(left=None, bottom=None, right=None, top=None, wspace=0, hspace=0) +f, axs = plt.subplots( + 1, 2, figsize=( + 10, 4), gridspec_kw=dict( + width_ratios=[ + 3, 6]), facecolor='#77C487') +plt.subplots_adjust( + left=None, + bottom=None, + right=None, + top=None, + wspace=0, + hspace=0) -#generate y-axis order for all figures based on frequency -freq_order = pd.value_counts(df_plt_Bot10_today['Com_Name']).iloc[-readings:].index +# generate y-axis order for all figures based on frequency +freq_order = pd.value_counts( + df_plt_Bot10_today['Com_Name']).iloc[-readings:].index -#make color for max confidence --> this groups by name and calculates max conf +# make color for max confidence --> this groups by name and calculates max conf confmax = df_plt_Bot10_today.groupby('Com_Name')['Confidence'].max() confmax = confmax.reindex(freq_order) # probably wrong order . . . how to sort by no. of detections ? @@ -128,33 +174,53 @@ confmax = confmax.reindex(freq_order) norm = plt.Normalize(confmax.values.min(), confmax.values.max()) colors = plt.cm.Reds(norm(confmax)) -#Generate frequency plot -plot=sns.countplot(y='Com_Name', data = df_plt_Bot10_today, palette = colors, order=freq_order, ax=axs[0]) +# Generate frequency plot +plot = sns.countplot( + y='Com_Name', + data=df_plt_Bot10_today, + palette=colors, + order=freq_order, + ax=axs[0]) - - -#Try plot grid lines between bars - problem at the moment plots grid lines on bars - want between bars -z=plot.get_ymajorticklabels() -plot.set_yticklabels(['\n'.join(textwrap.wrap(ticklabel.get_text(),15)) for ticklabel in plot.get_yticklabels()], fontsize = 10) +# Try plot grid lines between bars - problem at the moment plots grid +# lines on bars - want between bars +z = plot.get_ymajorticklabels() +plot.set_yticklabels(['\n'.join(textwrap.wrap(ticklabel.get_text(), 15)) + for ticklabel in plot.get_yticklabels()], fontsize=10) plot.set(ylabel=None) plot.set(xlabel="Detections") -#Generate crosstab matrix for heatmap plot +# Generate crosstab matrix for heatmap plot -heat = pd.crosstab(df_plt_Bot10_today['Com_Name'],df_plt_Bot10_today['Hour of Day']) -#Order heatmap Birds by frequency of occurrance -heat.index = pd.CategoricalIndex(heat.index, categories = freq_order) +heat = pd.crosstab( + df_plt_Bot10_today['Com_Name'], + df_plt_Bot10_today['Hour of Day']) +# Order heatmap Birds by frequency of occurrance +heat.index = pd.CategoricalIndex(heat.index, categories=freq_order) heat.sort_index(level=0, inplace=True) -hours_in_day = pd.Series(data = range(0,24)) -heat_frame = pd.DataFrame(data=0, index=heat.index, columns = hours_in_day) -heat=(heat+heat_frame).fillna(0) +hours_in_day = pd.Series(data=range(0, 24)) +heat_frame = pd.DataFrame(data=0, index=heat.index, columns=hours_in_day) +heat = (heat + heat_frame).fillna(0) -#Generatie heatmap plot -plot = sns.heatmap(heat, norm=LogNorm(), annot=True, fmt="g", annot_kws={"fontsize":7}, cmap = pal , square = False, cbar=False, linewidths = 0.5, linecolor = "Grey", ax=axs[1], yticklabels = False) -plot.set_xticklabels(plot.get_xticklabels(), rotation = 0, size = 7) +# Generatie heatmap plot +plot = sns.heatmap( + heat, + norm=LogNorm(), + annot=True, + fmt="g", + annot_kws={ + "fontsize": 7}, + cmap=pal, + square=False, + cbar=False, + linewidths=0.5, + linecolor="Grey", + ax=axs[1], + yticklabels=False) +plot.set_xticklabels(plot.get_xticklabels(), rotation=0, size=7) # Set heatmap border for _, spine in plot.spines.items(): @@ -162,12 +228,13 @@ for _, spine in plot.spines.items(): plot.set(ylabel=None) plot.set(xlabel="Hour of Day") -#Set combined plot layout and titles +# Set combined plot layout and titles f.subplots_adjust(top=0.9) -plt.suptitle("Bottom 10 Last Updated: "+ str(now.strftime("%Y-%m-%d %H:%M"))) +plt.suptitle("Bottom 10 Last Updated: " + str(now.strftime("%Y-%m-%d %H:%M"))) -#Save combined plot -savename=userDir + '/BirdSongs/Extracted/Charts/Combo2-'+str(now.strftime("%Y-%m-%d"))+'.png' +# Save combined plot +savename = userDir + '/BirdSongs/Extracted/Charts/Combo2-' + \ + str(now.strftime("%Y-%m-%d")) + '.png' plt.savefig(savename) plt.show() plt.close() From f81b2c301426201799a4a8f11fe5fed544c741c5 Mon Sep 17 00:00:00 2001 From: Jake Herbst Date: Wed, 11 May 2022 07:58:34 -0400 Subject: [PATCH 05/17] Removing unused import in daily_plot.py to satisfy flake8 linting --- scripts/daily_plot.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/daily_plot.py b/scripts/daily_plot.py index a3e8a42..be7d853 100755 --- a/scripts/daily_plot.py +++ b/scripts/daily_plot.py @@ -1,6 +1,5 @@ import sqlite3 import os -import configparser import pandas as pd import seaborn as sns import matplotlib.pyplot as plt From 5f5c320e61a5cc2dc6b38563bdbcebaa315a0c3a Mon Sep 17 00:00:00 2001 From: Jake Herbst Date: Wed, 11 May 2022 08:05:54 -0400 Subject: [PATCH 06/17] Updating script/privacy_server.py to satisfy pep8 style guide Used 'autopep8 --in-place --aggressive scripts/privacy_server.py' as initial style fixes. --- scripts/privacy_server.py | 295 ++++++++++++++++++++++++-------------- 1 file changed, 191 insertions(+), 104 deletions(-) diff --git a/scripts/privacy_server.py b/scripts/privacy_server.py index a103fff..63ae524 100755 --- a/scripts/privacy_server.py +++ b/scripts/privacy_server.py @@ -1,4 +1,19 @@ -import socket +from pathlib import Path +from tzlocal import get_localzone +import pytz +from time import sleep +import datetime +import sqlite3 +import requests +import json +from decimal import Decimal +import time +import math +import numpy as np +import librosa +import operator +import argparse +import socket import threading import os os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' @@ -6,25 +21,9 @@ os.environ['CUDA_VISIBLE_DEVICES'] = '' try: import tflite_runtime.interpreter as tflite -except: +except BaseException: from tensorflow import lite as tflite -import argparse -import operator -import librosa -import numpy as np -import math -import time -from decimal import Decimal -import json -import requests -import sqlite3 -import datetime -from time import sleep -import pytz -from tzlocal import get_localzone -from pathlib import Path - HEADER = 64 PORT = 5050 @@ -36,17 +35,18 @@ DISCONNECT_MESSAGE = "!DISCONNECT" server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: server.bind(ADDR) -except: +except BaseException: print("Waiting on socket") time.sleep(5) - # Open most recent Configuration and grab DB_PWD as a python variable userDir = os.path.expanduser('~') with open(userDir + '/BirdNET-Pi/scripts/thisrun.txt', 'r') as f: this_run = f.readlines() - audiofmt = "." + str(str(str([i for i in this_run if i.startswith('AUDIOFMT')]).split('=')[1]).split('\\')[0]) + audiofmt = "." + \ + str(str(str([i for i in this_run if i.startswith( + 'AUDIOFMT')]).split('=')[1]).split('\\')[0]) def loadModel(): @@ -60,7 +60,7 @@ def loadModel(): # Load TFLite model and allocate tensors. modelpath = userDir + '/BirdNET-Pi/model/BirdNET_6K_GLOBAL_MODEL.tflite' - myinterpreter = tflite.Interpreter(model_path=modelpath,num_threads=2) + myinterpreter = tflite.Interpreter(model_path=modelpath, num_threads=2) myinterpreter.allocate_tensors() # Get input and output tensors. @@ -82,6 +82,7 @@ def loadModel(): return myinterpreter + def loadCustomSpeciesList(path): slist = [] @@ -92,6 +93,7 @@ def loadCustomSpeciesList(path): return slist + def splitSignal(sig, rate, overlap, seconds=3.0, minlen=1.5): # Split signal with overlap @@ -102,23 +104,25 @@ def splitSignal(sig, rate, overlap, seconds=3.0, minlen=1.5): # End of signal? if len(split) < int(minlen * rate): break - + # Signal chunk too short? Fill with zeros. if len(split) < int(rate * seconds): temp = np.zeros((int(rate * seconds))) temp[:len(split)] = split split = temp - + sig_splits.append(split) return sig_splits + def readAudioData(path, overlap, sample_rate=48000): print('READING AUDIO DATA...', end=' ', flush=True) # Open file with librosa (uses ffmpeg or libav) - sig, rate = librosa.load(path, sr=sample_rate, mono=True, res_type='kaiser_fast') + sig, rate = librosa.load( + path, sr=sample_rate, mono=True, res_type='kaiser_fast') # Split audio into 3-second chunks chunks = splitSignal(sig, rate, overlap) @@ -127,11 +131,12 @@ def readAudioData(path, overlap, sample_rate=48000): return chunks + def convertMetadata(m): # Convert week to cosine if m[2] >= 1 and m[2] <= 48: - m[2] = math.cos(math.radians(m[2] * 7.5)) + 1 + m[2] = math.cos(math.radians(m[2] * 7.5)) + 1 else: m[2] = -1 @@ -144,14 +149,20 @@ def convertMetadata(m): return np.concatenate([m, mask]) + def custom_sigmoid(x, sensitivity=1.0): return 1 / (1.0 + np.exp(-sensitivity * x)) + def predict(sample, sensitivity): global INTERPRETER # Make a prediction - INTERPRETER.set_tensor(INPUT_LAYER_INDEX, np.array(sample[0], dtype='float32')) - INTERPRETER.set_tensor(MDATA_INPUT_INDEX, np.array(sample[1], dtype='float32')) + INTERPRETER.set_tensor( + INPUT_LAYER_INDEX, np.array( + sample[0], dtype='float32')) + INTERPRETER.set_tensor( + MDATA_INPUT_INDEX, np.array( + sample[1], dtype='float32')) INTERPRETER.invoke() prediction = INTERPRETER.get_tensor(OUTPUT_LAYER_INDEX)[0] @@ -162,26 +173,32 @@ def predict(sample, sensitivity): p_labels = dict(zip(CLASSES, p_sigmoid)) # Sort by score - p_sorted = sorted(p_labels.items(), key=operator.itemgetter(1), reverse=True) + p_sorted = sorted( + p_labels.items(), + key=operator.itemgetter(1), + reverse=True) # Remove species that are on blacklist for i in range(min(10, len(p_sorted))): if p_sorted[i][0] in ['Non-bird_Non-bird', 'Noise_Noise']: p_sorted[i] = (p_sorted[i][0], 0.0) - if p_sorted[i][0]=='Human_Human': - print("HUMAN SCORE:",str(p_sorted[i])) - HUMAN_FLAG=True + if p_sorted[i][0] == 'Human_Human': + print("HUMAN SCORE:", str(p_sorted[i])) + HUMAN_FLAG = True with open(userDir + '/BirdNET-Pi/HUMAN.txt', 'a') as rfile: - rfile.write(str(datetime.datetime.now())+str(p_sorted[i])+ '\n') + rfile.write(str(datetime.datetime.now()) + + str(p_sorted[i]) + '\n') # date_stamp=datetime.datetime.now().strftime("%d_%m_%y_%H:%M:%S") -# -# sf.write('./home/*/human_sample.wav',np.random.randn(10,2) , 44100) #sample[0] +# +# sf.write('./home/*/human_sample.wav',np.random.randn(10,2) , 44100) +# #sample[0] # Only return first the top ten results - #INCREASE THIS TO SEE IF HUMAN IS DETECTED MORE RELIABLY + # INCREASE THIS TO SEE IF HUMAN IS DETECTED MORE RELIABLY # print('P_SORTED-------', p_sorted) return p_sorted[:100] + def analyzeAudioData(chunks, lat, lon, week, sensitivity, overlap,): global INTERPRETER @@ -203,24 +220,24 @@ def analyzeAudioData(chunks, lat, lon, week, sensitivity, overlap,): # Make prediction p = predict([sig, mdata], sensitivity) # print("PPPPP",p) - HUMAN_DETECTED=False - #Catch if Human is recognized + HUMAN_DETECTED = False + # Catch if Human is recognized for x in range(len(p)): if "Human" in p[x][0]: -# print("HUMAN DETECTED!!",p[x][0]) - #clear list - HUMAN_DETECTED=True - print("CHUNK -----",c) - + # print("HUMAN DETECTED!!",p[x][0]) + # clear list + HUMAN_DETECTED = True + print("CHUNK -----", c) + # Save result and timestamp pred_end = pred_start + 3.0 - + if HUMAN_DETECTED == True: - p=[('Human_Human',0.0)]*10 - print("HUMAN DETECTED!!!",p) + p = [('Human_Human', 0.0)] * 10 + print("HUMAN DETECTED!!!", p) detections[str(pred_start) + ';' + str(pred_end)] = p - + pred_start = pred_end - overlap print('DONE! Time', int((time.time() - start) * 10) / 10.0, 'SECONDS') @@ -233,15 +250,23 @@ def writeResultsToFile(detections, min_conf, path): print('WRITING RESULTS TO', path, '...', end=' ') rcnt = 0 with open(path, 'w') as rfile: - rfile.write('Start (s);End (s);Scientific name;Common name;Confidence\n') + rfile.write( + 'Start (s);End (s);Scientific name;Common name;Confidence\n') for d in detections: for entry in detections[d]: - if entry[1] >= min_conf and ((entry[0] in INCLUDE_LIST or len(INCLUDE_LIST) == 0) and (entry[0] not in EXCLUDE_LIST or len(EXCLUDE_LIST) == 0) ): - rfile.write(d + ';' + entry[0].replace('_', ';') + ';' + str(entry[1]) + '\n') + if entry[1] >= min_conf and ((entry[0] in INCLUDE_LIST or len( + INCLUDE_LIST) == 0) and (entry[0] not in EXCLUDE_LIST or len(EXCLUDE_LIST) == 0)): + rfile.write(d + + ';' + + entry[0].replace('_', ';') + + ';' + + str(entry[1]) + + '\n') rcnt += 1 print('DONE! WROTE', rcnt, 'RESULTS.') return + def handle_client(conn, addr): global INCLUDE_LIST global EXCLUDE_LIST @@ -257,9 +282,9 @@ def handle_client(conn, addr): connected = False else: #print(f"[{addr}] {msg}") - + args = type('', (), {})() - + args.i = '' args.o = '' args.birdweather_id = '99999' @@ -270,8 +295,7 @@ def handle_client(conn, addr): args.sensitivity = 1.25 args.min_conf = 0.70 args.lat = -1 - args.lon = -1 - + args.lon = -1 for line in msg.split('||'): inputvars = line.split('=') @@ -298,14 +322,12 @@ def handle_client(conn, addr): elif inputvars[0] == 'lon': args.lon = float(inputvars[1]) - - # Load custom species lists - INCLUDED and EXCLUDED if not args.include_list == 'null': INCLUDE_LIST = loadCustomSpeciesList(args.include_list) else: INCLUDE_LIST = [] - + if not args.exclude_list == 'null': EXCLUDE_LIST = loadCustomSpeciesList(args.exclude_list) else: @@ -324,7 +346,8 @@ def handle_client(conn, addr): file_date = file_name.split('-birdnet-')[0] file_time = file_name.split('-birdnet-')[1] date_time_str = file_date + ' ' + file_time - date_time_obj = datetime.datetime.strptime(date_time_str, '%Y-%m-%d %H:%M:%S') + date_time_obj = datetime.datetime.strptime( + date_time_str, '%Y-%m-%d %H:%M:%S') #print('Date:', date_time_obj.date()) #print('Time:', date_time_obj.time()) print('Date-time:', date_time_obj) @@ -332,44 +355,47 @@ def handle_client(conn, addr): current_date = now.strftime("%Y-%m-%d") current_time = now.strftime("%H:%M:%S") current_iso8601 = now.astimezone(get_localzone()).isoformat() - + week_number = int(now.strftime("%V")) week = max(1, min(week_number, 48)) - sensitivity = max(0.5, min(1.0 - (args.sensitivity - 1.0), 1.5)) + sensitivity = max( + 0.5, min(1.0 - (args.sensitivity - 1.0), 1.5)) # Process audio data and get detections - detections = analyzeAudioData(audioData, args.lat, args.lon, week, sensitivity, args.overlap) + detections = analyzeAudioData( + audioData, args.lat, args.lon, week, sensitivity, args.overlap) # Write detections to output file min_conf = max(0.01, min(args.min_conf, 0.99)) writeResultsToFile(detections, min_conf, args.o) - - ############################################################################### - ############################################################################### - + + ################################################################### + ################################################################### + soundscape_uploaded = False # Write detections to Database myReturn = '' for i in detections: - myReturn += str(i) + '-' + str(detections[i][0]) + '\n' - - + myReturn += str(i) + '-' + str(detections[i][0]) + '\n' + with open(userDir + '/BirdNET-Pi/BirdDB.txt', 'a') as rfile: for d in detections: for entry in detections[d]: - if entry[1] >= min_conf and ((entry[0] in INCLUDE_LIST or len(INCLUDE_LIST) == 0) and (entry[0] not in EXCLUDE_LIST or len(EXCLUDE_LIST) == 0) ): - rfile.write(str(current_date) + ';' + str(current_time) + ';' + entry[0].replace('_', ';') + ';' \ - + str(entry[1]) +";" + str(args.lat) + ';' + str(args.lon) + ';' + str(min_conf) + ';' + str(week) + ';' \ - + str(args.sensitivity) +';' + str(args.overlap) + '\n') - + if entry[1] >= min_conf and ((entry[0] in INCLUDE_LIST or len( + INCLUDE_LIST) == 0) and (entry[0] not in EXCLUDE_LIST or len(EXCLUDE_LIST) == 0)): + rfile.write(str(current_date) + ';' + str(current_time) + ';' + entry[0].replace('_', ';') + ';' + + str(entry[1]) + ";" + str(args.lat) + ';' + str( + args.lon) + ';' + str(min_conf) + ';' + str(week) + ';' + + str(args.sensitivity) + ';' + str(args.overlap) + '\n') + Date = str(current_date) Time = str(current_time) species = entry[0] - Sci_Name,Com_Name = species.split('_') + Sci_Name, Com_Name = species.split('_') score = entry[1] - Confidence = str(round(score*100)) + Confidence = str(round(score * 100)) Lat = str(args.lat) Lon = str(args.lon) Cutoff = str(args.min_conf) @@ -378,66 +404,127 @@ def handle_client(conn, addr): Overlap = str(args.overlap) Com_Name = Com_Name.replace("'", "") File_Name = Com_Name.replace(" ", "_") + '-' + Confidence + '-' + \ - Date.replace("/", "-") + '-birdnet-' + Time + audiofmt + Date.replace( + "/", "-") + '-birdnet-' + Time + audiofmt - #Connect to SQLite Database - try: - con = sqlite3.connect(userDir + '/BirdNET-Pi/scripts/birds.db') + # Connect to SQLite Database + try: + con = sqlite3.connect( + userDir + '/BirdNET-Pi/scripts/birds.db') cur = con.cursor() - cur.execute("INSERT INTO detections VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (Date, Time, Sci_Name, Com_Name, str(score), Lat, Lon, Cutoff, Week, Sens, Overlap, File_Name)) + cur.execute( + "INSERT INTO detections VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + (Date, + Time, + Sci_Name, + Com_Name, + str(score), + Lat, + Lon, + Cutoff, + Week, + Sens, + Overlap, + File_Name)) con.commit() con.close() - except: + except BaseException: print("Database busy") time.sleep(2) - print(str(current_date) + ';' + str(current_time) + ';' + entry[0].replace('_', ';') + ';' + str(entry[1]) + ';' + str(args.lat) + ';' + str(args.lon) + ';' + str(min_conf) + ';' + str(week) + ';' + str(args.sensitivity) +';' + str(args.overlap) + Com_Name.replace(" ", "_") + '-' + str(score) + '-' + str(current_date) + '-birdnet-' + str(current_time) + audiofmt + '\n') + print(str(current_date) + + ';' + + str(current_time) + + ';' + + entry[0].replace('_', ';') + + ';' + + str(entry[1]) + + ';' + + str(args.lat) + + ';' + + str(args.lon) + + ';' + + str(min_conf) + + ';' + + str(week) + + ';' + + str(args.sensitivity) + + ';' + + str(args.overlap) + + Com_Name.replace(" ", "_") + + '-' + + str(score) + + '-' + + str(current_date) + + '-birdnet-' + + str(current_time) + + audiofmt + + '\n') if birdweather_id != "99999": try: if soundscape_uploaded is False: # POST soundscape to server - soundscape_url = "https://app.birdweather.com/api/v1/stations/" + birdweather_id + "/soundscapes" + "?timestamp=" + current_iso8601 - + soundscape_url = "https://app.birdweather.com/api/v1/stations/" + \ + birdweather_id + "/soundscapes" + "?timestamp=" + current_iso8601 + with open(args.i, 'rb') as f: wav_data = f.read() - response = requests.post(url=soundscape_url, data=wav_data, headers={'Content-Type': 'application/octet-stream'}) - print("Soundscape POST Response Status - ", response.status_code) + response = requests.post( + url=soundscape_url, data=wav_data, headers={ + 'Content-Type': 'application/octet-stream'}) + print( + "Soundscape POST Response Status - ", response.status_code) sdata = response.json() soundscape_id = sdata['soundscape']['id'] soundscape_uploaded = True - + # POST detection to server - detection_url = "https://app.birdweather.com/api/v1/stations/" + birdweather_id + "/detections" + detection_url = "https://app.birdweather.com/api/v1/stations/" + \ + birdweather_id + "/detections" start_time = d.split(';')[0] end_time = d.split(';')[1] post_begin = "{ " - now_p_start = now + datetime.timedelta(seconds=float(start_time)) - current_iso8601 = now_p_start.astimezone(get_localzone()).isoformat() - post_timestamp = "\"timestamp\": \"" + current_iso8601 + "\"," - post_lat = "\"lat\": " + str(args.lat) + "," - post_lon = "\"lon\": " + str(args.lon) + "," - post_soundscape_id = "\"soundscapeId\": " + str(soundscape_id) + "," + now_p_start = now + \ + datetime.timedelta( + seconds=float(start_time)) + current_iso8601 = now_p_start.astimezone( + get_localzone()).isoformat() + post_timestamp = "\"timestamp\": \"" + current_iso8601 + "\"," + post_lat = "\"lat\": " + \ + str(args.lat) + "," + post_lon = "\"lon\": " + \ + str(args.lon) + "," + post_soundscape_id = "\"soundscapeId\": " + \ + str(soundscape_id) + "," post_soundscape_start_time = "\"soundscapeStartTime\": " + start_time + "," post_soundscape_end_time = "\"soundscapeEndTime\": " + end_time + "," - post_commonName = "\"commonName\": \"" + entry[0].split('_')[1] + "\"," - post_scientificName = "\"scientificName\": \"" + entry[0].split('_')[0] + "\"," + post_commonName = "\"commonName\": \"" + \ + entry[0].split('_')[1] + "\"," + post_scientificName = "\"scientificName\": \"" + \ + entry[0].split('_')[0] + "\"," post_algorithm = "\"algorithm\": " + "\"alpha\"" + "," - post_confidence = "\"confidence\": " + str(entry[1]) + post_confidence = "\"confidence\": " + \ + str(entry[1]) post_end = " }" - - post_json = post_begin + post_timestamp + post_lat + post_lon + post_soundscape_id + post_soundscape_start_time + post_soundscape_end_time + post_commonName + post_scientificName + post_algorithm + post_confidence + post_end + + post_json = post_begin + post_timestamp + post_lat + post_lon + post_soundscape_id + post_soundscape_start_time + \ + post_soundscape_end_time + post_commonName + post_scientificName + \ + post_algorithm + post_confidence + post_end print(post_json) - response = requests.post(detection_url, json=json.loads(post_json)) - print("Detection POST Response Status - ", response.status_code) - except: + response = requests.post( + detection_url, json=json.loads(post_json)) + print( + "Detection POST Response Status - ", response.status_code) + except BaseException: print("Cannot POST right now") conn.send(myReturn.encode(FORMAT)) - #time.sleep(3) + # time.sleep(3) + + conn.close() - conn.close() def start(): # Load model From a7e4ef3777cbc85ef6c4943c0a00fa30e5670864 Mon Sep 17 00:00:00 2001 From: Jake Herbst Date: Wed, 11 May 2022 08:07:51 -0400 Subject: [PATCH 07/17] Removing unused imports in privacy_server.py to satisfy flake8 linting --- scripts/privacy_server.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scripts/privacy_server.py b/scripts/privacy_server.py index 63ae524..5e7beb8 100755 --- a/scripts/privacy_server.py +++ b/scripts/privacy_server.py @@ -1,18 +1,14 @@ from pathlib import Path from tzlocal import get_localzone -import pytz -from time import sleep import datetime import sqlite3 import requests import json -from decimal import Decimal import time import math import numpy as np import librosa import operator -import argparse import socket import threading import os From 6a6826c59e6155b1092638b0fe501f4b770896e5 Mon Sep 17 00:00:00 2001 From: Jake Herbst Date: Wed, 11 May 2022 08:10:52 -0400 Subject: [PATCH 08/17] Correcting additional flake8 failures on privacy_server.py Corrected: E712 comparison to True should be 'if cond is True:' or 'if cond:' E265 block comment should start with '# ' --- scripts/privacy_server.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/privacy_server.py b/scripts/privacy_server.py index 5e7beb8..45084c9 100755 --- a/scripts/privacy_server.py +++ b/scripts/privacy_server.py @@ -228,7 +228,7 @@ def analyzeAudioData(chunks, lat, lon, week, sensitivity, overlap,): # Save result and timestamp pred_end = pred_start + 3.0 - if HUMAN_DETECTED == True: + if HUMAN_DETECTED is True: p = [('Human_Human', 0.0)] * 10 print("HUMAN DETECTED!!!", p) @@ -277,7 +277,7 @@ def handle_client(conn, addr): if msg == DISCONNECT_MESSAGE: connected = False else: - #print(f"[{addr}] {msg}") + # print(f"[{addr}] {msg}") args = type('', (), {})() @@ -335,7 +335,7 @@ def handle_client(conn, addr): audioData = readAudioData(args.i, args.overlap) # Get Date/Time from filename in case Pi gets behind - #now = datetime.now() + # now = datetime.now() full_file_name = args.i print('FULL FILENAME: -' + full_file_name + '-') file_name = Path(full_file_name).stem @@ -344,8 +344,8 @@ def handle_client(conn, addr): date_time_str = file_date + ' ' + file_time date_time_obj = datetime.datetime.strptime( date_time_str, '%Y-%m-%d %H:%M:%S') - #print('Date:', date_time_obj.date()) - #print('Time:', date_time_obj.time()) + # print('Date:', date_time_obj.date()) + # print('Time:', date_time_obj.time()) print('Date-time:', date_time_obj) now = date_time_obj current_date = now.strftime("%Y-%m-%d") From d5866f2b5245c4464c7b2e7dd254c2cc91a8c17d Mon Sep 17 00:00:00 2001 From: Jake Herbst Date: Wed, 11 May 2022 08:12:55 -0400 Subject: [PATCH 09/17] Removing unused variable HUMAN_FLAG from privacy_server.py Flake8 error was: F841 local variable 'HUMAN_FLAG' is assigned to but never used --- scripts/privacy_server.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/privacy_server.py b/scripts/privacy_server.py index 45084c9..2dfa88f 100755 --- a/scripts/privacy_server.py +++ b/scripts/privacy_server.py @@ -180,7 +180,6 @@ def predict(sample, sensitivity): p_sorted[i] = (p_sorted[i][0], 0.0) if p_sorted[i][0] == 'Human_Human': print("HUMAN SCORE:", str(p_sorted[i])) - HUMAN_FLAG = True with open(userDir + '/BirdNET-Pi/HUMAN.txt', 'a') as rfile: rfile.write(str(datetime.datetime.now()) + str(p_sorted[i]) + '\n') From 9b12feea5dcbffce31ef342cbd9b85462c073c6d Mon Sep 17 00:00:00 2001 From: Jake Herbst Date: Wed, 11 May 2022 08:16:21 -0400 Subject: [PATCH 10/17] Added a flake8 configuration ini file to configure line length. Configures flake8 to use 128 character lengths for "E501 line too long" --- .flake8 | 3 +++ scripts/privacy_server.py | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 .flake8 diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..467c30e --- /dev/null +++ b/.flake8 @@ -0,0 +1,3 @@ +[flake8] +max-line-length = 128 + diff --git a/scripts/privacy_server.py b/scripts/privacy_server.py index 2dfa88f..fefbf81 100755 --- a/scripts/privacy_server.py +++ b/scripts/privacy_server.py @@ -504,7 +504,9 @@ def handle_client(conn, addr): str(entry[1]) post_end = " }" - post_json = post_begin + post_timestamp + post_lat + post_lon + post_soundscape_id + post_soundscape_start_time + \ + post_json = post_begin + \ + post_timestamp + post_lat + post_lon + \ + post_soundscape_id + post_soundscape_start_time + \ post_soundscape_end_time + post_commonName + post_scientificName + \ post_algorithm + post_confidence + post_end print(post_json) From 7ad6bbe93e00d25c1d938ca6886951f11fb223ab Mon Sep 17 00:00:00 2001 From: Jake Herbst Date: Wed, 11 May 2022 08:23:42 -0400 Subject: [PATCH 11/17] Updating script/plotly_streamlit.py to correct pep8 violations Used 'autopep8 --in-place --aggressive scripts/plotly_streamlit.py' as initial style fixes. --- scripts/plotly_streamlit.py | 134 ++++++++++++++++++------------------ 1 file changed, 68 insertions(+), 66 deletions(-) diff --git a/scripts/plotly_streamlit.py b/scripts/plotly_streamlit.py index b42906d..4275cb5 100755 --- a/scripts/plotly_streamlit.py +++ b/scripts/plotly_streamlit.py @@ -34,22 +34,22 @@ st.markdown(""" @st.cache(hash_funcs={Connection: id}) -def get_connection(path:str): - return sqlite3.connect(path,check_same_thread=False) +def get_connection(path: str): + return sqlite3.connect(path, check_same_thread=False) def get_data(conn: Connection): - df1=pd.read_sql("SELECT * FROM detections", con=conn) + df1 = pd.read_sql("SELECT * FROM detections", con=conn) return df1 + conn = get_connection(URI_SQLITE_DB) # Read in the cereal data # df = load_data() -df=get_data(conn) -df2=df.copy() -df2['DateTime']=pd.to_datetime(df2['Date'] + " " + df2['Time']) -df2=df2.set_index('DateTime') - +df = get_data(conn) +df2 = df.copy() +df2['DateTime'] = pd.to_datetime(df2['Date'] + " " + df2['Time']) +df2 = df2.set_index('DateTime') # Filter on date range @@ -59,115 +59,117 @@ df2=df2.set_index('DateTime') # Date as slider Start_Date = pd.to_datetime(df2.index.min()).date() -End_Date = pd.to_datetime(df2.index.max()).date() +End_Date = pd.to_datetime(df2.index.max()).date() Date_Slider = st.slider('Date Range', - min_value = Start_Date-timedelta(days=1), - max_value = End_Date, - value=(Start_Date, - End_Date) - ) + min_value=Start_Date - timedelta(days=1), + max_value=End_Date, + value=(Start_Date, + End_Date) + ) - -filt = (df2.index >= pd.Timestamp(Date_Slider[0])) & (df2.index <= pd.Timestamp(Date_Slider[1]+timedelta(days=1))) +filt = (df2.index >= pd.Timestamp(Date_Slider[0])) & (df2.index <= pd.Timestamp(Date_Slider[1] + timedelta(days=1))) df2 = df2[filt] -#Create species count for selected date range +# Create species count for selected date range -Specie_Count=df2['Com_Name'].value_counts() +Specie_Count = df2['Com_Name'].value_counts() -#Create species treemap +# Create species treemap # Create Hourly Crosstab -hourly=pd.crosstab(df2['Com_Name'],df2.index.hour, dropna=False) +hourly = pd.crosstab(df2['Com_Name'], df2.index.hour, dropna=False) # Filter on species species = list(hourly.index) -cols1,cols2= st.columns((1,1)) +cols1, cols2 = st.columns((1, 1)) top_N = cols1.slider( 'Select Number of Birds to Show', - min_value = 1, - value=min(10,len(Specie_Count)) - ) + min_value=1, + value=min(10, len(Specie_Count)) +) top_N_species = (df2['Com_Name'].value_counts()[:top_N]) -specie = cols2.selectbox('Which bird would you like to explore for the dates '+str(Date_Slider[0])+' to '+str(Date_Slider[1])+'?', species, - index=species.index(list(top_N_species.index)[0])) +specie = cols2.selectbox('Which bird would you like to explore for the dates ' + str(Date_Slider[0]) + ' to ' + str(Date_Slider[1]) + '?', species, + index=species.index(list(top_N_species.index)[0])) -font_size=15 +font_size = 15 -#specie filter -filt=df2['Com_Name']==specie +# specie filter +filt = df2['Com_Name'] == specie -df_counts=df2[filt].resample('D').count() +df_counts = df2[filt].resample('D').count() fig = make_subplots( - rows=3, cols =2, - specs= [[{"type":"xy","rowspan":3}, {"type":"polar","rowspan":2}], [{"rowspan":1}, {"rowspan":1} ], [None, {"type":"xy","rowspan":1}]], - subplot_titles=('Top '+ str(top_N) + ' Species in Date Range '+str(Date_Slider[0])+' to '+str(Date_Slider[1])+'', - 'Total Detect:'+str('{:,}'.format(sum(df_counts.Time)))+ - ' Confidence Max:'+str('{:.2f}%'.format(max(df2[df2['Com_Name']==specie]['Confidence'])*100))+ - ' '+' Median:'+str('{:.2f}%'.format(np.median(df2[df2['Com_Name']==specie]['Confidence'])*100)) - ) + rows=3, cols=2, + specs=[[{"type": "xy", "rowspan": 3}, {"type": "polar", "rowspan": 2}], [ + {"rowspan": 1}, {"rowspan": 1}], [None, {"type": "xy", "rowspan": 1}]], + subplot_titles=('Top ' + str(top_N) + ' Species in Date Range ' + str(Date_Slider[0]) + ' to ' + str(Date_Slider[1]) + '', + 'Total Detect:' + str('{:,}'.format(sum(df_counts.Time))) + + ' Confidence Max:' + str('{:.2f}%'.format(max(df2[df2['Com_Name'] == specie]['Confidence']) * 100)) + + ' ' + ' Median:' + + str('{:.2f}%'.format(np.median(df2[df2['Com_Name'] == specie]['Confidence']) * 100)) ) -fig.layout.annotations[1].update(x=0.7,y=0.25, font_size=15) +) +fig.layout.annotations[1].update(x=0.7, y=0.25, font_size=15) -#Plot seen species for selected date range and number of species -fig.add_trace(go.Bar(y=top_N_species.index, x=top_N_species, orientation='h'), row=1,col=1) +# Plot seen species for selected date range and number of species +fig.add_trace(go.Bar(y=top_N_species.index, x=top_N_species, orientation='h'), row=1, col=1) fig.update_layout( margin=dict(l=0, r=0, t=50, b=0), - yaxis={'categoryorder':'total ascending'}) + yaxis={'categoryorder': 'total ascending'}) # Set 360 degrees, 24 hours for polar plot theta = np.linspace(0.0, 360, 24, endpoint=False) -d=pd.DataFrame(np.zeros((23,1))).squeeze() +d = pd.DataFrame(np.zeros((23, 1))).squeeze() detections = hourly.loc[specie] -detections=(d+detections).fillna(0) -fig.add_trace(go.Barpolar(r = detections, theta=theta), row=1, col=2) +detections = (d + detections).fillna(0) +fig.add_trace(go.Barpolar(r=detections, theta=theta), row=1, col=2) fig.update_layout( autosize=False, - width = 1000, - height = 500, + width=1000, + height=500, showlegend=False, - polar = dict( - radialaxis = dict( - tickfont_size = font_size, - showticklabels = True, - hoverformat = "#%{theta}:
Popularity: %{percent}
%{r}" - ), - angularaxis = dict( - tickfont_size= font_size, - rotation = -90, - direction = 'clockwise', + polar=dict( + radialaxis=dict( + tickfont_size=font_size, + showticklabels=True, + hoverformat="#%{theta}:
Popularity: %{percent}
%{r}" + ), + angularaxis=dict( + tickfont_size=font_size, + rotation=-90, + direction='clockwise', tickmode='array', - tickvals=[0,15,35,45,60,75,90,105,120,135,150,165,180,195,210,225,240,255,270,285,300,315,330,345], - ticktext=['12am','1am','2am','3am','4am','5am', '6am','7am','8am','9am','10am','11am','12pm','1pm','2pm','3pm','4pm','5pm', '6pm','7pm','8pm','9pm','10pm','11pm'], - hoverformat = "#%{theta}:
Popularity: %{percent}
%{r}" + tickvals=[0, 15, 35, 45, 60, 75, 90, 105, 120, 135, 150, 165, + 180, 195, 210, 225, 240, 255, 270, 285, 300, 315, 330, 345], + ticktext=['12am', '1am', '2am', '3am', '4am', '5am', '6am', '7am', '8am', '9am', '10am', '11am', + '12pm', '1pm', '2pm', '3pm', '4pm', '5pm', '6pm', '7pm', '8pm', '9pm', '10pm', '11pm'], + hoverformat="#%{theta}:
Popularity: %{percent}
%{r}" ), - ), - ) + ), +) - -daily=pd.crosstab(df2['Com_Name'],df2.index.date, dropna=False) +daily = pd.crosstab(df2['Com_Name'], df2.index.date, dropna=False) fig.add_trace(go.Bar(x=daily.columns, y=daily.loc[specie]), row=3, col=2) # container=st.container() # config={'displayModelBar': False} -st.plotly_chart(fig, use_container_width=True) #, config=config) +st.plotly_chart(fig, use_container_width=True) # , config=config) # cols3,cols4=st.columns((1,1)) -# +# # extract_date=Date_Slider -# +# # audio_file = open('/home/*/BirdSongs/Extracted/By_Date/2022-03-22/Yellow-streaked_Greenbul/Yellow-streaked_Greenbul-77-2022-03-22-birdnet-15:04:28.mp3', 'rb') # audio_bytes = audio_file.read() # cols4.audio(audio_bytes, format='audio/mp3') From d707010843b097d8675d9a480dc23d7836bcbfec Mon Sep 17 00:00:00 2001 From: Jake Herbst Date: Wed, 11 May 2022 08:29:55 -0400 Subject: [PATCH 12/17] Correcting additional flake8 failures on plotly_streamlit.py Corrected: F401 imported but unused E501 line too long --- scripts/plotly_streamlit.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/scripts/plotly_streamlit.py b/scripts/plotly_streamlit.py index 4275cb5..1adea4f 100755 --- a/scripts/plotly_streamlit.py +++ b/scripts/plotly_streamlit.py @@ -4,8 +4,7 @@ import pandas as pd import numpy as np import plotly.graph_objects as go from plotly.subplots import make_subplots -from datetime import timedelta, datetime -from pathlib import Path +from datetime import timedelta import sqlite3 from sqlite3 import Connection @@ -93,8 +92,11 @@ top_N = cols1.slider( top_N_species = (df2['Com_Name'].value_counts()[:top_N]) -specie = cols2.selectbox('Which bird would you like to explore for the dates ' + str(Date_Slider[0]) + ' to ' + str(Date_Slider[1]) + '?', species, - index=species.index(list(top_N_species.index)[0])) +specie = cols2.selectbox( + 'Which bird would you like to explore for the dates ' + + str(Date_Slider[0]) + ' to ' + str(Date_Slider[1]) + '?', + species, + index=species.index(list(top_N_species.index)[0])) font_size = 15 @@ -109,12 +111,16 @@ fig = make_subplots( rows=3, cols=2, specs=[[{"type": "xy", "rowspan": 3}, {"type": "polar", "rowspan": 2}], [ {"rowspan": 1}, {"rowspan": 1}], [None, {"type": "xy", "rowspan": 1}]], - subplot_titles=('Top ' + str(top_N) + ' Species in Date Range ' + str(Date_Slider[0]) + ' to ' + str(Date_Slider[1]) + '', - 'Total Detect:' + str('{:,}'.format(sum(df_counts.Time))) + - ' Confidence Max:' + str('{:.2f}%'.format(max(df2[df2['Com_Name'] == specie]['Confidence']) * 100)) + - ' ' + ' Median:' + - str('{:.2f}%'.format(np.median(df2[df2['Com_Name'] == specie]['Confidence']) * 100)) - ) + subplot_titles=( + 'Top ' + str(top_N) + + ' Species in Date Range ' + str(Date_Slider[0]) + + ' to ' + str(Date_Slider[1]) + + '', + 'Total Detect:' + str('{:,}'.format(sum(df_counts.Time))) + + ' Confidence Max:' + str('{:.2f}%'.format(max(df2[df2['Com_Name'] == specie]['Confidence']) * 100)) + + ' ' + ' Median:' + + str('{:.2f}%'.format(np.median(df2[df2['Com_Name'] == specie]['Confidence']) * 100)) + ) ) fig.layout.annotations[1].update(x=0.7, y=0.25, font_size=15) @@ -170,6 +176,6 @@ st.plotly_chart(fig, use_container_width=True) # , config=config) # # extract_date=Date_Slider # -# audio_file = open('/home/*/BirdSongs/Extracted/By_Date/2022-03-22/Yellow-streaked_Greenbul/Yellow-streaked_Greenbul-77-2022-03-22-birdnet-15:04:28.mp3', 'rb') +# audio_file = open('/home/*/BirdSongs/Extracted/By_Date/2022-03-22/Yellow-streaked_Greenbul/Yellow-streaked_Greenbul-77-2022-03-22-birdnet-15:04:28.mp3', 'rb') # noqa: E501 # audio_bytes = audio_file.read() # cols4.audio(audio_bytes, format='audio/mp3') From 4211eb19c641f8ab8d0a1c0a6f98e33509483f51 Mon Sep 17 00:00:00 2001 From: Jake Herbst Date: Wed, 11 May 2022 08:32:21 -0400 Subject: [PATCH 13/17] Updating script/server.py to correct pep8 violations Used 'autopep8 --in-place --aggressive scripts/server.py' as initial style fixes. --- scripts/server.py | 177 +++++++++++++++++++++++++++++----------------- 1 file changed, 114 insertions(+), 63 deletions(-) diff --git a/scripts/server.py b/scripts/server.py index d025f32..f6f4287 100755 --- a/scripts/server.py +++ b/scripts/server.py @@ -1,4 +1,19 @@ -import socket +from pathlib import Path +from tzlocal import get_localzone +import pytz +from time import sleep +import datetime +import sqlite3 +import requests +import json +from decimal import Decimal +import time +import math +import numpy as np +import librosa +import operator +import argparse +import socket import threading import os os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' @@ -6,25 +21,9 @@ os.environ['CUDA_VISIBLE_DEVICES'] = '' try: import tflite_runtime.interpreter as tflite -except: +except BaseException: from tensorflow import lite as tflite -import argparse -import operator -import librosa -import numpy as np -import math -import time -from decimal import Decimal -import json -import requests -import sqlite3 -import datetime -from time import sleep -import pytz -from tzlocal import get_localzone -from pathlib import Path - HEADER = 64 PORT = 5050 @@ -36,10 +35,9 @@ DISCONNECT_MESSAGE = "!DISCONNECT" server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: server.bind(ADDR) -except: +except BaseException: print("Waiting on socket") time.sleep(5) - # Open most recent Configuration and grab DB_PWD as a python variable @@ -60,7 +58,7 @@ def loadModel(): # Load TFLite model and allocate tensors. modelpath = userDir + '/BirdNET-Pi/model/BirdNET_6K_GLOBAL_MODEL.tflite' - myinterpreter = tflite.Interpreter(model_path=modelpath,num_threads=2) + myinterpreter = tflite.Interpreter(model_path=modelpath, num_threads=2) myinterpreter.allocate_tensors() # Get input and output tensors. @@ -83,6 +81,7 @@ def loadModel(): return myinterpreter + def loadCustomSpeciesList(path): slist = [] @@ -93,6 +92,7 @@ def loadCustomSpeciesList(path): return slist + def splitSignal(sig, rate, overlap, seconds=3.0, minlen=1.5): # Split signal with overlap @@ -103,17 +103,18 @@ def splitSignal(sig, rate, overlap, seconds=3.0, minlen=1.5): # End of signal? if len(split) < int(minlen * rate): break - + # Signal chunk too short? Fill with zeros. if len(split) < int(rate * seconds): temp = np.zeros((int(rate * seconds))) temp[:len(split)] = split split = temp - + sig_splits.append(split) return sig_splits + def readAudioData(path, overlap, sample_rate=48000): print('READING AUDIO DATA...', end=' ', flush=True) @@ -128,11 +129,12 @@ def readAudioData(path, overlap, sample_rate=48000): return chunks + def convertMetadata(m): # Convert week to cosine if m[2] >= 1 and m[2] <= 48: - m[2] = math.cos(math.radians(m[2] * 7.5)) + 1 + m[2] = math.cos(math.radians(m[2] * 7.5)) + 1 else: m[2] = -1 @@ -145,9 +147,11 @@ def convertMetadata(m): return np.concatenate([m, mask]) + def custom_sigmoid(x, sensitivity=1.0): return 1 / (1.0 + np.exp(-sensitivity * x)) + def predict(sample, sensitivity): global INTERPRETER # Make a prediction @@ -173,6 +177,7 @@ def predict(sample, sensitivity): # Only return first the top ten results return p_sorted[:10] + def analyzeAudioData(chunks, lat, lon, week, sensitivity, overlap,): global INTERPRETER @@ -203,6 +208,7 @@ def analyzeAudioData(chunks, lat, lon, week, sensitivity, overlap,): return detections + def writeResultsToFile(detections, min_conf, path): print('WRITING RESULTS TO', path, '...', end=' ') @@ -211,12 +217,14 @@ def writeResultsToFile(detections, min_conf, path): rfile.write('Start (s);End (s);Scientific name;Common name;Confidence\n') for d in detections: for entry in detections[d]: - if entry[1] >= min_conf and ((entry[0] in INCLUDE_LIST or len(INCLUDE_LIST) == 0) and (entry[0] not in EXCLUDE_LIST or len(EXCLUDE_LIST) == 0) ): + if entry[1] >= min_conf and ((entry[0] in INCLUDE_LIST or len(INCLUDE_LIST) == 0) + and (entry[0] not in EXCLUDE_LIST or len(EXCLUDE_LIST) == 0)): rfile.write(d + ';' + entry[0].replace('_', ';') + ';' + str(entry[1]) + '\n') rcnt += 1 print('DONE! WROTE', rcnt, 'RESULTS.') return + def handle_client(conn, addr): global INCLUDE_LIST global EXCLUDE_LIST @@ -232,9 +240,9 @@ def handle_client(conn, addr): connected = False else: #print(f"[{addr}] {msg}") - + args = type('', (), {})() - + args.i = '' args.o = '' args.birdweather_id = '99999' @@ -245,8 +253,7 @@ def handle_client(conn, addr): args.sensitivity = 1.25 args.min_conf = 0.70 args.lat = -1 - args.lon = -1 - + args.lon = -1 for line in msg.split('||'): inputvars = line.split('=') @@ -273,14 +280,12 @@ def handle_client(conn, addr): elif inputvars[0] == 'lon': args.lon = float(inputvars[1]) - - # Load custom species lists - INCLUDED and EXCLUDED if not args.include_list == 'null': INCLUDE_LIST = loadCustomSpeciesList(args.include_list) else: INCLUDE_LIST = [] - + if not args.exclude_list == 'null': EXCLUDE_LIST = loadCustomSpeciesList(args.exclude_list) else: @@ -307,7 +312,7 @@ def handle_client(conn, addr): current_date = now.strftime("%Y-%m-%d") current_time = now.strftime("%H:%M:%S") current_iso8601 = now.astimezone(get_localzone()).isoformat() - + week_number = int(now.strftime("%V")) week = max(1, min(week_number, 48)) @@ -319,32 +324,33 @@ def handle_client(conn, addr): # Write detections to output file min_conf = max(0.01, min(args.min_conf, 0.99)) writeResultsToFile(detections, min_conf, args.o) - - ############################################################################### - ############################################################################### - + + ############################################################################### + ############################################################################### + soundscape_uploaded = False # Write detections to Database myReturn = '' for i in detections: - myReturn += str(i) + '-' + str(detections[i][0]) + '\n' - - + myReturn += str(i) + '-' + str(detections[i][0]) + '\n' + with open(userDir + '/BirdNET-Pi/BirdDB.txt', 'a') as rfile: for d in detections: for entry in detections[d]: - if entry[1] >= min_conf and ((entry[0] in INCLUDE_LIST or len(INCLUDE_LIST) == 0) and (entry[0] not in EXCLUDE_LIST or len(EXCLUDE_LIST) == 0) ): - rfile.write(str(current_date) + ';' + str(current_time) + ';' + entry[0].replace('_', ';') + ';' \ - + str(entry[1]) +";" + str(args.lat) + ';' + str(args.lon) + ';' + str(min_conf) + ';' + str(week) + ';' \ - + str(args.sensitivity) +';' + str(args.overlap) + '\n') - + if entry[1] >= min_conf and ((entry[0] in INCLUDE_LIST or len(INCLUDE_LIST) == 0) and ( + entry[0] not in EXCLUDE_LIST or len(EXCLUDE_LIST) == 0)): + rfile.write(str(current_date) + ';' + str(current_time) + ';' + entry[0].replace('_', ';') + ';' + + str(entry[1]) + ";" + str(args.lat) + ';' + str(args.lon) + + ';' + str(min_conf) + ';' + str(week) + ';' + + str(args.sensitivity) + ';' + str(args.overlap) + '\n') + Date = str(current_date) Time = str(current_time) species = entry[0] - Sci_Name,Com_Name = species.split('_') + Sci_Name, Com_Name = species.split('_') score = entry[1] - Confidence = str(round(score*100)) + Confidence = str(round(score * 100)) Lat = str(args.lat) Lon = str(args.lon) Cutoff = str(args.min_conf) @@ -353,39 +359,82 @@ def handle_client(conn, addr): Overlap = str(args.overlap) Com_Name = Com_Name.replace("'", "") File_Name = Com_Name.replace(" ", "_") + '-' + Confidence + '-' + \ - Date.replace("/", "-") + '-birdnet-' + Time + audiofmt + Date.replace("/", "-") + '-birdnet-' + Time + audiofmt - #Connect to SQLite Database + # Connect to SQLite Database for attempt_number in range(3): - try: + try: con = sqlite3.connect(userDir + '/BirdNET-Pi/scripts/birds.db') cur = con.cursor() - cur.execute("INSERT INTO detections VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (Date, Time, Sci_Name, Com_Name, str(score), Lat, Lon, Cutoff, Week, Sens, Overlap, File_Name)) + cur.execute( + "INSERT INTO detections VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + (Date, + Time, + Sci_Name, + Com_Name, + str(score), + Lat, + Lon, + Cutoff, + Week, + Sens, + Overlap, + File_Name)) con.commit() con.close() break - except: + except BaseException: print("Database busy") time.sleep(2) - print(str(current_date) + ';' + str(current_time) + ';' + entry[0].replace('_', ';') + ';' + str(entry[1]) + ';' + str(args.lat) + ';' + str(args.lon) + ';' + str(min_conf) + ';' + str(week) + ';' + str(args.sensitivity) +';' + str(args.overlap) + Com_Name.replace(" ", "_") + '-' + str(score) + '-' + str(current_date) + '-birdnet-' + str(current_time) + audiofmt + '\n') + print(str(current_date) + + ';' + + str(current_time) + + ';' + + entry[0].replace('_', ';') + + ';' + + str(entry[1]) + + ';' + + str(args.lat) + + ';' + + str(args.lon) + + ';' + + str(min_conf) + + ';' + + str(week) + + ';' + + str(args.sensitivity) + + ';' + + str(args.overlap) + + Com_Name.replace(" ", "_") + + '-' + + str(score) + + '-' + + str(current_date) + + '-birdnet-' + + str(current_time) + + audiofmt + + '\n') if birdweather_id != "99999": try: if soundscape_uploaded is False: # POST soundscape to server - soundscape_url = "https://app.birdweather.com/api/v1/stations/" + birdweather_id + "/soundscapes" + "?timestamp=" + current_iso8601 - + soundscape_url = "https://app.birdweather.com/api/v1/stations/" + \ + birdweather_id + "/soundscapes" + "?timestamp=" + current_iso8601 + with open(args.i, 'rb') as f: wav_data = f.read() - response = requests.post(url=soundscape_url, data=wav_data, headers={'Content-Type': 'application/octet-stream'}) + response = requests.post( + url=soundscape_url, data=wav_data, headers={ + 'Content-Type': 'application/octet-stream'}) print("Soundscape POST Response Status - ", response.status_code) sdata = response.json() soundscape_id = sdata['soundscape']['id'] soundscape_uploaded = True - + # POST detection to server detection_url = "https://app.birdweather.com/api/v1/stations/" + birdweather_id + "/detections" start_time = d.split(';')[0] @@ -393,7 +442,7 @@ def handle_client(conn, addr): post_begin = "{ " now_p_start = now + datetime.timedelta(seconds=float(start_time)) current_iso8601 = now_p_start.astimezone(get_localzone()).isoformat() - post_timestamp = "\"timestamp\": \"" + current_iso8601 + "\"," + post_timestamp = "\"timestamp\": \"" + current_iso8601 + "\"," post_lat = "\"lat\": " + str(args.lat) + "," post_lon = "\"lon\": " + str(args.lon) + "," post_soundscape_id = "\"soundscapeId\": " + str(soundscape_id) + "," @@ -404,18 +453,20 @@ def handle_client(conn, addr): post_algorithm = "\"algorithm\": " + "\"alpha\"" + "," post_confidence = "\"confidence\": " + str(entry[1]) post_end = " }" - - post_json = post_begin + post_timestamp + post_lat + post_lon + post_soundscape_id + post_soundscape_start_time + post_soundscape_end_time + post_commonName + post_scientificName + post_algorithm + post_confidence + post_end + + post_json = post_begin + post_timestamp + post_lat + post_lon + post_soundscape_id + post_soundscape_start_time + \ + post_soundscape_end_time + post_commonName + post_scientificName + post_algorithm + post_confidence + post_end print(post_json) response = requests.post(detection_url, json=json.loads(post_json)) print("Detection POST Response Status - ", response.status_code) - except: + except BaseException: print("Cannot POST right now") conn.send(myReturn.encode(FORMAT)) - #time.sleep(3) + # time.sleep(3) + + conn.close() - conn.close() def start(): # Load model From 5020f4dd060bd5074cb55af53b3fbf9314ab28ce Mon Sep 17 00:00:00 2001 From: Jake Herbst Date: Wed, 11 May 2022 08:36:15 -0400 Subject: [PATCH 14/17] Correcting additional flake8 failures on server.py Corrected: F401 imported but unused E265 block comment should start with '# ' E501 line too long --- scripts/server.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/scripts/server.py b/scripts/server.py index f6f4287..dbb27f1 100755 --- a/scripts/server.py +++ b/scripts/server.py @@ -1,18 +1,14 @@ from pathlib import Path from tzlocal import get_localzone -import pytz -from time import sleep import datetime import sqlite3 import requests import json -from decimal import Decimal import time import math import numpy as np import librosa import operator -import argparse import socket import threading import os @@ -239,7 +235,7 @@ def handle_client(conn, addr): if msg == DISCONNECT_MESSAGE: connected = False else: - #print(f"[{addr}] {msg}") + # print(f"[{addr}] {msg}") args = type('', (), {})() @@ -297,7 +293,7 @@ def handle_client(conn, addr): audioData = readAudioData(args.i, args.overlap) # Get Date/Time from filename in case Pi gets behind - #now = datetime.now() + # now = datetime.now() full_file_name = args.i print('FULL FILENAME: -' + full_file_name + '-') file_name = Path(full_file_name).stem @@ -305,8 +301,8 @@ def handle_client(conn, addr): file_time = file_name.split('-birdnet-')[1] date_time_str = file_date + ' ' + file_time date_time_obj = datetime.datetime.strptime(date_time_str, '%Y-%m-%d %H:%M:%S') - #print('Date:', date_time_obj.date()) - #print('Time:', date_time_obj.time()) + # print('Date:', date_time_obj.date()) + # print('Time:', date_time_obj.time()) print('Date-time:', date_time_obj) now = date_time_obj current_date = now.strftime("%Y-%m-%d") @@ -436,7 +432,8 @@ def handle_client(conn, addr): soundscape_uploaded = True # POST detection to server - detection_url = "https://app.birdweather.com/api/v1/stations/" + birdweather_id + "/detections" + detection_url = "https://app.birdweather.com/api/v1/stations/" + \ + birdweather_id + "/detections" start_time = d.split(';')[0] end_time = d.split(';')[1] post_begin = "{ " @@ -454,8 +451,11 @@ def handle_client(conn, addr): post_confidence = "\"confidence\": " + str(entry[1]) post_end = " }" - post_json = post_begin + post_timestamp + post_lat + post_lon + post_soundscape_id + post_soundscape_start_time + \ - post_soundscape_end_time + post_commonName + post_scientificName + post_algorithm + post_confidence + post_end + post_json = post_begin + \ + post_timestamp + post_lat + post_lon + \ + post_soundscape_id + post_soundscape_start_time + \ + post_soundscape_end_time + post_commonName + post_scientificName + \ + post_algorithm + post_confidence + post_end print(post_json) response = requests.post(detection_url, json=json.loads(post_json)) print("Detection POST Response Status - ", response.status_code) From 6d71ca9c2a237c5f6bc9a8c8e48eb8cb55bd4f55 Mon Sep 17 00:00:00 2001 From: Jake Herbst Date: Wed, 11 May 2022 08:39:09 -0400 Subject: [PATCH 15/17] Removing exclude list from Flake8 now that all python scripts are corrected --- .github/workflows/ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5764bb8..d2f684a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,5 +21,3 @@ jobs: - name: Run Flake8 Lint uses: py-actions/flake8@v2 - with: - exclude: "./scripts/daily_plot.py,./scripts/server.py,./scripts/plotly_streamlit.py,./scripts/privacy_server.py" From 2dd2b40ca5888d9ef6b1e054abc7e690ff724938 Mon Sep 17 00:00:00 2001 From: Jake Herbst Date: Wed, 11 May 2022 08:48:37 -0400 Subject: [PATCH 16/17] Updating script/server.py to correct pep8 violations Used 'autopep8 --in-place --aggressive scripts/server.py' as initial style fixes. --- scripts/server.py | 42 +++++++++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/scripts/server.py b/scripts/server.py index 715b349..f383b13 100755 --- a/scripts/server.py +++ b/scripts/server.py @@ -43,7 +43,8 @@ userDir = os.path.expanduser('~') with open(userDir + '/BirdNET-Pi/scripts/thisrun.txt', 'r') as f: this_run = f.readlines() audiofmt = "." + str(str(str([i for i in this_run if i.startswith('AUDIOFMT')]).split('=')[1]).split('\\')[0]) - priv_thresh = float("." + str(str(str([i for i in this_run if i.startswith('PRIVACY_THRESHOLD')]).split('=')[1]).split('\\')[0]))/10 + priv_thresh = float( + "." + str(str(str([i for i in this_run if i.startswith('PRIVACY_THRESHOLD')]).split('=')[1]).split('\\')[0])) / 10 def loadModel(): @@ -173,12 +174,12 @@ def predict(sample, sensitivity): # # # Remove species that are on blacklist - human_cutoff = max(10,int(len(p_sorted)*priv_thresh)) + human_cutoff = max(10, int(len(p_sorted) * priv_thresh)) for i in range(min(10, len(p_sorted))): - if p_sorted[i][0]=='Human_Human': + if p_sorted[i][0] == 'Human_Human': with open(userDir + '/BirdNET-Pi/HUMAN.txt', 'a') as rfile: - rfile.write(str(datetime.datetime.now())+str(p_sorted[i])+ ' ' + str(human_cutoff)+ '\n') + rfile.write(str(datetime.datetime.now()) + str(p_sorted[i]) + ' ' + str(human_cutoff) + '\n') return p_sorted[:human_cutoff] @@ -204,19 +205,19 @@ def analyzeAudioData(chunks, lat, lon, week, sensitivity, overlap,): # Make prediction p = predict([sig, mdata], sensitivity) # print("PPPPP",p) - HUMAN_DETECTED=False + HUMAN_DETECTED = False - #Catch if Human is recognized + # Catch if Human is recognized for x in range(len(p)): if "Human" in p[x][0]: - HUMAN_DETECTED=True + HUMAN_DETECTED = True # Save result and timestamp pred_end = pred_start + 3.0 - #If human detected set all detections to human to make sure voices are not saved + # If human detected set all detections to human to make sure voices are not saved if HUMAN_DETECTED == True: - p=[('Human_Human',0.0)]*10 + p = [('Human_Human', 0.0)] * 10 detections[str(pred_start) + ';' + str(pred_end)] = p @@ -226,12 +227,15 @@ def analyzeAudioData(chunks, lat, lon, week, sensitivity, overlap,): # print('DETECTIONS:::::',detections) return detections -def sendAppriseNotifications(species,confidence): + +def sendAppriseNotifications(species, confidence): if os.path.exists(userDir + '/BirdNET-Pi/apprise.txt') and os.path.getsize(userDir + '/BirdNET-Pi/apprise.txt') > 0: with open(userDir + '/BirdNET-Pi/scripts/thisrun.txt', 'r') as f: this_run = f.readlines() - title = str(str(str([i for i in this_run if i.startswith('APPRISE_NOTIFICATION_TITLE')]).split('=')[1]).split('\\')[0]).replace('"', '') - body = str(str(str([i for i in this_run if i.startswith('APPRISE_NOTIFICATION_BODY')]).split('=')[1]).split('\\')[0]).replace('"', '') + title = str(str(str([i for i in this_run if i.startswith('APPRISE_NOTIFICATION_TITLE')] + ).split('=')[1]).split('\\')[0]).replace('"', '') + body = str(str(str([i for i in this_run if i.startswith('APPRISE_NOTIFICATION_BODY')] + ).split('=')[1]).split('\\')[0]).replace('"', '') if str(str(str([i for i in this_run if i.startswith('APPRISE_NOTIFY_EACH_DETECTION')]).split('=')[1]).split('\\')[0]) == "1": @@ -241,10 +245,17 @@ def sendAppriseNotifications(species,confidence): apobj.add(config) apobj.notify( - body=body.replace("$sciname",species.split("_")[0]).replace("$comname",species.split("_")[1]).replace("$confidence",confidence), + body=body.replace( + "$sciname", + species.split("_")[0]).replace( + "$comname", + species.split("_")[1]).replace( + "$confidence", + confidence), title=title, ) + def writeResultsToFile(detections, min_conf, path): print('WRITING RESULTS TO', path, '...', end=' ') @@ -253,8 +264,9 @@ def writeResultsToFile(detections, min_conf, path): rfile.write('Start (s);End (s);Scientific name;Common name;Confidence\n') for d in detections: for entry in detections[d]: - if entry[1] >= min_conf and ((entry[0] in INCLUDE_LIST or len(INCLUDE_LIST) == 0) and (entry[0] not in EXCLUDE_LIST or len(EXCLUDE_LIST) == 0) ): - sendAppriseNotifications(str(entry[0]),str(entry[1])); + if entry[1] >= min_conf and ((entry[0] in INCLUDE_LIST or len(INCLUDE_LIST) == 0) + and (entry[0] not in EXCLUDE_LIST or len(EXCLUDE_LIST) == 0)): + sendAppriseNotifications(str(entry[0]), str(entry[1])) rfile.write(d + ';' + entry[0].replace('_', ';') + ';' + str(entry[1]) + '\n') rcnt += 1 print('DONE! WROTE', rcnt, 'RESULTS.') From 264380a37f76064c10ba8c633c70502bc9b2b570 Mon Sep 17 00:00:00 2001 From: Jake Herbst Date: Wed, 11 May 2022 08:52:01 -0400 Subject: [PATCH 17/17] Fixing flake8 errors in script/server.py F401 'argparse' imported but unused F821 undefined name 'os' F821 undefined name 'socket' F821 undefined name 'threading' E712 comparison to True should be 'if cond is True:' or 'if cond:' --- scripts/server.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/scripts/server.py b/scripts/server.py index f383b13..42c7868 100755 --- a/scripts/server.py +++ b/scripts/server.py @@ -1,16 +1,15 @@ -import argparse +import os +import socket +import threading import operator import librosa import numpy as np import math import time -from decimal import Decimal import json import requests import sqlite3 import datetime -from time import sleep -import pytz from tzlocal import get_localzone from pathlib import Path import apprise @@ -216,7 +215,7 @@ def analyzeAudioData(chunks, lat, lon, week, sensitivity, overlap,): pred_end = pred_start + 3.0 # If human detected set all detections to human to make sure voices are not saved - if HUMAN_DETECTED == True: + if HUMAN_DETECTED is True: p = [('Human_Human', 0.0)] * 10 detections[str(pred_start) + ';' + str(pred_end)] = p @@ -237,7 +236,7 @@ def sendAppriseNotifications(species, confidence): body = str(str(str([i for i in this_run if i.startswith('APPRISE_NOTIFICATION_BODY')] ).split('=')[1]).split('\\')[0]).replace('"', '') - if str(str(str([i for i in this_run if i.startswith('APPRISE_NOTIFY_EACH_DETECTION')]).split('=')[1]).split('\\')[0]) == "1": + if str(str(str([i for i in this_run if i.startswith('APPRISE_NOTIFY_EACH_DETECTION')]).split('=')[1]).split('\\')[0]) == "1": # noqa E501 apobj = apprise.Apprise() config = apprise.AppriseConfig()