Merge pull request #308 from jmherbst/python-lint

Python linting

.flake8

@@ -0,0 +1,2 @@
+[flake8]
+max-line-length = 160

.github/workflows/python-ci.yml

@@ -13,9 +13,6 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v3
-        with:
-          # This is a bit heavy handed, but we need `origin/main` to get a ref to diff against
-          fetch-depth: 0
 
       - name: Setup Python
         uses: actions/setup-python@v3
@@ -29,5 +26,4 @@ jobs:
 
       - name: Run Flake8 Lint
         run: |
-          DIFF="$(git --no-pager diff -u $(git merge-base HEAD origin/main) -- '**/*.py')"
+          flake8
-          echo "$DIFF" | flake8 --diff

scripts/analyze.py

@@ -11,6 +11,7 @@ ADDR = (SERVER, PORT)
 client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
 client.connect(ADDR)
 
+
 def send(msg):
     message = msg.encode(FORMAT)
     msg_length = len(message)
@@ -20,6 +21,7 @@ def send(msg):
     client.send(message)
     print(client.recv(2048).decode(FORMAT))
 
+
 def main():
 
     global INCLUDE_LIST
@@ -28,16 +30,52 @@ def main():
     # Parse passed arguments
     parser = argparse.ArgumentParser()
     parser.add_argument('--i', help='Path to input file.')
-    parser.add_argument('--o', default='result.csv', help='Path to output file. Defaults to result.csv.')
+    parser.add_argument(
-    parser.add_argument('--lat', type=float, default=-1, help='Recording location latitude. Set -1 to ignore.')
+        '--o',
-    parser.add_argument('--lon', type=float, default=-1, help='Recording location longitude. Set -1 to ignore.')
+        default='result.csv',
-    parser.add_argument('--week', type=int, default=-1, help='Week of the year when the recording was made. Values in [1, 48] (4 weeks per month). Set -1 to ignore.')
+        help='Path to output file. Defaults to result.csv.')
-    parser.add_argument('--overlap', type=float, default=0.0, help='Overlap in seconds between extracted spectrograms. Values in [0.0, 2.9]. Defaults tp 0.0.')
+    parser.add_argument(
-    parser.add_argument('--sensitivity', type=float, default=1.0, help='Detection sensitivity; Higher values result in higher sensitivity. Values in [0.5, 1.5]. Defaults to 1.0.')
+        '--lat',
-    parser.add_argument('--min_conf', type=float, default=0.1, help='Minimum confidence threshold. Values in [0.01, 0.99]. Defaults to 0.1.')   
+        type=float,
-    parser.add_argument('--include_list', default='null', help='Path to text file containing a list of included species. Not used if not provided.')
+        default=-1,
-    parser.add_argument('--exclude_list', default='null', help='Path to text file containing a list of excluded species. Not used if not provided.')
+        help='Recording location latitude. Set -1 to ignore.')
-    parser.add_argument('--birdweather_id', default='99999', help='Private Station ID for BirdWeather.')    
+    parser.add_argument(
+        '--lon',
+        type=float,
+        default=-1,
+        help='Recording location longitude. Set -1 to ignore.')
+    parser.add_argument(
+        '--week',
+        type=int,
+        default=-1,
+        help='Week of the year when the recording was made. Values in [1, 48] (4 weeks per month). Set -1 to ignore.')
+    parser.add_argument(
+        '--overlap',
+        type=float,
+        default=0.0,
+        help='Overlap in seconds between extracted spectrograms. Values in [0.0, 2.9]. Defaults tp 0.0.')
+    parser.add_argument(
+        '--sensitivity',
+        type=float,
+        default=1.0,
+        help='Detection sensitivity; Higher values result in higher sensitivity. Values in [0.5, 1.5]. Defaults to 1.0.')
+    parser.add_argument(
+        '--min_conf',
+        type=float,
+        default=0.1,
+        help='Minimum confidence threshold. Values in [0.01, 0.99]. Defaults to 0.1.')
+    parser.add_argument(
+        '--include_list',
+        default='null',
+        help='Path to text file containing a list of included species. Not used if not provided.')
+    parser.add_argument(
+        '--exclude_list',
+        default='null',
+        help='Path to text file containing a list of excluded species. Not used if not provided.')
+    parser.add_argument(
+        '--birdweather_id',
+        default='99999',
+        help='Private Station ID for BirdWeather.')
 
     args = parser.parse_args()
 
@@ -73,10 +111,13 @@ def main():
 ###############################################################################
 ###############################################################################
 
+
 if __name__ == '__main__':
 
     main()
 
     # Example calls
     # python3 analyze.py --i 'example/XC558716 - Soundscape.mp3' --lat 35.4244 --lon -120.7463 --week 18
-    # python3 analyze.py --i 'example/XC563936 - Soundscape.mp3' --lat 47.6766 --lon -122.294 --week 11 --overlap 1.5 --min_conf 0.25 --sensitivity 1.25 --custom_list 'example/custom_species_list.txt'
+    # python3 analyze.py --i 'example/XC563936 - Soundscape.mp3' --lat 47.6766
+    # --lon -122.294 --week 11 --overlap 1.5 --min_conf 0.25 --sensitivity
+    # 1.25 --custom_list 'example/custom_species_list.txt'

scripts/daily_plot.py

@@ -1,6 +1,5 @@
 import sqlite3
 import os
-import configparser
 import pandas as pd
 import seaborn as sns
 import matplotlib.pyplot as plt
@@ -72,8 +71,6 @@ colors = plt.cm.Greens(norm(confmax))
 plot = sns.countplot(y='Com_Name', data=df_plt_top10_today, palette=colors, order=freq_order, ax=axs[0])
 
 
-
-
 # Try plot grid lines between bars - problem at the moment plots grid lines on bars - want between bars
 z = plot.get_ymajorticklabels()
 plot.set_yticklabels(['\n'.join(textwrap.wrap(ticklabel.get_text(), 15)) for ticklabel in plot.get_yticklabels()], fontsize=10)
@@ -94,7 +91,20 @@ heat_frame = pd.DataFrame(data=0, index=heat.index, columns = hours_in_day)
 heat = (heat + heat_frame).fillna(0)
 
 # Generatie heatmap plot
-plot = sns.heatmap(heat, norm=LogNorm(),  annot=True,  annot_kws={"fontsize":7}, fmt="g", cmap = pal , square = False, cbar=False, linewidths = 0.5, linecolor = "Grey", ax=axs[1], yticklabels = False)
+plot = sns.heatmap(
+    heat,
+    norm=LogNorm(),
+    annot=True,
+    annot_kws={
+        "fontsize": 7},
+    fmt="g",
+    cmap=pal,
+    square=False,
+    cbar=False,
+    linewidths=0.5,
+    linecolor="Grey",
+    ax=axs[1],
+    yticklabels=False)
 plot.set_xticklabels(plot.get_xticklabels(), rotation=0, size=7)
 
 # Set heatmap border
@@ -142,8 +152,6 @@ colors = plt.cm.Reds(norm(confmax))
 plot = sns.countplot(y='Com_Name', data=df_plt_Bot10_today, palette=colors, order=freq_order, ax=axs[0])
 
 
-
-
 # Try plot grid lines between bars - problem at the moment plots grid lines on bars - want between bars
 z = plot.get_ymajorticklabels()
 plot.set_yticklabels(['\n'.join(textwrap.wrap(ticklabel.get_text(), 15)) for ticklabel in plot.get_yticklabels()], fontsize=10)
@@ -163,7 +171,20 @@ heat_frame = pd.DataFrame(data=0, index=heat.index, columns = hours_in_day)
 heat = (heat + heat_frame).fillna(0)
 
 # Generatie heatmap plot
-plot = sns.heatmap(heat, norm=LogNorm(),  annot=True, fmt="g",  annot_kws={"fontsize":7}, cmap = pal , square = False, cbar=False, linewidths = 0.5, linecolor = "Grey", ax=axs[1], yticklabels = False)
+plot = sns.heatmap(
+    heat,
+    norm=LogNorm(),
+    annot=True,
+    fmt="g",
+    annot_kws={
+        "fontsize": 7},
+    cmap=pal,
+    square=False,
+    cbar=False,
+    linewidths=0.5,
+    linecolor="Grey",
+    ax=axs[1],
+    yticklabels=False)
 plot.set_xticklabels(plot.get_xticklabels(), rotation=0, size=7)
 
 # Set heatmap border

scripts/plotly_streamlit.py

@@ -4,8 +4,7 @@ import pandas as pd
 import numpy as np
 import plotly.graph_objects as go
 from plotly.subplots import make_subplots
-from datetime import timedelta, datetime
+from datetime import timedelta
-from pathlib import Path
 import sqlite3
 from sqlite3 import Connection
 import plotly.express as px
@@ -43,6 +42,7 @@ def get_data(conn: Connection):
     df1 = pd.read_sql("SELECT * FROM detections", con=conn)
     return df1
 
+
 conn = get_connection(URI_SQLITE_DB)
 # Read in the cereal data
 # df = load_data()
@@ -52,7 +52,6 @@ df2['DateTime']=pd.to_datetime(df2['Date'] + " " + df2['Time'])
 df2 = df2.set_index('DateTime')
 
 
-
 # Filter on date range
 # Date as calendars
 # Start_Date = pd.to_datetime(st.sidebar.date_input('Which date do you want to start?', value = df2.index.min()))
@@ -70,14 +69,22 @@ Date_Slider = cols1.slider('Date Range',
                            )
 
 
-
 filt = (df2.index >= pd.Timestamp(Date_Slider[0])) & (df2.index <= pd.Timestamp(Date_Slider[1] + timedelta(days=1)))
 df2 = df2[filt]
 
 st.write('<style>div.row-widget.stRadio > div{flex-direction:row;justify-content: left;} </style>', unsafe_allow_html=True)
 st.write('<style>div.st-bf{flex-direction:column;} div.st-ag{font-weight:bold;padding-left:2px;}</style>', unsafe_allow_html=True)
 
-resample_sel=cols2.radio("Select Resample Resolution - To downsample and make run faster select longer period, Daily provides a view on detections at 15 min intervals through the day", ('1 minute', '5 minutes', '10 minutes', 'Hourly', 'Daily'))
+resample_sel = cols2.radio(
+    '''
+    Select Resample Resolution - To downsample and make run faster select longer period,
+    Daily provides a view on detections at 15 min intervals through the day
+    ''',
+    ('1 minute',
+     '5 minutes',
+     '10 minutes',
+     'Hourly',
+     'Daily'))
 
 resample_times = {'1 minute': '1min',
                   '5 minutes': '5min',
@@ -124,13 +131,20 @@ filt=df2['Com_Name']==specie
 df_counts = sum(df5 == specie)
 
 
-
-
 if resample_time != '1D':
     fig = make_subplots(
         rows=3, cols=2,
         specs=[[{"type": "xy", "rowspan": 3}, {"type": "polar", "rowspan": 2}], [{"rowspan": 1}, {"rowspan": 1}], [None, {"type": "xy", "rowspan": 1}]],
-                    subplot_titles=('<b>Top '+ str(top_N) + ' Species in Date Range '+str(Date_Slider[0])+' to '+str(Date_Slider[1])+' for '+str(resample_sel)+' sampling interval.'+'</b>',
+        subplot_titles=('<b>Top ' +
+                        str(top_N) +
+                        ' Species in Date Range ' +
+                        str(Date_Slider[0]) +
+                        ' to ' +
+                        str(Date_Slider[1]) +
+                        ' for ' +
+                        str(resample_sel) +
+                        ' sampling interval.' +
+                        '</b>',
                         'Total Detect:' + str('{:,}'.format(df_counts)) +
                         '   Confidence Max:' + str('{:.2f}%'.format(max(df2[df2['Com_Name'] == specie]['Confidence']) * 100)) +
                         '   ' + '   Median:' + str('{:.2f}%'.format(np.median(df2[df2['Com_Name'] == specie]['Confidence']) * 100))
@@ -145,7 +159,6 @@ if resample_time != '1D':
         margin=dict(l=0, r=0, t=50, b=0),
         yaxis={'categoryorder': 'total ascending'})
 
-
     # Set 360 degrees, 24 hours for polar plot
     theta = np.linspace(0.0, 360, 24, endpoint=False)
 
@@ -154,9 +167,6 @@ if resample_time != '1D':
 
     detections2 = pd.crosstab(df3, df3.index.hour)
 
-
-
-
     d = pd.DataFrame(np.zeros((23, 1))).squeeze()
     detections = hourly.loc[specie]
     detections = (d + detections).fillna(0)
@@ -178,14 +188,13 @@ if resample_time != '1D':
                 direction='clockwise',
                 tickmode='array',
                 tickvals=[0, 15, 35, 45, 60, 75, 90, 105, 120, 135, 150, 165, 180, 195, 210, 225, 240, 255, 270, 285, 300, 315, 330, 345],
-                ticktext=['12am','1am','2am','3am','4am','5am', '6am','7am','8am','9am','10am','11am','12pm','1pm','2pm','3pm','4pm','5pm', '6pm','7pm','8pm','9pm','10pm','11pm'],
+                ticktext=['12am', '1am', '2am', '3am', '4am', '5am', '6am', '7am', '8am', '9am', '10am', '11am',
+                          '12pm', '1pm', '2pm', '3pm', '4pm', '5pm', '6pm', '7pm', '8pm', '9pm', '10pm', '11pm'],
                 hoverformat="#%{theta}: <br>Popularity: %{percent} </br> %{r}"
             ),
         ),
     )
 
-
-
     daily = pd.crosstab(df5, df5.index.date, dropna=False)
 
     fig.add_trace(go.Bar(x=daily.columns, y=daily.loc[specie]), row=3, col=2)

scripts/server.py

@@ -1,3 +1,15 @@
+import apprise
+from pathlib import Path
+from tzlocal import get_localzone
+import datetime
+import sqlite3
+import requests
+import json
+import time
+import math
+import numpy as np
+import librosa
+import operator
 import socket
 import threading
 import os
@@ -6,26 +18,9 @@ os.environ['CUDA_VISIBLE_DEVICES'] = ''
 
 try:
     import tflite_runtime.interpreter as tflite
-except:
+except BaseException:
     from tensorflow import lite as tflite
 
-import argparse
-import operator
-import librosa
-import numpy as np
-import math
-import time
-from decimal import Decimal
-import json
-import requests
-import sqlite3
-import datetime
-from time import sleep
-import pytz
-from tzlocal import get_localzone
-from pathlib import Path
-import apprise
-
 
 HEADER = 64
 PORT = 5050
@@ -37,12 +32,11 @@ DISCONNECT_MESSAGE = "!DISCONNECT"
 server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
 try:
     server.bind(ADDR)
-except:
+except BaseException:
     print("Waiting on socket")
     time.sleep(5)
 
 
-
 # Open most recent Configuration and grab DB_PWD as a python variable
 userDir = os.path.expanduser('~')
 with open(userDir + '/BirdNET-Pi/scripts/thisrun.txt', 'r') as f:
@@ -85,6 +79,7 @@ def loadModel():
 
     return myinterpreter
 
+
 def loadCustomSpeciesList(path):
 
     slist = []
@@ -95,6 +90,7 @@ def loadCustomSpeciesList(path):
 
     return slist
 
+
 def splitSignal(sig, rate, overlap, seconds=3.0, minlen=1.5):
 
     # Split signal with overlap
@@ -116,6 +112,7 @@ def splitSignal(sig, rate, overlap, seconds=3.0, minlen=1.5):
 
     return sig_splits
 
+
 def readAudioData(path, overlap, sample_rate=48000):
 
     print('READING AUDIO DATA...', end=' ', flush=True)
@@ -130,6 +127,7 @@ def readAudioData(path, overlap, sample_rate=48000):
 
     return chunks
 
+
 def convertMetadata(m):
 
     # Convert week to cosine
@@ -147,9 +145,11 @@ def convertMetadata(m):
 
     return np.concatenate([m, mask])
 
+
 def custom_sigmoid(x, sensitivity=1.0):
     return 1 / (1.0 + np.exp(-sensitivity * x))
 
+
 def predict(sample, sensitivity):
     global INTERPRETER
     # Make a prediction
@@ -181,6 +181,7 @@ def predict(sample, sensitivity):
 
     return p_sorted[:human_cutoff]
 
+
 def analyzeAudioData(chunks, lat, lon, week, sensitivity, overlap,):
     global INTERPRETER
 
@@ -213,7 +214,7 @@ def analyzeAudioData(chunks, lat, lon, week, sensitivity, overlap,):
         pred_end = pred_start + 3.0
 
         # If human detected set all detections to human to make sure voices are not saved
-        if HUMAN_DETECTED == True:
+        if HUMAN_DETECTED is True:
             p = [('Human_Human', 0.0)] * 10
 
         detections[str(pred_start) + ';' + str(pred_end)] = p
@@ -224,6 +225,7 @@ def analyzeAudioData(chunks, lat, lon, week, sensitivity, overlap,):
 #    print('DETECTIONS:::::',detections)
     return detections
 
+
 def sendAppriseNotifications(species, confidence):
     if os.path.exists(userDir + '/BirdNET-Pi/apprise.txt') and os.path.getsize(userDir + '/BirdNET-Pi/apprise.txt') > 0:
         with open(userDir + '/BirdNET-Pi/scripts/thisrun.txt', 'r') as f:
@@ -266,10 +268,11 @@ def sendAppriseNotifications(species,confidence):
                     )
 
                 con.close()
-            except:
+            except BaseException:
                 print("Database busy")
                 time.sleep(2)
 
+
 def writeResultsToFile(detections, min_conf, path):
 
     print('WRITING RESULTS TO', path, '...', end=' ')
@@ -279,12 +282,13 @@ def writeResultsToFile(detections, min_conf, path):
         for d in detections:
             for entry in detections[d]:
                 if entry[1] >= min_conf and ((entry[0] in INCLUDE_LIST or len(INCLUDE_LIST) == 0) and (entry[0] not in EXCLUDE_LIST or len(EXCLUDE_LIST) == 0)):
-                    sendAppriseNotifications(str(entry[0]),str(entry[1]));
+                    sendAppriseNotifications(str(entry[0]), str(entry[1]))
                     rfile.write(d + ';' + entry[0].replace('_', ';') + ';' + str(entry[1]) + '\n')
                     rcnt += 1
     print('DONE! WROTE', rcnt, 'RESULTS.')
     return
 
+
 def handle_client(conn, addr):
     global INCLUDE_LIST
     global EXCLUDE_LIST
@@ -315,7 +319,6 @@ def handle_client(conn, addr):
                 args.lat = -1
                 args.lon = -1
 
-
                 for line in msg.split('||'):
                     inputvars = line.split('=')
                     if inputvars[0] == 'i':
@@ -341,8 +344,6 @@ def handle_client(conn, addr):
                     elif inputvars[0] == 'lon':
                         args.lon = float(inputvars[1])
 
-
-                   
                 # Load custom species lists - INCLUDED and EXCLUDED
                 if not args.include_list == 'null':
                     INCLUDE_LIST = loadCustomSpeciesList(args.include_list)
@@ -398,13 +399,13 @@ def handle_client(conn, addr):
                 for i in detections:
                     myReturn += str(i) + '-' + str(detections[i][0]) + '\n'
 
-                
                 with open(userDir + '/BirdNET-Pi/BirdDB.txt', 'a') as rfile:
                     for d in detections:
                         for entry in detections[d]:
-                            if entry[1] >= min_conf and ((entry[0] in INCLUDE_LIST or len(INCLUDE_LIST) == 0) and (entry[0] not in EXCLUDE_LIST or len(EXCLUDE_LIST) == 0) ):
+                            if entry[1] >= min_conf and ((entry[0] in INCLUDE_LIST or len(INCLUDE_LIST) == 0)
-                                rfile.write(str(current_date) + ';' + str(current_time) + ';' + entry[0].replace('_', ';') + ';' \
+                                                         and (entry[0] not in EXCLUDE_LIST or len(EXCLUDE_LIST) == 0)):
-                                + str(entry[1]) +";" + str(args.lat) + ';' + str(args.lon) + ';' + str(min_conf) + ';' + str(week) + ';' \
+                                rfile.write(str(current_date) + ';' + str(current_time) + ';' + entry[0].replace('_', ';') + ';'
+                                            + str(entry[1]) + ";" + str(args.lat) + ';' + str(args.lon) + ';' + str(min_conf) + ';' + str(week) + ';'
                                             + str(args.sensitivity) + ';' + str(args.overlap) + '\n')
 
                                 Date = str(current_date)
@@ -428,23 +429,55 @@ def handle_client(conn, addr):
                                     try:
                                         con = sqlite3.connect(userDir + '/BirdNET-Pi/scripts/birds.db')
                                         cur = con.cursor()
-                                        cur.execute("INSERT INTO detections VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (Date, Time, Sci_Name, Com_Name, str(score), Lat, Lon, Cutoff, Week, Sens, Overlap, File_Name))
+                                        cur.execute("INSERT INTO detections VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (Date, Time,
+                                                    Sci_Name, Com_Name, str(score), Lat, Lon, Cutoff, Week, Sens, Overlap, File_Name))
 
                                         con.commit()
                                         con.close()
                                         break
-                                    except:
+                                    except BaseException:
                                         print("Database busy")
                                         time.sleep(2)
 
-                                print(str(current_date) + ';' + str(current_time) + ';' + entry[0].replace('_', ';') + ';' + str(entry[1]) + ';' + str(args.lat) + ';' + str(args.lon) + ';' + str(min_conf) + ';' + str(week) + ';' + str(args.sensitivity) +';' + str(args.overlap) + Com_Name.replace(" ", "_") + '-' + str(score) + '-' + str(current_date) + '-birdnet-' + str(current_time) + audiofmt  + '\n')
+                                print(str(current_date) +
+                                      ';' +
+                                      str(current_time) +
+                                      ';' +
+                                      entry[0].replace('_', ';') +
+                                      ';' +
+                                      str(entry[1]) +
+                                      ';' +
+                                      str(args.lat) +
+                                      ';' +
+                                      str(args.lon) +
+                                      ';' +
+                                      str(min_conf) +
+                                      ';' +
+                                      str(week) +
+                                      ';' +
+                                      str(args.sensitivity) +
+                                      ';' +
+                                      str(args.overlap) +
+                                      Com_Name.replace(" ", "_") +
+                                      '-' +
+                                      str(score) +
+                                      '-' +
+                                      str(current_date) +
+                                      '-birdnet-' +
+                                      str(current_time) +
+                                      audiofmt +
+                                      '\n')
 
                                 if birdweather_id != "99999":
                                     try:
 
                                         if soundscape_uploaded is False:
                                             # POST soundscape to server
-                                            soundscape_url = "https://app.birdweather.com/api/v1/stations/" + birdweather_id +  "/soundscapes" + "?timestamp=" + current_iso8601
+                                            soundscape_url = 'https://app.birdweather.com/api/v1/stations/' + \
+                                                birdweather_id + \
+                                                '/soundscapes' + \
+                                                '?timestamp=' + \
+                                                current_iso8601
 
                                             with open(args.i, 'rb') as f:
                                                 wav_data = f.read()
@@ -473,11 +506,12 @@ def handle_client(conn, addr):
                                         post_confidence = "\"confidence\": " + str(entry[1])
                                         post_end = " }"
 
-                                        post_json = post_begin + post_timestamp + post_lat + post_lon + post_soundscape_id + post_soundscape_start_time + post_soundscape_end_time + post_commonName + post_scientificName + post_algorithm + post_confidence + post_end
+                                        post_json = post_begin + post_timestamp + post_lat + post_lon + post_soundscape_id + post_soundscape_start_time + \
+                                            post_soundscape_end_time + post_commonName + post_scientificName + post_algorithm + post_confidence + post_end
                                         print(post_json)
                                         response = requests.post(detection_url, json=json.loads(post_json))
                                         print("Detection POST Response Status - ", response.status_code)
-                                    except:
+                                    except BaseException:
                                         print("Cannot POST right now")
                 conn.send(myReturn.encode(FORMAT))
 
@@ -485,6 +519,7 @@ def handle_client(conn, addr):
 
     conn.close()
 
+
 def start():
     # Load model
     global INTERPRETER, INCLUDE_LIST, EXCLUDE_LIST