Merge pull request #308 from jmherbst/python-lint

Python linting
2022-05-24 12:51:59 -04:00
parent 3a95504f04 166ca7fcf8
commit f9cc6b6aa5
6 changed files with 399 additions and 295 deletions
@@ -0,0 +1,2 @@
+[flake8]
+max-line-length = 160
@@ -13,9 +13,6 @@ jobs:
    steps:
      - name: Checkout
        uses: actions/checkout@v3
-        with:
-          # This is a bit heavy handed, but we need `origin/main` to get a ref to diff against
-          fetch-depth: 0

      - name: Setup Python
        uses: actions/setup-python@v3
@@ -29,5 +26,4 @@ jobs:

      - name: Run Flake8 Lint
        run: |
-          DIFF="$(git --no-pager diff -u $(git merge-base HEAD origin/main) -- '**/*.py')"
-          echo "$DIFF" | flake8 --diff
+          flake8
@@ -11,6 +11,7 @@ ADDR = (SERVER, PORT)
 client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
 client.connect(ADDR)

+
 def send(msg):
    message = msg.encode(FORMAT)
    msg_length = len(message)
@@ -20,6 +21,7 @@ def send(msg):
    client.send(message)
    print(client.recv(2048).decode(FORMAT))

+
 def main():

    global INCLUDE_LIST
@@ -28,16 +30,52 @@ def main():
    # Parse passed arguments
    parser = argparse.ArgumentParser()
    parser.add_argument('--i', help='Path to input file.')
-    parser.add_argument('--o', default='result.csv', help='Path to output file. Defaults to result.csv.')
-    parser.add_argument('--lat', type=float, default=-1, help='Recording location latitude. Set -1 to ignore.')
-    parser.add_argument('--lon', type=float, default=-1, help='Recording location longitude. Set -1 to ignore.')
-    parser.add_argument('--week', type=int, default=-1, help='Week of the year when the recording was made. Values in [1, 48] (4 weeks per month). Set -1 to ignore.')
-    parser.add_argument('--overlap', type=float, default=0.0, help='Overlap in seconds between extracted spectrograms. Values in [0.0, 2.9]. Defaults tp 0.0.')
-    parser.add_argument('--sensitivity', type=float, default=1.0, help='Detection sensitivity; Higher values result in higher sensitivity. Values in [0.5, 1.5]. Defaults to 1.0.')
-    parser.add_argument('--min_conf', type=float, default=0.1, help='Minimum confidence threshold. Values in [0.01, 0.99]. Defaults to 0.1.')   
-    parser.add_argument('--include_list', default='null', help='Path to text file containing a list of included species. Not used if not provided.')
-    parser.add_argument('--exclude_list', default='null', help='Path to text file containing a list of excluded species. Not used if not provided.')
-    parser.add_argument('--birdweather_id', default='99999', help='Private Station ID for BirdWeather.')    
+    parser.add_argument(
+        '--o',
+        default='result.csv',
+        help='Path to output file. Defaults to result.csv.')
+    parser.add_argument(
+        '--lat',
+        type=float,
+        default=-1,
+        help='Recording location latitude. Set -1 to ignore.')
+    parser.add_argument(
+        '--lon',
+        type=float,
+        default=-1,
+        help='Recording location longitude. Set -1 to ignore.')
+    parser.add_argument(
+        '--week',
+        type=int,
+        default=-1,
+        help='Week of the year when the recording was made. Values in [1, 48] (4 weeks per month). Set -1 to ignore.')
+    parser.add_argument(
+        '--overlap',
+        type=float,
+        default=0.0,
+        help='Overlap in seconds between extracted spectrograms. Values in [0.0, 2.9]. Defaults tp 0.0.')
+    parser.add_argument(
+        '--sensitivity',
+        type=float,
+        default=1.0,
+        help='Detection sensitivity; Higher values result in higher sensitivity. Values in [0.5, 1.5]. Defaults to 1.0.')
+    parser.add_argument(
+        '--min_conf',
+        type=float,
+        default=0.1,
+        help='Minimum confidence threshold. Values in [0.01, 0.99]. Defaults to 0.1.')
+    parser.add_argument(
+        '--include_list',
+        default='null',
+        help='Path to text file containing a list of included species. Not used if not provided.')
+    parser.add_argument(
+        '--exclude_list',
+        default='null',
+        help='Path to text file containing a list of excluded species. Not used if not provided.')
+    parser.add_argument(
+        '--birdweather_id',
+        default='99999',
+        help='Private Station ID for BirdWeather.')

    args = parser.parse_args()

@@ -73,10 +111,13 @@ def main():
 ###############################################################################
 ###############################################################################

+
 if __name__ == '__main__':

    main()

    # Example calls
    # python3 analyze.py --i 'example/XC558716 - Soundscape.mp3' --lat 35.4244 --lon -120.7463 --week 18
-    # python3 analyze.py --i 'example/XC563936 - Soundscape.mp3' --lat 47.6766 --lon -122.294 --week 11 --overlap 1.5 --min_conf 0.25 --sensitivity 1.25 --custom_list 'example/custom_species_list.txt'
+    # python3 analyze.py --i 'example/XC563936 - Soundscape.mp3' --lat 47.6766
+    # --lon -122.294 --week 11 --overlap 1.5 --min_conf 0.25 --sensitivity
+    # 1.25 --custom_list 'example/custom_species_list.txt'
@@ -1,6 +1,5 @@
 import sqlite3
 import os
-import configparser
 import pandas as pd
 import seaborn as sns
 import matplotlib.pyplot as plt
@@ -72,8 +71,6 @@ colors = plt.cm.Greens(norm(confmax))
 plot = sns.countplot(y='Com_Name', data=df_plt_top10_today, palette=colors, order=freq_order, ax=axs[0])


-
-
 # Try plot grid lines between bars - problem at the moment plots grid lines on bars - want between bars
 z = plot.get_ymajorticklabels()
 plot.set_yticklabels(['\n'.join(textwrap.wrap(ticklabel.get_text(), 15)) for ticklabel in plot.get_yticklabels()], fontsize=10)
@@ -94,7 +91,20 @@ heat_frame = pd.DataFrame(data=0, index=heat.index, columns = hours_in_day)
 heat = (heat + heat_frame).fillna(0)

 # Generatie heatmap plot
-plot = sns.heatmap(heat, norm=LogNorm(),  annot=True,  annot_kws={"fontsize":7}, fmt="g", cmap = pal , square = False, cbar=False, linewidths = 0.5, linecolor = "Grey", ax=axs[1], yticklabels = False)
+plot = sns.heatmap(
+    heat,
+    norm=LogNorm(),
+    annot=True,
+    annot_kws={
+        "fontsize": 7},
+    fmt="g",
+    cmap=pal,
+    square=False,
+    cbar=False,
+    linewidths=0.5,
+    linecolor="Grey",
+    ax=axs[1],
+    yticklabels=False)
 plot.set_xticklabels(plot.get_xticklabels(), rotation=0, size=7)

 # Set heatmap border
@@ -142,8 +152,6 @@ colors = plt.cm.Reds(norm(confmax))
 plot = sns.countplot(y='Com_Name', data=df_plt_Bot10_today, palette=colors, order=freq_order, ax=axs[0])


-
-
 # Try plot grid lines between bars - problem at the moment plots grid lines on bars - want between bars
 z = plot.get_ymajorticklabels()
 plot.set_yticklabels(['\n'.join(textwrap.wrap(ticklabel.get_text(), 15)) for ticklabel in plot.get_yticklabels()], fontsize=10)
@@ -163,7 +171,20 @@ heat_frame = pd.DataFrame(data=0, index=heat.index, columns = hours_in_day)
 heat = (heat + heat_frame).fillna(0)

 # Generatie heatmap plot
-plot = sns.heatmap(heat, norm=LogNorm(),  annot=True, fmt="g",  annot_kws={"fontsize":7}, cmap = pal , square = False, cbar=False, linewidths = 0.5, linecolor = "Grey", ax=axs[1], yticklabels = False)
+plot = sns.heatmap(
+    heat,
+    norm=LogNorm(),
+    annot=True,
+    fmt="g",
+    annot_kws={
+        "fontsize": 7},
+    cmap=pal,
+    square=False,
+    cbar=False,
+    linewidths=0.5,
+    linecolor="Grey",
+    ax=axs[1],
+    yticklabels=False)
 plot.set_xticklabels(plot.get_xticklabels(), rotation=0, size=7)

 # Set heatmap border
@@ -4,8 +4,7 @@ import pandas as pd
 import numpy as np
 import plotly.graph_objects as go
 from plotly.subplots import make_subplots
-from datetime import timedelta, datetime
-from pathlib import Path
+from datetime import timedelta
 import sqlite3
 from sqlite3 import Connection
 import plotly.express as px
@@ -43,6 +42,7 @@ def get_data(conn: Connection):
    df1 = pd.read_sql("SELECT * FROM detections", con=conn)
    return df1

+
 conn = get_connection(URI_SQLITE_DB)
 # Read in the cereal data
 # df = load_data()
@@ -52,7 +52,6 @@ df2['DateTime']=pd.to_datetime(df2['Date'] + " " + df2['Time'])
 df2 = df2.set_index('DateTime')


-
 # Filter on date range
 # Date as calendars
 # Start_Date = pd.to_datetime(st.sidebar.date_input('Which date do you want to start?', value = df2.index.min()))
@@ -70,14 +69,22 @@ Date_Slider = cols1.slider('Date Range',
                           )


-
 filt = (df2.index >= pd.Timestamp(Date_Slider[0])) & (df2.index <= pd.Timestamp(Date_Slider[1] + timedelta(days=1)))
 df2 = df2[filt]

 st.write('<style>div.row-widget.stRadio > div{flex-direction:row;justify-content: left;} </style>', unsafe_allow_html=True)
 st.write('<style>div.st-bf{flex-direction:column;} div.st-ag{font-weight:bold;padding-left:2px;}</style>', unsafe_allow_html=True)

-resample_sel=cols2.radio("Select Resample Resolution - To downsample and make run faster select longer period, Daily provides a view on detections at 15 min intervals through the day", ('1 minute', '5 minutes', '10 minutes', 'Hourly', 'Daily'))
+resample_sel = cols2.radio(
+    '''
+    Select Resample Resolution - To downsample and make run faster select longer period,
+    Daily provides a view on detections at 15 min intervals through the day
+    ''',
+    ('1 minute',
+     '5 minutes',
+     '10 minutes',
+     'Hourly',
+     'Daily'))

 resample_times = {'1 minute': '1min',
                  '5 minutes': '5min',
@@ -124,13 +131,20 @@ filt=df2['Com_Name']==specie
 df_counts = sum(df5 == specie)


-
-
 if resample_time != '1D':
    fig = make_subplots(
        rows=3, cols=2,
        specs=[[{"type": "xy", "rowspan": 3}, {"type": "polar", "rowspan": 2}], [{"rowspan": 1}, {"rowspan": 1}], [None, {"type": "xy", "rowspan": 1}]],
-                    subplot_titles=('<b>Top '+ str(top_N) + ' Species in Date Range '+str(Date_Slider[0])+' to '+str(Date_Slider[1])+' for '+str(resample_sel)+' sampling interval.'+'</b>',
+        subplot_titles=('<b>Top ' +
+                        str(top_N) +
+                        ' Species in Date Range ' +
+                        str(Date_Slider[0]) +
+                        ' to ' +
+                        str(Date_Slider[1]) +
+                        ' for ' +
+                        str(resample_sel) +
+                        ' sampling interval.' +
+                        '</b>',
                        'Total Detect:' + str('{:,}'.format(df_counts)) +
                        '   Confidence Max:' + str('{:.2f}%'.format(max(df2[df2['Com_Name'] == specie]['Confidence']) * 100)) +
                        '   ' + '   Median:' + str('{:.2f}%'.format(np.median(df2[df2['Com_Name'] == specie]['Confidence']) * 100))
@@ -145,7 +159,6 @@ if resample_time != '1D':
        margin=dict(l=0, r=0, t=50, b=0),
        yaxis={'categoryorder': 'total ascending'})

-
    # Set 360 degrees, 24 hours for polar plot
    theta = np.linspace(0.0, 360, 24, endpoint=False)

@@ -154,9 +167,6 @@ if resample_time != '1D':

    detections2 = pd.crosstab(df3, df3.index.hour)

-
-
-
    d = pd.DataFrame(np.zeros((23, 1))).squeeze()
    detections = hourly.loc[specie]
    detections = (d + detections).fillna(0)
@@ -178,14 +188,13 @@ if resample_time != '1D':
                direction='clockwise',
                tickmode='array',
                tickvals=[0, 15, 35, 45, 60, 75, 90, 105, 120, 135, 150, 165, 180, 195, 210, 225, 240, 255, 270, 285, 300, 315, 330, 345],
-                ticktext=['12am','1am','2am','3am','4am','5am', '6am','7am','8am','9am','10am','11am','12pm','1pm','2pm','3pm','4pm','5pm', '6pm','7pm','8pm','9pm','10pm','11pm'],
+                ticktext=['12am', '1am', '2am', '3am', '4am', '5am', '6am', '7am', '8am', '9am', '10am', '11am',
+                          '12pm', '1pm', '2pm', '3pm', '4pm', '5pm', '6pm', '7pm', '8pm', '9pm', '10pm', '11pm'],
                hoverformat="#%{theta}: <br>Popularity: %{percent} </br> %{r}"
            ),
        ),
    )

-
-
    daily = pd.crosstab(df5, df5.index.date, dropna=False)

    fig.add_trace(go.Bar(x=daily.columns, y=daily.loc[specie]), row=3, col=2)
@@ -1,3 +1,15 @@
+import apprise
+from pathlib import Path
+from tzlocal import get_localzone
+import datetime
+import sqlite3
+import requests
+import json
+import time
+import math
+import numpy as np
+import librosa
+import operator
 import socket
 import threading
 import os
@@ -6,26 +18,9 @@ os.environ['CUDA_VISIBLE_DEVICES'] = ''

 try:
    import tflite_runtime.interpreter as tflite
-except:
+except BaseException:
    from tensorflow import lite as tflite

-import argparse
-import operator
-import librosa
-import numpy as np
-import math
-import time
-from decimal import Decimal
-import json
-import requests
-import sqlite3
-import datetime
-from time import sleep
-import pytz
-from tzlocal import get_localzone
-from pathlib import Path
-import apprise
-

 HEADER = 64
 PORT = 5050
@@ -37,12 +32,11 @@ DISCONNECT_MESSAGE = "!DISCONNECT"
 server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
 try:
    server.bind(ADDR)
-except:
+except BaseException:
    print("Waiting on socket")
    time.sleep(5)


-
 # Open most recent Configuration and grab DB_PWD as a python variable
 userDir = os.path.expanduser('~')
 with open(userDir + '/BirdNET-Pi/scripts/thisrun.txt', 'r') as f:
@@ -85,6 +79,7 @@ def loadModel():

    return myinterpreter

+
 def loadCustomSpeciesList(path):

    slist = []
@@ -95,6 +90,7 @@ def loadCustomSpeciesList(path):

    return slist

+
 def splitSignal(sig, rate, overlap, seconds=3.0, minlen=1.5):

    # Split signal with overlap
@@ -116,6 +112,7 @@ def splitSignal(sig, rate, overlap, seconds=3.0, minlen=1.5):

    return sig_splits

+
 def readAudioData(path, overlap, sample_rate=48000):

    print('READING AUDIO DATA...', end=' ', flush=True)
@@ -130,6 +127,7 @@ def readAudioData(path, overlap, sample_rate=48000):

    return chunks

+
 def convertMetadata(m):

    # Convert week to cosine
@@ -147,9 +145,11 @@ def convertMetadata(m):

    return np.concatenate([m, mask])

+
 def custom_sigmoid(x, sensitivity=1.0):
    return 1 / (1.0 + np.exp(-sensitivity * x))

+
 def predict(sample, sensitivity):
    global INTERPRETER
    # Make a prediction
@@ -181,6 +181,7 @@ def predict(sample, sensitivity):

    return p_sorted[:human_cutoff]

+
 def analyzeAudioData(chunks, lat, lon, week, sensitivity, overlap,):
    global INTERPRETER

@@ -213,7 +214,7 @@ def analyzeAudioData(chunks, lat, lon, week, sensitivity, overlap,):
        pred_end = pred_start + 3.0

        # If human detected set all detections to human to make sure voices are not saved
-        if HUMAN_DETECTED == True:
+        if HUMAN_DETECTED is True:
            p = [('Human_Human', 0.0)] * 10

        detections[str(pred_start) + ';' + str(pred_end)] = p
@@ -224,6 +225,7 @@ def analyzeAudioData(chunks, lat, lon, week, sensitivity, overlap,):
 #    print('DETECTIONS:::::',detections)
    return detections

+
 def sendAppriseNotifications(species, confidence):
    if os.path.exists(userDir + '/BirdNET-Pi/apprise.txt') and os.path.getsize(userDir + '/BirdNET-Pi/apprise.txt') > 0:
        with open(userDir + '/BirdNET-Pi/scripts/thisrun.txt', 'r') as f:
@@ -266,10 +268,11 @@ def sendAppriseNotifications(species,confidence):
                    )

                con.close()
-            except:
+            except BaseException:
                print("Database busy")
                time.sleep(2)

+
 def writeResultsToFile(detections, min_conf, path):

    print('WRITING RESULTS TO', path, '...', end=' ')
@@ -279,12 +282,13 @@ def writeResultsToFile(detections, min_conf, path):
        for d in detections:
            for entry in detections[d]:
                if entry[1] >= min_conf and ((entry[0] in INCLUDE_LIST or len(INCLUDE_LIST) == 0) and (entry[0] not in EXCLUDE_LIST or len(EXCLUDE_LIST) == 0)):
-                    sendAppriseNotifications(str(entry[0]),str(entry[1]));
+                    sendAppriseNotifications(str(entry[0]), str(entry[1]))
                    rfile.write(d + ';' + entry[0].replace('_', ';') + ';' + str(entry[1]) + '\n')
                    rcnt += 1
    print('DONE! WROTE', rcnt, 'RESULTS.')
    return

+
 def handle_client(conn, addr):
    global INCLUDE_LIST
    global EXCLUDE_LIST
@@ -315,7 +319,6 @@ def handle_client(conn, addr):
                args.lat = -1
                args.lon = -1

-
                for line in msg.split('||'):
                    inputvars = line.split('=')
                    if inputvars[0] == 'i':
@@ -341,8 +344,6 @@ def handle_client(conn, addr):
                    elif inputvars[0] == 'lon':
                        args.lon = float(inputvars[1])

-
-                   
                # Load custom species lists - INCLUDED and EXCLUDED
                if not args.include_list == 'null':
                    INCLUDE_LIST = loadCustomSpeciesList(args.include_list)
@@ -398,13 +399,13 @@ def handle_client(conn, addr):
                for i in detections:
                    myReturn += str(i) + '-' + str(detections[i][0]) + '\n'

-                
                with open(userDir + '/BirdNET-Pi/BirdDB.txt', 'a') as rfile:
                    for d in detections:
                        for entry in detections[d]:
-                            if entry[1] >= min_conf and ((entry[0] in INCLUDE_LIST or len(INCLUDE_LIST) == 0) and (entry[0] not in EXCLUDE_LIST or len(EXCLUDE_LIST) == 0) ):
-                                rfile.write(str(current_date) + ';' + str(current_time) + ';' + entry[0].replace('_', ';') + ';' \
-                                + str(entry[1]) +";" + str(args.lat) + ';' + str(args.lon) + ';' + str(min_conf) + ';' + str(week) + ';' \
+                            if entry[1] >= min_conf and ((entry[0] in INCLUDE_LIST or len(INCLUDE_LIST) == 0)
+                                                         and (entry[0] not in EXCLUDE_LIST or len(EXCLUDE_LIST) == 0)):
+                                rfile.write(str(current_date) + ';' + str(current_time) + ';' + entry[0].replace('_', ';') + ';'
+                                            + str(entry[1]) + ";" + str(args.lat) + ';' + str(args.lon) + ';' + str(min_conf) + ';' + str(week) + ';'
                                            + str(args.sensitivity) + ';' + str(args.overlap) + '\n')

                                Date = str(current_date)
@@ -428,23 +429,55 @@ def handle_client(conn, addr):
                                    try:
                                        con = sqlite3.connect(userDir + '/BirdNET-Pi/scripts/birds.db')
                                        cur = con.cursor()
-                                        cur.execute("INSERT INTO detections VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (Date, Time, Sci_Name, Com_Name, str(score), Lat, Lon, Cutoff, Week, Sens, Overlap, File_Name))
+                                        cur.execute("INSERT INTO detections VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (Date, Time,
+                                                    Sci_Name, Com_Name, str(score), Lat, Lon, Cutoff, Week, Sens, Overlap, File_Name))

                                        con.commit()
                                        con.close()
                                        break
-                                    except:
+                                    except BaseException:
                                        print("Database busy")
                                        time.sleep(2)

-                                print(str(current_date) + ';' + str(current_time) + ';' + entry[0].replace('_', ';') + ';' + str(entry[1]) + ';' + str(args.lat) + ';' + str(args.lon) + ';' + str(min_conf) + ';' + str(week) + ';' + str(args.sensitivity) +';' + str(args.overlap) + Com_Name.replace(" ", "_") + '-' + str(score) + '-' + str(current_date) + '-birdnet-' + str(current_time) + audiofmt  + '\n')
+                                print(str(current_date) +
+                                      ';' +
+                                      str(current_time) +
+                                      ';' +
+                                      entry[0].replace('_', ';') +
+                                      ';' +
+                                      str(entry[1]) +
+                                      ';' +
+                                      str(args.lat) +
+                                      ';' +
+                                      str(args.lon) +
+                                      ';' +
+                                      str(min_conf) +
+                                      ';' +
+                                      str(week) +
+                                      ';' +
+                                      str(args.sensitivity) +
+                                      ';' +
+                                      str(args.overlap) +
+                                      Com_Name.replace(" ", "_") +
+                                      '-' +
+                                      str(score) +
+                                      '-' +
+                                      str(current_date) +
+                                      '-birdnet-' +
+                                      str(current_time) +
+                                      audiofmt +
+                                      '\n')

                                if birdweather_id != "99999":
                                    try:

                                        if soundscape_uploaded is False:
                                            # POST soundscape to server
-                                            soundscape_url = "https://app.birdweather.com/api/v1/stations/" + birdweather_id +  "/soundscapes" + "?timestamp=" + current_iso8601
+                                            soundscape_url = 'https://app.birdweather.com/api/v1/stations/' + \
+                                                birdweather_id + \
+                                                '/soundscapes' + \
+                                                '?timestamp=' + \
+                                                current_iso8601

                                            with open(args.i, 'rb') as f:
                                                wav_data = f.read()
@@ -473,11 +506,12 @@ def handle_client(conn, addr):
                                        post_confidence = "\"confidence\": " + str(entry[1])
                                        post_end = " }"

-                                        post_json = post_begin + post_timestamp + post_lat + post_lon + post_soundscape_id + post_soundscape_start_time + post_soundscape_end_time + post_commonName + post_scientificName + post_algorithm + post_confidence + post_end
+                                        post_json = post_begin + post_timestamp + post_lat + post_lon + post_soundscape_id + post_soundscape_start_time + \
+                                            post_soundscape_end_time + post_commonName + post_scientificName + post_algorithm + post_confidence + post_end
                                        print(post_json)
                                        response = requests.post(detection_url, json=json.loads(post_json))
                                        print("Detection POST Response Status - ", response.status_code)
-                                    except:
+                                    except BaseException:
                                        print("Cannot POST right now")
                conn.send(myReturn.encode(FORMAT))

@@ -485,6 +519,7 @@ def handle_client(conn, addr):

    conn.close()

+
 def start():
    # Load model
    global INTERPRETER, INCLUDE_LIST, EXCLUDE_LIST