claudecode/AvianVisitors
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Revert "Adding Flake8 Github Action for Python Linting "

Browse Source
This commit is contained in:
Patrick McGuire
2022-05-11 09:05:59 -04:00
committed by GitHub
parent bffd6a889d
commit 528802a0e6
6 changed files with 256 additions and 473 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/server.py
+94 -155
View File
@@ -1,27 +1,32 @@
import os
import socket
import socket
import threading
import operator
import librosa
import numpy as np
import math
import time
import json
import requests
import sqlite3
import datetime
from tzlocal import get_localzone
from pathlib import Path
import apprise
import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
os.environ['CUDA_VISIBLE_DEVICES'] = ''
try:
import tflite_runtime.interpreter as tflite
except BaseException:
except:
from tensorflow import lite as tflite
import argparse
import operator
import librosa
import numpy as np
import math
import time
from decimal import Decimal
import json
import requests
import sqlite3
import datetime
from time import sleep
import pytz
from tzlocal import get_localzone
from pathlib import Path
import apprise
HEADER = 64
PORT = 5050
SERVER = socket.gethostbyname(socket.gethostname())
@@ -32,9 +37,10 @@ DISCONNECT_MESSAGE = "!DISCONNECT"
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
server.bind(ADDR)
except BaseException:
except:
print("Waiting on socket")
time.sleep(5)
# Open most recent Configuration and grab DB_PWD as a python variable
@@ -42,8 +48,7 @@ userDir = os.path.expanduser('~')
with open(userDir + '/BirdNET-Pi/scripts/thisrun.txt', 'r') as f:
this_run = f.readlines()
audiofmt = "." + str(str(str([i for i in this_run if i.startswith('AUDIOFMT')]).split('=')[1]).split('\\')[0])
priv_thresh = float(
"." + str(str(str([i for i in this_run if i.startswith('PRIVACY_THRESHOLD')]).split('=')[1]).split('\\')[0])) / 10
priv_thresh = float("." + str(str(str([i for i in this_run if i.startswith('PRIVACY_THRESHOLD')]).split('=')[1]).split('\\')[0]))/10
def loadModel():
@@ -57,7 +62,7 @@ def loadModel():
# Load TFLite model and allocate tensors.
modelpath = userDir + '/BirdNET-Pi/model/BirdNET_6K_GLOBAL_MODEL.tflite'
myinterpreter = tflite.Interpreter(model_path=modelpath, num_threads=2)
myinterpreter = tflite.Interpreter(model_path=modelpath,num_threads=2)
myinterpreter.allocate_tensors()
# Get input and output tensors.
@@ -80,7 +85,6 @@ def loadModel():
return myinterpreter
def loadCustomSpeciesList(path):
slist = []
@@ -91,7 +95,6 @@ def loadCustomSpeciesList(path):
return slist
def splitSignal(sig, rate, overlap, seconds=3.0, minlen=1.5):
# Split signal with overlap
@@ -102,18 +105,17 @@ def splitSignal(sig, rate, overlap, seconds=3.0, minlen=1.5):
# End of signal?
if len(split) < int(minlen * rate):
break
# Signal chunk too short? Fill with zeros.
if len(split) < int(rate * seconds):
temp = np.zeros((int(rate * seconds)))
temp[:len(split)] = split
split = temp
sig_splits.append(split)
return sig_splits
def readAudioData(path, overlap, sample_rate=48000):
print('READING AUDIO DATA...', end=' ', flush=True)
@@ -128,12 +130,11 @@ def readAudioData(path, overlap, sample_rate=48000):
return chunks
def convertMetadata(m):
# Convert week to cosine
if m[2] >= 1 and m[2] <= 48:
m[2] = math.cos(math.radians(m[2] * 7.5)) + 1
m[2] = math.cos(math.radians(m[2] * 7.5)) + 1
else:
m[2] = -1
@@ -146,11 +147,9 @@ def convertMetadata(m):
return np.concatenate([m, mask])
def custom_sigmoid(x, sensitivity=1.0):
return 1 / (1.0 + np.exp(-sensitivity * x))
def predict(sample, sensitivity):
global INTERPRETER
# Make a prediction
@@ -167,22 +166,21 @@ def predict(sample, sensitivity):
# Sort by score
p_sorted = sorted(p_labels.items(), key=operator.itemgetter(1), reverse=True)
# #print("DATABASE SIZE:", len(p_sorted))
# #print("HUMAN-CUTOFF AT:", int(len(p_sorted)*priv_thresh)/10)
#
#
# # Remove species that are on blacklist
human_cutoff = max(10, int(len(p_sorted) * priv_thresh))
human_cutoff = max(10,int(len(p_sorted)*priv_thresh))
for i in range(min(10, len(p_sorted))):
if p_sorted[i][0] == 'Human_Human':
if p_sorted[i][0]=='Human_Human':
with open(userDir + '/BirdNET-Pi/HUMAN.txt', 'a') as rfile:
rfile.write(str(datetime.datetime.now()) + str(p_sorted[i]) + ' ' + str(human_cutoff) + '\n')
rfile.write(str(datetime.datetime.now())+str(p_sorted[i])+ ' ' + str(human_cutoff)+ '\n')
return p_sorted[:human_cutoff]
def analyzeAudioData(chunks, lat, lon, week, sensitivity, overlap,):
global INTERPRETER
@@ -204,57 +202,47 @@ def analyzeAudioData(chunks, lat, lon, week, sensitivity, overlap,):
# Make prediction
p = predict([sig, mdata], sensitivity)
# print("PPPPP",p)
HUMAN_DETECTED = False
# Catch if Human is recognized
HUMAN_DETECTED=False
#Catch if Human is recognized
for x in range(len(p)):
if "Human" in p[x][0]:
HUMAN_DETECTED = True
HUMAN_DETECTED=True
# Save result and timestamp
pred_end = pred_start + 3.0
# If human detected set all detections to human to make sure voices are not saved
if HUMAN_DETECTED is True:
p = [('Human_Human', 0.0)] * 10
#If human detected set all detections to human to make sure voices are not saved
if HUMAN_DETECTED == True:
p=[('Human_Human',0.0)]*10
detections[str(pred_start) + ';' + str(pred_end)] = p
pred_start = pred_end - overlap
print('DONE! Time', int((time.time() - start) * 10) / 10.0, 'SECONDS')
# print('DETECTIONS:::::',detections)
return detections
def sendAppriseNotifications(species, confidence):
def sendAppriseNotifications(species,confidence):
if os.path.exists(userDir + '/BirdNET-Pi/apprise.txt') and os.path.getsize(userDir + '/BirdNET-Pi/apprise.txt') > 0:
with open(userDir + '/BirdNET-Pi/scripts/thisrun.txt', 'r') as f:
this_run = f.readlines()
title = str(str(str([i for i in this_run if i.startswith('APPRISE_NOTIFICATION_TITLE')]
).split('=')[1]).split('\\')[0]).replace('"', '')
body = str(str(str([i for i in this_run if i.startswith('APPRISE_NOTIFICATION_BODY')]
).split('=')[1]).split('\\')[0]).replace('"', '')
title = str(str(str([i for i in this_run if i.startswith('APPRISE_NOTIFICATION_TITLE')]).split('=')[1]).split('\\')[0]).replace('"', '')
body = str(str(str([i for i in this_run if i.startswith('APPRISE_NOTIFICATION_BODY')]).split('=')[1]).split('\\')[0]).replace('"', '')
if str(str(str([i for i in this_run if i.startswith('APPRISE_NOTIFY_EACH_DETECTION')]).split('=')[1]).split('\\')[0]) == "1": # noqa E501
if str(str(str([i for i in this_run if i.startswith('APPRISE_NOTIFY_EACH_DETECTION')]).split('=')[1]).split('\\')[0]) == "1":
apobj = apprise.Apprise()
config = apprise.AppriseConfig()
config.add(userDir + '/BirdNET-Pi/apprise.txt')
apobj.add(config)
apobj.notify(
body=body.replace(
"$sciname",
species.split("_")[0]).replace(
"$comname",
species.split("_")[1]).replace(
"$confidence",
confidence),
body=body.replace("$sciname",species.split("_")[0]).replace("$comname",species.split("_")[1]).replace("$confidence",confidence),
title=title,
)
def writeResultsToFile(detections, min_conf, path):
print('WRITING RESULTS TO', path, '...', end=' ')
@@ -263,15 +251,13 @@ def writeResultsToFile(detections, min_conf, path):
rfile.write('Start (s);End (s);Scientific name;Common name;Confidence\n')
for d in detections:
for entry in detections[d]:
if entry[1] >= min_conf and ((entry[0] in INCLUDE_LIST or len(INCLUDE_LIST) == 0)
and (entry[0] not in EXCLUDE_LIST or len(EXCLUDE_LIST) == 0)):
sendAppriseNotifications(str(entry[0]), str(entry[1]))
if entry[1] >= min_conf and ((entry[0] in INCLUDE_LIST or len(INCLUDE_LIST) == 0) and (entry[0] not in EXCLUDE_LIST or len(EXCLUDE_LIST) == 0) ):
sendAppriseNotifications(str(entry[0]),str(entry[1]));
rfile.write(d + ';' + entry[0].replace('_', ';') + ';' + str(entry[1]) + '\n')
rcnt += 1
print('DONE! WROTE', rcnt, 'RESULTS.')
return
def handle_client(conn, addr):
global INCLUDE_LIST
global EXCLUDE_LIST
@@ -286,10 +272,10 @@ def handle_client(conn, addr):
if msg == DISCONNECT_MESSAGE:
connected = False
else:
# print(f"[{addr}] {msg}")
#print(f"[{addr}] {msg}")
args = type('', (), {})()
args.i = ''
args.o = ''
args.birdweather_id = '99999'
@@ -300,7 +286,8 @@ def handle_client(conn, addr):
args.sensitivity = 1.25
args.min_conf = 0.70
args.lat = -1
args.lon = -1
args.lon = -1
for line in msg.split('||'):
inputvars = line.split('=')
@@ -327,12 +314,14 @@ def handle_client(conn, addr):
elif inputvars[0] == 'lon':
args.lon = float(inputvars[1])
# Load custom species lists - INCLUDED and EXCLUDED
if not args.include_list == 'null':
INCLUDE_LIST = loadCustomSpeciesList(args.include_list)
else:
INCLUDE_LIST = []
if not args.exclude_list == 'null':
EXCLUDE_LIST = loadCustomSpeciesList(args.exclude_list)
else:
@@ -344,7 +333,7 @@ def handle_client(conn, addr):
audioData = readAudioData(args.i, args.overlap)
# Get Date/Time from filename in case Pi gets behind
# now = datetime.now()
#now = datetime.now()
full_file_name = args.i
print('FULL FILENAME: -' + full_file_name + '-')
file_name = Path(full_file_name).stem
@@ -352,14 +341,14 @@ def handle_client(conn, addr):
file_time = file_name.split('-birdnet-')[1]
date_time_str = file_date + ' ' + file_time
date_time_obj = datetime.datetime.strptime(date_time_str, '%Y-%m-%d %H:%M:%S')
# print('Date:', date_time_obj.date())
# print('Time:', date_time_obj.time())
#print('Date:', date_time_obj.date())
#print('Time:', date_time_obj.time())
print('Date-time:', date_time_obj)
now = date_time_obj
current_date = now.strftime("%Y-%m-%d")
current_time = now.strftime("%H:%M:%S")
current_iso8601 = now.astimezone(get_localzone()).isoformat()
week_number = int(now.strftime("%V"))
week = max(1, min(week_number, 48))
@@ -371,33 +360,32 @@ def handle_client(conn, addr):
# Write detections to output file
min_conf = max(0.01, min(args.min_conf, 0.99))
writeResultsToFile(detections, min_conf, args.o)
###############################################################################
###############################################################################
###############################################################################
###############################################################################
soundscape_uploaded = False
# Write detections to Database
myReturn = ''
for i in detections:
myReturn += str(i) + '-' + str(detections[i][0]) + '\n'
myReturn += str(i) + '-' + str(detections[i][0]) + '\n'
with open(userDir + '/BirdNET-Pi/BirdDB.txt', 'a') as rfile:
for d in detections:
for entry in detections[d]:
if entry[1] >= min_conf and ((entry[0] in INCLUDE_LIST or len(INCLUDE_LIST) == 0) and (
entry[0] not in EXCLUDE_LIST or len(EXCLUDE_LIST) == 0)):
rfile.write(str(current_date) + ';' + str(current_time) + ';' + entry[0].replace('_', ';') + ';'
+ str(entry[1]) + ";" + str(args.lat) + ';' + str(args.lon) +
';' + str(min_conf) + ';' + str(week) + ';'
+ str(args.sensitivity) + ';' + str(args.overlap) + '\n')
if entry[1] >= min_conf and ((entry[0] in INCLUDE_LIST or len(INCLUDE_LIST) == 0) and (entry[0] not in EXCLUDE_LIST or len(EXCLUDE_LIST) == 0) ):
rfile.write(str(current_date) + ';' + str(current_time) + ';' + entry[0].replace('_', ';') + ';' \
+ str(entry[1]) +";" + str(args.lat) + ';' + str(args.lon) + ';' + str(min_conf) + ';' + str(week) + ';' \
+ str(args.sensitivity) +';' + str(args.overlap) + '\n')
Date = str(current_date)
Time = str(current_time)
species = entry[0]
Sci_Name, Com_Name = species.split('_')
Sci_Name,Com_Name = species.split('_')
score = entry[1]
Confidence = str(round(score * 100))
Confidence = str(round(score*100))
Lat = str(args.lat)
Lon = str(args.lon)
Cutoff = str(args.min_conf)
@@ -406,91 +394,47 @@ def handle_client(conn, addr):
Overlap = str(args.overlap)
Com_Name = Com_Name.replace("'", "")
File_Name = Com_Name.replace(" ", "_") + '-' + Confidence + '-' + \
Date.replace("/", "-") + '-birdnet-' + Time + audiofmt
Date.replace("/", "-") + '-birdnet-' + Time + audiofmt
# Connect to SQLite Database
#Connect to SQLite Database
for attempt_number in range(3):
try:
try:
con = sqlite3.connect(userDir + '/BirdNET-Pi/scripts/birds.db')
cur = con.cursor()
cur.execute(
"INSERT INTO detections VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
(Date,
Time,
Sci_Name,
Com_Name,
str(score),
Lat,
Lon,
Cutoff,
Week,
Sens,
Overlap,
File_Name))
cur.execute("INSERT INTO detections VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (Date, Time, Sci_Name, Com_Name, str(score), Lat, Lon, Cutoff, Week, Sens, Overlap, File_Name))
con.commit()
con.close()
break
except BaseException:
except:
print("Database busy")
time.sleep(2)
print(str(current_date) +
';' +
str(current_time) +
';' +
entry[0].replace('_', ';') +
';' +
str(entry[1]) +
';' +
str(args.lat) +
';' +
str(args.lon) +
';' +
str(min_conf) +
';' +
str(week) +
';' +
str(args.sensitivity) +
';' +
str(args.overlap) +
Com_Name.replace(" ", "_") +
'-' +
str(score) +
'-' +
str(current_date) +
'-birdnet-' +
str(current_time) +
audiofmt +
'\n')
print(str(current_date) + ';' + str(current_time) + ';' + entry[0].replace('_', ';') + ';' + str(entry[1]) + ';' + str(args.lat) + ';' + str(args.lon) + ';' + str(min_conf) + ';' + str(week) + ';' + str(args.sensitivity) +';' + str(args.overlap) + Com_Name.replace(" ", "_") + '-' + str(score) + '-' + str(current_date) + '-birdnet-' + str(current_time) + audiofmt + '\n')
if birdweather_id != "99999":
try:
if soundscape_uploaded is False:
# POST soundscape to server
soundscape_url = "https://app.birdweather.com/api/v1/stations/" + \
birdweather_id + "/soundscapes" + "?timestamp=" + current_iso8601
soundscape_url = "https://app.birdweather.com/api/v1/stations/" + birdweather_id + "/soundscapes" + "?timestamp=" + current_iso8601
with open(args.i, 'rb') as f:
wav_data = f.read()
response = requests.post(
url=soundscape_url, data=wav_data, headers={
'Content-Type': 'application/octet-stream'})
response = requests.post(url=soundscape_url, data=wav_data, headers={'Content-Type': 'application/octet-stream'})
print("Soundscape POST Response Status - ", response.status_code)
sdata = response.json()
soundscape_id = sdata['soundscape']['id']
soundscape_uploaded = True
# POST detection to server
detection_url = "https://app.birdweather.com/api/v1/stations/" + \
birdweather_id + "/detections"
detection_url = "https://app.birdweather.com/api/v1/stations/" + birdweather_id + "/detections"
start_time = d.split(';')[0]
end_time = d.split(';')[1]
post_begin = "{ "
now_p_start = now + datetime.timedelta(seconds=float(start_time))
current_iso8601 = now_p_start.astimezone(get_localzone()).isoformat()
post_timestamp = "\"timestamp\": \"" + current_iso8601 + "\","
post_timestamp = "\"timestamp\": \"" + current_iso8601 + "\","
post_lat = "\"lat\": " + str(args.lat) + ","
post_lon = "\"lon\": " + str(args.lon) + ","
post_soundscape_id = "\"soundscapeId\": " + str(soundscape_id) + ","
@@ -501,23 +445,18 @@ def handle_client(conn, addr):
post_algorithm = "\"algorithm\": " + "\"alpha\"" + ","
post_confidence = "\"confidence\": " + str(entry[1])
post_end = " }"
post_json = post_begin + \
post_timestamp + post_lat + post_lon + \
post_soundscape_id + post_soundscape_start_time + \
post_soundscape_end_time + post_commonName + post_scientificName + \
post_algorithm + post_confidence + post_end
post_json = post_begin + post_timestamp + post_lat + post_lon + post_soundscape_id + post_soundscape_start_time + post_soundscape_end_time + post_commonName + post_scientificName + post_algorithm + post_confidence + post_end
print(post_json)
response = requests.post(detection_url, json=json.loads(post_json))
print("Detection POST Response Status - ", response.status_code)
except BaseException:
except:
print("Cannot POST right now")
conn.send(myReturn.encode(FORMAT))
# time.sleep(3)
conn.close()
#time.sleep(3)
conn.close()
def start():
# Load model