From d092d4fa7981fd13f65a679d1b143d0c80ee1134 Mon Sep 17 00:00:00 2001 From: mcguirepr89 Date: Fri, 6 May 2022 10:23:34 -0400 Subject: [PATCH 01/16] replaced radio buttons with slider --- scripts/advanced.php | 48 ++++++++++++++++++++------------------------ 1 file changed, 22 insertions(+), 26 deletions(-) diff --git a/scripts/advanced.php b/scripts/advanced.php index bfcd285..f3859df 100644 --- a/scripts/advanced.php +++ b/scripts/advanced.php @@ -109,27 +109,16 @@ if(isset($_GET['submit'])) { } } - if(isset($_GET["privacy_mode"])) { - $privacy_mode = $_GET["privacy_mode"]; - if(strcmp($config['PRIVACY_MODE'], "1") == 0 ) { + if(isset($_GET["privacy_threshold"])) { + $privacy_threshold = $_GET["privacy_threshold"]; + if(strcmp($config['PRIVACY_THRESHOLD'], "1") == 0 ) { $pmode = "on"; - }elseif(strcmp($config['PRIVACY_MODE'], "") == 0) { + }elseif(strcmp($config['PRIVACY_THRESHOLD'], "") == 0) { $pmode = "off"; } - if(strcmp($privacy_mode,$pmode) !== 0) { - $contents = preg_replace("/PRIVACY_MODE=.*/", "PRIVACY_MODE=$privacy_mode", $contents); - $contents2 = preg_replace("/PRIVACY_MODE=.*/", "PRIVACY_MODE=$privacy_mode", $contents2); - if(strcmp($privacy_mode,"on") == 0) { - exec('sudo sed -i \'s/\/usr\/local\/bin\/server.py/\/usr\/local\/bin\/privacy_server.py/g\' ../../BirdNET-Pi/templates/birdnet_server.service'); - exec('sudo systemctl daemon-reload'); - exec('restart_services.sh'); - header('Location: /log'); - } elseif(strcmp($privacy_mode,"off") == 0) { - exec('sudo sed -i \'s/\/usr\/local\/bin\/privacy_server.py/\/usr\/local\/bin\/server.py/g\' ../../BirdNET-Pi/templates/birdnet_server.service'); - exec('sudo systemctl daemon-reload'); - exec('restart_services.sh'); - header('Location: /log'); - } + if(strcmp($privacy_threshold,$pmode) !== 0) { + $contents = preg_replace("/PRIVACY_THRESHOLD=.*/", "PRIVACY_THRESHOLD=$privacy_threshold", $contents); + $contents2 = preg_replace("/PRIVACY_THRESHOLD=.*/", "PRIVACY_THRESHOLD=$privacy_threshold", $contents2); } } @@ -194,14 +183,21 @@ if (file_exists('./scripts/thisrun.txt')) { ?>

Advanced Settings

- - - -

Privacy mode can be set to 'on' or 'off' to configure analysis to be more sensitive to human detections. Privacy mode 'on' will purge any data that receives even a low Human confidence score. - Please note that changing this setting restarts services and replaces the running server. It will take about 90, so please be patient!

- +
+
+ +

Value: %

+
+ From 1a5e81a46fa5538dcb30ff5ec42539c1e3bcf5f5 Mon Sep 17 00:00:00 2001 From: mcguirepr89 Date: Fri, 6 May 2022 10:36:47 -0400 Subject: [PATCH 02/16] adding new PRIVACY_THRESHOLD variable and removing PRIVACY_MODE variable --- birdnet.conf-defaults | 11 +++++++---- scripts/install_config.sh | 11 +++++++---- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/birdnet.conf-defaults b/birdnet.conf-defaults index f21fcad..871d30e 100644 --- a/birdnet.conf-defaults +++ b/birdnet.conf-defaults @@ -120,11 +120,14 @@ CHANNELS=2 FULL_DISK=purge -## PRIVACY_MODE can be set to 'on' or 'off' to configure analysis to be more -## sensitive to human detections. PRIVACY_MODE 'on' will purge any data that -## receives even a low HUMAN_HUMAN confidence score. +## PRIVACY_THRESHOLD can be set to enable sensitivity to Human sounds. This +## setting is an effort to introduce privacy into the data collection. +## The PRIVACY_THRESHOLD value represents a percentage of the entire species +## list used during analysis. If a human sound is predicted anywhere within +## the precentile set below, no data is collected for that audio chunk. +## Valid range: 0-30 -PRIVACY_MODE=off +PRIVACY_THRESHOLD=0 ## RECORDING_LENGTH sets the length of the recording that BirdNET-Lite will ## analyze. diff --git a/scripts/install_config.sh b/scripts/install_config.sh index 351c3a2..ee456b3 100755 --- a/scripts/install_config.sh +++ b/scripts/install_config.sh @@ -133,11 +133,14 @@ CHANNELS=2 FULL_DISK=purge -## PRIVACY_MODE can be set to 'on' or 'off' to configure analysis to be more -## sensitive to human detections. PRIVACY_MODE 'on' will purge any data that -## receives even a low HUMAN_HUMAN confidence score. +## PRIVACY_THRESHOLD can be set to enable sensitivity to Human sounds. This +## setting is an effort to introduce privacy into the data collection. +## The PRIVACY_THRESHOLD value represents a percentage of the entire species +## list used during analysis. If a human sound is predicted anywhere within +## the precentile set below, no data is collected for that audio chunk. +## Valid range: 0-30 -PRIVACY_MODE=off +PRIVACY_THRESHOLD=0 ## RECORDING_LENGTH sets the length of the recording that BirdNET-Lite will ## analyze. From 5b4aec2ab8ed6b2ecec709140ff893af0181e973 Mon Sep 17 00:00:00 2001 From: mcguirepr89 Date: Fri, 6 May 2022 11:24:07 -0400 Subject: [PATCH 03/16] css for the slider advanced.php updates birdnet.conf now removed privacy_server.py added snippets to update_birdnet_snippets.sh --- homepage/style.css | 30 ++ scripts/advanced.php | 15 +- scripts/privacy_server.py | 456 ----------------------------- scripts/update_birdnet_snippets.sh | 8 +- 4 files changed, 41 insertions(+), 468 deletions(-) delete mode 100755 scripts/privacy_server.py diff --git a/homepage/style.css b/homepage/style.css index 6469e1c..ba5e360 100644 --- a/homepage/style.css +++ b/homepage/style.css @@ -349,6 +349,36 @@ button:hover { width: 20%; } +.slider { + -webkit-appearance: none; + width: 33%; + height: 15px; + border-radius: 5px; + background: #d3d3d3; + outline: none; + opacity: 0.7; + -webkit-transition: .2s; + transition: opacity .2s; +} + +.slider::-webkit-slider-thumb { + -webkit-appearance: none; + appearance: none; + width: 25px; + height: 25px; + border-radius: 50%; + background: #04AA6D; + cursor: pointer; +} + +.slider::-moz-range-thumb { + width: 25px; + height: 25px; + border-radius: 50%; + background: #04AA6D; + cursor: pointer; +} + #body::-webkit-scrollbar { # display:none #} diff --git a/scripts/advanced.php b/scripts/advanced.php index f3859df..87c1f89 100644 --- a/scripts/advanced.php +++ b/scripts/advanced.php @@ -111,12 +111,7 @@ if(isset($_GET['submit'])) { if(isset($_GET["privacy_threshold"])) { $privacy_threshold = $_GET["privacy_threshold"]; - if(strcmp($config['PRIVACY_THRESHOLD'], "1") == 0 ) { - $pmode = "on"; - }elseif(strcmp($config['PRIVACY_THRESHOLD'], "") == 0) { - $pmode = "off"; - } - if(strcmp($privacy_threshold,$pmode) !== 0) { + if(strcmp($privacy_threshold,$config['PRIVACY_THRESHOLD']) !== 0) { $contents = preg_replace("/PRIVACY_THRESHOLD=.*/", "PRIVACY_THRESHOLD=$privacy_threshold", $contents); $contents2 = preg_replace("/PRIVACY_THRESHOLD=.*/", "PRIVACY_THRESHOLD=$privacy_threshold", $contents2); } @@ -185,12 +180,12 @@ if (file_exists('./scripts/thisrun.txt')) {
- -

Value: %

+ +

Value: %

+

If a Human is predicted anywhere among the top predictions, the sample will be considered of human origin and no data will be collected.

From ca443ecbef0a49d663595345d148136c87bc51c0 Mon Sep 17 00:00:00 2001 From: ehpersonal38 <103586016+ehpersonal38@users.noreply.github.com> Date: Fri, 6 May 2022 13:32:03 -0400 Subject: [PATCH 08/16] oops --- scripts/advanced.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/advanced.php b/scripts/advanced.php index c71cd24..24f4879 100644 --- a/scripts/advanced.php +++ b/scripts/advanced.php @@ -115,6 +115,8 @@ if(isset($_GET['submit'])) { $contents = preg_replace("/PRIVACY_THRESHOLD=.*/", "PRIVACY_THRESHOLD=$privacy_threshold", $contents); $contents2 = preg_replace("/PRIVACY_THRESHOLD=.*/", "PRIVACY_THRESHOLD=$privacy_threshold", $contents2); } + + exec('sudo systemctl restart birdnet_server.service'); } if(isset($_GET["rec_card"])) { @@ -203,7 +205,7 @@ if (file_exists('./scripts/thisrun.txt')) { document.getElementById("predictionCount").innerHTML = parseInt((this.value * )/100); } -

If a Human is predicted anywhere among the top predictions, the sample will be considered of human origin and no data will be collected.

+

If a Human is predicted anywhere among the top predictions, the sample will be considered of human origin and no data will be collected.

From 57bbc83a4c23e4553dcd36590b582878ca9fc389 Mon Sep 17 00:00:00 2001 From: mcguirepr89 Date: Fri, 6 May 2022 13:46:02 -0400 Subject: [PATCH 09/16] using webroot for file check on `$count` --- scripts/advanced.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/advanced.php b/scripts/advanced.php index 24f4879..db3140c 100644 --- a/scripts/advanced.php +++ b/scripts/advanced.php @@ -165,8 +165,8 @@ if(isset($_GET['submit'])) { fwrite($fh2, $contents2); } -$count_included = count(file("/home/pi/BirdNET-Pi/include_species_list.txt")); -$count_labels = count(file("/home/pi/BirdNET-Pi/model/labels.txt")); +$count_included = count(file("./scripts/include_species_list.txt")); +$count_labels = count(file("./scripts/labels.txt")); if($count_included > 0) { $count = $count_included; From 4bfaf2b0d499c786840ffbd8a0d13798ecf05a33 Mon Sep 17 00:00:00 2001 From: CaiusX Date: Sun, 8 May 2022 16:55:10 +0200 Subject: [PATCH 10/16] server.py with Privacy Threshold --- scripts/server.py | 37 ++++++++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/scripts/server.py b/scripts/server.py index d025f32..514c793 100755 --- a/scripts/server.py +++ b/scripts/server.py @@ -47,6 +47,7 @@ userDir = os.path.expanduser('~') with open(userDir + '/BirdNET-Pi/scripts/thisrun.txt', 'r') as f: this_run = f.readlines() audiofmt = "." + str(str(str([i for i in this_run if i.startswith('AUDIOFMT')]).split('=')[1]).split('\\')[0]) + priv_thresh = float("." + str(str(str([i for i in this_run if i.startswith('PRIVACY_THRESHOLD')]).split('=')[1]).split('\\')[0])) def loadModel(): @@ -164,14 +165,23 @@ def predict(sample, sensitivity): # Sort by score p_sorted = sorted(p_labels.items(), key=operator.itemgetter(1), reverse=True) + + print("DATABASE SIZE:", len(p_sorted)) + print("HUMAN-CUTOFF AT:", int(len(p_sorted)*priv_thresh)/10) # Remove species that are on blacklist for i in range(min(10, len(p_sorted))): - if p_sorted[i][0] in ['Human_Human', 'Non-bird_Non-bird', 'Noise_Noise']: + if p_sorted[i][0] in ['Non-bird_Non-bird', 'Noise_Noise']: p_sorted[i] = (p_sorted[i][0], 0.0) + if p_sorted[i][0]=='Human_Human': +# print("HUMAN SCORE:",str(p_sorted[i])) +# HUMAN_FLAG=True + with open(userDir + '/BirdNET-Pi/HUMAN.txt', 'a') as rfile: + rfile.write(str(datetime.datetime.now())+str(p_sorted[i])+ '\n') + human_cutoff = max(10,int(len(p_sorted)*priv_thresh/10)) - # Only return first the top ten results - return p_sorted[:10] + + return p_sorted[:human_cutoff] def analyzeAudioData(chunks, lat, lon, week, sensitivity, overlap,): global INTERPRETER @@ -193,16 +203,33 @@ def analyzeAudioData(chunks, lat, lon, week, sensitivity, overlap,): # Make prediction p = predict([sig, mdata], sensitivity) - +# print("PPPPP",p) + HUMAN_DETECTED=False + #Catch if Human is recognized + for x in range(len(p)): + if "Human" in p[x][0]: +# print("HUMAN DETECTED!!",p[x][0]) + #clear list + HUMAN_DETECTED=True + print("CHUNK -----",c) + # Save result and timestamp pred_end = pred_start + 3.0 + + if HUMAN_DETECTED == True: + p=[('Human_Human',0.0)]*10 + print("HUMAN DETECTED!!!",p) + detections[str(pred_start) + ';' + str(pred_end)] = p + pred_start = pred_end - overlap print('DONE! Time', int((time.time() - start) * 10) / 10.0, 'SECONDS') - +# print('DETECTIONS:::::',detections) return detections + + def writeResultsToFile(detections, min_conf, path): print('WRITING RESULTS TO', path, '...', end=' ') From 45bcf1b977e6e249db86f3dfced5527034a52e21 Mon Sep 17 00:00:00 2001 From: CaiusX Date: Mon, 9 May 2022 10:37:32 +0200 Subject: [PATCH 11/16] Updated server.py removing excessive printing of testing garbage --- scripts/server.py | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/scripts/server.py b/scripts/server.py index 514c793..c1300b7 100755 --- a/scripts/server.py +++ b/scripts/server.py @@ -1,3 +1,4 @@ +#!/home/pi/BirdNET-Pi/birdnet/bin/python3 import socket import threading import os @@ -47,7 +48,7 @@ userDir = os.path.expanduser('~') with open(userDir + '/BirdNET-Pi/scripts/thisrun.txt', 'r') as f: this_run = f.readlines() audiofmt = "." + str(str(str([i for i in this_run if i.startswith('AUDIOFMT')]).split('=')[1]).split('\\')[0]) - priv_thresh = float("." + str(str(str([i for i in this_run if i.startswith('PRIVACY_THRESHOLD')]).split('=')[1]).split('\\')[0])) + priv_thresh = float("." + str(str(str([i for i in this_run if i.startswith('PRIVACY_THRESHOLD')]).split('=')[1]).split('\\')[0]))/10 def loadModel(): @@ -166,20 +167,17 @@ def predict(sample, sensitivity): # Sort by score p_sorted = sorted(p_labels.items(), key=operator.itemgetter(1), reverse=True) - print("DATABASE SIZE:", len(p_sorted)) - print("HUMAN-CUTOFF AT:", int(len(p_sorted)*priv_thresh)/10) +# #print("DATABASE SIZE:", len(p_sorted)) +# #print("HUMAN-CUTOFF AT:", int(len(p_sorted)*priv_thresh)/10) +# +# # Remove species that are on blacklist + + human_cutoff = max(10,int(len(p_sorted)*priv_thresh)) - # Remove species that are on blacklist for i in range(min(10, len(p_sorted))): - if p_sorted[i][0] in ['Non-bird_Non-bird', 'Noise_Noise']: - p_sorted[i] = (p_sorted[i][0], 0.0) if p_sorted[i][0]=='Human_Human': -# print("HUMAN SCORE:",str(p_sorted[i])) -# HUMAN_FLAG=True with open(userDir + '/BirdNET-Pi/HUMAN.txt', 'a') as rfile: - rfile.write(str(datetime.datetime.now())+str(p_sorted[i])+ '\n') - human_cutoff = max(10,int(len(p_sorted)*priv_thresh/10)) - + rfile.write(str(datetime.datetime.now())+str(p_sorted[i])+ ' ' + str(human_cutoff)+ '\n') return p_sorted[:human_cutoff] @@ -205,20 +203,18 @@ def analyzeAudioData(chunks, lat, lon, week, sensitivity, overlap,): p = predict([sig, mdata], sensitivity) # print("PPPPP",p) HUMAN_DETECTED=False + #Catch if Human is recognized for x in range(len(p)): if "Human" in p[x][0]: -# print("HUMAN DETECTED!!",p[x][0]) - #clear list HUMAN_DETECTED=True - print("CHUNK -----",c) # Save result and timestamp pred_end = pred_start + 3.0 + #If human detected set all detections to human to make sure voices are not saved if HUMAN_DETECTED == True: p=[('Human_Human',0.0)]*10 - print("HUMAN DETECTED!!!",p) detections[str(pred_start) + ';' + str(pred_end)] = p From f12b81a40d5df29de0ec8bd3a238c6c3ee03b959 Mon Sep 17 00:00:00 2001 From: Patrick McGuire <60325264+mcguirepr89@users.noreply.github.com> Date: Mon, 9 May 2022 12:54:12 -0400 Subject: [PATCH 12/16] Update server.py --- scripts/server.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/server.py b/scripts/server.py index c1300b7..b523884 100755 --- a/scripts/server.py +++ b/scripts/server.py @@ -1,4 +1,3 @@ -#!/home/pi/BirdNET-Pi/birdnet/bin/python3 import socket import threading import os From bfa69330ab196944dd1956a9e40b4ff1905874d6 Mon Sep 17 00:00:00 2001 From: mcguirepr89 Date: Mon, 9 May 2022 13:40:49 -0400 Subject: [PATCH 13/16] reducing options to be realistic --- scripts/advanced.php | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/advanced.php b/scripts/advanced.php index db3140c..20e2cd8 100644 --- a/scripts/advanced.php +++ b/scripts/advanced.php @@ -114,9 +114,8 @@ if(isset($_GET['submit'])) { if(strcmp($privacy_threshold,$config['PRIVACY_THRESHOLD']) !== 0) { $contents = preg_replace("/PRIVACY_THRESHOLD=.*/", "PRIVACY_THRESHOLD=$privacy_threshold", $contents); $contents2 = preg_replace("/PRIVACY_THRESHOLD=.*/", "PRIVACY_THRESHOLD=$privacy_threshold", $contents2); + exec('restart_services.sh'); } - - exec('sudo systemctl restart birdnet_server.service'); } if(isset($_GET["rec_card"])) { @@ -191,7 +190,7 @@ if (file_exists('./scripts/thisrun.txt')) {
- +

Value: %

-

If a Human is predicted anywhere among the top predictions, the sample will be considered of human origin and no data will be collected.

+

If a Human is predicted anywhere among the top predictions, the sample will be considered of human origin and no data will be collected. Start with 1% and move up as needed.