Feat : limit the number of audio files to keep per species (#121)

* New disk usage settings

* Use sudo for rm, just in case

* Remove set -x

* Correct code for png

* Correct today

* Correct dates

* Accepts date format of different distributions

* Avoid message

* Very significant processing time optimisation

* Update disk_species_clean.sh

* Avoid locking database

* Update disk_species_clean.sh

* Disable set -x (again)

* Add number of remaining files

* Add new variables to update_birdnet_snippets.sh

* Consolidate disk management elements

* Update disk_species_clean.sh

* Update disk_species_clean.sh

* Protect last 7 days

* Add options to advanced.php, in same style

* Remove from config

* Update config.php

* Reset max_disk_usage to hard 95

* Improve layout

* Remove automatic addition of MAX_FILE_SPECIES if not defined https://github.com/alexbelgium/hassio-addons/issues/1458

* Cosmetic white-listed changed to whitelist

* Add comma in text

* Update scripts/advanced.php

Co-authored-by: Nachtzuster <Nachtzuster@users.noreply.github.com>

* Update crontab if less than 5 #birdnet occurence (= all scripts not installed)

* Made .sh file executable

* Remove unused field k5 (second field of year) ; add explanations

* Call functions without ()

* Uninstall current cron before reinstalling them

* Update update_birdnet_snippets.sh

* Use $HOME instead of $mydir

---------

Co-authored-by: Nachtzuster <Nachtzuster@users.noreply.github.com>

homepage/views.php

@@ -142,7 +142,7 @@ if(isset($_GET['view'])){
       <button type=\"submit\" name=\"view\" value=\"Webterm\" form=\"views\">Web Terminal</button>
       <button type=\"submit\" name=\"view\" value=\"Included\" form=\"views\">Custom Species List</button>
       <button type=\"submit\" name=\"view\" value=\"Excluded\" form=\"views\">Excluded Species List</button>
-      <button type=\"submit\" name=\"view\" value=\"Whitelisted\" form=\"views\">White-listed Species List</button>
+      <button type=\"submit\" name=\"view\" value=\"Whitelisted\" form=\"views\">Whitelist Species List</button>
       </form>
       </div>";
   }

scripts/advanced.php

@@ -130,6 +130,13 @@ if(isset($_GET['submit'])) {
     }
   }
 
+if (isset($_GET["max_files_species"])) {
+    $max_files_species = $_GET["max_files_species"];
+    if (strcmp($max_files_species, $config['MAX_FILES_SPECIES']) !== 0) {
+        $contents = preg_replace("/MAX_FILES_SPECIES=.*/", "MAX_FILES_SPECIES=$max_files_species", $contents);
+    }
+}
+	
   if(isset($_GET["privacy_threshold"])) {
     $privacy_threshold = $_GET["privacy_threshold"];
     if(strcmp($privacy_threshold,$config['PRIVACY_THRESHOLD']) !== 0) {
@@ -274,12 +281,21 @@ $newconfig = get_config();
       </td></tr></table><br>
       
       <table class="settingstable"><tr><td>
-      <h2>Full Disk Behaviour</h2>
+      <h2>Disk Management</h2>
       <label for="purge">
       <input name="full_disk" type="radio" id="purge" value="purge" <?php if (strcmp($newconfig['FULL_DISK'], "purge") == 0) { echo "checked"; }?>>Purge</label>
       <label for="keep">
       <input name="full_disk" type="radio" id="keep" value="keep" <?php if (strcmp($newconfig['FULL_DISK'], "keep") == 0) { echo "checked"; }?>>Keep</label>
       <p>When the disk becomes full, you can choose to 'purge' old files to make room for new ones or 'keep' your data and stop all services instead.<br>Note: you can exclude specific files from 'purge' on the Recordings page.</p>
+      <br>
+      <label for="max_files_species">Amount of files to keep for each specie :</label>
+      <input name="max_files_species" type="number" style="width:6em;" min="0" step="1" value="<?php print($newconfig['MAX_FILES_SPECIES']);?>"/>
+      </td></tr><tr><td>
+      If different than 0 (keep all), defines the maximum number of files to be kept for each species, with priority give to files with highest confidence. 
+      This value does not take into account the last 7 days (protected by default).
+      </td></tr><tr><td>
+      Note only the spectrogram and audio files are deleted, the obsevation data remains in the database.
+      The files protected through the "lock" icon are also not affected.
       </td></tr></table><br>
       <table class="settingstable"><tr><td>
 

scripts/disk_species_clean.sh

@@ -0,0 +1,77 @@
+#!/bin/bash
+
+# KEEP ONLY THE NUMBER OF FILES PER SPECIES DEFINED IN THE OPTIONS
+
+source /etc/birdnet/birdnet.conf
+base_dir="$HOME/BirdSongs/Extracted/By_Date"
+max_files_species="${MAX_FILES_SPECIES:-1000}"
+cd "$base_dir" || true
+
+# If max_files_species is not higher than 1, exit
+if [[ "$max_files_species" -lt 1 ]]; then
+    exit 0
+fi
+
+# Get unique species
+bird_names=$(
+    sqlite3 -readonly "$HOME"/BirdNET-Pi/scripts/birds.db <<EOF
+.mode column
+.headers off
+SELECT DISTINCT Com_Name FROM detections;
+.quit
+EOF
+)
+
+# Sanitize the bird names (remove single quotes and replace spaces with underscores)
+sanitized_names="$(echo "$bird_names" | tr ' ' '_' | tr -d "'" | grep '[[:alnum:]]')"
+# Remove trailing underscores
+sanitized_names=$(echo "$sanitized_names" | sed 's/_*$//')
+# Define how date works
+dateformat=""
+if test "$(date -d "-7 days" '+%Y-%m-%d' 2>/dev/null)"; then
+    dateformat=" days"
+fi
+
+# find detail :
+# In the base folders (that corresponds to the BirdSongs/By_date)
+# Look for all folders that have the correct species names whatever the date
+# Look for files that have the correct format (containing a date), and that have an extension
+# For all That are not *.png (as the objective is to limit the number of audio files)
+# That were not taken in the past 7 days (= that don't contain the date from that past 7 days). $dateformat is configured as a different variables, as ubuntu accepts "5 days" while alpine accepts only "5"
+# That are not included in the file disk_check_exclude.txt that lists files protected from purge
+# If the specie name had a "-" in it, it must be converted to "=" to ensure that we have always the same number of "-" separated fields in the filename
+# Sort by confidence level (field 4 separated by -)
+# Sort by date (1 for year, 2 for month, 3 for days)
+# Remove the top x files, corresponding to the files best matching the criteria of confidence + age ; this corresponds to the number of file to keep (in addition to protected files
+# Rename species that had a = in their name to - (we don't need anymore - separated fields)
+# Duplicate all lines to append .png at the end, to remove the linked png
+# This appends a fake "temp" file, so that the sudo rm has at least one file to delete and does not hang
+# Delete files, then once all files are deleted echo the number of remaining files
+
+# Read each line from the variable and echo the species
+while read -r species; do
+    echo -n "$species : "
+    species_san="${species/-/=}"
+    # Dummy file to execute the rm using xargs even if no files are there. Best solution found for code speed
+    touch temp
+    find */"$species" -type f -name "*[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]*.*" \
+        -not -name "*.png" \
+        -not -name "*$(date -d "-7$dateformat" '+%Y-%m-%d')*" \
+        -not -name "*$(date -d "-6$dateformat" '+%Y-%m-%d')*" \
+        -not -name "*$(date -d "-5$dateformat" '+%Y-%m-%d')*" \
+        -not -name "*$(date -d "-4$dateformat" '+%Y-%m-%d')*" \
+        -not -name "*$(date -d "-3$dateformat" '+%Y-%m-%d')*" \
+        -not -name "*$(date -d "-2$dateformat" '+%Y-%m-%d')*" \
+        -not -name "*$(date -d "-1$dateformat" '+%Y-%m-%d')*" \
+        -not -name "*$(date '+%Y-%m-%d')*" |
+        grep -vFf "$HOME/BirdNET-Pi/scripts/disk_check_exclude.txt" |
+        sed "s|$species|$species_san|g" |
+        sort -t'-' -k4,4nr -k1,1nr -k2,2nr -k3,3nr |
+        tail -n +"$((max_files_species + 1))" |
+        sed "s|$species_san|$species|g" |
+        sed 'p; s/\(\.[^.]*\)$/\1.png/' |
+        awk 'BEGIN{print "temp"} {print}' |
+        xargs sudo rm && echo "success ($(find */"$species" -type f -name "*[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]*.*" \
+        -not -name "*.png" | wc -l)) remaining" || echo "failed ($?)"
+# rm to be changed to touch or echo if you want to test without deletion
+done <<<"$sanitized_names"

scripts/install_config.sh

@@ -129,6 +129,19 @@ INFO_SITE="ALLABOUTBIRDS"
 
 COLOR_SCHEME="light"
 
+#------------------------------  Disk Management  ------------------------------#
+## FULL_DISK can be set to configure how the system reacts to a full disk
+## purge = Remove the oldest day's worth of recordings
+## keep = Keep all data and 'stop_core_services.sh'
+
+FULL_DISK=purge
+
+## Maximum amount of files to keep for a given specie (0 = keep all)
+## Files from the last 7 days, and files protected from purge, are not taken into
+## account in this number
+
+MAX_FILES_SPECIES=0
+
 ################################################################################
 #--------------------------------  Defaults  ----------------------------------#
 ################################################################################
@@ -199,12 +212,6 @@ FREQSHIFT_PITCH=-1500
 
 CHANNELS=2
 
-## FULL_DISK can be set to configure how the system reacts to a full disk
-## purge = Remove the oldest day's worth of recordings
-## keep = Keep all data and 'stop_core_services.sh'
-
-FULL_DISK=purge
-
 ## PRIVACY_THRESHOLD can be set to enable sensitivity to Human sounds. This
 ## setting is an effort to introduce privacy into the data collection.
 ## The PRIVACY_THRESHOLD value represents a percentage of the entire species

scripts/update_birdnet_snippets.sh

@@ -94,6 +94,10 @@ if ! grep -E '^COLOR_SCHEME=' /etc/birdnet/birdnet.conf &>/dev/null;then
   echo "COLOR_SCHEME=\"light\"" >> /etc/birdnet/birdnet.conf
 fi
 
+if ! grep -E '^MAX_FILES_SPECIES=' /etc/birdnet/birdnet.conf &>/dev/null;then
+  echo "MAX_FILES_SPECIES=\"0\"" >> /etc/birdnet/birdnet.conf
+fi
+
 [ -d $RECS_DIR/StreamData ] || sudo_with_user mkdir -p $RECS_DIR/StreamData
 [ -L ${EXTRACTED}/spectrogram.png ] || sudo_with_user ln -sf ${RECS_DIR}/StreamData/spectrogram.png ${EXTRACTED}/spectrogram.png
 
@@ -180,6 +184,13 @@ if [ -L /usr/local/bin/birdnet_analysis.sh ];then
   rm -f /usr/local/bin/birdnet_analysis.sh
 fi
 
+# Clean state and update cron if all scripts are not installed
+if [ "$(grep -o "#birdnet" /etc/crontab | wc -l)" -lt 5 ]; then
+  sudo sed -i '/birdnet/,+1d' /etc/crontab
+  sed "s/\$USER/$USER/g" "$HOME"/BirdNET-Pi/templates/cleanup.cron >> /etc/crontab
+  sed "s/\$USER/$USER/g" "$HOME"/BirdNET-Pi/templates/weekly_report.cron >> /etc/crontab
+fi
+
 # update snippets above
 
 systemctl daemon-reload

templates/cleanup.cron

@@ -1,6 +1,8 @@
 #birdnet
 */5 * * * * $USER /usr/local/bin/disk_check.sh >/dev/null 2>&1
 #birdnet
+0 2 * * * $USER /usr/local/bin/disk_species_clean.sh >/dev/null 2>&1
+#birdnet
 */3 * * * * $USER /usr/local/bin/cleanup.sh >/dev/null 2>&1
 #birdnet
 @reboot $USER /usr/local/bin/cleanup.sh >/dev/null 2>&1

version.md

@@ -39,7 +39,7 @@
 - BirdWeather Support
 - Bug Fix for systemd-networkd-wait-online.service
 - Bug Fix for `install_noip2.sh` for NoIP DUC Support
-- New `disk_check.sh` utitlity/crontab entry to `stop_core_services.sh`
+- New `disk_check.sh` utility/crontab entry to `stop_core_services.sh`
   when disk space is greater than 95%
 
 # main v0.9 -- pre-installed image