From b77f579e35cddc6c32d1c2f116fee4b5cf3f413a Mon Sep 17 00:00:00 2001 From: Alexandre <44178713+alexbelgium@users.noreply.github.com> Date: Fri, 12 Jul 2024 18:18:27 +0200 Subject: [PATCH] Feat : limit the number of audio files to keep per species (#121) * New disk usage settings * Use sudo for rm, just in case * Remove set -x * Correct code for png * Correct today * Correct dates * Accepts date format of different distributions * Avoid message * Very significant processing time optimisation * Update disk_species_clean.sh * Avoid locking database * Update disk_species_clean.sh * Disable set -x (again) * Add number of remaining files * Add new variables to update_birdnet_snippets.sh * Consolidate disk management elements * Update disk_species_clean.sh * Update disk_species_clean.sh * Protect last 7 days * Add options to advanced.php, in same style * Remove from config * Update config.php * Reset max_disk_usage to hard 95 * Improve layout * Remove automatic addition of MAX_FILE_SPECIES if not defined https://github.com/alexbelgium/hassio-addons/issues/1458 * Cosmetic white-listed changed to whitelist * Add comma in text * Update scripts/advanced.php Co-authored-by: Nachtzuster * Update crontab if less than 5 #birdnet occurence (= all scripts not installed) * Made .sh file executable * Remove unused field k5 (second field of year) ; add explanations * Call functions without () * Uninstall current cron before reinstalling them * Update update_birdnet_snippets.sh * Use $HOME instead of $mydir --------- Co-authored-by: Nachtzuster --- homepage/views.php | 2 +- scripts/advanced.php | 18 ++++++- scripts/disk_species_clean.sh | 77 ++++++++++++++++++++++++++++++ scripts/install_config.sh | 19 +++++--- scripts/update_birdnet_snippets.sh | 11 +++++ templates/cleanup.cron | 2 + version.md | 2 +- 7 files changed, 122 insertions(+), 9 deletions(-) create mode 100755 scripts/disk_species_clean.sh diff --git a/homepage/views.php b/homepage/views.php index b2f2205..81d744d 100644 --- a/homepage/views.php +++ b/homepage/views.php @@ -142,7 +142,7 @@ if(isset($_GET['view'])){ - + "; } diff --git a/scripts/advanced.php b/scripts/advanced.php index fc6f430..c4d4caf 100644 --- a/scripts/advanced.php +++ b/scripts/advanced.php @@ -130,6 +130,13 @@ if(isset($_GET['submit'])) { } } +if (isset($_GET["max_files_species"])) { + $max_files_species = $_GET["max_files_species"]; + if (strcmp($max_files_species, $config['MAX_FILES_SPECIES']) !== 0) { + $contents = preg_replace("/MAX_FILES_SPECIES=.*/", "MAX_FILES_SPECIES=$max_files_species", $contents); + } +} + if(isset($_GET["privacy_threshold"])) { $privacy_threshold = $_GET["privacy_threshold"]; if(strcmp($privacy_threshold,$config['PRIVACY_THRESHOLD']) !== 0) { @@ -274,12 +281,21 @@ $newconfig = get_config();
-

Full Disk Behaviour

+

Disk Management

When the disk becomes full, you can choose to 'purge' old files to make room for new ones or 'keep' your data and stop all services instead.
Note: you can exclude specific files from 'purge' on the Recordings page.

+
+ + +
+ If different than 0 (keep all), defines the maximum number of files to be kept for each species, with priority give to files with highest confidence. + This value does not take into account the last 7 days (protected by default). +
+ Note only the spectrogram and audio files are deleted, the obsevation data remains in the database. + The files protected through the "lock" icon are also not affected.

diff --git a/scripts/disk_species_clean.sh b/scripts/disk_species_clean.sh new file mode 100755 index 0000000..2526ea1 --- /dev/null +++ b/scripts/disk_species_clean.sh @@ -0,0 +1,77 @@ +#!/bin/bash + +# KEEP ONLY THE NUMBER OF FILES PER SPECIES DEFINED IN THE OPTIONS + +source /etc/birdnet/birdnet.conf +base_dir="$HOME/BirdSongs/Extracted/By_Date" +max_files_species="${MAX_FILES_SPECIES:-1000}" +cd "$base_dir" || true + +# If max_files_species is not higher than 1, exit +if [[ "$max_files_species" -lt 1 ]]; then + exit 0 +fi + +# Get unique species +bird_names=$( + sqlite3 -readonly "$HOME"/BirdNET-Pi/scripts/birds.db </dev/null)"; then + dateformat=" days" +fi + +# find detail : +# In the base folders (that corresponds to the BirdSongs/By_date) +# Look for all folders that have the correct species names whatever the date +# Look for files that have the correct format (containing a date), and that have an extension +# For all That are not *.png (as the objective is to limit the number of audio files) +# That were not taken in the past 7 days (= that don't contain the date from that past 7 days). $dateformat is configured as a different variables, as ubuntu accepts "5 days" while alpine accepts only "5" +# That are not included in the file disk_check_exclude.txt that lists files protected from purge +# If the specie name had a "-" in it, it must be converted to "=" to ensure that we have always the same number of "-" separated fields in the filename +# Sort by confidence level (field 4 separated by -) +# Sort by date (1 for year, 2 for month, 3 for days) +# Remove the top x files, corresponding to the files best matching the criteria of confidence + age ; this corresponds to the number of file to keep (in addition to protected files +# Rename species that had a = in their name to - (we don't need anymore - separated fields) +# Duplicate all lines to append .png at the end, to remove the linked png +# This appends a fake "temp" file, so that the sudo rm has at least one file to delete and does not hang +# Delete files, then once all files are deleted echo the number of remaining files + +# Read each line from the variable and echo the species +while read -r species; do + echo -n "$species : " + species_san="${species/-/=}" + # Dummy file to execute the rm using xargs even if no files are there. Best solution found for code speed + touch temp + find */"$species" -type f -name "*[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]*.*" \ + -not -name "*.png" \ + -not -name "*$(date -d "-7$dateformat" '+%Y-%m-%d')*" \ + -not -name "*$(date -d "-6$dateformat" '+%Y-%m-%d')*" \ + -not -name "*$(date -d "-5$dateformat" '+%Y-%m-%d')*" \ + -not -name "*$(date -d "-4$dateformat" '+%Y-%m-%d')*" \ + -not -name "*$(date -d "-3$dateformat" '+%Y-%m-%d')*" \ + -not -name "*$(date -d "-2$dateformat" '+%Y-%m-%d')*" \ + -not -name "*$(date -d "-1$dateformat" '+%Y-%m-%d')*" \ + -not -name "*$(date '+%Y-%m-%d')*" | + grep -vFf "$HOME/BirdNET-Pi/scripts/disk_check_exclude.txt" | + sed "s|$species|$species_san|g" | + sort -t'-' -k4,4nr -k1,1nr -k2,2nr -k3,3nr | + tail -n +"$((max_files_species + 1))" | + sed "s|$species_san|$species|g" | + sed 'p; s/\(\.[^.]*\)$/\1.png/' | + awk 'BEGIN{print "temp"} {print}' | + xargs sudo rm && echo "success ($(find */"$species" -type f -name "*[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]*.*" \ + -not -name "*.png" | wc -l)) remaining" || echo "failed ($?)" +# rm to be changed to touch or echo if you want to test without deletion +done <<<"$sanitized_names" diff --git a/scripts/install_config.sh b/scripts/install_config.sh index 04f5ff3..5fdf452 100755 --- a/scripts/install_config.sh +++ b/scripts/install_config.sh @@ -129,6 +129,19 @@ INFO_SITE="ALLABOUTBIRDS" COLOR_SCHEME="light" +#------------------------------ Disk Management ------------------------------# +## FULL_DISK can be set to configure how the system reacts to a full disk +## purge = Remove the oldest day's worth of recordings +## keep = Keep all data and 'stop_core_services.sh' + +FULL_DISK=purge + +## Maximum amount of files to keep for a given specie (0 = keep all) +## Files from the last 7 days, and files protected from purge, are not taken into +## account in this number + +MAX_FILES_SPECIES=0 + ################################################################################ #-------------------------------- Defaults ----------------------------------# ################################################################################ @@ -199,12 +212,6 @@ FREQSHIFT_PITCH=-1500 CHANNELS=2 -## FULL_DISK can be set to configure how the system reacts to a full disk -## purge = Remove the oldest day's worth of recordings -## keep = Keep all data and 'stop_core_services.sh' - -FULL_DISK=purge - ## PRIVACY_THRESHOLD can be set to enable sensitivity to Human sounds. This ## setting is an effort to introduce privacy into the data collection. ## The PRIVACY_THRESHOLD value represents a percentage of the entire species diff --git a/scripts/update_birdnet_snippets.sh b/scripts/update_birdnet_snippets.sh index 7905c18..ca37cf6 100755 --- a/scripts/update_birdnet_snippets.sh +++ b/scripts/update_birdnet_snippets.sh @@ -94,6 +94,10 @@ if ! grep -E '^COLOR_SCHEME=' /etc/birdnet/birdnet.conf &>/dev/null;then echo "COLOR_SCHEME=\"light\"" >> /etc/birdnet/birdnet.conf fi +if ! grep -E '^MAX_FILES_SPECIES=' /etc/birdnet/birdnet.conf &>/dev/null;then + echo "MAX_FILES_SPECIES=\"0\"" >> /etc/birdnet/birdnet.conf +fi + [ -d $RECS_DIR/StreamData ] || sudo_with_user mkdir -p $RECS_DIR/StreamData [ -L ${EXTRACTED}/spectrogram.png ] || sudo_with_user ln -sf ${RECS_DIR}/StreamData/spectrogram.png ${EXTRACTED}/spectrogram.png @@ -180,6 +184,13 @@ if [ -L /usr/local/bin/birdnet_analysis.sh ];then rm -f /usr/local/bin/birdnet_analysis.sh fi +# Clean state and update cron if all scripts are not installed +if [ "$(grep -o "#birdnet" /etc/crontab | wc -l)" -lt 5 ]; then + sudo sed -i '/birdnet/,+1d' /etc/crontab + sed "s/\$USER/$USER/g" "$HOME"/BirdNET-Pi/templates/cleanup.cron >> /etc/crontab + sed "s/\$USER/$USER/g" "$HOME"/BirdNET-Pi/templates/weekly_report.cron >> /etc/crontab +fi + # update snippets above systemctl daemon-reload diff --git a/templates/cleanup.cron b/templates/cleanup.cron index 03a84f5..b07f021 100644 --- a/templates/cleanup.cron +++ b/templates/cleanup.cron @@ -1,6 +1,8 @@ #birdnet */5 * * * * $USER /usr/local/bin/disk_check.sh >/dev/null 2>&1 #birdnet +0 2 * * * $USER /usr/local/bin/disk_species_clean.sh >/dev/null 2>&1 +#birdnet */3 * * * * $USER /usr/local/bin/cleanup.sh >/dev/null 2>&1 #birdnet @reboot $USER /usr/local/bin/cleanup.sh >/dev/null 2>&1 diff --git a/version.md b/version.md index 62cf729..86e4d12 100644 --- a/version.md +++ b/version.md @@ -39,7 +39,7 @@ - BirdWeather Support - Bug Fix for systemd-networkd-wait-online.service - Bug Fix for `install_noip2.sh` for NoIP DUC Support -- New `disk_check.sh` utitlity/crontab entry to `stop_core_services.sh` +- New `disk_check.sh` utility/crontab entry to `stop_core_services.sh` when disk space is greater than 95% # main v0.9 -- pre-installed image