Feat : limit the number of audio files to keep per species (#121)
* New disk usage settings * Use sudo for rm, just in case * Remove set -x * Correct code for png * Correct today * Correct dates * Accepts date format of different distributions * Avoid message * Very significant processing time optimisation * Update disk_species_clean.sh * Avoid locking database * Update disk_species_clean.sh * Disable set -x (again) * Add number of remaining files * Add new variables to update_birdnet_snippets.sh * Consolidate disk management elements * Update disk_species_clean.sh * Update disk_species_clean.sh * Protect last 7 days * Add options to advanced.php, in same style * Remove from config * Update config.php * Reset max_disk_usage to hard 95 * Improve layout * Remove automatic addition of MAX_FILE_SPECIES if not defined https://github.com/alexbelgium/hassio-addons/issues/1458 * Cosmetic white-listed changed to whitelist * Add comma in text * Update scripts/advanced.php Co-authored-by: Nachtzuster <Nachtzuster@users.noreply.github.com> * Update crontab if less than 5 #birdnet occurence (= all scripts not installed) * Made .sh file executable * Remove unused field k5 (second field of year) ; add explanations * Call functions without () * Uninstall current cron before reinstalling them * Update update_birdnet_snippets.sh * Use $HOME instead of $mydir --------- Co-authored-by: Nachtzuster <Nachtzuster@users.noreply.github.com>
This commit is contained in:
Alexandre
+1
-1
@@ -142,7 +142,7 @@ if(isset($_GET['view'])){
|
||||
<button type=\"submit\" name=\"view\" value=\"Webterm\" form=\"views\">Web Terminal</button>
|
||||
<button type=\"submit\" name=\"view\" value=\"Included\" form=\"views\">Custom Species List</button>
|
||||
<button type=\"submit\" name=\"view\" value=\"Excluded\" form=\"views\">Excluded Species List</button>
|
||||
<button type=\"submit\" name=\"view\" value=\"Whitelisted\" form=\"views\">White-listed Species List</button>
|
||||
<button type=\"submit\" name=\"view\" value=\"Whitelisted\" form=\"views\">Whitelist Species List</button>
|
||||
</form>
|
||||
</div>";
|
||||
}
|
||||
|
||||
+17
-1
@@ -130,6 +130,13 @@ if(isset($_GET['submit'])) {
|
||||
}
|
||||
}
|
||||
|
||||
if (isset($_GET["max_files_species"])) {
|
||||
$max_files_species = $_GET["max_files_species"];
|
||||
if (strcmp($max_files_species, $config['MAX_FILES_SPECIES']) !== 0) {
|
||||
$contents = preg_replace("/MAX_FILES_SPECIES=.*/", "MAX_FILES_SPECIES=$max_files_species", $contents);
|
||||
}
|
||||
}
|
||||
|
||||
if(isset($_GET["privacy_threshold"])) {
|
||||
$privacy_threshold = $_GET["privacy_threshold"];
|
||||
if(strcmp($privacy_threshold,$config['PRIVACY_THRESHOLD']) !== 0) {
|
||||
@@ -274,12 +281,21 @@ $newconfig = get_config();
|
||||
</td></tr></table><br>
|
||||
|
||||
<table class="settingstable"><tr><td>
|
||||
<h2>Full Disk Behaviour</h2>
|
||||
<h2>Disk Management</h2>
|
||||
<label for="purge">
|
||||
<input name="full_disk" type="radio" id="purge" value="purge" <?php if (strcmp($newconfig['FULL_DISK'], "purge") == 0) { echo "checked"; }?>>Purge</label>
|
||||
<label for="keep">
|
||||
<input name="full_disk" type="radio" id="keep" value="keep" <?php if (strcmp($newconfig['FULL_DISK'], "keep") == 0) { echo "checked"; }?>>Keep</label>
|
||||
<p>When the disk becomes full, you can choose to 'purge' old files to make room for new ones or 'keep' your data and stop all services instead.<br>Note: you can exclude specific files from 'purge' on the Recordings page.</p>
|
||||
<br>
|
||||
<label for="max_files_species">Amount of files to keep for each specie :</label>
|
||||
<input name="max_files_species" type="number" style="width:6em;" min="0" step="1" value="<?php print($newconfig['MAX_FILES_SPECIES']);?>"/>
|
||||
</td></tr><tr><td>
|
||||
If different than 0 (keep all), defines the maximum number of files to be kept for each species, with priority give to files with highest confidence.
|
||||
This value does not take into account the last 7 days (protected by default).
|
||||
</td></tr><tr><td>
|
||||
Note only the spectrogram and audio files are deleted, the obsevation data remains in the database.
|
||||
The files protected through the "lock" icon are also not affected.
|
||||
</td></tr></table><br>
|
||||
<table class="settingstable"><tr><td>
|
||||
|
||||
|
||||
Executable
+77
@@ -0,0 +1,77 @@
|
||||
#!/bin/bash
|
||||
|
||||
# KEEP ONLY THE NUMBER OF FILES PER SPECIES DEFINED IN THE OPTIONS
|
||||
|
||||
source /etc/birdnet/birdnet.conf
|
||||
base_dir="$HOME/BirdSongs/Extracted/By_Date"
|
||||
max_files_species="${MAX_FILES_SPECIES:-1000}"
|
||||
cd "$base_dir" || true
|
||||
|
||||
# If max_files_species is not higher than 1, exit
|
||||
if [[ "$max_files_species" -lt 1 ]]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Get unique species
|
||||
bird_names=$(
|
||||
sqlite3 -readonly "$HOME"/BirdNET-Pi/scripts/birds.db <<EOF
|
||||
.mode column
|
||||
.headers off
|
||||
SELECT DISTINCT Com_Name FROM detections;
|
||||
.quit
|
||||
EOF
|
||||
)
|
||||
|
||||
# Sanitize the bird names (remove single quotes and replace spaces with underscores)
|
||||
sanitized_names="$(echo "$bird_names" | tr ' ' '_' | tr -d "'" | grep '[[:alnum:]]')"
|
||||
# Remove trailing underscores
|
||||
sanitized_names=$(echo "$sanitized_names" | sed 's/_*$//')
|
||||
# Define how date works
|
||||
dateformat=""
|
||||
if test "$(date -d "-7 days" '+%Y-%m-%d' 2>/dev/null)"; then
|
||||
dateformat=" days"
|
||||
fi
|
||||
|
||||
# find detail :
|
||||
# In the base folders (that corresponds to the BirdSongs/By_date)
|
||||
# Look for all folders that have the correct species names whatever the date
|
||||
# Look for files that have the correct format (containing a date), and that have an extension
|
||||
# For all That are not *.png (as the objective is to limit the number of audio files)
|
||||
# That were not taken in the past 7 days (= that don't contain the date from that past 7 days). $dateformat is configured as a different variables, as ubuntu accepts "5 days" while alpine accepts only "5"
|
||||
# That are not included in the file disk_check_exclude.txt that lists files protected from purge
|
||||
# If the specie name had a "-" in it, it must be converted to "=" to ensure that we have always the same number of "-" separated fields in the filename
|
||||
# Sort by confidence level (field 4 separated by -)
|
||||
# Sort by date (1 for year, 2 for month, 3 for days)
|
||||
# Remove the top x files, corresponding to the files best matching the criteria of confidence + age ; this corresponds to the number of file to keep (in addition to protected files
|
||||
# Rename species that had a = in their name to - (we don't need anymore - separated fields)
|
||||
# Duplicate all lines to append .png at the end, to remove the linked png
|
||||
# This appends a fake "temp" file, so that the sudo rm has at least one file to delete and does not hang
|
||||
# Delete files, then once all files are deleted echo the number of remaining files
|
||||
|
||||
# Read each line from the variable and echo the species
|
||||
while read -r species; do
|
||||
echo -n "$species : "
|
||||
species_san="${species/-/=}"
|
||||
# Dummy file to execute the rm using xargs even if no files are there. Best solution found for code speed
|
||||
touch temp
|
||||
find */"$species" -type f -name "*[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]*.*" \
|
||||
-not -name "*.png" \
|
||||
-not -name "*$(date -d "-7$dateformat" '+%Y-%m-%d')*" \
|
||||
-not -name "*$(date -d "-6$dateformat" '+%Y-%m-%d')*" \
|
||||
-not -name "*$(date -d "-5$dateformat" '+%Y-%m-%d')*" \
|
||||
-not -name "*$(date -d "-4$dateformat" '+%Y-%m-%d')*" \
|
||||
-not -name "*$(date -d "-3$dateformat" '+%Y-%m-%d')*" \
|
||||
-not -name "*$(date -d "-2$dateformat" '+%Y-%m-%d')*" \
|
||||
-not -name "*$(date -d "-1$dateformat" '+%Y-%m-%d')*" \
|
||||
-not -name "*$(date '+%Y-%m-%d')*" |
|
||||
grep -vFf "$HOME/BirdNET-Pi/scripts/disk_check_exclude.txt" |
|
||||
sed "s|$species|$species_san|g" |
|
||||
sort -t'-' -k4,4nr -k1,1nr -k2,2nr -k3,3nr |
|
||||
tail -n +"$((max_files_species + 1))" |
|
||||
sed "s|$species_san|$species|g" |
|
||||
sed 'p; s/\(\.[^.]*\)$/\1.png/' |
|
||||
awk 'BEGIN{print "temp"} {print}' |
|
||||
xargs sudo rm && echo "success ($(find */"$species" -type f -name "*[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]*.*" \
|
||||
-not -name "*.png" | wc -l)) remaining" || echo "failed ($?)"
|
||||
# rm to be changed to touch or echo if you want to test without deletion
|
||||
done <<<"$sanitized_names"
|
||||
@@ -129,6 +129,19 @@ INFO_SITE="ALLABOUTBIRDS"
|
||||
|
||||
COLOR_SCHEME="light"
|
||||
|
||||
#------------------------------ Disk Management ------------------------------#
|
||||
## FULL_DISK can be set to configure how the system reacts to a full disk
|
||||
## purge = Remove the oldest day's worth of recordings
|
||||
## keep = Keep all data and 'stop_core_services.sh'
|
||||
|
||||
FULL_DISK=purge
|
||||
|
||||
## Maximum amount of files to keep for a given specie (0 = keep all)
|
||||
## Files from the last 7 days, and files protected from purge, are not taken into
|
||||
## account in this number
|
||||
|
||||
MAX_FILES_SPECIES=0
|
||||
|
||||
################################################################################
|
||||
#-------------------------------- Defaults ----------------------------------#
|
||||
################################################################################
|
||||
@@ -199,12 +212,6 @@ FREQSHIFT_PITCH=-1500
|
||||
|
||||
CHANNELS=2
|
||||
|
||||
## FULL_DISK can be set to configure how the system reacts to a full disk
|
||||
## purge = Remove the oldest day's worth of recordings
|
||||
## keep = Keep all data and 'stop_core_services.sh'
|
||||
|
||||
FULL_DISK=purge
|
||||
|
||||
## PRIVACY_THRESHOLD can be set to enable sensitivity to Human sounds. This
|
||||
## setting is an effort to introduce privacy into the data collection.
|
||||
## The PRIVACY_THRESHOLD value represents a percentage of the entire species
|
||||
|
||||
@@ -94,6 +94,10 @@ if ! grep -E '^COLOR_SCHEME=' /etc/birdnet/birdnet.conf &>/dev/null;then
|
||||
echo "COLOR_SCHEME=\"light\"" >> /etc/birdnet/birdnet.conf
|
||||
fi
|
||||
|
||||
if ! grep -E '^MAX_FILES_SPECIES=' /etc/birdnet/birdnet.conf &>/dev/null;then
|
||||
echo "MAX_FILES_SPECIES=\"0\"" >> /etc/birdnet/birdnet.conf
|
||||
fi
|
||||
|
||||
[ -d $RECS_DIR/StreamData ] || sudo_with_user mkdir -p $RECS_DIR/StreamData
|
||||
[ -L ${EXTRACTED}/spectrogram.png ] || sudo_with_user ln -sf ${RECS_DIR}/StreamData/spectrogram.png ${EXTRACTED}/spectrogram.png
|
||||
|
||||
@@ -180,6 +184,13 @@ if [ -L /usr/local/bin/birdnet_analysis.sh ];then
|
||||
rm -f /usr/local/bin/birdnet_analysis.sh
|
||||
fi
|
||||
|
||||
# Clean state and update cron if all scripts are not installed
|
||||
if [ "$(grep -o "#birdnet" /etc/crontab | wc -l)" -lt 5 ]; then
|
||||
sudo sed -i '/birdnet/,+1d' /etc/crontab
|
||||
sed "s/\$USER/$USER/g" "$HOME"/BirdNET-Pi/templates/cleanup.cron >> /etc/crontab
|
||||
sed "s/\$USER/$USER/g" "$HOME"/BirdNET-Pi/templates/weekly_report.cron >> /etc/crontab
|
||||
fi
|
||||
|
||||
# update snippets above
|
||||
|
||||
systemctl daemon-reload
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
#birdnet
|
||||
*/5 * * * * $USER /usr/local/bin/disk_check.sh >/dev/null 2>&1
|
||||
#birdnet
|
||||
0 2 * * * $USER /usr/local/bin/disk_species_clean.sh >/dev/null 2>&1
|
||||
#birdnet
|
||||
*/3 * * * * $USER /usr/local/bin/cleanup.sh >/dev/null 2>&1
|
||||
#birdnet
|
||||
@reboot $USER /usr/local/bin/cleanup.sh >/dev/null 2>&1
|
||||
|
||||
+1
-1
@@ -39,7 +39,7 @@
|
||||
- BirdWeather Support
|
||||
- Bug Fix for systemd-networkd-wait-online.service
|
||||
- Bug Fix for `install_noip2.sh` for NoIP DUC Support
|
||||
- New `disk_check.sh` utitlity/crontab entry to `stop_core_services.sh`
|
||||
- New `disk_check.sh` utility/crontab entry to `stop_core_services.sh`
|
||||
when disk space is greater than 95%
|
||||
|
||||
# main v0.9 -- pre-installed image
|
||||
|
||||
Reference in New Issue
Block a user