From b136e07b51c8facc16b41a37487b675c7c9d84d2 Mon Sep 17 00:00:00 2001 From: Jake Herbst Date: Mon, 16 May 2022 21:32:30 -0400 Subject: [PATCH 01/14] Removing the 'sed' added python shebangs --- scripts/update_birdnet_snippets.sh | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/scripts/update_birdnet_snippets.sh b/scripts/update_birdnet_snippets.sh index e9c89d3..2f0a618 100755 --- a/scripts/update_birdnet_snippets.sh +++ b/scripts/update_birdnet_snippets.sh @@ -5,13 +5,10 @@ trap 'exit 1' SIGINT SIGHUP USER=$(awk -F: '/1000/ {print $1}' /etc/passwd) HOME=$(awk -F: '/1000/ {print $6}' /etc/passwd) my_dir=$HOME/BirdNET-Pi/scripts -if ! grep python3 <(head -n1 $my_dir/analyze.py) &>/dev/null;then - echo "Ensure all python scripts use the virtual environment" - sudo -u$USER sed -si "1 i\\#\!$HOME/BirdNET-Pi/birdnet/bin/python3" $my_dir/*.py -fi + if ! grep PRIVACY_THRESHOLD /etc/birdnet/birdnet.conf &>/dev/null;then sudo -u$USER echo "PRIVACY_THRESHOLD=0" >> /etc/birdnet/birdnet.conf - git -C $HOME/BirdNET-Pi rm $my_dir/privacy_server.py + git -C $HOME/BirdNET-Pi rm $my_dir/privacy_server.py fi if [ -f $my_dir/privacy_server ] || [ -L /usr/local/bin/privacy_server.py ];then rm -f $my_dir/privacy_server.py From 670d3f90cac39e74ddaab987ab26ef23f4642793 Mon Sep 17 00:00:00 2001 From: Jake Herbst Date: Mon, 16 May 2022 21:46:05 -0400 Subject: [PATCH 02/14] Adding python3 virtualenv to python systemd services --- scripts/install_services.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/scripts/install_services.sh b/scripts/install_services.sh index b909977..a63febe 100755 --- a/scripts/install_services.sh +++ b/scripts/install_services.sh @@ -9,6 +9,8 @@ config_file=$my_dir/birdnet.conf export USER=$USER export HOME=$HOME +export PYTHON_VIRTUAL_ENV="$HOME/BirdNET-Pi/birdnet/bin/python3" + install_depends() { apt install -y debian-keyring debian-archive-keyring apt-transport-https curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/gpg.key' | sudo tee /etc/apt/trusted.gpg.d/caddy-stable.asc @@ -66,7 +68,7 @@ Restart=always Type=simple RestartSec=10 User=${USER} -ExecStart=/usr/local/bin/server.py +ExecStart=$PYTHON_VIRTUAL_ENV /usr/local/bin/server.py [Install] WantedBy=multi-user.target EOF @@ -116,7 +118,7 @@ create_necessary_dirs() { sudo -u ${USER} ln -fs $my_dir/exclude_species_list.txt $my_dir/scripts sudo -u ${USER} ln -fs $my_dir/include_species_list.txt $my_dir/scripts - sudo -u ${USER} ln -fs $my_dir/homepage/* ${EXTRACTED} + sudo -u ${USER} ln -fs $my_dir/homepage/* ${EXTRACTED} sudo -u ${USER} ln -fs $my_dir/model/labels.txt ${my_dir}/scripts sudo -u ${USER} ln -fs $my_dir/scripts ${EXTRACTED} sudo -u ${USER} ln -fs $my_dir/scripts/play.php ${EXTRACTED} @@ -325,7 +327,7 @@ Restart=always RestartSec=120 Type=simple User=$USER -ExecStart=/usr/local/bin/daily_plot.py +ExecStart=$PYTHON_VIRTUAL_ENV /usr/local/bin/daily_plot.py [Install] WantedBy=multi-user.target EOF @@ -386,7 +388,7 @@ install_phpsysinfo() { } config_icecast() { - if [ -f /etc/icecast2/icecast.xml ];then + if [ -f /etc/icecast2/icecast.xml ];then cp /etc/icecast2/icecast.xml{,.prebirdnetpi} fi sed -i 's/>adminbirdnet Date: Mon, 16 May 2022 21:46:24 -0400 Subject: [PATCH 03/14] Adding pythond3 virtualenv to birdnet analysis python call --- scripts/birdnet_analysis.sh | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/scripts/birdnet_analysis.sh b/scripts/birdnet_analysis.sh index b9dab47..92ac137 100755 --- a/scripts/birdnet_analysis.sh +++ b/scripts/birdnet_analysis.sh @@ -29,8 +29,8 @@ fi INCLUDE_LIST="$HOME/BirdNET-Pi/include_species_list.txt" EXCLUDE_LIST="$HOME/BirdNET-Pi/exclude_species_list.txt" -if [ ! -f ${INCLUDE_LIST} ];then - touch ${INCLUDE_LIST} && +if [ ! -f ${INCLUDE_LIST} ];then + touch ${INCLUDE_LIST} && chmod g+rw ${INCLUDE_LIST} fi if [ ! -f ${EXCLUDE_LIST} ];then @@ -59,8 +59,8 @@ get_files() { # Takes one argument: # - {DIRECTORY} move_analyzed() { - for i in "${files[@]}";do - j="${i}.csv" + for i in "${files[@]}";do + j="${i}.csv" if [ -f "${1}/${j}" ];then if [ ! -d "${1}-Analyzed" ];then mkdir -p "${1}-Analyzed" && echo "'Analyzed' directory created" @@ -75,7 +75,9 @@ move_analyzed() { # Uses one argument: # - {DIRECTORY} run_analysis() { - sleep .5 + PYTHON_VIRTUAL_ENV="$HOME/BirdNET-Pi/birdnet/bin/python3" + + sleep .5 ### TESTING NEW WEEK CALCULATION WEEK_OF_YEAR="$(echo "($(date +%m)-1) * 4" | bc -l)" @@ -123,7 +125,7 @@ run_analysis() { BIRDWEATHER_ID_PARAM="" BIRDWEATHER_ID_LOG="" fi - echo analyze.py \ + echo $PYTHON_VIRTUAL_ENV analyze.py \ --i "${1}/${i}" \ --o "${1}/${i}.csv" \ --lat "${LATITUDE}" \ @@ -135,7 +137,7 @@ run_analysis() { ${INCLUDEPARAM} \ ${EXCLUDEPARAM} \ ${BIRDWEATHER_ID_LOG} - analyze.py \ + $PYTHON_VIRTUAL_ENV analyze.py \ --i "${1}/${i}" \ --o "${1}/${i}.csv" \ --lat "${LATITUDE}" \ From 873dfb12d69d1242e5fd4e98c2ef80f6a6392ed6 Mon Sep 17 00:00:00 2001 From: Jake Herbst Date: Mon, 16 May 2022 22:16:29 -0400 Subject: [PATCH 04/14] Fixing analyze.py location --- scripts/birdnet_analysis.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/birdnet_analysis.sh b/scripts/birdnet_analysis.sh index 92ac137..94ea8c5 100755 --- a/scripts/birdnet_analysis.sh +++ b/scripts/birdnet_analysis.sh @@ -76,6 +76,7 @@ move_analyzed() { # - {DIRECTORY} run_analysis() { PYTHON_VIRTUAL_ENV="$HOME/BirdNET-Pi/birdnet/bin/python3" + DIR="$HOME/BirdNET-Pi/scripts" sleep .5 @@ -125,7 +126,7 @@ run_analysis() { BIRDWEATHER_ID_PARAM="" BIRDWEATHER_ID_LOG="" fi - echo $PYTHON_VIRTUAL_ENV analyze.py \ + echo $PYTHON_VIRTUAL_ENV "$DIR\analyze.py" \ --i "${1}/${i}" \ --o "${1}/${i}.csv" \ --lat "${LATITUDE}" \ @@ -137,7 +138,7 @@ run_analysis() { ${INCLUDEPARAM} \ ${EXCLUDEPARAM} \ ${BIRDWEATHER_ID_LOG} - $PYTHON_VIRTUAL_ENV analyze.py \ + $PYTHON_VIRTUAL_ENV $DIR/analyze.py \ --i "${1}/${i}" \ --o "${1}/${i}.csv" \ --lat "${LATITUDE}" \ From 54a6ecf68bf581705bb51bb2a5b4045f53fe057d Mon Sep 17 00:00:00 2001 From: Jake Herbst Date: Tue, 17 May 2022 19:13:13 -0400 Subject: [PATCH 05/14] Removing stash+pop git workflow from update_birdnet.sh --- scripts/update_birdnet.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/update_birdnet.sh b/scripts/update_birdnet.sh index a699670..9c6b865 100755 --- a/scripts/update_birdnet.sh +++ b/scripts/update_birdnet.sh @@ -7,10 +7,8 @@ HOME=$(awk -F: '/1000/ {print $6}' /etc/passwd) my_dir=$HOME/BirdNET-Pi/scripts sudo -u$USER git -C $my_dir rm privacy_server.py -sudo -u${USER} git -C $my_dir stash sudo -u${USER} git -C $my_dir pull -f sudo systemctl daemon-reload -sudo -u${USER} git -C $my_dir stash pop sudo ln -sf $my_dir/* /usr/local/bin/ # The script below handles changes to the host system From 02bf7ffe077b284379cbb16f02bb96c9f43ed1a7 Mon Sep 17 00:00:00 2001 From: Jake Herbst Date: Tue, 17 May 2022 19:27:47 -0400 Subject: [PATCH 06/14] Adding a snippet to ensure python systemd services use the python virtual-env --- scripts/update_birdnet_snippets.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/update_birdnet_snippets.sh b/scripts/update_birdnet_snippets.sh index 2f0a618..11d3823 100755 --- a/scripts/update_birdnet_snippets.sh +++ b/scripts/update_birdnet_snippets.sh @@ -14,6 +14,11 @@ if [ -f $my_dir/privacy_server ] || [ -L /usr/local/bin/privacy_server.py ];then rm -f $my_dir/privacy_server.py rm -f /usr/local/bin/privacy_server.py fi + +# Adds python virtual-env to the python systemd services +sudo -E sed -i "s|ExecStart=.*|ExecStart=$HOME/BirdNET-Pi/birdnet/bin/python3 /usr/local/bin/server.py|" ~/BirdNET-Pi/templates/birdnet_server.service +sudo -E sed -i "s|ExecStart=.*|ExecStart=$HOME/BirdNET-Pi/birdnet/bin/python3 /usr/local/bin/daily_plot.py|" ~/BirdNET-Pi/templates/chart_viewer.service + if grep privacy ~/BirdNET-Pi/templates/birdnet_server.service &>/dev/null;then sudo -E sed -i 's/privacy_server.py/server.py/g' \ ~/BirdNET-Pi/templates/birdnet_server.service From a515e05e15364bc4b15621296161c7d42dd990ab Mon Sep 17 00:00:00 2001 From: Patrick McGuire <60325264+mcguirepr89@users.noreply.github.com> Date: Tue, 17 May 2022 19:36:31 -0400 Subject: [PATCH 07/14] Adding daemon-reload --- scripts/update_birdnet_snippets.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/update_birdnet_snippets.sh b/scripts/update_birdnet_snippets.sh index 11d3823..908999d 100755 --- a/scripts/update_birdnet_snippets.sh +++ b/scripts/update_birdnet_snippets.sh @@ -18,6 +18,7 @@ fi # Adds python virtual-env to the python systemd services sudo -E sed -i "s|ExecStart=.*|ExecStart=$HOME/BirdNET-Pi/birdnet/bin/python3 /usr/local/bin/server.py|" ~/BirdNET-Pi/templates/birdnet_server.service sudo -E sed -i "s|ExecStart=.*|ExecStart=$HOME/BirdNET-Pi/birdnet/bin/python3 /usr/local/bin/daily_plot.py|" ~/BirdNET-Pi/templates/chart_viewer.service +sudo systemctl daemon-reload if grep privacy ~/BirdNET-Pi/templates/birdnet_server.service &>/dev/null;then sudo -E sed -i 's/privacy_server.py/server.py/g' \ From 1f7fc63afa6a122017a2efb6cd057db0754847e2 Mon Sep 17 00:00:00 2001 From: Patrick McGuire <60325264+mcguirepr89@users.noreply.github.com> Date: Tue, 17 May 2022 19:43:37 -0400 Subject: [PATCH 08/14] reloading the daemons and restarting services Since the service files have changes, we need to `sudo systemctl daemon-reload` and then we need to restart those services. But it is easier to just run `restart_services.sh`, so I've done that instead --- scripts/update_birdnet_snippets.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/update_birdnet_snippets.sh b/scripts/update_birdnet_snippets.sh index 908999d..038683f 100755 --- a/scripts/update_birdnet_snippets.sh +++ b/scripts/update_birdnet_snippets.sh @@ -16,9 +16,11 @@ if [ -f $my_dir/privacy_server ] || [ -L /usr/local/bin/privacy_server.py ];then fi # Adds python virtual-env to the python systemd services -sudo -E sed -i "s|ExecStart=.*|ExecStart=$HOME/BirdNET-Pi/birdnet/bin/python3 /usr/local/bin/server.py|" ~/BirdNET-Pi/templates/birdnet_server.service -sudo -E sed -i "s|ExecStart=.*|ExecStart=$HOME/BirdNET-Pi/birdnet/bin/python3 /usr/local/bin/daily_plot.py|" ~/BirdNET-Pi/templates/chart_viewer.service -sudo systemctl daemon-reload +if ! grep 'BirdNET-Pi/birdnet/' $HOME/BirdNET-Pi/templates/birdnet_server.service || ! grep 'BirdNET-Pi/birdnet' $HOME/BirdNET-Pi/templates/chart_viewer.service;then + sudo -E sed -i "s|ExecStart=.*|ExecStart=$HOME/BirdNET-Pi/birdnet/bin/python3 /usr/local/bin/server.py|" ~/BirdNET-Pi/templates/birdnet_server.service + sudo -E sed -i "s|ExecStart=.*|ExecStart=$HOME/BirdNET-Pi/birdnet/bin/python3 /usr/local/bin/daily_plot.py|" ~/BirdNET-Pi/templates/chart_viewer.service + sudo systemctl daemon-reload && restart_services.sh +fi if grep privacy ~/BirdNET-Pi/templates/birdnet_server.service &>/dev/null;then sudo -E sed -i 's/privacy_server.py/server.py/g' \ From a786e51da5fe4b48e7bb6ed091d55ae7826700fc Mon Sep 17 00:00:00 2001 From: Jake Herbst Date: Wed, 18 May 2022 13:58:32 -0400 Subject: [PATCH 09/14] Updating update_birdnet.sh to make more robust and offer functionality to switch branches for testing remote dev branches --- scripts/update_birdnet.sh | 46 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/scripts/update_birdnet.sh b/scripts/update_birdnet.sh index 9c6b865..af72fe5 100755 --- a/scripts/update_birdnet.sh +++ b/scripts/update_birdnet.sh @@ -2,12 +2,54 @@ # Update BirdNET-Pi's Git Repo source /etc/birdnet/birdnet.conf trap 'exit 1' SIGINT SIGHUP + +usage() { echo "Usage: $0 [-r ] [-b ]" 1>&2; exit 1; } + +# Defaults +remote="origin" +branch="main" + +while getopts ":r:b:" o; do + case "${o}" in + r) + remote=${OPTARG} + + # Ensure that git remote is configured + git remote show $remote 2&> /dev/null || { + echo "Error: remote '$remote' not found. Add the upstream remote to your repository and try again." + exit 1 + } + ;; + b) + branch=${OPTARG} + ;; + *) + usage + ;; + esac +done +shift $((OPTIND-1)) + USER=$(awk -F: '/1000/ {print $1}' /etc/passwd) HOME=$(awk -F: '/1000/ {print $6}' /etc/passwd) my_dir=$HOME/BirdNET-Pi/scripts -sudo -u$USER git -C $my_dir rm privacy_server.py -sudo -u${USER} git -C $my_dir pull -f + +sudo_with_user () { + set -x + sudo -u $USER "$@" + set +x +} + +# Reset current HEAD to remove any local changes +sudo_with_user git reset --hard + +# Fetches latest changes +sudo_with_user git fetch $remote $branch + +# Switches git to specified branch +sudo_with_user git switch -C $branch --track $remote/$branch + sudo systemctl daemon-reload sudo ln -sf $my_dir/* /usr/local/bin/ From 4b09e658cda5e6248abe7746c0494afb3616f80a Mon Sep 17 00:00:00 2001 From: Patrick McGuire <60325264+mcguirepr89@users.noreply.github.com> Date: Wed, 18 May 2022 17:24:51 -0400 Subject: [PATCH 10/14] I think we actually want to redirect stdout --- scripts/update_birdnet.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/update_birdnet.sh b/scripts/update_birdnet.sh index af72fe5..4a0ded6 100755 --- a/scripts/update_birdnet.sh +++ b/scripts/update_birdnet.sh @@ -15,7 +15,7 @@ while getopts ":r:b:" o; do remote=${OPTARG} # Ensure that git remote is configured - git remote show $remote 2&> /dev/null || { + git remote show $remote 1&> /dev/null || { echo "Error: remote '$remote' not found. Add the upstream remote to your repository and try again." exit 1 } From ac0bccebfc17ba8ee1906ce29e8cf59b6c7eb9b6 Mon Sep 17 00:00:00 2001 From: Patrick McGuire <60325264+mcguirepr89@users.noreply.github.com> Date: Fri, 20 May 2022 09:55:17 -0400 Subject: [PATCH 11/14] removed hardcoded API key --- scripts/stats.php | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/stats.php b/scripts/stats.php index fce469b..e1be3b9 100644 --- a/scripts/stats.php +++ b/scripts/stats.php @@ -176,7 +176,13 @@ while($results=$result3->fetchArray(SQLITE3_ASSOC)){ ob_flush(); flush(); - $flickrjson = json_decode(file_get_contents("https://www.flickr.com/services/rest/?method=flickr.photos.search&api_key=2158f6c5d66e89778bd0d340953f1bcf&text=\"".str_replace('_', '+', $comname)."\"&license=2%2C3%2C4%2C5%2C6%2C9&sort=relevance&per_page=15&format=json&nojsoncallback=1"), true)["photos"]["photo"]; + if (file_exists('./scripts/thisrun.txt')) { + $config = parse_ini_file('./scripts/thisrun.txt'); + } elseif (file_exists('./scripts/firstrun.ini')) { + $config = parse_ini_file('./scripts/firstrun.ini'); + } + $flickrjson = json_decode(file_get_contents("https://www.flickr.com/services/rest/?method=flickr.photos.search&api_key=".$config["FLICKR_API_KEY"]."&text=\"".str_replace('_', '+', $comname)."\"&license=2%2C3%2C4%2C5%2C6%2C9&sort=relevance&per_page=15&format=json&nojsoncallback=1"), true)["photos"]["photo"]; + foreach ($flickrjson as $val) { $iter++; $modaltext = "https://flickr.com/photos/".$val["owner"]."/".$val["id"]; From a580402d52f0a7e7135fb31c707468a50fdcb0b7 Mon Sep 17 00:00:00 2001 From: Jake Herbst Date: Fri, 20 May 2022 09:57:20 -0400 Subject: [PATCH 12/14] Fixing the git remote check to suppress stdout+stderr and actually work correctly --- scripts/update_birdnet.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/update_birdnet.sh b/scripts/update_birdnet.sh index 4a0ded6..f7fc305 100755 --- a/scripts/update_birdnet.sh +++ b/scripts/update_birdnet.sh @@ -15,7 +15,7 @@ while getopts ":r:b:" o; do remote=${OPTARG} # Ensure that git remote is configured - git remote show $remote 1&> /dev/null || { + git remote show $remote > /dev/null 2>&1 || { echo "Error: remote '$remote' not found. Add the upstream remote to your repository and try again." exit 1 } From 24e20fb438a1d000a9f1bf97214126a19e8305fe Mon Sep 17 00:00:00 2001 From: Patrick McGuire <60325264+mcguirepr89@users.noreply.github.com> Date: Fri, 20 May 2022 10:04:06 -0400 Subject: [PATCH 13/14] This needs a proper test to work --- scripts/update_birdnet.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/scripts/update_birdnet.sh b/scripts/update_birdnet.sh index f7fc305..bea167b 100755 --- a/scripts/update_birdnet.sh +++ b/scripts/update_birdnet.sh @@ -12,13 +12,12 @@ branch="main" while getopts ":r:b:" o; do case "${o}" in r) - remote=${OPTARG} - - # Ensure that git remote is configured - git remote show $remote > /dev/null 2>&1 || { + if git remote show $remote &>/dev/null;then + remote=${OPTARG} + else echo "Error: remote '$remote' not found. Add the upstream remote to your repository and try again." exit 1 - } + fi ;; b) branch=${OPTARG} From 7a58b1cc54e79607be7a0a653c37108c0833d22b Mon Sep 17 00:00:00 2001 From: Jake Herbst Date: Fri, 20 May 2022 12:37:06 -0400 Subject: [PATCH 14/14] Updating git remote check to use remote string from argument --- scripts/update_birdnet.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/update_birdnet.sh b/scripts/update_birdnet.sh index bea167b..93dd5b8 100755 --- a/scripts/update_birdnet.sh +++ b/scripts/update_birdnet.sh @@ -12,9 +12,11 @@ branch="main" while getopts ":r:b:" o; do case "${o}" in r) - if git remote show $remote &>/dev/null;then - remote=${OPTARG} - else + remote=${OPTARG} + git remote show $remote > /dev/null 2>&1 + ret_val=$? + + if [ $ret_val -ne 0 ]; then echo "Error: remote '$remote' not found. Add the upstream remote to your repository and try again." exit 1 fi