restart services if analysis hangs

scripts/birdnet_analysis.sh

@@ -140,7 +140,7 @@ run_analysis() {
 ${INCLUDEPARAM} \
 ${EXCLUDEPARAM} \
 ${BIRDWEATHER_ID_LOG}
-    $PYTHON_VIRTUAL_ENV $DIR/analyze.py \
+     output=$($PYTHON_VIRTUAL_ENV $DIR/analyze.py \
       --i "${1}/${i}" \
       --o "${1}/${i}.csv" \
       --lat "${LATITUDE}" \
@@ -151,12 +151,20 @@ ${BIRDWEATHER_ID_LOG}
       --min_conf "${CONFIDENCE}" \
       ${INCLUDEPARAM} \
       ${EXCLUDEPARAM} \
-      ${BIRDWEATHER_ID_PARAM}
+      ${BIRDWEATHER_ID_PARAM})
+
+    echo -e "${output}"
+
+    if [ -z "$output" ] || [ "${#output}" -lt 5 ]; then
+        echo "BirdNET Analysis Server is hung or down, attempting to restart it..."
+        sudo ~/BirdNET-Pi/scripts/restart_services.sh
+    else
         if [ ! -z $HEARTBEAT_URL ]; then
           echo "Performing Heartbeat"
           IP=`curl -s ${HEARTBEAT_URL}`
           echo "Heartbeat: $IP"
         fi
+    fi
   done
 }
 
@@ -169,8 +177,15 @@ run_birdnet() {
   run_analysis "${1}"
 }
 
+count=0
 until grep 5050 <(netstat -tulpn 2>&1) &> /dev/null 2>&1; do
   sleep 1
+  count=$((count+1))
+  if [[ $count -eq 30 ]]; then
+    echo "BirdNET Analysis Server is hung or down, attempting to restart it..."
+    sudo ~/BirdNET-Pi/scripts/restart_services.sh
+    count=0
+  fi
 done
 
 if [ $(find ${RECS_DIR}/StreamData -maxdepth 1 -name '*wav' 2>/dev/null| wc -l) -gt 0 ];then