time to test installation

scripts/daily_plot.py.newer

@@ -0,0 +1,141 @@
+#!/home/pi/BirdNET-Pi/birdnet/bin/python3
+
+import os
+import configparser
+import pandas as pd
+import seaborn as sns
+import numpy as np
+import matplotlib.pyplot as plt
+from matplotlib.colors import LogNorm
+from datetime import datetime
+import textwrap
+import sqlite3
+
+conn = sqlite3.connect('/home/pi/BirdNET-Pi/scripts/birds.db')
+df = pd.read_sql_query("SELECT * from detections", conn)
+cursor = conn.cursor()
+
+table_rows = cursor.fetchall()
+
+#df=pd.DataFrame(table_rows)
+
+#Convert Date and Time Fields to Panda's format
+df['Date']=pd.to_datetime(df['Date'])
+df['Time']=pd.to_datetime(df['Time'], unit='ns')
+
+
+#Add round hours to dataframe
+df['Hour of Day'] = [r.hour for r in df.Time]
+
+#Create separate dataframes for separate locations
+df_plt=df #Default to use the whole Dbase
+
+#Get todays readings
+now = datetime.now()
+df_plt_today = df_plt[df_plt['Date']==now.strftime("%Y-%m-%d")]
+
+#Set number of species to report
+#For ALL
+readings = len(df_plt_today['Com_Name'].value_counts())
+# Uncomment for user selection
+readings = 10
+
+
+plt_top10_today = (df_plt_today['Com_Name'].value_counts()[:readings])
+
+df_plt_top10_today = df_plt_today[df_plt_today.Com_Name.isin(plt_top10_today.index)]
+
+#Set Palette for graphics
+pal = "Greens"
+
+#Set up plot axes and titles
+
+# f, axs = plt.subplots(1, 3, figsize = (10, 5 * vert_scale), gridspec_kw=dict(width_ratios=[3, 2, 5]))
+
+vert_scale = readings / 10
+f, axs = plt.subplots(1, 2, figsize = (10, 5 * vert_scale), gridspec_kw=dict(width_ratios=[3, 6]), facecolor='#77C487')
+plt.subplots_adjust(left=None, bottom=None, right=None, top=None, wspace=0, hspace=0)
+
+#generate y-axis order for all figures based on frequency
+freq_order = pd.value_counts(df_plt_top10_today['Com_Name']).iloc[:readings].index
+
+# this groups by name and calculates mean/max conf
+confmax = df_plt_top10_today.groupby('Com_Name')['Confidence'].max()
+confavg = df_plt_top10_today.groupby('Com_Name')['Confidence'].mean()
+#reorder confmax/avg to detection frequency order
+confmax = confmax.reindex(freq_order)
+confavg = confavg.reindex(freq_order)
+# norm avg values for color palette
+norm = plt.Normalize(confavg.values.min(), confavg.values.max())
+# bars of frequency plot based on avg color palette
+colors = plt.cm.Greens(norm(confavg))
+
+#Generate frequency plot
+plot=sns.countplot(y='Com_Name', data = df_plt_top10_today, palette = colors,  order=freq_order, ax=axs[0])
+
+# for container in axs[0].containers:
+#     axs[0].bar_label(containers)
+
+# Function to show value on bars - from https://stackoverflow.com/questions/43214978/seaborn-barplot-displaying-values
+def show_values_on_bars(ax,label):
+    i = 0
+    for p in ax.patches:
+        _x = p.get_x() + p.get_width()* 0.9
+        _y = p.get_y() + p.get_height() / 2
+        value = '{:.0%}'.format(label[i])
+        # Uncomment for Species Count Total
+        # value = '{:,}'.format(p.get_width())
+        ax.text(_x, _y, value, ha='center', va='center', size=8, fontweight='bold', color='darkgreen', bbox=dict(facecolor='lightgrey',pad = 4.0))
+        i=i+1
+
+# Prints Max Confidence on bars     
+show_values_on_bars(axs[0],confmax)
+
+#Try plot grid lines between bars - problem at the moment plots grid lines on bars - want between bars
+# plot.grid(True, axis='y')
+z=plot.get_ymajorticklabels()
+plot.set_yticklabels(['\n'.join(textwrap.wrap(ticklabel.get_text(),15)) for ticklabel in plot.get_yticklabels()], fontsize = 10)
+plot.set(ylabel=None)
+plot.set(xlabel="Detections")
+# Comma formatting for when your Detections are >1,000
+# current_values=plot.gca().get_xticks()
+# plt.gca().set_xticklabels(['{:,0f}'.format(x) for x in current_values])
+
+#If you want violin/box plots uncomment here and ** above
+# plot = sns.boxenplot(x=df_plt_top10_today['Confidence']*100,color='Green',  y=df_plt_top10_today['Com_Name'], ax=axs[1],order=freq_order)
+# plot.set(xlabel="Confidence", ylabel=None,yticklabels=[])
+
+
+#Generate crosstab matrix for heatmap plot
+
+heat = pd.crosstab(df_plt_top10_today['Com_Name'],df_plt_top10_today['Hour of Day'])
+#Order heatmap Birds by frequency of occurrance
+heat.index = pd.CategoricalIndex(heat.index, categories = freq_order)
+heat.sort_index(level=0, inplace=True)
+
+
+hours_in_day = pd.Series(data = range(0,24))
+heat_frame = pd.DataFrame(data=0, index=heat.index, columns = hours_in_day)
+heat=(heat+heat_frame).fillna(0)
+
+#Generatie heatmap plot
+plot = sns.heatmap(heat, norm=LogNorm(),  annot=True, fmt="g", annot_kws={"fontsize":7}, cmap = pal , square = False, cbar=False, linewidths = 0.5, linecolor = "Grey", ax=axs[1], yticklabels = False)
+plot.set_xticklabels(plot.get_xticklabels(), rotation = 0, size = 8)
+
+# Set heatmap border
+for _, spine in plot.spines.items():
+    spine.set_visible(True)
+
+plot.set(ylabel=None)
+plot.set(xlabel="Hour of Day")
+#Set combined plot layout and titles
+#f.tight_layout()
+#f.subplots_adjust(top=0.95)
+#f.suptitle("DAILY OVERVIEW FOR "+ str(now.strftime("%d-%m-%Y %H:%M")),x=0.5,y=1.5,va='top')
+
+#Save combined plot
+savename='/home/pi/BirdSongs/Extracted/Charts/Combo-'+str(now.strftime("%Y-%m-%d"))+'.png'
+plt.savefig(savename)
+plt.show()
+plt.close()
+

scripts/extra_info.sh

@@ -13,28 +13,44 @@ revbinary=$(echo $binary|rev)
 if echo $binary | grep 1 ;then
   echo "ISSUES DETECTED"
   if [ ${revbinary:0:1} -eq 1 &> /dev/null ];then
-    echo "Under-voltage detected"
+    message="Under-voltage detected"
+    echo "$message"
+    dmesg -H | grep -i voltage
   fi
   if [ ${revbinary:1:1} -eq 1 &> /dev/null ];then
-    echo "Arm frequency capped"
+    message="Arm frequency capped"
+    echo "$message"
+    dmesg -H | grep -i frequen
   fi
   if [ ${revbinary:2:1} -eq 1 &> /dev/null ];then
-    echo "Currently Throttled"
+    message="Currently Throttled"
+    echo "$message"
+    dmesg -H | grep -i throttl
   fi
   if [ ${revbinary:3:1} -eq 1 &> /dev/null ];then
-    echo "Soft temperatue limit active"
+    message="Soft temperatue limit active"
+    echo "$message"
+    dmesg -H | grep -i temperature 
   fi
   if [ ${revbinary:16:1} -eq 1 &> /dev/null ];then
-    echo "Under-voltage has occurred"
+    message="Under-voltage has occurred"
+    echo "$message"
+    dmesg -H | grep -i voltage
   fi
   if [ ${revbinary:17:1} -eq 1 &> /dev/null ];then
-    echo "Arm frequency capping has occurred"
+    message="Arm frequency capping has occurred"
+    echo "$message"
+    dmesg -H | grep -i frequen
   fi
   if [ ${revbinary:18:1} -eq 1 &> /dev/null ];then
-    echo "Throttling has occurred"
+    message="Throttling has occurred"
+    echo "$message"
+    dmesg -H | grep -i throttl
   fi
   if [ ${revbinary:19:1} -eq 1 &> /dev/null ];then
-    echo "Soft temperature limit has occurred"
+    message="Soft temperature limit has occurred"
+    echo "$message"
+    dmesg -H | grep -i temperature 
   fi
 fi
 echo "....................................Clock Speeds................................"

scripts/install_services.sh

@@ -84,7 +84,7 @@ Restart=on-failure
 RestartSec=3
 Type=simple
 User=${USER}
-ExecStart=/usr/bin/env bash -c 'while true;do extract_new_birdsounds.sh;sleep ${RECORDING_LENGTH};done'
+ExecStart=/usr/bin/env bash -c 'while true;do extract_new_birdsounds.sh;sleep 3;done'
 [Install]
 WantedBy=multi-user.target
 EOF
@@ -152,9 +152,9 @@ create_necessary_dirs() {
   sudo -u ${USER} ln -fs $(dirname ${my_dir})/scripts/history.php ${EXTRACTED}
   sudo -u ${USER} ln -fs $(dirname ${my_dir})/homepage/images/favicon.ico ${EXTRACTED}
   sudo -u ${USER} ln -fs ${HOME}/phpsysinfo ${EXTRACTED}
-  sudo -u ${USER} cp -f $(dirname ${my_dir})/templates/phpsysinfo.ini ${HOME}/phpsysinfo/
+  sudo -u ${USER} ln -fs $(dirname ${my_dir})/templates/phpsysinfo.ini ${HOME}/phpsysinfo/
-  sudo -u ${USER} cp -f $(dirname ${my_dir})/templates/green_bootstrap.css ${HOME}/phpsysinfo/templates/
+  sudo -u ${USER} ln -fs $(dirname ${my_dir})/templates/green_bootstrap.css ${HOME}/phpsysinfo/templates/
-  sudo -u ${USER} cp -f $(dirname ${my_dir})/templates/index_bootstrap.html ${HOME}/phpsysinfo/templates/html
+  sudo -u ${USER} ln -fs $(dirname ${my_dir})/templates/index_bootstrap.html ${HOME}/phpsysinfo/templates/html
 
   echo "Setting Wttr.in URL to "${LATITUDE}", "${LONGITUDE}""
   sudo -u${USER} sed -i "s/https:\/\/v2.wttr.in\//https:\/\/v2.wttr.in\/"${LATITUDE},${LONGITUDE}"/g" $(dirname ${my_dir})/homepage/menu.html

scripts/overview.php

@@ -7,26 +7,50 @@ $myDate = date('Y-m-d');
 $chart = "Combo-$myDate.png";
 
 $db = new SQLite3('/home/pi/BirdNET-Pi/scripts/birds.db', SQLITE3_OPEN_CREATE | SQLITE3_OPEN_READWRITE);
+if($db == False) {
+  echo "Database is busy";
+  header("refresh: 0;");
+}
 
 $statement = $db->prepare('SELECT COUNT(*) FROM detections');
+if($statement == False) {
+  echo "Database is busy";
+  header("refresh: 0;");
+}
 $result = $statement->execute();
 $totalcount = $result->fetchArray(SQLITE3_ASSOC);
 
 $statement2 = $db->prepare('SELECT COUNT(*) FROM detections WHERE Date == DATE(\'now\')');
+if($statement2 == False) {
+  echo "Database is busy";
+  header("refresh: 0;");
+}
 $result2 = $statement2->execute();
 $todaycount = $result2->fetchArray(SQLITE3_ASSOC);
 
 $statement3 = $db->prepare('SELECT COUNT(*) FROM detections WHERE TIME >= TIME(\'now\', \'localtime\', \'-1 hour\')');
+if($statement3 == False) {
+  echo "Database is busy";
+  header("refresh: 0;");
+}
 $result3 = $statement3->execute();
 $hourcount = $result3->fetchArray(SQLITE3_ASSOC);
 
-$statement4 = $db->prepare('SELECT Com_Name, Sci_Name, Date, Time, Confidence, File_Name FROM detections ORDER BY Time ASC LIMIT 1');
+$statement4 = $db->prepare('SELECT Com_Name, Sci_Name, Date, Time, Confidence, File_Name FROM detections ORDER BY Date DESC, Time DESC LIMIT 1');
+if($statement4 == False) {
+  echo "Database is busy";
+  header("refresh: 0;");
+}
 $result4 = $statement4->execute();
 $mostrecent = $result4->fetchArray(SQLITE3_ASSOC);
 $comname = preg_replace('/ /', '_', $mostrecent['Com_Name']);
 $scilink = preg_replace('/ /', '_', $mostrecent['Sci_Name']);
 
 $statement5 = $db->prepare('SELECT COUNT(DISTINCT(Com_Name)) FROM detections WHERE Date == Date(\'now\')');
+if($statement5 == False) {
+  echo "Database is busy";
+  header("refresh: 0;");
+}
 $result5 = $statement5->execute();
 $speciestally = $result5->fetchArray(SQLITE3_ASSOC);
 ?>
@@ -54,6 +78,12 @@ table, th {
 th {
   padding: 0 5px;
 }
+button {
+  background-color: rgb(219, 295, 235);
+  border:none;
+  font-size:large;
+  cursor:pointer;
+}
 .center {
   display: block;
   margin-left: 5px;
@@ -101,7 +131,8 @@ th {
       <tr>
         <th>Most Recent Detection</th>
 	<td><a href="https://wikipedia.org/wiki/<?php echo $scilink;?>" target="top"/><?php echo $mostrecent['Sci_Name'];?></a></td>
-	<td><a href="/By_Date/<?php echo $mostrecent['Date']."/".$comname;?>"><?php echo $mostrecent['Com_Name'];?></a></td>
+        <form action="/stats.php" name="species" method="POST">
+	<td><button type="submit" name="species" value="<?php echo $mostrecent['Com_Name'];?>"><?php echo $mostrecent['Com_Name'];?></button></td></form>
 	<td><a href="/By_Date/<?php echo$myDate."/".$comname."/".$mostrecent['File_Name'];?>" target="footer"/><?php echo $mostrecent['Date']." ".$mostrecent['Time'];?></a></td>
 	<td><?php echo $mostrecent['Confidence'];?></td>
       </tr>

scripts/server.py

@@ -353,12 +353,16 @@ def handle_client(conn, addr):
                                         Date.replace("/", "-") + '-birdnet-' + Time + audiofmt
 
                                 #Connect to SQLite Database
-                                con = sqlite3.connect('/home/pi/BirdNET-Pi/scripts/birds.db')
+                                try: 
-                                cur = con.cursor()
+                                    con = sqlite3.connect('/home/pi/BirdNET-Pi/scripts/birds.db')
-                                cur.execute("INSERT INTO detections VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (Date, Time, Sci_Name, Com_Name, str(score), Lat, Lon, Cutoff, Week, Sens, Overlap, File_Name))
+                                    cur = con.cursor()
+                                    cur.execute("INSERT INTO detections VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (Date, Time, Sci_Name, Com_Name, str(score), Lat, Lon, Cutoff, Week, Sens, Overlap, File_Name))
 
-                                con.commit()
+                                    con.commit()
-                                con.close()
+                                    con.close()
+                                except:
+                                    print("Database busy")
+                                    time.sleep(2)
                                 print(str(current_date) + ';' + str(current_time) + ';' + entry[0].replace('_', ';') + ';' + str(entry[1]) + ';' + str(args.lat) + ';' + str(args.lon) + ';' + str(min_conf) + ';' + str(week) + ';' + str(args.sensitivity) +';' + str(args.overlap) + Com_Name.replace(" ", "_") + '-' + str(score) + '-' + str(current_date) + '-birdnet-' + str(current_time) + audiofmt  + '\n')
 
                                 if birdweather_id != "99999":

scripts/stats.php

@@ -4,17 +4,33 @@ ini_set('display_startup_errors', 1);
 error_reporting(E_ALL);
 
 $db = new SQLite3('/home/pi/BirdNET-Pi/scripts/birds.db', SQLITE3_OPEN_CREATE | SQLITE3_OPEN_READWRITE);
-
+if($db == False) {
+	echo "Database busy";
+	header("refresh: 0;");
+}
 $statement = $db->prepare('SELECT Com_Name, COUNT(*), MAX(Confidence), Sci_Name FROM detections GROUP BY Com_Name ORDER BY COUNT(*) DESC');
+if($statement == False) {
+	echo "Database busy";
+	header("refresh: 0;");
+}
 $result = $statement->execute();
 
+
 $statement2 = $db->prepare('SELECT Com_Name FROM detections GROUP BY Com_Name ORDER BY Com_Name ASC');
+if($statement2 == False) {
+	echo "Database busy";
+	header("refresh: 0;");
+}
 $result2 = $statement2->execute();
 
 if(isset($_POST['species'])){
   $selection = $_POST['species'];
   $statement3 = $db->prepare("SELECT Com_Name, Sci_Name, COUNT(*), MAX(Confidence) from detections
     WHERE Com_Name = '$selection'");
+  if($statement3 == False) {
+  	echo "Database busy";
+  	header("refresh: 0;");
+  }
   $result3 = $statement3->execute();
 }
 ?>
@@ -80,6 +96,12 @@ a {
 	text-align: center;
 }
 
+button {
+  background-color: rgb(219, 295, 235);
+  border:none;
+  font-size:large;
+  cursor:pointer;
+}
 
 img {
 	width:75%;
@@ -143,7 +165,8 @@ $comlink = "/By_Date/".date('Y-m-d')."/".$comname;
 $sciname = preg_replace('/ /', '_', $results['Sci_Name']);
 ?>
       <tr>
-      <td><?php echo $results['Com_Name'];?></td>
+      <form action="stats.php" method="POST">
+      <td><button type="submit" name="species" value="<?php echo $results['Com_Name'];?>"><?php echo $results['Com_Name'];?></button></td></form>
       <td><?php echo $results['COUNT(*)'];?></td>
       <td><?php echo $results['MAX(Confidence)'];?></td>
       </tr>

scripts/viewdb.php

@@ -6,29 +6,58 @@ error_reporting(E_ALL);
 header("refresh: 30;");
 
 $db = new SQLite3('/home/pi/BirdNET-Pi/scripts/birds.db', SQLITE3_OPEN_CREATE | SQLITE3_OPEN_READWRITE);
+if($db == False){
+  echo "Database is busy";
+  header("refresh: 0;");
+}
 
 $statement0 = $db->prepare('SELECT Time, Com_Name, Sci_Name, Confidence, File_Name FROM detections WHERE Date == Date(\'now\') ORDER BY Time DESC');
+if($statement0 == False){
+  echo "Database is busy";
+  header("refresh: 0;");
+}
 $result0 = $statement0->execute();
 
 $statement1 = $db->prepare('SELECT COUNT(*) FROM detections');
+if($statement1 == False){
+  echo "Database is busy";
+  header("refresh: 0;");
+}
 $result1 = $statement1->execute();
 $totalcount = $result1->fetchArray(SQLITE3_ASSOC);
 
 $statement2 = $db->prepare('SELECT COUNT(*) FROM detections WHERE Date == DATE(\'now\')');
+if($statement2 == False){
+  echo "Database is busy";
+  header("refresh: 0;");
+}
 $result2 = $statement2->execute();
 $todaycount = $result2->fetchArray(SQLITE3_ASSOC);
 
 $statement3 = $db->prepare('SELECT COUNT(*) FROM detections WHERE TIME >= TIME(\'now\', \'localtime\', \'-1 hour\')');
+if($statement3 == False){
+  echo "Database is busy";
+  header("refresh: 0;");
+}
 $result3 = $statement3->execute();
 $hourcount = $result3->fetchArray(SQLITE3_ASSOC);
 
 $statement4 = $db->prepare('SELECT Com_Name, Sci_Name, Time, Confidence FROM detections LIMIT 1');
+if($statement4 == False){
+  echo "Database is busy";
+  header("refresh: 0;");
+}
 $result4 = $statement4->execute();
 $mostrecent = $result4->fetchArray(SQLITE3_ASSOC);
 
 $statement5 = $db->prepare('SELECT COUNT(DISTINCT(Com_Name)) FROM detections');
+if($statement4 == False){
+  echo "Database is busy";
+  header("refresh: 0;");
+}
 $result5 = $statement5->execute();
 $speciestally = $result5->fetchArray(SQLITE3_ASSOC);
+
 ?>
 
 <!DOCTYPE html>

templates/phpsysinfo.ini

@@ -397,7 +397,7 @@ HIDE_TOTALS=false
 ; Example : HIDE_NETWORK_INTERFACE="eth0,sit0"
 ;           HIDE_NETWORK_INTERFACE=true  //hide all network interfaces
 ;
-HIDE_NETWORK_INTERFACE="lo"
+HIDE_NETWORK_INTERFACE=""
 
 
 ; Use a regular expression in the name of a hidden network interface (e.g. HIDE_NETWORK_INTERFACE="docker.*")
@@ -785,7 +785,7 @@ USE_REGEX=false
 ;
 ; string contains a list of process names that are checked, names are seperated by a comma (on WinNT names must end with '.exe')
 ;
-PROCESSES="avahi-daemon,gotty,sshd,mariadbd,ffmpeg,vncserver-x11-serviced,nxd,bash"
+PROCESSES="avahi-daemon,gotty,sshd,ffmpeg,icecast2,arecord,server.py"
 
 
 [quotas]