From 286aa095f9dcc581fa978eafcff634b2e9dc8c05 Mon Sep 17 00:00:00 2001 From: mcguirepr89 Date: Mon, 21 Feb 2022 16:33:01 -0500 Subject: [PATCH] time to test installation --- scripts/daily_plot.py.newer | 141 ++++++++++++++++++++++++++++++++++++ scripts/extra_info.sh | 32 ++++++-- scripts/install_services.sh | 8 +- scripts/overview.php | 35 ++++++++- scripts/server.py | 14 ++-- scripts/stats.php | 27 ++++++- scripts/viewdb.php | 29 ++++++++ templates/phpsysinfo.ini | 4 +- 8 files changed, 267 insertions(+), 23 deletions(-) create mode 100755 scripts/daily_plot.py.newer diff --git a/scripts/daily_plot.py.newer b/scripts/daily_plot.py.newer new file mode 100755 index 0000000..8e1729a --- /dev/null +++ b/scripts/daily_plot.py.newer @@ -0,0 +1,141 @@ +#!/home/pi/BirdNET-Pi/birdnet/bin/python3 + +import os +import configparser +import pandas as pd +import seaborn as sns +import numpy as np +import matplotlib.pyplot as plt +from matplotlib.colors import LogNorm +from datetime import datetime +import textwrap +import sqlite3 + +conn = sqlite3.connect('/home/pi/BirdNET-Pi/scripts/birds.db') +df = pd.read_sql_query("SELECT * from detections", conn) +cursor = conn.cursor() + +table_rows = cursor.fetchall() + +#df=pd.DataFrame(table_rows) + +#Convert Date and Time Fields to Panda's format +df['Date']=pd.to_datetime(df['Date']) +df['Time']=pd.to_datetime(df['Time'], unit='ns') + + +#Add round hours to dataframe +df['Hour of Day'] = [r.hour for r in df.Time] + +#Create separate dataframes for separate locations +df_plt=df #Default to use the whole Dbase + +#Get todays readings +now = datetime.now() +df_plt_today = df_plt[df_plt['Date']==now.strftime("%Y-%m-%d")] + +#Set number of species to report +#For ALL +readings = len(df_plt_today['Com_Name'].value_counts()) +# Uncomment for user selection +readings = 10 + + +plt_top10_today = (df_plt_today['Com_Name'].value_counts()[:readings]) + +df_plt_top10_today = df_plt_today[df_plt_today.Com_Name.isin(plt_top10_today.index)] + +#Set Palette for graphics +pal = "Greens" + +#Set up plot axes and titles + +# f, axs = plt.subplots(1, 3, figsize = (10, 5 * vert_scale), gridspec_kw=dict(width_ratios=[3, 2, 5])) + +vert_scale = readings / 10 +f, axs = plt.subplots(1, 2, figsize = (10, 5 * vert_scale), gridspec_kw=dict(width_ratios=[3, 6]), facecolor='#77C487') +plt.subplots_adjust(left=None, bottom=None, right=None, top=None, wspace=0, hspace=0) + +#generate y-axis order for all figures based on frequency +freq_order = pd.value_counts(df_plt_top10_today['Com_Name']).iloc[:readings].index + +# this groups by name and calculates mean/max conf +confmax = df_plt_top10_today.groupby('Com_Name')['Confidence'].max() +confavg = df_plt_top10_today.groupby('Com_Name')['Confidence'].mean() +#reorder confmax/avg to detection frequency order +confmax = confmax.reindex(freq_order) +confavg = confavg.reindex(freq_order) +# norm avg values for color palette +norm = plt.Normalize(confavg.values.min(), confavg.values.max()) +# bars of frequency plot based on avg color palette +colors = plt.cm.Greens(norm(confavg)) + +#Generate frequency plot +plot=sns.countplot(y='Com_Name', data = df_plt_top10_today, palette = colors, order=freq_order, ax=axs[0]) + +# for container in axs[0].containers: +# axs[0].bar_label(containers) + +# Function to show value on bars - from https://stackoverflow.com/questions/43214978/seaborn-barplot-displaying-values +def show_values_on_bars(ax,label): + i = 0 + for p in ax.patches: + _x = p.get_x() + p.get_width()* 0.9 + _y = p.get_y() + p.get_height() / 2 + value = '{:.0%}'.format(label[i]) + # Uncomment for Species Count Total + # value = '{:,}'.format(p.get_width()) + ax.text(_x, _y, value, ha='center', va='center', size=8, fontweight='bold', color='darkgreen', bbox=dict(facecolor='lightgrey',pad = 4.0)) + i=i+1 + +# Prints Max Confidence on bars +show_values_on_bars(axs[0],confmax) + +#Try plot grid lines between bars - problem at the moment plots grid lines on bars - want between bars +# plot.grid(True, axis='y') +z=plot.get_ymajorticklabels() +plot.set_yticklabels(['\n'.join(textwrap.wrap(ticklabel.get_text(),15)) for ticklabel in plot.get_yticklabels()], fontsize = 10) +plot.set(ylabel=None) +plot.set(xlabel="Detections") +# Comma formatting for when your Detections are >1,000 +# current_values=plot.gca().get_xticks() +# plt.gca().set_xticklabels(['{:,0f}'.format(x) for x in current_values]) + +#If you want violin/box plots uncomment here and ** above +# plot = sns.boxenplot(x=df_plt_top10_today['Confidence']*100,color='Green', y=df_plt_top10_today['Com_Name'], ax=axs[1],order=freq_order) +# plot.set(xlabel="Confidence", ylabel=None,yticklabels=[]) + + +#Generate crosstab matrix for heatmap plot + +heat = pd.crosstab(df_plt_top10_today['Com_Name'],df_plt_top10_today['Hour of Day']) +#Order heatmap Birds by frequency of occurrance +heat.index = pd.CategoricalIndex(heat.index, categories = freq_order) +heat.sort_index(level=0, inplace=True) + + +hours_in_day = pd.Series(data = range(0,24)) +heat_frame = pd.DataFrame(data=0, index=heat.index, columns = hours_in_day) +heat=(heat+heat_frame).fillna(0) + +#Generatie heatmap plot +plot = sns.heatmap(heat, norm=LogNorm(), annot=True, fmt="g", annot_kws={"fontsize":7}, cmap = pal , square = False, cbar=False, linewidths = 0.5, linecolor = "Grey", ax=axs[1], yticklabels = False) +plot.set_xticklabels(plot.get_xticklabels(), rotation = 0, size = 8) + +# Set heatmap border +for _, spine in plot.spines.items(): + spine.set_visible(True) + +plot.set(ylabel=None) +plot.set(xlabel="Hour of Day") +#Set combined plot layout and titles +#f.tight_layout() +#f.subplots_adjust(top=0.95) +#f.suptitle("DAILY OVERVIEW FOR "+ str(now.strftime("%d-%m-%Y %H:%M")),x=0.5,y=1.5,va='top') + +#Save combined plot +savename='/home/pi/BirdSongs/Extracted/Charts/Combo-'+str(now.strftime("%Y-%m-%d"))+'.png' +plt.savefig(savename) +plt.show() +plt.close() + diff --git a/scripts/extra_info.sh b/scripts/extra_info.sh index 5438b8f..129a226 100755 --- a/scripts/extra_info.sh +++ b/scripts/extra_info.sh @@ -13,28 +13,44 @@ revbinary=$(echo $binary|rev) if echo $binary | grep 1 ;then echo "ISSUES DETECTED" if [ ${revbinary:0:1} -eq 1 &> /dev/null ];then - echo "Under-voltage detected" + message="Under-voltage detected" + echo "$message" + dmesg -H | grep -i voltage fi if [ ${revbinary:1:1} -eq 1 &> /dev/null ];then - echo "Arm frequency capped" + message="Arm frequency capped" + echo "$message" + dmesg -H | grep -i frequen fi if [ ${revbinary:2:1} -eq 1 &> /dev/null ];then - echo "Currently Throttled" + message="Currently Throttled" + echo "$message" + dmesg -H | grep -i throttl fi if [ ${revbinary:3:1} -eq 1 &> /dev/null ];then - echo "Soft temperatue limit active" + message="Soft temperatue limit active" + echo "$message" + dmesg -H | grep -i temperature fi if [ ${revbinary:16:1} -eq 1 &> /dev/null ];then - echo "Under-voltage has occurred" + message="Under-voltage has occurred" + echo "$message" + dmesg -H | grep -i voltage fi if [ ${revbinary:17:1} -eq 1 &> /dev/null ];then - echo "Arm frequency capping has occurred" + message="Arm frequency capping has occurred" + echo "$message" + dmesg -H | grep -i frequen fi if [ ${revbinary:18:1} -eq 1 &> /dev/null ];then - echo "Throttling has occurred" + message="Throttling has occurred" + echo "$message" + dmesg -H | grep -i throttl fi if [ ${revbinary:19:1} -eq 1 &> /dev/null ];then - echo "Soft temperature limit has occurred" + message="Soft temperature limit has occurred" + echo "$message" + dmesg -H | grep -i temperature fi fi echo "....................................Clock Speeds................................" diff --git a/scripts/install_services.sh b/scripts/install_services.sh index 62c6bcb..6116da7 100755 --- a/scripts/install_services.sh +++ b/scripts/install_services.sh @@ -84,7 +84,7 @@ Restart=on-failure RestartSec=3 Type=simple User=${USER} -ExecStart=/usr/bin/env bash -c 'while true;do extract_new_birdsounds.sh;sleep ${RECORDING_LENGTH};done' +ExecStart=/usr/bin/env bash -c 'while true;do extract_new_birdsounds.sh;sleep 3;done' [Install] WantedBy=multi-user.target EOF @@ -152,9 +152,9 @@ create_necessary_dirs() { sudo -u ${USER} ln -fs $(dirname ${my_dir})/scripts/history.php ${EXTRACTED} sudo -u ${USER} ln -fs $(dirname ${my_dir})/homepage/images/favicon.ico ${EXTRACTED} sudo -u ${USER} ln -fs ${HOME}/phpsysinfo ${EXTRACTED} - sudo -u ${USER} cp -f $(dirname ${my_dir})/templates/phpsysinfo.ini ${HOME}/phpsysinfo/ - sudo -u ${USER} cp -f $(dirname ${my_dir})/templates/green_bootstrap.css ${HOME}/phpsysinfo/templates/ - sudo -u ${USER} cp -f $(dirname ${my_dir})/templates/index_bootstrap.html ${HOME}/phpsysinfo/templates/html + sudo -u ${USER} ln -fs $(dirname ${my_dir})/templates/phpsysinfo.ini ${HOME}/phpsysinfo/ + sudo -u ${USER} ln -fs $(dirname ${my_dir})/templates/green_bootstrap.css ${HOME}/phpsysinfo/templates/ + sudo -u ${USER} ln -fs $(dirname ${my_dir})/templates/index_bootstrap.html ${HOME}/phpsysinfo/templates/html echo "Setting Wttr.in URL to "${LATITUDE}", "${LONGITUDE}"" sudo -u${USER} sed -i "s/https:\/\/v2.wttr.in\//https:\/\/v2.wttr.in\/"${LATITUDE},${LONGITUDE}"/g" $(dirname ${my_dir})/homepage/menu.html diff --git a/scripts/overview.php b/scripts/overview.php index c8c62c9..edbc53f 100644 --- a/scripts/overview.php +++ b/scripts/overview.php @@ -7,26 +7,50 @@ $myDate = date('Y-m-d'); $chart = "Combo-$myDate.png"; $db = new SQLite3('/home/pi/BirdNET-Pi/scripts/birds.db', SQLITE3_OPEN_CREATE | SQLITE3_OPEN_READWRITE); +if($db == False) { + echo "Database is busy"; + header("refresh: 0;"); +} $statement = $db->prepare('SELECT COUNT(*) FROM detections'); +if($statement == False) { + echo "Database is busy"; + header("refresh: 0;"); +} $result = $statement->execute(); $totalcount = $result->fetchArray(SQLITE3_ASSOC); $statement2 = $db->prepare('SELECT COUNT(*) FROM detections WHERE Date == DATE(\'now\')'); +if($statement2 == False) { + echo "Database is busy"; + header("refresh: 0;"); +} $result2 = $statement2->execute(); $todaycount = $result2->fetchArray(SQLITE3_ASSOC); $statement3 = $db->prepare('SELECT COUNT(*) FROM detections WHERE TIME >= TIME(\'now\', \'localtime\', \'-1 hour\')'); +if($statement3 == False) { + echo "Database is busy"; + header("refresh: 0;"); +} $result3 = $statement3->execute(); $hourcount = $result3->fetchArray(SQLITE3_ASSOC); -$statement4 = $db->prepare('SELECT Com_Name, Sci_Name, Date, Time, Confidence, File_Name FROM detections ORDER BY Time ASC LIMIT 1'); +$statement4 = $db->prepare('SELECT Com_Name, Sci_Name, Date, Time, Confidence, File_Name FROM detections ORDER BY Date DESC, Time DESC LIMIT 1'); +if($statement4 == False) { + echo "Database is busy"; + header("refresh: 0;"); +} $result4 = $statement4->execute(); $mostrecent = $result4->fetchArray(SQLITE3_ASSOC); $comname = preg_replace('/ /', '_', $mostrecent['Com_Name']); $scilink = preg_replace('/ /', '_', $mostrecent['Sci_Name']); $statement5 = $db->prepare('SELECT COUNT(DISTINCT(Com_Name)) FROM detections WHERE Date == Date(\'now\')'); +if($statement5 == False) { + echo "Database is busy"; + header("refresh: 0;"); +} $result5 = $statement5->execute(); $speciestally = $result5->fetchArray(SQLITE3_ASSOC); ?> @@ -54,6 +78,12 @@ table, th { th { padding: 0 5px; } +button { + background-color: rgb(219, 295, 235); + border:none; + font-size:large; + cursor:pointer; +} .center { display: block; margin-left: 5px; @@ -101,7 +131,8 @@ th { Most Recent Detection - "> +
+
" target="footer"/> diff --git a/scripts/server.py b/scripts/server.py index 7aa6f1d..3916582 100755 --- a/scripts/server.py +++ b/scripts/server.py @@ -353,12 +353,16 @@ def handle_client(conn, addr): Date.replace("/", "-") + '-birdnet-' + Time + audiofmt #Connect to SQLite Database - con = sqlite3.connect('/home/pi/BirdNET-Pi/scripts/birds.db') - cur = con.cursor() - cur.execute("INSERT INTO detections VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (Date, Time, Sci_Name, Com_Name, str(score), Lat, Lon, Cutoff, Week, Sens, Overlap, File_Name)) + try: + con = sqlite3.connect('/home/pi/BirdNET-Pi/scripts/birds.db') + cur = con.cursor() + cur.execute("INSERT INTO detections VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (Date, Time, Sci_Name, Com_Name, str(score), Lat, Lon, Cutoff, Week, Sens, Overlap, File_Name)) - con.commit() - con.close() + con.commit() + con.close() + except: + print("Database busy") + time.sleep(2) print(str(current_date) + ';' + str(current_time) + ';' + entry[0].replace('_', ';') + ';' + str(entry[1]) + ';' + str(args.lat) + ';' + str(args.lon) + ';' + str(min_conf) + ';' + str(week) + ';' + str(args.sensitivity) +';' + str(args.overlap) + Com_Name.replace(" ", "_") + '-' + str(score) + '-' + str(current_date) + '-birdnet-' + str(current_time) + audiofmt + '\n') if birdweather_id != "99999": diff --git a/scripts/stats.php b/scripts/stats.php index 597d35f..1c684c8 100644 --- a/scripts/stats.php +++ b/scripts/stats.php @@ -4,17 +4,33 @@ ini_set('display_startup_errors', 1); error_reporting(E_ALL); $db = new SQLite3('/home/pi/BirdNET-Pi/scripts/birds.db', SQLITE3_OPEN_CREATE | SQLITE3_OPEN_READWRITE); - +if($db == False) { + echo "Database busy"; + header("refresh: 0;"); +} $statement = $db->prepare('SELECT Com_Name, COUNT(*), MAX(Confidence), Sci_Name FROM detections GROUP BY Com_Name ORDER BY COUNT(*) DESC'); +if($statement == False) { + echo "Database busy"; + header("refresh: 0;"); +} $result = $statement->execute(); + $statement2 = $db->prepare('SELECT Com_Name FROM detections GROUP BY Com_Name ORDER BY Com_Name ASC'); +if($statement2 == False) { + echo "Database busy"; + header("refresh: 0;"); +} $result2 = $statement2->execute(); if(isset($_POST['species'])){ $selection = $_POST['species']; $statement3 = $db->prepare("SELECT Com_Name, Sci_Name, COUNT(*), MAX(Confidence) from detections WHERE Com_Name = '$selection'"); + if($statement3 == False) { + echo "Database busy"; + header("refresh: 0;"); + } $result3 = $statement3->execute(); } ?> @@ -80,6 +96,12 @@ a { text-align: center; } +button { + background-color: rgb(219, 295, 235); + border:none; + font-size:large; + cursor:pointer; +} img { width:75%; @@ -143,7 +165,8 @@ $comlink = "/By_Date/".date('Y-m-d')."/".$comname; $sciname = preg_replace('/ /', '_', $results['Sci_Name']); ?> - +
+
diff --git a/scripts/viewdb.php b/scripts/viewdb.php index 6d8dd68..3bd7238 100644 --- a/scripts/viewdb.php +++ b/scripts/viewdb.php @@ -6,29 +6,58 @@ error_reporting(E_ALL); header("refresh: 30;"); $db = new SQLite3('/home/pi/BirdNET-Pi/scripts/birds.db', SQLITE3_OPEN_CREATE | SQLITE3_OPEN_READWRITE); +if($db == False){ + echo "Database is busy"; + header("refresh: 0;"); +} $statement0 = $db->prepare('SELECT Time, Com_Name, Sci_Name, Confidence, File_Name FROM detections WHERE Date == Date(\'now\') ORDER BY Time DESC'); +if($statement0 == False){ + echo "Database is busy"; + header("refresh: 0;"); +} $result0 = $statement0->execute(); $statement1 = $db->prepare('SELECT COUNT(*) FROM detections'); +if($statement1 == False){ + echo "Database is busy"; + header("refresh: 0;"); +} $result1 = $statement1->execute(); $totalcount = $result1->fetchArray(SQLITE3_ASSOC); $statement2 = $db->prepare('SELECT COUNT(*) FROM detections WHERE Date == DATE(\'now\')'); +if($statement2 == False){ + echo "Database is busy"; + header("refresh: 0;"); +} $result2 = $statement2->execute(); $todaycount = $result2->fetchArray(SQLITE3_ASSOC); $statement3 = $db->prepare('SELECT COUNT(*) FROM detections WHERE TIME >= TIME(\'now\', \'localtime\', \'-1 hour\')'); +if($statement3 == False){ + echo "Database is busy"; + header("refresh: 0;"); +} $result3 = $statement3->execute(); $hourcount = $result3->fetchArray(SQLITE3_ASSOC); $statement4 = $db->prepare('SELECT Com_Name, Sci_Name, Time, Confidence FROM detections LIMIT 1'); +if($statement4 == False){ + echo "Database is busy"; + header("refresh: 0;"); +} $result4 = $statement4->execute(); $mostrecent = $result4->fetchArray(SQLITE3_ASSOC); $statement5 = $db->prepare('SELECT COUNT(DISTINCT(Com_Name)) FROM detections'); +if($statement4 == False){ + echo "Database is busy"; + header("refresh: 0;"); +} $result5 = $statement5->execute(); $speciestally = $result5->fetchArray(SQLITE3_ASSOC); + ?> diff --git a/templates/phpsysinfo.ini b/templates/phpsysinfo.ini index 90bab10..ccf35df 100644 --- a/templates/phpsysinfo.ini +++ b/templates/phpsysinfo.ini @@ -397,7 +397,7 @@ HIDE_TOTALS=false ; Example : HIDE_NETWORK_INTERFACE="eth0,sit0" ; HIDE_NETWORK_INTERFACE=true //hide all network interfaces ; -HIDE_NETWORK_INTERFACE="lo" +HIDE_NETWORK_INTERFACE="" ; Use a regular expression in the name of a hidden network interface (e.g. HIDE_NETWORK_INTERFACE="docker.*") @@ -785,7 +785,7 @@ USE_REGEX=false ; ; string contains a list of process names that are checked, names are seperated by a comma (on WinNT names must end with '.exe') ; -PROCESSES="avahi-daemon,gotty,sshd,mariadbd,ffmpeg,vncserver-x11-serviced,nxd,bash" +PROCESSES="avahi-daemon,gotty,sshd,ffmpeg,icecast2,arecord,server.py" [quotas]