claudecode/AvianVisitors
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rewritten php and some python for SQLITE

Browse Source
This commit is contained in:
mcguirepr89
2022-02-19 13:09:32 -05:00
parent f30ac0b8d2
commit 857fd981a9
10 changed files with 122 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/daily_plot.py
+7 -19
View File
@@ -1,9 +1,8 @@
#!/home/pi/BirdNET-Pi/birdnet/bin/python3
import mysql.connector as sql
import sqlite3
import os
import configparser
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
@@ -11,25 +10,14 @@ from matplotlib.colors import LogNorm
from datetime import datetime
import textwrap
#Extract DB_PWD from thisrun.txt
with open('/home/pi/BirdNET-Pi/thisrun.txt', 'r') as f:
this_run = f.readlines()
db_pwd = str(str(str([i for i in this_run if i.startswith('DB_PWD')]).split('=')[1]).split('\\')[0])
conn = sqlite3.connect('/home/pi/BirdNET-Pi/scripts/birds.db')
df = pd.read_sql_query("SELECT * from detections", conn)
cursor = conn.cursor()
cursor.execute('SELECT * FROM detections')
table_rows = cursor.fetchall()
db_connection = sql.connect(host='localhost',
database='birds',
user='birder',
password=db_pwd)
db_cursor=db_connection.cursor(dictionary=True)
db_cursor.execute('SELECT * FROM detections')
table_rows = db_cursor.fetchall()
df=pd.DataFrame(table_rows)
#df=pd.DataFrame(table_rows)
#Convert Date and Time Fields to Panda's format
df['Date']=pd.to_datetime(df['Date'])
scripts/extract_new_birdsounds.sh
+6 -62
View File
@@ -61,14 +61,13 @@ for h in "${SCAN_DIRS[@]}";do
SCIENTIFIC_NAME=""$(echo ${line} \
| awk -F\; '!/birdnet/{print $3}')""
CONFIDENCE=""$(echo ${line} \
| awk -F\; '{print $5}' \
| cut -d'.' -f2)""
CONFIDENCE_SCORE="${CONFIDENCE:0:2}%"
| awk -F\; '{print $5}')""
#CONFIDENCE_SCORE="${CONFIDENCE:0:2}%"
CONFIDENCE_SCORE="$(printf %.0f $(echo "scale=2; ${CONFIDENCE} * 100" | bc))"
NEWFILE="${COMMON_NAME// /_}-${CONFIDENCE_SCORE}-${OLDFILE//.wav/.${AUDIOFMT}}"
echo "NEWFILE=$NEWFILE"
NEWSPECIES_BYDATE="${EXTRACTED}/By_Date/${DATE}/${COMMON_NAME// /_}"
NEWSPECIES_BY_COMMON="${EXTRACTED}/By_Common_Name/${COMMON_NAME// /_}"
NEWSPECIES_BY_SCIENCE="${EXTRACTED}/By_Scientific_Name/${SCIENTIFIC_NAME// /_}"
# If the extracted file already exists, move on
if [[ -f "${NEWSPECIES_BYDATE}/${NEWFILE}" ]];then
@@ -85,16 +84,6 @@ for h in "${SCAN_DIRS[@]}";do
# it is created
[[ -d "${NEWSPECIES_BYDATE}" ]] || mkdir -p "${NEWSPECIES_BYDATE}"
# If a directory does not already exist for the species (by-species),
# it is created.
[[ -d "${NEWSPECIES_BY_COMMON}" ]] || mkdir -p "${NEWSPECIES_BY_COMMON}"
# If a directory does not already exist for the species (by-species),
# it is created.
[[ -d "${NEWSPECIES_BY_SCIENCE}" ]] || mkdir -p "${NEWSPECIES_BY_SCIENCE}"
# If there are already 20 extracted entries for a given species
# for today, remove the oldest file and create the new one.
# if [[ "$(find ${NEWSPECIES_BYDATE} | wc -l)" -ge 20 ]];then
@@ -131,9 +120,6 @@ for h in "${SCAN_DIRS[@]}";do
sox "${h}/${OLDFILE}" "${NEWSPECIES_BYDATE}/${NEWFILE}" \
trim "${START}" "${END}"
#ffmpeg -hide_banner -loglevel error -nostdin -i "${h}/${OLDFILE}" \
# -acodec copy -ss "${START}" -to "${END}"\
# "${NEWSPECIES_BYDATE}/${NEWFILE}"
# Create spectrogram for extraction
sox "${NEWSPECIES_BYDATE}/${NEWFILE}" -n remix 1 rate 24k spectrogram \
@@ -141,48 +127,6 @@ for h in "${SCAN_DIRS[@]}";do
-c "${NEWSPECIES_BYDATE}/${NEWFILE}" \
-o "${NEWSPECIES_BYDATE}/${NEWFILE}.png"
# Remove the oldest symbolic links that would make the directory have more
# than 20 entries.
if [[ "$(find ${NEWSPECIES_BY_COMMON} | wc -l)" -ge 40 ]];then
echo "20 ${SPECIES}s, already! Removing the oldest by-species and making a new one"
cd ${NEWSPECIES_BY_COMMON} || exit 1
ls -1t . | tail -n +40 | xargs -r rm -vv
ln -fs "${NEWSPECIES_BYDATE}/${NEWFILE}"\
"${NEWSPECIES_BY_COMMON}/${NEWFILE}"
ln -fs "${NEWSPECIES_BYDATE}/${NEWFILE}.png"\
"${NEWSPECIES_BY_COMMON}/${NEWFILE}.png"
echo "Success! New extraction for ${COMMON_NAME}"
else
# Make symbolic link of the extraction to add to By_Common_Name
ln -fs "${NEWSPECIES_BYDATE}/${NEWFILE}"\
"${NEWSPECIES_BY_COMMON}/${NEWFILE}"
ln -fs "${NEWSPECIES_BYDATE}/${NEWFILE}.png"\
"${NEWSPECIES_BY_COMMON}/${NEWFILE}.png"
fi
# Remove the oldest symbolic links that would made the directory have more
# than 20 entries.
if [[ "$(find ${NEWSPECIES_BY_SCIENCE} | wc -l)" -ge 40 ]];then
echo "20 ${SPECIES}s, already! Removing the oldest by-species and making a new one"
cd ${NEWSPECIES_BY_SCIENCE} || exit 1
ls -1t . | tail -n +40 | xargs -r rm -vv
ln -fs "${NEWSPECIES_BYDATE}/${NEWFILE}"\
"${NEWSPECIES_BY_SCIENCE}/${NEWFILE}"
ln -fs "${NEWSPECIES_BYDATE}/${NEWFILE}.png"\
"${NEWSPECIES_BY_SCIENCE}/${NEWFILE}.png"
echo "Success! New extraction for ${COMMON_NAME}"
else
ln -fs "${NEWSPECIES_BYDATE}/${NEWFILE}"\
"${NEWSPECIES_BY_SCIENCE}/${NEWFILE}"
ln -fs "${NEWSPECIES_BYDATE}/${NEWFILE}.png"\
"${NEWSPECIES_BY_SCIENCE}/${NEWFILE}.png"
fi
# Finally, 'a' is incremented by one to allow multiple extractions per
# species per minute.
#a=$((a + 1))
done < "${TMPFILE}"
# Once each line of the TMPFILE has been processed, the TMPFILE is emptied
@@ -192,11 +136,11 @@ for h in "${SCAN_DIRS[@]}";do
# Rename files that have been processed so that they are not processed on the
# next extraction.
[[ -d "${PROCESSED}" ]] || mkdir "${PROCESSED}"
echo "Moving processed files to ${PROCESSED}"
#echo "Moving processed files to ${PROCESSED}"
mv ${h}/* ${PROCESSED} &> /dev/null || continue
done
echo "Linking Processed files to "${EXTRACTED}/Processed" web directory"
#echo "Linking Processed files to "${EXTRACTED}/Processed" web directory"
# After all audio extractions have taken place, a directory is created to house
# the original WAVE and .txt files used for this extraction processs.
if [[ ! -L ${EXTRACTED}/Processed ]] || [[ ! -e ${EXTRACTED}/Processed ]];then
scripts/history.php
+9 -1
View File
@@ -10,6 +10,14 @@ $theDate = date('Y-m-d');
}
$chart = "Combo-$theDate.png";
$chart2 = "Combo2-$theDate.png";
$db = new SQLite3('/home/pi/BirdNET-Pi/scripts/birds.db', SQLITE3_OPEN_CREATE | SQLITE3_OPEN_READWRITE);
$statement1 = $db->prepare("SELECT COUNT(*) FROM detections
WHERE Date == \"$theDate\"");
$result1 = $statement1->execute();
$totalcount = $result1->fetchArray(SQLITE3_ASSOC);
?>
<head>
@@ -52,7 +60,7 @@ hr {
<table>
<tr>
<th>Total Detections For The Day</th>
<td></td>
<td><?php echo $totalcount['COUNT(*)'];?></td>
</tr>
</table>
</div>
scripts/overview.php
+6 -6
View File
@@ -20,10 +20,10 @@ $statement3 = $db->prepare('SELECT COUNT(*) FROM detections WHERE TIME >= TIME(\
$result3 = $statement3->execute();
$hourcount = $result3->fetchArray(SQLITE3_ASSOC);
$statement4 = $db->prepare('SELECT Com_Name, Sci_Name, Time, Confidence FROM detections LIMIT 1');
$statement4 = $db->prepare('SELECT Com_Name, Sci_Name, Date, Time, Confidence, File_Name FROM detections ORDER BY Time ASC LIMIT 1');
$result4 = $statement4->execute();
$mostrecent = $result4->fetchArray(SQLITE3_ASSOC);
$comlink = preg_replace('/ /', '_', $mostrecent['Com_Name']);
$comname = preg_replace('/ /', '_', $mostrecent['Com_Name']);
$scilink = preg_replace('/ /', '_', $mostrecent['Sci_Name']);
$statement5 = $db->prepare('SELECT COUNT(DISTINCT(Com_Name)) FROM detections WHERE Date == Date(\'now\')');
@@ -70,8 +70,8 @@ a {
<table>
<tr>
<th>Most Recent Detection</th>
<td><a href="/By_Date/<?php echo $myDate."/".$comlink;?>"><?php echo $mostrecent['Com_Name'];?></a></td>
<td><a href="/By_Date/<?php echo$myDate;?>"/><?php echo $mostrecent['Time'];?></a></td>
<td><a href="/By_Date/<?php echo $myDate."/".$comname;?>"><?php echo $mostrecent['Com_Name'];?></a></td>
<td><a href="/By_Date/<?php echo$myDate."/".$comname."/".$mostrecent['File_Name'];?>" target="footer"/><?php echo $mostrecent['Date']." ".$mostrecent['Time'];?></a></td>
<td><?php echo $mostrecent['Confidence'];?></td>
<td><a href="https://wikipedia.org/wiki/<?php echo $scilink;?>" target="top"/>More Info</a></td>
</tr>
@@ -91,7 +91,7 @@ a {
<tr>
<th>Number of Detections</th>
<td><?php echo $totalcount['COUNT(*)'];?></td>
<td><?php echo $todaycount['COUNT(*)'];?></td>
<td><a href="/By_Date/<?php echo date('Y-m-d');?>"/><?php echo $todaycount['COUNT(*)'];?></a></td>
<td><?php echo $hourcount['COUNT(*)'];?></td>
</tr>
</table>
@@ -100,7 +100,7 @@ a {
<table>
<tr>
<th>Species Detected Today</th>
<td><?php echo $speciestally['COUNT(DISTINCT(Com_Name))'];?></td>
<td><a href="/stats.php"/><?php echo $speciestally['COUNT(DISTINCT(Com_Name))'];?></a></td>
</tr>
</table>
</div>
scripts/server.py
+2 -2
View File
@@ -302,7 +302,7 @@ def handle_client(conn, addr):
#print('Time:', date_time_obj.time())
print('Date-time:', date_time_obj)
now = date_time_obj
current_date = now.strftime("%Y/%m/%d")
current_date = now.strftime("%Y-%m-%d")
current_time = now.strftime("%H:%M:%S")
current_iso8601 = now.astimezone(get_localzone()).isoformat()
@@ -342,7 +342,7 @@ def handle_client(conn, addr):
species = entry[0]
Sci_Name,Com_Name = species.split('_')
score = entry[1]
Confidence = "{:.0%}".format(score)
Confidence = str(round(score*100))
Lat = str(args.lat)
Lon = str(args.lon)
Cutoff = str(args.min_conf)
scripts/stats.php
+49 -32
View File
@@ -3,6 +3,20 @@ ini_set('display_errors', 1);
ini_set('display_startup_errors', 1);
error_reporting(E_ALL);
$db = new SQLite3('/home/pi/BirdNET-Pi/scripts/birds.db', SQLITE3_OPEN_CREATE | SQLITE3_OPEN_READWRITE);
$statement = $db->prepare('SELECT Com_Name, COUNT(*), MAX(Confidence), Sci_Name FROM detections GROUP BY Com_Name ORDER BY COUNT(*) DESC');
$result = $statement->execute();
$statement2 = $db->prepare('SELECT Com_Name FROM detections GROUP BY Com_Name ORDER BY Com_Name ASC');
$result2 = $statement2->execute();
if(isset($_POST['species'])){
$selection = $_POST['species'];
$statement3 = $db->prepare("SELECT Com_Name, Sci_Name, COUNT(*), MAX(Confidence) from detections
WHERE Com_Name = '$selection'");
$result3 = $statement3->execute();
}
?>
<!DOCTYPE html>
@@ -116,21 +130,6 @@ form {
<section>
<div class="row">
<div class="column first">
<h3>Number of Detections</h3>
<table>
<tr>
<th>Total</th>
<th>Today</th>
<th>Last Hour</th>
<th>Number of Unique Species</th>
</tr>
<tr>
<td></td>
<td></td>
<td></td>
<td></td>
</tr>
</table>
<h3>Summary</h3>
<table>
<tr>
@@ -138,22 +137,40 @@ form {
<th>Occurrences</th>
<th>Max Confidence Score</th>
</tr>
<?php
while($results=$result->fetchArray(SQLITE3_ASSOC))
{
$comname = preg_replace('/ /', '_', $results['Com_Name']);
$comlink = "/By_Date/".date('Y-m-d')."/".$comname;
$sciname = preg_replace('/ /', '_', $results['Sci_Name']);
?>
<tr>
<td><a href="../By_Common_Name/"></a></td>
<td></td>
<td></td>
<td><?php echo $results['Com_Name'];?></td>
<td><?php echo $results['COUNT(*)'];?></td>
<td><?php echo $results['MAX(Confidence)'];?></td>
</tr>
<?php
}
?>
</table>
</div>
<div class="column">
<form action="stats.php" method="POST">
<h3>Species Stats</h3>
<select name="species" >
<option value=""></option>
<option value=""></option>"
<option value="<?php if(isset($_POST['species'])){echo $selection;}?>"><?php if(isset($_POST['species'])){echo $selection;}else{echo "--Choose Species--";}?></option>
<?php
while($results=$result2->fetchArray(SQLITE3_ASSOC))
{
$comname = preg_replace('/ /', '_', $results['Com_Name']);
$comlink = "/By_Date/".date('Y-m-d')."/".$comname;
$sciname = preg_replace('/ /', '_', $results['Sci_Name']);
?>
<option value="<?php echo $results['Com_Name'];?>"><?php echo $results['Com_Name'];?></option>
<?php
}
?>
</select>
</p>
<button type="submit" class="block"/>Show Species Statistics</button>
</form>
<?php if(isset($_POST['species'])){
@@ -171,22 +188,22 @@ form {
ob_flush();
flush();
while($rows = $specificstats->fetch_assoc()) {
$count = $rows['COUNT(*)'];
$maxconf = $rows['MAX(Confidence)'];
$name = $rows['Com_Name'];
$sciname = $rows['Sci_Name'];
$dbname = preg_replace('/ /', '_', $rows['Com_Name']);
while($results=$result3->fetchArray(SQLITE3_ASSOC)){
$count = $results['COUNT(*)'];
$maxconf = $results['MAX(Confidence)'];
$name = $results['Com_Name'];
$sciname = $results['Sci_Name'];
$dbname = preg_replace('/ /', '_', $results['Com_Name']);
$dbname = preg_replace('/\'/', '', $dbname);
$dbsciname = preg_replace('/ /', '_', $rows['Sci_Name']);
$dbsciname = preg_replace('/ /', '_', $results['Sci_Name']);
$imagelink = shell_exec("/home/pi/BirdNET-Pi/scripts/get_image.sh $dbsciname");
$imagecitation = shell_exec("/home/pi/BirdNET-Pi/scripts/get_citation.sh $dbsciname");
$str= "<tr>
<td><a href=\"../By_Common_Name/$dbname\"/>$name</a></td>
<td><a href=\"../By_Scientific_Name/$dbsciname\"/>$sciname</a></td>
<td>$name</td>
<td><a href=\"https://wikipedia.org/wiki/$dbsciname\" target=\"top\"/>$sciname</a></td>
<td>$count</td>
<td>$maxconf</td>
<td><a href=\"https://wikipedia.org/wiki/$dbsciname\" target=\"top\"/>Wikipedia</a>, <a href=\"https://allaboutbirds.org/guide/$dbname\" target=\"top\"/>All About Birds</a>
<td><a href=\"https://allaboutbirds.org/guide/$dbname\" target=\"top\"/>All About Birds</a>
</tr>
</table>";
echo str_pad($str, 4096);
scripts/stop_core_services.sh
+1 -1
View File
@@ -6,7 +6,7 @@ services=(birdnet_recording.service
birdnet_analysis.service
birdnet_server.service
chart_viewer.service
extraction.timer
extraction.service
spectrogram_viewer.service)
for i in "${services[@]}";do
scripts/update_birdnet2.sh
-1
View File
@@ -48,7 +48,6 @@ spectrogram_viewer.service
pushed_notifications.service
livestream.service
icecast2.service
extraction.timer
extraction.service
chart_viewer.service
birdnet_recording.service
scripts/viewdb.php
+42 -9
View File
@@ -4,6 +4,31 @@ ini_set('display_startup_errors', 1);
error_reporting(E_ALL);
header("refresh: 30;");
$db = new SQLite3('/home/pi/BirdNET-Pi/scripts/birds.db', SQLITE3_OPEN_CREATE | SQLITE3_OPEN_READWRITE);
$statement0 = $db->prepare('SELECT Time, Com_Name, Sci_Name, Confidence, File_Name FROM detections WHERE Date == Date(\'now\') ORDER BY Time DESC');
$result0 = $statement0->execute();
$statement1 = $db->prepare('SELECT COUNT(*) FROM detections');
$result1 = $statement1->execute();
$totalcount = $result1->fetchArray(SQLITE3_ASSOC);
$statement2 = $db->prepare('SELECT COUNT(*) FROM detections WHERE Date == DATE(\'now\')');
$result2 = $statement2->execute();
$todaycount = $result2->fetchArray(SQLITE3_ASSOC);
$statement3 = $db->prepare('SELECT COUNT(*) FROM detections WHERE TIME >= TIME(\'now\', \'localtime\', \'-1 hour\')');
$result3 = $statement3->execute();
$hourcount = $result3->fetchArray(SQLITE3_ASSOC);
$statement4 = $db->prepare('SELECT Com_Name, Sci_Name, Time, Confidence FROM detections LIMIT 1');
$result4 = $statement4->execute();
$mostrecent = $result4->fetchArray(SQLITE3_ASSOC);
$statement5 = $db->prepare('SELECT COUNT(DISTINCT(Com_Name)) FROM detections');
$result5 = $statement5->execute();
$speciestally = $result5->fetchArray(SQLITE3_ASSOC);
?>
<!DOCTYPE html>
@@ -36,10 +61,10 @@ a {
<th>Number of Unique Species</th>
</tr>
<tr>
<td></td>
<td></td>
<td></td>
<td></td>
<td><?php echo $totalcount['COUNT(*)'];?></td>
<td><a href="/By_Date/<?php echo date('Y-m-d');?>"/><?php echo $todaycount['COUNT(*)'];?></a></td>
<td><?php echo $hourcount['COUNT(*)'];?></td>
<td><a href="/stats.php"/><?php echo $speciestally['COUNT(DISTINCT(Com_Name))'];?></a></td>
</tr>
</table>
</div>
@@ -53,11 +78,19 @@ a {
<th>Confidence</th>
<th>Links</th>
</tr>
<?php
while($todaytable=$result0->fetchArray(SQLITE3_ASSOC))
{
$comname = preg_replace('/ /', '_', $todaytable['Com_Name']);
$comlink = "/By_Date/".date('Y-m-d')."/".$comname."/".$todaytable['File_Name'];
$sciname = preg_replace('/ /', '_', $todaytable['Sci_Name']);
?>
<tr>
<td></td>
<td><a href="/By_Scientific_Name/"/></a></td>
<td><a href="/By_Common_Name/"/></a></td>
<td></td>
<td><a class="a2" href="https://allaboutbirds.org/guide/" target="top">All About Birds</a>, <a class="a2" href="https://wikipedia.org/wiki/" target="top">Wikipedia</a></td>
<td><a href="<?php echo $comlink;?>" target="footer"/><?php echo $todaytable['Time'];?></a></td>
<td><?php echo $todaytable['Sci_Name'];?></td>
<td><?php echo $todaytable['Com_Name'];?></td>
<td><?php echo $todaytable['Confidence'];?></td>
<td><a class="a2" href="https://allaboutbirds.org/guide/<?php echo $comname;?>" target="top">All About Birds</a>, <a class="a2" href="https://wikipedia.org/wiki/<?php echo $sciname;?>" target="top">Wikipedia</a></td>
<?php }?>
</tr>
</table>