diff --git a/.gitignore b/.gitignore index b6e4761..50425de 100644 --- a/.gitignore +++ b/.gitignore @@ -1,129 +1,11 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -pip-wheel-metadata/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -.python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ +*.pyc +*.flac +model/BirdNET_Soundscape_Model.pkl +.vscode +datasets/ +birdnet.conf +miniforge +scripts/install_miniforge.sh +IdentifiedSoFar.txt +IdentifiedSoFar.txt.bak +Birders_Guide_Installer_Configuration.txt diff --git a/Birders_Guide_Installer.sh b/Birders_Guide_Installer.sh new file mode 100755 index 0000000..e0d04b2 --- /dev/null +++ b/Birders_Guide_Installer.sh @@ -0,0 +1,423 @@ +#!/usr/bin/env bash +set -e +my_dir=${HOME}/BirdNET-Lite +trap '${my_dir}/scripts/dump_logs.sh && exit' EXIT SIGHUP SIGINT + + +if [ "$(uname -m)" != "aarch64" ];then + echo "BirdNET-Lite requires a 64-bit OS. +It looks like your operating system is using $(uname -m), +but would need to be aarch64. +Please take a look at https://birdnetwiki.pmcgui.xyz for more +information" + exit 1 +fi + +install_zram_swap() { + echo + echo "Configuring zram.service" + sudo touch /etc/modules-load.d/zram.conf + echo 'zram' | sudo tee /etc/modules-load.d/zram.conf + sudo touch /etc/modprobe.d/zram.conf + echo 'options zram num_devices=1' | sudo tee /etc/modprobe.d/zram.conf + sudo touch /etc/udev/rules.d/99-zram.rules + echo 'KERNEL=="zram0", ATTR{disksize}="4G",TAG+="systemd"' \ + | sudo tee /etc/udev/rules.d/99-zram.rules + sudo touch /etc/systemd/system/zram.service + echo "Installing zram.service" + cat << EOF | sudo tee /etc/systemd/system/zram.service &> /dev/null +[Unit] +Description=Swap with zram +After=multi-user.target + +[Service] +Type=oneshot +RemainAfterExit=true +ExecStartPre=/sbin/mkswap /dev/zram0 +ExecStart=/sbin/swapon /dev/zram0 +ExecStop=/sbin/swapoff /dev/zram0 + +[Install] +WantedBy=multi-user.target +EOF + sudo systemctl enable zram + echo + echo "Installing stage 2 installation script now." + cd ~ + curl -s -O "https://raw.githubusercontent.com/mcguirepr89/BirdNET-Lite/rpi4/Birders_Guide_Installer.sh" + chmod +x Birders_Guide_Installer.sh + cat << EOF | sudo tee /etc/systemd/user/birdnet-system-installer.service &> /dev/null +[Unit] +Description=A BirdNET-Lite Installation Script Service +After=graphical.target network-online.target + +[Service] +Type=simple +Restart=on-failure +RestartSec=3s +ExecStart=lxterminal -e /home/pi/Birders_Guide_Installer.sh + +[Install] +WantedBy=default.target +EOF + systemctl --user enable birdnet-system-installer.service + echo + echo "Stage 1 complete" + touch ${HOME}/stage_1_complete + echo + echo "Rebooting the system in 5 seconds" + sleep 5 + sudo reboot +} + +stage_1() { + echo + echo "Beginning Stage 1" + echo + echo "Ensuring the system is up-to-date." + sudo apt -qq update + sudo apt -qqy full-upgrade + sudo apt -y autoremove --purge + echo "System Updated!" + if ! which git &> /dev/null; then + echo "Installing git" + sudo apt install -qqy git + fi + ZRAM="$(swapon --show=SIZE,NAME | awk -FG '!/SIZE/ && /zram/ {print $1}')" + [ ! -z ${ZRAM} ] || ZRAM=0 + if [ ${ZRAM} -lt 4 ];then + install_zram_swap + else + echo "Stage 1 complete" + stage_2 + exit + fi +} + +stage_2() { + echo + echo "Beginning stage 2" + echo + echo "Checking for an internet connection to continue . . ." + until ping -c 1 google.com &> /dev/null; do + sleep 1 + done + echo "Connected!" + echo + if [ ! -d ${my_dir} ];then + cd ~ || exit 1 + echo "Cloning the BirdNET-Lite repository in your home directory" + git clone https://github.com/mcguirepr89/BirdNET-Lite.git !/BirdNET-Lite + fi + + if [ -f ${my_dir}/Birders_Guide_Installer_Configuration.txt ];then + echo + echo + echo "Follow the instructions to fill out the LATITUDE and LONGITUDE variables +and set the passwords for the live audio stream. Save the file after editing +and then close the Mouse Pad editing window to continue." + mousepad ${my_dir}/Birders_Guide_Installer_Configuration.txt &> /dev/null + while pgrep mouse &> /dev/null;do + sleep 1 + done + source ${my_dir}/Birders_Guide_Installer_Configuration.txt || exit 1 + else + echo "Something went wrong. I can't find the configuration file." + exit 1 + fi + + if [ -z ${LATITUDE} ] || [ -z ${LONGITUDE} ] || [ -z ${CADDY_PWD} ] || [ -z ${ICE_PWD} ];then + echo + echo + echo "It looks like you haven't filled out the Birders_Guide_Installer_Configuration.txt file +completely. + +Open that file to edit it. (Go to the folder icon in the top left and look for the \"BirdNET-Lite\" +folder and double-click the file called \"Birders_Guide_Installer_Configuration.txt\" +Enter the latitude and longitude of where the BirdNET-Lite will be. +You can find this information at https://maps.google.com + +Find your location on the map and right click to find your coordinates. +After you have filled out the configuration file, you can re-run this script. Just do the exact +same things you did to start this (copying and pasting from the Wiki) to try again. +Press Enter to close this window. +Good luck!" + read + exit 1 + fi + echo "Installing the BirdNET-Lite configuration file." + [ -f ${my_dir}/soundcard_params.txt ] || touch ${my_dir}/soundcard_params.txt + SOUND_PARAMS="${HOME}/BirdNET-Lite/soundcard_params.txt" + SOUND_CARD="$(sudo -u pi aplay -L \ + | grep -e '^hw' \ + | cut -d, -f1 \ + | grep -ve 'vc4' -e 'Head' -e 'PCH' \ + | uniq)" + script -c "arecord -D ${SOUND_CARD} --dump-hw-params" -a ${SOUND_PARAMS} &> /dev/null + install_birdnet_config || exit 1 + echo "Installing the BirdNET-Lite" + if ${my_dir}/scripts/install_birdnet.sh << EOF ; then + +n +EOF +echo "The next time you power on the raspberry pi, all of the services will start up automatically. + +The installation has finished. Press Enter to close this window." + read + else + echo "Something went wrong during installation. Open a github issue or email mcguirepr89@gmail.com" + fi +} + +install_birdnet_config() { + cat << EOF > ${my_dir}/birdnet.conf +################################################################################ +# Configuration settings for BirdNET as a service # +################################################################################ + +#___________The four variables below are the only that are required.___________# + +## BIRDNET_USER should be the non-root user systemd should use to execute each +## service. + +BIRDNET_USER=pi + +## RECS_DIR is the location birdnet_analysis.service will look for the data-set +## it needs to analyze. Be sure this directory is readable and writable for +## the BIRDNET_USER. If you are going to be accessing a remote data-set, you +## still need to set this, as this will be where the remote directory gets +## mounted locally. See REMOTE_RECS_DIR below for mounting remote data-sets. + +RECS_DIR=${HOME}/BirdSongs + +## LATITUDE and LONGITUDE are self-explanatroy. Find them easily at +## maps.google.com. Only go to the thousanths place for these variables +## Example: these coordinates would indicate the Eiffel Tower in Paris, France. +## LATITUDE=48.858 +## LONGITUDE=2.294 + +LATITUDE="${LATITUDE}" +LONGITUDE="${LONGITUDE}" + +################################################################################ +#------------------------------ Extraction Service ---------------------------# + +# Keep this EMPTY if you do not want this device to perform the extractions # + +## DO_EXTRACTIONS is simply a setting for enabling the extraction.service. +## Set this to Y or y to enable extractions. + +DO_EXTRACTIONS=y + +################################################################################ +#----------------------------- Recording Service ----------------------------# + +# Keep this EMPTY if you do not want this device to perform the recording. # + +## DO_RECORDING is simply a setting for enabling the 24/7 birdnet_recording.service. +## Set this to Y or y to enable recording. + +DO_RECORDING=y + +################################################################################ +#----------------- Mounting a remote directory with systemd -----------------# +#_______________The four variables below can be set to enable a_______________# +#___________________systemd.mount for analysis, extraction,____________________# +#______________________________or file-serving_________________________________# + +# Leave these settings EMPTY if your data-set is local. # + +## REMOTE is simply a setting for enabling the systemd.mount to use a remote +## filesystem for the data storage and service. +## Set this to Y or y to enable the systemd.mount. + +REMOTE= + +## REMOTE_HOST is the IP address, hostname, or domain name SSH should use to +## connect for FUSE to mount its remote directories locally. + +REMOTE_HOST= + +## REMOTE_USER is the user SSH will use to connect to the REMOTE_HOST. + +REMOTE_USER= + +## REMOTE_RECS_DIR is the directory on the REMOTE_HOST which contains the +## data-set SSHFS should mount to this system for local access. This is NOT the +## directory where you will access the data on this machine. See RECS_DIR for +## that. + +REMOTE_RECS_DIR= + +################################################################################ +#----------------------- Web-hosting/Caddy File-server -----------------------# +#__________The two variables below can be set to enable web access_____________# +#____________to your data,(e.g., extractions, raw data, live___________________# +#______________audio stream, BirdNET.selection.txt files)______________________# + +# Leave these EMPTY if you do not want to enable web access # + +## EXTRACTIONS_URL is the URL where the extractions, data-set, and live-stream +## will be web-hosted. If you do not own a domain, or would just prefer to keep +## BirdNET-Lite on your local network, you can set this to http://localhost. +## Setting this (even to http://localhost) will also allow you to enable the +## GoTTY web logging features below. + +EXTRACTIONS_URL=http://raspberrypi.local + +## CADDY_PWD is the plaintext password (that will be hashed) and used to access +## the "Processed" directory and live audio stream. This MUST be set if you +## choose to enable this feature. + +CADDY_PWD=${CADDY_PWD} + +################################################################################ +#------------------------- Live Audio Stream --------------------------------# +#_____________The variable below configures/enables the live___________________# +#_____________________________audio stream.____________________________________# + +# Keep this EMPTY if you do not wish to enable the live stream # +# or if this device is not doing the recording # + +## ICE_PWD is the password that icecast2 will use to authenticate ffmpeg as a +## trusted source for the stream. You will never need to enter this manually +## anywhere other than here. + +ICE_PWD=${ICE_PWD} + +################################################################################ +#------------------- Mobile Notifications via Pushed.co ---------------------# +#____________The two variables below enable mobile notifications_______________# +#_____________See https://pushed.co/quick-start-guide to get___________________# +#_________________________these values for your app.___________________________# + +# Keep these EMPTY if haven't setup a Pushed.co App yet. # + +## Pushed.co App Key and App Secret + +PUSHED_APP_KEY=${PUSHED_APP_KEY} +PUSHED_APP_SECRET=${PUSHED_APP_SECRET} + +################################################################################ +#------------------------------- NoMachine ----------------------------------# +#_____________The variable below can be set include NoMachine__________________# +#_________________remote desktop software to be installed._____________________# + +# Keep this EMPTY if you do not want to install NoMachine. # + +## INSTALL_NOMACHINE is simply a setting that can be enabled to install +## NoMachine alongside the BirdNET-Lite for remote desktop access. This in- +## staller assumes personal use. Please reference the LICENSE file included +## in this repository for more information. +## Set this to Y or y to install NoMachine alongside the BirdNET-Lite + +INSTALL_NOMACHINE=y + +################################################################################ +#-------------------------------- Defaults ----------------------------------# +#______The six variables below are default settings that you (probably)________# +#__________________don't need to change at all, but can._______________________# + +## REC_CARD is the sound card you would want the birdnet_recording.service to +## use. This setting is irrelevant if you are not planning on doing data +## collection via recording on this machine. The command substitution below +## looks for a USB microphone's dsnoop alsa device. The dsnoop device lets +## birdnet_recording.service and livestream.service share the raw audio stream +## from the microphone. If you would like to use a different microphone than +## what this produces, or if your microphone does not support creating a +## dsnoop device, you can set this explicitly from a list of the available +## devices from the output of running 'aplay -L' + +REC_CARD="\$(sudo -u pi aplay -L \ + | grep dsnoop \ + | cut -d, -f1 \ + | grep -ve 'vc4' -e 'Head' -e 'PCH' \ + | uniq)" + +## PROCESSED is the directory where the formerly 'Analyzed' files are moved +## after extractions have been made from them. This includes both WAVE and +## BirdNET.selection.txt files. + +PROCESSED=${RECS_DIR}/Processed + +## EXTRACTED is the directory where the extracted audio selections are moved. + +EXTRACTED=${RECS_DIR}/Extracted + +## IDFILE is the file that keeps a complete list of every spececies that +## BirdNET has identified from your data-set. It is persistent across +## data-sets, so would need to be whiped clean through deleting or renaming +## it. A backup is automatically made from this variable each time it is +## updated (structure: ${IDFILE}.bak), and would also need to be removed +## or renamed to start a new file between data-sets. Alternately, you can +## change this variable between data-sets to preserve records of disparate +## data-sets according to name. + +IDFILE=${HOME}/BirdNET-Lite/IdentifiedSoFar.txt + +## OVERLAP is the value in seconds which BirdNET should use when analyzing +## the data. The values must be between 0.0-2.9. + +OVERLAP="0.0" + +## CONFIDENCE is the minimum confidence level from 0.0-1.0 BirdNET's analysis +## should reach before creating an entry in the BirdNET.selection.txt file. +## Don't set this to 1.0 or you won't have any results. + +CONFIDENCE="0.7" + +################################################################################ +#------------------------------ Auto-Generated ------------------------------# +#_____________________The variables below are auto-generated___________________# +#______________________________during installation_____________________________# + +## CHANNELS holds the variabel that corresponds to the number of channels the +## sound card supports. + +CHANNELS=$(awk '/CHANN/ { print $2 }' ${SOUND_PARAMS} | sed 's/\r$//') + +# Don't the three below +## ANALYZED is where the extraction.service looks for audio and +## BirdNET.selection.txt files after they have been processed by the +## birdnet_analysis.service. This is NOT where the analyzed files are moved -- +## analyzed files are always created within the same directory +## birdnet_analysis.service finds them. + +ANALYZED=${RECS_DIR}/*/*Analyzed + +## SYSTEMD_MOUNT is created from the RECS_DIR variable to comply with systemd +## mount naming requirements. + +SYSTEMD_MOUNT=$(echo ${RECS_DIR#/} | tr / -).mount + +## VENV is the virtual environment where the the BirdNET python build is found, +## i.e, VENV is the virtual environment miniforge built for BirdNET. + +VENV=${my_dir}/miniforge/envs/birdnet +EOF + [ -d /etc/birdnet ] || sudo mkdir /etc/birdnet + sudo ln -sf ${my_dir}/birdnet.conf /etc/birdnet/birdnet.conf +} +echo " +Welcome to the Birders Guide Installer script! + +The installer runs in two stages: +Stage 1 configures and enables the zRAM kernel module and allocates 4G + to its swapping size if needed. This will trigger a reboot. +Stage 1 also ensures the system is up to date. +Stage 2 guides you through configuring the essentials and installs the full BirdNET-Lite." + + +if [ ! -f ${HOME}/stage_1_complete ] ;then + stage_1 +else + stage_2 + if [ -f ${HOME}/Birders_Guide_Installer.sh ];then + rm ${HOME}/Birders_Guide_Installer.sh + fi + rm ${HOME}/stage_1_complete + ${HOME}/scripts/dump_logs.sh + systemctl --user disable --now birdnet-system-installer.service + sudo rm -f /etc/systemd/user/birdnet-system-installer.service +fi + diff --git a/Birders_Guide_Installer_Configuration.txt b/Birders_Guide_Installer_Configuration.txt new file mode 100644 index 0000000..ad5e960 --- /dev/null +++ b/Birders_Guide_Installer_Configuration.txt @@ -0,0 +1,37 @@ +# Birders Guide Installer Configuration File +#####NOTE: the '#' sign needs to precede the lines where they are currently +##### DO NOT DELETE ANY OF THEM!!! + +########################################################################### +# Fill this out in order to install BirdNET-Lite +# +# 1. To find this information, you can go to https://maps.google.com +# 1a. Find the location on the map where you will be putting the BirdNET-Lite +# 1b. Right-click the location to view the latitude and longitude and click them to +# copy them to your clip-board. +# 2. Enter the latitude and longitude here. Be certain not to mix them up and +# only go to the thousandth's decimal place. The first number is the latitude, +# and the second is longitude. See the example below. +# 3. Set a password that you will use to access your live audio stream. This is the +# "CADDY_PWD" field. You can set this to anything, but keep it alpha-numeric. +# You will probably also want to write this one down if you ever plan to listen +# to the live stream. +# 4. Set a password that that IceCast2 will use to authenticate the stream source. +# You can set this to anything also, but keep it alpha-numeric. You will never use +# this again, so just set it to your favorite word. +# 5. FILE SAVE!!! Don't forget to save this file after editing. +# 6. Close this window to continue the installation. +# Example: these coordinates would indicate the Eiffel Tower in Paris, France. +#LATITUDE=48.858 +#LONGITUDE=2.294 +LATITUDE= +LONGITUDE= +CADDY_PWD= +ICE_PWD= + +# This section can be left alone. If you setup a Pushed.co +# mobile application, you can enter the App secret and secret +# key in order to enable phone notifications for new species +# detections. +PUSHED_APP_SECRET= +PUSHED_APP_KEY= diff --git a/README.md b/README.md index 9e2a5fc..d1ea8b7 100644 --- a/README.md +++ b/README.md @@ -1,110 +1,112 @@ -# BirdNET-Lite -TFLite version of BirdNET. Bird sound recognition for more than 6,000 species worldwide. +# BirdNET-Lite for arm64/aarch64 (Raspberry Pi 4) +### Built on https://github.com/kahst/BirdNET -- checkout the Wiki at [BirdNETWiki.pmcgui.xyz](https://birdnetwiki.pmcgui.xyz) -Center for Conservation Bioacoustics, Cornell Lab of Ornithology, Cornell University +This project offers an installation script for BirdNET as a systemd service on arm64 (aarch64) Debian-based operating systems, namely RaspiOS. The installation script offers to walk the user through setting up the '*birdnet.conf*' main configuration file interactively, or can read from an existing '*birdnet.conf*'. A variety of configurations can be attained through this installation script. -Go to https://birdnet.cornell.edu to learn more about the project. +BirdNET-Lite can be configured with the following optional services: +- A 24/7 recording script that can be easily configured to use any available sound card +- An extraction service that extracts the audio selections identified by BirdNET by date and species +- A Caddy instance that serves the extracted files and live audio stream (icecast2) (requires dsnoop capable mic) +- A species list updating and notification script supporting mobile notifications via Pushed.co (sorry, Android users, Pushed.co doesn't seem to work for you) +- NoMachine remote desktop software (for personal use only) -Want to use BirdNET to analyze a large dataset? Don't hesitate to contact us: ccb-birdnet@cornell.edu +An installation one-liner is available [HERE](https://birdnetwiki.pmcgui.xyz/wiki/Birder%27s_Guide_to_BirdNET-Lite#Install_BirdNET-Lite) for RaspiOS-ARM64 meeting the prequisites below. It installs all services listed above. +- Prerequisites: + - An updated RaspiOS for AArch64 that has locale, WiFi, time-zone, and pi user password set. A guide is available [here](https://birdnetwiki.pmcgui.xyz/wiki/Birder%27s_Guide_to_BirdNET-Lite#Install_the_base_operating_system_.28OS.29). 64GB SD card for best performance. + - A USB microphone (dsnoop capable to enable live audio stream). + - Running the installer from within the Raspberry Pi's desktop environment (i.e., not over SSH -- for SSH installations, see installation options 2 & 3) -# Setup (Ubuntu 18.04) -TFLite for x86 platforms comes with the standard Tensorflow package. If you are on a different platform, you need to install a dedicated version of TFLite (e.g., a pre-compiled version for Raspberry Pi). See Raspberry Pi 4B installation instructions below. +## What the installation does +1. Looks for a *'birdnet.conf'* file in the *BirdNET-Lite* main directory +1. If a *'birdnet.conf'* file exists and is filled out properly, the installation is nearly + non-interactive and builds the system based off of the services configured in the *'birdnet.conf'* file +1. If the installer cannot find a *'birdnet.conf'* file, the installation is interactive and will + walk the user through creating the '*birdnet.conf'* file interactively. +1. Installs the following system dependencies: + - ffmpeg + - libblas-dev + - liblapack-dev + - caddy (for web access to extractions) + - icecast2 (live audio stream) + - alsa-utils (for recording) + - sshfs (to mount remote sound file directories) +1. Installs BirdNET-Lite scripts in */usr/local/bin* +1. Installs all selected services based on '*birdnet.conf*' +1. Installs *miniforge* for the aarch64 architecture using the current release from https://github.com/conda-forge/miniforge +1. Builds BirdNET in miniforge's *'birdnet'* virtual environment +1. Enables (but does not start) the services -## x86 -We need to setup TF2.3+ for BirdNET. First, we install Python 3 and pip: +## What you should know before any installation +1. The licensing information for the software that is used (see [LICENSE](https://raw.githubusercontent.com/mcguirepr89/BirdNET-Lite/BirdNET-Lite-for-raspi4/LICENSE)). +1. The **latitude** and **longitude** where the bird recordings take place. Google maps is an easy way to find these (right-clicking the location). +1. In order for the live audio stream to work at the same time as the birdnet_recording.service, the microphone needs to be dsnoop capable. If you are wondering whether your mic supports creating the dsnoop device, you can use `aplay -L | awk -F, '/dsn/ {print $1}' | grep -ve 'vc4' -e 'Head' -e 'PCH' | uniq` to check. (No output means your microphone does not support creating a dsnoop device and therefore cannot also provide an audio stream while recording. The birdnet_recording.service, however, should not be affected by this and the installation one-liner can still be used. The live stream link simply will not work.) -``` -sudo apt-get update -sudo apt-get install python3-dev python3-pip -sudo pip3 install --upgrade pip -``` +## What you should know for a manual installation +1. The **local directory** where the recordings should be found on your local computer. BirdNET-Lite supports setting up a systemd.mount for automounting remote directories. So for instance, if the actual recordings live on RemoteHost's `/home/user/recordings` directory, but you would like them to be found on your device at `/home/me/BirdNET-recordings`, then `/home/me/BirdNET-recordings` will be your answer to that question. +1. If mounting the recordings directory from a remote host, you need to know the **remote hostname, username, and password** to connect to it via SSH, as well as the **absolute path of the recordings on the remote host**. +1. If you are using a special microphone or have multiple sound cards and would like to specify which to use for recording, you can edit the `~/BirdNET-Lite/birdnet.conf` file before the installation and set the **REC_CARD** variable to the sound card of your choice. Copy your desired sound card line from the output of `aplay -L | awk -F, '/^dsn:/ { print $1 }'`(prefered), or `aplay -L | awk -F, '/^hw:/ { print $1 }'`(if prefered is not available). +1. If you would like to take advantage of Caddy's automatic handling of SSL certificates to be able to host a public website where your friends can hear your bird sounds, forward ports 80 and 443 to the host you want to serve the files. You may also want to purchase a domain name. + - *Note: If you're just keeping this on your local network, **be sure to set your extraction URL to something 'http://'** (on RaspiOS, I recommend 'http://raspberrypi.local') to disable Caddy's automatic HTTPS. Alternatively, you may edit the `/etc/caddy/Caddyfile` after installation and add the `tls internal` directive to the site block to have Caddy issue a self-signed certificate for an HTTPS connection.* +1. If you would like to take advantage of BirdNET-Lite's ability to send New Species mobile notifications, you can easily setup a Pushed.co notification app (see the #TODOs at the bottom for more info). After setting up your application, make note of your **App Key** and **App Secret** -- you will need these to enable mobile notifications for new species. + - *Note for Android users: it seems that the Pushed.co Mobile App does not work for Android devices, which is a huge bummer. If anyone knows of an Android alternative, or if anyone might be able to come up with a home-spun notification system, please let me know.* -Then, we can install Tensorflow with: +## How to install +#### Option 1 (Recommended) -- Install All Services +1. In the terminal run: `curl -s https://raw.githubusercontent.com/mcguirepr89/BirdNET-Lite/BirdNET-Lite-for-raspi4/Birders_Guide_Installer.sh | bash` -``` -sudo pip3 install tensorflow -``` +##### Options 2 & 3 require you setup 4GB of swapping. That step is included in the directions below. +#### Option 2 -- Pre-fill birdnet.conf +1. In the terminal run `git clone https://github.com/mcguirepr89/BirdNET-Lite.git ~/BirdNET-Lite` +1. You can copy the included *'birdnet.conf-defaults'* template to create and configure the BirdNET-Lite + to your needs before running the installer. Issue `cp ~/BirdNET-Lite/birdnet.conf-defaults ~/BirdNET-Lite/birdnet.conf`. + Edit the new *'birdnet.conf'* file to suit your needs and save it. + If you choose this method, the installation will be (nearly) non-interactive. +1. Setup zRAM swapping. Run `~/BirdNET-Lite/scripts/install_zram_service.sh && sudo reboot` +1. After the reboot, run `~/BirdNET-Lite/scripts/install_birdnet.sh` +#### Option 3 -- Interactive Installation +1. In the terminal run `git clone https://github.com/mcguirepr89/BirdNET-Lite.git ~/BirdNET-Lite` +1. Setup zRAM swapping. Run `~/BirdNET-Lite/scripts/install_zram_service.sh && sudo reboot` +1. After the reboot, run `~/BirdNET-Lite/scripts/install_birdnet.sh` +1. Follow the installation prompts to configure the BirdNET-Lite to your needs. +- Note: The installation should be run as a regular user. If run on an OS other than RaspiOS, be sure the regular user is in the sudoers file or the sudo group. -TFLite on x86 platform currently only supports CPUs. +## Access your BirdNET-Lite +If you configured BirdNET-Lite with the Caddy webserver, you can access the extractions locally at -Note: Make sure to set `CUDA_VISIBLE_DEVICES=""` in your environment variables. Or set `os.environ['CUDA_VISIBLE_DEVICES'] = ''` at the top of your Python script. +- http://birdnetsystem.local -In this example, we use Librosa to open audio files. Install Librosa with: +You can also view the log output for the birdnet_analysis.service and extraction.service at -``` -sudo pip3 install librosa -sudo apt-get install ffmpeg -``` +- http://birdlog.local +- http://extractionlog.local -You can use any other audio lib if you like, or pass raw audio signals to the model. +and the BirdNET-Lite Statistics Report at +- http://birdstats.local -If you don't use Librosa, make sure to install NumPy: +If you opt to also install NoMachine alongside the BirdNET-Lite, you can also access BirdNET-Lite +remotely following the address information that can be found on the NoMachine's server information page. -``` -sudo pip3 install numpy -``` +## Examples +These are examples of my personal instance of the BirdNET-Lite on a Raspberry Pi 4B. + - https://birdsounds.pmcgui.xyz -- My BirdNET-Lite Extractions page + - https://birdlog.pmcgui.xyz -- My 'birdlog' birdnet_analysis.service log + - https://extraction.pmcgui.xyz -- My 'extractionlog' extraction.service log + - https://birdstats.pmcgui.xyz -- My 'birdstats' BirdNET-Lite Report -Note: BirdNET expects 3-second chunks of raw audio data, sampled at 48 kHz. +## How to reconfigure the system +At any time, you can completely reconfigure the system to select or remove features. To reconfigure the system, simply run the included "reconfigure_birdnet.sh" script (as the regular user) and follow the prompts to create a new birdnet.conf file and install new services: `~/BirdNET-Lite/scripts/reconfigure_birdnet.sh` -## Raspberry Pi 4B running AArch64 OS -These steps install BirdNET-Lite on a Raspberry Pi 4B running an AArch64 OS using pre-built TFLite binaries. -1. Install dependencies
`sudo apt install swig libjpeg-dev zlib1g-dev python3-dev unzip wget python3-pip curl git cmake make` -1. Update pip, whell, and setuptools:
`sudo pip3 install --upgrade pip wheel setuptools` -1. Fetch pre-built binaries: - 1. `curl -sc /tmp/cookie "https://drive.google.com/uc?export=download&id=1dlEbugFDJXs-YDBCUC6WjADVtIttWxZA" > /dev/null` - 1. `CODE="$(awk '/_warning_/ {print $NF}' /tmp/cookie)"` - 1. `curl -Lb /tmp/cookie "https://drive.google.com/uc?export=download&confirm=${CODE}&id=1dlEbugFDJXs-YDBCUC6WjADVtIttWxZA" -o tflite_runtime-2.6.0-cp37-none-linux_aarch64.whl` -1. `sudo pip3 install --upgrade tflite_runtime-2.6.0-cp37-none-linux_aarch64.whl` -1. `sudo pip3 install librosa` -1. `sudo apt-get install ffmpeg` -1. `git clone https://github.com/kahst/BirdNET-Lite.git` -1. `cd BirdNET-Lite/`
and test: -1. `python3 analyze.py --i 'example/XC558716 - Soundscape.mp3' --lat 35.4244 --lon -120.7463 --week 18` +## How to uninstall BirdNET-Lite +To remove BirdNET-Lite, run the included '*uninstall.sh*' script as the regular user. +1. Issue `/usr/local/bin/uninstall.sh && cd ~ && rm -drf BirdNET-Lite` -# Usage +## Troubleshooting +**General** -- At anytime, you can run the included `~/BirdNET-Lite/dump_logs.sh` script to create a compressed tar ball of system logs that may provide a helpful overview of the system services. In addition, you can upload it in a new issue along with a description of what you are experiencing. dump_logs.sh scrubs password information, but does retain LATITUDE and LONGITUDE information. If at all concerned with privacy, you're welcome to send them to me via email at mailto:mcguirepr89@gmail.com. -You can run BirdNET via the command line. You can add a few parameters that affect the output. +**Audio** -- If you have problems with the _bridnet_recording.service_ or _livestream.service_, try setting the REC_CARD setting in the _birdnet.conf_ file to `REC_CARD=default` and the CHANNELS variable to `CHANNELS=2`. This works for two very different microphones I have, so it may work for you. If it does, please let me know, as I may change the code as a result. Also, during installation, a file is created called `~/BirdNET-Lite/soundcard_params.txt` that may provide helpful information for customized settings. -The input parameters include: - -``` ---i, Path to input file. ---o, Path to output file. Defaults to result.csv. ---lat, Recording location latitude. Set -1 to ignore. ---lon, Recording location longitude. Set -1 to ignore. ---week, Week of the year when the recording was made. Values in [1, 48] (4 weeks per month). Set -1 to ignore. ---overlap, Overlap in seconds between extracted spectrograms. Values in [0.0, 2.9]. Defaults tp 0.0. ---sensitivity, Detection sensitivity; Higher values result in higher sensitivity. Values in [0.5, 1.5]. Defaults to 1.0. ---min_conf, Minimum confidence threshold. Values in [0.01, 0.99]. Defaults to 0.1. ---custom_list, Path to text file containing a list of species. Not used if not provided. -``` - -Note: A custom species list needs to contain one species label per line. Take a look at the `model/label.txt` for the correct species label. Only labels from this text file are valid. You can find an example of a valid custom list in the 'example' folder. - -Here are two example commands to run this BirdNET version: - -``` - -python3 analyze.py --i 'example/XC558716 - Soundscape.mp3' --lat 35.4244 --lon -120.7463 --week 18 - -python3 analyze.py --i 'example/XC563936 - Soundscape.mp3' --lat 47.6766 --lon -122.294 --week 11 --overlap 1.5 --min_conf 0.25 --sensitivity 1.25 --custom_list 'example/custom_species_list.txt' - -``` - -Note: Please make sure to provide lat, lon, and week. BirdNET will work without these values, but the results might be less reliable. - -The results of the anlysis will be stored in a result file in CSV format. All confidence values are raw prediction scores and should be post-processed to eliminate occasional false-positive results. - -# Contact us - -Please don't hesitate to contact us if you have any issues with the code or if you have any other remarks or questions. - -Our e-mail address: ccb-birdnet@cornell.edu - -We are always open for a collaboration with you. - -# Funding - -This project is supported by Jake Holshuh (Cornell class of ’69). The Arthur Vining Davis Foundations also kindly support our efforts. +**Installation** -- The installer _should_ always create a compressed set of system logs whether it succeeds or fails. Its location is `~/BirdNET-Lite/logs.tar.gz`. Take a look through there or feel free to create a new issue and upload it along with a description of what you are experiencing. +### TODO & Notes: +1. I ought to add the steps to setup a Pushed.co application for the mobile notifications feature. Here is a link for now https://pushed.co/quick-start-guide diff --git a/birdnet.conf-defaults b/birdnet.conf-defaults new file mode 100644 index 0000000..aff886a --- /dev/null +++ b/birdnet.conf-defaults @@ -0,0 +1,224 @@ +################################################################################ +# Configuration settings for BirdNET as a service # +################################################################################ +INSTALL_DATE="$(date "+%D")" +#___________The four variables below are the only that are required.___________# + +## BIRDNET_USER should be the non-root user systemd should use to execute each +## service. + +BIRDNET_USER= + +## RECS_DIR is the location birdnet_analysis.service will look for the data-set +## it needs to analyze. Be sure this directory is readable and writable for +## the BIRDNET_USER. If you are going to be accessing a remote data-set, you +## still need to set this, as this will be where the remote directory gets +## mounted locally. See REMOTE_RECS_DIR below for mounting remote data-sets. + +RECS_DIR= + +## LATITUDE and LONGITUDE are self-explanatroy. Find them easily at +## maps.google.com. Only go to the thousanths place for these variables +## Example: these coordinates would indicate the Eiffel Tower in Paris, France. +## LATITUDE=48.858 +## LONGITUDE=2.294 + +LATITUDE= +LONGITUDE= + +################################################################################ +#------------------------------ Extraction Service ---------------------------# + +# Keep this EMPTY if you do not want this device to perform the extractions # + +## DO_EXTRACTIONS is simply a setting for enabling the extraction.service. +## Set this to Y or y to enable extractions. + +DO_EXTRACTIONS= + +################################################################################ +#----------------------------- Recording Service ----------------------------# + +# Keep this EMPTY if you do not want this device to perform the recording. # + +## DO_RECORDING is simply a setting for enabling the 24/7 birdnet_recording.service. +## Set this to Y or y to enable recording. + +DO_RECORDING= + +################################################################################ +#----------------- Mounting a remote directory with systemd -----------------# +#_______________The four variables below can be set to enable a_______________# +#___________________systemd.mount for analysis, extraction,____________________# +#______________________________or file-serving_________________________________# + +# Leave these settings EMPTY if your data-set is local. # + +## REMOTE is simply a setting for enabling the systemd.mount to use a remote +## filesystem for the data storage and service. +## Set this to Y or y to enable the systemd.mount. + +REMOTE= + +## REMOTE_HOST is the IP address, hostname, or domain name SSH should use to +## connect for FUSE to mount its remote directories locally. + +REMOTE_HOST= + +## REMOTE_USER is the user SSH will use to connect to the REMOTE_HOST. + +REMOTE_USER= + +## REMOTE_RECS_DIR is the directory on the REMOTE_HOST which contains the +## data-set SSHFS should mount to this system for local access. This is NOT the +## directory where you will access the data on this machine. See RECS_DIR for +## that. + +REMOTE_RECS_DIR= + +################################################################################ +#----------------------- Web-hosting/Caddy File-server -----------------------# +#__________The two variables below can be set to enable web access_____________# +#____________to your data,(e.g., extractions, raw data, live___________________# +#______________audio stream, BirdNET.selection.txt files)______________________# + +# Leave these EMPTY if you do not want to enable web access # + +## EXTRACTIONS_URL is the URL where the extractions, data-set, and live-stream +## will be web-hosted. If you do not own a domain, or would just prefer to keep +## BirdNET-Lite on your local network, you can set this to http://localhost. +## Setting this (even to http://localhost) will also allow you to enable the +## GoTTY web logging features below. + +EXTRACTIONS_URL= + +## CADDY_PWD is the plaintext password (that will be hashed) and used to access +## the "Processed" directory and live audio stream. This MUST be set if you +## choose to enable this feature. + +CADDY_PWD= + +################################################################################ +#------------------------- Live Audio Stream --------------------------------# +#_____________The variable below configures/enables the live___________________# +#_____________________________audio stream.____________________________________# + +# Keep this EMPTY if you do not wish to enable the live stream # +# or if this device is not doing the recording # + +## ICE_PWD is the password that icecast2 will use to authenticate ffmpeg as a +## trusted source for the stream. You will never need to enter this manually +## anywhere other than here. + +ICE_PWD= + +################################################################################ +#------------------- Mobile Notifications via Pushed.co ---------------------# +#____________The two variables below enable mobile notifications_______________# +#_____________See https://pushed.co/quick-start-guide to get___________________# +#_________________________these values for your app.___________________________# + +# Keep these EMPTY if haven't setup a Pushed.co App yet. # + +## Pushed.co App Key and App Secret + +PUSHED_APP_KEY= +PUSHED_APP_SECRET= + +################################################################################ +#------------------------------- NoMachine ----------------------------------# +#_____________The variable below can be set include NoMachine__________________# +#_________________remote desktop software to be installed._____________________# + +# Keep this EMPTY if you do not want to install NoMachine. # + +## INSTALL_NOMACHINE is simply a setting that can be enabled to install +## NoMachine alongside the BirdNET-Lite for remote desktop access. This in- +## staller assumes personal use. Please reference the LICENSE file included +## in this repository for more information. +## Set this to Y or y to install NoMachine alongside the BirdNET-Lite + +INSTALL_NOMACHINE= + +################################################################################ +#-------------------------------- Defaults ----------------------------------# +#______The seven variables below are default settings that you (probably)______# +#__________________don't need to change at all, but can._______________________# + +## REC_CARD is the sound card you would want the birdnet_recording.service to +## use. This setting is irrelevant if you are not planning on doing data +## collection via recording on this machine. The command substitution below +## looks for a USB microphone's dsnoop alsa device. The dsnoop device lets +## birdnet_recording.service and livestream.service share the raw audio stream +## from the microphone. If you would like to use a different microphone than +## what this produces, or if your microphone does not support creating a +## dsnoop device, you can set this explicitly from a list of the available +## devices from the output of running 'aplay -L' + +REC_CARD="\$(sudo -u pi aplay -L \ + | grep dsnoop \ + | cut -d, -f1 \ + | grep -ve 'vc4' -e 'Head' -e 'PCH' \ + | uniq)" + +## PROCESSED is the directory where the formerly 'Analyzed' files are moved +## after extractions have been made from them. This includes both WAVE and +## BirdNET.selection.txt files. + +PROCESSED=${RECS_DIR}/Processed + +## EXTRACTED is the directory where the extracted audio selections are moved. + +EXTRACTED=${RECS_DIR}/Extracted + +## IDFILE is the file that keeps a complete list of every spececies that +## BirdNET has identified from your data-set. It is persistent across +## data-sets, so would need to be whiped clean through deleting or renaming +## it. A backup is automatically made from this variable each time it is +## updated (structure: ${IDFILE}.bak), and would also need to be removed +## or renamed to start a new file between data-sets. Alternately, you can +## change this variable between data-sets to preserve records of disparate +## data-sets according to name. + +IDFILE=${HOME}/BirdNET-Lite/IdentifiedSoFar.txt + +## OVERLAP is the value in seconds which BirdNET should use when analyzing +## the data. The values must be between 0.0-2.9. + +OVERLAP="0.0" + +## CONFIDENCE is the minimum confidence level from 0.0-1.0 BirdNET's analysis +## should reach before creating an entry in the BirdNET.selection.txt file. +## Don't set this to 1.0 or you won't have any results. + +CONFIDENCE="0.7" + +################################################################################ +#------------------------------ Auto-Generated ------------------------------# +#_____________________The variables below are auto-generated___________________# +#______________________________during installation_____________________________# + +## CHANNELS holds the variabel that corresponds to the number of channels the +## sound card supports. + +CHANNELS=${CHANNELS} + +# Don't the three below + +## ANALYZED is where the extraction.service looks for audio and +## BirdNET.selection.txt files after they have been processed by the +## birdnet_analysis.service. This is NOT where the analyzed files are moved -- +## analyzed files are always created within the same directory +## birdnet_analysis.service finds them. + +ANALYZED=${RECS_DIR}/*/*Analyzed + +## SYSTEMD_MOUNT is created from the RECS_DIR variable to comply with systemd +## mount naming requirements. + +SYSTEMD_MOUNT=$(echo ${RECS_DIR#/} | tr / -).mount + +## VENV is the virtual environment where the the BirdNET python build is found, +## i.e, VENV is the virtual environment miniforge built for BirdNET. + +VENV=$(dirname ${my_dir})/miniforge/envs/birdnet diff --git a/scripts/birdnet_analysis.sh b/scripts/birdnet_analysis.sh new file mode 100755 index 0000000..6d68a52 --- /dev/null +++ b/scripts/birdnet_analysis.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# Runs BirdNET in virtual environment +#set -x +source /etc/birdnet/birdnet.conf +DAYS=( +"2 days ago" +"yesterday" +"today" +) + +# Create an array of the day's audio files +# Uses 1st argument: +# - {DIRECTORY} +get_files() { + echo "Starting get_files() for ${1}" + files=($( find ${1} -maxdepth 1 -name '*wav' \ + | sort \ + | awk -F "/" '{print $NF}' )) + [ -n "${files[1]}" ] && echo "Files loaded" +} + +# Move all files that have been analyzed already into newly created "Analyzed" +# directory +# Uses 1st argument: +# - {DIRECTORY} +move_analyzed() { + echo "Starting move_analyzed() for ${1}" + for i in "${files[@]}";do + j="$(echo "${i}" | cut -d'.' -f1).BirdNET.selections.txt" + if [ -f "${1}/${j}" ];then + if [ ! -d "${1}-Analyzed" ];then + mkdir -vvvvvvvp "${1}-Analyzed" && echo "'Analyzed' directory created" + fi + echo "Moving analyzed files to new directory" + mv -vv "${1}/${i}" "${1}-Analyzed/" + mv -vv "${1}/${j}" "${1}-Analyzed/" + fi +done +} + +# Run BirdNET analysis on the remaining WAVE files for the day +# Uses 1st and 2nd arguments: +# - {DIRECTORY} +# - {"today", "yesterday", "2 days ago",...} +run_analysis() { + echo "Starting run_analysis() for ${1}" + WEEK=$(date --date="${2}" +"%U") + cd ${HOME}/BirdNET-Lite || exit 1 + "${VENV}"/bin/python analyze.py \ + --i "${1}" \ + --lat "${LATITUDE}" \ + --lon "${LONGITUDE}" \ + --week "${WEEK}" \ + --overlap "${OVERLAP}" \ + --min_conf "${CONFIDENCE}" +} + +# The three main functions +# Requires 2 arguments: +# - {DIRECTORY} +# - {"today", "yesterday", "2 days ago",...} +run_birdnet() { + echo "Starting run_birdnet() in \"${1}\" for \""${2}"\"" + sleep 1 + get_files "${1}" + sleep 1 + move_analyzed "${1}" + sleep 1 + run_analysis "${1}" "${2}" +} + +if [ $(find ${RECS_DIR} -maxdepth 1 -name '*wav' | wc -l) -gt 0 ];then + run_birdnet "${RECS_DIR}" "today" +fi + +for i in ${!DAYS[@]};do + DIRECTORY="$RECS_DIR/$(date --date="${DAYS[$i]}" "+%B-%Y/%d-%A")" + if [ $(find ${DIRECTORY} -name '*wav' | wc -l) -gt 0 ];then + run_birdnet "${DIRECTORY}" "${DAYS[$i]}" + fi +done diff --git a/scripts/birdnet_recording.sh b/scripts/birdnet_recording.sh new file mode 100755 index 0000000..4ee59c8 --- /dev/null +++ b/scripts/birdnet_recording.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -x +source /etc/birdnet/birdnet.conf + +if pgrep arecord &> /dev/null ;then + echo "Recording" +else + if [ -z ${REC_CARD} ];then + arecord -f S16_LE -c${CHANNELS} -r48000 -t wav --max-file-time 60 \ + --use-strftime ${RECS_DIR}/%B-%Y/%d-%A/%F-birdnet-%I:%M%P.wav + else + arecord -f S16_LE -c${CHANNELS} -r48000 -t wav --max-file-time 60 \ + -D "${REC_CARD}" --use-strftime \ + ${RECS_DIR}/%B-%Y/%d-%A/%F-birdnet-%I:%M%P.wav + fi +fi diff --git a/scripts/birdnet_stats.sh b/scripts/birdnet_stats.sh new file mode 100755 index 0000000..70fa88b --- /dev/null +++ b/scripts/birdnet_stats.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# BirdNET Stats Page +trap 'setterm --cursor on' EXIT +source /etc/birdnet/birdnet.conf +setterm --cursor off + +while true;do +cat << "EOF" + .+"+.+"+.+"+.+"+.+"+.+"+.+"+.+"+.+"+.+"+.+"+.+"+.+"+.+"+. +( _ ____ __ _ ) + ) |_)o.__||\ ||_ |__(_ __|_ _ ._ _ |_) _ ._ _ .__|_ ( +( |_)||(_|| \||_ | __)\/_> |_(/_| | | | \(/_|_)(_)| |_ ) + ) / | ( + "+.+"+.+"+.+"+.+"+.+"+.+"+.+"+.+"+.+"+.+"+.+"+.+"+.+"+.+" +EOF +if [ "$(find ${EXTRACTED} -name '*.wav' | wc -l)" -ge 1 ];then + a=$( find "${EXTRACTED}" -name '*.wav' \ + | awk -F "/" '{print $NF}' \ + | cut -d'-' -f1 \ + | sort -n \ + | tail -n1 ) +else + a=0 +fi +echo +SOFAR=$(wc -l ${IDFILE}| cut -d' ' -f1) +echo " -$a detections so far" +echo +echo " -$SOFAR species identified so far" +while read -r line;do + echo " + $line" +done < ${IDFILE} +echo +echo -n "Listening since "${INSTALL_DATE}"" +sleep 180 +clear +done diff --git a/scripts/cleanup.sh b/scripts/cleanup.sh new file mode 100755 index 0000000..27c8998 --- /dev/null +++ b/scripts/cleanup.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +source /etc/birdnet/birdnet.conf + +cd "${PROCESSED}" || exit 1 +FIND_DATE=*$(date --date="2 days ago" "+%F")* +find . -name "${FIND_DATE}" -exec rm -rfv {} + diff --git a/scripts/disk_usage.sh b/scripts/disk_usage.sh new file mode 100755 index 0000000..d07af5a --- /dev/null +++ b/scripts/disk_usage.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +# Disk Usage Indicator for TMUX +df -h / | awk '/\// {print "Disk Size="$2", Used="$3", Available="$4", "$5" of disk used"}' diff --git a/scripts/dump_logs.sh b/scripts/dump_logs.sh new file mode 100755 index 0000000..e3f7150 --- /dev/null +++ b/scripts/dump_logs.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# A comprehensive log dumper +# set -x # Uncomment to debug +source /etc/birdnet/birdnet.conf &> /dev/null +LOG_DIR="${HOME}/BirdNET-Lite/logs" +SERVICES=(avahi-alias@.service +birdnet_analysis.service +birdnet_log.service +birdnet_recording.service +birdstats.service +birdterminal.service +caddy.service +extraction_log.service +extraction.service +extraction.timer +icecast2.service +livestream.service +${SYSTEMD_MOUNT}) + +# Create logs directory +[ -d ${LOG_DIR} ] || mkdir ${LOG_DIR} + +# Create services logs +for i in "${SERVICES[@]}";do + if [ -L /etc/systemd/system/multi-user.target.wants/${i} ];then + journalctl -u ${i} -n 100 --no-pager > ${LOG_DIR}/${i}.log + cp -L /etc/systemd/system/multi-user.target.wants/${i} ${LOG_DIR}/${i} + fi +done + +# Create password-removed birdnet.conf +sed -e '/PWD=/d' ${HOME}/BirdNET-Lite/birdnet.conf > ${LOG_DIR}/birdnet.conf + +# Create password-removed Caddyfile +if [ -f /etc/caddy/Caddyfile ];then + sed -e '/basicauth/,+2d' /etc/caddy/Caddyfile > ${LOG_DIR}/Caddyfile +fi + +# Get sound card specs +SOUND_CARD="$(aplay -L \ + | awk -F, '/^hw:/ {print $1}' \ + | grep -ve 'vc4' -e 'Head' -e 'PCH' \ + | uniq)" +echo "SOUND_CARD=${SOUND_CARD}" > ${LOG_DIR}/soundcard +script -c "arecord -D ${SOUND_CARD} --dump-hw-params" -a ${LOG_DIR}/soundcard &> /dev/null + +# Get system info +CALLS=("df -h" "free -h" "ifconfig" "find ${RECS_DIR}") + +for i in "${CALLS[@]}";do + ${i} >> ${LOG_DIR}/sysinfo + echo " +=============================================================================== +=============================================================================== + +" >> ${LOG_DIR}/sysinfo +done + +# TAR the logs into a ball +tar --remove-files -cvpzf ${HOME}/BirdNET-Lite/logs.tar.gz ${LOG_DIR} &> /dev/null +# Finished +echo "Your compressed logs are located at ${HOME}/BirdNET-Lite/logs.tar.gz" diff --git a/scripts/extract_new_birdsounds.sh b/scripts/extract_new_birdsounds.sh new file mode 100755 index 0000000..e5d1c6d --- /dev/null +++ b/scripts/extract_new_birdsounds.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env bash +# Exit when any command fails +#set -x +set -e +# Keep track of the last executed command +trap 'last_command=$current_command; current_command=$BASH_COMMAND' DEBUG +# Echo an error message before exiting +trap 'echo "\"${last_command}\" command exited with code $?."' EXIT +# Remove temporary file +trap 'rm -f $TMPFILE' EXIT + +source /etc/birdnet/birdnet.conf + +# Set Variables +TMPFILE=$(mktemp) +# SCAN_DIRS are all directories marked "Analyzed" +SCAN_DIRS=($(find ${ANALYZED} -type d | sort )) + +# This sets our while loop integer iterator 'a' -- it first checks whether +# there are any extractions, and if so, this instance of the extraction will +# start the 'a' value where the most recent instance left off. +# Ex: If the last extraction file is 189-%date%species.wav, 'a' will be 190. +# Else, 'a' starts at 1 +if [ "$(find ${EXTRACTED} -name '*.wav' | wc -l)" -ge 1 ];then + a=$(( $( find "${EXTRACTED}" -name '*.wav' \ + | awk -F "/" '{print $NF}' \ + | cut -d'-' -f1 \ + | sort -n \ + | tail -n1 ) + 1 )) +else + a=1 +fi + +echo "Starting numbering at ${a}" + +for h in "${SCAN_DIRS[@]}";do + echo "Creating the TMPFILE" + # The TMPFILE is created from each "Selection" txt file BirdNET creates + # within each "Analyzed" directory + # Field 1: Original WAVE file name + # Field 2: Extraction start time in seconds + # Field 3: Extraction end time in seconds + # Field 4: New WAVE file name to use + # Field 5: The species name + # Iterates over each "Analyzed" directory + for i in $(find ${h} -name '*txt' | sort );do + # Iterates over each '.txt' file found in each "Analyzed" directory + # to create the TMPFILE + sort -k 6n "$i" \ + | awk '/Spect/ {print}' \ + >> $TMPFILE + done + + # The extraction reads each line of the TMPFILE and sets the variables ffmpeg + # will use. + while read -r line;do + a=$a + DATE="$(echo "${line}" \ + | awk '{print $5}' \ + | awk -F- '{print $1"-"$2"-"$3}')" + OLDFILE="$(echo "${line}" | awk '{print $5}')" + START="$(echo "${line}" | awk '{print $6}')" + END="$(echo "${line}" | awk '{print $7}')" + SPECIES=""$(echo ${line//\'} \ + | awk '{for(i=11;i<=NF;++i)printf $i""FS ; print ""}' \ + | cut -d'0' -f1 \ + | xargs)"" + NEWFILE="${SPECIES// /_}-${OLDFILE}" + NEWSPECIES_BYDATE="${EXTRACTED}/By_Date/${DATE}/${SPECIES// /_}" + NEWSPECIES_BYSPEC="${EXTRACTED}/By_Species/${SPECIES// /_}" + + # If the extracted file already exists, increment the 'a' variable once + # but move onto the next line of the TMPFILE for extraction. + if [[ -f "${NEWSPECIES_BYDATE}/${a}-${NEWFILE}" ]];then + echo "Extraction exists. Moving on" + a=$((a+1)) + continue + fi + + + echo "Checking for ${h}/${OLDFILE}" + # Before extracting the "Selection," the script checks to be sure the + # original WAVE file still exists. + [[ -f "${h}/${OLDFILE}" ]] || continue + + + echo "Checking for ${NEWSPECIES_BYDATE}" + # If a directory does not already exist for the species (by date), + # it is created + [[ -d "${NEWSPECIES_BYDATE}" ]] || mkdir -p "${NEWSPECIES_BYDATE}" + + + echo "Checking for ${NEWSPECIES_BYSPEC}" + # If a directory does not already exist for the species (by-species), + # it is created. + [[ -d "${NEWSPECIES_BYSPEC}" ]] || mkdir -p "${NEWSPECIES_BYSPEC}" + + + # If there are already 20 extracted entries for a given species + # for today, remove the oldest file and create the new one. + if [[ "$(find ${NEWSPECIES_BYDATE} | wc -l)" -ge 21 ]];then + echo "20 ${SPECIES}s, already! Removing the oldest by-date and making a new one" + cd ${NEWSPECIES_BYDATE} || exit 1 + ls -1t . | tail -n +21 | xargs -r rm -vv + fi + + echo "Extracting audio . . . " + # If the above tests have passed, then the extraction happens. + # After creating the extracted files by-date, and a directory tree + # structured by-species, symbolic links are made to populate the new + # directory. + + ffmpeg -hide_banner -loglevel error -nostdin -i "${h}/${OLDFILE}" \ + -acodec copy -ss "${START}" -to "${END}"\ + "${NEWSPECIES_BYDATE}/${a}-${NEWFILE}" + if [[ "$(find ${NEWSPECIES_BYSPEC} | wc -l)" -ge 21 ]];then + echo "20 ${SPECIES}s, already! Removing the oldest by-species and making a new one" + cd ${NEWSPECIES_BYSPEC} || exit 1 + ls -1t . | tail -n +21 | xargs -r rm -vv + ln -fs "${NEWSPECIES_BYDATE}/${a}-${NEWFILE}"\ + "${NEWSPECIES_BYSPEC}/${a}-${NEWFILE}" + echo "Success! New extraction for ${SPECIES}" + else + ln -fs "${NEWSPECIES_BYDATE}/${a}-${NEWFILE}"\ + "${NEWSPECIES_BYSPEC}/${a}-${NEWFILE}" + fi + + + # Finally, 'a' is incremented by one to allow multiple extractions per + # species per minute. + a=$((a + 1)) + + done < "${TMPFILE}" + + echo -e "\n\n\nFINISHED!!! Processed extractions for ${h}" + # Once each line of the TMPFILE has been processed, the TMPFILE is emptied + # for the next iteration of the for loop. + >"${TMPFILE}" + + # Rename files that have been processed so that they are not processed on the + # next extraction. + [[ -d "${PROCESSED}" ]] || mkdir "${PROCESSED}" + echo "Moving processed files to ${PROCESSED}" + mv -v ${h}/* ${PROCESSED} || continue +done + +echo "Linking Processed files to "${EXTRACTED}/Processed" web directory" +# After all audio extractions have taken place, a directory is created to house +# the original WAVE and .txt files used for this extraction processs. +if [[ ! -L ${EXTRACTED}/Processed ]] || [[ ! -e ${EXTRACTED}/Processed ]];then + ln -sf ${PROCESSED} ${EXTRACTED}/Processed +fi + + + +# That's all! +echo "Finished -- the extracted sections are in: +$(find -L ${EXTRACTED} -maxdepth 1)" diff --git a/scripts/install_birdnet.sh b/scripts/install_birdnet.sh new file mode 100755 index 0000000..3221497 --- /dev/null +++ b/scripts/install_birdnet.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +# Install BirdNET script +#set -x # debugging +set -e # exit installation if anything fails +my_dir=$(realpath $(dirname $0)) +trap '${my_dir}/dump_logs.sh && echo -e "\n\nExiting the installation. Goodbye!" && exit 1' SIGINT +cd $my_dir || exit 1 + +if [ "$(uname -m)" != "aarch64" ];then + echo "BirdNET-Lite requires a 64-bit OS. +It looks like your operating system is using $(uname -m), +but would need to be aarch64. +Please take a look at https://birdnetwiki.pmcgui.xyz for more +information" + exit 1 +fi + +#Install/Configure /etc/birdnet/birdnet.conf +./install_config.sh || exit 1 +sudo ./install_services.sh || exit 1 +source /etc/birdnet/birdnet.conf + +CODE="$(awk '/_warning_/ {print $NF}' /tmp/cookie)" +TFLITE_URL="https://drive.google.com/uc?export=download&id=1dlEbugFDJXs-YDBCUC6WjADVtIttWxZA" +TF_COOKIE="https://drive.google.com/uc?export=download&confirm=${CODE}&id=1dlEbugFDJXs-YDBCUC6WjADVtIttWxZA" +APT_DEPS=(swig ffmpeg wget unzip curl cmake make) +LIBS_MODULES=(libjpeg-dev zlib1g-dev python3-dev python3-pip) + +install_deps() { + echo " Checking dependencies" + sudo apt update &> /dev/null + for i in "${LIBS_MODULES[@]}";do + if [ $(apt list --installed 2>/dev/null | grep "$i" | wc -l) -le 0 ];then + echo " Installing $i" + sudo apt -y install ${i} &> /dev/null + else + echo " $i is installed!" + fi + done + + for i in "${APT_DEPS[@]}";do + if ! which $i &>/dev/null ;then + echo " Installing $i" + sudo apt -y install ${i} &> /dev/null + else + echo " $i is installed!" + fi + done +} + +install_birdnet() { + set -e + cd ~/BirdNET-Lite || exit 1 + echo "Upgrading pip, whell, and setuptools" + sudo pip3 install --upgrade pip wheel setuptools + echo "Fetching the TFLite pre-built binaries" + curl -sc /tmp/cookie ${TFLITE_URL} > /dev/null + curl -Lb /tmp/cookie ${TF_COOKIE} -o tflite_runtime-2.6.0-cp37-none-linux_aarch64.whl + echo "Installing the new TFLite bin wheel" + sudo pip3 install --upgrade tflite_runtime-2.6.0-cp37-none-linux_aarch64.whl + echo "Installing colorama==0.4.4" + sudo pip3 install colorama==0.4.4 + echo "Installing librosa" + sudo pip3 install librosa +} + +echo " +This script will do the following: +#1: Install the following BirdNET system dependencies: +- ffmpeg +- swig +- libjpeg-dev +- zlib1g-dev +- python3-dev +- curl +- cmake +- make +- wget +#2: Copies the systemd .service and .mount files and enables those chosen +#3: Adds cron environments and jobs chosen" + +echo +read -sp "\ +Be sure you have read the software license before installing. This is +available in the BirdNET-Lite directory as "LICENSE" +If you DO NOT want to install BirdNET and the birdnet_analysis.service, +press Ctrl+C to cancel. If you DO wish to install BirdNET and the +birdnet_analysis.service, press ENTER to continue with the installation." +echo +echo + +[ -d ${RECS_DIR} ] || mkdir -p ${RECS_DIR} &> /dev/null + +install_deps +if [ ! -d ${VENV} ];then + install_birdnet +fi + +echo " BirdNet is installed!! + + To start the service manually, issue: + 'sudo systemctl start birdnet_analysis' + To monitor the service logs, issue: + 'journalctl -fu birdnet_analysis' + To stop the service manually, issue: + 'sudo systemctl stop birdnet_analysis' + To stop and disable the service, issue: + 'sudo systemctl disable --now birdnet_analysis.service' + + Visit + http://birdnetsystem.local to see your extractions, + http://birdlog.local to see the log output of the birdnet_analysis.service, + http://extractionlog.local to see the log output of the extraction.service, and + http://birdstats.local to see the BirdNET-Lite Report" +echo +read -n1 -p " Would you like to run the birdnet_analysis.service now?" YN +echo +case $YN in + [Yy] ) sudo systemctl start birdnet_analysis.service \ + && journalctl -fu birdnet_analysis;; +* ) echo " Thanks for installing BirdNET-Lite!! + I hope it was helpful!";; +esac diff --git a/scripts/install_config.sh b/scripts/install_config.sh new file mode 100755 index 0000000..7d56e1a --- /dev/null +++ b/scripts/install_config.sh @@ -0,0 +1,412 @@ +#!/usr/bin/env bash +# Creates and installs the /etc/birdnet/birdnet.conf file +#set -x # Uncomment to enable debugging +set -e +trap 'exit 1' SIGINT SIGHUP + +my_dir=$(realpath $(dirname $0)) +BIRDNET_CONF="$(dirname ${my_dir})/birdnet.conf" + +get_RECS_DIR() { + read -p "What is the full path to your recordings directory (locally)? " RECS_DIR +} + +get_LATITUDE() { + read -p "What is the latitude where the recordings were made? " LATITUDE +} + +get_LONGITUDE() { + read -p "What is the longitude where the recordings were made? " LONGITUDE +} + +get_DO_EXTRACTIONS() { + while true; do + read -n1 -p "Do you want this device to perform the extractions? " DO_EXTRACTIONS + echo + case $DO_EXTRACTIONS in + [Yy] ) break;; + [Nn] ) break;; + * ) echo "You must answer with Yes or No (y or n)";; + esac + done +} + +get_DO_RECORDING() { + while true; do + read -n1 -p "Is this device also doing the recording? " DO_RECORDING + echo + case $DO_RECORDING in + [Yy] ) break;; + [Nn] ) break;; + * ) echo "You must answer with Yes or No (y or n)";; + esac + done +} + +get_REMOTE() { + while true; do + read -n1 -p "Are the recordings mounted on a remote file system?" REMOTE + echo + case $REMOTE in + [Yy] ) + read -p "What is the remote hostname or IP address for the recorder? " REMOTE_HOST + read -p "Who is the remote user? " REMOTE_USER + read -p "What is the absolute path of the recordings directory on the remote host? " REMOTE_RECS_DIR + break;; + [Nn] ) break;; + * ) echo "Please answer Yes or No (y or n)";; + esac + done +} + +get_EXTRACTIONS_URL() { + while true;do + read -n1 -p "Would you like to access the extractions via a web browser? + + *Note: It is recommended, (but not required), that you run the web + server on the same host that does the extractions. If the extraction + service and web server are on different hosts, the \"By_Species\" and + \"Processed\" symbolic links won't work. The \"By-Date\" extractions, + however, will work as expected." CADDY_SERVICE + echo + case $CADDY_SERVICE in + [Yy] ) read -p "What URL would you like to publish the extractions to? + *Note: Set this to http://localhost if you do not want to make the + extractions publically available: " EXTRACTIONS_URL + get_CADDY_PWD + get_ICE_PWD + break;; + [Nn] ) EXTRACTIONS_URL= CADDY_PWD= ICE_PWD=;break;; + * ) echo "Please answer Yes or No";; + esac + done +} + +get_CADDY_PWD() { + if [ -z ${CADDY_PWD} ]; then + while true; do + read -p "Please set a password to protect your data: " CADDY_PWD + case $CADDY_PWD in + "" ) echo "The password cannot be empty. Please try again.";; + * ) break;; + esac + done + fi +} + +get_ICE_PWD() { + if [ ! -z ${CADDY_PWD} ] && [[ ${DO_RECORDING} =~ [Yy] ]];then + while true; do + read -n1 -p "Would you like to enable the live audio streaming service?" LIVE_STREAM + echo + case $LIVE_STREAM in + [Yy] ) + read -p "Please set the icecast password. Use only alphanumeric characters." ICE_PWD + echo + case ${ICE_PWD} in + "" ) echo "The password cannot be empty. Please try again.";; + *) break;; + esac + break;; + [Nn] ) break;; + * ) echo "You must answer Yes or No (y or n).";; + esac + done + fi +} + +get_PUSHED() { + while true; do + read -n1 -p "Do you have a free App key to receive mobile notifications via Pushed.co?" YN + echo + case $YN in + [Yy] ) read -p "Enter your Pushed.co App Key: " PUSHED_APP_KEY + read -p "Enter your Pushed.co App Key Secret: " PUSHED_APP_SECRET + break;; + [Nn] ) PUSHED_APP_KEY= + PUSHED_APP_SECRET= + break;; + * ) echo "A simple Yea or Nay will do";; + esac + done +} + +get_INSTALL_NOMACHINE() { + while true; do + read -n1 -p "Would you like to also install NoMachine for remote desktop access?" INSTALL_NOMACHINE + echo + case $INSTALL_NOMACHINE in + [Yy] ) break;; + [Nn] ) break;; + * ) echo "You must answer with Yes or No (y or n)";; + esac + done +} + +get_CHANNELS() { + [ -f $(dirname ${my_dir})/soundcard_params.txt ] || touch $(dirname ${my_dir})/soundcard_params.txt + SOUND_PARAMS=$(dirname ${my_dir})/soundcard_params.txt + SOUND_CARD="$(sudo -u ${USER} aplay -L \ + | awk -F, '/^hw:/ {print $1}' \ + | grep -ve 'vc4' -e 'Head' -e 'PCH' \ + | uniq)" + script -c "arecord -D ${SOUND_CARD} --dump-hw-params" -a "${SOUND_PARAMS}" &> /dev/null + + CHANNELS=$(awk '/CHANN/ { print $2 }' "${SOUND_PARAMS}" | sed 's/\r$//') + echo "Number of channels available: ${CHANNELS}" +} + + +configure() { + get_RECS_DIR + get_LATITUDE + get_LONGITUDE + get_DO_EXTRACTIONS + get_DO_RECORDING + get_REMOTE + get_EXTRACTIONS_URL + get_PUSHED + get_INSTALL_NOMACHINE + get_CHANNELS +} + +install_birdnet_conf() { + cat << EOF > $(dirname ${my_dir})/birdnet.conf +################################################################################ +# Configuration settings for BirdNET as a service # +################################################################################ +INSTALL_DATE="$(date "+%D")" +#___________The four variables below are the only that are required.___________# + +## BIRDNET_USER should be the non-root user systemd should use to execute each +## service. + +BIRDNET_USER=${USER} + +## RECS_DIR is the location birdnet_analysis.service will look for the data-set +## it needs to analyze. Be sure this directory is readable and writable for +## the BIRDNET_USER. If you are going to be accessing a remote data-set, you +## still need to set this, as this will be where the remote directory gets +## mounted locally. See REMOTE_RECS_DIR below for mounting remote data-sets. + +RECS_DIR=${RECS_DIR} + +## LATITUDE and LONGITUDE are self-explanatroy. Find them easily at +## maps.google.com. Only go to the thousanths place for these variables +## Example: these coordinates would indicate the Eiffel Tower in Paris, France. +## LATITUDE=48.858 +## LONGITUDE=2.294 + +LATITUDE="${LATITUDE}" +LONGITUDE="${LONGITUDE}" + +################################################################################ +#------------------------------ Extraction Service ---------------------------# + +# Keep this EMPTY if you do not want this device to perform the extractions # + +## DO_EXTRACTIONS is simply a setting for enabling the extraction.service. +## Set this to Y or y to enable extractions. + +DO_EXTRACTIONS=${DO_EXTRACTIONS} + +################################################################################ +#----------------------------- Recording Service ----------------------------# + +# Keep this EMPTY if you do not want this device to perform the recording. # + +## DO_RECORDING is simply a setting for enabling the 24/7 birdnet_recording.service. +## Set this to Y or y to enable recording. + +DO_RECORDING=${DO_RECORDING} + +################################################################################ +#----------------- Mounting a remote directory with systemd -----------------# +#_______________The four variables below can be set to enable a_______________# +#___________________systemd.mount for analysis, extraction,____________________# +#______________________________or file-serving_________________________________# + +# Leave these settings EMPTY if your data-set is local. # + +## REMOTE is simply a setting for enabling the systemd.mount to use a remote +## filesystem for the data storage and service. +## Set this to Y or y to enable the systemd.mount. + +REMOTE=${REMOTE} + +## REMOTE_HOST is the IP address, hostname, or domain name SSH should use to +## connect for FUSE to mount its remote directories locally. + +REMOTE_HOST=${REMOTE_HOST} + +## REMOTE_USER is the user SSH will use to connect to the REMOTE_HOST. + +REMOTE_USER=${REMOTE_USER} + +## REMOTE_RECS_DIR is the directory on the REMOTE_HOST which contains the +## data-set SSHFS should mount to this system for local access. This is NOT the +## directory where you will access the data on this machine. See RECS_DIR for +## that. + +REMOTE_RECS_DIR=${REMOTE_RECS_DIR} + +################################################################################ +#----------------------- Web-hosting/Caddy File-server -----------------------# +#__________The two variables below can be set to enable web access_____________# +#____________to your data,(e.g., extractions, raw data, live___________________# +#______________audio stream, BirdNET.selection.txt files)______________________# + +# Leave these EMPTY if you do not want to enable web access # + +## EXTRACTIONS_URL is the URL where the extractions, data-set, and live-stream +## will be web-hosted. If you do not own a domain, or would just prefer to keep +## BirdNET-Lite on your local network, you can set this to http://localhost. +## Setting this (even to http://localhost) will also allow you to enable the +## GoTTY web logging features below. + +EXTRACTIONS_URL=${EXTRACTIONS_URL} + +## CADDY_PWD is the plaintext password (that will be hashed) and used to access +## the "Processed" directory and live audio stream. This MUST be set if you +## choose to enable this feature. + +CADDY_PWD=${CADDY_PWD} + +################################################################################ +#------------------------- Live Audio Stream --------------------------------# +#_____________The variable below configures/enables the live___________________# +#_____________________________audio stream.____________________________________# + +# Keep this EMPTY if you do not wish to enable the live stream # +# or if this device is not doing the recording # + +## ICE_PWD is the password that icecast2 will use to authenticate ffmpeg as a +## trusted source for the stream. You will never need to enter this manually +## anywhere other than here. + +ICE_PWD=${ICE_PWD} + +################################################################################ +#------------------- Mobile Notifications via Pushed.co ---------------------# +#____________The two variables below enable mobile notifications_______________# +#_____________See https://pushed.co/quick-start-guide to get___________________# +#_________________________these values for your app.___________________________# + +# Keep these EMPTY if haven't setup a Pushed.co App yet. # + +## Pushed.co App Key and App Secret + +PUSHED_APP_KEY=${PUSHED_APP_KEY} +PUSHED_APP_SECRET=${PUSHED_APP_SECRET} + +################################################################################ +#------------------------------- NoMachine ----------------------------------# +#_____________The variable below can be set include NoMachine__________________# +#_________________remote desktop software to be installed._____________________# + +# Keep this EMPTY if you do not want to install NoMachine. # + +## INSTALL_NOMACHINE is simply a setting that can be enabled to install +## NoMachine alongside the BirdNET-Lite for remote desktop access. This in- +## staller assumes personal use. Please reference the LICENSE file included +## in this repository for more information. +## Set this to Y or y to install NoMachine alongside the BirdNET-Lite + +INSTALL_NOMACHINE=${INSTALL_NOMACHINE} + +################################################################################ +#-------------------------------- Defaults ----------------------------------# +#________The six variables below are default settings that you (probably)______# +#__________________don't need to change at all, but can._______________________# + +## REC_CARD is the sound card you would want the birdnet_recording.service to +## use. This setting is irrelevant if you are not planning on doing data +## collection via recording on this machine. The command substitution below +## looks for a USB microphone's dsnoop alsa device. The dsnoop device lets +## birdnet_recording.service and livestream.service share the raw audio stream +## from the microphone. If you would like to use a different microphone than +## what this produces, or if your microphone does not support creating a +## dsnoop device, you can set this explicitly from a list of the available +## devices from the output of running 'aplay -L' + +REC_CARD="\$(sudo -u pi aplay -L \ + | grep dsnoop \ + | cut -d, -f1 \ + | grep -ve 'vc4' -e 'Head' -e 'PCH' \ + | uniq)" + +## PROCESSED is the directory where the formerly 'Analyzed' files are moved +## after extractions have been made from them. This includes both WAVE and +## BirdNET.selection.txt files. + +PROCESSED=${RECS_DIR}/Processed + +## EXTRACTED is the directory where the extracted audio selections are moved. + +EXTRACTED=${RECS_DIR}/Extracted + +## IDFILE is the file that keeps a complete list of every spececies that +## BirdNET has identified from your data-set. It is persistent across +## data-sets, so would need to be whiped clean through deleting or renaming +## it. A backup is automatically made from this variable each time it is +## updated (structure: ${IDFILE}.bak), and would also need to be removed +## or renamed to start a new file between data-sets. Alternately, you can +## change this variable between data-sets to preserve records of disparate +## data-sets according to name. + +IDFILE=${HOME}/BirdNET-Lite/IdentifiedSoFar.txt + +## OVERLAP is the value in seconds which BirdNET should use when analyzing +## the data. The values must be between 0.0-2.9. + +OVERLAP="0.0" + +## CONFIDENCE is the minimum confidence level from 0.0-1.0 BirdNET's analysis +## should reach before creating an entry in the BirdNET.selection.txt file. +## Don't set this to 1.0 or you won't have any results. + +CONFIDENCE="0.7" + +################################################################################ +#------------------------------ Auto-Generated ------------------------------# +#_____________________The variables below are auto-generated___________________# +#______________________________during installation_____________________________# + +## CHANNELS holds the variabel that corresponds to the number of channels the +## sound card supports. + +CHANNELS=${CHANNELS} + +# Don't touch the three below + +## ANALYZED is where the extraction.service looks for audio and +## BirdNET.selection.txt files after they have been processed by the +## birdnet_analysis.service. This is NOT where the analyzed files are moved -- +## analyzed files are always created within the same directory +## birdnet_analysis.service finds them. + +ANALYZED=${RECS_DIR}/*/*Analyzed + +## SYSTEMD_MOUNT is created from the RECS_DIR variable to comply with systemd +## mount naming requirements. + +SYSTEMD_MOUNT=$(echo ${RECS_DIR#/} | tr / -).mount + +## VENV is the virtual environment where the the BirdNET python build is found, +## i.e, VENV is the virtual environment miniforge built for BirdNET. + +VENV=$(dirname ${my_dir})/miniforge/envs/birdnet +EOF + [ -d /etc/birdnet ] || sudo mkdir /etc/birdnet + sudo ln -sf $(dirname ${my_dir})/birdnet.conf /etc/birdnet/birdnet.conf +} + +# Checks for a birdnet.conf file in the BirdNET-Lite directory for a +# non-interactive installation. Otherwise,the installation is interactive. +if [ -f ${BIRDNET_CONF} ];then + source ${BIRDNET_CONF} + install_birdnet_conf +else + configure + install_birdnet_conf +fi diff --git a/scripts/install_services.sh b/scripts/install_services.sh new file mode 100755 index 0000000..99795a3 --- /dev/null +++ b/scripts/install_services.sh @@ -0,0 +1,441 @@ +#!/usr/bin/env bash +# This installs the services that have been selected +#set -x # Uncomment to enable debugging +trap 'rm -f ${TMPFILE}' EXIT +trap 'exit 1' SIGINT SIGHUP +my_dir=$(realpath $(dirname $0)) +TMPFILE=$(mktemp) +nomachine_url="https://download.nomachine.com/download/7.6/Arm/nomachine_7.6.2_3_arm64.deb" +gotty_url="https://github.com/yudai/gotty/releases/download/v1.0.1/gotty_linux_arm.tar.gz" +CONFIG_FILE="$(dirname ${my_dir})/birdnet.conf" + +install_scripts() { + echo "Installing BirdNET-Lite scripts to /usr/local/bin" + ln -sf ${my_dir}/* /usr/local/bin/ +} + +install_birdnet_analysis() { + echo "Installing the birdnet_analysis.service" + cat << EOF > /etc/systemd/system/birdnet_analysis.service +[Unit] +Description=BirdNET Analysis +[Service] +Restart=always +RuntimeMaxSec=10800 +Type=simple +RestartSec=3 +User=${USER} +ExecStart=/usr/local/bin/birdnet_analysis.sh +[Install] +WantedBy=multi-user.target +EOF + systemctl enable birdnet_analysis.service +} + +install_extraction_service() { + echo "Installing the extraction.service and extraction.timer" + cat << EOF > /etc/systemd/system/extraction.service +[Unit] +Description=BirdNET BirdSound Extraction +[Service] +Restart=on-failure +RestartSec=3 +Type=simple +User=${USER} +ExecStart=/usr/local/bin/extract_new_birdsounds.sh +[Install] +WantedBy=multi-user.target +EOF + cat << EOF > /etc/systemd/system/extraction.timer +[Unit] +Description=BirdNET BirdSound Extraction Timer +Requires=extraction.service + +[Timer] +Unit=extraction.service +OnCalendar=*:0/10 + +[Install] +WantedBy=multi-user.target +EOF + systemctl enable extraction.timer + systemctl enable extraction.service + echo "Adding the species_updater.cron" + if ! crontab -u ${USER} -l &> /dev/null;then + crontab -u ${USER} $(dirname ${my_dir})/templates/species_updater.cron &> /dev/null + else + crontab -u ${USER} -l > ${TMPFILE} + cat $(dirname ${my_dir})/templates/species_updater.cron >> ${TMPFILE} + crontab -u ${USER} "${TMPFILE}" &> /dev/null + fi +} + +create_necessary_dirs() { + echo "Creating necessary directories" + [ -d ${EXTRACTED} ] || sudo -u ${USER} mkdir -p ${EXTRACTED} + [ -d ${EXTRACTED}/By_Date ] || sudo -u ${USER} mkdir -p ${EXTRACTED}/By_Date + [ -d ${EXTRACTED}/By_Species ] || sudo -u ${USER} mkdir -p ${EXTRACTED}/By_Species + [ -d ${PROCESSED} ] || sudo -u ${USER} mkdir -p ${PROCESSED} +} + +install_alsa() { + echo "Checking for alsa-utils" + if which arecord &> /dev/null ;then + echo "alsa-utils installed" + else + echo "Installing alsa-utils" + apt -qqq update + apt install -qqy alsa-utils + echo "alsa-utils installed" + fi +} + +install_recording_service() { + echo "Installing birdnet_recording.service" + cat << EOF > /etc/systemd/system/birdnet_recording.service +[Unit] +Description=BirdNET Recording + +[Service] +Environment=XDG_RUNTIME_DIR=/run/user/1000 +Restart=always +Type=simple +RestartSec=3 +User=${USER} +ExecStart=/usr/local/bin/birdnet_recording.sh + +[Install] +WantedBy=multi-user.target +EOF + systemctl enable birdnet_recording.service +} + + +install_sshfs() { + echo "Checking for SSHFS to mount remote filesystem" + if ! which sshfs &> /dev/null ;then + echo "Installing SSHFS" + apt -qqq update + apt install -qqqy sshfs + fi +} + +setup_sshkeys() { + echo "Setting up SSH keys for SSHFS" + echo "Adding remote host key to ${HOME}/.ssh/known_hosts" + ssh-keyscan -H ${REMOTE_HOST} >> ${HOME}/.ssh/known_hosts + chown ${USER}:${USER} ${HOME}/.ssh/known_hosts &> /dev/null + if [ ! -f ${HOME}/.ssh/id_ed25519.pub ];then + echo "Creating a new key" + ssh-keygen -t ed25519 -f ${HOME}/.ssh/id_ed25519 -P "" + fi + chown -R ${USER}:${USER} ${HOME}/.ssh/ &> /dev/null + echo "Copying public key to ${REMOTE_HOST}" + ssh-copy-id ${REMOTE_USER}@${REMOTE_HOST} +} + +install_systemd_mount() { + echo "Installing systemd.mount" + cat << EOF > /etc/systemd/system/${SYSTEMD_MOUNT} +[Unit] +Description=Mount remote fs with sshfs +DefaultDependencies=no +Conflicts=umount.target +After=network-online.target +Before=umount.target +Wants=network-online.target +[Install] +WantedBy=multi-user.target +[Mount] +What=${REMOTE_USER}@${REMOTE_HOST}:${REMOTE_RECS_DIR} +Where=${RECS_DIR} +Type=fuse.sshfs +Options=delay_connect,_netdev,allow_other,IdentityFile=${HOME}/.ssh/id_ed25519,reconnect,ServerAliveInterval=30,ServerAliveCountMax=5,x-systemd.automount,uid=1000,gid=1000 +TimeoutSec=60 +EOF +} + +install_caddy() { + if ! which caddy &> /dev/null ;then + echo "Installing Caddy" + curl -1sLf \ + 'https://dl.cloudsmith.io/public/caddy/stable/setup.deb.sh' \ + | sudo -E bash + apt -qq update + apt install -qqy caddy + systemctl enable --now caddy + else + echo "Caddy is installed" + systemctl enable --now caddy + fi +} + +install_Caddyfile() { + echo "Installing the Caddyfile" + [ -d /etc/caddy ] || mkdir /etc/caddy + cp $(dirname ${my_dir})/templates/index.html ${EXTRACTED}/ + if [ -f /etc/caddy/Caddyfile ];then + cp /etc/caddy/Caddyfile{,.original} + fi + HASHWORD=$(caddy hash-password -plaintext ${CADDY_PWD}) + cat << EOF > /etc/caddy/Caddyfile +${EXTRACTIONS_URL} { + root * ${EXTRACTED} + file_server browse + basicauth /Processed* { + birdnet ${HASHWORD} + } + basicauth /stream { + birdnet ${HASHWORD} + } + reverse_proxy /stream localhost:8000 +} + +http://birdnetsystem.local { + root * ${EXTRACTED} + file_server browse + basicauth /Processed* { + birdnet ${HASHWORD} + } + basicauth /stream { + birdnet ${HASHWORD} + } + reverse_proxy /stream localhost:8000 +} + +http://birdlog.local { + reverse_proxy localhost:8080 +} + +http://extractionlog.local { + reverse_proxy localhost:8888 +} + +http://birdstats.local { + reverse_proxy localhost:9090 +} +EOF + systemctl reload caddy +} + +install_avahi_aliases() { + echo "Installing Avahi Services" + if ! which avahi-publish &> /dev/null; then + echo "Installing avahi-utils" + apt install -y avahi-utils &> /dev/null + fi + echo "Installing avahi-alias service" + cat << 'EOF' > /etc/systemd/system/avahi-alias@.service +[Unit] +Description=Publish %I as alias for %H.local via mdns +After=network.target network-online.target +Requires=network-online.target + +[Service] +Restart=always +Type=simple +ExecStart=/bin/bash -c "/usr/bin/avahi-publish -a -R %I $(avahi-resolve -4 -n %H.local | cut -f 2)" + +[Install] +WantedBy=multi-user.target +EOF + systemctl enable --now avahi-alias@birdnetsystem.local.service + systemctl enable --now avahi-alias@birdlog.local.service + systemctl enable --now avahi-alias@extractionlog.local.service + systemctl enable --now avahi-alias@birdstats.local.service +} + +install_gotty_logs() { + echo "Installing GoTTY logging" + if ! which gotty &> /dev/null;then + echo "Installing GoTTY binary" + wget -c ${gotty_url} -O - | tar -xz -C /usr/local/bin/ + fi + sudo -u ${USER} ln -sf $(dirname ${my_dir})/templates/gotty \ + ${HOME}/.gotty + echo "Installing the birdnet_log.service" + cat << EOF > /etc/systemd/system/birdnet_log.service +[Unit] +Description=BirdNET Analysis Log + +[Service] +Restart=on-failure +RestartSec=3 +Type=simple +User=${USER} +Environment=TERM=xterm-256color +ExecStart=/usr/local/bin/gotty -p 8080 --title-format "BirdNET-Lite Log" journalctl -fu birdnet_analysis.service + +[Install] +WantedBy=multi-user.target +EOF + systemctl enable --now birdnet_log.service + echo "Installing the extraction_log.service" + cat << EOF > /etc/systemd/system/extraction_log.service +[Unit] +Description=BirdNET Extraction Log + +[Service] +Restart=on-failure +RestartSec=3 +Type=simple +User=${USER} +Environment=TERM=xterm-256color +ExecStart=/usr/local/bin/gotty -p 8888 --title-format "Extractions Log" journalctl -fu extraction.service + +[Install] +WantedBy=multi-user.target +EOF + systemctl enable --now extraction_log.service + echo "Installing the birdstats.service" + cat << EOF > /etc/systemd/system/birdstats.service +[Unit] +Description=BirdNET Statistics Log + +[Service] +Restart=on-failure +RestartSec=3 +Type=simple +User=${USER} +Environment=TERM=xterm-256color +ExecStart=/usr/local/bin/gotty -p 9090 --title-format "BirdNET-Lite Statistics" /usr/local/bin/birdnet_stats.sh + +[Install] +WantedBy=multi-user.target +EOF + systemctl enable --now birdstats.service +} + + +install_icecast() { + if ! which icecast2;then + echo "Installing IceCast2" + apt -qq update + echo "icecast2 icecast2/icecast-setup boolean false" | debconf-set-selections + apt install -qqy icecast2 + config_icecast + systemctl enable --now icecast2 + /etc/init.d/icecast2 start + else + echo "Icecast2 is installed" + config_icecast + systemctl enable --now icecast2 + /etc/init.d/icecast2 start + fi +} + +config_icecast() { + if [ -f /etc/icecast2/icecast.xml ];then + cp /etc/icecast2/icecast.xml{,.prebirdnetsystem} + fi + sed -i 's/>adminbirdnet.*<\/${i}password>/<${i}password>${ICE_PWD}<\/${i}password>/g" /etc/icecast2/icecast.xml + done +} + +install_livestream_service() { + echo "Installing Live Stream service" + cat << EOF > /etc/systemd/system/livestream.service +[Unit] +Description=BirdNET-Lite Live Stream + +[Service] +Environment=XDG_RUNTIME_DIR=/run/user/1000 +Restart=always +Type=simple +RestartSec=3 +User=${USER} +ExecStart=/usr/local/bin/livestream.sh + +[Install] +WantedBy=multi-user.target +EOF + systemctl enable --now livestream.service +} + +install_nomachine() { + echo "Installing NoMachine" + cd ~ + curl -s -O "${nomachine_url}" + apt install -y ${HOME}/nomachine_7.6.2_3_arm64.deb + rm -f ${HOME}/nomachine_7.6.2_3_arm64.deb +} + +install_systemd_overrides() { + for i in caddy birdnet_analysis extraction birdnet_recording;do + if [ -f /etc/systemd/system/${i}.service ];then + [ -d /etc/systemd/system/${i}.d ] || mkdir /etc/systemd/system/${i}.d + echo "Installing the systemd overrides.conf for the ${i}.service" + cat << EOF > /etc/systemd/system/${i}.d/overrides.conf +[Unit] +After=network.target network-online.target ${SYSTEMD_MOUNT} +Requires=network-online.target ${SYSTEMD_MOUNT} +EOF + fi + done +} + +install_cleanup_cron() { + echo "Installing the cleanup.cron" + if ! crontab -u ${USER} -l &> /dev/null;then + crontab -u ${USER} $(dirname ${my_dir})/templates/cleanup.cron &> /dev/null + else + crontab -u ${USER} -l > ${TMPFILE} + cat $(dirname ${my_dir})/templates/cleanup.cron >> ${TMPFILE} + crontab -u ${USER} "${TMPFILE}" &> /dev/null + fi +} + +install_selected_services() { + install_scripts + install_birdnet_analysis + + if [[ "${DO_EXTRACTIONS}" =~ [Yy] ]];then + install_extraction_service + create_necessary_dirs + fi + + if [[ "${DO_RECORDING}" =~ [Yy] ]];then + install_alsa + install_recording_service + fi + + if [[ "${REMOTE}" =~ [Yy] ]];then + install_sshfs + setup_sshkeys + install_systemd_mount + fi + + if [ ! -z "${EXTRACTIONS_URL}" ];then + install_caddy + install_Caddyfile + install_avahi_aliases + install_gotty_logs + fi + + if [ ! -z "${ICE_PWD}" ];then + install_icecast + install_livestream_service + fi + + if [ ! -z "${INSTALL_NOMACHINE}" ];then + install_nomachine + fi + + if [[ "${REMOTE}" =~ [Yy] ]];then + install_systemd_overrides + fi + + install_cleanup_cron +} + +if [ -f ${CONFIG_FILE} ];then + source ${CONFIG_FILE} + USER=${BIRDNET_USER} + HOME="$(getent passwd ${BIRDNET_USER} | cut -d: -f6)" + install_selected_services +else + echo "Unable to find a configuration file. Please make sure that $CONFIG_FILE exists." +fi diff --git a/scripts/install_tmux_services.sh b/scripts/install_tmux_services.sh new file mode 100755 index 0000000..30f470b --- /dev/null +++ b/scripts/install_tmux_services.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# This script installs a web-based terminal @ http://birdterminal.local +# ONLY run this additional script if you trust everyone on your +# local network completely as the credentials are sent WITHOUT +# any SSL/TLS encryption. For a secure remote connection to your +# BirdNET-Lite command line, consider enabling SSH on the +# Raspberry Pi and using another Linux machine or an SSH +# client software, or you can alternately add the 'tls internal' +# directive to the Caddyfile to add a self-signed certificate for TLS/SSL +# encryption. For remote desktop access, NoMachine can be installed. +source /etc/birdnet/birdnet.conf +my_dir=$(realpath $(dirname $0)) + +# Install tmux from version control +install_tmux() { + DEPENDS=( + automake + autoconf + libevent-2* + libevent-dev + ncurses-bin + ncurses-base + ncurses-term + libncurses-dev + build-essential + bison + pkg-config + gcc + ) + + if which tmux &>/dev/null; then + echo "tmux is installed" + else + + sudo apt update && sudo apt -y install "${DEPENDS[@]}" + + cd ${HOME} && git clone https://github.com/tmux/tmux.git + cd tmux + sh autogen.sh + ./configure && make && sudo make install + cd && rm -drf ./tmux + sudo ln -sf "$(dirname ${my_dir})/templates/tmux.conf" /etc/tmux.conf + fi +} + +install_web_terminal() { + cat << EOF | sudo tee /etc/systemd/system/birdterminal.service +[Unit] +Description=A BirdNET-Lite Web Terminal + +[Service] +Restart=on-failure +RestartSec=3 +Type=simple +User=${BIRDNET_USER} +Environment=TERM=xterm-256color +ExecStart=/usr/local/bin/gotty -w --title-format "Login!" -p 9111 tmux new -A -s Login sudo bash -c login + +[Install] +WantedBy=multi-user.target +EOF + HASHWORD="$(caddy hash-password -plaintext "${CADDY_PWD}")" + cat << EOF | sudo tee -a /etc/caddy/Caddyfile +http://birdterminal.local { + reverse_proxy localhost:9111 + basicauth { + birdnet "${HASHWORD}" + } +} +EOF + sudo systemctl enable --now birdterminal.service + sudo systemctl enable --now avahi-alias@birdterminal.local.service + sudo systemctl restart caddy +} +install_tmux +install_web_terminal diff --git a/scripts/install_zram_service.sh b/scripts/install_zram_service.sh new file mode 100755 index 0000000..edebb66 --- /dev/null +++ b/scripts/install_zram_service.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +echo "Configuring zram.service" +sudo touch /etc/modules-load.d/zram.conf +echo 'zram' | sudo tee /etc/modules-load.d/zram.conf +sudo touch /etc/modprobe.d/zram.conf +echo 'options zram num_devices=1' | sudo tee /etc/modprobe.d/zram.conf +sudo touch /etc/udev/rules.d/99-zram.rules +echo 'KERNEL=="zram0", ATTR{disksize}="4G",TAG+="systemd"' \ + | sudo tee /etc/udev/rules.d/99-zram.rules +sudo touch /etc/systemd/system/zram.service +echo "Installing zram.service" +cat << EOF | sudo tee /etc/systemd/system/zram.service +[Unit] +Description=Swap with zram +After=multi-user.target +[Service] +Type=oneshot +RemainAfterExit=true +ExecStartPre=/sbin/mkswap /dev/zram0 +ExecStart=/sbin/swapon /dev/zram0 +ExecStop=/sbin/swapoff /dev/zram0 +[Install] +WantedBy=multi-user.target +EOF +sudo systemctl enable zram +echo "You'll need to reboot for this to take effect." diff --git a/scripts/livestream.sh b/scripts/livestream.sh new file mode 100755 index 0000000..bdebba7 --- /dev/null +++ b/scripts/livestream.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Live Audio Stream Service Script +source /etc/birdnet/birdnet.conf + +if [ -z ${REC_CARD} ];then + echo "Stream not supported" +else + ffmpeg -loglevel 52 -ac ${CHANNELS} -f alsa -i ${REC_CARD} -acodec libmp3lame \ + -b:a 320k -ac ${CHANNELS} -content_type 'audio/mpeg' \ + -f mp3 icecast://source:${ICE_PWD}@localhost:8000/stream -re +fi diff --git a/scripts/pretty_date.sh b/scripts/pretty_date.sh new file mode 100755 index 0000000..b40d3bd --- /dev/null +++ b/scripts/pretty_date.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Pretty date suffixes +TODAY="$(date +%e)" + +if [[ $TODAY == ' 1' ]] || [[ $TODAY == 21 ]] || [[ $TODAY == 31 ]]; then + SUFFIX="st" +elif [[ $TODAY == ' 2' ]] || [[ $TODAY == 22 ]];then + SUFFIX="nd" +elif [[ $TODAY == ' 3' ]] || [[ $TODAY == 23 ]];then + SUFFIX="rd" +else + SUFFIX="th" +fi +echo "$SUFFIX" diff --git a/scripts/reconfigure_birdnet.sh b/scripts/reconfigure_birdnet.sh new file mode 100755 index 0000000..7c46dfb --- /dev/null +++ b/scripts/reconfigure_birdnet.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# Reconfigure the BirdNET-Lite +source /etc/birdnet/birdnet.conf +uninstall.sh +${HOME}/BirdNET-Lite/scripts/install_config.sh +sudo ${HOME}/BirdNET-Lite/scripts/install_services.sh +echo "BirdNET-Lite has now been reconfigured." diff --git a/scripts/species_notifier.sh b/scripts/species_notifier.sh new file mode 100755 index 0000000..d5b6b5c --- /dev/null +++ b/scripts/species_notifier.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Sends a notification if a new species is detected +# set -x +trap 'rm -f $TMPFILE' SIGINT SIGHUP EXIT + +source /etc/birdnet/birdnet.conf + +TMPFILE=$(mktemp) + +[ -f ${IDFILE} ] || touch ${IDFILE} +cat "${IDFILE}" > "${TMPFILE}" + +/usr/local/bin/update_species.sh &> /dev/null + +if ! diff "${IDFILE}" "${TMPFILE}"; then + SPECIES=("$(diff "${IDFILE}" "${TMPFILE}" \ + | awk '/ /dev/null +SCRIPTS=(/usr/local/bin/birdnet_analysis.sh +/usr/local/bin/birdnet_recording.sh +/usr/local/bin/birdnet_stats.sh +/usr/local/bin/cleanup.sh +/usr/local/bin/disk_usage.sh +/usr/local/bin/dump_logs.sh +/usr/local/bin/extract_new_birdsounds.sh +/usr/local/bin/install_birdnet.sh +/usr/local/bin/install_config.sh +/usr/local/bin/install_services.sh +/usr/local/bin/install_tmux_services.sh +/usr/local/bin/install_zram_service.sh +/usr/local/bin/livestream.sh +/usr/local/bin/pretty_date.sh +/usr/local/bin/reconfigure_birdnet.sh +/usr/local/bin/species_notifier.sh +/usr/local/bin/uninstall.sh +/usr/local/bin/update_species.sh +${HOME}/.gotty) + + +SERVICES=(avahi-alias@birdlog.local.service +avahi-alias@birdnetsystem.local.service +avahi-alias@birdstats.local.service +avahi-alias@extractionlog.local.service +avahi-alias@birdterminal.local.service +birdnet_analysis.d +birdnet_analysis.service +birdnet_log.service +birdnet_recording.d +birdnet_recording.service +birdstats.service +birdterminal.service +caddy.d +caddy.service +extraction_log.service +extraction.d +extraction.service +extraction.timer +livestream.service +${SYSTEMD_MOUNT}) + +remove_services() { + for i in "${SERVICES[@]}"; do + if [ -L /etc/systemd/system/multi-user.target.wants/"${i}" ];then + sudo systemctl disable --now "${i}" + fi + if [ -f /etc/systemd/system/"${i}" ];then + sudo rm /etc/systemd/system/"${i}" + fi + if [ -d /etc/systemd/system/"${i}" ];then + sudo rm -drf /etc/systemd/system/"${i}" + fi + done + remove_icecast + remove_crons +} + +remove_crons() { + TMPFILE=$(mktemp) + crontab -l | sed -e '/birdnet/,+1d' > "${TMPFILE}" + crontab "${TMPFILE}" +} + +remove_icecast() { + if [ -f /etc/init.d/icecast2 ];then + sudo /etc/init.d/icecast2 stop + sudo systemctl disable --now icecast2 + fi +} + +remove_scripts() { + for i in "${SCRIPTS[@]}";do + if [ -L "${i}" ];then + sudo rm -v "${i}" + fi + done +} + +remove_services +remove_scripts +if [ -d /etc/birdnet ];then sudo rm -drf /etc/birdnet;fi +if [ -f ${HOME}/BirdNET-Lite/birdnet.conf ];then sudo rm -f ${HOME}/BirdNET-Lite/birdnet.conf;fi +echo "Uninstall finished. Remove this directory with 'rm -drfv' to finish." diff --git a/scripts/update_species.sh b/scripts/update_species.sh new file mode 100755 index 0000000..d1c9214 --- /dev/null +++ b/scripts/update_species.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Update the species list +# set -x +trap 'rm -f "$TMPFILE"' SIGINT SIGTERM EXIT + +source /etc/birdnet/birdnet.conf + +TMPFILE=$(mktemp) || exit 1 + +[ -f ${IDFILE} ] || touch ${IDFILE} + +IDFILEBAKUP="${IDFILE}.bak" + +if [ $(find ${ANALYZED} -name '*txt' | wc -l) -ge 1 ];then + sort $(find ${ANALYZED} -name '*txt') \ + | awk '/Spect/ {for(i=11;i<=NF;++i)printf $i""FS ; print ""}' \ + | cut -d'0' -f1 \ + | sort -u > "$TMPFILE" + cat "$IDFILE" >> "$TMPFILE" + cp "$IDFILE" "$IDFILEBAKUP" + sort -u "$TMPFILE" > "$IDFILE" + cat "$IDFILE" +else + cat "$IDFILE" +fi diff --git a/templates/Caddyfile_livestream b/templates/Caddyfile_livestream new file mode 100644 index 0000000..f2c2768 --- /dev/null +++ b/templates/Caddyfile_livestream @@ -0,0 +1,31 @@ +${EXTRACTIONS_URL} { + root * ${EXTRACTED} + file_server browse + basicauth ${PROCESSED}* { + birdnet ${HASHWORD} + } + basicauth /stream { + birdnet ${HASHWORD} + } + reverse_proxy /stream localhost:8000 +} +http://birdnetsystem.local { + root * ${EXTRACTED} + file_server browse + basicauth ${PROCESSED}* { + birdnet ${HASHWORD} + } + basicauth /stream { + birdnet ${HASHWORD} + } + reverse_proxy /stream localhost:8000 +} +http://birdlog.local { + reverse_proxy localhost:8080 +} +http://extractionlog.local { + reverse_proxy localhost:8888 +} +http://birdterminal.local { + reverse_proxy localhost:9111 +} diff --git a/templates/Caddyfile_nostream b/templates/Caddyfile_nostream new file mode 100644 index 0000000..05b3a77 --- /dev/null +++ b/templates/Caddyfile_nostream @@ -0,0 +1,29 @@ +${EXTRACTIONS_URL} { + root * ${EXTRACTED} + file_server browse + basicauth ${PROCESSED}* { + birdnet ${HASHWORD} + } + basicauth /stream { + birdnet ${HASHWORD} + } +} +http://birdnetsystem.local { + root * ${EXTRACTED} + file_server browse + basicauth ${PROCESSED}* { + birdnet ${HASHWORD} + } + basicauth /stream { + birdnet ${HASHWORD} + } +} +http://birdlog.local { + reverse_proxy localhost:8080 +} +http://extractionlog.local { + reverse_proxy localhost:8888 +} +http://birdterminal.local { + reverse_proxy localhost:9111 +} diff --git a/templates/SYSTEMD_MOUNT-overrides.conf b/templates/SYSTEMD_MOUNT-overrides.conf new file mode 100644 index 0000000..e9182f7 --- /dev/null +++ b/templates/SYSTEMD_MOUNT-overrides.conf @@ -0,0 +1,3 @@ +[Unit] +After=network.target network-online.target ${SYSTEMD_MOUNT} +Requires=network-online.target ${SYSTEMD_MOUNT} diff --git a/templates/SYSTEMD_MOUNT.mount b/templates/SYSTEMD_MOUNT.mount new file mode 100644 index 0000000..957fb36 --- /dev/null +++ b/templates/SYSTEMD_MOUNT.mount @@ -0,0 +1,17 @@ +[Unit] +Description=Mount remote fs with sshfs +DefaultDependencies=no +Conflicts=umount.target +After=network-online.target +Before=umount.target +Wants=network-online.target + +[Install] +WantedBy=multi-user.target + +[Mount] +What="${REMOTE_USER}"@"${REMOTE_HOST}":"${REMOTE_RECS_DIR}" +Where="${RECS_DIR}" +Type=fuse.sshfs +Options=delay_connect,_netdev,allow_other,IdentityFile=${HOME}/.ssh/id_ed25519,reconnect,ServerAliveInterval=30,ServerAliveCountMax=5,x-systemd.automount,uid=1000,gid=1000 +TimeoutSec=60 diff --git a/templates/avahi-alias@.service b/templates/avahi-alias@.service new file mode 100644 index 0000000..6bcc3e7 --- /dev/null +++ b/templates/avahi-alias@.service @@ -0,0 +1,10 @@ +[Unit] +Description=Publish %I as alias for %H.local via mdns +After=network.target network-online.target +Requires=network-online.target +[Service] +Restart=always +Type=simple +ExecStart=/bin/bash -c "/usr/bin/avahi-publish -a -R %I $(avahi-resolve -4 -n %H.local | cut -f 2)" +[Install] +WantedBy=multi-user.target diff --git a/templates/birdnet_analysis.service b/templates/birdnet_analysis.service new file mode 100644 index 0000000..0986adb --- /dev/null +++ b/templates/birdnet_analysis.service @@ -0,0 +1,14 @@ +[Unit] +Description=BirdNET Analysis +Requires=${SYSTEMD_MOUNT} +After=network-online.target ${SYSTEMD_MOUNT} + +[Service] +Restart=always +Type=simple +RestartSec=3 +User=${BIRDNET_USER} +ExecStart=/usr/local/bin/birdnet_analysis.sh + +[Install] +WantedBy=multi-user.target diff --git a/templates/birdnet_log.service b/templates/birdnet_log.service new file mode 100644 index 0000000..75f5c75 --- /dev/null +++ b/templates/birdnet_log.service @@ -0,0 +1,11 @@ +[Unit] +Description=BirdNET Analysis Log +[Service] +Restart=on-failure +RestartSec=3 +Type=simple +User=${BIRDNET_USER} +Environment=TERM=xterm-256color +ExecStart=/usr/local/bin/gotty -p 8080 --title-format "BirdNET-Lite Log" journalctl -fu birdnet_analysis.service +[Install] +WantedBy=multi-user.target diff --git a/templates/birdnet_recording.service b/templates/birdnet_recording.service new file mode 100644 index 0000000..6f7c4f4 --- /dev/null +++ b/templates/birdnet_recording.service @@ -0,0 +1,11 @@ +[Unit] +Description=BirdNET Recording +[Service] +Environment=XDG_RUNTIME_DIR=/run/user/1000 +Restart=always +Type=simple +RestartSec=3 +User=${BIRDNET_USER} +ExecStart=/usr/local/bin/birdnet_recording.sh +[Install] +WantedBy=multi-user.target diff --git a/templates/birdstats.service b/templates/birdstats.service new file mode 100644 index 0000000..10ddf7e --- /dev/null +++ b/templates/birdstats.service @@ -0,0 +1,11 @@ +[Unit] +Description=BirdNET Statistics Log +[Service] +Restart=on-failure +RestartSec=3 +Type=simple +User=${BIRDNET_USER} +Environment=TERM=xterm-256color +ExecStart=/usr/local/bin/gotty -p 9090 --title-format "BirdNET-Lite Statistics" /usr/local/bin/birdnet_stats.sh +[Install] +WantedBy=multi-user.target diff --git a/templates/birdterminal.service b/templates/birdterminal.service new file mode 100644 index 0000000..d564374 --- /dev/null +++ b/templates/birdterminal.service @@ -0,0 +1,13 @@ +[Unit] +Description=A BirdNET-Lite Web Terminal + +[Service] +Restart=on-failure +RestartSec=3 +Type=simple +User=${BIRDNET_USER} +Environment=TERM=xterm-256color +ExecStart=/usr/local/bin/gotty -w --title-format "Login!" -p 9111 tmux new -A -s Login sudo bash -c login + +[Install] +WantedBy=multi-user.target diff --git a/templates/cleanup.cron b/templates/cleanup.cron new file mode 100644 index 0000000..f6002a0 --- /dev/null +++ b/templates/cleanup.cron @@ -0,0 +1,2 @@ +#birdnet +0 1 * * * /usr/local/bin/cleanup.sh &> /dev/null diff --git a/templates/extraction.service b/templates/extraction.service new file mode 100644 index 0000000..00d3f73 --- /dev/null +++ b/templates/extraction.service @@ -0,0 +1,10 @@ +[Unit] +Description=BirdNET BirdSound Extraction +[Service] +Restart=on-failure +RestartSec=3 +Type=simple +User=${BIRDNET_USER} +ExecStart=/usr/local/bin/extract_new_birdsounds.sh +[Install] +WantedBy=multi-user.target diff --git a/templates/extraction.timer b/templates/extraction.timer new file mode 100644 index 0000000..7816a0c --- /dev/null +++ b/templates/extraction.timer @@ -0,0 +1,8 @@ +[Unit] +Description=BirdNET BirdSound Extraction Timer +Requires=extraction.service +[Timer] +Unit=extraction.service +OnCalendar=*:0/10 +[Install] +WantedBy=multi-user.target diff --git a/templates/extraction_log.service b/templates/extraction_log.service new file mode 100644 index 0000000..76d87db --- /dev/null +++ b/templates/extraction_log.service @@ -0,0 +1,11 @@ +[Unit] +Description=BirdNET Extraction Log +[Service] +Restart=on-failure +RestartSec=3 +Type=simple +User=${BIRDNET_USER} +Environment=TERM=xterm-256color +ExecStart=/usr/local/bin/gotty -p 8888 --title-format "Extractions Log" journalctl -fu extraction.service +[Install] +WantedBy=multi-user.target diff --git a/templates/gotty b/templates/gotty new file mode 100644 index 0000000..e4fe5da --- /dev/null +++ b/templates/gotty @@ -0,0 +1,5 @@ +preferences { + font_size = 16 + foreground_color = "rgb(0, 0, 0)" + background_color = "rgb(119, 196, 135)" +} diff --git a/templates/index.html b/templates/index.html new file mode 100644 index 0000000..5ac0121 --- /dev/null +++ b/templates/index.html @@ -0,0 +1,445 @@ + + + + BirdNET-Lite + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+

+ BirdNET-Lite Extractions +

+
+
+
+
+ 2 directories + 2 files + +
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + Name + + Size + + Modified +
+ + + By_Date + + —
+ + + By_Species + + —
+ + + Processed + + 28 B
+ + + stream + + 0 B
+
+
+ + + + diff --git a/templates/livestream.service b/templates/livestream.service new file mode 100644 index 0000000..1c37350 --- /dev/null +++ b/templates/livestream.service @@ -0,0 +1,11 @@ +[Unit] +Description=BirdNET-Lite Live Stream +[Service] +Environment=XDG_RUNTIME_DIR=/run/user/1000 +Restart=always +Type=simple +RestartSec=3 +User=${BIRDNET_USER} +ExecStart=ffmpeg -loglevel 52 -ac 2 -f alsa -i ${REC_CARD} -acodec libmp3lame -b:a 320k -ac 2 -content_type 'audio/mpeg' -f mp3 icecast://source:${ICE_PWD}@localhost:8000/stream -re +[Install] +WantedBy=multi-user.target diff --git a/templates/species_updater.cron b/templates/species_updater.cron new file mode 100644 index 0000000..6f48223 --- /dev/null +++ b/templates/species_updater.cron @@ -0,0 +1,2 @@ +#birdnet +*/5 * * * * /usr/local/bin/species_notifier.sh &> /dev/null diff --git a/templates/tmux.conf b/templates/tmux.conf new file mode 100644 index 0000000..9af3364 --- /dev/null +++ b/templates/tmux.conf @@ -0,0 +1,31 @@ +# This is the tmux.conf +set-option -g base-index 1 +set-option -g pane-base-index 1 +set-option -g allow-rename off +set-option -g clock-mode-style 12 +set-option -g clock-mode-colour brightgreen +set-option -g window-status-current-style bg=brightgreen,fg=black +set-option -g window-status-separator '| |' +set-window-option -g mode-keys vi + + +set-option -g status-interval 5 +set-option -g status 2 +set-option -g status-format[1] '#[bg=cyan,fg=black]#[align=left][BirdNET-Lite] #[bg=cyan]#[align=centre]#(vcgencmd measure_temp) #(vcgencmd get_throttled)#[align=right]#(~/BirdNET-Lite/scripts/disk_usage.sh)%l:%M%p' +set-option -g status-style bg=magenta,fg=cyan +set-option -g status-justify centre +set-option -g status-left-length 50 +set-option -g status-left '#[fg=brightyellow]arm clock #(vcgencmd measure_clock arm)' +set-option -g status-right '#[fg=brightyellow]%A, %B %e#(~/BirdNET-Lite/scripts/pretty_date.sh)' + +bind r source-file /etc/tmux.conf \; display "Reloaded!" + +bind -n M-1 select-window -t 1 +bind -n M-2 select-window -t 2 +bind -n M-3 select-window -t 3 +bind -n M-4 select-window -t 4 +bind -n M-5 select-window -t 5 +bind -n M-6 select-window -t 6 +bind -n M-7 select-window -t 7 +bind -n M-8 select-window -t 8 +bind -n M-TAB next