claudecode/AvianVisitors
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

flickr db: keep a local cache of flickr urls to improve ui responsiveness

Browse Source
This commit is contained in:
frederik
2024-03-24 18:37:39 +01:00
parent 1c41fc3d8f
commit 0d5eadb880
3 changed files with 214 additions and 161 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/common.php
+172
View File
@@ -88,3 +88,175 @@ function debug_log($message) {
}
error_log($message . "\n", 3, $_SERVER['DOCUMENT_ROOT'] . "/debug_log.log");
}
define('DB', './scripts/flickr.db');
class Flickr {
private $flickr_api_key = null;
private $args = "&license=2%2C3%2C4%2C5%2C6%2C9&orientation=square,portrait";
private $blacklisted_ids = [];
private $db = null;
private $licenses_urls = [];
private $labels_flickr = null;
private $flickr_email = null;
private $comnameprefix = "%20bird";
public function __construct() {
$tbl_def = "CREATE TABLE images (sci_name VARCHAR(63) NOT NULL PRIMARY KEY, com_en_name VARCHAR(63) NOT NULL, image_url VARCHAR(63) NOT NULL, title VARCHAR(31) NOT NULL, id VARCHAR(31) NOT NULL UNIQUE, author_url VARCHAR(63) NOT NULL, license_url VARCHAR(63) NOT NULL, date_created DATE)";
try {
$db = new SQLite3(DB, SQLITE3_OPEN_READWRITE);
} catch (Exception $ex) {
$db = new SQLite3(DB);
$db->exec($tbl_def);
$db->exec('CREATE TABLE source (ID INTEGER PRIMARY KEY, email VARCHAR(63), uid VARCHAR(63), date_created DATE)');
}
$db->busyTimeout(1000);
$this->db = $db;
$blacklisted = get_home() . "/BirdNET-Pi/scripts/blacklisted_images.txt";
if (file_exists($blacklisted)) {
$blacklisted_file = file($blacklisted);
if ($blacklisted_file) {
$this->blacklisted_ids = array_map('trim', $blacklisted_file);
}
}
$this->flickr_api_key = get_config()["FLICKR_API_KEY"];
$this->flickr_email = get_config()["FLICKR_FILTER_EMAIL"];
$source = $this->get_uid_from_db();
if ($source['email'] !== $this->flickr_email) {
// reset the DB
$this->db->exec("DROP TABLE images; " . $tbl_def);
if (!empty($this->flickr_email)) {
$source = $this->get_uid_from_db();
if ($source['email'] !== $this->flickr_email) {
$this->get_uid_from_flickr();
$source = $this->get_uid_from_db();
}
} else {
$this->set_uid_in_db("");
}
}
if (!empty($this->flickr_email)) {
$this->args = "&user_id=" . $source['uid'];
$this->comnameprefix = "";
}
}
public function get_image($sci_name) {
$image = $this->get_image_from_db($sci_name);
if ($image !== false && in_array($image['id'], $this->blacklisted_ids)) {
$image = false;
}
if ($image !== false) {
$now = new DateTime();
$datetime = DateTime::createFromFormat("Y-m-d", $image['date_created']);
$interval = $now->diff($datetime);
// use the last digit of the id as a semi random number, so not all entries expire at the same time
$expire_days = 15 + intval($image['id'][-1]);
if ($interval->days > $expire_days) {
$image = false;
}
}
if ($image === false) {
$this->get_from_flickr($sci_name);
$image = $this->get_image_from_db($sci_name);
}
return $image;
}
private function get_image_from_db($sci_name) {
$statement0 = $this->db->prepare('SELECT sci_name, com_en_name, image_url, title, id, author_url, license_url, date_created FROM images WHERE sci_name == :sci_name');
$statement0->bindValue(':sci_name', $sci_name);
$result = $statement0->execute();
$row = $result->fetchArray(SQLITE3_ASSOC);
return $row;
}
private function set_image_in_db($sci_name, $com_en_name, $image_url, $title, $id, $author_url, $license_url) {
$statement0 = $this->db->prepare("INSERT OR REPLACE INTO images VALUES (:sci_name, :com_en_name, :image_url, :title, :id, :author_url, :license_url, DATE(\"now\"))");
$statement0->bindValue(':sci_name', $sci_name);
$statement0->bindValue(':com_en_name', $com_en_name);
$statement0->bindValue(':image_url', $image_url);
$statement0->bindValue(':title', $title);
$statement0->bindValue(':id', $id);
$statement0->bindValue(':author_url', $author_url);
$statement0->bindValue(':license_url', $license_url);
$statement0->execute();
}
private function get_from_flickr($sci_name) {
$engname = $this->get_com_en_name($sci_name);
$flickrjson = json_decode(file_get_contents("https://www.flickr.com/services/rest/?method=flickr.photos.search&api_key=" . $this->flickr_api_key . "&text=" . str_replace(" ", "%20", $engname) . $this->comnameprefix . "&sort=relevance" . $this->args . "&per_page=5&media=photos&format=json&nojsoncallback=1"), true)["photos"]["photo"];
// could be null!!
// Find the first photo that is not blacklisted or is not the specific blacklisted id
$photo = null;
foreach ($flickrjson as $flickrphoto) {
if ($flickrphoto["id"] !== "4892923285" && !in_array($flickrphoto["id"], $this->blacklisted_ids)) {
$photo = $flickrphoto;
break;
}
}
$license_url = "https://api.flickr.com/services/rest/?method=flickr.photos.getInfo&api_key=" . $this->flickr_api_key . "&photo_id=" . $photo["id"] . "&format=json&nojsoncallback=1";
$license_response = file_get_contents($license_url);
$license_id = json_decode($license_response, true)["photo"]["license"];
$license_url = $this->get_license_url($license_id);
$authorlink = "https://flickr.com/people/" . $photo["owner"];
$imageurl = 'https://farm' . $photo["farm"] . '.static.flickr.com/' . $photo["server"] . '/' . $photo["id"] . '_' . $photo["secret"] . '.jpg';
$this->set_image_in_db($sci_name, $engname, $imageurl, $photo["title"], $photo["id"], $authorlink, $license_url);
}
private function get_license_url($id) {
if (empty($this->licenses_urls)) {
$licenses_url = "https://api.flickr.com/services/rest/?method=flickr.photos.licenses.getInfo&api_key=" . $this->flickr_api_key . "&format=json&nojsoncallback=1";
$licenses_response = file_get_contents($licenses_url);
$licenses_data = json_decode($licenses_response, true)["licenses"]["license"];
foreach ($licenses_data as $license) {
$license_id = $license["id"];
$license_url = $license["url"];
$this->licenses_urls[$license_id] = $license_url;
}
}
return $this->licenses_urls[$id];
}
public function get_uid_from_db() {
$statement0 = $this->db->prepare('SELECT email, uid, date_created FROM source');
$result = $statement0->execute();
$row = $result->fetchArray(SQLITE3_ASSOC);
return $row;
}
private function set_uid_in_db($uid) {
$statement0 = $this->db->prepare("INSERT OR REPLACE INTO source VALUES (1, :email, :uid, DATE(\"now\"))");
$statement0->bindValue(':email', $this->flickr_email);
$statement0->bindValue(':uid', $uid);
$result = $statement0->execute();
$row = $result->fetchArray(SQLITE3_ASSOC);
return $row;
}
private function get_uid_from_flickr() {
$uid = json_decode(file_get_contents("https://www.flickr.com/services/rest/?method=flickr.people.findByEmail&api_key=" . $this->flickr_api_key . "&find_email=" . $this->flickr_email . "&format=json&nojsoncallback=1"), true)["user"]["nsid"];
$this->set_uid_in_db($uid);
}
private function get_com_en_name($sci_name) {
if ($this->labels_flickr === null) {
$this->labels_flickr = file(get_home() . "/BirdNET-Pi/model/labels_flickr.txt");
}
$engname = null;
foreach ($this->labels_flickr as $label) {
if (strpos($label, $sci_name) !== false) {
$engname = trim(explode("_", $label)[1]);
break;
}
}
return $engname;
}
}
scripts/overview.php
+22 -82
View File
@@ -61,105 +61,45 @@ if(isset($_GET['ajax_detections']) && $_GET['ajax_detections'] == "true" && isse
$_SESSION['images'] = [];
}
$iterations = 0;
$lines;
$licenses_urls = array();
$flickr = null;
// hopefully one of the 5 most recent detections has an image that is valid, we'll use that one as the most recent detection until the newer ones get their images created
while($mostrecent = $result4->fetchArray(SQLITE3_ASSOC)) {
$comname = preg_replace('/ /', '_', $mostrecent['Com_Name']);
$sciname = preg_replace('/ /', '_', $mostrecent['Sci_Name']);
$comname = preg_replace('/\'/', '', $comname);
$filename = "/By_Date/".$mostrecent['Date']."/".$comname."/".$mostrecent['File_Name'];
$args = "&license=2%2C3%2C4%2C5%2C6%2C9&orientation=square,portrait";
$comnameprefix = "%20bird";
// check to make sure the image actually exists, sometimes it takes a minute to be created\
if(file_exists($home."/BirdSongs/Extracted".$filename.".png")){
if($_GET['previous_detection_identifier'] == $filename) { die(); }
if($_GET['only_name'] == "true") { echo $comname.",".$filename;die(); }
$iterations++;
// check to make sure the image actually exists, sometimes it takes a minute to be created\
if(file_exists($home."/BirdSongs/Extracted".$filename.".png")){
if($_GET['previous_detection_identifier'] == $filename) { die(); }
if($_GET['only_name'] == "true") { echo $comname.",".$filename;die(); }
$iterations++;
if (!empty($config["FLICKR_API_KEY"])) {
if(!empty($config["FLICKR_FILTER_EMAIL"])) {
if(!isset($_SESSION["FLICKR_FILTER_EMAIL"])) {
unset($_SESSION['images']);
$_SESSION['FLICKR_FILTER_EMAIL'] = json_decode(file_get_contents("https://www.flickr.com/services/rest/?method=flickr.people.findByEmail&api_key=".$config["FLICKR_API_KEY"]."&find_email=".$config["FLICKR_FILTER_EMAIL"]."&format=json&nojsoncallback=1"), true)["user"]["nsid"];
}
$args = "&user_id=".$_SESSION['FLICKR_FILTER_EMAIL'];
$comnameprefix = "";
} else {
if(isset($_SESSION["FLICKR_FILTER_EMAIL"])) {
unset($_SESSION["FLICKR_FILTER_EMAIL"]);
unset($_SESSION['images']);
}
if ($flickr === null) {
$flickr = new Flickr();
}
if ($_SESSION["FLICKR_FILTER_EMAIL"] !== $flickr->get_uid_from_db()['uid']) {
if (isset($_SESSION["FLICKR_FILTER_EMAIL"])) {
$_SESSION['images'] = [];
}
$_SESSION["FLICKR_FILTER_EMAIL"] = $flickr->get_uid_from_db()['uid'];
}
// if we already searched flickr for this species before, use the previous image rather than doing an unneccesary api call
$key = array_search($comname, array_column($_SESSION['images'], 0));
if($key !== false) {
if ($key !== false) {
$image = $_SESSION['images'][$key];
} else {
// Get license information if we haven't already
if (empty($licenses_urls)) {
$licenses_url = "https://api.flickr.com/services/rest/?method=flickr.photos.licenses.getInfo&api_key=".$config["FLICKR_API_KEY"]."&format=json&nojsoncallback=1";
$licenses_response = file_get_contents($licenses_url);
$licenses_data = json_decode($licenses_response, true)["licenses"]["license"];
foreach ($licenses_data as $license) {
$license_id = $license["id"];
$license_name = $license["name"];
$license_url = $license["url"];
$licenses_urls[$license_id] = $license_url;
}
}
// only open the file once per script execution
if(!isset($lines)) {
$lines = file($home."/BirdNET-Pi/model/labels_flickr.txt");
}
// convert sci name to English name
foreach($lines as $line){
if(strpos($line, $mostrecent['Sci_Name']) !== false){
$engname = trim(explode("_", $line)[1]);
break;
}
}
// Read the blacklisted image ids from the file into an array
$blacklisted_file = file($home."/BirdNET-Pi/scripts/blacklisted_images.txt");
if ($blacklisted_file !== false) {
$blacklisted_ids = array_map('trim', $blacklisted_file);
} else {
$blacklisted_ids = [];
}
// Make the API call
$flickrjson = json_decode(file_get_contents("https://www.flickr.com/services/rest/?method=flickr.photos.search&api_key=".$config["FLICKR_API_KEY"]."&text=".str_replace(" ", "%20", $engname).$comnameprefix."&sort=relevance".$args."&per_page=5&media=photos&format=json&nojsoncallback=1"), true)["photos"]["photo"];
// Find the first photo that is not blacklisted or is not the specific blacklisted id
$photo = null;
foreach ($flickrjson as $flickrphoto) {
if ($flickrphoto["id"] !== "4892923285" && !in_array($flickrphoto["id"], $blacklisted_ids)) {
$photo = $flickrphoto;
break;
}
}
$license_url = "https://api.flickr.com/services/rest/?method=flickr.photos.getInfo&api_key=".$config["FLICKR_API_KEY"]."&photo_id=".$photo["id"]."&format=json&nojsoncallback=1";
$license_response = file_get_contents($license_url);
$license_info = json_decode($license_response, true)["photo"]["license"];
$license_url = $licenses_urls[$license_info];
$modaltext = "https://flickr.com/photos/".$photo["owner"]."/".$photo["id"];
$authorlink = "https://flickr.com/people/".$photo["owner"];
$imageurl = 'https://farm' .$photo["farm"]. '.static.flickr.com/' .$photo["server"]. '/' .$photo["id"]. '_' .$photo["secret"]. '.jpg';
array_push($_SESSION['images'], array($comname,$imageurl,$photo["title"], $modaltext, $authorlink, $license_url));
$image = $_SESSION['images'][count($_SESSION['images'])-1];
$flickr_cache = $flickr->get_image($mostrecent['Sci_Name']);
$modaltext = $flickr_cache["author_url"] . "/" . $flickr_cache["id"];
array_push($_SESSION["images"], array($comname, $flickr_cache["image_url"], $flickr_cache["title"], $modaltext, $flickr_cache["author_url"], $flickr_cache["license_url"]));
$image = $_SESSION['images'][count($_SESSION['images']) - 1];
}
}
?>
?>
<style>
.fade-in {
opacity: 1;
scripts/todays_detections.php
+20 -79
View File
@@ -184,97 +184,38 @@ if(isset($_GET['ajax_detections']) && $_GET['ajax_detections'] == "true" ) {
$_SESSION['images'] = [];
}
$iterations = 0;
$lines=null;
$licenses_urls = array();
$flickr = null;
while($todaytable=$result0->fetchArray(SQLITE3_ASSOC))
{
$iterations++;
$comname = preg_replace('/ /', '_', $todaytable['Com_Name']);
$comname = preg_replace('/\'/', '', $comname);
$filename = "/By_Date/".date('Y-m-d')."/".$comname."/".$todaytable['File_Name'];
$filename_formatted = $todaytable['Date']."/".$comname."/".$todaytable['File_Name'];
$sciname = preg_replace('/ /', '_', $todaytable['Sci_Name']);
$args = "&license=2%2C3%2C4%2C5%2C6%2C9&orientation=square,portrait";
$comnameprefix = "%20bird";
if (!empty($config["FLICKR_API_KEY"]) && (isset($_GET['display_limit']) || isset($_GET['hard_limit']) || $_GET['kiosk'] == true) ) {
$comname = preg_replace('/ /', '_', $todaytable['Com_Name']);
$comname = preg_replace('/\'/', '', $comname);
$filename = "/By_Date/".date('Y-m-d')."/".$comname."/".$todaytable['File_Name'];
$filename_formatted = $todaytable['Date']."/".$comname."/".$todaytable['File_Name'];
$sciname = preg_replace('/ /', '_', $todaytable['Sci_Name']);
if(!empty($config["FLICKR_FILTER_EMAIL"])) {
if(!isset($_SESSION["FLICKR_FILTER_EMAIL"])) {
if (!empty($config["FLICKR_API_KEY"]) && (isset($_GET['display_limit']) || isset($_GET['hard_limit']) || $_GET['kiosk'] == true) ) {
if ($flickr === null) {
$flickr = new Flickr();
}
if (isset($_SESSION["FLICKR_FILTER_EMAIL"]) && $_SESSION["FLICKR_FILTER_EMAIL"] !== $flickr->get_uid_from_db()['uid']) {
unset($_SESSION['images']);
$_SESSION['FLICKR_FILTER_EMAIL'] = json_decode(file_get_contents("https://www.flickr.com/services/rest/?method=flickr.people.findByEmail&api_key=".$config["FLICKR_API_KEY"]."&find_email=".$config["FLICKR_FILTER_EMAIL"]."&format=json&nojsoncallback=1"), true)["user"]["nsid"];
}
$args = "&user_id=".$_SESSION['FLICKR_FILTER_EMAIL'];
$comnameprefix = "";
} else {
if(isset($_SESSION["FLICKR_FILTER_EMAIL"])) {
unset($_SESSION["FLICKR_FILTER_EMAIL"]);
unset($_SESSION['images']);
}
}
// if we already searched flickr for this species before, use the previous image rather than doing an unneccesary api call
$key = array_search($comname, array_column($_SESSION['images'], 0));
if($key !== false) {
$image = $_SESSION['images'][$key];
} else {
// Get license information if we haven't already
if (empty($licenses_urls)) {
$licenses_url = "https://api.flickr.com/services/rest/?method=flickr.photos.licenses.getInfo&api_key=".$config["FLICKR_API_KEY"]."&format=json&nojsoncallback=1";
$licenses_response = file_get_contents($licenses_url);
$licenses_data = json_decode($licenses_response, true)["licenses"]["license"];
foreach ($licenses_data as $license) {
$license_id = $license["id"];
$license_name = $license["name"];
$license_url = $license["url"];
$licenses_urls[$license_id] = $license_url;
}
$_SESSION["FLICKR_FILTER_EMAIL"] = $flickr->get_uid_from_db()['uid'];
}
// only open the file once per script execution
if(!isset($lines)) {
$lines = file($home."/BirdNET-Pi/model/labels_flickr.txt");
}
// convert sci name to English name
foreach($lines as $line){
if(strpos($line, $todaytable['Sci_Name']) !== false){
$engname = trim(explode("_", $line)[1]);
break;
}
}
// Read the blacklisted image ids from the file into an array
$blacklisted_file = file($home."/BirdNET-Pi/scripts/blacklisted_images.txt");
if ($blacklisted_file) {
$blacklisted_ids = array_map('trim', $blacklisted_file);
// if we already searched flickr for this species before, use the previous image rather than doing an unneccesary api call
$key = array_search($comname, array_column($_SESSION['images'], 0));
if ($key !== false) {
$image = $_SESSION['images'][$key];
} else {
$blacklisted_ids = [];
$flickr_cache = $flickr->get_image($todaytable['Sci_Name']);
$modaltext = $flickr_cache["author_url"] . "/" . $flickr_cache["id"];
array_push($_SESSION["images"], array($comname, $flickr_cache["image_url"], $flickr_cache["title"], $modaltext, $flickr_cache["author_url"], $flickr_cache["license_url"]));
$image = $_SESSION['images'][count($_SESSION['images']) - 1];
}
// Make the API call
$flickrjson = json_decode(file_get_contents("https://www.flickr.com/services/rest/?method=flickr.photos.search&api_key=".$config["FLICKR_API_KEY"]."&text=".str_replace(" ", "%20", $engname).$comnameprefix."&sort=relevance".$args."&per_page=5&media=photos&format=json&nojsoncallback=1"), true)["photos"]["photo"];
// Find the first photo that is not blacklisted or is not the specific blacklisted id
$photo = null;
foreach ($flickrjson as $flickrphoto) {
if ($flickrphoto["id"] !== "4892923285" && !in_array($flickrphoto["id"], $blacklisted_ids)) {
$photo = $flickrphoto;
break;
}
}
$license_url = "https://api.flickr.com/services/rest/?method=flickr.photos.getInfo&api_key=".$config["FLICKR_API_KEY"]."&photo_id=".$photo["id"]."&format=json&nojsoncallback=1";
$license_response = file_get_contents($license_url);
$license_info = json_decode($license_response, true)["photo"]["license"];
$license_url = $licenses_urls[$license_info];
$modaltext = "https://flickr.com/photos/".$photo["owner"]."/".$photo["id"];
$authorlink = "https://flickr.com/people/".$photo["owner"];
$imageurl = 'https://farm' .$photo["farm"]. '.static.flickr.com/' .$photo["server"]. '/' .$photo["id"]. '_' .$photo["secret"]. '.jpg';
array_push($_SESSION['images'], array($comname,$imageurl,$photo["title"], $modaltext, $authorlink, $license_url));
$image = $_SESSION['images'][count($_SESSION['images'])-1];
}
}
?>
<?php if(isset($_GET['display_limit']) && is_numeric($_GET['display_limit'])){ ?>
<tr class="relative" id="<?php echo $iterations; ?>">