Lazy load local files and threshold
This commit is contained in:
Alexandre
+96
-77
@@ -7,14 +7,44 @@ require_once __DIR__ . '/common.php';
|
||||
ensure_authenticated();
|
||||
|
||||
$home = get_home();
|
||||
// Open database read-only for typical operations; enable writes only for deletions
|
||||
|
||||
/* ---------- disk species counts (AJAX endpoint) ---------- */
|
||||
if (isset($_GET['diskcounts'])) {
|
||||
header('Content-Type: application/json');
|
||||
$script = __DIR__ . '/disk_species_count.sh';
|
||||
$cmd = 'HOME=' . escapeshellarg($home) . ' bash ' . escapeshellarg($script) . ' 2>&1';
|
||||
$output = @shell_exec($cmd);
|
||||
$counts = [];
|
||||
if ($output !== null) {
|
||||
foreach (preg_split('/\\r?\\n/', $output) as $line) {
|
||||
$line = trim($line);
|
||||
if ($line === '') continue;
|
||||
if (preg_match('/^([0-9]+(?:\\.[0-9]+)?)(k?)\\s*:\\s*(.+)$/i', $line, $m)) {
|
||||
$num = (float)$m[1];
|
||||
if (strtolower($m[2]) === 'k') $num *= 1000;
|
||||
$counts[$m[3]] = (int)round($num);
|
||||
}
|
||||
}
|
||||
}
|
||||
echo json_encode($counts, JSON_UNESCAPED_UNICODE);
|
||||
exit;
|
||||
}
|
||||
|
||||
/* ---------- DB open (RO unless deleting) ---------- */
|
||||
$flags = isset($_GET['delete']) ? SQLITE3_OPEN_READWRITE : SQLITE3_OPEN_READONLY;
|
||||
$db = new SQLite3(__DIR__ . '/birds.db', $flags);
|
||||
$db->busyTimeout(1000);
|
||||
$db->exec("
|
||||
PRAGMA journal_mode=WAL; -- safe read concurrency
|
||||
PRAGMA synchronous=NORMAL; -- cheaper fsyncs (read-mostly)
|
||||
PRAGMA temp_store=MEMORY; -- temp data stays in RAM
|
||||
PRAGMA cache_size=-80000; -- ~80MB page cache (tune to your RAM)
|
||||
PRAGMA mmap_size=268435456; -- 256MB mmap (set 0 if kernel disallows)
|
||||
");
|
||||
|
||||
/* Paths / lists */
|
||||
$base_symlink = $home . '/BirdSongs/Extracted/By_Date';
|
||||
$base = realpath($base_symlink); // used only for safety checks
|
||||
$base = realpath($base_symlink); // safety checks
|
||||
|
||||
$confirm_file = __DIR__ . '/confirmed_species_list.txt';
|
||||
$exclude_file = __DIR__ . '/exclude_species_list.txt';
|
||||
@@ -32,16 +62,11 @@ $config = get_config();
|
||||
$sf_thresh = isset($config['SF_THRESH']) ? (float)$config['SF_THRESH'] : 0.0;
|
||||
|
||||
/* ---------- helpers ---------- */
|
||||
function join_path(...$parts): string {
|
||||
return preg_replace('#/+#', '/', implode('/', $parts));
|
||||
}
|
||||
function can_unlink(string $p): bool {
|
||||
return is_link($p) || is_file($p);
|
||||
}
|
||||
function join_path(...$parts): string { return preg_replace('#/+#', '/', implode('/', $parts)); }
|
||||
function can_unlink(string $p): bool { return is_link($p) || is_file($p); }
|
||||
function under_base(string $path, string $base): bool {
|
||||
if ($base === false) return false;
|
||||
$baseReal = rtrim(realpath($base) ?: $base, DIRECTORY_SEPARATOR);
|
||||
|
||||
$resolved = realpath($path);
|
||||
if ($resolved === false) {
|
||||
$parent = realpath(dirname($path));
|
||||
@@ -51,50 +76,31 @@ function under_base(string $path, string $base): bool {
|
||||
return $resolved === $baseReal || strpos($resolved, $baseReal . DIRECTORY_SEPARATOR) === 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Collect detection count, files to delete (unique), first scientific name,
|
||||
* and dirs to try rmdir later — for a given species.
|
||||
* NOTE: Row-wise enumeration (no aggregates) so we gather every file.
|
||||
*/
|
||||
/* Collect files/dirs for a species */
|
||||
function collect_species_targets(SQLite3 $db, string $species, string $home, $base): array {
|
||||
$stmt = $db->prepare('SELECT Date, Com_Name, Sci_Name, File_Name
|
||||
FROM detections
|
||||
WHERE Com_Name = :name');
|
||||
$stmt = $db->prepare('SELECT Date, Com_Name, Sci_Name, File_Name FROM detections WHERE Com_Name = :name');
|
||||
ensure_db_ok($stmt);
|
||||
$stmt->bindValue(':name', $species, SQLITE3_TEXT);
|
||||
$res = $stmt->execute();
|
||||
|
||||
$count = 0; $files = []; $dirs = []; $sci = null;
|
||||
|
||||
while ($row = $res->fetchArray(SQLITE3_ASSOC)) {
|
||||
$count++;
|
||||
if ($sci === null) $sci = $row['Sci_Name'];
|
||||
$count++; if ($sci === null) $sci = $row['Sci_Name'];
|
||||
$dir = str_replace([' ', "'"], ['_', ''], $row['Com_Name']);
|
||||
|
||||
$candidates = [
|
||||
join_path($home, 'BirdSongs/Extracted/By_Date', $row['Date'], $dir, $row['File_Name']),
|
||||
join_path($home, 'BirdSongs/Extracted/By_Date/shifted', $row['Date'], $dir, $row['File_Name']),
|
||||
];
|
||||
|
||||
foreach ($candidates as $c) {
|
||||
if (can_unlink($c) && under_base($c, $base)) {
|
||||
$files[$c] = true; $dirs[] = dirname($c); continue;
|
||||
}
|
||||
if (can_unlink($c) && under_base($c, $base)) { $files[$c] = true; $dirs[] = dirname($c); continue; }
|
||||
$d = realpath(dirname($c));
|
||||
if ($d !== false) {
|
||||
$alt = $d . DIRECTORY_SEPARATOR . basename($c);
|
||||
if (can_unlink($alt) && under_base($alt, $base)) {
|
||||
$files[$alt] = true; $dirs[] = dirname($alt);
|
||||
if (can_unlink($alt) && under_base($alt, $base)) { $files[$alt] = true; $dirs[] = dirname($alt); }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return [
|
||||
'count' => $count,
|
||||
'files' => array_keys($files),
|
||||
'dirs' => array_values(array_unique($dirs)),
|
||||
'sci' => $sci,
|
||||
];
|
||||
return ['count'=>$count, 'files'=>array_keys($files), 'dirs'=>array_values(array_unique($dirs)), 'sci'=>$sci];
|
||||
}
|
||||
|
||||
/* ---------- toggle exclude/whitelist/confirmed ---------- */
|
||||
@@ -117,7 +123,7 @@ if (isset($_GET['toggle'], $_GET['species'], $_GET['action'])) {
|
||||
header('Content-Type: text/plain'); echo 'OK'; exit;
|
||||
}
|
||||
|
||||
/* ---------- count (keeps your old "getcounts=" API) ---------- */
|
||||
/* ---------- count ---------- */
|
||||
if (isset($_GET['getcounts'])) {
|
||||
header('Content-Type: application/json');
|
||||
if ($base === false) { http_response_code(500); exit(json_encode(['error' => 'Base directory not found'])); }
|
||||
@@ -126,7 +132,7 @@ if (isset($_GET['getcounts'])) {
|
||||
echo json_encode(['count' => $info['count'], 'files' => count($info['files'])]); exit;
|
||||
}
|
||||
|
||||
/* ---------- delete (keeps your old "delete=" API) ---------- */
|
||||
/* ---------- delete ---------- */
|
||||
if (isset($_GET['delete'])) {
|
||||
header('Content-Type: application/json');
|
||||
if ($base === false) { http_response_code(500); exit(json_encode(['error' => 'Base directory not found'])); }
|
||||
@@ -138,24 +144,19 @@ if (isset($_GET['delete'])) {
|
||||
if (!under_base($fp, $base)) continue;
|
||||
if (can_unlink($fp) && @unlink($fp)) {
|
||||
$deleted++;
|
||||
// thumbnails: "file.wav.png" and "file.png"
|
||||
foreach ([$fp . '.png', preg_replace('/\.[^.]+$/', '.png', $fp)] as $png) {
|
||||
if (can_unlink($png)) @unlink($png);
|
||||
}
|
||||
}
|
||||
}
|
||||
foreach ($info['dirs'] as $dir) {
|
||||
if (under_base($dir, $base)) @rmdir($dir); // best effort
|
||||
}
|
||||
foreach ($info['dirs'] as $dir) { if (under_base($dir, $base)) @rmdir($dir); }
|
||||
|
||||
// DB rows
|
||||
$del = $db->prepare('DELETE FROM detections WHERE Com_Name = :name');
|
||||
ensure_db_ok($del);
|
||||
$del->bindValue(':name', $species, SQLITE3_TEXT);
|
||||
$del->execute();
|
||||
$lines_deleted = $db->changes();
|
||||
|
||||
// Remove from confirmed list
|
||||
if ($info['sci'] !== null && file_exists($confirm_file)) {
|
||||
$identifier = str_replace("'", '', $info['sci']);
|
||||
$lines = array_values(array_filter($confirmed_species, fn($l) => $l !== $identifier));
|
||||
@@ -167,15 +168,9 @@ if (isset($_GET['delete'])) {
|
||||
|
||||
/* ---------- query species aggregates ---------- */
|
||||
$sql = <<<SQL
|
||||
SELECT
|
||||
Com_Name,
|
||||
Sci_Name,
|
||||
COUNT(*) AS Count,
|
||||
MAX(Confidence) AS MaxConfidence,
|
||||
MAX(Date) AS LastSeen
|
||||
SELECT Com_Name, Sci_Name, COUNT(*) AS Count, MAX(Confidence) AS MaxConfidence, MAX(Date) AS LastSeen
|
||||
FROM detections
|
||||
GROUP BY Com_Name, Sci_Name
|
||||
ORDER BY Com_Name COLLATE NOCASE;
|
||||
GROUP BY Com_Name, Sci_Name;
|
||||
SQL;
|
||||
$result = $db->query($sql);
|
||||
?>
|
||||
@@ -190,8 +185,7 @@ $result = $db->query($sql);
|
||||
<div class="centered">
|
||||
<!-- Search with persistence -->
|
||||
<div class="toolbar">
|
||||
<input id="q" type="text" placeholder="Filter species… (name, scientific)"
|
||||
title="Type to filter; persists across reloads">
|
||||
<input id="q" type="text" placeholder="Filter species… (name, scientific)" title="Type to filter; persists across reloads">
|
||||
<small id="matchCount"></small>
|
||||
</div>
|
||||
|
||||
@@ -223,8 +217,7 @@ $result = $db->query($sql);
|
||||
$lastSeenSort = $lastSeen ? (strtotime($lastSeen) ?: 0) : 0;
|
||||
$lastSeenDisplay = htmlspecialchars($lastSeen, ENT_QUOTES);
|
||||
|
||||
$common_link = "<a href='views.php?view=Recordings&species="
|
||||
. rawurlencode($row['Sci_Name']) . "'>{$common}</a>";
|
||||
$common_link = "<a href='views.php?view=Recordings&species=" . rawurlencode($row['Sci_Name']) . "'>{$common}</a>";
|
||||
|
||||
$is_confirmed = in_array($identifier_sci, $confirmed_species, true);
|
||||
$is_excluded = in_array($identifier, $excluded_species, true);
|
||||
@@ -246,8 +239,8 @@ $result = $db->query($sql);
|
||||
. "<td>{$common_link}</td>"
|
||||
. "<td><i>{$scient}</i></td>"
|
||||
. "<td>{$count}</td>"
|
||||
. "<td data-sort='{$max_confidence}'>{$max_confidence}%</td>" /* Max Confidence */
|
||||
. "<td data-sort=\"{$lastSeenSort}\">{$lastSeenDisplay}</td>" /* Last Seen */
|
||||
. "<td data-sort='{$max_confidence}'>{$max_confidence}%</td>"
|
||||
. "<td data-sort=\"{$lastSeenSort}\">{$lastSeenDisplay}</td>"
|
||||
. "<td class='threshold' data-sort='0'>0.0000</td>"
|
||||
. "<td data-sort='".($is_confirmed?0:1)."'>".$confirm_cell."</td>"
|
||||
. "<td data-sort='".($is_excluded?0:1)."'>".$excl_cell."</td>"
|
||||
@@ -262,13 +255,11 @@ $result = $db->query($sql);
|
||||
<script>
|
||||
const scriptsBase = 'scripts/';
|
||||
const sfThresh = <?php echo json_encode($sf_thresh, JSON_UNESCAPED_UNICODE); ?>;
|
||||
|
||||
// tiny fetch helper
|
||||
const get = (url) => fetch(url, {cache:'no-store'}).then(r => r.text());
|
||||
|
||||
// ---------- load thresholds and colorize ----------
|
||||
/* ---------- Probability (thresholds) auto-load ---------- */
|
||||
function loadThresholds() {
|
||||
get(scriptsBase + 'config.php?threshold=0').then(text => {
|
||||
return get(scriptsBase + 'config.php?threshold=0').then(text => {
|
||||
const lines = (text || '').split(/\r?\n/);
|
||||
const map = Object.create(null);
|
||||
for (const line of lines) {
|
||||
@@ -293,31 +284,62 @@ function loadThresholds() {
|
||||
cell.dataset.sort = v.toFixed(4);
|
||||
}
|
||||
});
|
||||
}).catch(() => {
|
||||
console.warn('Probability load failed.');
|
||||
});
|
||||
}
|
||||
document.addEventListener('DOMContentLoaded', loadThresholds);
|
||||
|
||||
// ---------- toggles / delete ----------
|
||||
/* ---------- Files on Disk column auto-load ---------- */
|
||||
function addDiskCounts() {
|
||||
return get(scriptsBase + 'species_tools.php?diskcounts=1').then(t => {
|
||||
let counts; try { counts = JSON.parse(t); } catch { console.warn('Could not parse disk counts'); return; }
|
||||
|
||||
const table = document.getElementById('speciesTable');
|
||||
const headerRow = table.tHead.rows[0];
|
||||
|
||||
// Insert header before last column (Delete)
|
||||
const deleteHeader = headerRow.lastElementChild;
|
||||
const th = document.createElement('th');
|
||||
th.textContent = 'Files on Disk';
|
||||
headerRow.insertBefore(th, deleteHeader);
|
||||
|
||||
const colIndex = headerRow.cells.length - 2; // new column index
|
||||
th.addEventListener('click', () => sortTable(colIndex));
|
||||
|
||||
const decoder = document.createElement('textarea');
|
||||
document.querySelectorAll('#speciesTable tbody tr').forEach(tr => {
|
||||
decoder.innerHTML = tr.getAttribute('data-comname') || '';
|
||||
const name = decoder.value;
|
||||
const lookup = name.replace(/'/g, '');
|
||||
const count = counts[lookup] || 0;
|
||||
const td = document.createElement('td');
|
||||
td.textContent = count;
|
||||
td.dataset.sort = count;
|
||||
tr.insertBefore(td, tr.lastElementChild); // before Delete cell
|
||||
});
|
||||
}).catch(() => {
|
||||
console.warn('Disk counts load failed.');
|
||||
});
|
||||
}
|
||||
|
||||
/* ---------- toggles / delete ---------- */
|
||||
function toggleSpecies(list, species, action) {
|
||||
get(scriptsBase + 'species_tools.php?toggle=' + list + '&species=' + encodeURIComponent(species) + '&action=' + action)
|
||||
.then(t => { if (t.trim() === 'OK') location.reload(); });
|
||||
}
|
||||
|
||||
function deleteSpecies(species) {
|
||||
get(scriptsBase + 'species_tools.php?getcounts=' + encodeURIComponent(species)).then(t => {
|
||||
let info; try { info = JSON.parse(t); } catch { alert('Could not parse count response'); return; }
|
||||
if (!confirm('Delete ' + info.count + ' detections and local audio and png files for ' + species + '?')) return;
|
||||
get(scriptsBase + 'species_tools.php?delete=' + encodeURIComponent(species)).then(t2 => {
|
||||
try {
|
||||
const res = JSON.parse(t2);
|
||||
alert('Deleted ' + res.lines + ' detections and ' + res.files + ' files for ' + species);
|
||||
} catch { alert('Deletion complete'); }
|
||||
try { const res = JSON.parse(t2); alert('Deleted ' + res.lines + ' detections and ' + res.files + ' files for ' + species); }
|
||||
catch { alert('Deletion complete'); }
|
||||
location.reload();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// ---------- Sorting with persistence ----------
|
||||
/* ---------- Sorting with persistence ---------- */
|
||||
function sortTable(n) {
|
||||
const table = document.getElementById('speciesTable');
|
||||
const tbody = table.tBodies[0];
|
||||
@@ -332,13 +354,8 @@ function sortTable(n) {
|
||||
});
|
||||
rows.forEach(r => tbody.appendChild(r));
|
||||
table.setAttribute('data-sort-' + n, asc ? 'asc' : 'desc');
|
||||
|
||||
try {
|
||||
localStorage.setItem('speciesSortCol', String(n));
|
||||
localStorage.setItem('speciesSortAsc', asc ? '1' : '0');
|
||||
} catch(e){}
|
||||
try { localStorage.setItem('speciesSortCol', String(n)); localStorage.setItem('speciesSortAsc', asc ? '1' : '0'); } catch(e){}
|
||||
}
|
||||
|
||||
function applySavedSort() {
|
||||
const table = document.getElementById('speciesTable');
|
||||
const col = parseInt(localStorage.getItem('speciesSortCol') || '', 10);
|
||||
@@ -349,10 +366,9 @@ function applySavedSort() {
|
||||
if ((asc === '1') !== isAscNow) sortTable(col);
|
||||
}
|
||||
|
||||
// ---------- Search with persistence ----------
|
||||
/* ---------- Search with persistence ---------- */
|
||||
const q = document.getElementById('q');
|
||||
const matchCount = document.getElementById('matchCount');
|
||||
|
||||
function applyFilter() {
|
||||
const needle = (q.value || '').trim().toLowerCase();
|
||||
let shown = 0, total = 0;
|
||||
@@ -366,12 +382,15 @@ function applyFilter() {
|
||||
matchCount.textContent = total ? `${shown} / ${total}` : '';
|
||||
try { localStorage.setItem('speciesFilter', q.value); } catch(e){}
|
||||
}
|
||||
|
||||
q.addEventListener('input', applyFilter);
|
||||
|
||||
/* ---------- boot ---------- */
|
||||
document.addEventListener('DOMContentLoaded', () => {
|
||||
try { const saved = localStorage.getItem('speciesFilter'); if (saved !== null) q.value = saved; } catch(e){}
|
||||
applyFilter();
|
||||
applySavedSort();
|
||||
// Auto-load both heavy enrichments
|
||||
loadThresholds();
|
||||
addDiskCounts();
|
||||
});
|
||||
</script>
|
||||
|
||||
Reference in New Issue
Block a user