Lazy load local files and threshold

scripts/species_tools.php

@@ -7,14 +7,44 @@ require_once __DIR__ . '/common.php';
 ensure_authenticated();
 
 $home = get_home();
-// Open database read-only for typical operations; enable writes only for deletions
+
+/* ---------- disk species counts (AJAX endpoint) ---------- */
+if (isset($_GET['diskcounts'])) {
+    header('Content-Type: application/json');
+    $script = __DIR__ . '/disk_species_count.sh';
+    $cmd    = 'HOME=' . escapeshellarg($home) . ' bash ' . escapeshellarg($script) . ' 2>&1';
+    $output = @shell_exec($cmd);
+    $counts = [];
+    if ($output !== null) {
+        foreach (preg_split('/\\r?\\n/', $output) as $line) {
+            $line = trim($line);
+            if ($line === '') continue;
+            if (preg_match('/^([0-9]+(?:\\.[0-9]+)?)(k?)\\s*:\\s*(.+)$/i', $line, $m)) {
+                $num = (float)$m[1];
+                if (strtolower($m[2]) === 'k') $num *= 1000;
+                $counts[$m[3]] = (int)round($num);
+            }
+        }
+    }
+    echo json_encode($counts, JSON_UNESCAPED_UNICODE);
+    exit;
+}
+
+/* ---------- DB open (RO unless deleting) ---------- */
 $flags = isset($_GET['delete']) ? SQLITE3_OPEN_READWRITE : SQLITE3_OPEN_READONLY;
 $db   = new SQLite3(__DIR__ . '/birds.db', $flags);
 $db->busyTimeout(1000);
+$db->exec("
+  PRAGMA journal_mode=WAL;       -- safe read concurrency
+  PRAGMA synchronous=NORMAL;     -- cheaper fsyncs (read-mostly)
+  PRAGMA temp_store=MEMORY;      -- temp data stays in RAM
+  PRAGMA cache_size=-80000;     -- ~80MB page cache (tune to your RAM)
+  PRAGMA mmap_size=268435456;    -- 256MB mmap (set 0 if kernel disallows)
+");
 
 /* Paths / lists */
 $base_symlink   = $home . '/BirdSongs/Extracted/By_Date';
-$base           = realpath($base_symlink); // used only for safety checks
+$base           = realpath($base_symlink); // safety checks
 
 $confirm_file   = __DIR__ . '/confirmed_species_list.txt';
 $exclude_file   = __DIR__ . '/exclude_species_list.txt';
@@ -32,16 +62,11 @@ $config    = get_config();
 $sf_thresh = isset($config['SF_THRESH']) ? (float)$config['SF_THRESH'] : 0.0;
 
 /* ---------- helpers ---------- */
-function join_path(...$parts): string {
+function join_path(...$parts): string { return preg_replace('#/+#', '/', implode('/', $parts)); }
-  return preg_replace('#/+#', '/', implode('/', $parts));
+function can_unlink(string $p): bool { return is_link($p) || is_file($p); }
-}
-function can_unlink(string $p): bool {
-  return is_link($p) || is_file($p);
-}
 function under_base(string $path, string $base): bool {
   if ($base === false) return false;
   $baseReal = rtrim(realpath($base) ?: $base, DIRECTORY_SEPARATOR);
-
   $resolved = realpath($path);
   if ($resolved === false) {
     $parent = realpath(dirname($path));
@@ -51,50 +76,31 @@ function under_base(string $path, string $base): bool {
   return $resolved === $baseReal || strpos($resolved, $baseReal . DIRECTORY_SEPARATOR) === 0;
 }
 
-/**
+/* Collect files/dirs for a species */
- * Collect detection count, files to delete (unique), first scientific name,
- * and dirs to try rmdir later — for a given species.
- * NOTE: Row-wise enumeration (no aggregates) so we gather every file.
- */
 function collect_species_targets(SQLite3 $db, string $species, string $home, $base): array {
-  $stmt = $db->prepare('SELECT Date, Com_Name, Sci_Name, File_Name
+  $stmt = $db->prepare('SELECT Date, Com_Name, Sci_Name, File_Name FROM detections WHERE Com_Name = :name');
-                        FROM detections
-                        WHERE Com_Name = :name');
   ensure_db_ok($stmt);
   $stmt->bindValue(':name', $species, SQLITE3_TEXT);
   $res = $stmt->execute();
 
   $count = 0; $files = []; $dirs = []; $sci = null;
-
   while ($row = $res->fetchArray(SQLITE3_ASSOC)) {
-    $count++;
+    $count++; if ($sci === null) $sci = $row['Sci_Name'];
-    if ($sci === null) $sci = $row['Sci_Name'];
     $dir = str_replace([' ', "'"], ['_', ''], $row['Com_Name']);
-
     $candidates = [
       join_path($home, 'BirdSongs/Extracted/By_Date',         $row['Date'], $dir, $row['File_Name']),
       join_path($home, 'BirdSongs/Extracted/By_Date/shifted', $row['Date'], $dir, $row['File_Name']),
     ];
-
     foreach ($candidates as $c) {
-      if (can_unlink($c) && under_base($c, $base)) {
+      if (can_unlink($c) && under_base($c, $base)) { $files[$c] = true; $dirs[] = dirname($c); continue; }
-        $files[$c] = true; $dirs[] = dirname($c); continue;
-      }
       $d = realpath(dirname($c));
       if ($d !== false) {
         $alt = $d . DIRECTORY_SEPARATOR . basename($c);
-        if (can_unlink($alt) && under_base($alt, $base)) {
+        if (can_unlink($alt) && under_base($alt, $base)) { $files[$alt] = true; $dirs[] = dirname($alt); }
-          $files[$alt] = true; $dirs[] = dirname($alt);
-        }
       }
     }
   }
-  return [
+  return ['count'=>$count, 'files'=>array_keys($files), 'dirs'=>array_values(array_unique($dirs)), 'sci'=>$sci];
-    'count' => $count,
-    'files' => array_keys($files),
-    'dirs'  => array_values(array_unique($dirs)),
-    'sci'   => $sci,
-  ];
 }
 
 /* ---------- toggle exclude/whitelist/confirmed ---------- */
@@ -117,7 +123,7 @@ if (isset($_GET['toggle'], $_GET['species'], $_GET['action'])) {
   header('Content-Type: text/plain'); echo 'OK'; exit;
 }
 
-/* ---------- count (keeps your old "getcounts=" API) ---------- */
+/* ---------- count ---------- */
 if (isset($_GET['getcounts'])) {
   header('Content-Type: application/json');
   if ($base === false) { http_response_code(500); exit(json_encode(['error' => 'Base directory not found'])); }
@@ -126,7 +132,7 @@ if (isset($_GET['getcounts'])) {
   echo json_encode(['count' => $info['count'], 'files' => count($info['files'])]); exit;
 }
 
-/* ---------- delete (keeps your old "delete=" API) ---------- */
+/* ---------- delete ---------- */
 if (isset($_GET['delete'])) {
   header('Content-Type: application/json');
   if ($base === false) { http_response_code(500); exit(json_encode(['error' => 'Base directory not found'])); }
@@ -138,24 +144,19 @@ if (isset($_GET['delete'])) {
     if (!under_base($fp, $base)) continue;
     if (can_unlink($fp) && @unlink($fp)) {
       $deleted++;
-      // thumbnails: "file.wav.png" and "file.png"
       foreach ([$fp . '.png', preg_replace('/\.[^.]+$/', '.png', $fp)] as $png) {
         if (can_unlink($png)) @unlink($png);
       }
     }
   }
-  foreach ($info['dirs'] as $dir) {
+  foreach ($info['dirs'] as $dir) { if (under_base($dir, $base)) @rmdir($dir); }
-    if (under_base($dir, $base)) @rmdir($dir); // best effort
-  }
 
-  // DB rows
   $del = $db->prepare('DELETE FROM detections WHERE Com_Name = :name');
   ensure_db_ok($del);
   $del->bindValue(':name', $species, SQLITE3_TEXT);
   $del->execute();
   $lines_deleted = $db->changes();
 
-  // Remove from confirmed list
   if ($info['sci'] !== null && file_exists($confirm_file)) {
     $identifier = str_replace("'", '', $info['sci']);
     $lines = array_values(array_filter($confirmed_species, fn($l) => $l !== $identifier));
@@ -167,15 +168,9 @@ if (isset($_GET['delete'])) {
 
 /* ---------- query species aggregates ---------- */
 $sql = <<<SQL
-SELECT
+SELECT Com_Name, Sci_Name, COUNT(*) AS Count, MAX(Confidence) AS MaxConfidence, MAX(Date) AS LastSeen
-  Com_Name,
-  Sci_Name,
-  COUNT(*)        AS Count,
-  MAX(Confidence) AS MaxConfidence,
-  MAX(Date)       AS LastSeen
 FROM detections
-GROUP BY Com_Name, Sci_Name
+GROUP BY Com_Name, Sci_Name;
-ORDER BY Com_Name COLLATE NOCASE;
 SQL;
 $result = $db->query($sql);
 ?>
@@ -190,27 +185,26 @@ $result = $db->query($sql);
 <div class="centered">
   <!-- Search with persistence -->
   <div class="toolbar">
-    <input id="q" type="text" placeholder="Filter species… (name, scientific)"
+    <input id="q" type="text" placeholder="Filter species… (name, scientific)" title="Type to filter; persists across reloads">
-           title="Type to filter; persists across reloads">
     <small id="matchCount"></small>
   </div>
 
-<table id="speciesTable">
+  <table id="speciesTable">
-  <thead>
+    <thead>
-    <tr>
+      <tr>
-      <th onclick="sortTable(0)">Common Name</th>
+        <th onclick="sortTable(0)">Common Name</th>
-      <th onclick="sortTable(1)">Scientific Name</th>
+        <th onclick="sortTable(1)">Scientific Name</th>
-      <th onclick="sortTable(2)">Identifications</th>
+        <th onclick="sortTable(2)">Identifications</th>
-      <th onclick="sortTable(3)">Max Confidence</th>
+        <th onclick="sortTable(3)">Max Confidence</th>
-      <th onclick="sortTable(4)">Last Seen</th>
+        <th onclick="sortTable(4)">Last Seen</th>
-      <th onclick="sortTable(5)">Probability</th>
+        <th onclick="sortTable(5)">Probability</th>
-      <th onclick="sortTable(6)">Confirmed</th>
+        <th onclick="sortTable(6)">Confirmed</th>
-      <th onclick="sortTable(7)">Excluded</th>
+        <th onclick="sortTable(7)">Excluded</th>
-      <th onclick="sortTable(8)">Whitelisted</th>
+        <th onclick="sortTable(8)">Whitelisted</th>
-      <th>Delete</th>
+        <th>Delete</th>
-    </tr>
+      </tr>
-  </thead>
+    </thead>
-  <tbody>
+    <tbody>
 <?php while ($row = $result->fetchArray(SQLITE3_ASSOC)) {
   $common = htmlspecialchars($row['Com_Name'], ENT_QUOTES);
   $scient = htmlspecialchars($row['Sci_Name'], ENT_QUOTES);
@@ -223,8 +217,7 @@ $result = $db->query($sql);
   $lastSeenSort = $lastSeen ? (strtotime($lastSeen) ?: 0) : 0;
   $lastSeenDisplay = htmlspecialchars($lastSeen, ENT_QUOTES);
 
-  $common_link = "<a href='views.php?view=Recordings&species="
+  $common_link = "<a href='views.php?view=Recordings&species=" . rawurlencode($row['Sci_Name']) . "'>{$common}</a>";
-    . rawurlencode($row['Sci_Name']) . "'>{$common}</a>";
 
   $is_confirmed   = in_array($identifier_sci, $confirmed_species, true);
   $is_excluded    = in_array($identifier, $excluded_species, true);
@@ -246,8 +239,8 @@ $result = $db->query($sql);
      . "<td>{$common_link}</td>"
      . "<td><i>{$scient}</i></td>"
      . "<td>{$count}</td>"
-     . "<td data-sort='{$max_confidence}'>{$max_confidence}%</td>"   /* Max Confidence */
+     . "<td data-sort='{$max_confidence}'>{$max_confidence}%</td>"
-     . "<td data-sort=\"{$lastSeenSort}\">{$lastSeenDisplay}</td>"   /* Last Seen */
+     . "<td data-sort=\"{$lastSeenSort}\">{$lastSeenDisplay}</td>"
      . "<td class='threshold' data-sort='0'>0.0000</td>"
      . "<td data-sort='".($is_confirmed?0:1)."'>".$confirm_cell."</td>"
      . "<td data-sort='".($is_excluded?0:1)."'>".$excl_cell."</td>"
@@ -255,20 +248,18 @@ $result = $db->query($sql);
      . "<td><img style='cursor:pointer;max-width:20px' src='images/delete.svg' onclick=\"deleteSpecies('".addslashes($row['Com_Name'])."')\"></td>"
      . "</tr>";
 } ?>
-  </tbody>
+    </tbody>
-</table>
+  </table>
 </div>
 
 <script>
 const scriptsBase = 'scripts/';
 const sfThresh = <?php echo json_encode($sf_thresh, JSON_UNESCAPED_UNICODE); ?>;
-
-// tiny fetch helper
 const get = (url) => fetch(url, {cache:'no-store'}).then(r => r.text());
 
-// ---------- load thresholds and colorize ----------
+/* ---------- Probability (thresholds) auto-load ---------- */
 function loadThresholds() {
-  get(scriptsBase + 'config.php?threshold=0').then(text => {
+  return get(scriptsBase + 'config.php?threshold=0').then(text => {
     const lines = (text || '').split(/\r?\n/);
     const map = Object.create(null);
     for (const line of lines) {
@@ -293,31 +284,62 @@ function loadThresholds() {
         cell.dataset.sort = v.toFixed(4);
       }
     });
+  }).catch(() => {
+    console.warn('Probability load failed.');
   });
 }
-document.addEventListener('DOMContentLoaded', loadThresholds);
 
-// ---------- toggles / delete ----------
+/* ---------- Files on Disk column auto-load ---------- */
+function addDiskCounts() {
+  return get(scriptsBase + 'species_tools.php?diskcounts=1').then(t => {
+    let counts; try { counts = JSON.parse(t); } catch { console.warn('Could not parse disk counts'); return; }
+
+    const table = document.getElementById('speciesTable');
+    const headerRow = table.tHead.rows[0];
+
+    // Insert header before last column (Delete)
+    const deleteHeader = headerRow.lastElementChild;
+    const th = document.createElement('th');
+    th.textContent = 'Files on Disk';
+    headerRow.insertBefore(th, deleteHeader);
+
+    const colIndex = headerRow.cells.length - 2; // new column index
+    th.addEventListener('click', () => sortTable(colIndex));
+
+    const decoder = document.createElement('textarea');
+    document.querySelectorAll('#speciesTable tbody tr').forEach(tr => {
+      decoder.innerHTML = tr.getAttribute('data-comname') || '';
+      const name = decoder.value;
+      const lookup = name.replace(/'/g, '');
+      const count = counts[lookup] || 0;
+      const td = document.createElement('td');
+      td.textContent = count;
+      td.dataset.sort = count;
+      tr.insertBefore(td, tr.lastElementChild); // before Delete cell
+    });
+  }).catch(() => {
+    console.warn('Disk counts load failed.');
+  });
+}
+
+/* ---------- toggles / delete ---------- */
 function toggleSpecies(list, species, action) {
   get(scriptsBase + 'species_tools.php?toggle=' + list + '&species=' + encodeURIComponent(species) + '&action=' + action)
     .then(t => { if (t.trim() === 'OK') location.reload(); });
 }
-
 function deleteSpecies(species) {
   get(scriptsBase + 'species_tools.php?getcounts=' + encodeURIComponent(species)).then(t => {
     let info; try { info = JSON.parse(t); } catch { alert('Could not parse count response'); return; }
     if (!confirm('Delete ' + info.count + ' detections and local audio and png files for ' + species + '?')) return;
     get(scriptsBase + 'species_tools.php?delete=' + encodeURIComponent(species)).then(t2 => {
-      try {
+      try { const res = JSON.parse(t2); alert('Deleted ' + res.lines + ' detections and ' + res.files + ' files for ' + species); }
-        const res = JSON.parse(t2);
+      catch { alert('Deletion complete'); }
-        alert('Deleted ' + res.lines + ' detections and ' + res.files + ' files for ' + species);
-      } catch { alert('Deletion complete'); }
       location.reload();
     });
   });
 }
 
-// ---------- Sorting with persistence ----------
+/* ---------- Sorting with persistence ---------- */
 function sortTable(n) {
   const table = document.getElementById('speciesTable');
   const tbody = table.tBodies[0];
@@ -332,13 +354,8 @@ function sortTable(n) {
   });
   rows.forEach(r => tbody.appendChild(r));
   table.setAttribute('data-sort-' + n, asc ? 'asc' : 'desc');
-
+  try { localStorage.setItem('speciesSortCol', String(n)); localStorage.setItem('speciesSortAsc', asc ? '1' : '0'); } catch(e){}
-  try {
-    localStorage.setItem('speciesSortCol', String(n));
-    localStorage.setItem('speciesSortAsc', asc ? '1' : '0');
-  } catch(e){}
 }
-
 function applySavedSort() {
   const table = document.getElementById('speciesTable');
   const col = parseInt(localStorage.getItem('speciesSortCol') || '', 10);
@@ -349,10 +366,9 @@ function applySavedSort() {
   if ((asc === '1') !== isAscNow) sortTable(col);
 }
 
-// ---------- Search with persistence ----------
+/* ---------- Search with persistence ---------- */
 const q = document.getElementById('q');
 const matchCount = document.getElementById('matchCount');
-
 function applyFilter() {
   const needle = (q.value || '').trim().toLowerCase();
   let shown = 0, total = 0;
@@ -366,12 +382,15 @@ function applyFilter() {
   matchCount.textContent = total ? `${shown} / ${total}` : '';
   try { localStorage.setItem('speciesFilter', q.value); } catch(e){}
 }
-
 q.addEventListener('input', applyFilter);
 
+/* ---------- boot ---------- */
 document.addEventListener('DOMContentLoaded', () => {
   try { const saved = localStorage.getItem('speciesFilter'); if (saved !== null) q.value = saved; } catch(e){}
   applyFilter();
   applySavedSort();
+  // Auto-load both heavy enrichments
+  loadThresholds();
+  addDiskCounts();
 });
 </script>