From ee5fffd8fb4475e0bf49e4e80aa7ceb5f63cc5c9 Mon Sep 17 00:00:00 2001 From: Twarner491 Date: Mon, 8 Jun 2026 12:00:36 -0700 Subject: [PATCH] [FEAT] scripts: adversarial species-ID and anatomy verifier --- avian/scripts/verify.py | 222 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 222 insertions(+) create mode 100644 avian/scripts/verify.py diff --git a/avian/scripts/verify.py b/avian/scripts/verify.py new file mode 100644 index 0000000..906a52a --- /dev/null +++ b/avian/scripts/verify.py @@ -0,0 +1,222 @@ +#!/usr/bin/env python3 +"""AvianVisitors - adversarial species-ID + anatomy check on illustrations. + +An independent quality gate for the generated library. Each illustration +goes through a fresh Gemini Vision call that is NOT told the target +species: it's asked to identify the bird, count wings/legs/heads/tails, +and flag any twig, perch, or anatomical anomaly. The guess is then +compared to the intended species. This catches drift that passes a quick +visual review - a stylized bird that reads as the wrong species, an extra +wing, a stray perch the prompt said not to draw. + +Results are appended to verify-results.csv (slug, pose, target, guess, +match, confidence, anatomy counts, flags). + +Usage: + export GEMINI_API_KEY='your-key' + python3 verify.py --labels labels.txt # whole library + python3 verify.py --labels labels.txt calypte-anna # one slug +""" +from __future__ import annotations +import argparse +import base64 +import json +import os +import re +import sys +import time +import urllib.error +import urllib.request +from pathlib import Path + +GEMINI_URL = ( + "https://generativelanguage.googleapis.com/v1beta/models/" + "gemini-2.5-flash:generateContent" +) + +VERIFY_PROMPT = """You are a rigorous ornithologist examining a stylized kachō-e woodblock-style bird illustration. The bird in the image is intended to be a {target_com} ({target_sci}). + +Analyze the image and respond ONLY with a valid JSON object (no other text, no markdown fences) with these fields: + +{{ + "guessed_species_sci": "", + "guessed_species_com": "", + "guess_confidence": "", + "matches_target": , + "wing_count": , + "leg_count": , + "head_count": , + "tail_count": , + "has_stick_or_perch": , + "diagnostic_features_present": "", + "diagnostic_features_missing": "", + "anatomy_issues": "", + "style_assessment": "" +}} + +Be honest. If the bird looks more like a different species, say so. If the anatomy has issues, say so. Empty strings for fields where there's nothing to report.""" + + +def slugify(sci: str) -> str: + """Match avian/frontend/apt.js slugify() exactly.""" + return re.sub(r"[^a-z0-9]+", "-", sci.lower()).strip("-") + + +def load_labels(path: Path) -> dict[str, tuple[str, str]]: + """Parse a Sci|Com label file into {slug: (sci, com)}.""" + out = {} + for line in path.read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#") or "|" not in line: + continue + sci, com = (s.strip() for s in line.split("|", 1)) + out[slugify(sci)] = (sci, com) + return out + + +def call_gemini(api_key: str, parts: list) -> dict: + payload = {"contents": [{"parts": parts}]} + req = urllib.request.Request( + GEMINI_URL, + data=json.dumps(payload).encode(), + headers={"Content-Type": "application/json", "x-goog-api-key": api_key}, + method="POST", + ) + backoff = 4.0 + for attempt in range(4): + try: + with urllib.request.urlopen(req, timeout=120) as r: + return json.loads(r.read()) + except urllib.error.HTTPError as e: + if e.code in (429, 500, 502, 503, 504) and attempt < 3: + time.sleep(backoff); backoff *= 2; continue + raise RuntimeError(f"HTTP {e.code}: {e.read().decode(errors='ignore')[:300]}") + except urllib.error.URLError: + if attempt < 3: + time.sleep(backoff); backoff *= 2; continue + raise + raise RuntimeError("retries exhausted") + + +def extract_json(resp: dict) -> dict | None: + for cand in resp.get("candidates", []): + for part in cand.get("content", {}).get("parts", []): + text = part.get("text", "").strip() + if not text: + continue + if text.startswith("```"): + lines = text.split("\n") + text = "\n".join(lines[1:-1] if lines[-1].startswith("```") else lines[1:]) + try: + return json.loads(text) + except json.JSONDecodeError: + start, end = text.find("{"), text.rfind("}") + if start >= 0 and end > start: + try: + return json.loads(text[start:end + 1]) + except json.JSONDecodeError: + pass + return None + + +def verify_one(api_key: str, png: Path, sci: str, com: str) -> dict | None: + parts = [ + {"text": VERIFY_PROMPT.format(target_sci=sci, target_com=com)}, + {"inlineData": {"mimeType": "image/png", + "data": base64.b64encode(png.read_bytes()).decode()}}, + ] + return extract_json(call_gemini(api_key, parts)) + + +CSV_HEADER = ("slug,pose,target_sci,guessed_sci,guessed_com,matches,confidence," + "wings,legs,head,tail,has_stick,diag_present,diag_missing," + "anatomy_issues,style\n") + + +def csv_row(slug: str, pose: int, sci: str, v: dict) -> str: + def q(key): + return '"' + str(v.get(key, "")).replace('"', "'") + '"' + return ",".join([ + slug, str(pose), sci.replace(",", " "), + str(v.get("guessed_species_sci", "")).replace(",", " "), + str(v.get("guessed_species_com", "")).replace(",", " "), + str(v.get("matches_target", False)), str(v.get("guess_confidence", "")), + str(v.get("wing_count", "")), str(v.get("leg_count", "")), + str(v.get("head_count", "")), str(v.get("tail_count", "")), + str(v.get("has_stick_or_perch", "")), + q("diagnostic_features_present"), q("diagnostic_features_missing"), + q("anatomy_issues"), str(v.get("style_assessment", "")), + ]) + "\n" + + +def main() -> int: + here = Path(__file__).resolve().parents[1] + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("slugs", nargs="*", help="Slugs to verify. Default: all in --dir.") + ap.add_argument("--labels", type=Path, required=True, + help="Sci|Com label file (same one passed to pregen.py)") + ap.add_argument("--dir", type=Path, default=here / "assets" / "illustrations", + help="Illustration directory (default: avian/assets/illustrations/)") + ap.add_argument("--out", type=Path, default=Path("verify-results.csv"), + help="CSV output path (default: ./verify-results.csv)") + ap.add_argument("--gemini-key", help="Gemini API key (or GEMINI_API_KEY env)") + args = ap.parse_args() + + api_key = args.gemini_key or os.environ.get("GEMINI_API_KEY", "") + if not api_key: + print("error: GEMINI_API_KEY required (--gemini-key or env)", file=sys.stderr) + return 2 + labels = load_labels(args.labels) + + if args.slugs: + pngs = [args.dir / f"{s}.png" for s in args.slugs] + else: + pngs = sorted(args.dir.glob("*.png")) + if not args.out.exists(): + args.out.write_text(CSV_HEADER) + + print(f"verifying {len(pngs)} illustrations against {len(labels)} labels\n") + mismatches = 0 + for png in pngs: + if not png.exists(): + print(f" [skip] missing {png.name}") + continue + name = png.stem + pose, slug = (2, name[:-2]) if name.endswith("-2") else (1, name) + if slug not in labels: + print(f" [skip] no label for {slug}") + continue + sci, com = labels[slug] + try: + v = verify_one(api_key, png, sci, com) + except Exception as e: + print(f" [fail] {png.name}: {e}", file=sys.stderr) + continue + if not v: + print(f" [fail] {png.name}: could not parse response", file=sys.stderr) + continue + + match = v.get("matches_target", False) + tag = "[ok] " if match else "[MISS] " + print(f" {tag}{png.name}: reads as {v.get('guessed_species_com','?')} " + f"(conf={v.get('guess_confidence','?')})" + ("" if match else f", expected {com}")) + if not match: + mismatches += 1 + flags = [] + if v.get("wing_count", 2) != 2: flags.append(f"wings={v.get('wing_count')}") + if v.get("leg_count", 2) and v.get("leg_count", 2) > 2: flags.append(f"legs={v.get('leg_count')}") + if v.get("has_stick_or_perch"): flags.append("has perch/stick") + if v.get("anatomy_issues"): flags.append(str(v["anatomy_issues"])) + if v.get("diagnostic_features_missing"): flags.append(f"missing: {v['diagnostic_features_missing']}") + if flags: + print(f" [warn] {'; '.join(flags)}") + with args.out.open("a") as f: + f.write(csv_row(slug, pose, sci, v)) + + print(f"\ndone. {mismatches} mismatch(es). results -> {args.out}") + return 0 + + +if __name__ == "__main__": + sys.exit(main())