[FEAT] scripts: adversarial species-ID and anatomy verifier

2026-06-08 12:00:36 -07:00
parent 23fa8311f8
commit ee5fffd8fb
1 changed files with 222 additions and 0 deletions
@@ -0,0 +1,222 @@
 #!/usr/bin/env python3
 """AvianVisitors - adversarial species-ID + anatomy check on illustrations.
 An independent quality gate for the generated library. Each illustration
 goes through a fresh Gemini Vision call that is NOT told the target
 species: it's asked to identify the bird, count wings/legs/heads/tails,
 and flag any twig, perch, or anatomical anomaly. The guess is then
 compared to the intended species. This catches drift that passes a quick
 visual review - a stylized bird that reads as the wrong species, an extra
 wing, a stray perch the prompt said not to draw.
 Results are appended to verify-results.csv (slug, pose, target, guess,
 match, confidence, anatomy counts, flags).
 Usage:
    export GEMINI_API_KEY='your-key'
    python3 verify.py --labels labels.txt                 # whole library
    python3 verify.py --labels labels.txt calypte-anna    # one slug
 """
 from __future__ import annotations
 import argparse
 import base64
 import json
 import os
 import re
 import sys
 import time
 import urllib.error
 import urllib.request
 from pathlib import Path
 GEMINI_URL = (
    "https://generativelanguage.googleapis.com/v1beta/models/"
    "gemini-2.5-flash:generateContent"
 )
 VERIFY_PROMPT = """You are a rigorous ornithologist examining a stylized kachō-e woodblock-style bird illustration. The bird in the image is intended to be a {target_com} ({target_sci}).
 Analyze the image and respond ONLY with a valid JSON object (no other text, no markdown fences) with these fields:
 {{
  "guessed_species_sci": "<your best guess at the scientific name, Latin binomial, e.g. 'Calypte anna'>",
  "guessed_species_com": "<your best guess at the English common name>",
  "guess_confidence": "<low | medium | high>",
  "matches_target": <true if your guess matches {target_sci} or {target_com}, otherwise false>,
  "wing_count": <integer number of wings visible>,
  "leg_count": <integer number of legs/feet visible>,
  "head_count": <integer number of heads>,
  "tail_count": <integer number of tails>,
  "has_stick_or_perch": <true if any twig, stick, branch, perch, leaf, or substrate is visible in the image; false if the bird floats alone>,
  "diagnostic_features_present": "<comma-separated list of species-diagnostic field marks you can see, e.g. 'red cap, pink breast, streaked back, conical bill'>",
  "diagnostic_features_missing": "<features the species SHOULD have but you don't see, or empty string if all match>",
  "anatomy_issues": "<any anomalies (extra wings, missing feet, deformed beak), or empty string>",
  "style_assessment": "<one of: 'true kachō-e' | 'kachō-e-influenced watercolor' | 'field guide illustration' | 'photographic'>"
 }}
 Be honest. If the bird looks more like a different species, say so. If the anatomy has issues, say so. Empty strings for fields where there's nothing to report."""
 def slugify(sci: str) -> str:
    """Match avian/frontend/apt.js slugify() exactly."""
    return re.sub(r"[^a-z0-9]+", "-", sci.lower()).strip("-")
 def load_labels(path: Path) -> dict[str, tuple[str, str]]:
    """Parse a Sci|Com label file into {slug: (sci, com)}."""
    out = {}
    for line in path.read_text().splitlines():
        line = line.strip()
        if not line or line.startswith("#") or "|" not in line:
            continue
        sci, com = (s.strip() for s in line.split("|", 1))
        out[slugify(sci)] = (sci, com)
    return out
 def call_gemini(api_key: str, parts: list) -> dict:
    payload = {"contents": [{"parts": parts}]}
    req = urllib.request.Request(
        GEMINI_URL,
        data=json.dumps(payload).encode(),
        headers={"Content-Type": "application/json", "x-goog-api-key": api_key},
        method="POST",
    )
    backoff = 4.0
    for attempt in range(4):
        try:
            with urllib.request.urlopen(req, timeout=120) as r:
                return json.loads(r.read())
        except urllib.error.HTTPError as e:
            if e.code in (429, 500, 502, 503, 504) and attempt < 3:
                time.sleep(backoff); backoff *= 2; continue
            raise RuntimeError(f"HTTP {e.code}: {e.read().decode(errors='ignore')[:300]}")
        except urllib.error.URLError:
            if attempt < 3:
                time.sleep(backoff); backoff *= 2; continue
            raise
    raise RuntimeError("retries exhausted")
 def extract_json(resp: dict) -> dict | None:
    for cand in resp.get("candidates", []):
        for part in cand.get("content", {}).get("parts", []):
            text = part.get("text", "").strip()
            if not text:
                continue
            if text.startswith("```"):
                lines = text.split("\n")
                text = "\n".join(lines[1:-1] if lines[-1].startswith("```") else lines[1:])
            try:
                return json.loads(text)
            except json.JSONDecodeError:
                start, end = text.find("{"), text.rfind("}")
                if start >= 0 and end > start:
                    try:
                        return json.loads(text[start:end + 1])
                    except json.JSONDecodeError:
                        pass
    return None
 def verify_one(api_key: str, png: Path, sci: str, com: str) -> dict | None:
    parts = [
        {"text": VERIFY_PROMPT.format(target_sci=sci, target_com=com)},
        {"inlineData": {"mimeType": "image/png",
                        "data": base64.b64encode(png.read_bytes()).decode()}},
    ]
    return extract_json(call_gemini(api_key, parts))
 CSV_HEADER = ("slug,pose,target_sci,guessed_sci,guessed_com,matches,confidence,"
              "wings,legs,head,tail,has_stick,diag_present,diag_missing,"
              "anatomy_issues,style\n")
 def csv_row(slug: str, pose: int, sci: str, v: dict) -> str:
    def q(key):
        return '"' + str(v.get(key, "")).replace('"', "'") + '"'
    return ",".join([
        slug, str(pose), sci.replace(",", " "),
        str(v.get("guessed_species_sci", "")).replace(",", " "),
        str(v.get("guessed_species_com", "")).replace(",", " "),
        str(v.get("matches_target", False)), str(v.get("guess_confidence", "")),
        str(v.get("wing_count", "")), str(v.get("leg_count", "")),
        str(v.get("head_count", "")), str(v.get("tail_count", "")),
        str(v.get("has_stick_or_perch", "")),
        q("diagnostic_features_present"), q("diagnostic_features_missing"),
        q("anatomy_issues"), str(v.get("style_assessment", "")),
    ]) + "\n"
 def main() -> int:
    here = Path(__file__).resolve().parents[1]
    ap = argparse.ArgumentParser(description=__doc__,
                                 formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("slugs", nargs="*", help="Slugs to verify. Default: all in --dir.")
    ap.add_argument("--labels", type=Path, required=True,
                    help="Sci|Com label file (same one passed to pregen.py)")
    ap.add_argument("--dir", type=Path, default=here / "assets" / "illustrations",
                    help="Illustration directory (default: avian/assets/illustrations/)")
    ap.add_argument("--out", type=Path, default=Path("verify-results.csv"),
                    help="CSV output path (default: ./verify-results.csv)")
    ap.add_argument("--gemini-key", help="Gemini API key (or GEMINI_API_KEY env)")
    args = ap.parse_args()
    api_key = args.gemini_key or os.environ.get("GEMINI_API_KEY", "")
    if not api_key:
        print("error: GEMINI_API_KEY required (--gemini-key or env)", file=sys.stderr)
        return 2
    labels = load_labels(args.labels)
    if args.slugs:
        pngs = [args.dir / f"{s}.png" for s in args.slugs]
    else:
        pngs = sorted(args.dir.glob("*.png"))
    if not args.out.exists():
        args.out.write_text(CSV_HEADER)
    print(f"verifying {len(pngs)} illustrations against {len(labels)} labels\n")
    mismatches = 0
    for png in pngs:
        if not png.exists():
            print(f"  [skip] missing {png.name}")
            continue
        name = png.stem
        pose, slug = (2, name[:-2]) if name.endswith("-2") else (1, name)
        if slug not in labels:
            print(f"  [skip] no label for {slug}")
            continue
        sci, com = labels[slug]
        try:
            v = verify_one(api_key, png, sci, com)
        except Exception as e:
            print(f"  [fail] {png.name}: {e}", file=sys.stderr)
            continue
        if not v:
            print(f"  [fail] {png.name}: could not parse response", file=sys.stderr)
            continue
        match = v.get("matches_target", False)
        tag = "[ok]   " if match else "[MISS] "
        print(f"  {tag}{png.name}: reads as {v.get('guessed_species_com','?')} "
              f"(conf={v.get('guess_confidence','?')})" + ("" if match else f", expected {com}"))
        if not match:
            mismatches += 1
        flags = []
        if v.get("wing_count", 2) != 2: flags.append(f"wings={v.get('wing_count')}")
        if v.get("leg_count", 2) and v.get("leg_count", 2) > 2: flags.append(f"legs={v.get('leg_count')}")
        if v.get("has_stick_or_perch"): flags.append("has perch/stick")
        if v.get("anatomy_issues"): flags.append(str(v["anatomy_issues"]))
        if v.get("diagnostic_features_missing"): flags.append(f"missing: {v['diagnostic_features_missing']}")
        if flags:
            print(f"         [warn] {'; '.join(flags)}")
        with args.out.open("a") as f:
            f.write(csv_row(slug, pose, sci, v))
    print(f"\ndone. {mismatches} mismatch(es). results -> {args.out}")
    return 0
 if __name__ == "__main__":
    sys.exit(main())