#!/usr/bin/env python3
"""Tranco top-10k crawl, RUN 2 (2026-08-11) — image-SEO fundamentals.

Direct descendant of tranco_crawl_2026-07-31_RECOVERED.py (the tool that produced
the published alt-text study: 306,634 images / 11.3% no-alt / Tranco list ZJG2G).

SAME universe (Tranco ZJG2G), SAME request budget (one homepage GET + one ranged
llms.txt GET per domain — zero extra requests per image, we never fetch images),
SAME honesty rails (homepage-only, static HTML, no JS, denominator = reachable).

WHAT'S NEW: per-<img> attribute parsing, so we can answer questions no published
source answers — filename quality at population scale, and JOINT compliance
(how many sites get several things right at once).

The alt columns are recomputed identically to run 1 on purpose: they are the
control. Run-2 alt numbers will NOT match run 1 exactly (different date, sites
change) and must NEVER be substituted for the locked 2026-07-31 figures.
"""
import csv, re, sys, socket, urllib.request, urllib.error, ssl, gzip, io, os, json, collections
from concurrent.futures import ThreadPoolExecutor

socket.setdefaulttimeout(10)

BASE = "/Users/arnoldasarny/Downloads/image-seo/keywords/06-raw-data"
IN_CSV = f"{BASE}/tranco-ZJG2G-top10k.csv"
OUT_CSV = f"{BASE}/2026-08-11_tranco-top10k-image-fundamentals.csv"
OUT_SAMPLES = f"{BASE}/2026-08-11_filename-samples.json"
N = 10000
THREADS = 32
UA = "Mozilla/5.0 (compatible; image-alt-study/1.0; +https://imageseo.io)"
CTX = ssl.create_default_context()
CTX.check_hostname = False
CTX.verify_mode = ssl.CERT_NONE

IMG_RE = re.compile(rb"<img\b[^>]*>", re.I)
PICTURE_RE = re.compile(rb"<picture\b", re.I)
HTML_START = (b"<!doctype", b"<html", b"<head", b"<body")


# Pre-compiled once per attribute. Building these per call made the parser
# CPU-bound (~9 patterns x ~300k images = millions of pattern constructions);
# the first run was pinned at 100% CPU doing regex bookkeeping, not crawling.
_ATTR_CACHE = {}


def _attr_re(name: str):
    r = _ATTR_CACHE.get(name)
    if r is None:
        r = re.compile(
            rb"\b" + name.encode() + rb"\s*=\s*(\"([^\"]*)\"|'([^']*)'|([^\s>\"']+))",
            re.I)
        _ATTR_CACHE[name] = r
    return r


for _n in ("alt", "src", "data-src", "srcset", "data-srcset", "sizes",
           "loading", "width", "height"):
    _attr_re(_n)


def attr(tag: bytes, name: str):
    """Return attribute value as str, '' if present-but-empty, None if absent."""
    m = _attr_re(name).search(tag)
    if not m:
        return None
    v = m.group(2) if m.group(2) is not None else (
        m.group(3) if m.group(3) is not None else (m.group(4) or b""))
    try:
        return v.decode("utf-8", "replace").strip()
    except Exception:
        return ""


def has_bare_attr(tag: bytes, name: str) -> bool:
    return re.search(rb"\b" + name.encode() + rb"\b", tag, re.I) is not None


EXT_RE = re.compile(r"\.([a-z0-9]{2,5})(?:$|[?#])", re.I)
FMT_MAP = {"jpg": "jpg", "jpeg": "jpg", "jfif": "jpg", "png": "png", "webp": "webp",
           "avif": "avif", "gif": "gif", "svg": "svg", "bmp": "other", "ico": "other",
           "tif": "other", "tiff": "other", "heic": "other", "jxl": "other"}

# Filename-quality classification. Deliberately conservative: anything we are not
# confident about lands in 'other', never in a headline bucket.
GENERIC_RE = re.compile(
    r"^(img|dsc|dscn|dscf|image|images|photo|photos|pic|pics|picture|screenshot|"
    r"screen[-_ ]?shot|untitled|unnamed|download|file|upload|asset|banner|logo|"
    r"icon|thumb|thumbnail|default|placeholder|temp|new|final|copy|scaled|"
    r"cropped|resized|output|capture|snap|shot)"
    # Flat character class, NOT a nested quantifier. An earlier version used
    # (?:[-_ ]?[0-9]{1,6})* here, which is catastrophic backtracking: one long
    # numeric filename pinned a worker thread at 100% CPU for minutes and stalled
    # the whole crawl. A single starred class is linear and matches the same
    # shapes (IMG_4032, screenshot-2024-01-05, photo1).
    r"[-_ 0-9]*"
    r"(?:[-_ ](copy|final|new|edited|small|large|scaled))?$", re.I)
HASH_RE = re.compile(r"^[0-9a-f]{16,}$", re.I)              # hex digest
WORD_RE = re.compile(r"[A-Za-z]{3,}")


def looks_like_hash(core: str) -> bool:
    """Opaque machine-generated blob: no word separators, long, letters AND digits
    mixed. Conservative on purpose — a real hyphenated phrase can never match."""
    if re.search(r"[-_ ]", core) or len(core) < 16:
        return False
    return bool(re.search(r"[A-Za-z]", core)) and bool(re.search(r"\d", core))


def classify_filename(src: str):
    """-> ('descriptive'|'generic'|'hash'|'numeric'|'other'|None, stem)"""
    if not src or src.startswith("data:"):
        return None, ""
    path = src.split("?")[0].split("#")[0]
    stem = path.rstrip("/").split("/")[-1]
    if not stem:
        return None, ""
    stem = re.sub(r"\.[A-Za-z0-9]{2,5}$", "", stem)          # strip extension
    if not stem:
        return None, ""
    # strip common responsive size suffixes so "hero-image-1024x768" reads as descriptive
    core = re.sub(r"[-_]\d{2,4}x\d{2,4}$", "", stem)
    core = re.sub(r"[-_](?:scaled|thumb|small|medium|large|\d{2,4}w)$", "", core, flags=re.I)
    if GENERIC_RE.match(core):
        return "generic", stem
    if HASH_RE.match(core):
        return "hash", stem
    if re.fullmatch(r"[0-9]{1,}", core):
        return "numeric", stem
    if looks_like_hash(core):
        return "hash", stem
    words = WORD_RE.findall(core)
    # descriptive = at least two real word tokens, separated by - or _ (the shape
    # every guide tells you to use: red-leather-handbag.jpg)
    if len(words) >= 2 and re.search(r"[-_]", core):
        return "descriptive", stem
    if len(words) == 1 and len(core) >= 6 and not re.search(r"\d{3,}", core):
        return "other", stem
    return "other", stem


HOST_RE = re.compile(r"^(?:https?:)?//([^/?#]+)", re.I)


def host_of(url: str) -> str:
    if "//" not in url[:8]:          # relative src — same host, skip the regex
        return ""
    m = HOST_RE.match(url)
    return (m.group(1) or "").lower().lstrip("www.") if m else ""


def fetch(url, max_bytes, ranged=False):
    req = urllib.request.Request(url, headers={
        "User-Agent": UA, "Accept": "*/*", "Accept-Encoding": "gzip",
        **({"Range": "bytes=0-2048"} if ranged else {}),
    })
    with urllib.request.urlopen(req, timeout=8, context=CTX) as r:
        body = r.read(max_bytes)
        if r.headers.get("Content-Encoding") == "gzip":
            try:
                body = gzip.GzipFile(fileobj=io.BytesIO(body)).read(max_bytes)
            except Exception:
                pass
        return r.status, r.headers.get("Content-Type", ""), body


def try_hosts(domain, path, max_bytes, ranged=False):
    last = "unreachable"
    for host in (domain, "www." + domain):
        try:
            st, ct, body = fetch(f"https://{host}{path}", max_bytes, ranged)
            return st, ct, body, host
        except urllib.error.HTTPError as e:
            last = str(e.code)
        except Exception:
            last = "unreachable"
    return last, "", b"", ""


FIELDS = ["rank", "domain", "home_status", "home_host",
          "img_total", "img_alt_nonempty", "img_alt_empty", "img_alt_missing",
          "fmt_jpg", "fmt_png", "fmt_webp", "fmt_avif", "fmt_gif", "fmt_svg",
          "fmt_other", "fmt_none",
          "load_lazy", "load_eager", "load_none",
          "has_srcset", "has_sizes", "dims_both", "dims_partial", "dims_none",
          "picture_tags", "src_offhost", "src_datauri",
          "fn_descriptive", "fn_generic", "fn_hash", "fn_numeric", "fn_other",
          "llms", "llms_status"]

samples = collections.defaultdict(list)


def crawl(row):
    rank, domain = row
    rec = {k: 0 for k in FIELDS}
    rec["rank"], rec["domain"] = rank, domain

    st, ct, body, host = try_hosts(domain, "/", 700_000)
    rec["home_status"], rec["home_host"] = st, host
    if st == 200 and body:
        low = body[:512].lower().lstrip()
        if not low.startswith(HTML_START) and b"<" not in body[:512]:
            body = b""
        page_host = host.lower().lstrip("www.")
        rec["picture_tags"] = len(PICTURE_RE.findall(body))
        for tag in IMG_RE.findall(body):
            rec["img_total"] += 1
            a = attr(tag, "alt")
            if a is None:
                rec["img_alt_missing"] += 1
            elif a == "":
                rec["img_alt_empty"] += 1
            else:
                rec["img_alt_nonempty"] += 1

            src = attr(tag, "src") or attr(tag, "data-src") or ""
            if src.startswith("data:"):
                rec["src_datauri"] += 1
                rec["fmt_none"] += 1
            else:
                m = EXT_RE.search(src)
                key = FMT_MAP.get(m.group(1).lower(), "other") if m else None
                rec["fmt_" + key if key else "fmt_none"] += 1
                h = host_of(src)
                if h and page_host and h != page_host and not h.endswith("." + page_host):
                    rec["src_offhost"] += 1

            cls, stem = classify_filename(src)
            if cls:
                rec["fn_" + cls] += 1
                if len(samples[cls]) < 60 and stem:
                    samples[cls].append(stem[:80])

            loading = (attr(tag, "loading") or "").lower()
            if loading == "lazy":
                rec["load_lazy"] += 1
            elif loading == "eager":
                rec["load_eager"] += 1
            else:
                rec["load_none"] += 1

            if attr(tag, "srcset") or attr(tag, "data-srcset"):
                rec["has_srcset"] += 1
            if attr(tag, "sizes"):
                rec["has_sizes"] += 1
            w, hgt = attr(tag, "width"), attr(tag, "height")
            if w and hgt:
                rec["dims_both"] += 1
            elif w or hgt:
                rec["dims_partial"] += 1
            else:
                rec["dims_none"] += 1

    lst, lct, lbody, _ = try_hosts(domain, "/llms.txt", 2048, ranged=True)
    rec["llms_status"] = lst
    if lst in (200, 206) and lbody:
        t = lbody.decode("utf-8", "replace").strip()
        tl = t.lower()
        if not t:
            rec["llms"] = "empty200"
        elif tl.lstrip().startswith("<!doctype") or "<html" in tl[:400]:
            rec["llms"] = "soft404"
        elif t.lstrip().startswith("#"):
            rec["llms"] = "ok"
        else:
            rec["llms"] = "ok_loose"
    else:
        rec["llms"] = f"http{lst}" if str(lst).isdigit() else "unreachable"
    return rec


def main():
    rows = []
    with open(IN_CSV) as f:
        for line in f:
            parts = line.strip().split(",")
            if len(parts) >= 2:
                rows.append((parts[0], parts[1]))
            if len(rows) >= N:
                break
    print(f"loaded {len(rows)} domains", flush=True)

    done = 0
    with open(OUT_CSV, "w", newline="") as out:
        w = csv.DictWriter(out, fieldnames=FIELDS)
        w.writeheader()
        with ThreadPoolExecutor(max_workers=THREADS) as ex:
            for rec in ex.map(crawl, rows):
                w.writerow(rec)
                done += 1
                if done % 500 == 0:
                    out.flush()
                    print(f"{done}/{len(rows)}", flush=True)

    with open(OUT_SAMPLES, "w") as f:
        json.dump({k: v for k, v in samples.items()}, f, indent=1)
    print("DONE", OUT_CSV, flush=True)


if __name__ == "__main__":
    main()