"""
Fresh census of the Alignerr public job board, 2026-09-01.

Purpose: independently re-verify the clone finding before AIPayList publishes it.
Reads only the platform's own public list endpoint. No auth, no scraping of detail
pages, one polite request per page.

Outputs a JSON summary plus the raw rows, so every number in the published piece
can be recomputed by anyone who runs this file.
"""

import json
import time
import urllib.request
from collections import Counter, defaultdict
from pathlib import Path

OUT = Path(__file__).parent
API = "https://www.alignerr.com/api/jobs?limit=120&offset={}"
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0 Safari/537.36"


def fetch(offset):
    """One page, with backoff. The board resets the connection under sustained
    polling, so a single reset is not evidence the board ended."""
    last = None
    for attempt in range(6):
        try:
            req = urllib.request.Request(API.format(offset), headers={"User-Agent": UA})
            with urllib.request.urlopen(req, timeout=60) as r:
                return json.loads(r.read().decode("utf-8"))
        except Exception as exc:
            last = exc
            wait = 3 * (attempt + 1)
            print("  retry {} at offset {} after {} ({}s)".format(attempt + 1, offset, type(exc).__name__, wait), flush=True)
            time.sleep(wait)
    raise RuntimeError("gave up at offset {}: {}".format(offset, last))


def main():
    # Resume support: a reset partway through should not cost the whole crawl.
    raw_path = OUT / "alignerr-raw.json"
    rows = json.loads(raw_path.read_text(encoding="utf-8")) if raw_path.exists() else []
    offset = (len(rows) // 120) * 120
    rows = rows[:offset]
    pages = offset // 120
    if offset:
        print("resuming from offset {} with {} rows".format(offset, len(rows)), flush=True)
    while True:
        payload = fetch(offset)
        batch = payload.get("jobs") or []
        if not batch:
            break
        rows.extend(batch)
        pages += 1
        offset += 120
        print("page {:>3}  offset {:>6}  running total {:>6}".format(pages, offset, len(rows)), flush=True)
        if pages % 10 == 0:
            raw_path.write_text(json.dumps(rows), encoding="utf-8")
        if pages > 80:
            print("SAFETY STOP at 80 pages")
            break
        time.sleep(1.2)

    (OUT / "alignerr-raw.json").write_text(json.dumps(rows), encoding="utf-8")

    titles = Counter(r.get("title", "") for r in rows)
    locations = Counter(r.get("location", "") for r in rows)
    pays = Counter(r.get("pay", "") for r in rows)

    # A "distinct job" under AIPayList's collapse rule is one title plus one pay wording.
    # Description is deliberately NOT part of the key: the description is the part that
    # varies, because the city is slotted into it.
    groups = defaultdict(list)
    for r in rows:
        groups[(r.get("title", ""), r.get("pay", ""))].append(r)

    biggest = sorted(groups.items(), key=lambda kv: -len(kv[1]))[:15]

    # For the largest group, how much do the descriptions actually differ?
    top_key, top_rows = biggest[0]
    descs = [r.get("description", "") for r in top_rows]
    lens = sorted(len(d) for d in descs)
    common_prefix = 0
    if len(descs) > 1:
        a, b = descs[0], descs[1]
        while common_prefix < min(len(a), len(b)) and a[common_prefix] == b[common_prefix]:
            common_prefix += 1

    summary = {
        "captured_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
        "endpoint": "https://www.alignerr.com/api/jobs?limit=120&offset=N",
        "total_listings": len(rows),
        "pages_fetched": pages,
        "distinct_titles": len(titles),
        "distinct_jobs_title_plus_pay": len(groups),
        "inflation_factor": round(len(rows) / max(len(groups), 1), 1),
        "distinct_locations": len(locations),
        "location_values": locations.most_common(10),
        "distinct_pay_bands": len(pays),
        "top_15_groups": [
            {"title": k[0], "pay": k[1], "listings": len(v)} for k, v in biggest
        ],
        "largest_group_description_check": {
            "title": top_key[0],
            "pay": top_key[1],
            "listings": len(top_rows),
            "description_length_min": lens[0] if lens else None,
            "description_length_max": lens[-1] if lens else None,
            "chars_identical_before_first_difference": common_prefix,
            "sample_a": descs[0][:260] if descs else "",
            "sample_b": descs[1][:260] if len(descs) > 1 else "",
        },
        "listings_saying_remote": locations.get("Remote", 0),
        "share_remote_pct": round(100.0 * locations.get("Remote", 0) / max(len(rows), 1), 1),
    }

    (OUT / "alignerr-census.json").write_text(json.dumps(summary, indent=2), encoding="utf-8")
    print(json.dumps(summary, indent=2)[:4000])


if __name__ == "__main__":
    main()