"""Enumerează categoriile etimologice românești din Wiktionary.

Fiecare categorie e populată automat de șabloanele {{inh|ro|X}} și {{bor|ro|X}}
din articole, deci lista e exact mulțimea cuvintelor pe care Wiktionary le
atribuie unui donator. Rezultat: raw/categories.json — pentru fiecare strat,
lista de lexeme.
"""
import json, os, sys, time, urllib.parse, urllib.request

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from strata import ALL_CATS

API = "https://en.wiktionary.org/w/api.php"
UA = "straturi-unui-cuvant/1.0 (https://straturi.mariuscomper.uk/)"
OUT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "raw", "categories.json")


def api(params, tries=5):
    params = dict(params, format="json", formatversion="2", maxlag="5")
    url = API + "?" + urllib.parse.urlencode(params)
    for attempt in range(tries):
        try:
            req = urllib.request.Request(url, headers={"User-Agent": UA})
            with urllib.request.urlopen(req, timeout=90) as r:
                d = json.load(r)
            if "error" in d and d["error"].get("code") == "maxlag":
                time.sleep(5 * (attempt + 1))
                continue
            return d
        except Exception as exc:  # noqa: BLE001
            if attempt == tries - 1:
                raise
            print(f"    retry {attempt + 1}: {exc}", file=sys.stderr)
            time.sleep(3 * (attempt + 1))
    raise RuntimeError("unreachable")


def members(cat):
    out, cont = [], None
    while True:
        p = {
            "action": "query", "list": "categorymembers",
            "cmtitle": "Category:" + cat, "cmlimit": "500", "cmnamespace": "0",
            "cmprop": "title",
        }
        if cont:
            p["cmcontinue"] = cont
        d = api(p)
        out += [m["title"] for m in d.get("query", {}).get("categorymembers", [])]
        cont = d.get("continue", {}).get("cmcontinue")
        if not cont:
            return out
        time.sleep(0.15)


def main():
    result = {}
    for stratum, cat in ALL_CATS:
        t0 = time.time()
        try:
            ms = members(cat)
        except Exception as exc:  # noqa: BLE001
            print(f"  FAIL {cat}: {exc}", file=sys.stderr)
            continue
        result.setdefault(stratum, {})[cat] = ms
        print(f"  {len(ms):>6}  {stratum:<16} {cat}  ({time.time() - t0:.0f}s)", flush=True)
        time.sleep(0.15)
    os.makedirs(os.path.dirname(OUT), exist_ok=True)
    with open(OUT, "w", encoding="utf8") as f:
        json.dump(result, f, ensure_ascii=False)
    tot = sum(len(v) for c in result.values() for v in c.values())
    uniq = len({w for c in result.values() for v in c.values() for w in v})
    print(f"\nintrări de categorie: {tot}   lexeme distincte: {uniq}")
    print(f"scris în {OUT}")


if __name__ == "__main__":
    main()
