instruments / chronology / cyclic/src_wikisource.py · raw

The wikisource fetcher (contact address replaced). A research program Kieran shapes and a model (the "seat") writes down. "I" in these files is the seat unless a line is quoted as his. The first write-up explained the misses instead of counting them; Kieran's objection ("this is cyclic") is in the file, and so is the retraction. Snapshot of 2026-10-15 from the private repo; local addresses and paths scrubbed, nothing else changed.

"""zh.wikisource pages → wikitext JSON, one file per page, manifest line each. Usage: src_wikisource.py --topic T --titles "A" "B" [--tag name]"""
import argparse, json, re, sys, time
from srcutil import fetch, sha, position, record, outdir
API = "https://zh.wikisource.org/w/api.php"
def get(title):
    import urllib.parse
    url = API + "?" + urllib.parse.urlencode({"action": "parse", "page": title, "prop": "wikitext|revid", "format": "json"})
    r = fetch(url, timeout=60, ua="research_projects/0.1 ([email protected])")
    body = r[0] if isinstance(r, tuple) else r
    if isinstance(body, bytes): body = body.decode("utf-8", "replace")
    return url, body
def main():
    ap = argparse.ArgumentParser(); ap.add_argument("--topic", required=True); ap.add_argument("--titles", nargs="+", required=True); ap.add_argument("--tag", default="page"); a = ap.parse_args()
    od = outdir(a.topic, "wikisource")
    for t in a.titles:
        try:
            url, body = get(t); d = json.loads(body)
            if "error" in d: print("MISSING", t, d["error"].get("info")); continue
            w = d["parse"]["wikitext"]["*"]; h = sha(body.encode()); name = re.sub(r"[/ ]+", "-", t)
            p = od / f"{a.tag}-{name}.{h[:12]}.json"; p.write_bytes(body.encode())
            n = len(re.findall(r"日有?[食蝕]之?|日[食蝕]", w))
            record(a.topic, {"connector": "wikisource", "id": t, "url": url, "path": f"wikisource/{p.name}", "sha256": h, "revid": d["parse"].get("revid"), "chars": len(w), "eclipse_mentions": n, **position("src_wikisource")})
            print(f"{t:40s} chars {len(w):7d}  eclipse mentions {n:4d}  → {p.name}")
        except Exception as e: print("ERR", t, e)
        time.sleep(0.5)
if __name__ == "__main__": main()

back to chronology