"""Crawl the built site through the real server: status, clean URLs, links, metadata, headings. Writes ROUTES.md."""
import threading, urllib.request, urllib.error, re, sys
from http.server import ThreadingHTTPServer
import server
from html.parser import HTMLParser
import os
B = os.environ.get("BASE")
if not B:
    srv = ThreadingHTTPServer(("127.0.0.1", 8765), server.H); threading.Thread(target=srv.serve_forever, daemon=True).start(); B = "http://127.0.0.1:8765"
def get(u):
    try:
        r = urllib.request.urlopen(B + u); return r.status, r.read().decode("utf8", "ignore")
    except urllib.error.HTTPError as e: return e.code, e.read().decode("utf8", "ignore")
class P(HTMLParser):
    def __init__(s): super().__init__(); s.links=[]; s.h1=0; s.title=""; s.desc=""; s.canon=""; s._t=False; s.lang=""
    def handle_starttag(s, t, a):
        a = dict(a)
        if t=="a" and "href" in a: s.links.append(a["href"])
        if t=="h1": s.h1+=1
        if t=="title": s._t=True
        if t=="html": s.lang=a.get("lang","")
        if t=="meta" and a.get("name")=="description": s.desc=a.get("content","")
        if t=="link" and a.get("rel")=="canonical": s.canon=a.get("href","")
    def handle_endtag(s, t):
        if t=="title": s._t=False
    def handle_data(s, d):
        if s._t: s.title+=d
sm = get("/sitemap.xml")[1]
routes = [re.sub(r"^https?://[^/]+", "", u) or "/" for u in re.findall(r"<loc>(.*?)</loc>", sm)] + ["/search", "/sw/search"]
fails = []; rows = []; titles = {}; descs = {}; checked = {}
def chk(u):
    if u not in checked: checked[u] = get(u)[0]
    return checked[u]
for r in routes:
    st, h = get(r); p = P(); p.feed(h); issues = []
    if st != 200: issues.append(f"status {st}")
    if p.h1 != 1: issues.append(f"h1 count {p.h1}")
    exp = "sw" if r == "/sw" or r.startswith("/sw/") else "en"
    if p.lang != exp: issues.append(f"lang {p.lang} != {exp}")
    if "hreflang=\"sw\"" not in h and r not in ("/404",): issues.append("no hreflang")
    if not p.canon: issues.append("no canonical")
    if r != "/search":
        if p.title in titles: issues.append("dup title with " + titles[p.title])
        if p.desc in descs: issues.append("dup desc with " + descs[p.desc])
    titles[p.title] = r; descs[p.desc] = r
    for l in p.links:
        if l.startswith(("http", "mailto", "tel:", "#", "data:")): continue
        path = l.split("#")[0].split("?")[0]
        if ".html" in path: issues.append("extension link " + l)
        elif chk(path) != 200: issues.append("broken link " + l)
    rows.append((r, p.title.strip(), p.desc, "PASS" if not issues else "FAIL: " + "; ".join(issues)))
    if issues: fails.append((r, issues))
for u, exp in [("/does-not-exist", 404), ("/about.html", 404), ("/about/", 200), ("/robots.txt", 200), ("/assets/search-index.json", 200), ("/search?q=board", 200), ("/sw/search?q=afya", 200), ("/sw", 200), ("/sw/policies/child-protection", 200), ("/policies/child-protection.html", 404)]:
    s = get(u)[0]
    if s != exp: fails.append((u, [f"expected {exp} got {s}"]))
# orphan check: every route linked from some page
linked = set()
for r in routes:
    p = P(); p.feed(get(r)[1]); linked |= {l.split("#")[0].split("?")[0].rstrip("/") or "/" for l in p.links}
orph = [r for r in routes if r not in linked]
if orph: fails.append(("orphans", orph))
with open("ROUTES.md", "w") as f:
    f.write("# Route inventory (generated by test.py)\n\n| Route | Title | Verification |\n|---|---|---|\n")
    for r, t, d, s in rows: f.write(f"| `{r}` | {t} | {s} |\n")
print(len(routes), "routes crawled;", len(fails), "failures")
for x in fails: print(x)
sys.exit(1 if fails else 0)