"""Unified crawl of all 정읍시 rows.
For each menuCd URL: capture bbs_total count (board vs page), redirect-loop flag.
Aggregate left-nav (ul.dep2/3/4) anchor target=_blank across ALL pages -> menuCd map.
Outputs _report.json for review before writing the xlsx."""
import urllib.request, urllib.error, ssl, re, io, sys, json
from concurrent.futures import ThreadPoolExecutor
from _navmap import NavParser

ctx = ssl.create_default_context(); ctx.check_hostname = False; ctx.verify_mode = ssl.CERT_NONE
BASE = 'https://www.jeongeup.go.kr/index.jeongeup?menuCd='

def fetch(url):
    req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0'})
    return urllib.request.urlopen(req, timeout=25, context=ctx).read().decode('utf-8', 'replace')

TOTAL_RE = re.compile(r'bbs_total[^>]*>[^<]*<strong>\s*([\d,]+)\s*</strong>')

def crawl_one(item):
    r, code, url = item
    out = {'r': r, 'code': code, 'count': None, 'err': None, 'nav': []}
    try:
        h = fetch(url)
    except urllib.error.HTTPError as e:
        out['err'] = 'redirect_loop' if e.code in (301, 302) else 'http%d' % e.code
        return out
    except Exception as e:
        out['err'] = type(e).__name__
        return out
    m = TOTAL_RE.search(h)
    if m:
        out['count'] = int(m.group(1).replace(',', ''))
    p = NavParser(); p.feed(h)
    out['nav'] = [(a['code'], a['blank']) for a in p.results if a['code']]
    return out

if __name__ == '__main__':
    data = json.load(io.open('_dump.json', encoding='utf-8'))
    items = []
    for row in data:
        k = row.get('K', '')
        mc = re.search(r'menuCd=(DOM_\d+)', k)
        if mc and 'jeongeup.go.kr/index.jeongeup' in k:
            items.append((row['r'], mc.group(1), k))
    results = []
    with ThreadPoolExecutor(max_workers=8) as ex:
        for res in ex.map(crawl_one, items):
            results.append(res)
    # aggregate nav target map across all pages
    navmap = {}
    for res in results:
        for code, blank in res['nav']:
            navmap.setdefault(code, set()).add(blank)
    navblank = {c: (True in v) for c, v in navmap.items()}
    report = {'pages': [{'r': x['r'], 'code': x['code'], 'count': x['count'], 'err': x['err']} for x in results],
              'navblank': navblank}
    json.dump(report, io.open('_report.json', 'w', encoding='utf-8'), ensure_ascii=False, indent=0)
    errs = [x for x in results if x['err']]
    counts = [x for x in results if x['count'] is not None]
    print('crawled:', len(results), '| with bbs_total:', len(counts), '| errors:', len(errs))
    print('distinct nav menuCd:', len(navblank), '| nav target=_blank:', sum(1 for v in navblank.values() if v))
    print('errors:', [(x['r'], x['err']) for x in errs])