import sys, io, re, ssl, json, urllib.request
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
from concurrent.futures import ThreadPoolExecutor
ctx = ssl.create_default_context(); ctx.check_hostname=False; ctx.verify_mode=ssl.CERT_NONE
BASE = 'https://www.jangsu.go.kr/index.jangsu?menuCd='
def fetch(cd):
    u = BASE + cd
    return urllib.request.urlopen(urllib.request.Request(u, headers={'User-Agent':'Mozilla/5.0'}), context=ctx, timeout=25).read().decode('utf-8','replace')

COMMON = ('kogl.or.kr','elis.go.kr','jeonbuk.work.go.kr','/index.jangsu')

def content_seg(h):
    m = re.search(r'id="contents"(.*?)(?:<footer|id="footer|<!-- //CONTENTS)', h, re.S)
    seg = m.group(1) if m else h
    return re.sub(r'<script.*?</script>', '', seg, flags=re.S)

def analyze(cd):
    try:
        h = fetch(cd)
    except Exception as e:
        return {'err': str(e)[:50]}
    seg = content_seg(h)
    txt = re.sub(r'\s+', ' ', re.sub(r'<[^>]+>', ' ', seg)).strip()
    # 탭
    taps = []
    tm = re.search(r'<ul class="taps[^"]*">(.*?)</ul>', h, re.S)
    if tm:
        for am in re.finditer(r'menuCd=(DOM_000000\d{12})"[^>]*>(?:<span>)?\s*([^<]+)', tm.group(1)):
            taps.append((am.group(1), am.group(2).strip()))
    # 외부 사이트
    exts = re.findall(r'href="(https?://[^"]+)"', seg)
    real_ext = [e for e in exts if not any(c in e for c in COMMON)]
    is_site = bool(real_ext) and bool(re.search(r'해당\s*사이트로\s*이동|홈페이지\s*바로가기|버튼을\s*클릭', txt))
    # 게시판
    is_board = 'bbs_list' in seg
    mcount = None
    bm = re.search(r'전체\s*([\d,]+)\s*건', txt)
    if bm: mcount = int(bm.group(1).replace(',', ''))
    # 공공누리
    om = re.search(r'공공누리\s*제?\s*(\d)\s*유형', txt)
    otype = om.group(1) + '유형' if om else '미부착'
    # 이미지(본문 실이미지 — 데코 제외)
    imgs = re.findall(r'<img[^>]+src="([^"]+)"', seg)
    deco = re.compile(r'opentype|kogl|btn_|icon|blank|move|/template/|/images/|no_img', re.I)
    realimg = [s for s in imgs if not deco.search(s)]
    if is_site:
        L = '사이트'; K = real_ext[0]; M=None; N=None; O='미부착'
    elif is_board:
        L = '게시판'; K = BASE+cd; M = mcount if mcount is not None else 0
        N = '어문' if M and M>0 else '없음'; O = otype
    else:
        L = '페이지'; K = BASE+cd; M = 1
        N = '어문,이미지' if len(realimg)>=2 else '어문'; O = otype
    return {'taps':taps,'L':L,'K':K,'M':M,'N':N,'O':O,'site':is_site,'board':is_board}

tree = json.load(open(r'D:\01.프로젝트\DB수집\작업파일\_스크립트\_jangsu_tree.json', encoding='utf-8'))
대, 중, 소 = tree['대'], tree['중'], [tuple(x) for x in tree['소']]

# 1차: 모든 소분류 분석
print(f'소분류 {len(소)}개 크롤 시작...', file=sys.stderr)
res = {}
def run(cd_full):
    return cd_full, analyze(cd_full)
with ThreadPoolExecutor(max_workers=8) as ex:
    for cd_full, r in ex.map(run, [x[3] for x in 소]):
        res[cd_full] = r

# 2차: 탭 자식 분석 (소분류에 탭 있으면)
tab_cds = set()
for cd, r in res.items():
    if r.get('taps'):
        for tcd, _ in r['taps']:
            tab_cds.add(tcd)
tab_res = {}
with ThreadPoolExecutor(max_workers=8) as ex:
    for cd_full, r in ex.map(run, sorted(tab_cds)):
        tab_res[cd_full] = r
print(f'소분류 {len(res)} + 탭 {len(tab_res)} 분석 완료', file=sys.stderr)

# 행 빌드
rows = []  # dict: 대,중,소,세부(G),K,L,M,N,O
errs = []
for d, e, f, cd, lab in 소:
    r = res.get(cd, {})
    if r.get('err'): errs.append((cd, lab, r['err']))
    Dl, El = 대.get(d, ''), 중.get(d+e, '')
    taps = r.get('taps') or []
    if taps:
        for tcd, tlab in taps:
            tr = tab_res.get(tcd, {})
            rows.append({'D':Dl,'E':El,'F':lab,'G':tlab,'K':tr.get('K', BASE+tcd),
                         'L':tr.get('L','페이지'),'M':tr.get('M',1),'N':tr.get('N','어문'),'O':tr.get('O','미부착')})
    else:
        rows.append({'D':Dl,'E':El,'F':lab,'G':None,'K':r.get('K', BASE+cd),
                     'L':r.get('L','페이지'),'M':r.get('M',1),'N':r.get('N','어문'),'O':r.get('O','미부착')})

json.dump({'rows':rows,'errs':errs}, open(r'D:\01.프로젝트\DB수집\작업파일\_스크립트\_jangsu_rows.json','w',encoding='utf-8'), ensure_ascii=False, indent=1)
from collections import Counter
print(f'총 행 {len(rows)} | L분포 {dict(Counter(x["L"] for x in rows))} | 탭전개 소분류 {sum(1 for r in res.values() if r.get("taps"))} | 에러 {len(errs)}')
print('saved _jangsu_rows.json')