# -*- coding: utf-8 -*-
# 괴산군 통합 수집: 메가메뉴 트리 + 본문탭 전개 + L/M/O 분류 (정적 크롤)
import sys,io,json,re,time,urllib.request,ssl
from bs4 import BeautifulSoup
sys.stdout=io.TextIOWrapper(sys.stdout.buffer,encoding='utf-8')
ctx=ssl.create_default_context(); ctx.check_hostname=False; ctx.verify_mode=ssl.CERT_NONE
BASE='https://www.goesan.go.kr'
HOST='goesan.go.kr'
def absu(u):
    if not u: return u
    if u.startswith('http'): return u
    if u.startswith('/'): return BASE+u
    return BASE+'/'+u
_cache={}
def get(u):
    if u in _cache: return _cache[u]
    h=''
    for _ in range(2):
        try:
            req=urllib.request.Request(u,headers={'User-Agent':'Mozilla/5.0'})
            h=urllib.request.urlopen(req,timeout=25,context=ctx).read().decode('utf-8','ignore'); break
        except Exception: time.sleep(1)
    _cache[u]=h; return h

def kind(u):
    if not u: return 'none'
    if u.startswith('http') and HOST not in u: return '사이트'
    if 'eminwon' in u: return '사이트'
    if re.search(r'selectBbsNttList|selectOthbcInfoList|BbsNtt',u): return '게시판'
    if 'contents.do' in u: return '페이지'
    if 'addBbsNttView' in u: return '페이지'
    return '페이지'

def board_M(html):
    m=re.search(r'게시물[^\d]{0,15}([\d,]+)', html)
    if m:
        try: return int(m.group(1).replace(',',''))
        except: return 0
    return 0
def has_opentype(html):
    mm=re.findall(r'img_opentype0?(\d)', html)
    return sorted(set(int(x) for x in mm)) if mm else []

def page_tabs(html, self_url):
    s=BeautifulSoup(html,'html.parser')
    cont=s.select_one('#contents,.contents,.sub_content,.sub_contents,#content,.content_in') or s
    selfkey=re.search(r'key=(\d+)',self_url)
    selfkey=selfkey.group(1) if selfkey else None
    for ul in cont.find_all('ul'):
        cls=' '.join(ul.get('class') or [])
        if not re.search(r'tab',cls,re.I): continue
        anchors=ul.find_all('a')
        if len(anchors)<2: continue
        tabs=[]
        for a in anchors:
            lab=a.get_text(strip=True); href=a.get('href')
            if not lab: continue
            tabs.append((lab,href))
        # exclude self/전체
        out=[]
        for lab,href in tabs:
            k=re.search(r'key=(\d+)',href or '')
            if lab in ('전체','전체보기') and (not k or (selfkey and k.group(1)==selfkey)): continue
            if href and href.rstrip('/')==self_url.rstrip('/'): continue
            out.append((lab,href))
        if out: return out
    return []

tree=json.load(open('_gs_tree.json',encoding='utf-8'))
nodes=[]   # each: dict
def classify_url(u):
    u=absu(u); k=kind(u); M=None; O=[]; det=None
    if k=='게시판':
        h=get(u); M=board_M(h); O=has_opentype(h)
        s=BeautifulSoup(h,'html.parser')
        for a in s.find_all('a'):
            href=a.get('href') or ''
            if re.search(r'NttView|nttNo=|addBbsNttView',href): det=absu(href); break
    elif k=='페이지':
        h=get(u); O=has_opentype(h)
    return k,M,O,det

print('트리노드',len(tree),flush=True)
for i,(d1,d2,d3,d4,url) in enumerate(tree):
    u=absu(url); k,M,O,det=classify_url(url)
    node={'d':[d1,d2,d3,d4,''],'url':u,'kind':k,'M':M,'O':O,'det':det}
    nodes.append(node)
    # 본문탭 전개 (페이지만)
    if k=='페이지' and 'contents.do' in u:
        tabs=page_tabs(get(u), u)
        for lab,href in tabs:
            tu=absu(href); tk,tM,tO,tdet=classify_url(href)
            # 탭은 부모 leaf 다음 레벨
            dd=[d1,d2,d3,d4,'']
            # 부모 leaf 깊이 찾기
            depth=max(idx for idx in range(4) if dd[idx]) if any(dd[:4]) else 0
            child=dd[:]
            if depth>=3: child[4]=lab          # d4 있으면 탭=H(5번째)
            else: child[depth+1]=lab            # 아니면 한단계 아래
            nodes.append({'d':child,'url':tu,'kind':tk,'M':tM,'O':tO,'det':tdet,'tab':True})
    if i%20==0: print('%d/%d %s'%(i+1,len(tree),d3 or d2 or d1),flush=True)

json.dump(nodes,open('_gs_nodes.json','w',encoding='utf-8'),ensure_ascii=False)
from collections import Counter
print('총행',len(nodes),'| L:',Counter(n['kind'] for n in nodes))
print('탭전개행',sum(1 for n in nodes if n.get('tab')))
print('O부착행',sum(1 for n in nodes if n['O']))