# -*- coding: utf-8 -*-
# 충북형(eGov 메가메뉴) 범용 수집: 트리 + 본문탭 + L/M/O
# 사용: python _cb_crawl.py <host> <sitemap_url> <out_nodes.json>
import sys,io,json,re,time,urllib.request,ssl
from bs4 import BeautifulSoup
sys.stdout=io.TextIOWrapper(sys.stdout.buffer,encoding='utf-8')
ctx=ssl.create_default_context(); ctx.check_hostname=False; ctx.verify_mode=ssl.CERT_NONE
HOST=sys.argv[1]; SITEMAP=sys.argv[2]; OUT=sys.argv[3]
BASE='https://'+HOST
def absu(u):
    if not u: return u
    if u.startswith('http'): return u
    if u.startswith('/'): return BASE+u
    return BASE+'/'+u
_cache={}
def get(u):
    if u in _cache: return _cache[u]
    h=''
    for _ in range(2):
        try:
            req=urllib.request.Request(u,headers={'User-Agent':'Mozilla/5.0'})
            h=urllib.request.urlopen(req,timeout=25,context=ctx).read().decode('utf-8','ignore'); break
        except Exception: time.sleep(1)
    _cache[u]=h; return h
def kind(u):
    if not u: return 'none'
    if u.startswith('http') and HOST not in u: return '사이트'
    if 'eminwon' in u: return '사이트'
    if re.search(r'selectBbsNttList|selectOthbcInfoList|BbsNtt',u): return '게시판'
    return '페이지'
def board_M(html):
    m=re.search(r'게시물[^\d]{0,15}([\d,]+)', html)
    if m:
        try: return int(m.group(1).replace(',',''))
        except: return 0
    return 0
def has_opentype(html):  # full-page(빠른 1차); 본문한정은 _cb_kogl로 후처리
    mm=re.findall(r'img_opentype0?(\d)', html); return sorted(set(int(x) for x in mm)) if mm else []
def page_tabs(html, self_url):
    # 본문 콘텐츠 탭만: 탭바가 '자기 페이지 key'를 포함(self-referential)해야 인정 → 전역위젯 배제
    s=BeautifulSoup(html,'html.parser')
    cont=s.select_one('#contents,.contents,.sub_content,.sub_contents,#content,.content_in') or s
    selfk=re.search(r'key=(\d+)',self_url); selfk=selfk.group(1) if selfk else None
    if not selfk: return []
    for ul in cont.find_all('ul'):
        cls=' '.join(ul.get('class') or [])
        if not re.search(r'tab',cls,re.I): continue
        anchors=ul.find_all('a')
        if len(anchors)<2: continue
        keys=[re.search(r'key=(\d+)',a.get('href') or '') for a in anchors]
        keys=[k.group(1) for k in keys if k]
        # 가드: 탭앵커가 전부 key= 형식이고 self key를 포함해야 진짜 본문탭
        if selfk not in keys: continue
        if len(keys)<len(anchors)-1: continue   # 대부분 key링크여야(아이콘/외부 위젯 배제)
        out=[]
        for a in anchors:
            lab=a.get_text(strip=True); href=a.get('href')
            if not lab: continue
            k=re.search(r'key=(\d+)',href or '')
            if not k: continue
            if k.group(1)==selfk: continue          # 자기자신(전체/현재) 제외
            out.append((lab,href))
        if out: return out
    return []
# 1) 트리 파싱 (depth1~4 megamenu)
h=get(SITEMAP); s=BeautifulSoup(h,'html.parser')
tree=[]
for li1 in s.select('li.depth1_item'):
    a1=li1.select_one('a.depth1_text');
    if not a1: continue
    n1=a1.get_text(strip=True)
    for li2 in li1.select('li.depth2_item'):
        a2=li2.select_one('a.depth2_text');
        if not a2: continue
        n2=a2.get_text(strip=True)
        d3=li2.select('li.depth3_item')
        if not d3: tree.append((n1,n2,'','',a2.get('href'))); continue
        for li3 in d3:
            a3=li3.select_one('a.depth3_text');
            if not a3: continue
            n3=a3.get_text(strip=True)
            d4=li3.select('li.depth4_item a.depth4_text')
            if not d4: tree.append((n1,n2,n3,'',a3.get('href')))
            else:
                for a4 in d4: tree.append((n1,n2,n3,a4.get_text(strip=True),a4.get('href')))
print('트리노드',len(tree),flush=True)
# 2) 분류 + 탭전개
def classify(u):
    u=absu(u); k=kind(u); M=None; O=[]; det=None
    if k=='게시판':
        hh=get(u); M=board_M(hh); O=has_opentype(hh)
        ss=BeautifulSoup(hh,'html.parser')
        for a in ss.find_all('a'):
            href=a.get('href') or ''
            if re.search(r'NttView|nttNo=|addBbsNttView',href): det=absu(href); break
    elif k=='페이지':
        O=has_opentype(get(u))
    return k,M,O,det
nodes=[]
menu_urls=set(absu(t[4]) for t in tree if t[4])   # 메뉴에 이미 있는 url(탭 중복 방지)
seen_tab=set()
for i,(d1,d2,d3,d4,url) in enumerate(tree):
    u=absu(url); k,M,O,det=classify(url)
    if k=='none' and not u: continue   # url 없는 빈 헤더 노드 제외
    nodes.append({'d':[d1,d2,d3,d4,''],'url':u,'kind':k,'M':M,'O':O,'det':det})
    if k=='페이지' and re.search(r'(contents|sub)\.do',u):
        for lab,href in page_tabs(get(u),u):
            tu=absu(href)
            if tu in menu_urls or tu in seen_tab: continue   # 메뉴/기존탭 중복 제외
            seen_tab.add(tu)
            tk,tM,tO,tdet=classify(href)
            dd=[d1,d2,d3,d4,'']; depth=max((idx for idx in range(4) if dd[idx]),default=0)
            child=dd[:]
            if depth>=3: child[4]=lab
            else: child[depth+1]=lab
            nodes.append({'d':child,'url':tu,'kind':tk,'M':tM,'O':tO,'det':tdet,'tab':True})
    if i%25==0: print('%d/%d'%(i+1,len(tree)),flush=True)
json.dump(nodes,open(OUT,'w',encoding='utf-8'),ensure_ascii=False)
from collections import Counter
print('총행',len(nodes),'| L',dict(Counter(n['kind'] for n in nodes)),'| 탭',sum(1 for n in nodes if n.get('tab')),'| O',sum(1 for n in nodes if n['O']))