# -*- coding: utf-8 -*-
"""정읍 사전정보공표목록(BBS_0000171) 3단 카테고리 트리 크롤 -> _gongpyo.json.
cat1(col5)>cat2(col3)>cat3(col4 leaf). 각 leaf M=총게시물 + view링크(O용)."""
import re, json, os, time, urllib.request
from concurrent.futures import ThreadPoolExecutor
HDR = {'User-Agent': 'Mozilla/5.0'}
DIR = os.path.dirname(os.path.abspath(__file__))
BID = 'BBS_0000171'
MENU = 'DOM_000000104003008000'
BASE = 'https://www.jeongeup.go.kr/board/list.jeongeup?boardId=%s&menuCd=%s' % (BID, MENU)
TOTAL = re.compile(r'class="bbs_total"[^>]*>[^<]*<strong>\s*([\d,]+)')
pool = ThreadPoolExecutor(max_workers=6)

def get(u, minlen=3000):
    for _ in range(4):
        try:
            r = urllib.request.urlopen(urllib.request.Request(u.replace(' ', '%20'), headers=HDR), timeout=25).read().decode('utf-8', 'replace')
            if len(r) >= minlen:
                return r
        except Exception:
            pass
        time.sleep(0.5)
    return r if 'r' in dir() else ''

def tab_items(h, tabid):
    """div#<tabid> 안의 li>a 에서 (c1,c2,c3,label) 추출."""
    m = re.search(r'id="%s"[^>]*>(.*?)</div>\s*(?:<div|$)' % tabid, h, re.S)
    seg = m.group(1) if m else ''
    out = []
    for a in re.finditer(r'<a href="([^"]*)"[^>]*>(.*?)</a>', seg, re.S):
        href, lab = a.group(1), re.sub(r'<[^>]+>', '', a.group(2)).strip()
        c1 = re.search(r'categoryCode1=([^&#]+)', href)
        c2 = re.search(r'categoryCode2=([^&#]+)', href)
        c3 = re.search(r'categoryCode3=([^&#]+)', href)
        out.append({'c1': c1.group(1) if c1 else '', 'c2': c2.group(1) if c2 else '',
                    'c3': c3.group(1) if c3 else '', 'label': lab})
    return out

def url_of(c1, c2, c3):
    return '%s&categoryCode1=%s&categoryCode2=%s&categoryCode3=%s' % (BASE, c1, c2, c3)

# 1) cat1 목록 (아무 fetch에나 들어있음)
h0 = get(url_of('A', 'A_01', 'A_01_001'))
cat1 = tab_items(h0, 'categoryCode1_tab')
print('cat1:', [(c['c1'], c['label']) for c in cat1])

tree = []
for c1 in cat1:
    # cat1 선택 -> cat2 목록
    hc1 = get(url_of(c1['c1'], c1['c1'] + '_01', c1['c1'] + '_01_001'))
    cat2 = tab_items(hc1, 'categoryCode2_tab')
    if not cat2:
        cat2 = [{'c1': c1['c1'], 'c2': c1['c1'] + '_01', 'c3': '', 'label': ''}]
    for c2 in cat2:
        hc2 = get(url_of(c2['c1'], c2['c2'], c2['c2'] + '_001'))
        cat3 = tab_items(hc2, 'categoryCode3_tab')
        if not cat3:
            cat3 = [{'c1': c2['c1'], 'c2': c2['c2'], 'c3': c2['c2'] + '_001', 'label': ''}]
        for c3 in cat3:
            tree.append({'c1': c1['c1'], 'c1_label': c1['label'],
                         'c2': c2['c2'], 'c2_label': c2['label'],
                         'c3': c3['c3'], 'c3_label': c3['label']})
    print(' cat1=%s(%s) cat2=%d leaves so far=%d' % (c1['c1'], c1['label'], len(cat2), len(tree)), flush=True)

print('TOTAL leaves:', len(tree))

# 2) 각 leaf M + view링크
def leaf_data(node):
    u = url_of(node['c1'], node['c2'], node['c3'])
    h = get(u)
    m = TOTAL.search(h)
    M = int(m.group(1).replace(',', '')) if m else None
    views = [v for v in re.findall(r'view\.jeongeup\?([^"\']+)', h) if 'boardId=' + BID in v]
    node['M'] = M
    node['url'] = u
    node['nview'] = len(views)
    node['views'] = views[:8]
    return node

tree = list(pool.map(leaf_data, tree))
json.dump(tree, open(os.path.join(DIR, '_gongpyo.json'), 'w', encoding='utf-8'), ensure_ascii=False, indent=0)
tot = sum(x['M'] or 0 for x in tree)
print('leaves=%d  M합=%d  M>0 leaves=%d  M=None=%d'
      % (len(tree), tot, sum(1 for x in tree if x['M']), sum(1 for x in tree if x['M'] is None)))