# -*- coding: utf-8 -*-
"""전주시 신규 leaf phase2-4 분류 + 본문탭 전수 스캔. 기존 매칭행은 값 보존."""
import requests, warnings, sys, re, json, time
from concurrent.futures import ThreadPoolExecutor
warnings.filterwarnings('ignore'); sys.stdout.reconfigure(encoding='utf-8', line_buffering=True)
from bs4 import BeautifulSoup
DIR=r'작업파일\광역_사이트맵\전북특별자치도\12.전주시'
BASE='https://www.jeonju.go.kr'
S=requests.Session(); S.headers['User-Agent']='Mozilla/5.0'

TAB_SELS=['.tab','ul.tab','.tab_menu','.tabmenu','.tab_wrap','.contab','.tab_con',
          '.tab_list','.sub_tab','[role=tablist]','.tabContent','.tab_area']

def fetch(u):
    r=S.get(u,timeout=25,verify=False); r.encoding='utf-8'
    return BeautifulSoup(r.text,'html5lib')

def board_count(cu_):
    try:
        soup=fetch(BASE+'/index.9is?contentUid='+cu_)
        if not soup.select_one('table.bbs_table'):
            return None,soup
        end=None
        for a in soup.select('.paging a'):
            if a.get_text(strip=True)=='끝': end=a.get('href')
        if not end:
            # single page: count numbered rows
            tb=soup.select_one('table.bbs_table')
            n=sum(1 for tr in tb.select('tbody tr') if (tr.find('td') and tr.find('td').get_text(strip=True).isdigit()))
            return n,soup
        m=re.search(r'page=(\d+)',end); last=int(m.group(1)) if m else 1
        eu=BASE+end if end.startswith('/') else end
        s2=fetch(eu); tb=s2.select_one('table.bbs_table')
        lastn=sum(1 for tr in tb.select('tbody tr') if (tr.find('td') and tr.find('td').get_text(strip=True).isdigit())) if tb else 0
        return (last-1)*10+lastn, soup
    except Exception as e:
        return -1,None

def classify(rec):
    if 'match' in rec:
        mt=rec['match']
        rec.update(L=mt['L'],M=mt['M'],N=mt['N'],O=mt['O'],P=mt['P'],Q=mt['Q'],src='keep')
        return rec
    if rec['external']:
        rec.update(L='사이트',M='',N='',O='',P='',Q='',src='ext'); return rec
    cnt,soup=board_count(rec['cu'])
    # tab scan
    if soup is not None:
        cont=soup.select_one('#content') or soup
        tabs=[]
        for sel in TAB_SELS:
            for e in cont.select(sel):
                txt=e.get_text(' ',strip=True)
                if e.find('a') or e.find('li'): tabs.append((sel,txt[:60]))
        rec['tabs']=tabs[:3]
    if cnt is None:
        rec.update(L='페이지',M=1,N='어문',O='미부착',P='',Q='',src='page')
    elif cnt==-1:
        rec.update(L='페이지',M=1,N='어문',O='미부착',P='',Q='',src='err')
    else:
        rec.update(L='게시판',M=cnt,N=('없음' if cnt==0 else '어문'),O='미부착',P='',Q='',src='board')
    return rec

if __name__=='__main__':
    rows=json.load(open(DIR+r'\_tree.json',encoding='utf-8'))
    todo=[r for r in rows if 'match' not in r and not r['external']]
    print('total leaves:',len(rows),'to-fetch:',len(todo))
    done=0
    with ThreadPoolExecutor(max_workers=12) as ex:
        for r in ex.map(classify,rows):
            done+=1
            if done%50==0: print('  ..%d/%d'%(done,len(rows)))
    json.dump(rows,open(DIR+r'\_rows.json','w',encoding='utf-8'),ensure_ascii=False,indent=1)
    from collections import Counter
    print('L분포:',Counter(r['L'] for r in rows))
    tabbed=[r for r in rows if r.get('tabs')]
    print('본문탭 발견 페이지:',len(tabbed))
    for r in tabbed[:20]: print('   TAB',r['path'][-1],r['tabs'])
    print('boards:',[(r['path'][-1],r['M']) for r in rows if r['L']=='게시판'][:40])