import requests, sys, io, re, json
from bs4 import BeautifulSoup
from concurrent.futures import ThreadPoolExecutor
import openpyxl, urllib3
urllib3.disable_warnings()
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
H={'User-Agent':'Mozilla/5.0'}
CMSBASE='https://www.asan.go.kr/main/cms/'

wb=openpyxl.load_workbook('충청남도_아산시.xlsx'); ws=wb.active
scan={o['row']:o for o in json.load(open('_scan_out.json',encoding='utf-8'))}

def no_of(u):
    m=re.search(r'[?&]no=(\d+)',str(u or '')); return m.group(1) if m else None
def norm_url(u):
    if not u: return None
    u=str(u).strip().rstrip('/')
    u=re.sub(r'^https?://','',u).replace('www.','')
    return u.lower()
def resolve(href):
    href=str(href).strip()
    if href.startswith('?'): return CMSBASE+href
    if href.startswith('/'): return 'https://www.asan.go.kr'+href
    return href

# existing keys -> set of rows
no2rows={}; url2rows={}
for r in range(3,271):
    u=ws.cell(r,11).value
    if not u: continue
    n=no_of(u);  no2rows.setdefault(n,[]).append(r) if n else None
    url2rows.setdefault(norm_url(u),[]).append(r)

def classify_html(url):
    """return (L,M) for a fetched url"""
    try:
        resp=requests.get(url,headers=H,timeout=25,verify=False,allow_redirects=True)
        final=resp.url; resp.encoding=resp.apparent_encoding or 'utf-8'
        s=BeautifulSoup(resp.text,'html.parser')
        dom=re.sub(r'^https?://','',final).split('/')[0]
        if 'asan.go.kr' not in dom:
            return ('사이트','', dom)
        # media portal
        if dom=='media.asan.go.kr':
            return ('게시판',0, dom)
        cnt=None
        for t in s.select('table'):
            cap=t.find('caption')
            if cap:
                m=re.search(r':\s*([\d,]+)\s*/\s*\d+',cap.get_text())
                if m: cnt=int(m.group(1).replace(',','')); break
        bbstbl=False
        for t in s.select('table'):
            ths=[th.get_text(strip=True) for th in t.select('th')]
            if '제목' in ths and ('번호' in ths or '작성일' in ths or '등록일' in ths or '게시일자' in ths):
                bbstbl=True; break
        # seip_sechul paginated table
        seip = 'seip_sechul' in final and bool(s.select('.paging a, .pagination a'))
        if cnt is not None: return ('게시판',cnt,dom)
        if bbstbl or seip: return ('게시판',0,dom)
        return ('페이지',1,dom)
    except Exception as e:
        return ('ERR',str(e)[:50],None)

# ---- existing rows final L/M ----
INFLATED={'어문,이미지,영상','어문,이미지,영상,오디오','어문,영상'}
existing_plan={}
def fix_existing(r):
    o=scan.get(r,{})
    url=ws.cell(r,11).value
    newL=o.get('newL'); cnt=o.get('cnt')
    dom=o.get('final_dom') or ''
    # refine with special domains
    if o.get('newL')=='ERR':
        # external sites that errored -> 사이트 if not asan, else keep page
        newL='사이트' if 'asan.go.kr' not in (dom or norm_url(url) or '') else '페이지'
    if dom=='media.asan.go.kr': newL='게시판'; cnt=None
    # seip
    if url and 'seip_sechul' in str(url): newL='게시판'; cnt=None
    # M
    if newL=='게시판': M = cnt if cnt is not None else 0
    elif newL=='페이지': M=1
    else: M=''
    return r,newL,M
for r in range(16,271):
    if ws.cell(r,11).value: existing_plan[r]=fix_existing(r)

# ---- expansion groups (URL+no dedup) ----
groups=[]  # each: {row, col, children:[(label,url,L,M,N,O,site)]}
def deepest_col(r):
    for c in range(10,4,-1):  # J..E (cols10..5)
        if ws.cell(r,c).value not in (None,''): return c
    return 5
def is_container(r,col):
    # cell merged vertically across >1 row => it's a parent/category container
    for mr in ws.merged_cells.ranges:
        if mr.min_col==col and mr.min_row<=r<=mr.max_row and mr.max_row>mr.min_row:
            return True
    return False
new_children_urls=set()
plan_groups=[]
for r in range(16,271):
    o=scan.get(r)
    if not o or not o.get('ptab'): continue
    cur_no=no_of(ws.cell(r,11).value)
    incl=[]  # children to add
    for txt,href in o['ptab']:
        tu=resolve(href); tn=no_of(tu); tnorm=norm_url(tu)
        rows_no=no2rows.get(tn,[]) if tn else []
        rows_url=url2rows.get(tnorm,[])
        other=[x for x in set(rows_no+rows_url) if x!=r]
        if other:  # exists elsewhere -> sibling nav, skip
            continue
        incl.append((txt.strip(),tu,tn))
    # skip container(대분류/category) rows: their tabs belong to a child menu item
    if is_container(r,deepest_col(r)):
        continue
    # only expand if >=2 genuine children (self + new)
    if len(incl)>=2:
        plan_groups.append({'row':r,'col':deepest_col(r),'incl':incl})
        for t,u,n in incl: new_children_urls.add(u)

# classify all new child urls
uniq=sorted(new_children_urls)
clsmap={}
with ThreadPoolExecutor(max_workers=12) as ex:
    res=list(ex.map(lambda u:(u,classify_html(u)), uniq))
for u,lm in res: clsmap[u]=lm

# attach
for g in plan_groups:
    kids=[]
    for t,u,n in g['incl']:
        L,M,dom=clsmap[u]
        if L=='ERR': L='사이트'; M=''  # external/unreachable -> site
        N='' if L=='사이트' else '어문'
        O='' if L=='사이트' else '미부착'
        site='외부링크' if L=='사이트' else ''
        kids.append({'label':t,'url':u,'L':L,'M':M,'N':N,'O':O,'S':site})
    g['kids']=kids

json.dump({'existing':{str(k):v for k,v in existing_plan.items()},'groups':plan_groups},
          open('_plan_out.json','w',encoding='utf-8'),ensure_ascii=False,indent=1)

# summary
from collections import Counter
cL=Counter(v[1] for v in existing_plan.values())
print('=== EXISTING rows 16-270 final L ===',dict(cL))
print(f'\n=== EXPANSION GROUPS: {len(plan_groups)} ===')
tot_new=0
for g in plan_groups:
    r=g['row']; lbl=ws.cell(r,g['col']).value
    nk=len(g['kids']); tot_new+=nk-1  # -1 because self row reused
    from openpyxl.utils import get_column_letter
    print(f"  r{r} [{get_column_letter(g['col'])}={lbl}] -> {nk} children (+{nk-1} new):")
    print('       '+' | '.join(f"{k['label']}({k['L']}{k['M']})" for k in g['kids']))
print(f'\nTOTAL new rows added: {tot_new}')
print('current data rows 3-270 =',sum(1 for r in range(3,271) if ws.cell(r,11).value))
print('after rebuild approx =', sum(1 for r in range(3,271) if ws.cell(r,11).value)+tot_new)