import requests, sys, io, re, json
from bs4 import BeautifulSoup
from concurrent.futures import ThreadPoolExecutor
import openpyxl, urllib3
urllib3.disable_warnings()
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
H = {'User-Agent':'Mozilla/5.0'}

wb=openpyxl.load_workbook('충청남도_아산시.xlsx'); ws=wb.active
rows=[]
for r in range(16,271):
    url=ws.cell(r,11).value; L=ws.cell(r,12).value; M=ws.cell(r,13).value
    label=ws.cell(r,7).value or ws.cell(r,6).value or ws.cell(r,5).value or ws.cell(r,4).value
    if url: rows.append((r,label,url,L,M))

def classify(item):
    r,label,url,L,M=item
    res={'row':r,'label':label,'url':url,'oldL':L}
    try:
        resp=requests.get(url,headers=H,timeout=25,verify=False,allow_redirects=True)
        final=resp.url
        resp.encoding=resp.apparent_encoding or 'utf-8'
        s=BeautifulSoup(resp.text,'html.parser')
        dom=re.sub(r'^https?://','',final).split('/')[0]
        res['final_dom']=dom
        # page_tab
        ptabs=[(a.get_text(strip=True),a.get('href')) for a in s.select('div.page_tab ul li a') if a.get('href','').strip() not in ('','#')]
        res['ptab']=ptabs
        # board detection
        cnt=None
        # 1) lc06 caption [name:N/page]
        for t in s.select('table'):
            cap=t.find('caption')
            if cap:
                m=re.search(r':\s*([\d,]+)\s*/\s*\d+',cap.get_text())
                if m: cnt=int(m.group(1).replace(',','')); break
        # 2) total counters
        if cnt is None:
            txt=s.get_text(' ',strip=True)
            m=re.search(r'(?:전체|총|Total)\s*([\d,]+)\s*건',txt)
            if m: cnt=int(m.group(1).replace(',',''))
        # signals
        has_paging=bool(s.select('.paging a, .pagination a, .page_num a, nav.paging a'))
        has_search=bool(s.select('input[name=searchKeyword], input[name=keyword], input[name=sword], input[type=search]'))
        has_bbs_table=False
        for t in s.select('table'):
            ths=[th.get_text(strip=True) for th in t.select('th')]
            if '제목' in ths and ('번호' in ths or '작성일' in ths or '등록일' in ths or '게시일자' in ths):
                has_bbs_table=True; break
        is_board = (cnt is not None) or has_bbs_table or (has_paging and has_search)
        res['cnt']=cnt; res['paging']=has_paging; res['search']=has_search; res['bbstbl']=has_bbs_table
        if 'asan.go.kr' not in dom:
            res['newL']='사이트'
        else:
            res['newL']='게시판' if is_board else '페이지'
        res['status']=resp.status_code
    except Exception as e:
        res['err']=str(e)[:80]; res['newL']='ERR'
    return res

out=[]
with ThreadPoolExecutor(max_workers=12) as ex:
    for res in ex.map(classify, rows):
        out.append(res)
json.dump(out,open('_scan_out.json','w',encoding='utf-8'),ensure_ascii=False,indent=1)

# report
chg=[o for o in out if o.get('oldL')!=o.get('newL') and o.get('newL')!='ERR']
print(f'TOTAL {len(out)} | changes {len(chg)} | errors {len([o for o in out if o.get("newL")=="ERR"])}')
print('\n=== L CHANGES ===')
for o in chg:
    print(f"  r{o['row']} {o['label']}: {o['oldL']}→{o['newL']} cnt={o.get('cnt')}")
print('\n=== page_tab GROUPS ===')
for o in out:
    if o.get('ptab'):
        print(f"  r{o['row']} {o['label']} ({len(o['ptab'])} tabs): {[t[0] for t in o['ptab']]}")
print('\n=== ERRORS ===')
for o in out:
    if o.get('newL')=='ERR': print(f"  r{o['row']} {o['label']}: {o.get('err')}")