import sys, io, json
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
import openpyxl, urllib.request, ssl, re
from concurrent.futures import ThreadPoolExecutor

P = r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청남도\10.아산시\충청남도_아산시.xlsx'
wb = openpyxl.load_workbook(P)
ws = wb.active

ctx = ssl.create_default_context(); ctx.check_hostname=False; ctx.verify_mode=ssl.CERT_NONE
hdr={'User-Agent':'Mozilla/5.0'}
def fetch(url):
    req=urllib.request.Request(url, headers=hdr)
    with urllib.request.urlopen(req, context=ctx, timeout=25) as r:
        return r.read().decode('utf-8','replace')

# collect target rows 37+: asan.go.kr pages, L in 페이지/게시판/None
targets=[]
for r in range(37, ws.max_row+1):
    k = ws[f'K{r}'].value
    l = ws[f'L{r}'].value
    o = ws[f'O{r}'].value
    if not k: continue
    if 'asan.go.kr' not in str(k): continue
    if l == '사이트': continue
    targets.append((r, str(k), l, o))

print(f'targets: {len(targets)} rows', file=sys.stderr)

def work(t):
    r,k,l,o = t
    try:
        html = fetch(k)
    except Exception as e:
        return (r,k,l,o,'ERR:'+str(e)[:60], None, None)
    # opentype mark
    mt = re.search(r'img_opentype_(\d)\.gif', html)
    otype = mt.group(1) if mt else None
    # in-page tab nav: ul with class containing openInfo / ng0X tab, listing pg= links
    tabs=None
    for um in re.finditer(r'<ul[^>]*class="([^"]*)"[^>]*>(.*?)</ul>', html, re.S):
        cls=um.group(1); body=um.group(2)
        if re.search(r'openInfo|ng0\d', cls):
            items=re.findall(r'<a[^>]*href="([^"]*)"[^>]*>(.*?)</a>', body, re.S)
            items=[(h, re.sub(r'<[^>]+>','',t).strip()) for h,t in items]
            if len(items)>=2 and any('pg=' in h or 'no=' in h for h,_ in items):
                tabs={'cls':cls.strip(),'items':items}
                break
    return (r,k,l,o,None,otype,tabs)

res=[]
with ThreadPoolExecutor(max_workers=10) as ex:
    for out in ex.map(work, targets):
        res.append(out)

# Report A: opentype marks where O is 미부착/empty (needs fixing)
print('\n=== A) img_opentype present but O=미부착/empty (FIX CANDIDATES) ===')
for r,k,l,o,err,otype,tabs in sorted(res):
    if otype and (not o or str(o).strip()=='미부착'):
        print(f'r{r}: TYPE={otype} L={l} O={o!r}  K={k}')

print('\n=== A2) img_opentype present AND O already set (verify match) ===')
for r,k,l,o,err,otype,tabs in sorted(res):
    if otype and o and str(o).strip()!='미부착':
        print(f'r{r}: TYPE={otype} O={o!r} L={l}  K={k}')

print('\n=== B) in-page openInfo/ngXX tabs found ===')
for r,k,l,o,err,otype,tabs in sorted(res):
    if tabs:
        print(f'r{r}: cls={tabs["cls"]} n={len(tabs["items"])} L={l}  K={k}')
        for h,t in tabs['items']:
            print(f'      {h}  | {t}')

print('\n=== C) fetch errors ===')
for r,k,l,o,err,otype,tabs in sorted(res):
    if err:
        print(f'r{r}: {err}  K={k}')

# dump json for later use
json.dump([{'r':r,'k':k,'l':l,'o':o,'err':err,'otype':otype,'tabs':tabs} for r,k,l,o,err,otype,tabs in res],
          open(r'D:\01.프로젝트\DB수집\작업파일\_스크립트\_asan_crawl.json','w',encoding='utf-8'), ensure_ascii=False, indent=1)
print('\nsaved _asan_crawl.json', file=sys.stderr)
