# -*- coding: utf-8 -*-
import openpyxl, urllib.request, ssl, re, glob, os, sys, time, http.cookiejar
ctx=ssl.create_default_context(); ctx.check_hostname=False; ctx.verify_mode=ssl.CERT_NONE
cj=http.cookiejar.CookieJar()
op=urllib.request.build_opener(urllib.request.HTTPSHandler(context=ctx), urllib.request.HTTPCookieProcessor(cj))
op.addheaders=[('User-Agent','Mozilla/5.0')]
def fetch(u): return op.open(u,timeout=20).read().decode('utf-8','ignore')
NO=int(sys.argv[1])
BASE=os.path.dirname(os.path.abspath(__file__))
d=[x for x in glob.glob(os.path.join(BASE,'*')) if os.path.isdir(x) and os.path.basename(x).startswith(str(NO)+'.')][0]
xp=[p for p in glob.glob(os.path.join(d,'*.xlsx')) if not os.path.basename(p).startswith(('_','~')) and 'conflict' not in os.path.basename(p)][0]
ws=openpyxl.load_workbook(xp).active
def norm(u): return re.sub(r'\?.*','',str(u).split('#')[0]).rstrip('/').lower() if u else ''
rows=[(r,ws.cell(r,11).value) for r in range(3,ws.max_row+1) if ws.cell(r,2).value is not None and ws.cell(r,11).value]
host=re.match(r'(https?://[^/]+)', str(rows[0][1])).group(1)
sheetset=set(norm(u) for _,u in rows)
# tab widget: ul whose class contains tab and has >=2 <li><a> with hrefs
TABRE=re.compile(r'<ul[^>]*class="([^"]*(?:tab|Tab)[^"]*)"[^>]*>(.*?)</ul>', re.S)
seen=set(); groups=[]
n=0
for r,u in rows:
    if 'http' in str(u) and host.split('//')[1] not in str(u): continue   # internal only
    try: html=fetch(str(u))
    except: continue
    n+=1
    for m in TABRE.finditer(html):
        cls=m.group(1); items=[]
        for a in re.finditer(r'<a[^>]*href="([^"]*)"[^>]*>(.*?)</a>', m.group(2), re.S):
            href=a.group(1); t=re.sub(r'<[^>]+>','',a.group(2)).strip()
            if t and href and href not in ('#',) and 'javascript:void' not in href:
                full=href if href.startswith('http') else host+('' if href.startswith('/') else '/')+href
                items.append((t,full))
        if len(items)>=2:
            key=tuple(norm(h) for _,h in items)
            if key in seen: continue
            seen.add(key)
            missing=[(t,h) for t,h in items if norm(h) not in sheetset]
            if missing:
                groups.append((r,cls,items,missing))
print('site %d: %d페이지 스캔, 탭위젯그룹(미수집탭보유) %d'%(NO,n,len(groups)))
for r,cls,items,missing in groups[:25]:
    print('  r%d [%s] 미수집%d: %s'%(r,cls[:20],len(missing),' | '.join(t for t,_ in missing[:6])))