# -*- coding: utf-8 -*-
"""
범용 탭 스캐너 (읽기전용).
사용: python -X utf8 _tab_scan.py "<xlsx경로>" [--json out.json] [--workers 16]
- 시트의 K열 내부 URL들을 동시 fetch
- 각 페이지에서 '탭 컨테이너(ul/div)'를 구조 기반으로 탐지
- 탭 링크를 site/board/page/anchor 로 분류
- 탭 타깃이 시트에 없으면 '누락(missing)' 으로 집계
출력: 벤더추정 + 누락그룹 수 + 예상 추가행 수 + 그룹 상세
"""
import openpyxl, requests, sys, re, json, argparse
from bs4 import BeautifulSoup
from urllib.parse import urljoin, urlparse
from concurrent.futures import ThreadPoolExecutor, as_completed
from collections import Counter
import urllib3
urllib3.disable_warnings()
sys.stdout.reconfigure(encoding='utf-8')

UA = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'}
TAB_KEYWORDS = ('tab', 'nav_', '_nav', 'depth', 'dep4', 'slave', 'sub_tab', 'subtab', 'lnbtab', 'snbtab')
ON_CLASSES = ('on', 'active', 'select', 'current', 'sel', 'now')

def norm_url(u):
    if not u: return ''
    u = u.split('#')[0].rstrip('/')
    u = re.sub(r'^https?://(www\.)?', '', u)
    return u.lower()

def is_tab_container(ul):
    """ul이 탭 컨테이너인지 구조 기반 판정."""
    lis = ul.find_all('li', recursive=False)
    if len(lis) < 2:
        # 일부 벤더는 li 비직속. 모든 li 중 a 가진 것 사용
        lis = [li for li in ul.find_all('li') if li.find('a')]
        if len(lis) < 2:
            return None
    anchors = [li.find('a') for li in lis if li.find('a')]
    if len(anchors) < 2:
        return None
    cls = ' '.join(ul.get('class') or []).lower()
    kw = any(k in cls for k in TAB_KEYWORDS)
    has_on = any(any(o in ' '.join((li.get('class') or [])).lower() for o in ON_CLASSES) for li in lis)
    # 링크 경로 유사성: 같은 디렉토리 또는 공통 쿼리키
    hrefs = [a.get('href', '') for a in anchors if a.get('href')]
    if len(hrefs) < 2:
        return None
    paths = [urlparse(h).path for h in hrefs]
    common_dir = len(set(p.rsplit('/', 1)[0] for p in paths)) == 1
    qkeys = [tuple(sorted(re.findall(r'[?&](\w+)=', h))) for h in hrefs]
    common_q = len(set(qkeys)) == 1 and qkeys and qkeys[0]
    score = sum([kw, has_on, common_dir, bool(common_q)])
    if (kw and (has_on or common_dir or common_q)) or (has_on and (common_dir or common_q)):
        return anchors
    return None

def classify(a, href, domain):
    target = (a.get('target') or '')
    cls = ' '.join(a.get('class') or []).lower()
    host = urlparse(href).netloc.lower()
    h = href.split('#')[0]
    if not h or href.strip().startswith('#') or href.strip().lower().startswith('javascript'):
        return 'anchor'
    if target == '_blank' or 'link_3th' in cls or (host and domain not in host):
        return 'site'
    if 'selectboardlist' in h.lower() or '/bbs/' in h.lower() or 'bbsmstr' in h.lower():
        return 'board'
    return 'page'

def scan_xlsx(xlsx, workers=16, lo=3):
    class A: pass
    args = A(); args.xlsx = xlsx; args.workers = workers; args.lo = lo; args.json = None
    wb = openpyxl.load_workbook(args.xlsx)
    ws = wb.active
    # data rows
    rows = []
    for r in range(args.lo, ws.max_row + 1):
        c = ws.cell(r, 11)
        url = c.hyperlink.target if c.hyperlink else (c.value if isinstance(c.value, str) and c.value.startswith('http') else None)
        L = ws.cell(r, 12).value
        F = ws.cell(r, 6).value; G = ws.cell(r, 7).value
        if url:
            rows.append({'r': r, 'url': url, 'L': L, 'F': F, 'G': G})
    sheet_norm = set(norm_url(x['url']) for x in rows)
    # dominant domain
    hosts = Counter(urlparse(x['url']).netloc.replace('www.', '') for x in rows if urlparse(x['url']).netloc)
    domain = hosts.most_common(1)[0][0] if hosts else ''
    # internal content pages to scan
    targets = [x for x in rows if x['L'] in ('페이지', '게시판') and domain in urlparse(x['url']).netloc]
    uniq = {}
    for x in targets:
        uniq.setdefault(norm_url(x['url']), x)

    sess = requests.Session(); sess.headers.update(UA)
    def fetch(x):
        try:
            rr = sess.get(x['url'], timeout=20, verify=False)
            rr.encoding = rr.apparent_encoding or 'utf-8'
            return x, rr.text
        except Exception as e:
            return x, None

    groups = []
    container_classes = Counter()
    with ThreadPoolExecutor(max_workers=args.workers) as ex:
        futs = [ex.submit(fetch, x) for x in uniq.values()]
        for fu in as_completed(futs):
            x, html = fu.result()
            if not html: continue
            soup = BeautifulSoup(html, 'html.parser')
            best = None
            for ul in soup.find_all('ul'):
                anchors = is_tab_container(ul)
                if not anchors: continue
                # build tabs
                tabs = []
                for a in anchors:
                    href = urljoin(x['url'], a.get('href', '').strip())
                    kind = classify(a, href, domain)
                    tabs.append({'label': ' '.join(a.get_text().split())[:30], 'href': href, 'kind': kind, 'norm': norm_url(href)})
                # only real page/board tabs count
                pb = [t for t in tabs if t['kind'] in ('page', 'board')]
                if len(pb) < 2: continue
                missing = [t for t in pb if t['norm'] not in sheet_norm]
                if not missing: continue
                cand = {'cls': ' '.join(ul.get('class') or []), 'n': len(pb), 'missing': len(missing),
                        'tabs': tabs}
                if best is None or cand['missing'] > best['missing']:
                    best = cand
            if best:
                container_classes[best['cls']] += 1
                groups.append({'r': x['r'], 'F': x['F'], 'url': x['url'],
                               'cls': best['cls'], 'tabs_n': best['n'], 'missing': best['missing']})
    total_missing = sum(g['missing'] for g in groups)
    out = {
        'xlsx': args.xlsx, 'domain': domain, 'data_rows': len(rows),
        'scanned_pages': len(uniq), 'tab_groups': len(groups),
        'est_new_rows': total_missing,
        'container_classes': dict(container_classes.most_common()),
        'groups': sorted(groups, key=lambda g: -g['missing']),
    }
    return out

def main():
    ap = argparse.ArgumentParser()
    ap.add_argument('xlsx')
    ap.add_argument('--json', default=None)
    ap.add_argument('--workers', type=int, default=16)
    ap.add_argument('--lo', type=int, default=3)
    args = ap.parse_args()
    out = scan_xlsx(args.xlsx, args.workers, args.lo)
    if args.json:
        json.dump(out, open(args.json, 'w', encoding='utf-8'), ensure_ascii=False, indent=1)
    print(json.dumps({k: v for k, v in out.items() if k != 'groups'}, ensure_ascii=False, indent=1))
    print('--- top groups ---')
    for g in out['groups'][:15]:
        print(f"  R{g['r']:>3} {str(g['F'])[:18]:18} cls='{g['cls']}' tabs={g['tabs_n']} missing={g['missing']}")

if __name__ == '__main__':
    main()
