# -*- coding: utf-8 -*-
"""미지 탭 클래스 발견: 표본 페이지 본문에서 '같은 본문에 2+ 실제링크를 가진 UL'의
class 를 빈도순 집계한다. 사이트 전역 nav(거의 모든 페이지에 동일 링크집합) 제외 목적.

사용: python -X utf8 _tab_discover.py <엑셀> <도메인> [표본수=40]
"""
import sys, warnings, ssl
from collections import Counter
from urllib.parse import urlsplit
from concurrent.futures import ThreadPoolExecutor, as_completed

import openpyxl, requests
from requests.adapters import HTTPAdapter
from urllib3.util.ssl_ import create_urllib3_context
from bs4 import BeautifulSoup

warnings.filterwarnings('ignore')
H = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120 Safari/537.36'}


class WeakSSLAdapter(HTTPAdapter):
    def init_poolmanager(self, *a, **k):
        ctx = create_urllib3_context(); ctx.set_ciphers('DEFAULT@SECLEVEL=0')
        ctx.options |= 0x4; ctx.check_hostname = False; ctx.verify_mode = ssl.CERT_NONE
        k['ssl_context'] = ctx; return super().init_poolmanager(*a, **k)


S = requests.Session(); S.headers.update(H)
if '--weak-ssl' in sys.argv:
    S.mount('https://', WeakSSLAdapter())


def norm(u):
    s = urlsplit(u); return (s.path.rstrip('/')).lower()


def fetch(r, u):
    try:
        x = S.get(u, timeout=15, verify=False)
        return r, u, x.content
    except Exception:
        return r, u, None


def main():
    xlsx, domain = sys.argv[1], sys.argv[2]
    nsamp = int([a for a in sys.argv[3:] if a.isdigit()][0]) if any(a.isdigit() for a in sys.argv[3:]) else 40
    wb = openpyxl.load_workbook(xlsx, read_only=True); ws = wb.active
    urls = []
    for row in ws.iter_rows(min_row=3, min_col=11, max_col=11, values_only=True):
        u = row[0]
        if isinstance(u, str) and domain in u:
            urls.append(u)
    wb.close()
    step = max(1, len(urls) // nsamp)
    sample = urls[::step][:nsamp]
    print(f'표본 {len(sample)}/{len(urls)}  ({domain})')

    # class별 등장 페이지 수 / 링크집합 다양성
    cls_pages = Counter()
    cls_linksets = {}
    with ThreadPoolExecutor(max_workers=8) as ex:
        futs = [ex.submit(fetch, i, u) for i, u in enumerate(sample)]
        for f in as_completed(futs):
            r, u, html = f.result()
            if not html:
                continue
            soup = BeautifulSoup(html, 'html.parser')
            for ul in soup.find_all(['ul', 'ol']):
                links = []
                for a in ul.find_all('a'):
                    t = a.get_text(strip=True)
                    h = (a.get('href') or '').strip()
                    if t and h and not h.startswith(('#', 'javascript:')):
                        links.append(norm(h))
                if len(links) < 2:
                    continue
                cls = ' '.join(ul.get('class') or []).strip() or '(no-class)'
                cls_pages[cls] += 1
                cls_linksets.setdefault(cls, set()).add(tuple(links))

    print('\n=== UL/OL class별 (2+링크) — 등장페이지수 / 서로다른 링크집합수 ===')
    print('(전역 nav = 등장多+집합1 / 본문탭 = 집합 다양) \n')
    for cls, cnt in cls_pages.most_common(40):
        nset = len(cls_linksets[cls])
        flag = '★탭후보' if nset >= 3 and cnt >= 3 else ''
        print(f'  {cnt:3d}p / {nset:3d}집합  [{cls}]  {flag}')


if __name__ == '__main__':
    main()