# -*- coding: utf-8 -*-
"""KISA 14행~끝 전수 sub_tab_menu 스캔. 읽기전용 보고."""
import sys, io, re, json, time
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
import requests, urllib3
from bs4 import BeautifulSoup
from urllib.parse import urljoin
from concurrent.futures import ThreadPoolExecutor, as_completed
import openpyxl
urllib3.disable_warnings()

XLSX = r"D:\01.프로젝트\DB수집\작업파일\공공기관3\4.한국인터넷진흥원\한국인터넷진흥원.xlsx"
BASE = "https://www.kisa.or.kr"
H = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36'}

wb = openpyxl.load_workbook(XLSX)
ws = wb.active
rows = []
for r in range(3, ws.max_row + 1):
    b = ws.cell(r, 2).value
    if b is None:
        break
    url = ws.cell(r, 11).value
    rows.append((r, url, ws.cell(r,4).value, ws.cell(r,5).value, ws.cell(r,6).value, ws.cell(r,7).value))

# 동향분석(r14)부터 끝까지
START_ROW = 14
scope = [x for x in rows if x[0] >= START_ROW]
print(f"스캔 대상 {len(scope)}행 (행 {START_ROW}~{rows[-1][0]})")

sess = requests.Session(); sess.headers.update(H)

def fetch(url):
    try:
        rr = sess.get(url, timeout=15, verify=False, allow_redirects=True)
        meta = re.search(rb'charset=["\']?\s*([\w-]+)', rr.content[:4096], re.I)
        rr.encoding = meta.group(1).decode(errors='ignore') if meta else rr.apparent_encoding
        if rr.status_code == 200:
            return BeautifulSoup(rr.text, 'html.parser')
    except Exception as e:
        return None
    return None

def scan(task):
    r, url = task[0], task[1]
    if not url or not isinstance(url, str) or not url.startswith('http') or 'kisa.or.kr' not in url:
        return (r, url, None)
    soup = fetch(url)
    if soup is None:
        return (r, url, 'FETCH_FAIL')
    tabs_found = []
    for ul in soup.select('ul.sub_tab_menu'):
        tabs = []
        for li in ul.find_all('li', recursive=False):
            a = li.find('a')
            if not a:
                continue
            href = a.get('href', '')
            label = a.get_text(strip=True)
            active = ('active' in (li.get('class') or [])) or ('active' in (a.get('class') or []))
            tabs.append({'label': label, 'href': urljoin(BASE, href) if href else '', 'active': active})
        if tabs:
            tabs_found.append(tabs)
    return (r, url, tabs_found if tabs_found else [])

results = {}
with ThreadPoolExecutor(max_workers=8) as ex:
    futs = [ex.submit(scan, t) for t in scope]
    for f in as_completed(futs):
        r, url, res = f.result()
        results[r] = (url, res)

out = []
for r, url, d, e, fcol, g in scope:
    url2, res = results.get(r, (url, None))
    leaf = g or fcol or e or d
    if res == 'FETCH_FAIL':
        print(f"r{r} [{leaf}] {url}  -> FETCH_FAIL")
    elif res:
        for ti, tabs in enumerate(res):
            labels = [('*' if t['active'] else '')+t['label'] for t in tabs]
            print(f"r{r} [{leaf}] {url}  -> sub_tab_menu({len(tabs)}): {labels}")
            out.append({'row': r, 'leaf': leaf, 'url': url, 'tabs': tabs})

print("\n=== sub_tab_menu 보유 행 수:", len(out))
json.dump(out, open(r"D:\01.프로젝트\DB수집\작업파일\공공기관3\_temp\_kisa_tabs.json", 'w', encoding='utf-8'), ensure_ascii=False, indent=2)