# -*- coding: utf-8 -*-
"""공공기관 Phase 1: 사이트맵/메가메뉴 → 메뉴트리 D~K (Playwright 렌더 + 범용 중첩리스트 추출).

출력: D:\\01.프로젝트\\DB수집\\공공기관\\{기관}.xlsx  (평면배치)
시트명: {번호}_{기관}
사용: python _공공기관_phase1.py [기관명 ...]   (인자 없으면 OVERRIDES 정의된 전체)
"""
import sys, io, os, re, shutil, json, warnings
if __name__ == '__main__':
    sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
from copy import copy
from urllib.parse import urljoin, urlparse
import openpyxl
from bs4 import BeautifulSoup
from openpyxl.styles import Alignment, Font

warnings.filterwarnings('ignore')
TEMPLATE = r'D:\01.프로젝트\DB수집\자료_취합_예시.xlsx'
OUTDIR = r'D:\01.프로젝트\DB수집\작업파일\공공기관2'
PROBE = r'D:\01.프로젝트\DB수집\_스크립트\_공공기관2_probe.json'
UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'

# 기관별 오버라이드: sitemap URL + 컨테이너 셀렉터(선택) + 렌더 대기(wait). probe json을 기본값으로.
OVERRIDES = {
    '국립생태원': {'sitemap': 'https://www.nie.re.kr/nie/main/main.do', 'wait': 5000},
    '국토안전관리원': {'use_home': True, 'sel': '.all-menu', 'wait': 4500},
    '국립중앙의료원': {'use_home': True, 'sel': '#siteMap', 'wait': 4500},
    '건설근로자공제회': {'use_home': True, 'wait': 4500},
    '국민연금공단': {'sitemap': 'https://www.nps.or.kr/main.do', 'sel': '.allmenu-container', 'wait': 4000},
    '국민건강보험공단': {'sitemap': 'https://www.nhis.or.kr/nhis/index.do', 'sel': 'nav.head-gnb', 'wait': 4000},
}

JS_PATS = [
    # goMenuPage('MENUID','/realpath',...) 형: 2번째 인자가 실제 URL
    re.compile(r"""go\w*(?:Menu|Page|Move)\w*\s*\(\s*['"][^'"]*['"]\s*,\s*['"]([^'"]+)['"]""", re.I),
    re.compile(r"""go(?:Menu|SubMenu|Page|Link|Url|View|Move)\s*\(\s*['"]([^'"]+)['"]""", re.I),
    re.compile(r"""(?:location\.href|window\.location(?:\.href)?)\s*=\s*(?:encodeURI\()?\s*['"]([^'"]+)['"]"""),
    re.compile(r"""window\.open\s*\(\s*['"]([^'"]+)['"]"""),
    re.compile(r"""(?:fn_?\w*|aLink|movePage|menuMove)\s*\(\s*['"]([^'"]+)['"]""", re.I),
]


def js_href(a):
    s = (a.get('href', '') or '') + ' ' + (a.get('onclick', '') or '')
    for p in JS_PATS:
        mm = p.search(s)
        if mm:
            u = mm.group(1).strip()
            if u.startswith(('/', 'http', './', '?', '../')):
                return u
    return ''


def clean(s):
    return re.sub(r'\s+', ' ', (s or '')).strip().replace('\xa0', '')


def render(url, wait=2500):
    from playwright.sync_api import sync_playwright
    with sync_playwright() as p:
        b = p.chromium.launch()
        pg = b.new_page(user_agent=UA, viewport={'width': 1440, 'height': 2400})
        try:
            pg.goto(url, timeout=30000, wait_until='domcontentloaded')
        except Exception:
            pass
        pg.wait_for_timeout(wait)
        html = pg.content()
        final = pg.url
        b.close()
        return final, html


SITEMAP_HINTS = ('sitemap', 'site_map', 'sitemapwrap', 'site-map', 'allmenu', 'all_menu', 'all-menu',
                 'menu_all', 'menuall', 'totalmenu', 'total_menu', 'totmenu', 'full_menu', 'fullmenu',
                 'gnb_all', 'gnball')


def pick_container(soup, prefer=None):
    if prefer:
        el = soup.select_one(prefer)
        if el and len(el.find_all('a')) >= 8:
            return el
    cands = []
    for d in soup.find_all(['div', 'section', 'main', 'nav', 'ul']):
        cls = ' '.join(d.get('class', [])).lower() + ' ' + (d.get('id', '') or '').lower()
        clsn = cls.replace(' ', '').replace('-', '').replace('_', '')
        if any(x in cls for x in ['footer', 'aside']):
            continue
        ac = len(d.find_all('a'))
        ul = len(d.find_all('ul'))
        if ac < 12:
            continue
        score = ac + ul * 2
        # 사이트맵/전체메뉴 컨테이너 강력 우대
        if any(h in clsn for h in SITEMAP_HINTS):
            score += 1000
        # 전역 헤더/상단 gnb는 (사이트맵 페이지에선) 감점
        if 'header' in cls or ('gnb' in clsn and not any(h in clsn for h in SITEMAP_HINTS)):
            score -= 200
        cands.append((score, ac, d))
    if not cands:
        return None
    cands.sort(key=lambda x: x[0], reverse=True)
    return cands[0][2]


NOISE_LABELS = {'메뉴없음', '메뉴 없음', '홈', 'home', 'home으로', '홈으로', 'eng', 'english', '로그인',
                'login', '회원가입', '검색', 'search', '바로가기', '본문바로가기', '닫기', 'close', '전체메뉴',
                '사이트맵', 'sitemap'}


def node_depth(el, container):
    """container 내부에서 el 위의 ul/ol/dl 조상 개수 = 중첩 깊이."""
    d = 0
    p = el.parent
    while p is not None and p is not container:
        if getattr(p, 'name', None) in ('ul', 'ol', 'dl'):
            d += 1
        p = p.parent
    return d


def parse_table_sitemap(table):
    """테이블형 사이트맵: tr > th(대분류, 빈칸=이어받기) + th(중분류 a) + td(소분류 a들)."""
    rows = []
    curD = ''
    for tr in table.find_all('tr'):
        ths = tr.find_all('th', recursive=False)
        if ths:
            d_txt = clean(ths[0].get_text())
            # 첫 th에 a가 없고 span/텍스트면 대분류 라벨(빈칸이면 이어받기)
            if d_txt and not (len(ths) == 1 and ths[0].find('a')):
                # 첫 th가 곧 중분류 a 단독인 경우는 제외 위해 a 유무 확인
                if not ths[0].find('a') or len(ths) >= 2:
                    if not ths[0].find('a'):
                        curD = d_txt
        E = ''; Eh = ''
        if len(ths) >= 2:
            ea = ths[1].find('a')
            if ea:
                E = clean(ea.get_text()); Eh = js_href(ea) or (ea.get('href') or '')
        elif len(ths) == 1 and ths[0].find('a'):
            ea = ths[0].find('a'); E = clean(ea.get_text()); Eh = js_href(ea) or (ea.get('href') or '')
        td = tr.find('td')
        leaves = td.find_all('a') if td else []
        if leaves:
            for la in leaves:
                lab = clean(la.get_text())
                if not lab:
                    continue
                rows.append({'path': [curD, E, lab], 'href': js_href(la) or (la.get('href') or '')})
        elif E:
            rows.append({'path': [curD, E], 'href': Eh})
    return [r for r in rows if r['path'] and any(r['path'])]


def extract_rows(container):
    """앵커 중심 깊이추출: 각 a/heading의 ul조상 수로 컬럼 결정(정규화). 범용. 테이블형 자동전환."""
    tbls = container.find_all('table')
    if tbls and sum(len(t.find_all('th')) for t in tbls) >= 4 and len(container.find_all('ul')) <= 1:
        rows = []
        for t in tbls:
            rows += parse_table_sitemap(t)
        if rows:
            return rows
    nodes = []  # [depth, label, href]
    for el in container.find_all(['a', 'button', 'h2', 'h3', 'h4', 'h5', 'h6', 'dt']):
        name = el.name
        if name == 'a':
            href = (el.get('href') or '').strip()
            if href.startswith('#') or href.lower().startswith('javascript:') or not href:
                href = js_href(el)
            label = clean(el.get_text())
        elif name == 'button':
            cls = ' '.join(el.get('class', [])).lower()
            if not (el.get('aria-expanded') is not None or el.get('aria-haspopup')
                    or any(x in cls for x in ('depth', 'trigger', 'gnb', '1d', '2d', '3d', 'menu'))):
                continue
            href = ''
            label = clean(el.get('data-dir') or el.get('title') or el.get_text())
        else:
            if el.find('a'):  # heading 안의 a는 따로 잡힘 → 중복방지
                continue
            href = ''
            label = clean(el.get_text())
        if not label or label.lower() in NOISE_LABELS or len(label) > 60:
            continue
        nodes.append([node_depth(el, container), label, href])
    # 연속 중복 제거
    dedup = []
    for n in nodes:
        if dedup and dedup[-1] == n:
            continue
        dedup.append(n)
    nodes = dedup
    if not nodes:
        return []
    mind = min(n[0] for n in nodes)
    cols = [min(n[0] - mind, 6) for n in nodes]
    out = []
    path = [''] * 7
    for i, (depth, label, href) in enumerate(nodes):
        c = cols[i]
        path[c] = label
        for k in range(c + 1, 7):
            path[k] = ''
        is_leaf = (i == len(nodes) - 1) or (cols[i + 1] <= c)
        if href or is_leaf:
            out.append({'path': path[:c + 1], 'href': href})
    return out


def rows_to_dicts(raw):
    out = []
    for it in raw:
        p = it['path']
        href = it.get('href', '')
        if href.startswith('#') or href.lower().startswith('javascript:'):
            href = ''
        row = {'D': '', 'E': '', 'F': '', 'G': '', 'H': '', 'I': '', 'J': '', 'href': href}
        for i, lab in enumerate(p[:7]):
            row['DEFGHIJ'[i]] = lab
        out.append(row)
    return out


def write_excel(name, num, base, raw_rows):
    domain = urlparse(base).netloc
    output = os.path.join(OUTDIR, f'{num}.{name}', f'{name}.xlsx')

    def abs_url(href):
        if not href:
            return ''
        href = href.strip()
        if href.startswith(('javascript:', '#')):
            return ''
        if href.startswith(('http://', 'https://')):
            return href
        return urljoin(base + '/', href)

    def is_external(url):
        return url.startswith(('http://', 'https://')) and domain not in url

    # 부모-자식 URL 중복 제거 (전 깊이) — 부모행 D~lc 동일 + lc+1 채워짐 + K동일
    def colval(r, c):
        return r.get(c, '') or ''
    final = []
    i = 0
    removed = 0
    cols = 'DEFGHIJ'
    while i < len(raw_rows):
        row = raw_rows[i]
        # 마지막 채워진 컬럼 찾기
        lc = -1
        for ci, c in enumerate(cols):
            if colval(row, c) != '':
                lc = ci
        dup = False
        if i + 1 < len(raw_rows) and lc >= 0 and lc < 6:
            nxt = raw_rows[i + 1]
            same = all(colval(row, cols[k]) == colval(nxt, cols[k]) for k in range(lc + 1))
            if (same and colval(nxt, cols[lc + 1]) != '' and colval(row, cols[lc + 1]) == ''
                    and row.get('href', '') == nxt.get('href', '')):
                dup = True
        if dup:
            removed += 1
            i += 1
            continue
        final.append(row)
        i += 1

    if not final:
        print(f'  [{name}] 행 0개 — 스킵')
        return 0
    shutil.copy(TEMPLATE, output)
    wb = openpyxl.load_workbook(output)
    ws = wb.active
    ws.title = f'{num:02d}_{name}'
    HEADER = {'B1:R1', 'S1:W1', 'Y1:AA1'}
    for rng in [str(m) for m in ws.merged_cells.ranges if str(m) not in HEADER]:
        ws.unmerge_cells(rng)
    for row in ws.iter_rows(min_row=3, max_row=ws.max_row, min_col=1, max_col=ws.max_column):
        for cell in row:
            cell.value = None
    START = 3
    template_r = 3
    cur_max = ws.max_row
    for idx, item in enumerate(final, start=START):
        if idx > cur_max:
            for c in range(1, ws.max_column + 1):
                srcc = ws.cell(template_r, c)
                tgt = ws.cell(idx, c)
                if srcc.has_style:
                    tgt.font = copy(srcc.font); tgt.fill = copy(srcc.fill)
                    tgt.border = copy(srcc.border); tgt.alignment = copy(srcc.alignment)
                    tgt.number_format = srcc.number_format; tgt.protection = copy(srcc.protection)
        url = abs_url(item.get('href', ''))
        ws.cell(idx, 2).value = idx - 2
        ws.cell(idx, 3).value = name
        for ci, c in enumerate('DEFGHIJ'):
            ws.cell(idx, 4 + ci).value = item.get(c, '')
        ws.cell(idx, 11).value = url
        if is_external(url):
            ws.cell(idx, 19).value = '외부링크'
    END = START + len(final) - 1
    center = Alignment(horizontal='center', vertical='center', wrap_text=True)
    left = Alignment(horizontal='left', vertical='center', wrap_text=False)

    def merge_runs(col_letter, col_idx, group_cols=()):
        runs = []
        cur_val = ws.cell(START, col_idx).value
        cur_grp = tuple(ws.cell(START, g).value for g in group_cols)
        run_start = START
        for r in range(START + 1, END + 1):
            v = ws.cell(r, col_idx).value
            g = tuple(ws.cell(r, gg).value for gg in group_cols)
            if v == cur_val and g == cur_grp:
                continue
            if cur_val not in (None, '') and r - 1 > run_start:
                runs.append((run_start, r - 1))
            cur_val, cur_grp, run_start = v, g, r
        if cur_val not in (None, '') and END > run_start:
            runs.append((run_start, END))
        for s, e in runs:
            ws.merge_cells(f'{col_letter}{s}:{col_letter}{e}')
            ws.cell(s, col_idx).alignment = center
        return len(runs)

    n_f = merge_runs('F', 6, group_cols=(4, 5))
    n_e = merge_runs('E', 5, group_cols=(4,))
    n_d = merge_runs('D', 4)
    for r in range(START, END + 1):
        for c in (4, 5, 6, 7):
            if ws.cell(r, c).value is not None:
                ws.cell(r, c).alignment = center
    for r in range(1, END + 1):
        ws.row_dimensions[r].height = 15
    link_n = 0
    for r in range(START, END + 1):
        cell = ws.cell(r, 11)
        u = cell.value
        if u and isinstance(u, str) and u.startswith(('http://', 'https://')):
            cell.hyperlink = u
            old = cell.font
            cell.font = Font(name=old.name or '맑은 고딕', size=old.size or 11,
                             bold=old.bold, italic=old.italic, color='0000FF', underline='single')
            cell.alignment = left
            link_n += 1
    wb.save(output)
    ext_n = sum(1 for r in range(START, END + 1) if ws.cell(r, 19).value == '외부링크')
    print(f'  [{name}] 원본{len(raw_rows)}→중복{removed}→{len(final)}행 | 병합D{n_d}E{n_e}F{n_f} | 외부{ext_n} K링크{link_n} → {output}')
    return len(final)


def load_probe():
    with open(PROBE, encoding='utf-8') as f:
        return {r['name']: r for r in json.load(f)}


def main():
    probe = load_probe()
    targets = sys.argv[1:] if len(sys.argv) > 1 else list(probe.keys())
    for name in targets:
        p = probe.get(name)
        if not p:
            print(f'[{name}] probe 정보 없음 — 스킵'); continue
        ov = OVERRIDES.get(name, {})
        num = int(p['num'])
        base = p['base']
        if ov.get('use_home'):
            sm = p['home']
        else:
            sm = ov.get('sitemap') or p.get('sitemap') or p['home']
        wait = ov.get('wait', 3000)
        print(f"\n=== {num}.{name} === render {sm}")
        try:
            final, html = render(sm, wait=wait)
            soup = BeautifulSoup(html, 'html.parser')
            cont = pick_container(soup, ov.get('sel'))
            if not cont:
                print(f'  [{name}] 컨테이너 못찾음 (a없음)'); continue
            raw = extract_rows(cont)
            dicts = rows_to_dicts(raw)
            write_excel(name, num, base, dicts)
        except Exception as e:
            import traceback
            print(f'  [{name}] 실패: {e}')
            traceback.print_exc()


if __name__ == '__main__':
    main()
