DB_JOB/_스크립트/_공공기관_colfix_kalis.py

# -*- coding: utf-8 -*-
"""국토안전관리원 컬럼(D~J) 재구성 — all-menu dl/dt/dd 4단 트리에서 URL별 경로 추출 후 in-place 갱신.
L~Q·K·B·C 보존(URL 매칭). 백업 후 실행. 사용: python _공공기관_colfix_kalis.py
"""
import os, re, shutil, importlib.util, warnings
from copy import copy
from urllib.parse import urljoin
import openpyxl
from bs4 import BeautifulSoup
from openpyxl.styles import Alignment
warnings.filterwarnings('ignore')
spec = importlib.util.spec_from_file_location('p1', r'D:\01.프로젝트\DB수집\_스크립트\_공공기관_phase1.py')
m1 = importlib.util.module_from_spec(spec); spec.loader.exec_module(m1)
XLSX = r'D:\01.프로젝트\DB수집\공공기관\국토안전관리원.xlsx'
BASE = 'https://www.kalis.or.kr'


def clean(s):
    return re.sub(r'\s+', ' ', (s or '')).strip()


def walk(dl, path, rows):
    dt = dl.find('dt', recursive=False)
    a = dt.find('a') if dt else None
    label = clean(a.get_text()) if a else (clean(dt.get_text()) if dt else '')
    if label in ('전체메뉴 닫기', '닫기', '전체메뉴', ''):
        label = ''
    np = path + [label] if label else path
    dd = dl.find('dd', recursive=False)
    if not dd:
        return
    for ul in dd.find_all('ul', recursive=False):
        for li in ul.find_all('li', recursive=False):
            cdl = li.find('dl', recursive=False)
            if cdl:
                walk(cdl, np, rows)
            else:
                la = li.find('a')
                if la:
                    href = (la.get('href') or '').strip()
                    if href and not href.startswith(('#', 'javascript')):
                        rows.append({'path': np + [clean(la.get_text())], 'href': urljoin(BASE + '/', href)})


def render_expanded(url):
    """all-menu의 지연로드 섹션을 모두 클릭 펼친 뒤 HTML 반환."""
    from playwright.sync_api import sync_playwright
    with sync_playwright() as p:
        b = p.chromium.launch()
        pg = b.new_page(user_agent=m1.UA, viewport={'width': 1440, 'height': 2400})
        pg.goto(url, timeout=30000, wait_until='domcontentloaded')
        pg.wait_for_timeout(3500)
        # 전체메뉴 열기 버튼
        for sel in ['button[class*=allmenu]', 'a[class*=allmenu]', '[data-selector=allMenu] button', 'button[aria-label*=전체]']:
            try:
                el = pg.query_selector(sel)
                if el:
                    el.click(timeout=1500); pg.wait_for_timeout(800)
            except Exception:
                pass
        # 모든 펼침 토글 여러 패스 클릭
        for action in ['depth1', 'depth2', 'dropDown']:
            for _ in range(2):
                els = pg.query_selector_all(f'[data-action={action}]')
                for el in els:
                    try:
                        el.click(timeout=800); pg.wait_for_timeout(120)
                    except Exception:
                        pass
        pg.wait_for_timeout(500)
        # 혹시 남은 hidden 제거
        try:
            pg.evaluate("document.querySelectorAll('[hidden]').forEach(e=>e.removeAttribute('hidden'))")
        except Exception:
            pass
        html = pg.content()
        b.close()
        return html


def main():
    html = render_expanded(BASE + '/')
    soup = BeautifulSoup(html, 'html.parser')
    cont = soup.select_one('.all-menu')
    rows = []
    for dl in cont.select('dl.depth-dl'):
        walk(dl, [], rows)
    # URL -> path (대,중,소,leaf...) ; 첫 라벨은 빈문자('사업'의 부모 None일 수 있어 제거)
    url2path = {}
    for r in rows:
        p = [x for x in r['path'] if x]
        url2path[r['href'].rstrip('/')] = p
    print(f'추출 {len(rows)}행, URL맵 {len(url2path)}')

    shutil.copy(XLSX, XLSX.replace('.xlsx', '_backup_컬럼재구성전.xlsx'))
    wb = openpyxl.load_workbook(XLSX)
    ws = wb.active
    # 데이터행
    END = 2
    for r in range(3, ws.max_row + 1):
        if ws.cell(r, 2).value is None:
            break
        END = r
    # L~Q 스냅샷(검증용)
    snap = {r: [ws.cell(r, c).value for c in range(12, 18)] for r in range(3, END + 1)}
    # 먼저 병합 해제(MergedCell 쓰기금지 회피)
    HEADER = {'B1:R1', 'S1:W1', 'Y1:AA1'}
    for rng in [str(mm) for mm in ws.merged_cells.ranges if str(mm) not in HEADER]:
        ws.unmerge_cells(rng)
    matched = 0
    for r in range(3, END + 1):
        u = (ws.cell(r, 11).value or '').rstrip('/')
        p = url2path.get(u)
        if not p:
            continue
        # D~J 클리어 후 경로 기입(최대 7열 D..J)
        for c in range(4, 11):
            ws.cell(r, c).value = None
        for i, lab in enumerate(p[:7]):
            ws.cell(r, 4 + i).value = lab
        matched += 1
    center = Alignment(horizontal='center', vertical='center', wrap_text=True)

    def merge(col, ci, groups):
        runs = []; cv = ws.cell(3, ci).value; cg = tuple(ws.cell(3, g).value for g in groups); rs = 3
        for r in range(4, END + 1):
            v = ws.cell(r, ci).value; g = tuple(ws.cell(r, gg).value for gg in groups)
            if v == cv and g == cg:
                continue
            if cv not in (None, '') and r - 1 > rs:
                runs.append((rs, r - 1))
            cv, cg, rs = v, g, r
        if cv not in (None, '') and END > rs:
            runs.append((rs, END))
        for s, e in runs:
            ws.merge_cells(f'{col}{s}:{col}{e}'); ws.cell(s, ci).alignment = center
    merge('F', 6, (4, 5)); merge('E', 5, (4,)); merge('D', 4, ())
    for r in range(3, END + 1):
        for c in (4, 5, 6, 7):
            if ws.cell(r, c).value is not None:
                ws.cell(r, c).alignment = center
    wb.save(XLSX)
    # 검증: L~Q 불변
    wb2 = openpyxl.load_workbook(XLSX, data_only=True); ws2 = wb2.active
    diff = sum(1 for r in range(3, END + 1) if [ws2.cell(r, c).value for c in range(12, 18)] != snap[r])
    print(f'URL매칭 {matched}/{END-2}행 D~J재구성 | L~Q변경 {diff}행(0이어야 정상)')


if __name__ == '__main__':
    main()