DB_JOB/_스크립트/_gongju_recurse.py

# -*- coding: utf-8 -*-
"""공주시 전용: type1 탭 메뉴를 재귀 크롤링해 각 시트 행의 '진짜 수량'(잎 페이지 수)을
끝까지 세서 재계산. 잎 weight = 인페이지 #nav 개수(>=2) 또는 1.
하위 subtree에 게시판이 하나라도 있으면 그 행은 '합치지 않음'으로 플래그(M 미변경).

판별:
  · type1 메뉴 = ul.tab-ul(단 type3/#nav 제외) 안 같은도메인 .do 링크 집합 중 self 포함하는 것
  · #nav 개수 = ul.tab-ul 안 href^='#' 탭 수
  · 게시판 = 목록table/페이징/총N건/상세링크(view.do·mode=V·nttId·BBSMSTR)

사용: python -X utf8 _gongju_recurse.py            (DRY 리포트)
      python -X utf8 _gongju_recurse.py --write     (M 갱신)
"""
import re, sys, shutil, warnings
from urllib.parse import urljoin, urlsplit
from concurrent.futures import ThreadPoolExecutor, as_completed
import openpyxl, requests
from bs4 import BeautifulSoup
warnings.filterwarnings('ignore')

XLSX = r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청남도\2.공주시\충청남도_공주시.xlsx'
DOMAIN = 'gongju.go.kr'
H = {'User-Agent': 'Mozilla/5.0 Chrome/120 Safari/537.36'}
DETAIL = re.compile(r'(view\.do|mode=V|nttId|BBSMSTR|selectBoard|selectBbs)', re.I)
TOTAL = re.compile(r'총\s*[\d,]+\s*건')

S = requests.Session(); S.headers.update(H)
CACHE = {}   # url -> (menu_set, menu_list, nav, is_board)


def norm(u):
    s = urlsplit(u); return urljoin('http://x/', s.path).split('//', 1)[-1].rstrip('/').lower()


def analyze(url):
    try:
        html = S.get(url, timeout=15, verify=False).content
    except Exception:
        return (frozenset(), [], 0, False)
    soup = BeautifulSoup(html, 'html.parser')
    self_n = norm(url)
    nav = 0
    type1_groups = []
    for ul in soup.find_all('ul'):
        cls = ' '.join(ul.get('class') or []).lower()
        if 'tab-ul' not in cls:
            continue
        do_links = []
        navc = 0
        for a in ul.find_all('a'):
            h = (a.get('href') or '').strip()
            t = a.get_text(strip=True)
            if not t:
                continue
            if h.startswith('#'):
                navc += 1
            elif h and not h.startswith('javascript:'):
                au = urljoin(url, h)
                if DOMAIN in au and au.lower().endswith('.do'):
                    do_links.append((t, au))
        if navc >= 2:
            nav = max(nav, navc)
        if len(do_links) >= 2:
            type1_groups.append(do_links)
    # self 포함하는 type1 그룹 우선, 없으면 최대
    menu = []
    for g in type1_groups:
        if any(norm(u) == self_n for _, u in g):
            menu = g; break
    if not menu and type1_groups:
        menu = max(type1_groups, key=len)
    menu_set = frozenset(norm(u) for _, u in menu)
    # 게시판 판정
    body = soup.select_one('#txt') or soup
    txt = body.get_text(' ', strip=True)
    is_board = bool(TOTAL.search(txt)) or bool(body.select('.paging,.pagination,.board_paging')) \
        or any(DETAIL.search(a['href']) for a in body.find_all('a', href=True))
    return (menu_set, menu, nav, is_board)


def get(url):
    n = norm(url)
    if n not in CACHE:
        CACHE[n] = analyze(url)
    return CACHE[n]


def crawl(seed_urls):
    seen = set()
    queue = list(seed_urls)
    while queue:
        batch = [u for u in queue if norm(u) not in seen]
        for u in batch:
            seen.add(norm(u))
        queue = []
        with ThreadPoolExecutor(max_workers=8) as ex:
            futs = {ex.submit(get, u): u for u in batch}
            for f in as_completed(futs):
                _ms, menu, _nav, _b = f.result()
                for _, cu in menu:
                    if norm(cu) not in seen:
                        queue.append(cu)


def weight(url):
    _ms, _menu, nav, _b = get(url)
    return nav if nav >= 2 else 1


def expand(url, parent_set, path):
    n = norm(url)
    if n in path:
        return 1, False
    ms, menu, nav, is_board = get(url)
    if not menu or ms == parent_set:
        return (nav if nav >= 2 else 1), is_board
    tot = 0; board = is_board
    for _, cu in menu:
        w, b = expand(cu, ms, path | {n})
        tot += w; board = board or b
    return tot, board


def main():
    write = '--write' in sys.argv
    wb = openpyxl.load_workbook(XLSX); ws = wb.active
    rows = []
    seeds = []
    for r in range(3, ws.max_row + 1):
        u = ws.cell(r, 11).value
        if isinstance(u, str) and DOMAIN in u and u.lower().endswith('.do'):
            rows.append((r, u, ws.cell(r, 12).value, ws.cell(r, 13).value,
                         ws.cell(r, 6).value or ws.cell(r, 5).value or ws.cell(r, 4).value or ''))
            seeds.append(u)
    print(f'시트 .do 페이지행: {len(rows)}  — 재귀 크롤 시작...')
    crawl(seeds)
    print(f'크롤한 고유 페이지: {len(CACHE)}\n')

    changes = []; boards = []
    for r, u, L, M, cat in rows:
        if L != '페이지':
            continue
        newM, hasboard = expand(u, frozenset({norm(u)}), set())
        if hasboard:
            boards.append((r, cat, u, M, newM))
            continue
        if str(newM) != str(M):
            changes.append((r, cat, M, newM, u))

    changes.sort(key=lambda x: -(x[3] - (x[2] or 0)))
    print(f'=== 수량 변경(증가/감소) 대상: {len(changes)}행 ===')
    for r, cat, oldM, newM, u in changes:
        print(f'  행{r} {cat[:24]:<24} M {oldM} → {newM}   {u}')
    if boards:
        print(f'\n=== ⚠ 하위에 게시판 있어 합치지 않음(M 보류): {len(boards)}행 ===')
        for r, cat, u, M, nm in boards:
            print(f'  행{r} {cat[:24]:<24} (현 M={M}, 재귀={nm}) {u}')

    if write and changes:
        bak = XLSX.replace('.xlsx', '_backup_recurse전.xlsx')
        shutil.copy(XLSX, bak)
        for r, cat, oldM, newM, u in changes:
            ws.cell(r, 13).value = newM
        wb.save(XLSX)
        print(f'\n저장 완료({len(changes)}행 M갱신). 백업: {bak}')
    elif not write:
        print('\n(DRY — 적용하려면 --write)')


if __name__ == '__main__':
    main()