DB_JOB/_스크립트/_probe_chungbuk.py

"""충청북도 11개 시·군 사이트맵 URL/패턴 탐지."""
import re
import warnings
from urllib.parse import urljoin, urlparse
from concurrent.futures import ThreadPoolExecutor, as_completed

import requests
from bs4 import BeautifulSoup

warnings.filterwarnings('ignore')
UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
H = {'User-Agent': UA}

CITIES = [
    ('괴산군', 'https://www.goesan.go.kr/www/index.do'),
    ('단양군', 'https://www.danyang.go.kr/dy21/1'),
    ('보은군', 'https://www.boeun.go.kr/www/index.do'),
    ('영동군', 'https://www.yd21.go.kr/'),
    ('옥천군', 'https://www.oc.go.kr/www/'),
    ('음성군', 'https://www.eumseong.go.kr/www/index.do'),
    ('제천시', 'https://www.jecheon.go.kr/www/index.do'),
    ('증평군', 'https://www.jp.go.kr/kor.do'),
    ('진천군', 'https://www.jincheon.go.kr/home/intro.do'),
    ('청주시', 'https://www.cheongju.go.kr/www/index.do'),
    ('충주시', 'https://www.chungju.go.kr/www/index.do'),
]


def fetch(url, timeout=12):
    try:
        r = requests.get(url, headers=H, timeout=timeout, verify=False, allow_redirects=True)
        r.encoding = r.apparent_encoding
        if r.status_code == 200:
            return r.url, r.text
    except Exception as e:
        return None, f'ERR: {e}'
    return None, f'HTTP {r.status_code}'


def find_sitemap_anchor(html, base):
    soup = BeautifulSoup(html, 'html.parser')
    found = []
    for a in soup.find_all('a', href=True):
        txt = a.get_text(strip=True)
        href = a['href']
        if not (txt and href):
            continue
        if '사이트맵' in txt or '전체메뉴' in txt or 'sitemap' in href.lower() or 'allMenu' in href:
            full = urljoin(base, href)
            found.append((txt, full))
    seen = set()
    uniq = []
    for t, u in found:
        if u not in seen and u != base:
            seen.add(u)
            uniq.append((t, u))
    return uniq


def analyze_sitemap_page(html):
    """Score candidate containers by anchor count and class hints."""
    soup = BeautifulSoup(html, 'html.parser')
    candidates = []
    # Common containers
    for sel in [
        'div.sitemap.type1', 'div.sitemap.type2', 'div.sitemap',
        'div.sitemap_grep', 'div.amThum', 'ul.sitemap_list',
        '#sitemap', '#contents ul.sitemap', 'ul.sitemap',
        'ul.depth1_ul', 'ul.depth1-ul', 'ul.depth1',
        'ul.top_menu', '#gnb', 'nav#gnb', 'nav.gnb',
        '.allmenu', '.allMenu', '#allMenu',
        'div.menu_all', 'div.totalMenu',
    ]:
        for el in soup.select(sel):
            ac = len(el.find_all('a'))
            if ac < 30:
                continue
            cls = ' '.join(el.get('class', []))
            eid = el.get('id', '')
            candidates.append((ac, sel, f'{el.name}#{eid}.{cls}'))
    # Fallback: search any container by id/class containing 'sitemap' or 'allMenu'
    for el in soup.find_all(True, class_=True):
        cls = ' '.join(el.get('class', []))
        if re.search(r'\b(sitemap|allmenu|amthum|depth1)\b', cls, re.I):
            ac = len(el.find_all('a'))
            if 30 <= ac <= 2000:
                candidates.append((ac, f'class~{cls[:30]}', f'{el.name}.{cls}'))
    candidates.sort(reverse=True)
    return candidates[:5]


def probe(name, base):
    print(f'\n=== {name} === {base}')
    real, html = fetch(base)
    if not real:
        print(f'  메인 실패: {html}')
        return name, None
    print(f'  메인 OK: {real}')
    # 1) Find sitemap links from main page
    links = find_sitemap_anchor(html, real)
    print(f'  사이트맵 링크 후보: {len(links)}')
    for t, u in links[:5]:
        print(f'    "{t}" → {u}')
    # 2) Common paths to try
    parsed = urlparse(base)
    origin = f'{parsed.scheme}://{parsed.netloc}'
    common = [
        '/www/sitemap.do', '/www/sub.do?key=121', '/www/contents.do?key=121',
        '/kor/sitemap.do', '/kor/sitemap_11.do', '/kor/sitemap_1.do',
        '/sitemap.do', '/sitemap.html',
        '/www/sitemap/', '/main/sitemap.do',
        '/home/sitemap.do', '/dy21/sitemap.do',
        '/www/cms/sitemap.do',
    ]
    candidates = [u for _, u in links] + [origin + p for p in common]
    seen = set()
    best = None
    for c in candidates:
        if c in seen:
            continue
        seen.add(c)
        u2, h2 = fetch(c, timeout=10)
        if not u2:
            continue
        info = analyze_sitemap_page(h2)
        if info:
            top = info[0]
            if best is None or top[0] > best[1][0]:
                best = (c, top, info)
    if best:
        url, top, info = best
        print(f'  ★ 사이트맵 URL: {url}')
        for ac, sel, desc in info:
            print(f'    {sel}: a={ac} {desc[:80]}')
        return name, {'url': url, 'best_sel': info[0][1], 'a_count': info[0][0]}
    print('  사이트맵 못 찾음')
    return name, None


def main():
    results = {}
    with ThreadPoolExecutor(max_workers=6) as ex:
        futs = {ex.submit(probe, n, b): n for n, b in CITIES}
        for f in as_completed(futs):
            n, info = f.result()
            results[n] = info
    print('\n=== 요약 ===')
    for n, _ in CITIES:
        info = results.get(n)
        if info:
            print(f'  {n}: {info["url"]} [{info["best_sel"]}, a={info["a_count"]}]')
        else:
            print(f'  {n}: NOT FOUND')


if __name__ == '__main__':
    main()