DB_JOB/_스크립트/_probe_chungbuk3.py

"""실패한 4개 사이트 (단양·보은·영동·진천) 심층 탐색."""
import re
import ssl
import warnings
import requests
from requests.adapters import HTTPAdapter
from urllib3.util.ssl_ import create_urllib3_context
from bs4 import BeautifulSoup

warnings.filterwarnings('ignore')
UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
H = {'User-Agent': UA}


class WeakSSLAdapter(HTTPAdapter):
    """레거시 SSL/TLS handshake를 허용하는 어댑터 (영동군 등 구형 SSL)."""
    def init_poolmanager(self, *args, **kwargs):
        ctx = create_urllib3_context()
        ctx.set_ciphers('DEFAULT@SECLEVEL=0')
        ctx.options |= 0x4  # ssl.OP_LEGACY_SERVER_CONNECT (Python 3.12+)
        ctx.check_hostname = False
        ctx.verify_mode = ssl.CERT_NONE
        kwargs['ssl_context'] = ctx
        return super().init_poolmanager(*args, **kwargs)


def make_session(weak_ssl=False):
    s = requests.Session()
    s.headers.update(H)
    if weak_ssl:
        s.mount('https://', WeakSSLAdapter())
    return s


def fetch(url, session=None, timeout=20):
    s = session or requests.Session()
    if not session:
        s.headers.update(H)
    try:
        r = s.get(url, timeout=timeout, verify=False, allow_redirects=True)
        r.encoding = r.apparent_encoding
        return r.status_code, r.url, r.text
    except Exception as e:
        return 0, str(e), ''


def outline(el, d=0, max_lines=40, lines=None):
    if lines is None: lines = []
    if len(lines) >= max_lines: return lines
    name = el.name
    cls = ' '.join(el.get('class', []))
    eid = el.get('id', '')
    lbl = name
    if eid: lbl += f'#{eid}'
    if cls: lbl += '.' + cls.replace(' ', '.')
    if name == 'a':
        t = el.get_text(strip=True)[:40]
        h = el.get('href', '')[:80]
        lines.append('  ' * d + f'{lbl} "{t}" → {h}')
    else:
        lines.append('  ' * d + lbl)
    for c in el.find_all(recursive=False):
        if c.name in ('script', 'style'): continue
        outline(c, d+1, max_lines, lines)
        if len(lines) >= max_lines: return lines
    return lines


def analyze(html):
    soup = BeautifulSoup(html, 'html.parser')
    best = (0, '', '', None)
    for sel in [
        'div.depth.depth1', 'div.depth1', '#sitemap div.site_map_col', '#sitemap',
        'div.sitemap_grep', 'ul.sitemap_list', 'div.sitemap_box',
        'ul.depth1_ul', 'ul.depth1-ul', 'ul.depth1',
        'ul.sitemap', 'div.sitemap', 'div.amThum', '.allMenu', 'div.menu_all',
        'nav#gnb', 'nav.gnb', '#gnb',
    ]:
        for el in soup.select(sel):
            ac = len(el.find_all('a'))
            if ac > best[0]:
                cls = ' '.join(el.get('class', []))
                best = (ac, sel, cls, el)
    return best


# 단양군 - try /dy21/98
print('='*70)
print('단양군 — /dy21/98')
print('='*70)
sess = make_session()
code, real, html = fetch('https://www.danyang.go.kr/dy21/98', sess)
print(f'  HTTP {code}, len {len(html)}')
if code == 200:
    soup = BeautifulSoup(html, 'html.parser')
    # Search for the sitemap container
    print(f'  All elements w/ a>=50:')
    for el in soup.find_all(['div', 'ul', 'nav']):
        ac = len(el.find_all('a'))
        if 50 <= ac:
            cls = ' '.join(el.get('class', []))[:50]
            eid = el.get('id', '')
            print(f'    {el.name}#{eid}.{cls} a={ac}')
    best = analyze(html)
    if best[3]:
        print(f'  Best: {best[1]} cls={best[2]} a={best[0]}')
        for line in outline(best[3], max_lines=40):
            print('   ', line)


# 진천군 - try /home/main.do
print('\n' + '='*70)
print('진천군 — /home/main.do')
print('='*70)
code, real, html = fetch('https://www.jincheon.go.kr/home/main.do', sess)
print(f'  HTTP {code}, len {len(html)}')
if code == 200:
    soup = BeautifulSoup(html, 'html.parser')
    cands = []
    for a in soup.find_all('a', href=True):
        txt = a.get_text(strip=True)
        if '사이트맵' in txt or '전체메뉴' in txt or '누리집' in txt:
            from urllib.parse import urljoin
            cands.append((txt, urljoin(real, a['href'])))
    print('  사이트맵 후보:')
    for t, u in cands[:8]:
        print(f'    "{t}" → {u}')
    # Common paths
    from urllib.parse import urlparse
    parsed = urlparse(real)
    origin = f'{parsed.scheme}://{parsed.netloc}'
    test_urls = [u for _, u in cands] + [
        origin + '/home/sitemap.do',
        origin + '/home/contents.do?key=121',
        origin + '/home/sub.do?key=121',
    ]
    for url in test_urls:
        c2, r2, h2 = fetch(url, sess)
        if c2 != 200:
            continue
        best = analyze(h2)
        if best[0] >= 50:
            print(f'  ★ {url}: {best[1]} cls={best[2]} a={best[0]}')


# 보은군 - retry
print('\n' + '='*70)
print('보은군 — retry')
print('='*70)
code, real, html = fetch('https://www.boeun.go.kr/www/index.do', sess)
print(f'  HTTP {code}, len {len(html)}')
if code == 200:
    soup = BeautifulSoup(html, 'html.parser')
    cands = []
    for a in soup.find_all('a', href=True):
        txt = a.get_text(strip=True)
        if '사이트맵' in txt or '전체메뉴' in txt or '누리집 지도' in txt:
            from urllib.parse import urljoin
            cands.append((txt, urljoin(real, a['href'])))
    print('  사이트맵 후보:')
    for t, u in cands[:8]:
        print(f'    "{t}" → {u}')
    # Try probable paths
    from urllib.parse import urlparse
    parsed = urlparse(real)
    origin = f'{parsed.scheme}://{parsed.netloc}'
    test_urls = [u for _, u in cands] + [
        origin + '/www/sitemap.do',
        origin + '/www/sub.do?key=121',
    ]
    for url in set(test_urls):
        c2, r2, h2 = fetch(url, sess)
        if c2 != 200:
            continue
        best = analyze(h2)
        if best[0] >= 50:
            print(f'  ★ {url}: {best[1]} cls={best[2]} a={best[0]}')


# 영동군 - try with weak SSL adapter
print('\n' + '='*70)
print('영동군 — weak SSL adapter')
print('='*70)
sess_weak = make_session(weak_ssl=True)
code, real, html = fetch('https://www.yd21.go.kr/', sess_weak)
print(f'  HTTP {code}, len {len(html)}')
if code == 200:
    soup = BeautifulSoup(html, 'html.parser')
    cands = []
    for a in soup.find_all('a', href=True):
        txt = a.get_text(strip=True)
        if '사이트맵' in txt or '전체메뉴' in txt or '누리집' in txt:
            from urllib.parse import urljoin
            cands.append((txt, urljoin(real, a['href'])))
    print('  사이트맵 후보:')
    for t, u in cands[:8]:
        print(f'    "{t}" → {u}')
    from urllib.parse import urlparse
    parsed = urlparse(real)
    origin = f'{parsed.scheme}://{parsed.netloc}'
    test_urls = [u for _, u in cands] + [
        origin + '/kor/sitemap.do',
        origin + '/sitemap.html',
        origin + '/sitemap.do',
        origin + '/contents/contents.html?cid=2151',
    ]
    for url in set(test_urls):
        c2, r2, h2 = fetch(url, sess_weak)
        if c2 != 200:
            continue
        best = analyze(h2)
        if best[0] >= 50:
            print(f'  ★ {url}: {best[1]} cls={best[2]} a={best[0]}')