"""실패한 4개 사이트 (단양·보은·영동·진천) 심층 탐색.""" import re import ssl import warnings import requests from requests.adapters import HTTPAdapter from urllib3.util.ssl_ import create_urllib3_context from bs4 import BeautifulSoup warnings.filterwarnings('ignore') UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36' H = {'User-Agent': UA} class WeakSSLAdapter(HTTPAdapter): """레거시 SSL/TLS handshake를 허용하는 어댑터 (영동군 등 구형 SSL).""" def init_poolmanager(self, *args, **kwargs): ctx = create_urllib3_context() ctx.set_ciphers('DEFAULT@SECLEVEL=0') ctx.options |= 0x4 # ssl.OP_LEGACY_SERVER_CONNECT (Python 3.12+) ctx.check_hostname = False ctx.verify_mode = ssl.CERT_NONE kwargs['ssl_context'] = ctx return super().init_poolmanager(*args, **kwargs) def make_session(weak_ssl=False): s = requests.Session() s.headers.update(H) if weak_ssl: s.mount('https://', WeakSSLAdapter()) return s def fetch(url, session=None, timeout=20): s = session or requests.Session() if not session: s.headers.update(H) try: r = s.get(url, timeout=timeout, verify=False, allow_redirects=True) r.encoding = r.apparent_encoding return r.status_code, r.url, r.text except Exception as e: return 0, str(e), '' def outline(el, d=0, max_lines=40, lines=None): if lines is None: lines = [] if len(lines) >= max_lines: return lines name = el.name cls = ' '.join(el.get('class', [])) eid = el.get('id', '') lbl = name if eid: lbl += f'#{eid}' if cls: lbl += '.' + cls.replace(' ', '.') if name == 'a': t = el.get_text(strip=True)[:40] h = el.get('href', '')[:80] lines.append(' ' * d + f'{lbl} "{t}" → {h}') else: lines.append(' ' * d + lbl) for c in el.find_all(recursive=False): if c.name in ('script', 'style'): continue outline(c, d+1, max_lines, lines) if len(lines) >= max_lines: return lines return lines def analyze(html): soup = BeautifulSoup(html, 'html.parser') best = (0, '', '', None) for sel in [ 'div.depth.depth1', 'div.depth1', '#sitemap div.site_map_col', '#sitemap', 'div.sitemap_grep', 'ul.sitemap_list', 'div.sitemap_box', 'ul.depth1_ul', 'ul.depth1-ul', 'ul.depth1', 'ul.sitemap', 'div.sitemap', 'div.amThum', '.allMenu', 'div.menu_all', 'nav#gnb', 'nav.gnb', '#gnb', ]: for el in soup.select(sel): ac = len(el.find_all('a')) if ac > best[0]: cls = ' '.join(el.get('class', [])) best = (ac, sel, cls, el) return best # 단양군 - try /dy21/98 print('='*70) print('단양군 — /dy21/98') print('='*70) sess = make_session() code, real, html = fetch('https://www.danyang.go.kr/dy21/98', sess) print(f' HTTP {code}, len {len(html)}') if code == 200: soup = BeautifulSoup(html, 'html.parser') # Search for the sitemap container print(f' All elements w/ a>=50:') for el in soup.find_all(['div', 'ul', 'nav']): ac = len(el.find_all('a')) if 50 <= ac: cls = ' '.join(el.get('class', []))[:50] eid = el.get('id', '') print(f' {el.name}#{eid}.{cls} a={ac}') best = analyze(html) if best[3]: print(f' Best: {best[1]} cls={best[2]} a={best[0]}') for line in outline(best[3], max_lines=40): print(' ', line) # 진천군 - try /home/main.do print('\n' + '='*70) print('진천군 — /home/main.do') print('='*70) code, real, html = fetch('https://www.jincheon.go.kr/home/main.do', sess) print(f' HTTP {code}, len {len(html)}') if code == 200: soup = BeautifulSoup(html, 'html.parser') cands = [] for a in soup.find_all('a', href=True): txt = a.get_text(strip=True) if '사이트맵' in txt or '전체메뉴' in txt or '누리집' in txt: from urllib.parse import urljoin cands.append((txt, urljoin(real, a['href']))) print(' 사이트맵 후보:') for t, u in cands[:8]: print(f' "{t}" → {u}') # Common paths from urllib.parse import urlparse parsed = urlparse(real) origin = f'{parsed.scheme}://{parsed.netloc}' test_urls = [u for _, u in cands] + [ origin + '/home/sitemap.do', origin + '/home/contents.do?key=121', origin + '/home/sub.do?key=121', ] for url in test_urls: c2, r2, h2 = fetch(url, sess) if c2 != 200: continue best = analyze(h2) if best[0] >= 50: print(f' ★ {url}: {best[1]} cls={best[2]} a={best[0]}') # 보은군 - retry print('\n' + '='*70) print('보은군 — retry') print('='*70) code, real, html = fetch('https://www.boeun.go.kr/www/index.do', sess) print(f' HTTP {code}, len {len(html)}') if code == 200: soup = BeautifulSoup(html, 'html.parser') cands = [] for a in soup.find_all('a', href=True): txt = a.get_text(strip=True) if '사이트맵' in txt or '전체메뉴' in txt or '누리집 지도' in txt: from urllib.parse import urljoin cands.append((txt, urljoin(real, a['href']))) print(' 사이트맵 후보:') for t, u in cands[:8]: print(f' "{t}" → {u}') # Try probable paths from urllib.parse import urlparse parsed = urlparse(real) origin = f'{parsed.scheme}://{parsed.netloc}' test_urls = [u for _, u in cands] + [ origin + '/www/sitemap.do', origin + '/www/sub.do?key=121', ] for url in set(test_urls): c2, r2, h2 = fetch(url, sess) if c2 != 200: continue best = analyze(h2) if best[0] >= 50: print(f' ★ {url}: {best[1]} cls={best[2]} a={best[0]}') # 영동군 - try with weak SSL adapter print('\n' + '='*70) print('영동군 — weak SSL adapter') print('='*70) sess_weak = make_session(weak_ssl=True) code, real, html = fetch('https://www.yd21.go.kr/', sess_weak) print(f' HTTP {code}, len {len(html)}') if code == 200: soup = BeautifulSoup(html, 'html.parser') cands = [] for a in soup.find_all('a', href=True): txt = a.get_text(strip=True) if '사이트맵' in txt or '전체메뉴' in txt or '누리집' in txt: from urllib.parse import urljoin cands.append((txt, urljoin(real, a['href']))) print(' 사이트맵 후보:') for t, u in cands[:8]: print(f' "{t}" → {u}') from urllib.parse import urlparse parsed = urlparse(real) origin = f'{parsed.scheme}://{parsed.netloc}' test_urls = [u for _, u in cands] + [ origin + '/kor/sitemap.do', origin + '/sitemap.html', origin + '/sitemap.do', origin + '/contents/contents.html?cid=2151', ] for url in set(test_urls): c2, r2, h2 = fetch(url, sess_weak) if c2 != 200: continue best = analyze(h2) if best[0] >= 50: print(f' ★ {url}: {best[1]} cls={best[2]} a={best[0]}')