df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
136 lines
4.6 KiB
Python
136 lines
4.6 KiB
Python
"""충청북도 추가 분석:
|
|
1) depth1 패턴 구조 outline (괴산·청주·충주 샘플)
|
|
2) 단양·보은·영동·진천 사이트맵 재탐색
|
|
"""
|
|
import re
|
|
import warnings
|
|
import requests
|
|
from bs4 import BeautifulSoup
|
|
|
|
warnings.filterwarnings('ignore')
|
|
UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
|
|
H = {'User-Agent': UA}
|
|
|
|
|
|
def fetch(url, timeout=15):
|
|
try:
|
|
r = requests.get(url, headers=H, timeout=timeout, verify=False, allow_redirects=True)
|
|
r.encoding = r.apparent_encoding
|
|
return r.status_code, r.url, r.text
|
|
except Exception as e:
|
|
return 0, str(e), ''
|
|
|
|
|
|
def outline(el, d=0, max_lines=60, lines=None):
|
|
if lines is None: lines = []
|
|
if len(lines) >= max_lines: return lines
|
|
name = el.name
|
|
cls = ' '.join(el.get('class', []))
|
|
eid = el.get('id', '')
|
|
lbl = name
|
|
if eid: lbl += f'#{eid}'
|
|
if cls: lbl += '.' + cls.replace(' ', '.')
|
|
if name == 'a':
|
|
t = el.get_text(strip=True)[:40]
|
|
h = el.get('href', '')[:80]
|
|
lines.append(' ' * d + f'{lbl} "{t}" → {h}')
|
|
else:
|
|
lines.append(' ' * d + lbl)
|
|
for c in el.find_all(recursive=False):
|
|
if c.name in ('script', 'style'): continue
|
|
outline(c, d+1, max_lines, lines)
|
|
if len(lines) >= max_lines: return lines
|
|
return lines
|
|
|
|
|
|
# (1) Depth1 패턴 outline
|
|
print('='*70)
|
|
print('Depth1 pattern — 괴산군')
|
|
print('='*70)
|
|
_, _, html = fetch('https://www.goesan.go.kr/www/sitemap.do?key=28')
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
for sel in ['div.depth1', 'div.depth.depth1', '#sitemap', '.sitemap']:
|
|
el = soup.select_one(sel)
|
|
if el:
|
|
print(f'\n>>> selector: {sel} (a={len(el.find_all("a"))})')
|
|
for line in outline(el, max_lines=50):
|
|
print(' ', line)
|
|
break
|
|
|
|
print('\n' + '='*70)
|
|
print('Depth1 pattern — 청주시')
|
|
print('='*70)
|
|
_, _, html = fetch('https://www.cheongju.go.kr/www/sitemap.do?key=589')
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
for sel in ['#sitemap div.sitemap', 'div#sitemap.sitemap', 'div.sitemap']:
|
|
el = soup.select_one(sel)
|
|
if el:
|
|
print(f'\n>>> selector: {sel} (a={len(el.find_all("a"))})')
|
|
for line in outline(el, max_lines=50):
|
|
print(' ', line)
|
|
break
|
|
|
|
print('\n' + '='*70)
|
|
print('Depth1 pattern — 충주시')
|
|
print('='*70)
|
|
_, _, html = fetch('https://www.chungju.go.kr/www/sub.do?key=692')
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
for sel in ['#sitemap']:
|
|
el = soup.select_one(sel)
|
|
if el:
|
|
print(f'\n>>> selector: {sel} (a={len(el.find_all("a"))})')
|
|
for line in outline(el, max_lines=50):
|
|
print(' ', line)
|
|
break
|
|
|
|
|
|
# (2) 실패한 사이트들 재탐색
|
|
print('\n\n' + '='*70)
|
|
print('실패 사이트 재탐색')
|
|
print('='*70)
|
|
for name, url in [
|
|
('단양군', 'https://www.danyang.go.kr/dy21/1'),
|
|
('보은군', 'https://www.boeun.go.kr/www/index.do'),
|
|
('영동군', 'https://www.yd21.go.kr/'),
|
|
('진천군', 'https://www.jincheon.go.kr/home/intro.do'),
|
|
]:
|
|
print(f'\n--- {name} {url} ---')
|
|
code, real, html = fetch(url)
|
|
if code != 200:
|
|
print(f' 메인 실패: {code} {real}')
|
|
continue
|
|
print(f' 메인 OK: {real}')
|
|
# Find sitemap link
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
cands = []
|
|
for a in soup.find_all('a', href=True):
|
|
txt = a.get_text(strip=True)
|
|
if '사이트맵' in txt or '전체메뉴' in txt or '누리집' in txt or 'sitemap' in (a.get('href','')+txt).lower():
|
|
cands.append((txt, a['href']))
|
|
# Deduplicate
|
|
seen = set()
|
|
uniq = []
|
|
from urllib.parse import urljoin
|
|
for t, h in cands:
|
|
full = urljoin(real, h)
|
|
if full not in seen and full != real:
|
|
seen.add(full)
|
|
uniq.append((t, full))
|
|
for t, u in uniq[:5]:
|
|
print(f' 사이트맵 후보: "{t}" → {u}')
|
|
# Test first candidate
|
|
if uniq:
|
|
c, c_url = uniq[0]
|
|
c2, r2, h2 = fetch(c_url)
|
|
if c2 == 200:
|
|
s2 = BeautifulSoup(h2, 'html.parser')
|
|
best = (0, '', '')
|
|
for sel in ['div.depth1', 'div.depth.depth1', 'ul.depth1_ul', 'ul.depth1-ul',
|
|
'#sitemap', 'div.sitemap_grep', 'ul.sitemap', 'div.sitemap',
|
|
'div.sitemap_11', 'div.amThum']:
|
|
for el in s2.select(sel):
|
|
ac = len(el.find_all('a'))
|
|
if ac > best[0]:
|
|
best = (ac, sel, ' '.join(el.get('class', [])))
|
|
print(f' 사이트맵 분석: best_sel={best[1]} cls={best[2]} a={best[0]}')
|