df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
108 lines
4.0 KiB
Python
108 lines
4.0 KiB
Python
"""Detailed structural analysis of representative sitemap pages."""
|
|
import warnings
|
|
from urllib.parse import urljoin
|
|
|
|
import requests
|
|
from bs4 import BeautifulSoup
|
|
|
|
warnings.filterwarnings('ignore')
|
|
|
|
UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
|
|
H = {'User-Agent': UA}
|
|
|
|
SAMPLES = {
|
|
# name: (sitemap_url, selector_for_container)
|
|
'논산시': ('https://nonsan.go.kr/kor/html/sub07/0701.html', '.sitemap'),
|
|
'당진시': ('https://www.dangjin.go.kr/kor/sitemap_11.do', '#sitemap'),
|
|
'보령시': ('https://www.brcn.go.kr/kor/sitemap_11.do', '#sitemap'),
|
|
'서천군': ('https://www.seocheon.go.kr/kor/sitemap_11.do', '#sitemap'),
|
|
'청양군': ('https://www.cheongyang.go.kr/kor/sitemap_11.do', '#sitemap'),
|
|
'태안군': ('https://www.taean.go.kr/kor/sitemap_11.do', '#sitemap'),
|
|
'아산시': ('https://www.asan.go.kr/main/sitemap.do', '.sitemap'),
|
|
'예산군': ('https://www.yesan.go.kr/kor/sitemap.do', 'ul.sitemap'),
|
|
'천안시': ('https://www.cheonan.go.kr/kor/sitemap.do', 'ul.sitemap'),
|
|
'홍성군': ('https://www.hongseong.go.kr/kor/sitemap.do', 'div.sitemap'),
|
|
'부여군': ('https://www.buyeo.go.kr/html/kr/html/sub07/0701.html', '.sitemap'),
|
|
'서산시': ('https://www.seosan.go.kr/www/contents.do?key=151', '.sitemap'),
|
|
}
|
|
|
|
|
|
def fetch(url):
|
|
r = requests.get(url, headers=H, timeout=20, verify=False, allow_redirects=True)
|
|
r.encoding = r.apparent_encoding
|
|
return r.url, r.text, r.status_code
|
|
|
|
|
|
def dump_outline(el, indent=0, max_lines=80, lines=None):
|
|
if lines is None:
|
|
lines = []
|
|
if len(lines) >= max_lines:
|
|
return lines
|
|
name = el.name
|
|
cls = ' '.join(el.get('class', []))
|
|
eid = el.get('id', '')
|
|
label = f'{name}'
|
|
if eid:
|
|
label += f'#{eid}'
|
|
if cls:
|
|
label += f'.{cls.replace(" ", ".")}'
|
|
if name == 'a':
|
|
txt = el.get_text(strip=True)[:40]
|
|
href = el.get('href', '')[:60]
|
|
lines.append(' ' * indent + f'{label} "{txt}" → {href}')
|
|
else:
|
|
lines.append(' ' * indent + label)
|
|
for c in el.find_all(recursive=False):
|
|
if c.name in ('script', 'style'):
|
|
continue
|
|
dump_outline(c, indent + 1, max_lines, lines)
|
|
if len(lines) >= max_lines:
|
|
return lines
|
|
return lines
|
|
|
|
|
|
def main():
|
|
for name, (url, sel) in SAMPLES.items():
|
|
print(f'\n{"="*70}\n{name}: {url}\n{"="*70}')
|
|
try:
|
|
final, html, code = fetch(url)
|
|
except Exception as e:
|
|
print(f' ERR: {e}')
|
|
continue
|
|
if code != 200:
|
|
print(f' HTTP {code}')
|
|
# Try alternative locations
|
|
for alt in [url.replace('sub07', 'sub06'), url.replace('contents.do?key=151', 'sitemap.do')]:
|
|
try:
|
|
final, html, code = fetch(alt)
|
|
if code == 200:
|
|
print(f' 대체 OK: {alt}')
|
|
break
|
|
except Exception:
|
|
pass
|
|
else:
|
|
continue
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
el = soup.select_one(sel)
|
|
if el is None:
|
|
print(f' selector "{sel}" 매칭 실패')
|
|
# Find any container with many anchors
|
|
for d in soup.find_all(['div', 'section', 'main', 'ul']):
|
|
if len(d.find_all('a')) >= 100:
|
|
cls = ' '.join(d.get('class', []))
|
|
print(f' 대안 발견: {d.name}.{cls} id={d.get("id","")} (a={len(d.find_all("a"))})')
|
|
el = d
|
|
break
|
|
if el is None:
|
|
continue
|
|
print(f'\n 컨테이너: {el.name} class={el.get("class")} id={el.get("id")}')
|
|
print(f' anchors={len(el.find_all("a"))} dl={len(el.find_all("dl"))} ul={len(el.find_all("ul"))} li={len(el.find_all("li"))}')
|
|
print(' 구조 (max 80 lines):')
|
|
lines = dump_outline(el)
|
|
for l in lines:
|
|
print(' ', l)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|