df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
58 lines
2.0 KiB
Python
58 lines
2.0 KiB
Python
"""Look for sitemap-like containers directly in main pages."""
|
|
import re
|
|
import warnings
|
|
from urllib.parse import urljoin
|
|
|
|
import requests
|
|
from bs4 import BeautifulSoup
|
|
|
|
warnings.filterwarnings('ignore')
|
|
|
|
UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
|
|
H = {'User-Agent': UA}
|
|
|
|
|
|
def fetch(url):
|
|
r = requests.get(url, headers=H, timeout=15, verify=False, allow_redirects=True)
|
|
r.encoding = r.apparent_encoding
|
|
return r.url, r.text
|
|
|
|
|
|
# 부여군 — find anchor with text 사이트맵
|
|
print('\n=== 부여군: extract sitemap link ===')
|
|
real, html = fetch('https://www.buyeo.go.kr/html/kr/')
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
for a in soup.find_all('a', href=True):
|
|
txt = a.get_text(strip=True)
|
|
if '사이트맵' in txt:
|
|
full = urljoin(real, a['href'])
|
|
print(f' 사이트맵 → {full}')
|
|
# Also look in JS for sitemap URL
|
|
for s in re.findall(r"location\.(?:href|replace)\s*=\s*['\"]([^'\"]+)['\"]", html):
|
|
if 'sitemap' in s.lower():
|
|
print(f' JS sitemap → {s}')
|
|
|
|
|
|
# 아산시 — gnb-menu inside the page
|
|
print('\n=== 아산시: scan for inline gnb menu ===')
|
|
real, html = fetch('https://www.asan.go.kr/main/')
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
# Find any container with many anchors that looks menu-like
|
|
for el in soup.select('nav, [class*=gnb], [class*=menu]'):
|
|
a_count = len(el.find_all('a'))
|
|
if a_count >= 50:
|
|
cls = ' '.join(el.get('class', []))
|
|
eid = el.get('id', '')
|
|
print(f' {el.name}#{eid}.{cls} a={a_count}')
|
|
|
|
# 서산시 — same approach
|
|
print('\n=== 서산시: scan for inline menu ===')
|
|
real, html = fetch('https://www.seosan.go.kr/www/index.do')
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
for el in soup.select('nav, [class*=gnb], [class*=menu], [class*=allMenu]'):
|
|
a_count = len(el.find_all('a'))
|
|
if a_count >= 30:
|
|
cls = ' '.join(el.get('class', []))
|
|
eid = el.get('id', '')
|
|
print(f' {el.name}#{eid}.{cls} a={a_count}')
|