df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
82 lines
2.6 KiB
Python
82 lines
2.6 KiB
Python
"""Find sitemap URLs for: 논산시, 아산시, 부여군, 서산시."""
|
|
import re
|
|
import warnings
|
|
from urllib.parse import urljoin
|
|
|
|
import requests
|
|
from bs4 import BeautifulSoup
|
|
|
|
warnings.filterwarnings('ignore')
|
|
|
|
UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
|
|
H = {'User-Agent': UA}
|
|
|
|
|
|
def fetch(url):
|
|
try:
|
|
r = requests.get(url, headers=H, timeout=15, verify=False, allow_redirects=True)
|
|
r.encoding = r.apparent_encoding
|
|
return r.status_code, r.url, r.text
|
|
except Exception as e:
|
|
return 0, str(e), ''
|
|
|
|
|
|
def find_links_to(html, base, keyword='사이트맵'):
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
found = []
|
|
for a in soup.find_all('a', href=True):
|
|
txt = a.get_text(strip=True)
|
|
href = a['href']
|
|
if keyword in txt or 'sitemap' in href.lower() or 'allMenu' in href:
|
|
full = urljoin(base, href)
|
|
if full != base:
|
|
found.append((txt, full))
|
|
return found
|
|
|
|
|
|
CASES = [
|
|
('논산시', 'https://nonsan.go.kr/'),
|
|
('아산시', 'https://www.asan.go.kr/main/'),
|
|
('부여군', 'https://www.buyeo.go.kr/html/kr/'),
|
|
('서산시', 'https://www.seosan.go.kr/www/index.do'),
|
|
]
|
|
|
|
for name, base in CASES:
|
|
print(f'\n=== {name} {base} ===')
|
|
code, url, html = fetch(base)
|
|
if code != 200:
|
|
print(f' main 실패: {code}')
|
|
continue
|
|
links = find_links_to(html, url)
|
|
# Deduplicate
|
|
seen = set()
|
|
uniq = []
|
|
for t, u in links:
|
|
if u not in seen:
|
|
seen.add(u)
|
|
uniq.append((t, u))
|
|
print(f' 사이트맵/allMenu 링크 후보 ({len(uniq)}):')
|
|
for t, u in uniq[:10]:
|
|
print(f' "{t}" → {u}')
|
|
# For top candidate, fetch and analyze
|
|
for t, u in uniq[:3]:
|
|
c2, u2, h2 = fetch(u)
|
|
if c2 != 200:
|
|
print(f' [{u}] HTTP {c2}')
|
|
continue
|
|
soup = BeautifulSoup(h2, 'html.parser')
|
|
# Find best container
|
|
best = (0, None, None)
|
|
for sel in ['.sitemap_grep', '.sitemap', '#sitemap', '.allMenu', '#allMenu',
|
|
'div[class*=sitemap]', 'div[class*=allMenu]', 'ul.sitemap_list',
|
|
'ul.depth1_ul', 'ul.depth1-ul', '#gnb', 'div.menu_all']:
|
|
for el in soup.select(sel):
|
|
ac = len(el.find_all('a'))
|
|
if ac > best[0]:
|
|
best = (ac, sel, el)
|
|
if best[1]:
|
|
cls = ' '.join(best[2].get('class', []))
|
|
print(f' ★ {u2}: best={best[1]!r} cls={cls!r} a={best[0]}')
|
|
else:
|
|
print(f' [{u2}]: no container')
|