df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
100 lines
2.7 KiB
Python
100 lines
2.7 KiB
Python
"""Brute-force common sitemap paths for 아산시, 부여군, 서산시."""
|
|
import warnings
|
|
from urllib.parse import urlparse
|
|
|
|
import requests
|
|
from bs4 import BeautifulSoup
|
|
|
|
warnings.filterwarnings('ignore')
|
|
|
|
UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
|
|
H = {'User-Agent': UA}
|
|
|
|
|
|
def fetch(url):
|
|
try:
|
|
r = requests.get(url, headers=H, timeout=10, verify=False, allow_redirects=True)
|
|
r.encoding = r.apparent_encoding
|
|
return r.status_code, r.url, r.text
|
|
except Exception as e:
|
|
return 0, str(e), ''
|
|
|
|
|
|
def analyze(html):
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
best = (0, '', '')
|
|
for sel in ['.sitemap_grep', '.sitemap', '#sitemap', '.allMenu', '#allMenu',
|
|
'div[class*=sitemap]', 'div[class*=allMenu]', 'ul.sitemap_list',
|
|
'ul.depth1_ul', 'ul.depth1-ul', '#gnb', 'div.menu_all',
|
|
'div[class*=menu_all]']:
|
|
for el in soup.select(sel):
|
|
ac = len(el.find_all('a'))
|
|
if ac > best[0]:
|
|
best = (ac, sel, ' '.join(el.get('class', [])))
|
|
return best
|
|
|
|
|
|
PATHS = [
|
|
'/main/sitemap.do',
|
|
'/main/sub01_01.do',
|
|
'/main/sub.do?key=121',
|
|
'/main/sub.do?key=151',
|
|
'/sitemap.do',
|
|
'/main/contents.do?key=121',
|
|
'/main/contents.do?key=151',
|
|
'/main/sitemap.html',
|
|
'/main/sitemap',
|
|
'/main/menu.do',
|
|
'/main/allMenu.do',
|
|
'/main/totalMenu.do',
|
|
'/sub01_01.do',
|
|
]
|
|
|
|
ORIGINS = {
|
|
'아산시': 'https://www.asan.go.kr',
|
|
'부여군': 'https://www.buyeo.go.kr',
|
|
'서산시': 'https://www.seosan.go.kr',
|
|
}
|
|
|
|
# 부여군 prefix
|
|
BUYEO_PATHS = [
|
|
'/html/kr/sitemap.html',
|
|
'/html/kr/html/guide/0701.html',
|
|
'/html/kr/html/sub06/0701.html',
|
|
'/html/kr/html/sub07/0701.html',
|
|
'/html/kr/html/sub01/0101.html',
|
|
'/html/kr/sub.do?key=151',
|
|
]
|
|
|
|
# 서산시 prefix
|
|
SEOSAN_PATHS = [
|
|
'/www/sitemap.do',
|
|
'/www/sub.do?key=121',
|
|
'/www/sub.do?key=151',
|
|
'/www/menu_all.do',
|
|
'/www/allMenu.do',
|
|
'/www/contents.do?key=121',
|
|
'/www/contents.do?key=151',
|
|
'/www/contents.do?key=141',
|
|
'/www/contents.do?key=131',
|
|
]
|
|
|
|
|
|
def try_paths(name, origin, paths):
|
|
print(f'\n=== {name} ===')
|
|
for p in paths:
|
|
url = origin + p
|
|
code, real, html = fetch(url)
|
|
if code != 200:
|
|
continue
|
|
info = analyze(html)
|
|
if info[0] >= 50:
|
|
print(f' ★ {url}: a={info[0]} sel={info[1]!r} cls={info[2]!r}')
|
|
else:
|
|
print(f' {url}: a={info[0]} (인덱싱 부족)')
|
|
|
|
|
|
try_paths('아산시', ORIGINS['아산시'], PATHS)
|
|
try_paths('부여군', ORIGINS['부여군'], BUYEO_PATHS + PATHS)
|
|
try_paths('서산시', ORIGINS['서산시'], SEOSAN_PATHS)
|