df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
173 lines
6.4 KiB
Python
173 lines
6.4 KiB
Python
"""Probe sitemap pages directly using candidate URLs found in probe pass 1."""
|
|
import re
|
|
import warnings
|
|
from urllib.parse import urljoin, urlparse
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
|
|
import requests
|
|
from bs4 import BeautifulSoup
|
|
|
|
warnings.filterwarnings('ignore')
|
|
|
|
UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
|
|
H = {'User-Agent': UA}
|
|
|
|
# Round 1 results — best candidate sitemap URL per site
|
|
CANDIDATES = {
|
|
'논산시': 'https://nonsan.go.kr/kor/html/sub07/0701.html',
|
|
'당진시': 'https://www.dangjin.go.kr/kor/sitemap_11.do',
|
|
'보령시': 'https://www.brcn.go.kr/kor/sitemap_11.do',
|
|
'부여군': 'https://www.buyeo.go.kr/html/kr/sitemap.do',
|
|
'서산시': 'https://www.seosan.go.kr/www/sitemap.do',
|
|
'서천군': 'https://www.seocheon.go.kr/kor/sitemap_11.do',
|
|
'아산시': 'https://www.asan.go.kr/main/sitemap.do',
|
|
'예산군': 'https://www.yesan.go.kr/kor/sitemap.do',
|
|
'천안시': 'https://www.cheonan.go.kr/kor/sitemap.do',
|
|
'청양군': 'https://www.cheongyang.go.kr/kor/sitemap_11.do',
|
|
'태안군': 'https://www.taean.go.kr/kor/sitemap_11.do',
|
|
'홍성군': 'https://www.hongseong.go.kr/kor/sitemap.do',
|
|
}
|
|
|
|
# Alternative URLs to try if main candidate fails
|
|
ALTS = {
|
|
'부여군': ['https://www.buyeo.go.kr/html/kr/html/sub07/0701.html',
|
|
'https://www.buyeo.go.kr/html/kr/sitemap.html',
|
|
'https://www.buyeo.go.kr/html/kr/sitemap.do'],
|
|
'서산시': ['https://www.seosan.go.kr/www/sitemap.do',
|
|
'https://www.seosan.go.kr/www/contents.do?key=151'],
|
|
'아산시': ['https://www.asan.go.kr/main/sitemap.do',
|
|
'https://www.asan.go.kr/main/sub01_01.do'],
|
|
'예산군': ['https://www.yesan.go.kr/kor/sitemap.do',
|
|
'https://www.yesan.go.kr/kor/sitemap01.do'],
|
|
}
|
|
|
|
|
|
def fetch(url, timeout=15):
|
|
try:
|
|
r = requests.get(url, headers=H, timeout=timeout, verify=False, allow_redirects=True)
|
|
r.encoding = r.apparent_encoding
|
|
if r.status_code == 200:
|
|
return r.url, r.text
|
|
except Exception as e:
|
|
return None, f'ERR: {e}'
|
|
return None, f'HTTP {r.status_code}'
|
|
|
|
|
|
def deep_analyze(html):
|
|
"""Deeply analyze HTML to find sitemap container regardless of class name."""
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
# Try several common selectors
|
|
selectors = [
|
|
'.sitemap', '#sitemap', '.site_map', '.sitemap_wrap',
|
|
'.allMenu', '.allmenu', '#allMenu', '.all_menu', '.allMenuWrap',
|
|
'.contents .menu', '#content .sitemap', '.menu_all',
|
|
'.gnb_all', '.totalMenu', '.totMenu',
|
|
'div[class*=sitemap]', 'div[class*=allMenu]',
|
|
]
|
|
candidates = []
|
|
for sel in selectors:
|
|
for el in soup.select(sel):
|
|
# Count nested anchors as a measure of usefulness
|
|
a_count = len(el.find_all('a'))
|
|
if a_count >= 20:
|
|
candidates.append((a_count, sel, el))
|
|
if not candidates:
|
|
# Fallback — find any container with most anchors (excluding header/footer)
|
|
all_divs = soup.find_all(['div', 'section', 'main', 'article'])
|
|
for d in all_divs:
|
|
cls = ' '.join(d.get('class', []))
|
|
if 'header' in cls.lower() or 'footer' in cls.lower() or 'gnb' in cls.lower() and 'all' not in cls.lower():
|
|
continue
|
|
a_count = len(d.find_all('a'))
|
|
if a_count >= 50:
|
|
candidates.append((a_count, f'div.{cls}', d))
|
|
candidates.sort(reverse=True)
|
|
return candidates[:3]
|
|
|
|
|
|
def describe(el):
|
|
"""Describe DOM structure of an element."""
|
|
info = {
|
|
'tag': el.name,
|
|
'class': ' '.join(el.get('class', [])),
|
|
'id': el.get('id', ''),
|
|
'a_count': len(el.find_all('a')),
|
|
'dl_count': len(el.find_all('dl')),
|
|
'ul_count': len(el.find_all('ul')),
|
|
'li_count': len(el.find_all('li')),
|
|
}
|
|
# Identify pattern
|
|
dls_direct = el.find_all('dl', recursive=False)
|
|
uls_direct = el.find_all('ul', recursive=False)
|
|
info['dl_direct'] = len(dls_direct)
|
|
info['ul_direct'] = len(uls_direct)
|
|
if dls_direct and dls_direct[0].find('dt') and dls_direct[0].find('dd'):
|
|
info['pattern'] = 'A (dl>dt|dd>ul)'
|
|
elif uls_direct:
|
|
info['pattern'] = 'B (ul nested)'
|
|
else:
|
|
# Search one level deeper
|
|
nested_dl = []
|
|
nested_ul = []
|
|
for child in el.find_all(['div', 'section'], recursive=False):
|
|
nested_dl.extend(child.find_all('dl', recursive=False))
|
|
nested_ul.extend(child.find_all('ul', recursive=False))
|
|
if nested_dl:
|
|
info['pattern'] = f'A nested 1 deep (dl={len(nested_dl)})'
|
|
elif nested_ul:
|
|
info['pattern'] = f'B nested 1 deep (ul={len(nested_ul)})'
|
|
else:
|
|
info['pattern'] = 'UNKNOWN'
|
|
return info
|
|
|
|
|
|
def probe(name, url):
|
|
print(f'\n=== {name} === {url}')
|
|
u, html = fetch(url)
|
|
if not u:
|
|
print(f' 실패: {html}')
|
|
# Try alternatives
|
|
for alt in ALTS.get(name, []):
|
|
u, html = fetch(alt)
|
|
if u:
|
|
print(f' 대체 URL: {alt}')
|
|
break
|
|
else:
|
|
return name, None
|
|
print(f' 최종 URL: {u} ({len(html)} bytes)')
|
|
cands = deep_analyze(html)
|
|
if not cands:
|
|
print(' 사이트맵 컨테이너 못 찾음')
|
|
# Dump some snippets to help diagnose
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
for tag in ['title', 'h1', 'h2']:
|
|
for t in soup.find_all(tag)[:3]:
|
|
print(f' {tag}: {t.get_text(strip=True)[:80]}')
|
|
return name, None
|
|
for a_count, sel, el in cands:
|
|
info = describe(el)
|
|
print(f' [{a_count} anchors] selector="{sel}" → {info}')
|
|
# Return best candidate info
|
|
best_count, best_sel, best_el = cands[0]
|
|
return name, {'url': u, 'selector': best_sel, **describe(best_el)}
|
|
|
|
|
|
def main():
|
|
results = {}
|
|
with ThreadPoolExecutor(max_workers=4) as ex:
|
|
futs = {ex.submit(probe, n, u): n for n, u in CANDIDATES.items()}
|
|
for f in as_completed(futs):
|
|
n, info = f.result()
|
|
results[n] = info
|
|
print('\n=== 최종 요약 ===')
|
|
for n in CANDIDATES:
|
|
info = results.get(n)
|
|
if info:
|
|
print(f' {n}: {info["url"]} | sel={info["selector"]} | a={info["a_count"]} dl={info["dl_count"]} ul={info["ul_count"]} | {info["pattern"]}')
|
|
else:
|
|
print(f' {n}: NOT FOUND')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|