df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
55 lines
2.3 KiB
Python
55 lines
2.3 KiB
Python
"""문제 사이트들의 HTML을 들여다보고 정확한 breadcrumb 셀렉터 찾기."""
|
|
import sys, io, json, re
|
|
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
|
|
|
|
import requests, urllib3
|
|
urllib3.disable_warnings()
|
|
|
|
HEADERS = {
|
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36',
|
|
'Accept': 'text/html,application/xhtml+xml,*/*;q=0.9',
|
|
'Accept-Language': 'ko-KR,ko;q=0.9',
|
|
}
|
|
|
|
# 사이트별 샘플 URL
|
|
samples = {
|
|
'인천광역시': 'https://www.incheon.go.kr/IC010101', # 인천소식 > 새소식
|
|
'교육부': 'https://www.moe.go.kr/sub/infoRenew.do?m=011602&page=011602&s=moe',
|
|
'성평등가족부': 'https://www.mogef.go.kr/sp/geq/sp_geq_f001.do', # 정책정보 > 성평등 > X
|
|
'외교부': 'https://www.mofa.go.kr/www/wpge/m_3435/contents.do', # 영사 > 여행
|
|
'과학기술정보통신부': 'https://www.msit.go.kr/contents/cont.do?sCode=user&mPid=13&mId=227',
|
|
}
|
|
|
|
s = requests.Session()
|
|
s.headers.update(HEADERS)
|
|
|
|
from bs4 import BeautifulSoup
|
|
for site, url in samples.items():
|
|
print(f'\n========== {site} ==========')
|
|
print(f'URL: {url}')
|
|
try:
|
|
r = s.get(url, verify=False, timeout=20)
|
|
print(f'HTTP {r.status_code} / len {len(r.text)}')
|
|
if r.status_code != 200: continue
|
|
soup = BeautifulSoup(r.text, 'html.parser')
|
|
# 가능한 breadcrumb 클래스 모두
|
|
candidates = []
|
|
for el in soup.find_all(['div', 'p', 'ul', 'ol', 'nav']):
|
|
cls = ' '.join(el.get('class', []))
|
|
idv = el.get('id', '')
|
|
kw = (cls + ' ' + idv).lower()
|
|
if re.search(r'loc|path|bread|crumb|위치|where', kw):
|
|
txt = el.get_text(' ', strip=True)
|
|
if 5 < len(txt) < 250 and ('홈' in txt or '>' in txt or 'home' in txt.lower() or '·' in txt):
|
|
candidates.append((cls or idv, txt[:200]))
|
|
seen = set()
|
|
for cls, txt in candidates[:10]:
|
|
if cls in seen: continue
|
|
seen.add(cls)
|
|
print(f' [{cls}] -> {txt}')
|
|
# title도 다시
|
|
title = soup.title.get_text(' ', strip=True) if soup.title else ''
|
|
print(f' title: {title[:150]}')
|
|
except Exception as e:
|
|
print(f' ERR: {e}')
|