df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
79 lines
3.5 KiB
Python
79 lines
3.5 KiB
Python
"""사이트별 breadcrumb 셀렉터 식별 — 샘플 URL 1~2개로 탐색."""
|
|
import sys, io, urllib.request, ssl, json, re
|
|
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
|
|
|
|
ctx = ssl.create_default_context(); ctx.check_hostname = False; ctx.verify_mode = ssl.CERT_NONE
|
|
HEADERS = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36',
|
|
'Accept-Language': 'ko-KR,ko;q=0.9'}
|
|
|
|
def fetch(url, timeout=20):
|
|
try:
|
|
req = urllib.request.Request(url, headers=HEADERS)
|
|
with urllib.request.urlopen(req, timeout=timeout, context=ctx) as r:
|
|
data = r.read()
|
|
cs = 'utf-8'
|
|
m = re.search(r'charset=([^\s;]+)', r.headers.get('Content-Type', ''))
|
|
if m: cs = m.group(1)
|
|
try: html = data.decode(cs, errors='replace')
|
|
except: html = data.decode('utf-8', errors='replace')
|
|
return r.getcode(), html, r.geturl()
|
|
except Exception as e:
|
|
return 0, str(e)[:200], url
|
|
|
|
# 사이트별 샘플 URL (실 데이터에서 한 줄씩 — 가급적 main domain 페이지)
|
|
samples = {
|
|
'인천광역시': 'https://www.incheon.go.kr/IC010101',
|
|
'전라남도': 'https://www.jeonnam.go.kr/M6698/boardList.do?menuId=jeonnam0101020000',
|
|
'고용노동부': 'https://www.moel.go.kr/minwon/petition/fraud_list.do',
|
|
'과학기술정보통신부': 'https://www.msit.go.kr/contents/cont.do?sCode=user&mPid=13&mId=227',
|
|
'교육부': 'https://www.moe.go.kr/sub/infoRenew.do?m=011602&page=011602&s=moe',
|
|
'보건복지부': 'https://www.mohw.go.kr/menu.es?mid=a10101030100',
|
|
'성평등가족부': 'https://www.mogef.go.kr/sp/geq/sp_geq_f001.do',
|
|
'외교부': 'https://www.mofa.go.kr/www/wpge/m_3435/contents.do',
|
|
}
|
|
|
|
from bs4 import BeautifulSoup
|
|
|
|
# 흔한 breadcrumb 후보 셀렉터
|
|
SEL = [
|
|
'.location', '#location', 'p.location',
|
|
'.breadcrumb', '#breadcrumb', '[class*=bread]',
|
|
'.path', '#path', '.nav_path', '.cont_path',
|
|
'.now_loc', '.locate', '.crumbs',
|
|
'.sub_loc', '#subLocation', '.subLocation',
|
|
'[role=navigation][aria-label*=breadcrumb]',
|
|
'.bbs_loc', '.menu_loc',
|
|
]
|
|
|
|
results = {}
|
|
for site, url in samples.items():
|
|
print(f'\n=== {site} ===')
|
|
print(f' URL: {url}')
|
|
code, html, fu = fetch(url)
|
|
if code != 200 or not isinstance(html, str) or len(html) < 1000:
|
|
print(f' HTTP {code} / len={len(html) if isinstance(html, str) else 0} — skip')
|
|
results[site] = {'sample_url': url, 'breadcrumb_selector': None, 'error': f'HTTP {code}'}
|
|
continue
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
title = (soup.title.get_text(strip=True) if soup.title else '')
|
|
print(f' title: {title}')
|
|
found = None
|
|
for sel in SEL:
|
|
els = soup.select(sel)
|
|
if els:
|
|
for el in els:
|
|
text = el.get_text(' ', strip=True)
|
|
if text and len(text) < 300 and ('홈' in text or '>' in text or '|' in text or 'home' in text.lower() or '/' in text):
|
|
print(f' [{sel}] -> "{text[:150]}"')
|
|
if not found:
|
|
found = sel
|
|
break
|
|
results[site] = {'sample_url': url, 'breadcrumb_selector': found, 'title': title}
|
|
|
|
with open(r'D:\01.프로젝트\DB수집\2주차\검토_리포트\breadcrumb_selectors.json', 'w', encoding='utf-8') as f:
|
|
json.dump(results, f, ensure_ascii=False, indent=2)
|
|
|
|
print('\n[저장] breadcrumb_selectors.json')
|
|
for s, v in results.items():
|
|
print(f' {s}: {v.get("breadcrumb_selector")}')
|