"""사이트별 breadcrumb 셀렉터 식별 — 샘플 URL 1~2개로 탐색."""
import sys, io, urllib.request, ssl, json, re
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')

ctx = ssl.create_default_context(); ctx.check_hostname = False; ctx.verify_mode = ssl.CERT_NONE
HEADERS = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36',
           'Accept-Language': 'ko-KR,ko;q=0.9'}

def fetch(url, timeout=20):
    try:
        req = urllib.request.Request(url, headers=HEADERS)
        with urllib.request.urlopen(req, timeout=timeout, context=ctx) as r:
            data = r.read()
            cs = 'utf-8'
            m = re.search(r'charset=([^\s;]+)', r.headers.get('Content-Type', ''))
            if m: cs = m.group(1)
            try: html = data.decode(cs, errors='replace')
            except: html = data.decode('utf-8', errors='replace')
            return r.getcode(), html, r.geturl()
    except Exception as e:
        return 0, str(e)[:200], url

# 사이트별 샘플 URL (실 데이터에서 한 줄씩 — 가급적 main domain 페이지)
samples = {
    '인천광역시': 'https://www.incheon.go.kr/IC010101',
    '전라남도': 'https://www.jeonnam.go.kr/M6698/boardList.do?menuId=jeonnam0101020000',
    '고용노동부': 'https://www.moel.go.kr/minwon/petition/fraud_list.do',
    '과학기술정보통신부': 'https://www.msit.go.kr/contents/cont.do?sCode=user&mPid=13&mId=227',
    '교육부': 'https://www.moe.go.kr/sub/infoRenew.do?m=011602&page=011602&s=moe',
    '보건복지부': 'https://www.mohw.go.kr/menu.es?mid=a10101030100',
    '성평등가족부': 'https://www.mogef.go.kr/sp/geq/sp_geq_f001.do',
    '외교부': 'https://www.mofa.go.kr/www/wpge/m_3435/contents.do',
}

from bs4 import BeautifulSoup

# 흔한 breadcrumb 후보 셀렉터
SEL = [
    '.location', '#location', 'p.location',
    '.breadcrumb', '#breadcrumb', '[class*=bread]',
    '.path', '#path', '.nav_path', '.cont_path',
    '.now_loc', '.locate', '.crumbs',
    '.sub_loc', '#subLocation', '.subLocation',
    '[role=navigation][aria-label*=breadcrumb]',
    '.bbs_loc', '.menu_loc',
]

results = {}
for site, url in samples.items():
    print(f'\n=== {site} ===')
    print(f'  URL: {url}')
    code, html, fu = fetch(url)
    if code != 200 or not isinstance(html, str) or len(html) < 1000:
        print(f'  HTTP {code} / len={len(html) if isinstance(html, str) else 0} — skip')
        results[site] = {'sample_url': url, 'breadcrumb_selector': None, 'error': f'HTTP {code}'}
        continue
    soup = BeautifulSoup(html, 'html.parser')
    title = (soup.title.get_text(strip=True) if soup.title else '')
    print(f'  title: {title}')
    found = None
    for sel in SEL:
        els = soup.select(sel)
        if els:
            for el in els:
                text = el.get_text(' ', strip=True)
                if text and len(text) < 300 and ('홈' in text or '>' in text or '|' in text or 'home' in text.lower() or '/' in text):
                    print(f'  [{sel}] -> "{text[:150]}"')
                    if not found:
                        found = sel
                    break
    results[site] = {'sample_url': url, 'breadcrumb_selector': found, 'title': title}

with open(r'D:\01.프로젝트\DB수집\2주차\검토_리포트\breadcrumb_selectors.json', 'w', encoding='utf-8') as f:
    json.dump(results, f, ensure_ascii=False, indent=2)

print('\n[저장] breadcrumb_selectors.json')
for s, v in results.items():
    print(f'  {s}: {v.get("breadcrumb_selector")}')