"""각 사이트의 사이트맵 URL 탐색."""
import sys, io, urllib.request, urllib.error, ssl, re
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')

ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE

HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36',
    'Accept-Language': 'ko-KR,ko;q=0.9,en;q=0.8',
}

def fetch(url, timeout=15):
    try:
        req = urllib.request.Request(url, headers=HEADERS)
        with urllib.request.urlopen(req, timeout=timeout, context=ctx) as r:
            data = r.read()
            ct = r.headers.get('Content-Type', '')
            charset = 'utf-8'
            m = re.search(r'charset=([^\s;]+)', ct)
            if m:
                charset = m.group(1)
            try:
                return r.getcode(), data.decode(charset, errors='replace'), r.geturl()
            except:
                return r.getcode(), data.decode('utf-8', errors='replace'), r.geturl()
    except urllib.error.HTTPError as e:
        return e.code, '', url
    except Exception as e:
        return 0, f'ERR: {e}', url

# 사이트별 메인 + 후보 사이트맵 패턴
SITES = {
    '인천광역시': 'https://www.incheon.go.kr',
    '전라남도': 'https://www.jeonnam.go.kr',
    '고용노동부': 'https://www.moel.go.kr',
    '과학기술정보통신부': 'https://www.msit.go.kr',
    '교육부': 'https://www.moe.go.kr',
    '보건복지부': 'https://www.mohw.go.kr',
    '성평등가족부': 'https://www.mogef.go.kr',
    '외교부': 'https://www.mofa.go.kr',
}

# 1차 — 메인 페이지에서 'sitemap' 링크 찾기
import re
SITEMAP_KEY = re.compile(r'href=["\']([^"\']+)["\'][^>]*>[^<]*?(?:사이트맵|Sitemap|SITEMAP|site\s*map)', re.IGNORECASE)
SITEMAP_HREF = re.compile(r'href=["\']([^"\']*sitemap[^"\']*)["\']', re.IGNORECASE)

results = {}
for name, base in SITES.items():
    print(f'=== {name} ===')
    print(f'  base: {base}')
    candidates = set()
    for entry in ['/', '/main', '/index.do', '/index.jsp']:
        url = base + entry
        code, html, final = fetch(url)
        print(f'  {entry} -> HTTP {code} (final: {final})')
        if code == 200:
            # 텍스트 기반 검색
            for m in SITEMAP_KEY.findall(html):
                candidates.add(m)
            for m in SITEMAP_HREF.findall(html):
                candidates.add(m)
            break
    # 일반 후보
    common = ['/sitemap.do', '/sitemap.html', '/html/sub06/sitemap.html',
              '/kr/html/guide/0701.html', '/kr/sitemap.do',
              '/kor/sub05_01.do', '/main/contents.do?menuNo=00000']
    for c in common:
        candidates.add(c)

    # 각 후보 시도
    print(f'  후보: {len(candidates)}개')
    found = []
    for c in candidates:
        if c.startswith('http'):
            test_url = c
        elif c.startswith('//'):
            test_url = 'https:' + c
        elif c.startswith('/'):
            test_url = base + c
        elif c.startswith('javascript:') or c.startswith('#'):
            continue
        else:
            test_url = base + '/' + c
        if 'sitemap' not in test_url.lower() and '0701' not in test_url:
            continue
        code, html, final = fetch(test_url)
        if code == 200 and len(html) > 1000:
            # 사이트맵스러운지 확인 (dl/dt나 ul.depth, .sitemap 클래스 등)
            looks = ('class="sitemap"' in html.lower() or 'sitemap_wrap' in html.lower()
                     or html.count('<dl') > 2 or html.count('<dt') > 3)
            print(f'    {test_url} -> HTTP {code}, sitemap-like={looks}, len={len(html)}')
            if looks:
                found.append(test_url)
    results[name] = found
    print()

import json
with open(r'D:\01.프로젝트\DB수집\2주차\검토_리포트\sitemap_urls.json', 'w', encoding='utf-8') as f:
    json.dump(results, f, ensure_ascii=False, indent=2)
print('saved sitemap_urls.json')