"""Find sitemap URLs for: 논산시, 아산시, 부여군, 서산시."""
import re
import warnings
from urllib.parse import urljoin

import requests
from bs4 import BeautifulSoup

warnings.filterwarnings('ignore')

UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
H = {'User-Agent': UA}


def fetch(url):
    try:
        r = requests.get(url, headers=H, timeout=15, verify=False, allow_redirects=True)
        r.encoding = r.apparent_encoding
        return r.status_code, r.url, r.text
    except Exception as e:
        return 0, str(e), ''


def find_links_to(html, base, keyword='사이트맵'):
    soup = BeautifulSoup(html, 'html.parser')
    found = []
    for a in soup.find_all('a', href=True):
        txt = a.get_text(strip=True)
        href = a['href']
        if keyword in txt or 'sitemap' in href.lower() or 'allMenu' in href:
            full = urljoin(base, href)
            if full != base:
                found.append((txt, full))
    return found


CASES = [
    ('논산시', 'https://nonsan.go.kr/'),
    ('아산시', 'https://www.asan.go.kr/main/'),
    ('부여군', 'https://www.buyeo.go.kr/html/kr/'),
    ('서산시', 'https://www.seosan.go.kr/www/index.do'),
]

for name, base in CASES:
    print(f'\n=== {name} {base} ===')
    code, url, html = fetch(base)
    if code != 200:
        print(f'  main 실패: {code}')
        continue
    links = find_links_to(html, url)
    # Deduplicate
    seen = set()
    uniq = []
    for t, u in links:
        if u not in seen:
            seen.add(u)
            uniq.append((t, u))
    print(f'  사이트맵/allMenu 링크 후보 ({len(uniq)}):')
    for t, u in uniq[:10]:
        print(f'    "{t}" → {u}')
    # For top candidate, fetch and analyze
    for t, u in uniq[:3]:
        c2, u2, h2 = fetch(u)
        if c2 != 200:
            print(f'    [{u}] HTTP {c2}')
            continue
        soup = BeautifulSoup(h2, 'html.parser')
        # Find best container
        best = (0, None, None)
        for sel in ['.sitemap_grep', '.sitemap', '#sitemap', '.allMenu', '#allMenu',
                    'div[class*=sitemap]', 'div[class*=allMenu]', 'ul.sitemap_list',
                    'ul.depth1_ul', 'ul.depth1-ul', '#gnb', 'div.menu_all']:
            for el in soup.select(sel):
                ac = len(el.find_all('a'))
                if ac > best[0]:
                    best = (ac, sel, el)
        if best[1]:
            cls = ' '.join(best[2].get('class', []))
            print(f'    ★ {u2}: best={best[1]!r} cls={cls!r} a={best[0]}')
        else:
            print(f'    [{u2}]: no container')