"""Detailed structural analysis of representative sitemap pages."""
import warnings
from urllib.parse import urljoin

import requests
from bs4 import BeautifulSoup

warnings.filterwarnings('ignore')

UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
H = {'User-Agent': UA}

SAMPLES = {
    # name: (sitemap_url, selector_for_container)
    '논산시': ('https://nonsan.go.kr/kor/html/sub07/0701.html', '.sitemap'),
    '당진시': ('https://www.dangjin.go.kr/kor/sitemap_11.do', '#sitemap'),
    '보령시': ('https://www.brcn.go.kr/kor/sitemap_11.do', '#sitemap'),
    '서천군': ('https://www.seocheon.go.kr/kor/sitemap_11.do', '#sitemap'),
    '청양군': ('https://www.cheongyang.go.kr/kor/sitemap_11.do', '#sitemap'),
    '태안군': ('https://www.taean.go.kr/kor/sitemap_11.do', '#sitemap'),
    '아산시': ('https://www.asan.go.kr/main/sitemap.do', '.sitemap'),
    '예산군': ('https://www.yesan.go.kr/kor/sitemap.do', 'ul.sitemap'),
    '천안시': ('https://www.cheonan.go.kr/kor/sitemap.do', 'ul.sitemap'),
    '홍성군': ('https://www.hongseong.go.kr/kor/sitemap.do', 'div.sitemap'),
    '부여군': ('https://www.buyeo.go.kr/html/kr/html/sub07/0701.html', '.sitemap'),
    '서산시': ('https://www.seosan.go.kr/www/contents.do?key=151', '.sitemap'),
}


def fetch(url):
    r = requests.get(url, headers=H, timeout=20, verify=False, allow_redirects=True)
    r.encoding = r.apparent_encoding
    return r.url, r.text, r.status_code


def dump_outline(el, indent=0, max_lines=80, lines=None):
    if lines is None:
        lines = []
    if len(lines) >= max_lines:
        return lines
    name = el.name
    cls = ' '.join(el.get('class', []))
    eid = el.get('id', '')
    label = f'{name}'
    if eid:
        label += f'#{eid}'
    if cls:
        label += f'.{cls.replace(" ", ".")}'
    if name == 'a':
        txt = el.get_text(strip=True)[:40]
        href = el.get('href', '')[:60]
        lines.append('  ' * indent + f'{label} "{txt}" → {href}')
    else:
        lines.append('  ' * indent + label)
    for c in el.find_all(recursive=False):
        if c.name in ('script', 'style'):
            continue
        dump_outline(c, indent + 1, max_lines, lines)
        if len(lines) >= max_lines:
            return lines
    return lines


def main():
    for name, (url, sel) in SAMPLES.items():
        print(f'\n{"="*70}\n{name}: {url}\n{"="*70}')
        try:
            final, html, code = fetch(url)
        except Exception as e:
            print(f'  ERR: {e}')
            continue
        if code != 200:
            print(f'  HTTP {code}')
            # Try alternative locations
            for alt in [url.replace('sub07', 'sub06'), url.replace('contents.do?key=151', 'sitemap.do')]:
                try:
                    final, html, code = fetch(alt)
                    if code == 200:
                        print(f'  대체 OK: {alt}')
                        break
                except Exception:
                    pass
            else:
                continue
        soup = BeautifulSoup(html, 'html.parser')
        el = soup.select_one(sel)
        if el is None:
            print(f'  selector "{sel}" 매칭 실패')
            # Find any container with many anchors
            for d in soup.find_all(['div', 'section', 'main', 'ul']):
                if len(d.find_all('a')) >= 100:
                    cls = ' '.join(d.get('class', []))
                    print(f'    대안 발견: {d.name}.{cls} id={d.get("id","")} (a={len(d.find_all("a"))})')
                    el = d
                    break
        if el is None:
            continue
        print(f'\n  컨테이너: {el.name} class={el.get("class")} id={el.get("id")}')
        print(f'  anchors={len(el.find_all("a"))} dl={len(el.find_all("dl"))} ul={len(el.find_all("ul"))} li={len(el.find_all("li"))}')
        print('  구조 (max 80 lines):')
        lines = dump_outline(el)
        for l in lines:
            print('   ', l)


if __name__ == '__main__':
    main()