"""Probe sitemap pages directly using candidate URLs found in probe pass 1."""
import re
import warnings
from urllib.parse import urljoin, urlparse
from concurrent.futures import ThreadPoolExecutor, as_completed

import requests
from bs4 import BeautifulSoup

warnings.filterwarnings('ignore')

UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
H = {'User-Agent': UA}

# Round 1 results — best candidate sitemap URL per site
CANDIDATES = {
    '논산시': 'https://nonsan.go.kr/kor/html/sub07/0701.html',
    '당진시': 'https://www.dangjin.go.kr/kor/sitemap_11.do',
    '보령시': 'https://www.brcn.go.kr/kor/sitemap_11.do',
    '부여군': 'https://www.buyeo.go.kr/html/kr/sitemap.do',
    '서산시': 'https://www.seosan.go.kr/www/sitemap.do',
    '서천군': 'https://www.seocheon.go.kr/kor/sitemap_11.do',
    '아산시': 'https://www.asan.go.kr/main/sitemap.do',
    '예산군': 'https://www.yesan.go.kr/kor/sitemap.do',
    '천안시': 'https://www.cheonan.go.kr/kor/sitemap.do',
    '청양군': 'https://www.cheongyang.go.kr/kor/sitemap_11.do',
    '태안군': 'https://www.taean.go.kr/kor/sitemap_11.do',
    '홍성군': 'https://www.hongseong.go.kr/kor/sitemap.do',
}

# Alternative URLs to try if main candidate fails
ALTS = {
    '부여군': ['https://www.buyeo.go.kr/html/kr/html/sub07/0701.html',
              'https://www.buyeo.go.kr/html/kr/sitemap.html',
              'https://www.buyeo.go.kr/html/kr/sitemap.do'],
    '서산시': ['https://www.seosan.go.kr/www/sitemap.do',
              'https://www.seosan.go.kr/www/contents.do?key=151'],
    '아산시': ['https://www.asan.go.kr/main/sitemap.do',
              'https://www.asan.go.kr/main/sub01_01.do'],
    '예산군': ['https://www.yesan.go.kr/kor/sitemap.do',
              'https://www.yesan.go.kr/kor/sitemap01.do'],
}


def fetch(url, timeout=15):
    try:
        r = requests.get(url, headers=H, timeout=timeout, verify=False, allow_redirects=True)
        r.encoding = r.apparent_encoding
        if r.status_code == 200:
            return r.url, r.text
    except Exception as e:
        return None, f'ERR: {e}'
    return None, f'HTTP {r.status_code}'


def deep_analyze(html):
    """Deeply analyze HTML to find sitemap container regardless of class name."""
    soup = BeautifulSoup(html, 'html.parser')
    # Try several common selectors
    selectors = [
        '.sitemap', '#sitemap', '.site_map', '.sitemap_wrap',
        '.allMenu', '.allmenu', '#allMenu', '.all_menu', '.allMenuWrap',
        '.contents .menu', '#content .sitemap', '.menu_all',
        '.gnb_all', '.totalMenu', '.totMenu',
        'div[class*=sitemap]', 'div[class*=allMenu]',
    ]
    candidates = []
    for sel in selectors:
        for el in soup.select(sel):
            # Count nested anchors as a measure of usefulness
            a_count = len(el.find_all('a'))
            if a_count >= 20:
                candidates.append((a_count, sel, el))
    if not candidates:
        # Fallback — find any container with most anchors (excluding header/footer)
        all_divs = soup.find_all(['div', 'section', 'main', 'article'])
        for d in all_divs:
            cls = ' '.join(d.get('class', []))
            if 'header' in cls.lower() or 'footer' in cls.lower() or 'gnb' in cls.lower() and 'all' not in cls.lower():
                continue
            a_count = len(d.find_all('a'))
            if a_count >= 50:
                candidates.append((a_count, f'div.{cls}', d))
    candidates.sort(reverse=True)
    return candidates[:3]


def describe(el):
    """Describe DOM structure of an element."""
    info = {
        'tag': el.name,
        'class': ' '.join(el.get('class', [])),
        'id': el.get('id', ''),
        'a_count': len(el.find_all('a')),
        'dl_count': len(el.find_all('dl')),
        'ul_count': len(el.find_all('ul')),
        'li_count': len(el.find_all('li')),
    }
    # Identify pattern
    dls_direct = el.find_all('dl', recursive=False)
    uls_direct = el.find_all('ul', recursive=False)
    info['dl_direct'] = len(dls_direct)
    info['ul_direct'] = len(uls_direct)
    if dls_direct and dls_direct[0].find('dt') and dls_direct[0].find('dd'):
        info['pattern'] = 'A (dl>dt|dd>ul)'
    elif uls_direct:
        info['pattern'] = 'B (ul nested)'
    else:
        # Search one level deeper
        nested_dl = []
        nested_ul = []
        for child in el.find_all(['div', 'section'], recursive=False):
            nested_dl.extend(child.find_all('dl', recursive=False))
            nested_ul.extend(child.find_all('ul', recursive=False))
        if nested_dl:
            info['pattern'] = f'A nested 1 deep (dl={len(nested_dl)})'
        elif nested_ul:
            info['pattern'] = f'B nested 1 deep (ul={len(nested_ul)})'
        else:
            info['pattern'] = 'UNKNOWN'
    return info


def probe(name, url):
    print(f'\n=== {name} === {url}')
    u, html = fetch(url)
    if not u:
        print(f'  실패: {html}')
        # Try alternatives
        for alt in ALTS.get(name, []):
            u, html = fetch(alt)
            if u:
                print(f'  대체 URL: {alt}')
                break
        else:
            return name, None
    print(f'  최종 URL: {u} ({len(html)} bytes)')
    cands = deep_analyze(html)
    if not cands:
        print('  사이트맵 컨테이너 못 찾음')
        # Dump some snippets to help diagnose
        soup = BeautifulSoup(html, 'html.parser')
        for tag in ['title', 'h1', 'h2']:
            for t in soup.find_all(tag)[:3]:
                print(f'    {tag}: {t.get_text(strip=True)[:80]}')
        return name, None
    for a_count, sel, el in cands:
        info = describe(el)
        print(f'  [{a_count} anchors] selector="{sel}" → {info}')
    # Return best candidate info
    best_count, best_sel, best_el = cands[0]
    return name, {'url': u, 'selector': best_sel, **describe(best_el)}


def main():
    results = {}
    with ThreadPoolExecutor(max_workers=4) as ex:
        futs = {ex.submit(probe, n, u): n for n, u in CANDIDATES.items()}
        for f in as_completed(futs):
            n, info = f.result()
            results[n] = info
    print('\n=== 최종 요약 ===')
    for n in CANDIDATES:
        info = results.get(n)
        if info:
            print(f'  {n}: {info["url"]} | sel={info["selector"]} | a={info["a_count"]} dl={info["dl_count"]} ul={info["ul_count"]} | {info["pattern"]}')
        else:
            print(f'  {n}: NOT FOUND')


if __name__ == '__main__':
    main()