"""Inspect main pages of 아산시, 부여군, 서산시 — grep for 사이트맵 keyword anywhere."""
import re
import warnings
from urllib.parse import urljoin, urlparse

import requests

warnings.filterwarnings('ignore')

UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
H = {'User-Agent': UA}

PAGES = {
    '아산시': 'https://www.asan.go.kr/main/',
    '부여군': 'https://www.buyeo.go.kr/html/kr/',
    '서산시': 'https://www.seosan.go.kr/www/index.do',
}


def fetch(url):
    r = requests.get(url, headers=H, timeout=15, verify=False, allow_redirects=True)
    r.encoding = r.apparent_encoding
    return r.status_code, r.url, r.text


for name, url in PAGES.items():
    print(f'\n=== {name} === {url}')
    code, real, html = fetch(url)
    print(f'  HTTP {code}, len {len(html)}')
    # Search for 사이트맵 or sitemap or allMenu
    for keyword in ['사이트맵', 'sitemap', 'allMenu', '전체메뉴', 'totalMenu']:
        pat = re.compile(r'[\'"][^\'"]*' + keyword + r'[^\'"]*[\'"]', re.I)
        matches = pat.findall(html)[:10]
        if matches:
            print(f'  "{keyword}" 매칭:')
            for m in matches:
                print(f'    {m}')