DB_JOB/_스크립트/_공공기관2_probe.py

# -*- coding: utf-8 -*-
"""공공기관 31곳 사이트맵 페이지 발견 + 구조 분석 (Phase 0 정찰)."""
import sys, io, re, warnings, json
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
from urllib.parse import urljoin, urlparse
from concurrent.futures import ThreadPoolExecutor, as_completed
import requests
from bs4 import BeautifulSoup
import openpyxl

warnings.filterwarnings('ignore')
UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
H = {'User-Agent': UA}
STATUS = r'D:\01.프로젝트\DB수집\작업파일\공공기관2\공공기관2_작업현황.xlsx'

SITEMAP_WORDS = ['사이트맵', '사이트 맵', '전체메뉴', '전체 메뉴', 'sitemap', 'site map', 'allmenu', '전체보기']
COMMON_PATHS = [
    '/sitemap', '/sitemap.do', '/sitemap.jsp', '/sitemap.html', '/sitemap.asp', '/sitemap.aspx',
    '/siteMap.do', '/siteMap', '/site/main/sitemap', '/main/sitemap.do', '/cms/sitemap.do',
    '/kor/sitemap.do', '/html/sitemap.html', '/contents/sitemap.do', '/intro/sitemap.do',
    '/user/sitemap.do', '/web/sitemap.do', '/main/html/sitemap.html', '/sub/sitemap.do',
]


def fetch(url, timeout=15):
    try:
        r = requests.get(url, headers=H, timeout=timeout, verify=False, allow_redirects=True)
        ct = r.headers.get('content-type', '')
        if 'html' not in ct and 'xml' not in ct and r.status_code == 200 and 'text' not in ct:
            return r.url, None
        meta = re.search(rb'charset=["\']?\s*([\w-]+)', r.content[:4096], re.I)
        r.encoding = meta.group(1).decode(errors='ignore') if meta else r.apparent_encoding
        if r.status_code == 200:
            return r.url, r.text
    except Exception as e:
        return None, f'ERR:{type(e).__name__}'
    return None, f'HTTP{r.status_code}'


def find_sitemap_link(home_html, base):
    soup = BeautifulSoup(home_html, 'html.parser')
    found = []
    for a in soup.find_all('a', href=True):
        txt = a.get_text(strip=True).lower().replace(' ', '')
        href = a['href']
        title = (a.get('title', '') or '').lower()
        for w in SITEMAP_WORDS:
            ww = w.replace(' ', '')
            if ww in txt or ww in href.lower() or ww in title.replace(' ', ''):
                if not href.startswith(('javascript:', '#')):
                    found.append(urljoin(base + '/', href))
                break
    # dedup preserve order
    seen = set(); out = []
    for u in found:
        if u not in seen:
            seen.add(u); out.append(u)
    return out


def analyze(html):
    soup = BeautifulSoup(html, 'html.parser')
    selectors = [
        '.sitemap', '#sitemap', '.site_map', '#siteMap', '.siteMap', '.sitemap_wrap', '.sitemapWrap',
        '.allMenu', '.allmenu', '#allMenu', '.all_menu', '.allMenuWrap', '#allmenu',
        '.menu_all', '.gnb_all', '.totalMenu', '.totMenu', '.total_menu', '.full_menu',
        '#contents .sitemap', 'div[class*=sitemap]', 'div[class*=siteMap]', 'div[class*=allMenu]',
        'div[class*=allmenu]', 'div[id*=sitemap]', 'div[id*=siteMap]',
    ]
    cands = []
    for sel in selectors:
        try:
            for el in soup.select(sel):
                ac = len(el.find_all('a'))
                if ac >= 15:
                    cands.append((ac, sel, el))
        except Exception:
            pass
    if not cands:
        for d in soup.find_all(['div', 'section', 'main', 'nav']):
            cls = ' '.join(d.get('class', [])).lower()
            did = (d.get('id', '') or '').lower()
            if any(x in cls + did for x in ['footer', 'header']):
                continue
            ac = len(d.find_all('a'))
            if ac >= 40:
                cands.append((ac, f'<{d.name} class="{cls}" id="{did}">', d))
    cands.sort(key=lambda x: x[0], reverse=True)
    if not cands:
        return None
    ac, sel, el = cands[0]
    uls = el.find_all('ul', recursive=False)
    dls = el.find_all('dl', recursive=False)
    if not uls and not dls:
        for ch in el.find_all(['div', 'section'], recursive=False):
            uls += ch.find_all('ul', recursive=False)
            dls += ch.find_all('dl', recursive=False)
    pat = 'A(dl)' if dls else ('B(ul)' if uls else 'UNK')
    # max nesting depth of ul
    def depth(e, d=0):
        subs = e.find_all('ul', recursive=False) + [u for c in e.find_all('li', recursive=False) for u in c.find_all('ul', recursive=False)]
        return max([depth(s, d + 1) for s in subs], default=d)
    return {'sel': sel, 'a': ac, 'ul': len(el.find_all('ul')), 'dl': len(el.find_all('dl')),
            'li': len(el.find_all('li')), 'pat': pat, 'depth': depth(el)}


def load_sites():
    wb = openpyxl.load_workbook(STATUS, data_only=True)
    ws = wb['현황']
    sites = []
    for r in ws.iter_rows(min_row=2, values_only=True):
        if r[0] == '공공기관' and r[1]:
            num, name, url = r[1], r[2], r[11]
            if not url:
                continue
            if not url.startswith('http'):
                url = 'https://' + url
            sites.append({'num': num, 'name': name, 'home': url})
    return sites


def probe(site):
    name = site['name']
    home = site['home']
    pr = urlparse(home)
    base = f'{pr.scheme}://{pr.netloc}'
    res = {'num': site['num'], 'name': name, 'home': home, 'base': base, 'sitemap': None, 'struct': None, 'tried': []}
    u, html = fetch(home)
    if not u or not html:
        res['err'] = f'home fail: {html}'
        return res
    res['base'] = f"{urlparse(u).scheme}://{urlparse(u).netloc}"
    base = res['base']
    # 1) homepage sitemap link
    links = find_sitemap_link(html, base)
    # 2) common paths
    cand_urls = links + [base + p for p in COMMON_PATHS]
    best = None
    for cu in cand_urls[:20]:
        fu, fhtml = fetch(cu)
        res['tried'].append(cu)
        if fu and fhtml:
            st = analyze(fhtml)
            if st and st['a'] >= 20:
                best = (fu, st)
                break
    if best:
        res['sitemap'] = best[0]
        res['struct'] = best[1]
    else:
        # fallback: analyze homepage itself (mega menu)
        st = analyze(html)
        res['struct'] = st
        res['sitemap'] = None
    return res


def main():
    sites = load_sites()
    only = sys.argv[1:]
    if only:
        sites = [s for s in sites if s['name'] in only or str(s['num']) in only]
    print(f'정찰 대상 {len(sites)}곳\n')
    results = []
    with ThreadPoolExecutor(max_workers=6) as ex:
        futs = {ex.submit(probe, s): s for s in sites}
        for f in as_completed(futs):
            results.append(f.result())
    results.sort(key=lambda x: x['num'])
    for r in results:
        sm = r['sitemap'] or '(없음→메가메뉴)'
        st = r['struct']
        sts = f"a={st['a']} {st['pat']} d={st['depth']} sel={st['sel']}" if st else 'STRUCT없음'
        err = r.get('err', '')
        print(f"{r['num']:>2}. {r['name']}")
        print(f"    home={r['home']}")
        print(f"    sitemap={sm}")
        print(f"    {sts} {err}")
    out = r'D:\01.프로젝트\DB수집\_스크립트\_공공기관2_probe.json'
    with open(out, 'w', encoding='utf-8') as f:
        json.dump([{k: (v if k != 'struct' or v is None else v) for k, v in r.items() if k != 'tried'} for r in results], f, ensure_ascii=False, indent=1, default=str)
    print(f'\n→ {out}')


if __name__ == '__main__':
    main()