# -*- coding: utf-8 -*-
"""공주시 227~414행 재검수용 증거 수집(쓰기 없음).
각 URL을 크롤링해 게시판 글수/관광지 썸네일수/외부링크/공공누리마크/#nav탭수/본문이미지·영상 신호를 수집.
검증용으로 226행까지 검수에서 바뀐 행 일부도 같이 수집.
출력: _스크립트/_evidence.json + 콘솔 요약
"""
import sys, json, re, warnings
from urllib.parse import urlparse
from concurrent.futures import ThreadPoolExecutor, as_completed
import requests
from bs4 import BeautifulSoup
import openpyxl
warnings.filterwarnings('ignore')

XLSX = r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청남도\2.공주시\충청남도_공주시.xlsx'
H = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120 Safari/537.36'}
COMMON_IMG = re.compile(r'(flag\.jpg|slogan|/common/|/template/|btn_|/btn|icon|blank|_mark\.png|sns|share|loading)', re.I)


def fetch(row, url):
    try:
        r = requests.get(url, headers=H, timeout=25, verify=False)
        r.encoding = r.apparent_encoding or 'utf-8'
        return row, url, r.status_code, r.text
    except Exception as e:
        return row, url, None, ('ERR:%s' % e)


def analyze(url, html):
    ev = {}
    host = urlparse(url).netloc
    ev['host'] = host
    ev['external'] = ('gongju.go.kr' not in host)
    if not html or html.startswith('ERR:'):
        ev['error'] = html
        return ev
    s = BeautifulSoup(html, 'html.parser')
    # board total count
    cnt_el = s.select_one('.program--count strong')
    if cnt_el:
        m = re.sub(r'[^0-9]', '', cnt_el.get_text())
        ev['board_count'] = int(m) if m else None
        ev['is_board'] = True
    else:
        ev['is_board'] = False
    # tursmCn thumbnails (관광지)
    tt = len(re.findall(r'/thumbnail/tursmCn/', html))
    if tt:
        ev['tursm_count'] = tt
    # content images (static html, excludes template/common) — JS maps missed
    cimgs = []
    for im in s.find_all('img'):
        src = im.get('src') or im.get('data-src') or ''
        if src and not COMMON_IMG.search(src):
            cimgs.append(src)
    ev['content_img'] = len(cimgs)
    ev['content_img_sample'] = cimgs[:4]
    # video signals
    low = html.lower()
    ev['video'] = bool(re.search(r'youtube\.com/embed|youtu\.be/|player\.vimeo|<video|\.mp4|data-video', low))
    # KOGL / 공공누리
    kogl_hits = re.findall(r'(opentype0?[1-4]|kogl[_\-]?[1-4]?|공공누리)', html, re.I)
    ev['kogl'] = bool(kogl_hits)
    ev['kogl_sample'] = list(dict.fromkeys(kogl_hits))[:6]
    # explicit opentype number
    ot = re.findall(r'opentype0?([1-4])', html, re.I)
    if ot:
        ev['kogl_types'] = sorted(set(int(x) for x in ot))
    # #nav tab count (규칙 A)
    best = 0
    for ul in s.find_all('ul'):
        cls = ' '.join(ul.get('class') or []).lower()
        if 'tab-ul' in cls:
            anchors = [a for a in ul.find_all('a')
                       if (a.get('href') or '').strip().startswith('#') and a.get_text(strip=True)]
            if len(anchors) >= 2:
                best = max(best, len(anchors))
    if best:
        ev['nav_tabs'] = best
    return ev


def main():
    lo, hi = 227, 414
    if len(sys.argv) > 2:
        lo, hi = int(sys.argv[1]), int(sys.argv[2])
    wb = openpyxl.load_workbook(XLSX)
    ws = wb.active
    targets = []
    rowinfo = {}
    for r in range(lo, hi + 1):
        u = ws.cell(r, 11).value
        if isinstance(u, str) and u.strip().startswith('http'):
            targets.append((r, u.strip()))
            rowinfo[r] = {
                'D': ws.cell(r, 4).value, 'E': ws.cell(r, 5).value, 'F': ws.cell(r, 6).value,
                'G': ws.cell(r, 7).value, 'K': u.strip(), 'L': ws.cell(r, 12).value,
                'M': ws.cell(r, 13).value, 'N': ws.cell(r, 14).value, 'O': ws.cell(r, 15).value,
            }
    print('크롤 대상:', len(targets), '행', lo, '~', hi)
    out = {}
    done = 0
    with ThreadPoolExecutor(max_workers=10) as ex:
        futs = [ex.submit(fetch, r, u) for r, u in targets]
        for f in as_completed(futs):
            row, url, st, html = f.result()
            ev = analyze(url, html)
            ev['status'] = st
            ev['row'] = row
            out[row] = {**rowinfo[row], **ev}
            done += 1
            if done % 25 == 0:
                print('  ...', done, '/', len(targets))
    json.dump(out, open(r'D:\01.프로젝트\DB수집\_스크립트\_evidence.json', 'w', encoding='utf-8'),
              ensure_ascii=False, indent=1)
    print('저장: _evidence.json  (', len(out), '행 )')


if __name__ == '__main__':
    main()