"""충청북도 10개 시·군 Phase 2~4 일괄 처리 (보은군 제외 — DNS 차단).

매뉴얼: D:\\01.프로젝트\\DB수집\\사이트맵_수집_매뉴얼.md
"""
import re
import ssl
import sys
import time
import warnings
from urllib.parse import urljoin
from concurrent.futures import ThreadPoolExecutor, as_completed

import openpyxl
import requests
from requests.adapters import HTTPAdapter
from urllib3.util.ssl_ import create_urllib3_context
from bs4 import BeautifulSoup

warnings.filterwarnings('ignore')

UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
H = {'User-Agent': UA}


class WeakSSLAdapter(HTTPAdapter):
    def init_poolmanager(self, *args, **kwargs):
        ctx = create_urllib3_context()
        ctx.set_ciphers('DEFAULT@SECLEVEL=0')
        ctx.options |= 0x4
        ctx.check_hostname = False
        ctx.verify_mode = ssl.CERT_NONE
        kwargs['ssl_context'] = ctx
        return super().init_poolmanager(*args, **kwargs)


# ================================================================
# 정규식
# ================================================================
TOTAL_PAT = re.compile(r'총\s*(?:게시물\s*)?(\d[\d,]*)\s*(?:건|개|page|페이지)', re.I)
TOTAL_PAT_LOOSE = re.compile(r'(?:전체|총)\s*[:\-]?\s*(\d[\d,]*)\s*건', re.I)
KOGL_IMG_PAT = re.compile(r'(?:new_)?img_opentype(\d{2})\.png', re.I)
KOGL_LINK_PAT = re.compile(r'kogl\.or\.kr/info/licenseType(\d)', re.I)
YOUTUBE_PAT = re.compile(r'(?:youtube\.com|youtu\.be)', re.I)
VIDEO_EXT = re.compile(r'\.(mp4|webm|mov|avi)(?:\?|$)', re.I)
DETAIL_PAT = re.compile(r'(mode=V|view\.do|bbtSn=|seqRepeat=|nttId=|articleNo=|boardSeq=|menukey=)', re.I)


# 사이트별: xlsx, body selectors, weak_ssl 여부
SITES = {
    '괴산군': {'xlsx': r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청북도\1.괴산군\충청북도_괴산군.xlsx',
              'body_sel': ['#contents', '#txt', 'main'], 'weak_ssl': False},
    '단양군': {'xlsx': r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청북도\2.단양군\충청북도_단양군.xlsx',
              'body_sel': ['#contents', '#txt', 'main'], 'weak_ssl': False},
    '보은군': {'xlsx': r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청북도\3.보은군\충청북도_보은군.xlsx',
              'body_sel': ['#contents', '#txt', 'main'], 'weak_ssl': False},
    '영동군': {'xlsx': r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청북도\4.영동군\충청북도_영동군.xlsx',
              'body_sel': ['#txt', '#contents', 'main'], 'weak_ssl': True},
    '옥천군': {'xlsx': r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청북도\5.옥천군\충청북도_옥천군.xlsx',
              'body_sel': ['#contents', '#txt', 'main'], 'weak_ssl': False},
    '음성군': {'xlsx': r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청북도\6.음성군\충청북도_음성군.xlsx',
              'body_sel': ['#contents', '#txt', 'main'], 'weak_ssl': False},
    '제천시': {'xlsx': r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청북도\7.제천시\충청북도_제천시.xlsx',
              'body_sel': ['#contents', '#txt', 'main'], 'weak_ssl': False},
    '증평군': {'xlsx': r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청북도\8.증평군\충청북도_증평군.xlsx',
              'body_sel': ['#txt', '#contents', 'main'], 'weak_ssl': False},
    '진천군': {'xlsx': r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청북도\9.진천군\충청북도_진천군.xlsx',
              'body_sel': ['#contents', '#txt', 'main'], 'weak_ssl': False},
    '청주시': {'xlsx': r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청북도\10.청주시\충청북도_청주시.xlsx',
              'body_sel': ['#contents', '#txt', 'main'], 'weak_ssl': False},
    '충주시': {'xlsx': r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청북도\11.충주시\충청북도_충주시.xlsx',
              'body_sel': ['#contents', '#txt', 'main'], 'weak_ssl': False},
}


def make_session(weak_ssl=False):
    s = requests.Session()
    s.headers.update(H)
    if weak_ssl:
        s.mount('https://', WeakSSLAdapter())
    return s


def fetch(session, url, timeout=12):
    try:
        r = session.get(url, timeout=timeout, verify=False, allow_redirects=True)
        # 메타 charset 우선
        meta_charset = re.search(rb'<meta[^>]*charset=["\']?\s*([\w-]+)', r.content[:4096], re.I)
        if meta_charset:
            r.encoding = meta_charset.group(1).decode('ascii', errors='ignore')
        else:
            r.encoding = r.apparent_encoding
        if r.status_code == 200:
            return BeautifulSoup(r.text, 'html.parser')
    except Exception:
        pass
    return None


def get_body(soup, selectors):
    for sel in selectors:
        el = soup.select_one(sel)
        if el:
            return el
    return soup


def detect_form(body):
    has_paging = bool(body.select('.pagination, .paging, nav.paging, .page_nav'))
    text_inputs = [i for i in body.find_all('input')
                   if (i.get('type') or 'text').lower() in ('text', 'search')]
    has_search = len(text_inputs) >= 1
    txt = body.get_text(' ', strip=True)
    m = TOTAL_PAT.search(txt) or TOTAL_PAT_LOOSE.search(txt)
    total = None
    if m:
        digits = m.group(1).replace(',', '')
        if digits.isdigit():
            total = int(digits)
    is_board = has_paging or has_search or (total is not None)
    if is_board:
        return '게시판', total if total is not None else 0
    return '페이지', 1


def extract_detail_urls(body, base_url, limit=5):
    urls = []
    seen = set()
    for a in body.find_all('a', href=True):
        h = a['href']
        if not h or h.startswith('#'):
            continue
        if DETAIL_PAT.search(h):
            full = urljoin(base_url, h)
            if full not in seen:
                seen.add(full)
                urls.append(full)
                if len(urls) >= limit:
                    break
    return urls


def detect_media(body):
    has_text = len(body.get_text(strip=True)) > 30
    has_image = False
    for img in body.find_all('img'):
        src = img.get('src', '')
        if KOGL_IMG_PAT.search(src):
            continue
        if not src:
            continue
        has_image = True
        break
    has_video = False
    for iframe in body.find_all('iframe'):
        if YOUTUBE_PAT.search(iframe.get('src', '')):
            has_video = True
            break
    if not has_video:
        for a in body.find_all('a', href=True):
            if YOUTUBE_PAT.search(a['href']):
                has_video = True
                break
    if not has_video and body.find_all('video'):
        has_video = True
    if not has_video and VIDEO_EXT.search(str(body)):
        has_video = True
    return has_image, has_video, has_text


def n_string(has_text, has_image, has_video):
    parts = []
    if has_text: parts.append('어문')
    if has_image: parts.append('이미지')
    if has_video: parts.append('영상')
    return ','.join(parts) if parts else '없음'


def img_has_valid_anchor(img):
    p = img.parent
    while p is not None:
        if p.name == 'a':
            href = p.get('href', '')
            if href and not href.startswith('#') and not href.lower().startswith('javascript:'):
                return True
            return False
        p = p.parent
    return False


def detect_kogl(body):
    types = set()
    q_any_y = False
    q_any_n = False
    for a in body.find_all('a', href=True):
        m = KOGL_LINK_PAT.search(a['href'])
        if m:
            types.add(int(m.group(1)))
            q_any_y = True
    for img in body.find_all('img'):
        src = img.get('src', '')
        m = KOGL_IMG_PAT.search(src)
        if m:
            types.add(int(m.group(1)))
            if img_has_valid_anchor(img):
                q_any_y = True
            else:
                q_any_n = True
    for el in body.find_all(style=True):
        m = KOGL_IMG_PAT.search(el.get('style', ''))
        if m:
            types.add(int(m.group(1)))
            q_any_n = True
    if not types:
        return set(), None
    return types, ('Y' if q_any_y else 'N')


def process_row(session, url, body_selectors):
    out = {'L': '', 'M': '', 'N': '', 'O': '', 'P': '', 'Q': '', 'note': ''}
    soup = fetch(session, url)
    if soup is None:
        out['note'] = '접근 실패'
        return out
    body = get_body(soup, body_selectors)

    form, count = detect_form(body)
    out['L'] = form
    out['M'] = count if form == '게시판' else 1

    has_img, has_vid, has_txt = detect_media(body)
    types_main, q_main = detect_kogl(body)
    P = '게시판' if types_main else ''
    types_all = set(types_main)
    q_flags = []
    if q_main:
        q_flags.append(q_main)

    if form == '게시판':
        detail_urls = extract_detail_urls(body, url, limit=5)
        for du in detail_urls:
            d_soup = fetch(session, du, timeout=10)
            if not d_soup:
                continue
            d_body = get_body(d_soup, body_selectors)
            di, dv, dt = detect_media(d_body)
            has_img = has_img or di
            has_vid = has_vid or dv
            has_txt = has_txt or dt
            dt_types, dt_q = detect_kogl(d_body)
            if dt_types and not types_main and not P:
                P = '게시물'
            types_all |= dt_types
            if dt_q:
                q_flags.append(dt_q)

    out['N'] = n_string(has_txt, has_img, has_vid)

    if not types_all:
        out['O'] = '미부착'
    else:
        sorted_types = sorted(types_all)
        out['O'] = ','.join(f'{n}유형' for n in sorted_types)
        out['P'] = P if P else '게시판'
        out['Q'] = 'Y' if 'Y' in q_flags else 'N'
    return out


def run_site(name, xlsx, body_selectors, weak_ssl=False, workers=10):
    print(f'\n{"="*60}\n[{name}] {xlsx}\n{"="*60}')
    wb = openpyxl.load_workbook(xlsx)
    ws = wb.active
    START = 3
    END = START - 1
    for r in range(START, ws.max_row + 1):
        if ws.cell(r, 2).value is None:
            break
        END = r

    tasks = []
    for r in range(START, END + 1):
        url = ws.cell(r, 11).value
        is_ext = (ws.cell(r, 19).value == '외부링크')
        tasks.append((r, url, is_ext))
    n_ext = sum(1 for t in tasks if t[2])
    print(f'  처리 대상: 총 {len(tasks)}행 (외부링크 {n_ext})')

    t0 = time.time()
    results = {}
    session = make_session(weak_ssl=weak_ssl)

    def worker(task):
        row, url, is_ext = task
        if is_ext:
            return row, {'L': '사이트', 'M': '', 'N': '', 'O': '', 'P': '', 'Q': '', 'note': ''}
        if not url or not isinstance(url, str):
            return row, {'L': '', 'M': '', 'N': '', 'O': '', 'P': '', 'Q': '', 'note': 'URL 없음'}
        return row, process_row(session, url, body_selectors)

    done = 0
    with ThreadPoolExecutor(max_workers=workers) as ex:
        futs = [ex.submit(worker, t) for t in tasks]
        for fut in as_completed(futs):
            row, res = fut.result()
            results[row] = res
            done += 1
            if done % 50 == 0 or done == len(tasks):
                print(f'  진행 {done}/{len(tasks)} ({time.time()-t0:.0f}s)')

    for r in range(START, END + 1):
        res = results.get(r, {})
        if not res:
            continue
        if res.get('L'): ws.cell(r, 12).value = res['L']
        if res.get('M') != '': ws.cell(r, 13).value = res['M']
        if res.get('N'): ws.cell(r, 14).value = res['N']
        if res.get('O'): ws.cell(r, 15).value = res['O']
        if res.get('P'): ws.cell(r, 16).value = res['P']
        if res.get('Q'): ws.cell(r, 17).value = res['Q']
        if res.get('note'):
            existing = ws.cell(r, 19).value
            if not existing:
                ws.cell(r, 19).value = res['note']

    wb.save(xlsx)
    forms = {}
    attach = {'미부착': 0, '부착': 0, '기타': 0}
    q_dist = {'Y': 0, 'N': 0, '': 0}
    for r, res in results.items():
        forms[res.get('L', '')] = forms.get(res.get('L', ''), 0) + 1
        o = res.get('O', '')
        if o == '미부착': attach['미부착'] += 1
        elif o and '유형' in o: attach['부착'] += 1
        else: attach['기타'] += 1
        q = res.get('Q', '')
        q_dist[q] = q_dist.get(q, 0) + 1
    print(f'  [{name}] L분포 {forms} | 부착 {attach} | Q {q_dist} | 시간 {time.time()-t0:.0f}s')


def main():
    targets = sys.argv[1:] if len(sys.argv) > 1 else list(SITES.keys())
    total_t0 = time.time()
    for name in targets:
        if name not in SITES:
            print(f'  알 수 없음: {name}')
            continue
        cfg = SITES[name]
        try:
            run_site(name, cfg['xlsx'], cfg['body_sel'], weak_ssl=cfg.get('weak_ssl', False))
        except Exception as e:
            print(f'  [{name}] 실패: {e}')
            import traceback
            traceback.print_exc()
    print(f'\n총 소요: {time.time()-total_t0:.0f}s')


if __name__ == '__main__':
    main()