"""충청남도 11개 시·군 Phase 2~4 일괄 처리 (L/M/N/O/P/Q).

매뉴얼: D:\\01.프로젝트\\DB수집\\사이트맵_수집_매뉴얼.md
입력: 각 폴더의 {기관명}.xlsx
처리: K열 URL 접근 → L(게시판형태), M(수량), N(저작물 유형), O/P/Q(공공누리)
"""
import re
import sys
import time
import warnings
from urllib.parse import urljoin
from concurrent.futures import ThreadPoolExecutor, as_completed

import openpyxl
import requests
from bs4 import BeautifulSoup

warnings.filterwarnings('ignore')

UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
H = {'User-Agent': UA}

# ================================================================
# 정규식
# ================================================================
TOTAL_PAT = re.compile(r'총\s*(?:게시물\s*)?(\d[\d,]*)\s*(?:건|개|page|페이지)', re.I)
TOTAL_PAT_LOOSE = re.compile(r'(?:전체|총)\s*[:\-]?\s*(\d[\d,]*)\s*건', re.I)
KOGL_IMG_PAT = re.compile(r'(?:new_)?img_opentype(\d{2})\.png', re.I)
KOGL_LINK_PAT = re.compile(r'kogl\.or\.kr/info/licenseType(\d)', re.I)
YOUTUBE_PAT = re.compile(r'(?:youtube\.com|youtu\.be)', re.I)
VIDEO_EXT = re.compile(r'\.(mp4|webm|mov|avi)(?:\?|$)', re.I)


# ================================================================
# 사이트 설정
# ================================================================

SITES = {
    '논산시': {'xlsx': r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청남도\4.논산시\충청남도_논산시.xlsx',
             'body_sel': ['#txt', '#contents', 'main']},
    '당진시': {'xlsx': r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청남도\5.당진시\충청남도_당진시.xlsx',
             'body_sel': ['#txt', '#contents', 'main']},
    '보령시': {'xlsx': r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청남도\6.보령시\충청남도_보령시.xlsx',
             'body_sel': ['#txt', '#contents', 'main']},
    '부여군': {'xlsx': r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청남도\7.부여군\충청남도_부여군.xlsx',
             'body_sel': ['#txt', '#contents', 'main']},
    '서산시': {'xlsx': r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청남도\8.서산시\충청남도_서산시.xlsx',
             'body_sel': ['#contents', '#txt', 'main']},
    '서천군': {'xlsx': r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청남도\9.서천군\충청남도_서천군.xlsx',
             'body_sel': ['#txt', '#contents', 'main']},
    '아산시': {'xlsx': r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청남도\10.아산시\충청남도_아산시.xlsx',
             'body_sel': ['.contents', '#contents', '#txt', 'main']},
    '예산군': {'xlsx': r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청남도\11.예산군\충청남도_예산군.xlsx',
             'body_sel': ['#txt', '#contents', 'main']},
    '천안시': {'xlsx': r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청남도\12.천안시\충청남도_천안시.xlsx',
             'body_sel': ['#txt', '#contents', 'main']},
    '청양군': {'xlsx': r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청남도\13.청양군\충청남도_청양군.xlsx',
             'body_sel': ['#txt', '#contents', 'main']},
    '태안군': {'xlsx': r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청남도\14.태안군\충청남도_태안군.xlsx',
             'body_sel': ['#txt', '#contents', 'main']},
    '홍성군': {'xlsx': r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청남도\15.홍성군\충청남도_홍성군.xlsx',
             'body_sel': ['#txt', '#contents', 'main']},
}


# ================================================================
# 크롤링 공통 함수
# ================================================================


def fetch(url, timeout=12):
    try:
        r = requests.get(url, headers=H, timeout=timeout, verify=False, allow_redirects=True)
        r.encoding = r.apparent_encoding
        if r.status_code == 200:
            return BeautifulSoup(r.text, 'html.parser')
    except Exception:
        pass
    return None


def get_body(soup, selectors):
    for sel in selectors:
        el = soup.select_one(sel)
        if el:
            return el
    return soup


def detect_form(body):
    """L 판별. 페이징·검색·총건수 3요소 중 하나라도 있으면 '게시판'."""
    has_paging = bool(body.select('.pagination, .paging, nav.paging, .page_nav'))
    text_inputs = [i for i in body.find_all('input')
                   if (i.get('type') or 'text').lower() in ('text', 'search')]
    has_search = len(text_inputs) >= 1
    txt = body.get_text(' ', strip=True)
    m = TOTAL_PAT.search(txt)
    if not m:
        m = TOTAL_PAT_LOOSE.search(txt)
    total = None
    if m:
        digits = m.group(1).replace(',', '')
        if digits.isdigit():
            total = int(digits)
    is_board = has_paging or has_search or (total is not None)
    if is_board:
        return '게시판', total if total is not None else 0
    return '페이지', 1


DETAIL_PAT = re.compile(r'(mode=V|view\.do|bbtSn=|seqRepeat=|nttId=|articleNo=|boardSeq=)', re.I)


def extract_detail_urls(body, base_url, limit=5):
    urls = []
    seen = set()
    for a in body.find_all('a', href=True):
        h = a['href']
        if not h or h.startswith('#'):
            continue
        if DETAIL_PAT.search(h):
            full = urljoin(base_url, h)
            if full not in seen:
                seen.add(full)
                urls.append(full)
                if len(urls) >= limit:
                    break
    # Also try fn_search_detail JS pattern (공주시 형)
    if len(urls) < limit:
        for a in body.find_all('a'):
            onclick = a.get('onclick', '')
            m = re.search(r"fn_(?:search_)?detail\(['\"]([^'\"]+)['\"]", onclick)
            if m:
                ntt_id = m.group(1)
                # Build URL by replacing list.do with view.do?nttId=
                view_url = re.sub(r'list\.do[^\'"]*', f'view.do?nttId={ntt_id}', base_url)
                if view_url not in seen:
                    seen.add(view_url)
                    urls.append(view_url)
                    if len(urls) >= limit:
                        break
    return urls


def detect_media(body):
    has_text = len(body.get_text(strip=True)) > 30
    has_image = False
    for img in body.find_all('img'):
        src = img.get('src', '')
        if KOGL_IMG_PAT.search(src):
            continue
        if not src:
            continue
        has_image = True
        break
    has_video = False
    for iframe in body.find_all('iframe'):
        if YOUTUBE_PAT.search(iframe.get('src', '')):
            has_video = True
            break
    if not has_video:
        for a in body.find_all('a', href=True):
            if YOUTUBE_PAT.search(a['href']):
                has_video = True
                break
    if not has_video and body.find_all('video'):
        has_video = True
    if not has_video and VIDEO_EXT.search(str(body)):
        has_video = True
    return has_image, has_video, has_text


def n_string(has_text, has_image, has_video):
    parts = []
    if has_text:
        parts.append('어문')
    if has_image:
        parts.append('이미지')
    if has_video:
        parts.append('영상')
    return ','.join(parts) if parts else '없음'


def img_has_valid_anchor(img):
    p = img.parent
    while p is not None:
        if p.name == 'a':
            href = p.get('href', '')
            if href and not href.startswith('#') and not href.lower().startswith('javascript:'):
                return True
            return False
        p = p.parent
    return False


def detect_kogl(body):
    types = set()
    q_any_y = False
    q_any_n = False
    for a in body.find_all('a', href=True):
        m = KOGL_LINK_PAT.search(a['href'])
        if m:
            types.add(int(m.group(1)))
            q_any_y = True
    for img in body.find_all('img'):
        src = img.get('src', '')
        m = KOGL_IMG_PAT.search(src)
        if m:
            types.add(int(m.group(1)))
            if img_has_valid_anchor(img):
                q_any_y = True
            else:
                q_any_n = True
    for el in body.find_all(style=True):
        m = KOGL_IMG_PAT.search(el.get('style', ''))
        if m:
            types.add(int(m.group(1)))
            q_any_n = True
    if not types:
        return set(), None
    return types, ('Y' if q_any_y else 'N')


def process_row(url, body_selectors):
    out = {'L': '', 'M': '', 'N': '', 'O': '', 'P': '', 'Q': '', 'note': ''}
    soup = fetch(url)
    if soup is None:
        out['note'] = '접근 실패'
        return out
    body = get_body(soup, body_selectors)

    form, count = detect_form(body)
    out['L'] = form
    out['M'] = count if form == '게시판' else 1

    has_img, has_vid, has_txt = detect_media(body)
    types_main, q_main = detect_kogl(body)
    P = '게시판' if types_main else ''
    types_all = set(types_main)
    q_flags = []
    if q_main:
        q_flags.append(q_main)

    if form == '게시판':
        detail_urls = extract_detail_urls(body, url, limit=5)
        for du in detail_urls:
            d_soup = fetch(du, timeout=10)
            if not d_soup:
                continue
            d_body = get_body(d_soup, body_selectors)
            di, dv, dt = detect_media(d_body)
            has_img = has_img or di
            has_vid = has_vid or dv
            has_txt = has_txt or dt
            dt_types, dt_q = detect_kogl(d_body)
            if dt_types and not types_main and not P:
                P = '게시물'
            types_all |= dt_types
            if dt_q:
                q_flags.append(dt_q)

    out['N'] = n_string(has_txt, has_img, has_vid)

    if not types_all:
        out['O'] = '미부착'
    else:
        sorted_types = sorted(types_all)
        out['O'] = ','.join(f'{n}유형' for n in sorted_types)
        out['P'] = P if P else '게시판'
        out['Q'] = 'Y' if 'Y' in q_flags else 'N'

    return out


# ================================================================
# 사이트 단위 실행
# ================================================================


def run_site(name, xlsx, body_selectors, workers=10):
    print(f'\n{"="*60}\n[{name}] {xlsx}\n{"="*60}')
    wb = openpyxl.load_workbook(xlsx)
    ws = wb.active
    START = 3
    # 진짜 데이터 행만 가져옴 (B열에 순번 있어야)
    END = START - 1
    for r in range(START, ws.max_row + 1):
        if ws.cell(r, 2).value is None:
            break
        END = r

    tasks = []
    for r in range(START, END + 1):
        url = ws.cell(r, 11).value
        is_ext = (ws.cell(r, 19).value == '외부링크')
        tasks.append((r, url, is_ext))
    n_ext = sum(1 for t in tasks if t[2])
    print(f'  처리 대상: 총 {len(tasks)}행 (외부링크 {n_ext})')

    t0 = time.time()
    results = {}

    def worker(task):
        row, url, is_ext = task
        if is_ext:
            return row, {'L': '사이트', 'M': '', 'N': '', 'O': '', 'P': '', 'Q': '', 'note': ''}
        if not url or not isinstance(url, str):
            return row, {'L': '', 'M': '', 'N': '', 'O': '', 'P': '', 'Q': '', 'note': 'URL 없음'}
        return row, process_row(url, body_selectors)

    done = 0
    with ThreadPoolExecutor(max_workers=workers) as ex:
        futs = [ex.submit(worker, t) for t in tasks]
        for fut in as_completed(futs):
            row, res = fut.result()
            results[row] = res
            done += 1
            if done % 50 == 0 or done == len(tasks):
                print(f'  진행 {done}/{len(tasks)} ({time.time()-t0:.0f}s)')

    for r in range(START, END + 1):
        res = results.get(r, {})
        if not res:
            continue
        if res.get('L'):
            ws.cell(r, 12).value = res['L']
        if res.get('M') != '':
            ws.cell(r, 13).value = res['M']
        if res.get('N'):
            ws.cell(r, 14).value = res['N']
        if res.get('O'):
            ws.cell(r, 15).value = res['O']
        if res.get('P'):
            ws.cell(r, 16).value = res['P']
        if res.get('Q'):
            ws.cell(r, 17).value = res['Q']
        if res.get('note'):
            existing = ws.cell(r, 19).value
            if not existing:
                ws.cell(r, 19).value = res['note']

    wb.save(xlsx)
    forms = {}
    attach = {'미부착': 0, '부착': 0, '기타': 0}
    q_dist = {'Y': 0, 'N': 0, '': 0}
    for r, res in results.items():
        forms[res.get('L', '')] = forms.get(res.get('L', ''), 0) + 1
        o = res.get('O', '')
        if o == '미부착':
            attach['미부착'] += 1
        elif o and '유형' in o:
            attach['부착'] += 1
        else:
            attach['기타'] += 1
        q = res.get('Q', '')
        q_dist[q] = q_dist.get(q, 0) + 1
    print(f'  [{name}] L분포 {forms} | 부착 {attach} | Q {q_dist} | 시간 {time.time()-t0:.0f}s')


def main():
    targets = sys.argv[1:] if len(sys.argv) > 1 else list(SITES.keys())
    total_t0 = time.time()
    for name in targets:
        if name not in SITES:
            print(f'  알 수 없음: {name}')
            continue
        cfg = SITES[name]
        try:
            run_site(name, cfg['xlsx'], cfg['body_sel'])
        except Exception as e:
            print(f'  [{name}] 실패: {e}')
            import traceback
            traceback.print_exc()
    print(f'\n총 소요: {time.time()-total_t0:.0f}s')


if __name__ == '__main__':
    main()