DB_JOB/_스크립트/_jeonbuk_phase234_all.py

"""전북특별자치도 14개 시·군 + 제주특별자치도 2개 시 Phase 2~4 일괄 처리.

L(형태)·M(건수)·N(저작물유형)·O(공공누리)·P(부착위치)·Q(링크여부) 자동 채움.
매뉴얼: D:\\01.프로젝트\\DB수집\\사이트맵_수집_매뉴얼.md
KOGL(O열) 권위 판정은 별도 _recheck 단계에서 (feedback_kogl_image_rule).
"""
import re
import ssl
import sys
import time
import warnings
from urllib.parse import urljoin
from concurrent.futures import ThreadPoolExecutor, as_completed

import openpyxl
import requests
from requests.adapters import HTTPAdapter
from urllib3.util.ssl_ import create_urllib3_context
from bs4 import BeautifulSoup

warnings.filterwarnings('ignore')
try:
    sys.stdout.reconfigure(line_buffering=True)
except Exception:
    pass

UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
H = {'User-Agent': UA}
ROOT = r'D:\01.프로젝트\DB수집'


class WeakSSLAdapter(HTTPAdapter):
    def init_poolmanager(self, *args, **kwargs):
        ctx = create_urllib3_context()
        ctx.set_ciphers('DEFAULT@SECLEVEL=0')
        ctx.options |= 0x4
        ctx.check_hostname = False
        ctx.verify_mode = ssl.CERT_NONE
        kwargs['ssl_context'] = ctx
        return super().init_poolmanager(*args, **kwargs)


TOTAL_PAT = re.compile(r'총\s*(?:게시물\s*)?(\d[\d,]*)\s*(?:건|개|page|페이지)', re.I)
TOTAL_PAT_LOOSE = re.compile(r'(?:전체|총)\s*[:\-]?\s*(\d[\d,]*)\s*건', re.I)
KOGL_IMG_PAT = re.compile(r'(?:new_)?img_open(?:type|code)(\d{1,2})\.(?:png|jpe?g|gif)', re.I)
KOGL_LINK_PAT = re.compile(r'kogl\.or\.kr/info/licenseType(\d)', re.I)
YOUTUBE_PAT = re.compile(r'(?:youtube\.com|youtu\.be)', re.I)
VIDEO_EXT = re.compile(r'\.(mp4|webm|mov|avi)(?:\?|$)', re.I)
DETAIL_PAT = re.compile(
    r'(mode=V|view\.do|view\.9is|/view\b|bbtSn=|dataUid=|dataSid=|seqRepeat=|'
    r'nttId=|nttNo=|articleNo=|boardSeq=|bbsSeq=|not_ancmt|menukey=)', re.I)

BODY_SEL = ['#main-contents', '#content', '#contents', '.contents', '#txt',
            'main', '#container', '#sub']


def site(i, prov, name, weak=False):
    folder = fr'{ROOT}\작업파일\광역_사이트맵\{prov}\{i}.{name}'
    return name, {'xlsx': fr'{folder}\{prov}_{name}.xlsx', 'body_sel': BODY_SEL, 'weak_ssl': weak}


SITES = dict([
    site(1, '전북특별자치도', '고창군'),
    site(2, '전북특별자치도', '군산시'),
    site(3, '전북특별자치도', '김제시'),
    site(4, '전북특별자치도', '남원시'),
    site(5, '전북특별자치도', '무주군'),
    site(6, '전북특별자치도', '부안군'),
    site(7, '전북특별자치도', '순창군'),
    site(8, '전북특별자치도', '완주군'),
    site(9, '전북특별자치도', '익산시'),
    site(10, '전북특별자치도', '임실군'),
    site(11, '전북특별자치도', '장수군'),
    site(12, '전북특별자치도', '전주시'),
    site(13, '전북특별자치도', '정읍시'),
    site(14, '전북특별자치도', '진안군'),
    site(1, '제주특별자치도', '서귀포시'),
    site(2, '제주특별자치도', '제주시'),
])


def make_session(weak_ssl=False):
    s = requests.Session()
    s.headers.update(H)
    if weak_ssl:
        s.mount('https://', WeakSSLAdapter())
    return s


def fetch(session, url, timeout=5):
    try:
        r = session.get(url, timeout=timeout, verify=False, allow_redirects=True)
        meta = re.search(rb'<meta[^>]*charset=["\']?\s*([\w-]+)', r.content[:4096], re.I)
        r.encoding = meta.group(1).decode('ascii', errors='ignore') if meta else r.apparent_encoding
        if r.status_code == 200:
            return BeautifulSoup(r.text, 'html.parser')
    except Exception:
        pass
    return None


def get_body(soup, selectors):
    for sel in selectors:
        el = soup.select_one(sel)
        if el:
            return el
    return soup


def detect_form(body):
    has_paging = bool(body.select('.pagination, .paging, nav.paging, .page_nav, .board_paging'))
    text_inputs = [i for i in body.find_all('input')
                   if (i.get('type') or 'text').lower() in ('text', 'search')]
    has_search = len(text_inputs) >= 1
    txt = body.get_text(' ', strip=True)
    m = TOTAL_PAT.search(txt) or TOTAL_PAT_LOOSE.search(txt)
    total = None
    if m:
        digits = m.group(1).replace(',', '')
        if digits.isdigit():
            total = int(digits)
    is_board = has_paging or has_search or (total is not None)
    if is_board:
        return '게시판', total if total is not None else 0
    return '페이지', 1


def extract_detail_urls(body, base_url, limit=5):
    urls = []
    seen = set()
    for a in body.find_all('a', href=True):
        h = a['href']
        if not h or h.startswith('#'):
            continue
        if DETAIL_PAT.search(h):
            full = urljoin(base_url, h)
            if full not in seen:
                seen.add(full)
                urls.append(full)
                if len(urls) >= limit:
                    break
    return urls


def detect_media(body):
    has_text = len(body.get_text(strip=True)) > 30
    has_image = False
    for img in body.find_all('img'):
        src = img.get('src', '')
        if KOGL_IMG_PAT.search(src):
            continue
        if not src:
            continue
        has_image = True
        break
    has_video = False
    for iframe in body.find_all('iframe'):
        if YOUTUBE_PAT.search(iframe.get('src', '')):
            has_video = True
            break
    if not has_video:
        for a in body.find_all('a', href=True):
            if YOUTUBE_PAT.search(a['href']):
                has_video = True
                break
    if not has_video and body.find_all('video'):
        has_video = True
    if not has_video and VIDEO_EXT.search(str(body)):
        has_video = True
    return has_image, has_video, has_text


def n_string(has_text, has_image, has_video):
    parts = []
    if has_text: parts.append('어문')
    if has_image: parts.append('이미지')
    if has_video: parts.append('영상')
    return ','.join(parts) if parts else '없음'


def img_has_valid_anchor(img):
    p = img.parent
    while p is not None:
        if p.name == 'a':
            href = p.get('href', '')
            if href and not href.startswith('#') and not href.lower().startswith('javascript:'):
                return True
            return False
        p = p.parent
    return False


def detect_kogl(body):
    types = set()
    q_any_y = False
    q_any_n = False
    for a in body.find_all('a', href=True):
        m = KOGL_LINK_PAT.search(a['href'])
        if m:
            types.add(int(m.group(1)))
            q_any_y = True
    for img in body.find_all('img'):
        src = img.get('src', '')
        m = KOGL_IMG_PAT.search(src)
        if m:
            types.add(int(m.group(1)))
            if img_has_valid_anchor(img):
                q_any_y = True
            else:
                q_any_n = True
    for el in body.find_all(style=True):
        m = KOGL_IMG_PAT.search(el.get('style', ''))
        if m:
            types.add(int(m.group(1)))
            q_any_n = True
    if not types:
        return set(), None
    return types, ('Y' if q_any_y else 'N')


def process_row(session, url, body_selectors):
    out = {'L': '', 'M': '', 'N': '', 'O': '', 'P': '', 'Q': '', 'note': ''}
    soup = fetch(session, url)
    if soup is None:
        out['note'] = '접근 실패'
        return out
    body = get_body(soup, body_selectors)

    form, count = detect_form(body)
    out['L'] = form
    out['M'] = count if form == '게시판' else 1

    has_img, has_vid, has_txt = detect_media(body)
    types_main, q_main = detect_kogl(body)
    P = '게시판' if types_main else ''
    types_all = set(types_main)
    q_flags = []
    if q_main:
        q_flags.append(q_main)

    if form == '게시판':
        detail_urls = extract_detail_urls(body, url, limit=2)
        for du in detail_urls:
            d_soup = fetch(session, du, timeout=5)
            if not d_soup:
                continue
            d_body = get_body(d_soup, body_selectors)
            di, dv, dt = detect_media(d_body)
            has_img = has_img or di
            has_vid = has_vid or dv
            has_txt = has_txt or dt
            dt_types, dt_q = detect_kogl(d_body)
            if dt_types and not types_main and not P:
                P = '게시물'
            types_all |= dt_types
            if dt_q:
                q_flags.append(dt_q)

    out['N'] = n_string(has_txt, has_img, has_vid)

    if not types_all:
        out['O'] = '미부착'
    else:
        sorted_types = sorted(types_all)
        out['O'] = ','.join(f'{n}유형' for n in sorted_types)
        out['P'] = P if P else '게시판'
        out['Q'] = 'Y' if 'Y' in q_flags else 'N'
    return out


def run_site(name, xlsx, body_selectors, weak_ssl=False, workers=14):
    print(f'\n{"="*60}\n[{name}] {xlsx}\n{"="*60}')
    wb = openpyxl.load_workbook(xlsx)
    ws = wb.active
    START = 3
    END = START - 1
    for r in range(START, ws.max_row + 1):
        if ws.cell(r, 2).value is None:
            break
        END = r

    tasks = []
    for r in range(START, END + 1):
        url = ws.cell(r, 11).value
        is_ext = (ws.cell(r, 19).value == '외부링크')
        tasks.append((r, url, is_ext))
    n_ext = sum(1 for t in tasks if t[2])
    print(f'  처리 대상: 총 {len(tasks)}행 (외부링크 {n_ext})')

    t0 = time.time()
    results = {}
    session = make_session(weak_ssl=weak_ssl)

    def worker(task):
        row, url, is_ext = task
        if is_ext:
            return row, {'L': '사이트', 'M': '', 'N': '', 'O': '', 'P': '', 'Q': '', 'note': ''}
        if not url or not isinstance(url, str):
            return row, {'L': '', 'M': '', 'N': '', 'O': '', 'P': '', 'Q': '', 'note': 'URL 없음'}
        return row, process_row(session, url, body_selectors)

    done = 0
    with ThreadPoolExecutor(max_workers=workers) as ex:
        futs = [ex.submit(worker, t) for t in tasks]
        for fut in as_completed(futs):
            row, res = fut.result()
            results[row] = res
            done += 1
            if done % 50 == 0 or done == len(tasks):
                print(f'  진행 {done}/{len(tasks)} ({time.time()-t0:.0f}s)')

    for r in range(START, END + 1):
        res = results.get(r, {})
        if not res:
            continue
        if res.get('L'): ws.cell(r, 12).value = res['L']
        if res.get('M') != '': ws.cell(r, 13).value = res['M']
        if res.get('N'): ws.cell(r, 14).value = res['N']
        if res.get('O'): ws.cell(r, 15).value = res['O']
        if res.get('P'): ws.cell(r, 16).value = res['P']
        if res.get('Q'): ws.cell(r, 17).value = res['Q']
        if res.get('note'):
            existing = ws.cell(r, 19).value
            if not existing:
                ws.cell(r, 19).value = res['note']

    wb.save(xlsx)
    forms = {}
    attach = {'미부착': 0, '부착': 0, '기타': 0}
    q_dist = {'Y': 0, 'N': 0, '': 0}
    for r, res in results.items():
        forms[res.get('L', '')] = forms.get(res.get('L', ''), 0) + 1
        o = res.get('O', '')
        if o == '미부착': attach['미부착'] += 1
        elif o and '유형' in o: attach['부착'] += 1
        else: attach['기타'] += 1
        q = res.get('Q', '')
        q_dist[q] = q_dist.get(q, 0) + 1
    print(f'  [{name}] L분포 {forms} | 부착 {attach} | Q {q_dist} | 시간 {time.time()-t0:.0f}s')


def main():
    targets = sys.argv[1:] if len(sys.argv) > 1 else list(SITES.keys())
    total_t0 = time.time()
    for name in targets:
        if name not in SITES:
            print(f'  알 수 없음: {name}')
            continue
        cfg = SITES[name]
        try:
            run_site(name, cfg['xlsx'], cfg['body_sel'], weak_ssl=cfg.get('weak_ssl', False))
        except Exception as e:
            print(f'  [{name}] 실패: {e}')
            import traceback
            traceback.print_exc()
    print(f'\n총 소요: {time.time()-total_t0:.0f}s')


if __name__ == '__main__':
    main()