DB_JOB/_스크립트/_tab_phase234.py

# -*- coding: utf-8 -*-
"""탭 확장으로 새로 추가된 행만 골라 Phase 2~4(L/M/N/O/P) 수집.

- 대상: L(게시판형태)이 비어있고 K가 같은 도메인 http URL 인 행 = 신규 탭 행.
- L/M/N: 해당 기관 _phase234.py 의 get_body/detect_form/detect_media/extract_detail_urls 재사용.
- O/P : 확정 KOGL 규칙(_recheck_kogl_all 의 BROAD_IMG_PAT + detect_split + decide_O).
        이미지명 우선, 게시판은 같은도메인 상세 5건 추적, 이미지≠링크면 S열 '링크주소 오기'.
- 인코딩: 바이트로 받아 BeautifulSoup 자동판별(읍면동 등 오판 방지).
- 기존 행(L 이미 채워짐)은 절대 건드리지 않음.

사용: python -X utf8 _tab_phase234.py <엑셀경로> <Phase234 모듈경로> <도메인키워드> [body_sel(콤마)]
  - 개별형(공주시 _phase234.py: get_body(soup))     → body_sel 생략
  - 일괄형(_chungnam_phase234_all.py: get_body(soup, sel)) → body_sel 지정(미지정 시 #txt,#contents,main)
예) python -X utf8 _tab_phase234.py 충청남도/2.공주시/공주시_탭확장.xlsx 충청남도/2.공주시/_phase234.py gongju.go.kr
   python -X utf8 _tab_phase234.py 충청남도/4.논산시/충청남도_논산시.xlsx _chungnam_phase234_all.py nonsan.go.kr "#txt,#contents,main"
"""
import re
import sys
import time
import inspect
import importlib.util
import warnings
from urllib.parse import urlparse
from concurrent.futures import ThreadPoolExecutor, as_completed

import ssl
import openpyxl
import requests
from requests.adapters import HTTPAdapter
from urllib3.util.ssl_ import create_urllib3_context
from bs4 import BeautifulSoup

warnings.filterwarnings('ignore')

UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
H = {'User-Agent': UA}


class WeakSSLAdapter(HTTPAdapter):
    def init_poolmanager(self, *args, **kwargs):
        ctx = create_urllib3_context()
        ctx.set_ciphers('DEFAULT@SECLEVEL=0')
        ctx.options |= 0x4
        ctx.check_hostname = False
        ctx.verify_mode = ssl.CERT_NONE
        kwargs['ssl_context'] = ctx
        return super().init_poolmanager(*args, **kwargs)


SESSION = requests.Session()
SESSION.headers.update(H)

BROAD_IMG_PAT = re.compile(r'(?:new_)?img_open(?:type|code)(\d{1,2})\.(?:png|jpe?g|gif)', re.I)


def load_module(path):
    spec = importlib.util.spec_from_file_location('city_p234', path)
    mod = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(mod)
    return mod


def fetch_soup(url, timeout=14):
    try:
        r = SESSION.get(url, timeout=timeout, verify=False)
        if r.status_code == 200:
            return BeautifulSoup(r.content, 'html.parser')  # 바이트 → 자동 인코딩
    except Exception:
        pass
    return None


def valid(n):
    return 1 <= n <= 4


def detect_split(body, LINK_PAT):
    img_t, link_t = set(), set()
    for a in body.find_all('a', href=True):
        m = LINK_PAT.search(a['href'])
        if m and valid(int(m.group(1))):
            link_t.add(int(m.group(1)))
    blob = ' '.join(filter(None, (img.get('src', '') for img in body.find_all('img'))))
    blob += ' ' + ' '.join(el.get('style', '') for el in body.find_all(style=True))
    blob += ' ' + str(body)
    for m in BROAD_IMG_PAT.finditer(blob):
        n = int(m.group(1))
        if valid(n):
            img_t.add(n)
    return img_t, link_t


def decide_O(img_t, link_t):
    if img_t:
        return ','.join(f'{n}유형' for n in sorted(img_t)), (bool(link_t) and link_t != img_t)
    if link_t:
        if {1, 2, 3, 4}.issubset(link_t):
            return '미부착', False
        return ','.join(f'{n}유형' for n in sorted(link_t)), False
    return '미부착', False


def domain3(host):
    labels = (host or '').split('.')
    return '.'.join(labels[-3:]) if len(labels) >= 3 else host


def same_site(a, b):
    return domain3(urlparse(a).hostname) == domain3(urlparse(b).hostname)


def main():
    xlsx, p234_path, domain = sys.argv[1], sys.argv[2], sys.argv[3]
    if '--weak-ssl' in sys.argv:
        SESSION.mount('https://', WeakSSLAdapter())
    body_sel = None
    if len(sys.argv) > 4 and sys.argv[4].strip() and not sys.argv[4].startswith('--'):
        body_sel = [s.strip() for s in sys.argv[4].split(',') if s.strip()]
    mod = load_module(p234_path)
    LINK_PAT = mod.KOGL_LINK_PAT

    # get_body 시그니처 자동 대응: 일괄형은 (soup, selectors), 개별형은 (soup)
    needs_sel = len(inspect.signature(mod.get_body).parameters) >= 2
    sel = body_sel or ['#txt', '#contents', 'main']
    get_body = (lambda s: mod.get_body(s, sel)) if needs_sel else mod.get_body
    print(f'get_body 인자 {2 if needs_sel else 1}개 | body_sel={sel if needs_sel else "(미사용)"}')

    wb = openpyxl.load_workbook(xlsx)
    ws = wb.active

    targets = []
    for r in range(3, ws.max_row + 1):
        L = ws.cell(r, 12).value
        url = ws.cell(r, 11).value
        if L not in (None, '') :
            continue  # 기존 행 보존
        if not (isinstance(url, str) and domain in url):
            continue
        targets.append((r, url))
    print(f'대상 신규행: {len(targets)}개')

    def work(t):
        r, url = t
        soup = fetch_soup(url)
        if soup is None:
            return r, {'note': '접근 실패'}
        body = get_body(soup)
        form, count = mod.detect_form(body)
        has_img, has_vid, has_txt = mod.detect_media(body)
        img_t, link_t = detect_split(body, LINK_PAT)
        P_loc = '게시판' if img_t or link_t else ''
        if form == '게시판':
            for du in mod.extract_detail_urls(body, url, limit=5):
                if not same_site(url, du):
                    continue
                ds = fetch_soup(du, 10)
                if ds is None:
                    continue
                db = get_body(ds)
                di, dv, dt = mod.detect_media(db)
                has_img = has_img or di; has_vid = has_vid or dv; has_txt = has_txt or dt
                dimg, dlink = detect_split(db, LINK_PAT)
                if (dimg or dlink) and not P_loc:
                    P_loc = '게시물'
                img_t |= dimg; link_t |= dlink
        N = mod.n_string(has_txt, has_img, has_vid)
        O, mismatch = decide_O(img_t, link_t)
        P = '' if O == '미부착' else (P_loc or '게시판')
        return r, {'L': form, 'M': count if form == '게시판' else 1,
                   'N': N, 'O': O, 'P': P, 'mismatch': mismatch}

    t0 = time.time()
    results = {}
    done = 0
    with ThreadPoolExecutor(max_workers=8) as ex:
        futs = [ex.submit(work, t) for t in targets]
        for fut in as_completed(futs):
            r, res = fut.result()
            results[r] = res
            done += 1
            if done % 40 == 0:
                print(f'  진행 {done}/{len(targets)} ({time.time()-t0:.0f}s)')
    print(f'크롤링 완료 ({time.time()-t0:.0f}s)')

    fail = 0
    for r, res in results.items():
        if res.get('note'):
            fail += 1
            if not ws.cell(r, 19).value:
                ws.cell(r, 19).value = res['note']
            continue
        ws.cell(r, 12).value = res['L']
        ws.cell(r, 13).value = res['M']
        ws.cell(r, 14).value = res['N']
        ws.cell(r, 15).value = res['O']
        if res['P']:
            ws.cell(r, 16).value = res['P']
        if res['mismatch']:
            cur = (ws.cell(r, 19).value or '').strip()
            ws.cell(r, 19).value = '링크주소 오기' if not cur else cur + ' / 링크주소 오기'

    try:
        wb.save(xlsx)
        saved = xlsx
    except PermissionError:
        saved = xlsx.replace('.xlsx', '_LP.xlsx')
        wb.save(saved)
        print(f'!! 원본 잠김(Excel 열림). 대체 저장: {saved}')

    from collections import Counter
    Lc = Counter(res.get('L') for res in results.values())
    Oc = Counter(res.get('O') for res in results.values())
    print(f'저장: {xlsx}')
    print('L 분포:', dict(Lc))
    print('O 분포:', dict(Oc))
    print(f'접근실패: {fail}')


if __name__ == '__main__':
    main()