DB_JOB/_스크립트/_recheck_kogl_jeonbuk.py

# -*- coding: utf-8 -*-
"""전북(14)+제주(2) 공공누리(O열) 권위 재판정 — 충청도 _recheck_kogl_all.py 방식.

_jeonbuk_phase234_all 의 검출 로직(get_body/detect_form/extract_detail_urls/KOGL_LINK_PAT)을
그대로 재사용. 각 부착 행을 재크롤링하여:
  1) O열 = 이미지명(img_opentype/opencode) 유형 우선 권위 재판정
  2) 이미지 존재 & 이미지≠링크 → S열 비고 '링크주소 오기'
  3) 링크만 있고 1·2·3·4 전부 → 범례페이지로 미부착
규칙: feedback_kogl_image_rule. 상세추적은 같은 기관 도메인만.

사용:  python -X utf8 _recheck_kogl_jeonbuk.py [기관명 ...]
"""
import sys, os, re, shutil, warnings, openpyxl
from concurrent.futures import ThreadPoolExecutor
from urllib.parse import urlparse

sys.path.insert(0, r'D:\01.프로젝트\DB수집\_스크립트')
sys.stdout.reconfigure(encoding='utf-8')
warnings.filterwarnings('ignore')

import _jeonbuk_phase234_all as JB

O_COL, K_COL, S_COL = 15, 11, 19
WORKERS = 12
DETAIL_TIMEOUT = 5
DETAIL_LIMIT = 3

# (기관명, weak_ssl) — xlsx/body_sel 은 JB.SITES 에서 가져옴
TARGETS = [(name, JB.SITES[name].get('weak_ssl', False)) for name in JB.SITES]

BROAD_IMG_PAT = re.compile(r'(?:new_)?img_open(?:type|code)(\d{1,2})\.(?:png|jpe?g|gif)', re.I)


def _valid(n):
    return 1 <= n <= 4


def detect_split(body, LINK_PAT):
    img_t, link_t = set(), set()
    for a in body.find_all('a', href=True):
        m = LINK_PAT.search(a['href'])
        if m and _valid(int(m.group(1))):
            link_t.add(int(m.group(1)))
    blob = ' '.join(filter(None, (img.get('src', '') for img in body.find_all('img'))))
    blob += ' ' + ' '.join(el.get('style', '') for el in body.find_all(style=True))
    blob += ' ' + str(body)
    for m in BROAD_IMG_PAT.finditer(blob):
        n = int(m.group(1))
        if _valid(n):
            img_t.add(n)
    return img_t, link_t


def decide_O(img_t, link_t):
    if img_t:
        newO = ','.join(f'{n}유형' for n in sorted(img_t))
        return newO, bool(link_t) and (link_t != img_t)
    if link_t:
        if {1, 2, 3, 4}.issubset(link_t):
            return '미부착', False
        return ','.join(f'{n}유형' for n in sorted(link_t)), False
    return '미부착', False


def make_fetch(weak_ssl):
    sess = JB.make_session(weak_ssl)
    return lambda url, timeout=DETAIL_TIMEOUT: JB.fetch(sess, url, timeout)


def _domain(host):
    labels = (host or '').split('.')
    return '.'.join(labels[-3:]) if len(labels) >= 3 else host


def same_site(base_url, target_url):
    return _domain(urlparse(base_url).hostname) == _domain(urlparse(target_url).hostname)


def detect_row(url, body_sel, fetch_fn):
    soup = fetch_fn(url)
    if soup is None:
        return None, None, 'fetch_fail'
    body = JB.get_body(soup, body_sel)
    form, _ = JB.detect_form(body)
    img_t, link_t = detect_split(body, JB.KOGL_LINK_PAT)
    if form == '게시판':
        for du in JB.extract_detail_urls(body, url, limit=DETAIL_LIMIT):
            if not same_site(url, du):
                continue
            ds = fetch_fn(du, DETAIL_TIMEOUT)
            if ds is None:
                continue
            db = JB.get_body(ds, body_sel)
            di, dl = detect_split(db, JB.KOGL_LINK_PAT)
            img_t |= di
            link_t |= dl
    return img_t, link_t, 'ok'


def process_site(name, weak_ssl):
    cfg = JB.SITES[name]
    xlsx, body_sel = cfg['xlsx'], cfg['body_sel']
    print(f'\n{"="*70}\n■ {name}  ({os.path.relpath(xlsx)})')
    wb = openpyxl.load_workbook(xlsx)
    ws = wb.active
    targets = []
    for r in range(3, ws.max_row + 1):
        o = ws.cell(r, O_COL).value
        if not o or str(o).strip() == '미부착':
            continue
        url = ws.cell(r, K_COL).value
        if url:
            targets.append((r, str(url).strip(), o, ws.cell(r, S_COL).value))
    if not targets:
        print('  부착 행 없음 → 변경 없음')
        return dict(name=name, total=0, o_changed=0, to_none=0, mismatch=0, fetch_fail=0)

    print(f'  부착 행 {len(targets)}건 재크롤링 (workers={WORKERS}, weak_ssl={weak_ssl})')
    fetch_fn = make_fetch(weak_ssl)

    def work(t):
        r, url, oldO, oldS = t
        img_t, link_t, status = detect_row(url, body_sel, fetch_fn)
        return (r, url, oldO, oldS, img_t, link_t, status)

    results = []
    with ThreadPoolExecutor(max_workers=WORKERS) as ex:
        for res in ex.map(work, targets):
            results.append(res)

    o_changed = to_none = mismatch = fetch_fail = 0
    for r, url, oldO, oldS, img_t, link_t, status in sorted(results):
        if status != 'ok':
            fetch_fail += 1
            continue
        newO, do_flag = decide_O(img_t, link_t)
        if newO != str(oldO).strip():
            o_changed += 1
            if newO == '미부착':
                to_none += 1
            ws.cell(r, O_COL).value = newO
        if do_flag:
            mismatch += 1
            cur = (oldS or '').strip()
            if '링크주소 오기' not in cur:
                ws.cell(r, S_COL).value = '링크주소 오기' if not cur else cur + ' / 링크주소 오기'

    backup = os.path.join(os.path.dirname(xlsx), f'{name}_backup_kogl재판정전.xlsx')
    if not os.path.exists(backup):
        shutil.copy2(xlsx, backup)
    wb.save(xlsx)
    print(f'  ▶ {name} 완료: O변경 {o_changed} (미부착化 {to_none}) | 불일치비고 {mismatch} | fetch실패 {fetch_fail}')
    return dict(name=name, total=len(targets), o_changed=o_changed, to_none=to_none,
                mismatch=mismatch, fetch_fail=fetch_fail)


def main():
    sel = [a for a in sys.argv[1:] if not a.startswith('-')]
    sites = [t for t in TARGETS if (not sel or t[0] in sel)]
    print(f'대상 {len(sites)}개: {[s[0] for s in sites]}')
    summary = []
    for name, weak in sites:
        try:
            summary.append(process_site(name, weak))
        except Exception as e:
            import traceback; traceback.print_exc()
            summary.append(dict(name=name, total=-1, o_changed=0, to_none=0, mismatch=0, fetch_fail=0))

    print('\n' + '=' * 70 + '\n[전체 요약]')
    print(f'{"기관":<8}{"부착":>6}{"O변경":>7}{"미부착化":>8}{"불일치":>7}{"실패":>6}')
    for s in summary:
        print(f'{s["name"]:<8}{s["total"]:>6}{s["o_changed"]:>7}{s["to_none"]:>8}{s["mismatch"]:>7}{s["fetch_fail"]:>6}')
    tot = lambda k: sum(x[k] for x in summary if x[k] >= 0)
    print(f'\n합계: 부착 {tot("total")} | O변경 {tot("o_changed")} | 미부착化 {tot("to_none")} | 불일치비고 {tot("mismatch")} | fetch실패 {tot("fetch_fail")}')


if __name__ == '__main__':
    main()