DB_JOB/_스크립트/_tab_batch.py

# -*- coding: utf-8 -*-
"""수집완료(✅) 기관 일괄 본문탭(1-5b) 확장 + 신규행 Phase2~4 수집 오케스트레이터.

매뉴얼 1-5b 의 _tab_expand.py / _tab_phase234.py 를 기관 표대로 순차 호출한다.
- 검수완료(계룡시)·미처리(보은군)·이미 탭확장(공주시)는 제외.
- base/domain 은 각 엑셀 K열 URL에서 자동 도출(가장 흔한 host 기준).
- 영동군만 가중 SSL(--weak-ssl).

모드:
  python -X utf8 _tab_batch.py probe        # 설정·행수·base/domain 검증만
  python -X utf8 _tab_batch.py dry           # 전 기관 확장계획(읽기전용)만 산출
  python -X utf8 _tab_batch.py run [기관...]  # 확장(--write)+Phase234 실제 수행
"""
import os
import re
import sys
import subprocess
from collections import Counter
from urllib.parse import urlsplit

import openpyxl

HERE = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.dirname(HERE)
MAP = os.path.join(ROOT, '작업파일', '광역_사이트맵')

CN_ALL = os.path.join(HERE, '_chungnam_phase234_all.py')
CB_ALL = os.path.join(HERE, '_chungbuk_phase234_all.py')
JB_ALL = os.path.join(HERE, '_jeonbuk_phase234_all.py')
GEUMSAN = os.path.join(MAP, '충청남도', '3.금산군', '_phase234.py')
JB_SEL = '#main-contents,#content,#contents,.contents,#txt,main,#container,#sub'

# (광역, idx, 기관, 폴더명, phase234모듈, body_sel(콤마/None), weak_ssl)
SITES = [
    # 충청남도 (계룡=검수완료 제외, 공주=이미 탭확장 제외)
    ('충청남도', 3, '금산군', GEUMSAN, None, False),
    ('충청남도', 4, '논산시', CN_ALL, '#txt,#contents,main', False),
    ('충청남도', 5, '당진시', CN_ALL, '#txt,#contents,main', False),
    ('충청남도', 6, '보령시', CN_ALL, '#txt,#contents,main', False),
    ('충청남도', 7, '부여군', CN_ALL, '#txt,#contents,main', False),
    ('충청남도', 8, '서산시', CN_ALL, '#contents,#txt,main', False),
    ('충청남도', 9, '서천군', CN_ALL, '#txt,#contents,main', False),
    ('충청남도', 10, '아산시', CN_ALL, '#contents,main,#txt', False),
    ('충청남도', 11, '예산군', CN_ALL, '#txt,#contents,main', False),
    ('충청남도', 12, '천안시', CN_ALL, '#txt,#contents,main', False),
    ('충청남도', 13, '청양군', CN_ALL, '#txt,#contents,main', False),
    ('충청남도', 14, '태안군', CN_ALL, '#txt,#contents,main', False),
    ('충청남도', 15, '홍성군', CN_ALL, '#txt,#contents,main', False),
    # 충청북도 (보은=미처리 제외)
    ('충청북도', 1, '괴산군', CB_ALL, '#contents,#txt,main', False),
    ('충청북도', 2, '단양군', CB_ALL, '#contents,#txt,main', False),
    ('충청북도', 4, '영동군', CB_ALL, '#txt,#contents,main', True),
    ('충청북도', 5, '옥천군', CB_ALL, '#contents,#txt,main', False),
    ('충청북도', 6, '음성군', CB_ALL, '#contents,#txt,main', False),
    ('충청북도', 7, '제천시', CB_ALL, '#contents,#txt,main', False),
    ('충청북도', 8, '증평군', CB_ALL, '#txt,#contents,main', False),
    ('충청북도', 9, '진천군', CB_ALL, '#contents,#txt,main', False),
    ('충청북도', 10, '청주시', CB_ALL, '#contents,#txt,main', False),
    ('충청북도', 11, '충주시', CB_ALL, '#contents,#txt,main', False),
    # 전북특별자치도
    ('전북특별자치도', 1, '고창군', JB_ALL, JB_SEL, False),
    ('전북특별자치도', 2, '군산시', JB_ALL, JB_SEL, False),
    ('전북특별자치도', 3, '김제시', JB_ALL, JB_SEL, False),
    ('전북특별자치도', 4, '남원시', JB_ALL, JB_SEL, False),
    ('전북특별자치도', 5, '무주군', JB_ALL, JB_SEL, False),
    ('전북특별자치도', 6, '부안군', JB_ALL, JB_SEL, False),
    ('전북특별자치도', 7, '순창군', JB_ALL, JB_SEL, False),
    ('전북특별자치도', 8, '완주군', JB_ALL, JB_SEL, False),
    ('전북특별자치도', 9, '익산시', JB_ALL, JB_SEL, False),
    ('전북특별자치도', 10, '임실군', JB_ALL, JB_SEL, False),
    ('전북특별자치도', 11, '장수군', JB_ALL, JB_SEL, False),
    ('전북특별자치도', 12, '전주시', JB_ALL, JB_SEL, False),
    ('전북특별자치도', 13, '정읍시', JB_ALL, JB_SEL, False),
    ('전북특별자치도', 14, '진안군', JB_ALL, JB_SEL, False),
    # 제주특별자치도
    ('제주특별자치도', 1, '서귀포시', JB_ALL, JB_SEL, False),
    ('제주특별자치도', 2, '제주시', JB_ALL, JB_SEL, False),
]


def xlsx_path(prov, idx, name):
    return os.path.join(MAP, prov, f'{idx}.{name}', f'{prov}_{name}.xlsx')


def derive_base_domain(xlsx):
    """K열 URL에서 가장 흔한 host → base(scheme://host), domain(끝 3라벨)."""
    wb = openpyxl.load_workbook(xlsx, read_only=True)
    ws = wb.active
    hosts = Counter()
    scheme_by_host = {}
    nrow = 0
    for r in ws.iter_rows(min_row=3, min_col=11, max_col=11, values_only=True):
        u = r[0]
        if isinstance(u, str) and u.startswith('http'):
            nrow += 1
            sp = urlsplit(u)
            if sp.hostname:
                hosts[sp.hostname] += 1
                scheme_by_host.setdefault(sp.hostname, sp.scheme)
    # 데이터 행수(K 무관)
    maxrow = ws.max_row
    wb.close()
    if not hosts:
        return None, None, maxrow
    host = hosts.most_common(1)[0][0]
    base = f'{scheme_by_host[host]}://{host}'
    labels = host.split('.')
    domain = '.'.join(labels[-3:]) if len(labels) >= 3 else host
    return base, domain, maxrow


def run(cmd):
    print('  $', ' '.join(os.path.basename(c) if c.endswith('.py') else c for c in cmd[2:]))
    p = subprocess.run(cmd, capture_output=True, text=True, encoding='utf-8', errors='replace')
    out = (p.stdout or '') + (p.stderr or '')
    return out


def main():
    mode = sys.argv[1] if len(sys.argv) > 1 else 'probe'
    only = set(sys.argv[2:]) if len(sys.argv) > 2 else None
    py = [sys.executable, '-X', 'utf8']

    sites = [s for s in SITES if not only or s[2] in only]
    print(f'대상 기관: {len(sites)}개  (모드={mode})\n')

    for prov, idx, name, mod, sel, weak in sites:
        xp = xlsx_path(prov, idx, name)
        if not os.path.exists(xp):
            print(f'❌ {prov} {name}: 엑셀 없음 → {xp}\n')
            continue
        base, domain, maxrow = derive_base_domain(xp)
        tag = ' [weak-ssl]' if weak else ''
        print(f'━━━ {prov} {name}{tag}  (행~{maxrow}, base={base}, domain={domain})')

        if mode == 'probe':
            print(f'    모듈={os.path.basename(mod)}  body_sel={sel}\n')
            continue

        # 1) 항상 dry 로 먼저 확장계획 산출(읽기전용)
        ex = py + [os.path.join(HERE, '_tab_expand.py'), xp, base, domain]
        if weak:
            ex += ['--weak-ssl']
        out = run(ex)
        m = re.search(r'신규 추가 행:\s*(\d+)개', out)
        newcnt = int(m.group(1)) if m else 0
        grp = re.search(r'탭 확장 계획:\s*(\d+)개', out)
        print(f'    → 확장그룹 {grp.group(1) if grp else "?"}개 / 신규행 {newcnt}개')
        if mode == 'dry':
            print()
            continue

        # 2) run 모드 & 신규행 있을 때만 --write 후 Phase234
        if newcnt > 0:
            exw = ex + ['--write']
            run(exw)
            ph = py + [os.path.join(HERE, '_tab_phase234.py'), xp, mod, domain]
            if sel:
                ph += [sel]
            if weak:
                ph += ['--weak-ssl']
            out2 = run(ph)
            for line in out2.splitlines():
                if any(k in line for k in ('대상 신규행', 'L 분포', 'O 분포', '접근실패', '저장', '대체 저장')):
                    print('    ' + line)
        else:
            print('    (신규행 없음 → Phase234 생략)')
        print()

    print('\n=== 배치 종료 ===')


if __name__ == '__main__':
    main()