# -*- coding: utf-8 -*- """수집완료(✅) 기관 일괄 본문탭(1-5b) 확장 + 신규행 Phase2~4 수집 오케스트레이터. 매뉴얼 1-5b 의 _tab_expand.py / _tab_phase234.py 를 기관 표대로 순차 호출한다. - 검수완료(계룡시)·미처리(보은군)·이미 탭확장(공주시)는 제외. - base/domain 은 각 엑셀 K열 URL에서 자동 도출(가장 흔한 host 기준). - 영동군만 가중 SSL(--weak-ssl). 모드: python -X utf8 _tab_batch.py probe # 설정·행수·base/domain 검증만 python -X utf8 _tab_batch.py dry # 전 기관 확장계획(읽기전용)만 산출 python -X utf8 _tab_batch.py run [기관...] # 확장(--write)+Phase234 실제 수행 """ import os import re import sys import subprocess from collections import Counter from urllib.parse import urlsplit import openpyxl HERE = os.path.dirname(os.path.abspath(__file__)) ROOT = os.path.dirname(HERE) MAP = os.path.join(ROOT, '작업파일', '광역_사이트맵') CN_ALL = os.path.join(HERE, '_chungnam_phase234_all.py') CB_ALL = os.path.join(HERE, '_chungbuk_phase234_all.py') JB_ALL = os.path.join(HERE, '_jeonbuk_phase234_all.py') GEUMSAN = os.path.join(MAP, '충청남도', '3.금산군', '_phase234.py') JB_SEL = '#main-contents,#content,#contents,.contents,#txt,main,#container,#sub' # (광역, idx, 기관, 폴더명, phase234모듈, body_sel(콤마/None), weak_ssl) SITES = [ # 충청남도 (계룡=검수완료 제외, 공주=이미 탭확장 제외) ('충청남도', 3, '금산군', GEUMSAN, None, False), ('충청남도', 4, '논산시', CN_ALL, '#txt,#contents,main', False), ('충청남도', 5, '당진시', CN_ALL, '#txt,#contents,main', False), ('충청남도', 6, '보령시', CN_ALL, '#txt,#contents,main', False), ('충청남도', 7, '부여군', CN_ALL, '#txt,#contents,main', False), ('충청남도', 8, '서산시', CN_ALL, '#contents,#txt,main', False), ('충청남도', 9, '서천군', CN_ALL, '#txt,#contents,main', False), ('충청남도', 10, '아산시', CN_ALL, '#contents,main,#txt', False), ('충청남도', 11, '예산군', CN_ALL, '#txt,#contents,main', False), ('충청남도', 12, '천안시', CN_ALL, '#txt,#contents,main', False), ('충청남도', 13, '청양군', CN_ALL, '#txt,#contents,main', False), ('충청남도', 14, '태안군', CN_ALL, '#txt,#contents,main', False), ('충청남도', 15, '홍성군', CN_ALL, '#txt,#contents,main', False), # 충청북도 (보은=미처리 제외) ('충청북도', 1, '괴산군', CB_ALL, '#contents,#txt,main', False), ('충청북도', 2, '단양군', CB_ALL, '#contents,#txt,main', False), ('충청북도', 4, '영동군', CB_ALL, '#txt,#contents,main', True), ('충청북도', 5, '옥천군', CB_ALL, '#contents,#txt,main', False), ('충청북도', 6, '음성군', CB_ALL, '#contents,#txt,main', False), ('충청북도', 7, '제천시', CB_ALL, '#contents,#txt,main', False), ('충청북도', 8, '증평군', CB_ALL, '#txt,#contents,main', False), ('충청북도', 9, '진천군', CB_ALL, '#contents,#txt,main', False), ('충청북도', 10, '청주시', CB_ALL, '#contents,#txt,main', False), ('충청북도', 11, '충주시', CB_ALL, '#contents,#txt,main', False), # 전북특별자치도 ('전북특별자치도', 1, '고창군', JB_ALL, JB_SEL, False), ('전북특별자치도', 2, '군산시', JB_ALL, JB_SEL, False), ('전북특별자치도', 3, '김제시', JB_ALL, JB_SEL, False), ('전북특별자치도', 4, '남원시', JB_ALL, JB_SEL, False), ('전북특별자치도', 5, '무주군', JB_ALL, JB_SEL, False), ('전북특별자치도', 6, '부안군', JB_ALL, JB_SEL, False), ('전북특별자치도', 7, '순창군', JB_ALL, JB_SEL, False), ('전북특별자치도', 8, '완주군', JB_ALL, JB_SEL, False), ('전북특별자치도', 9, '익산시', JB_ALL, JB_SEL, False), ('전북특별자치도', 10, '임실군', JB_ALL, JB_SEL, False), ('전북특별자치도', 11, '장수군', JB_ALL, JB_SEL, False), ('전북특별자치도', 12, '전주시', JB_ALL, JB_SEL, False), ('전북특별자치도', 13, '정읍시', JB_ALL, JB_SEL, False), ('전북특별자치도', 14, '진안군', JB_ALL, JB_SEL, False), # 제주특별자치도 ('제주특별자치도', 1, '서귀포시', JB_ALL, JB_SEL, False), ('제주특별자치도', 2, '제주시', JB_ALL, JB_SEL, False), ] def xlsx_path(prov, idx, name): return os.path.join(MAP, prov, f'{idx}.{name}', f'{prov}_{name}.xlsx') def derive_base_domain(xlsx): """K열 URL에서 가장 흔한 host → base(scheme://host), domain(끝 3라벨).""" wb = openpyxl.load_workbook(xlsx, read_only=True) ws = wb.active hosts = Counter() scheme_by_host = {} nrow = 0 for r in ws.iter_rows(min_row=3, min_col=11, max_col=11, values_only=True): u = r[0] if isinstance(u, str) and u.startswith('http'): nrow += 1 sp = urlsplit(u) if sp.hostname: hosts[sp.hostname] += 1 scheme_by_host.setdefault(sp.hostname, sp.scheme) # 데이터 행수(K 무관) maxrow = ws.max_row wb.close() if not hosts: return None, None, maxrow host = hosts.most_common(1)[0][0] base = f'{scheme_by_host[host]}://{host}' labels = host.split('.') domain = '.'.join(labels[-3:]) if len(labels) >= 3 else host return base, domain, maxrow def run(cmd): print(' $', ' '.join(os.path.basename(c) if c.endswith('.py') else c for c in cmd[2:])) p = subprocess.run(cmd, capture_output=True, text=True, encoding='utf-8', errors='replace') out = (p.stdout or '') + (p.stderr or '') return out def main(): mode = sys.argv[1] if len(sys.argv) > 1 else 'probe' only = set(sys.argv[2:]) if len(sys.argv) > 2 else None py = [sys.executable, '-X', 'utf8'] sites = [s for s in SITES if not only or s[2] in only] print(f'대상 기관: {len(sites)}개 (모드={mode})\n') for prov, idx, name, mod, sel, weak in sites: xp = xlsx_path(prov, idx, name) if not os.path.exists(xp): print(f'❌ {prov} {name}: 엑셀 없음 → {xp}\n') continue base, domain, maxrow = derive_base_domain(xp) tag = ' [weak-ssl]' if weak else '' print(f'━━━ {prov} {name}{tag} (행~{maxrow}, base={base}, domain={domain})') if mode == 'probe': print(f' 모듈={os.path.basename(mod)} body_sel={sel}\n') continue # 1) 항상 dry 로 먼저 확장계획 산출(읽기전용) ex = py + [os.path.join(HERE, '_tab_expand.py'), xp, base, domain] if weak: ex += ['--weak-ssl'] out = run(ex) m = re.search(r'신규 추가 행:\s*(\d+)개', out) newcnt = int(m.group(1)) if m else 0 grp = re.search(r'탭 확장 계획:\s*(\d+)개', out) print(f' → 확장그룹 {grp.group(1) if grp else "?"}개 / 신규행 {newcnt}개') if mode == 'dry': print() continue # 2) run 모드 & 신규행 있을 때만 --write 후 Phase234 if newcnt > 0: exw = ex + ['--write'] run(exw) ph = py + [os.path.join(HERE, '_tab_phase234.py'), xp, mod, domain] if sel: ph += [sel] if weak: ph += ['--weak-ssl'] out2 = run(ph) for line in out2.splitlines(): if any(k in line for k in ('대상 신규행', 'L 분포', 'O 분포', '접근실패', '저장', '대체 저장')): print(' ' + line) else: print(' (신규행 없음 → Phase234 생략)') print() print('\n=== 배치 종료 ===') if __name__ == '__main__': main()