df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
79 lines
2.9 KiB
Python
79 lines
2.9 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""진천 L빈칸 행 재수집: URL의 빠진 /home/ 경로 복원 + Phase2~4(L/M/N/O/P/Q) 채움.
|
|
원본 수집이 jincheon.go.kr/sub.do?menukey= (404)로 저장 → /home/sub.do (200)로 교정.
|
|
사용: python -X utf8 _진천_refill.py [--write]
|
|
"""
|
|
import sys, os, re, shutil, importlib.util
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
import openpyxl
|
|
|
|
HERE=os.path.dirname(os.path.abspath(__file__))
|
|
sp=importlib.util.spec_from_file_location('cb',os.path.join(HERE,'_chungbuk_phase234_all.py'))
|
|
M=importlib.util.module_from_spec(sp); sp.loader.exec_module(M)
|
|
XLSX=r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청북도\9.진천군\충청북도_진천군.xlsx'
|
|
BODY=['#contents','#txt','main']
|
|
sess=M.make_session()
|
|
|
|
def home_variant(u):
|
|
# jincheon.go.kr/sub.do → jincheon.go.kr/home/sub.do (도메인 직후 bare sub.do만)
|
|
return re.sub(r'(jincheon\.go\.kr)/sub\.do', r'\1/home/sub.do', u)
|
|
|
|
def working_url(u):
|
|
s=M.fetch(sess,u)
|
|
if s is not None: return u,s
|
|
v=home_variant(u)
|
|
if v!=u:
|
|
s=M.fetch(sess,v)
|
|
if s is not None: return v,s
|
|
return None,None
|
|
|
|
def do_row(r,K):
|
|
wu,soup=working_url(K)
|
|
if soup is None:
|
|
return r,None,K
|
|
out=M.process_row(sess, wu, BODY) # L/M/N/O/P/Q + note
|
|
return r,out,wu
|
|
|
|
def main():
|
|
write='--write' in sys.argv
|
|
wb=openpyxl.load_workbook(XLSX); ws=wb.active
|
|
jobs=[]
|
|
for r in range(3,ws.max_row+1):
|
|
K=ws.cell(r,11).value; L=ws.cell(r,12).value
|
|
if isinstance(K,str) and K.startswith('http') and L in (None,''):
|
|
jobs.append((r,K))
|
|
print('대상 L빈칸+http행:',len(jobs))
|
|
res={}
|
|
with ThreadPoolExecutor(max_workers=6) as ex:
|
|
futs={ex.submit(do_row,r,K):r for r,K in jobs}
|
|
for f in as_completed(futs):
|
|
r,out,wu=f.result(); res[r]=(out,wu)
|
|
filled=fail=urlfix=0
|
|
for r,(out,wu) in sorted(res.items()):
|
|
K0=ws.cell(r,11).value
|
|
if out is None:
|
|
fail+=1; continue
|
|
if wu!=K0:
|
|
urlfix+=1
|
|
if write:
|
|
ws.cell(r,11).value=wu
|
|
cell=ws.cell(r,11)
|
|
if cell.hyperlink: cell.hyperlink.target=wu
|
|
if write:
|
|
ws.cell(r,12).value=out['L']; ws.cell(r,13).value=out['M']
|
|
ws.cell(r,14).value=out['N']
|
|
ws.cell(r,15).value=out['O']; ws.cell(r,16).value=out['P']; ws.cell(r,17).value=out['Q']
|
|
filled+=1
|
|
print(f'채움 {filled} (URL교정 {urlfix}) · 실패 {fail} · {"적용" if write else "DRY"}')
|
|
# 분포
|
|
from collections import Counter
|
|
c=Counter(v[0]['L'] for v in res.values() if v[0])
|
|
print('새 L분포(채운것):',dict(c))
|
|
if write and filled:
|
|
bak=XLSX.replace('.xlsx','_backup_진천재수집전.xlsx')
|
|
if not os.path.exists(bak): shutil.copy(XLSX,bak)
|
|
wb.save(XLSX)
|
|
print('저장+백업')
|
|
|
|
if __name__=='__main__': main()
|