DB_JOB/_스크립트/_홍성_재집계.py

# -*- coding: utf-8 -*-
"""홍성 B순번 34~끝: 게시판 M 재집계(총 게시물 N) + N 재판정(현행규칙) + F병합 누락 보정.
1~33(검수영역) 보존. 사용: python -X utf8 _홍성_재집계.py [--write]
"""
import sys, os, re, shutil, importlib.util, time
from concurrent.futures import ThreadPoolExecutor, as_completed
import openpyxl

HERE=os.path.dirname(os.path.abspath(__file__))
s=importlib.util.spec_from_file_location('m',os.path.join(HERE,'_chungnam_phase234_all.py'))
M=importlib.util.module_from_spec(s); s.loader.exec_module(M)
XLSX=r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청남도\15.홍성군\충청남도_홍성군.xlsx'
BODY=['#contents','#txt','main']
START_B=34  # 이 B순번 이상만 재처리

HONG_TOTAL=re.compile(r'총\s*게시물\s*([\d,]+)')
EMPTY=re.compile(r'게시물이?\s*없|등록된?\s*(?:게시물|자료|글)\s*가?\s*없|검색된\s*(?:게시물|결과)\s*가?\s*없')
DECO=re.compile(r'/common/|move\.png|no[-_]?img|blank|spacer|/ico|/btn|bullet|arrow|/bg|icon|see_btn|/sample|mimetype|/file_|filedown|btn_dir',re.I)
EXCL=re.compile(r'한눈에|흐름도|절차도|로고(?!송)|logo(?!song)|아이콘|배너|banner|신문고|relation_item|tracer|headline|copyright|copy_logo|popup|wa_mk|웹접근성|품질인증',re.I)
AUDIO=re.compile(r'\.(?:mp3|wav|m4a|ogg|flac)\b',re.I)
MAPPDF=re.compile(r'pdf|viewer\.html|/map|kakao|daum.*map',re.I)

def real_imgs(body):
    out=[]
    for img in body.find_all('img'):
        src=img.get('src') or ''
        if not src or M.KOGL_IMG_PAT.search(src) or DECO.search(src): continue
        if EXCL.search(src+' '+(img.get('alt') or '')): continue
        out.append(img)
    return out

def media2(body):
    txt=len(body.get_text(strip=True))>30
    img=len(real_imgs(body))>0
    vid=False
    for ifr in body.find_all('iframe'):
        sname=ifr.get('src') or ''
        if M.YOUTUBE_PAT.search(sname): vid=True
        elif MAPPDF.search(sname): img=True
    if not vid and (body.find('video') or body.find('a',href=M.YOUTUBE_PAT) or M.VIDEO_EXT.search(str(body))): vid=True
    aud=bool(body.find('audio')) or bool(AUDIO.search(str(body)))
    return img,vid,aud,txt

def njoin(t,i,v,a):
    p=[]
    if t:p.append('어문')
    if i:p.append('이미지')
    if v:p.append('영상')
    if a:p.append('오디오')
    return ','.join(p) if p else '없음'

def fetch(url):
    for k in range(3):
        soup=M.fetch(url)
        if soup is not None: return soup
        time.sleep(0.4*(k+1))
    return None

def process_board(url):
    soup=fetch(url)
    if soup is None: return None
    body=M.get_body(soup,BODY)
    txt=body.get_text(' ',strip=True)
    mm=HONG_TOTAL.search(txt)
    total=int(mm.group(1).replace(',','')) if mm else None
    details=M.extract_detail_urls(body,url,limit=10)
    rows=[li for li in body.select('table tbody tr, .bbs_list li, .board_list li, ul.list li') if li.find('a')]
    empty=bool(EMPTY.search(txt))
    if (total==0) or (not details and not rows and empty):
        return {'M':0,'N':'없음'}
    img,vid,aud,t=media2(body)
    for du in details[:10]:
        ds=M.fetch(du)
        if not ds: continue
        db=M.get_body(ds,BODY)
        i2,v2,a2,t2=media2(db)
        img=img or i2; vid=vid or v2; aud=aud or a2; t=t or t2
    Mval=total if total is not None else (len(details) or len(rows))
    return {'M':Mval,'N':njoin(t,img,vid,aud)}

def process_page(url):
    soup=fetch(url)
    if soup is None: return None
    body=M.get_body(soup,BODY)
    img,vid,aud,t=media2(body)
    return {'N':njoin(t,img,vid,aud)}

def fix_fmerge(ws, start_row):
    """F값 있는 행 + 직후 F=None & G채워진 행들 = F그룹 → 병합(미병합시)."""
    fm=set()
    for x in ws.merged_cells.ranges:
        if x.min_col==6==x.max_col:
            for rr in range(x.min_row,x.max_row+1): fm.add(rr)
    merged=0
    r=start_row
    while r<=ws.max_row:
        fv=ws.cell(r,6).value
        if fv in (None,'') or r in fm: r+=1; continue
        k=r+1
        while k<=ws.max_row and ws.cell(k,6).value in (None,'') and ws.cell(k,7).value not in (None,''):
            k+=1
        end=k-1
        if end>r:
            ws.merge_cells(start_row=r,start_column=6,end_row=end,end_column=6)
            merged+=1
        r=k
    return merged

def main():
    write='--write' in sys.argv
    wb=openpyxl.load_workbook(XLSX); ws=wb.active
    # start row = 첫 B>=34
    start=None
    for r in range(3,ws.max_row+1):
        b=ws.cell(r,2).value
        if isinstance(b,int) and b>=START_B: start=r; break
    print('재처리 시작행 r%d (B%s)'%(start,ws.cell(start,2).value))
    jobs=[]
    for r in range(start,ws.max_row+1):
        L=ws.cell(r,12).value; K=ws.cell(r,11).value
        if not(isinstance(K,str) and K.startswith('http')): continue
        if L=='게시판': jobs.append((r,'board',K))
        elif L=='페이지': jobs.append((r,'page',K))
    res={}
    with ThreadPoolExecutor(max_workers=6) as ex:
        futs={ex.submit(process_board if t=='board' else process_page,K):(r,t) for r,t,K in jobs}
        for f in as_completed(futs):
            r,t=futs[f]
            try: res[r]=(t,f.result())
            except: res[r]=(t,None)
    mchg=nchg=fail=0
    for r,(t,out) in sorted(res.items()):
        if out is None: fail+=1; continue
        if t=='board':
            oldM=ws.cell(r,13).value
            if out['M']!=oldM:
                mchg+=1
                if write: ws.cell(r,13).value=out['M']
        oldN=ws.cell(r,14).value
        if out['N']!=oldN:
            nchg+=1
            if write: ws.cell(r,14).value=out['N']
    fmerged=fix_fmerge(ws,start) if write else 0
    # dry: count fmerge gaps
    if not write:
        fmerged=fix_fmerge.__wrapped__ if False else None
    print('M변경 %d · N변경 %d · 접근실패 %d · %s'%(mchg,nchg,fail,'적용' if write else 'DRY'))
    if write:
        print('F병합 보정 %d그룹'%fmerged)
        bak=XLSX.replace('.xlsx','_backup_홍성재집계전.xlsx')
        if not os.path.exists(bak): shutil.copy(XLSX,bak)
        wb.save(XLSX); print('저장+백업')

if __name__=='__main__': main()