df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
94 lines
4.6 KiB
Python
94 lines
4.6 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""장수 N 재판정 (r81~끝). 어문/어문,이미지/영상/오디오/없음. 본문+게시판 상세top10 이미지규칙.
|
|
사용: python -X utf8 _장수_N재판정.py [--write]
|
|
"""
|
|
import sys, os, re, shutil, importlib.util
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
import openpyxl
|
|
HERE=os.path.dirname(os.path.abspath(__file__))
|
|
s=importlib.util.spec_from_file_location('m',os.path.join(HERE,'_jeonbuk_phase234_all.py'))
|
|
M=importlib.util.module_from_spec(s); s.loader.exec_module(M)
|
|
XLSX=r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\전북특별자치도\11.장수군\전북특별자치도_장수군.xlsx'
|
|
import sys as _s; START=int(next((a for a in _s.argv[1:] if a.isdigit()),81)); SESS=M.make_session()
|
|
DECO=re.compile(r'/common/|move\.png|no[-_]?img|blank|spacer|/ico|/btn|bullet|arrow|/bg|icon|see_btn|/sample|mimetype|/file_|filedown|btn_dir|/open\d|viewImg|/board/view|_link\d|governing_img|img_open(?:code|type)0[._]|/link\d',re.I)
|
|
EXCL=re.compile(r'한눈에|흐름도|절차도|로고(?!송)|logo(?!song)|아이콘|배너|banner|신문고|relation_item|tracer|headline|copyright|copy_logo|popup|wa_mk|웹접근성|품질인증|전용뷰어|뷰어로보기|바로가기|생활법령|전자관보|정보시스템',re.I)
|
|
AUDIO=re.compile(r'\.(?:mp3|wav|m4a|ogg|flac)\b',re.I)
|
|
MAPPDF=re.compile(r'pdf|viewer\.html|/map|kakao|daum.*map',re.I)
|
|
DETAIL=re.compile(r'view\.|/view\b|articleNo=|dataSid=|nttId=|not_ancmt|bbsView|seqRepeat=|mode=V|idx=|seq=',re.I)
|
|
EMPTY=re.compile(r'게시물이?\s*없|등록된?\s*(?:게시물|자료|글)\s*가?\s*없|검색된\s*(?:게시물|결과)\s*가?\s*없')
|
|
def realimg(b):
|
|
for img in b.find_all('img'):
|
|
src=img.get('src') or ''
|
|
if not src or M.KOGL_IMG_PAT.search(src) or DECO.search(src): continue
|
|
if EXCL.search(src+' '+(img.get('alt') or '')): continue
|
|
return True
|
|
return False
|
|
def media(b):
|
|
t=len(b.get_text(strip=True))>30; img=realimg(b); vid=False
|
|
for ifr in b.find_all('iframe'):
|
|
ss=ifr.get('src') or ''
|
|
if M.YOUTUBE_PAT.search(ss): vid=True
|
|
elif MAPPDF.search(ss): img=True
|
|
if not vid and (b.find('video') or b.find('a',href=M.YOUTUBE_PAT) or M.VIDEO_EXT.search(str(b))): vid=True
|
|
aud=bool(b.find('audio')) or bool(AUDIO.search(str(b)))
|
|
return img,vid,aud,t
|
|
def detail_urls(b,base,lim=10):
|
|
from urllib.parse import urljoin
|
|
out=[];seen=set()
|
|
for a in b.find_all('a',href=True):
|
|
h=a['href']
|
|
if h and not h.startswith('#') and DETAIL.search(h):
|
|
f=urljoin(base,h)
|
|
if f not in seen: seen.add(f);out.append(f)
|
|
if len(out)>=lim: break
|
|
return out
|
|
def njoin(t,i,v,a):
|
|
p=[]
|
|
if t:p.append('어문')
|
|
if i:p.append('이미지')
|
|
if v:p.append('영상')
|
|
if a:p.append('오디오')
|
|
return ','.join(p) if p else '없음'
|
|
def judge(K,L):
|
|
soup=None
|
|
for _ in range(3):
|
|
soup=M.fetch(SESS,K)
|
|
if soup is not None: break
|
|
if soup is None: return None
|
|
b=M.get_body(soup,M.BODY_SEL); txt=b.get_text(' ',strip=True)
|
|
img,vid,aud,t=media(b)
|
|
if L=='게시판':
|
|
dus=detail_urls(b,K,10)
|
|
if not dus and (EMPTY.search(txt) or not t): return '없음'
|
|
for du in dus:
|
|
ds=M.fetch(SESS,du)
|
|
if not ds: continue
|
|
db=M.get_body(ds,M.BODY_SEL)
|
|
i2,v2,a2,t2=media(db); img=img or i2;vid=vid or v2;aud=aud or a2;t=t or t2
|
|
return njoin(t,img,vid,aud)
|
|
def main():
|
|
write='--write' in sys.argv
|
|
wb=openpyxl.load_workbook(XLSX); ws=wb.active
|
|
jobs=[(r,ws.cell(r,11).value,ws.cell(r,12).value) for r in range(START,ws.max_row+1)
|
|
if isinstance(ws.cell(r,11).value,str) and ws.cell(r,11).value.startswith('http') and ws.cell(r,12).value in ('페이지','게시판')]
|
|
res={}
|
|
with ThreadPoolExecutor(max_workers=6) as ex:
|
|
futs={ex.submit(judge,K,L):r for r,K,L in jobs}
|
|
for f in as_completed(futs):
|
|
r=futs[f]
|
|
try: res[r]=f.result()
|
|
except: res[r]=None
|
|
chg=[]
|
|
for r in sorted(res):
|
|
n=res[r]
|
|
if n is None: continue
|
|
old=ws.cell(r,14).value
|
|
if n!=(old or ''): chg.append((r,old,n))
|
|
print('N변경 %d행:'%len(chg))
|
|
for r,o,n in chg[:60]: print(' r%d %s: %s→%s'%(r,ws.cell(r,7).value or ws.cell(r,6).value,o,n))
|
|
if write and chg:
|
|
shutil.copy(XLSX,XLSX.replace('.xlsx','_backup_N재판정전.xlsx'))
|
|
for r,o,n in chg: ws.cell(r,14).value=n
|
|
wb.save(XLSX); print('적용 %d행'%len(chg))
|
|
if __name__=='__main__': main()
|