df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
105 lines
4.8 KiB
Python
105 lines
4.8 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""익산 게시판 행 M/N/O/P/Q 수집(행범위 지정). 글수(전체N)·N(이미지규칙+상세top10)·KOGL(open0N+표준+링크, 상세top10).
|
|
사용: python -X utf8 _익산_mnopq.py <start> <end> [--write]
|
|
"""
|
|
import sys, os, re, shutil, importlib.util
|
|
from urllib.parse import urljoin
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
import openpyxl
|
|
|
|
HERE=os.path.dirname(os.path.abspath(__file__))
|
|
s=importlib.util.spec_from_file_location('m',os.path.join(HERE,'_jeonbuk_phase234_all.py'))
|
|
M=importlib.util.module_from_spec(s); s.loader.exec_module(m if False else M)
|
|
sk=importlib.util.spec_from_file_location('k',os.path.join(HERE,'_임실_kogl.py'))
|
|
KG=importlib.util.module_from_spec(sk); sk.loader.exec_module(KG)
|
|
XLSX=r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\전북특별자치도\9.익산시\전북특별자치도_익산시.xlsx'
|
|
SESS=M.make_session()
|
|
TOT=re.compile(r'전체\s*([\d,]+)|총\s*([\d,]+)\s*건|총\s*게시물\s*([\d,]+)')
|
|
DECO=re.compile(r'/common/|move\.png|no[-_]?img|blank|spacer|/ico|/btn|bullet|arrow|/bg|icon|see_btn|/sample|mimetype|/file_|filedown|btn_dir|/open\d',re.I)
|
|
EXCL=re.compile(r'한눈에|흐름도|절차도|로고(?!송)|logo(?!song)|아이콘|배너|banner|신문고|relation_item|tracer|headline|copyright|copy_logo|popup|wa_mk|웹접근성|품질인증',re.I)
|
|
AUDIO=re.compile(r'\.(?:mp3|wav|m4a|ogg|flac)\b',re.I)
|
|
MAPPDF=re.compile(r'pdf|viewer\.html|/map|kakao|daum.*map',re.I)
|
|
|
|
def real_imgs(body):
|
|
out=[]
|
|
for img in body.find_all('img'):
|
|
src=img.get('src') or ''
|
|
if not src or M.KOGL_IMG_PAT.search(src) or DECO.search(src): continue
|
|
if EXCL.search(src+' '+(img.get('alt') or '')): continue
|
|
out.append(img)
|
|
return out
|
|
def media(body):
|
|
txt=len(body.get_text(strip=True))>30
|
|
img=len(real_imgs(body))>0; vid=False
|
|
for ifr in body.find_all('iframe'):
|
|
ss=ifr.get('src') or ''
|
|
if M.YOUTUBE_PAT.search(ss): vid=True
|
|
elif MAPPDF.search(ss): img=True
|
|
if not vid and (body.find('video') or body.find('a',href=M.YOUTUBE_PAT) or M.VIDEO_EXT.search(str(body))): vid=True
|
|
aud=bool(body.find('audio')) or bool(AUDIO.search(str(body)))
|
|
return img,vid,aud,txt
|
|
def njoin(t,i,v,a):
|
|
p=[]
|
|
if t:p.append('어문')
|
|
if i:p.append('이미지')
|
|
if v:p.append('영상')
|
|
if a:p.append('오디오')
|
|
return ','.join(p) if p else '없음'
|
|
|
|
def collect(K):
|
|
soup=None
|
|
for _ in range(3):
|
|
soup=M.fetch(SESS,K)
|
|
if soup is not None: break
|
|
if soup is None: return None
|
|
body=M.get_body(soup,M.BODY_SEL); txt=body.get_text(' ',strip=True)
|
|
mm=TOT.search(txt); Mval=None
|
|
if mm: g=[x for x in mm.groups() if x]; Mval=int(g[0].replace(',',''))
|
|
else:
|
|
el=body.select_one('.total strong,.num,.board_total')
|
|
if el and el.get_text(strip=True).replace(',','').isdigit(): Mval=int(el.get_text(strip=True).replace(',',''))
|
|
img,vid,aud,t=media(body)
|
|
types,qy,qn=KG.kogl(body); P='게시판' if types else ''
|
|
dus=KG.detail_urls(body,K,10)
|
|
empty=bool(re.search(r'게시물이?\s*없|등록된?\s*(?:게시물|자료)\s*가?\s*없',txt))
|
|
if (Mval==0) or (not dus and empty):
|
|
return {'M':0,'N':'없음','O':'미부착','P':'','Q':''}
|
|
for du in dus:
|
|
ds=M.fetch(SESS,du)
|
|
if not ds: continue
|
|
db=M.get_body(ds,M.BODY_SEL)
|
|
i2,v2,a2,t2=media(db); img=img or i2;vid=vid or v2;aud=aud or a2;t=t or t2
|
|
dt,dqy,dqn=KG.kogl(db)
|
|
if dt:
|
|
if not types: P='게시물'
|
|
types|=dt; qy=qy or dqy; qn=qn or dqn
|
|
N=njoin(t,img,vid,aud)
|
|
if types:
|
|
O=','.join('%d유형'%n for n in sorted(types)); Q='Y' if qy else 'N'; P=P or '게시판'
|
|
else: O='미부착'; P='';Q=''
|
|
return {'M':Mval if Mval is not None else 0,'N':N,'O':O,'P':P,'Q':Q}
|
|
|
|
def main():
|
|
st,en=int(sys.argv[1]),int(sys.argv[2]); write='--write' in sys.argv
|
|
wb=openpyxl.load_workbook(XLSX); ws=wb.active
|
|
jobs=[r for r in range(st,en+1) if isinstance(ws.cell(r,11).value,str) and ws.cell(r,11).value.startswith('http')]
|
|
res={}
|
|
with ThreadPoolExecutor(max_workers=6) as ex:
|
|
futs={ex.submit(collect,ws.cell(r,11).value):r for r in jobs}
|
|
for f in as_completed(futs):
|
|
r=futs[f]
|
|
try: res[r]=f.result()
|
|
except: res[r]=None
|
|
for r in sorted(res):
|
|
o=res[r]
|
|
print('r%d %s'%(r,o))
|
|
if write:
|
|
shutil.copy(XLSX, XLSX.replace('.xlsx','_backup_mnopq전.xlsx'))
|
|
for r,o in res.items():
|
|
if not o: continue
|
|
ws.cell(r,13).value=o['M']; ws.cell(r,14).value=o['N']
|
|
ws.cell(r,15).value=o['O']; ws.cell(r,16).value=o['P']; ws.cell(r,17).value=o['Q']
|
|
wb.save(XLSX); print('적용 %d행'%len([1 for o in res.values() if o]))
|
|
|
|
if __name__=='__main__': main()
|