df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
110 lines
5.7 KiB
Python
110 lines
5.7 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""정읍 L(게시판/페이지)+N 재판정. N=페이징화살표(prev/next)·장식 제외. L=게시판신호(리스트>=3/페이징/총건수) 없으면 페이지.
|
|
사용: python -X utf8 _정읍_LN.py [--write]
|
|
"""
|
|
import sys, os, re, shutil, importlib.util
|
|
from urllib.parse import urljoin
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
import openpyxl
|
|
HERE=os.path.dirname(os.path.abspath(__file__))
|
|
s=importlib.util.spec_from_file_location('m',os.path.join(HERE,'_jeonbuk_phase234_all.py'))
|
|
M=importlib.util.module_from_spec(s); s.loader.exec_module(M)
|
|
XLSX=r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\전북특별자치도\13.정읍시\전북특별자치도_정읍시.xlsx'
|
|
SESS=M.make_session()
|
|
DECO=re.compile(r'/common/|move\.png|no[-_]?img|blank|spacer|/ico|/btn|bullet|arrow|/bg|icon|see_btn|/sample|mimetype|/file_|filedown|btn_dir|/open\d|viewImg|/board/view|_link\d|governing_img|img_open(?:code|type)0[._]|prev\d|next\d|/prev|/next|paging|btn_preview|ico_file',re.I)
|
|
EXCL=re.compile(r'한눈에|흐름도|절차도|로고(?!송)|logo(?!song)|아이콘|배너|banner|신문고|relation_item|tracer|headline|copyright|popup|wa_mk|웹접근성|품질인증|전용뷰어|첫\s*페이지|이전\s*\d|다음\s*\d|마지막\s*페이지|페이지로|미리보기',re.I)
|
|
AUDIO=re.compile(r'\.(?:mp3|wav|m4a|ogg|flac)\b',re.I)
|
|
MAPPDF=re.compile(r'pdf|viewer\.html|/map|kakao|daum.*map',re.I)
|
|
TOT=re.compile(r'전체\s*([\d,]+)|총\s*([\d,]+)\s*건|총\s*게시물\s*([\d,]+)')
|
|
DETAIL=re.compile(r'view\.|/view\b|articleNo=|dataSid=|nttId=|idx=|seq=|bbsView|not_ancmt|mode=V',re.I)
|
|
EMPTY=re.compile(r'게시물이?\s*없|등록된?\s*(?:게시물|자료|글)\s*가?\s*없')
|
|
def realimg(b):
|
|
for img in b.find_all('img'):
|
|
src=img.get('src') or ''
|
|
if not src or M.KOGL_IMG_PAT.search(src) or DECO.search(src): continue
|
|
if EXCL.search(src+' '+(img.get('alt') or '')): continue
|
|
return True
|
|
return False
|
|
def media(b):
|
|
t=len(b.get_text(strip=True))>30; img=realimg(b); vid=False
|
|
for ifr in b.find_all('iframe'):
|
|
ss=ifr.get('src') or ''
|
|
if M.YOUTUBE_PAT.search(ss): vid=True
|
|
elif MAPPDF.search(ss): img=True
|
|
if not vid and (b.find('video') or b.find('a',href=M.YOUTUBE_PAT) or M.VIDEO_EXT.search(str(b))): vid=True
|
|
aud=bool(b.find('audio')) or bool(AUDIO.search(str(b)))
|
|
return img,vid,aud,t
|
|
def njoin(t,i,v,a):
|
|
p=[]
|
|
if t:p.append('어문')
|
|
if i:p.append('이미지')
|
|
if v:p.append('영상')
|
|
if a:p.append('오디오')
|
|
return ','.join(p) if p else '없음'
|
|
def listrows(b):
|
|
return [li for li in b.select('table tbody tr, .bbs_list li, .board_list li, ul.list li, .board_list tr') if li.find('a',href=True) and DETAIL.search(str(li))]
|
|
def paging(b):
|
|
pg=[a for a in b.select('.paging a,.pagination a,.page a') if a.get_text(strip=True).isdigit()]
|
|
return len(pg)>=2
|
|
def detail_urls(b,base,lim=10):
|
|
out=[];seen=set()
|
|
for a in b.find_all('a',href=True):
|
|
h=a['href']
|
|
if h and not h.startswith('#') and DETAIL.search(h):
|
|
f=urljoin(base,h)
|
|
if f not in seen: seen.add(f);out.append(f)
|
|
if len(out)>=lim:break
|
|
return out
|
|
def judge(K,curL,curM):
|
|
soup=None
|
|
for _ in range(3):
|
|
soup=M.fetch(SESS,K)
|
|
if soup is not None: break
|
|
if soup is None: return None
|
|
b=M.get_body(soup,M.BODY_SEL); txt=b.get_text(' ',strip=True)
|
|
lr=listrows(b); pg=paging(b); mm=TOT.search(txt)
|
|
tot=int([x for x in mm.groups() if x][0].replace(',','')) if mm else None
|
|
is_board = (tot is not None) or (len(lr)>=3) or pg or (isinstance(curM,int) and curM>1 and curL=='게시판')
|
|
img,vid,aud,t=media(b)
|
|
if is_board:
|
|
L='게시판'
|
|
for du in detail_urls(b,K,10):
|
|
ds=M.fetch(SESS,du)
|
|
if not ds: continue
|
|
db=M.get_body(ds,M.BODY_SEL); i2,v2,a2,t2=media(db); img=img or i2;vid=vid or v2;aud=aud or a2;t=t or t2
|
|
if not detail_urls(b,K,1) and (EMPTY.search(txt) or not t): return {'L':'게시판','M':0,'N':'없음'}
|
|
Mv = tot if tot is not None else (curM if (isinstance(curM,int) and curM>1) else len(lr))
|
|
return {'L':'게시판','M':Mv,'N':njoin(t,img,vid,aud)}
|
|
return {'L':'페이지','M':1,'N':njoin(t,img,vid,aud)}
|
|
def main():
|
|
write='--write' in sys.argv
|
|
wb=openpyxl.load_workbook(XLSX); ws=wb.active
|
|
jobs=[(r,ws.cell(r,11).value,ws.cell(r,12).value,ws.cell(r,13).value) for r in range(3,ws.max_row+1)
|
|
if isinstance(ws.cell(r,11).value,str) and ws.cell(r,11).value.startswith('http') and ws.cell(r,12).value in ('페이지','게시판')]
|
|
res={}
|
|
with ThreadPoolExecutor(max_workers=6) as ex:
|
|
futs={ex.submit(judge,K,L,Mv):(r,L,Mv,ws.cell(r,14).value) for r,K,L,Mv in jobs}
|
|
for f in as_completed(futs):
|
|
r,L,Mv,Nv=futs[f]
|
|
try: res[r]=(f.result(),L,Mv,Nv)
|
|
except: res[r]=(None,L,Mv,Nv)
|
|
Lc=[];Nc=[]
|
|
for r in sorted(res):
|
|
o,L,Mv,Nv=res[r]
|
|
if not o: continue
|
|
if o['L']!=L: Lc.append((r,L,o['L'],Mv,o['M']))
|
|
if o['N']!=(Nv or ''): Nc.append((r,Nv,o['N']))
|
|
print('L변경 %d · N변경 %d'%(len(Lc),len(Nc)))
|
|
print('[L]')
|
|
for r,a,b2,m1,m2 in Lc: print(' r%d %s %s→%s M%s→%s'%(r,ws.cell(r,7).value or ws.cell(r,6).value,a,b2,m1,m2))
|
|
print('[N 표본]')
|
|
for r,a,b2 in Nc[:25]: print(' r%d %s: %s→%s'%(r,ws.cell(r,7).value or ws.cell(r,6).value,a,b2))
|
|
if write:
|
|
shutil.copy(XLSX,XLSX.replace('.xlsx','_backup_LN재판정전.xlsx'))
|
|
for r in res:
|
|
o=res[r][0]
|
|
if not o: continue
|
|
ws.cell(r,12).value=o['L']; ws.cell(r,13).value=o['M']; ws.cell(r,14).value=o['N']
|
|
wb.save(XLSX); print('적용')
|
|
if __name__=='__main__': main()
|