df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
104 lines
4.1 KiB
Python
104 lines
4.1 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""임실 공공누리(O/P/Q) 전수 재스캔. 임실 마크=open0N.jpg(alt 공공누리). 기존 img_opentypeN 정규식이 못잡음.
|
|
전 행(페이지+게시판, 게시판은 상세글 상위10) 재검. 매뉴얼 Phase4.
|
|
사용: python -X utf8 _임실_kogl.py [--write]
|
|
"""
|
|
import sys, os, re, shutil, importlib.util
|
|
from urllib.parse import urljoin
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
import openpyxl
|
|
|
|
HERE=os.path.dirname(os.path.abspath(__file__))
|
|
s=importlib.util.spec_from_file_location('m',os.path.join(HERE,'_jeonbuk_phase234_all.py'))
|
|
M=importlib.util.module_from_spec(s); s.loader.exec_module(M)
|
|
XLSX=r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\전북특별자치도\10.임실군\전북특별자치도_임실군.xlsx'
|
|
SESS=M.make_session()
|
|
STD=re.compile(r'(?:new_)?img_open(?:type|code)(\d{1,2})\.(?:png|jpe?g|gif)',re.I)
|
|
OPEN=re.compile(r'/open0?(\d)\.(?:jpg|jpeg|png|gif)',re.I)
|
|
ALT=re.compile(r'제?(\d)\s*유형')
|
|
LINK=re.compile(r'kogl\.or\.kr/info/licenseType(\d)',re.I)
|
|
DETAIL=re.compile(r'view\.imsil|/view\.|articleNo=|dataSid=|nttId=|not_ancmt|bbsView|seqRepeat=|mode=V',re.I)
|
|
|
|
def kogl(body):
|
|
types=set(); qy=False; qn=False
|
|
for img in body.find_all('img'):
|
|
src=img.get('src') or ''; alt=img.get('alt') or ''
|
|
t=None
|
|
mm=STD.search(src)
|
|
if mm: t=int(mm.group(1))
|
|
elif ('공공누리' in alt) or re.search(r'/open\d',src):
|
|
m2=OPEN.search(src) or ALT.search(alt)
|
|
if m2: t=int(m2.group(1))
|
|
if t and 1<=t<=4:
|
|
types.add(t)
|
|
par=img.find_parent('a')
|
|
if par and par.get('href','') and not par['href'].startswith(('#','javascript')): qy=True
|
|
else: qn=True
|
|
for a in body.find_all('a',href=True):
|
|
mm=LINK.search(a['href'])
|
|
if mm: types.add(int(mm.group(1))); qy=True
|
|
return types, qy, qn
|
|
|
|
def detail_urls(body, base, limit=10):
|
|
urls=[]; seen=set()
|
|
for a in body.find_all('a',href=True):
|
|
h=a['href']
|
|
if h and not h.startswith('#') and DETAIL.search(h):
|
|
full=urljoin(base,h)
|
|
if full not in seen: seen.add(full); urls.append(full)
|
|
if len(urls)>=limit: break
|
|
return urls
|
|
|
|
def scan_row(K, L):
|
|
soup=M.fetch(SESS,K)
|
|
if soup is None: return None
|
|
body=M.get_body(soup,M.BODY_SEL)
|
|
types,qy,qn=kogl(body); P='게시판' if types else ''
|
|
if L=='게시판':
|
|
for du in detail_urls(body,K,10):
|
|
ds=M.fetch(SESS,du)
|
|
if not ds: continue
|
|
db=M.get_body(ds,M.BODY_SEL)
|
|
dt,dqy,dqn=kogl(db)
|
|
if dt:
|
|
if not types: P='게시물'
|
|
types|=dt; qy=qy or dqy; qn=qn or dqn
|
|
if not types: return ('미부착','', '')
|
|
O=','.join('%d유형'%n for n in sorted(types))
|
|
Q='Y' if qy else 'N'
|
|
return (O, P or '게시판', Q)
|
|
|
|
def main():
|
|
write='--write' in sys.argv
|
|
wb=openpyxl.load_workbook(XLSX); ws=wb.active
|
|
jobs=[]
|
|
for r in range(3,ws.max_row+1):
|
|
K=ws.cell(r,11).value; L=ws.cell(r,12).value
|
|
if isinstance(K,str) and K.startswith('http') and L in ('페이지','게시판'):
|
|
jobs.append((r,K,L))
|
|
res={}
|
|
with ThreadPoolExecutor(max_workers=6) as ex:
|
|
futs={ex.submit(scan_row,K,L):r for r,K,L in jobs}
|
|
for f in as_completed(futs):
|
|
r=futs[f]
|
|
try: res[r]=f.result()
|
|
except: res[r]=None
|
|
found=[]
|
|
for r in sorted(res):
|
|
v=res[r]
|
|
if v is None: continue
|
|
O,Pp,Qq=v
|
|
cur=ws.cell(r,15).value
|
|
if O!='미부착' and O!=(cur or '미부착'):
|
|
found.append((r,cur,O,Pp,Qq))
|
|
print('공공누리 발견(기존과 다름) %d행:'%len(found))
|
|
for r,cur,O,Pp,Qq in found:
|
|
print(' r%d %s: %s→%s (P=%s Q=%s)'%(r,ws.cell(r,7).value or ws.cell(r,6).value,cur,O,Pp,Qq))
|
|
if write and found:
|
|
shutil.copy(XLSX, XLSX.replace('.xlsx','_backup_kogl재스캔전.xlsx'))
|
|
for r,cur,O,Pp,Qq in found:
|
|
ws.cell(r,15).value=O; ws.cell(r,16).value=Pp; ws.cell(r,17).value=Qq
|
|
wb.save(XLSX); print('적용 %d행 · 백업'%len(found))
|
|
|
|
if __name__=='__main__': main()
|