df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
68 lines
4.4 KiB
Python
68 lines
4.4 KiB
Python
# -*- coding: utf-8 -*-
|
|
# 충북 범용 N 몽타주 크롤러: python _cb_Ncrawl.py <엑셀경로> <출력json>
|
|
import openpyxl, json, sys, time, io
|
|
from playwright.sync_api import sync_playwright
|
|
sys.stdout=io.TextIOWrapper(sys.stdout.buffer,encoding='utf-8')
|
|
F=sys.argv[1]; OUT=sys.argv[2]
|
|
wb=openpyxl.load_workbook(F); ws=wb.active
|
|
DEC=r"opentype|open0|kogl|/logo|logo\.|btn_|/icon|ico_|icon_|banner|visual|symbol|sns|/bg_|bg\.|blank|spacer|bullet|arrow|move\.png|no[-_]?img|noimg|popup|/common/|/template|/skin|emblem|/sns|share|facebook|insta|blog|twitter|youtube_|/qr|qr\."
|
|
BAD=r"header|footer|gnb|lnb|snb|^nav|nav$|visual|main_|top_area|quick|banner|aside|skip|popup|familySite"
|
|
JS_DET=r'''()=>{const a=[...document.querySelectorAll('a')];
|
|
return a.filter(x=>/selectBbsNttView|addBbsNttView|BbsNttView|nttNo=|nttId=|articleNo=|selectBoardArticle|view\.9is|view\.jinan|view\.jeongeup|board\/view|dataUid=|contentUid=|boardUid=|idx=/i.test((x.href||'')+(x.getAttribute&&x.getAttribute('onclick')||''))).map(x=>x.href).filter(h=>h&&h.indexOf('#')!=0).slice(0,8);}'''
|
|
JS_IMG=r'''()=>{
|
|
const RB=new RegExp("%s","i"), RD=new RegExp("%s","i");
|
|
const bad=el=>{for(let n=el;n;n=n.parentElement){let c='';if(n.className){c=(typeof n.className==='string')?n.className:(n.className.baseVal||'');}c+=' '+(n.id||'');if(RB.test(c))return true;}return false;};
|
|
const dec=s=>RD.test(s||'');
|
|
const imgs=[...document.querySelectorAll('img')];
|
|
const real=imgs.filter(im=>{const w=Math.max(im.naturalWidth,im.clientWidth),h=Math.max(im.naturalHeight,im.clientHeight);return (w>=240||h>=240)&&!dec(im.getAttribute('src'))&&!bad(im);});
|
|
const maps=[...document.querySelectorAll('iframe,embed,object')].filter(e=>/map|daum|kakao|google|vworld|roadview/i.test(e.src||e.data||'')).length;
|
|
const pdf=[...document.querySelectorAll('iframe,embed,object')].filter(e=>/\.pdf/i.test(e.src||e.data||'')).length;
|
|
return {r:real.length, m:maps, p:pdf, srcs:real.slice(0,6).map(im=>im.src)};
|
|
}''' % (BAD.replace('\\','\\\\'),DEC.replace('\\','\\\\'))
|
|
rows=[]
|
|
for r in range(3,ws.max_row+1):
|
|
L=ws.cell(r,12).value; url=ws.cell(r,11).value; M=ws.cell(r,13).value; old=ws.cell(r,14).value
|
|
if L=='사이트' or not str(url or '').startswith('http'): continue
|
|
rows.append((r,L,M,url,old))
|
|
print('crawl 대상',len(rows),flush=True)
|
|
def nav(pg,u):
|
|
for _ in range(2):
|
|
try:
|
|
pg.goto(u,wait_until='domcontentloaded'); pg.wait_for_timeout(1100)
|
|
pg.evaluate("window.scrollTo(0,document.body.scrollHeight)"); pg.wait_for_timeout(500)
|
|
return True
|
|
except Exception: time.sleep(1)
|
|
return False
|
|
res={}
|
|
with sync_playwright() as p:
|
|
b=p.chromium.launch(); pg=b.new_page(viewport={'width':1280,'height':2400}); pg.set_default_timeout(18000)
|
|
for i,(r,L,M,url,old) in enumerate(rows):
|
|
det=old; why=''; srcs=[]
|
|
try:
|
|
if L=='게시판':
|
|
if M in (0,None,''): det='없음'; why='M0'
|
|
elif nav(pg,url):
|
|
views=pg.evaluate(JS_DET); hit=False; w=''
|
|
for v in views[:5]:
|
|
if v and v.startswith('http') and nav(pg,v):
|
|
d=pg.evaluate(JS_IMG)
|
|
if d['r']>0: hit=True; w='img%d'%d['r']; srcs=d['srcs']; break
|
|
det='어문,이미지' if hit else '어문'; why=('board '+w) if hit else 'board txt'
|
|
else: det=old; why='nav_fail'
|
|
else:
|
|
if nav(pg,url):
|
|
d=pg.evaluate(JS_IMG)
|
|
if d['r']>0 or d['m']>0 or d['p']>0: det='어문,이미지'; why='page img%d map%d pdf%d'%(d['r'],d['m'],d['p']); srcs=d['srcs']
|
|
else: det='어문'; why='page txt'
|
|
else: det=old; why='nav_fail'
|
|
except Exception as e: det=old; why='ERR:'+str(e)[:30]
|
|
res[str(r)]={'L':L,'old':old,'det':det,'why':why,'srcs':srcs,
|
|
'label':ws.cell(r,8).value or ws.cell(r,7).value or ws.cell(r,6).value or ws.cell(r,5).value or ws.cell(r,4).value}
|
|
if i%20==0:
|
|
json.dump(res,open(OUT,'w',encoding='utf-8'),ensure_ascii=False)
|
|
print('%d/%d r%d %s %s->%s (%s)'%(i+1,len(rows),r,L,old,det,why),flush=True)
|
|
b.close()
|
|
json.dump(res,open(OUT,'w',encoding='utf-8'),ensure_ascii=False)
|
|
cand=sum(1 for k in res if res[k]['det']=='어문,이미지')
|
|
print('완료 %d | 이미지후보 %d | ERR %d'%(len(res),cand,sum(1 for k in res if 'ERR' in res[k]['why'] or 'nav_fail' in res[k]['why'])))
|