df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
99 lines
5.9 KiB
Python
99 lines
5.9 KiB
Python
# -*- coding: utf-8 -*-
|
||
# 청양 175~끝 L/M/O(P/Q)/N재수집 크롤. 결과 _cy4.json
|
||
import openpyxl, json, io, sys, re, time
|
||
from urllib.parse import urlparse, urljoin
|
||
from playwright.sync_api import sync_playwright
|
||
sys.stdout=io.TextIOWrapper(sys.stdout.buffer,encoding='utf-8')
|
||
F=r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청남도\13.청양군\충청남도_청양군.xlsx'
|
||
ws=openpyxl.load_workbook(F).active
|
||
INT=('cheongyang.go.kr','cy.go.kr')
|
||
def isint(h): return any(x in (h or '') for x in INT)
|
||
rows=[]
|
||
for r in range(175,ws.max_row+1):
|
||
hl=ws.cell(r,11).hyperlink; u=(hl.target if hl else ws.cell(r,11).value) or ''
|
||
lab=str(ws.cell(r,8).value or ws.cell(r,7).value or ws.cell(r,6).value or ws.cell(r,5).value or '')
|
||
rows.append((r,str(u),lab))
|
||
print('대상',len(rows),flush=True)
|
||
|
||
JS_PAGE=r'''()=>{
|
||
const t=document.body?document.body.innerText:'';
|
||
const ifr=[...document.querySelectorAll('iframe')].map(f=>f.src||'');
|
||
const eminwon=ifr.some(s=>/eminwon|emwp|OfrNotAncmt|jsp\/ofr/i.test(s));
|
||
const totM=(t.match(/총\s*게시물\s*[:：]?\s*([\d,]+)/)||t.match(/총\s*([\d,]+)\s*건/)||t.match(/게시물\s*수\s*[:：]?\s*([\d,]+)/));
|
||
const av=[...document.querySelectorAll('a')].filter(a=>/selectBoardArticle|nttId=|articleNo=|artclView|boardArticle|bbsSeq|dataSid/i.test(a.href||''));
|
||
// 공공누리 마크(본문)
|
||
function koglScan(root){
|
||
const imgs=[...root.querySelectorAll('img')].filter(im=>/opentype0?(\d)/i.test(im.getAttribute('src')||''));
|
||
const types=new Set(); let link=false;
|
||
for(const im of imgs){
|
||
const m=/opentype0?(\d)/i.exec(im.getAttribute('src')||''); if(m) types.add(parseInt(m[1]));
|
||
for(let n=im;n;n=n.parentElement){ if(n.tagName==='A'&&n.getAttribute('href')&&n.getAttribute('href')!=='#'){link=true;break;} }
|
||
let box=im.closest('div,p,td,li')||im.parentElement;
|
||
for(let up=0;box&&up<3;up++){ if([...box.querySelectorAll('a')].some(a=>a.getAttribute('href')&&a.getAttribute('href')!=='#'&&/kogl/i.test(a.href||''))){link=true;break;} box=box.parentElement; }
|
||
}
|
||
// div.open_type 텍스트형
|
||
if(root.querySelector('.open_type,.opentype,.kogl')){ if(!types.size) types.add(4); }
|
||
return {types:[...types].sort(),link};
|
||
}
|
||
const main=document.querySelector('#contents,.contents,.sub_contents,.sub_container,#container,main')||document.body;
|
||
const kogl=koglScan(main);
|
||
// 본문 콘텐츠 이미지(N후보)
|
||
const DEC=/\/common\/|move\.png|no[-_]?img|\/icon|ico_|bul_|btn_|bg_|banner|popup|logo|opentype0?\d|sns_|btn\.|arrow|prev|next|\.svg/i;
|
||
const imgs=[...main.querySelectorAll('img')].filter(im=>{const s=im.getAttribute('src')||''; if(DEC.test(s))return false; const w=im.naturalWidth||im.width||0,h=im.naturalHeight||im.height||0; return (w>=250||h>=250);}).map(im=>im.src).slice(0,8);
|
||
const video=!!document.querySelector('iframe[src*="youtube"],iframe[src*="youtu.be"],video,iframe[src*="player"]');
|
||
const audio=!!document.querySelector('audio,a[href*=".mp3"],a[href*=".wav"],a[href*=".m4a"]');
|
||
return {total:totM?totM[1].replace(/,/g,''):null, av:av.map(a=>a.href).slice(0,3), avn:av.length, eminwon, kogl, imgs, video, audio};
|
||
}'''
|
||
def nav(pg,u,wait='domcontentloaded'):
|
||
for _ in range(2):
|
||
try: pg.goto(u,wait_until=wait,timeout=22000); pg.wait_for_timeout(1300); return True
|
||
except Exception: time.sleep(1)
|
||
return False
|
||
res={}
|
||
with sync_playwright() as p:
|
||
b=p.chromium.launch(); pg=b.new_page(viewport={'width':1300,'height':2400}); pg.set_default_timeout(22000)
|
||
for r,u,lab in rows:
|
||
rec={'lab':lab,'u':u}
|
||
if not u.startswith('http'):
|
||
rec.update(L='페이지',rel=True); res[r]=rec; continue
|
||
host=urlparse(u).netloc.lower()
|
||
if not isint(host):
|
||
rec.update(L='사이트',ext=host); res[r]=rec
|
||
print('r%d %-16s 사이트(ext)'%(r,lab[:16]),flush=True); continue
|
||
try:
|
||
if not nav(pg,u): rec.update(L='ERR'); res[r]=rec; print('r%d ERR'%r,flush=True); continue
|
||
fh=urlparse(pg.url).netloc.lower()
|
||
if not isint(fh): rec.update(L='사이트',ext=fh); res[r]=rec; print('r%d %s 사이트(redir)'%(r,lab[:14]),flush=True); continue
|
||
d=pg.evaluate(JS_PAGE)
|
||
urlboard=bool(re.search(r'selectBoardList|/cop/bbs/BBSMSTR|selectBbsNttList',u))
|
||
isboard = urlboard or d['eminwon'] or d['avn']>=3 or (d['total'] is not None and d['avn']>=1)
|
||
kogl=d['kogl']; imgs=d['imgs']; video=d['video']; audio=d['audio']
|
||
if isboard:
|
||
rec['L']='게시판'
|
||
# M
|
||
M=None
|
||
if d['total'] is not None: M=int(d['total'])
|
||
# 상세글에서 공공누리/이미지 재확인
|
||
detail_kogl=kogl; detail_imgs=list(imgs)
|
||
if d['av']:
|
||
try:
|
||
if nav(pg,d['av'][0]):
|
||
dd=pg.evaluate(JS_PAGE)
|
||
if dd['kogl']['types']: detail_kogl=dd['kogl']
|
||
detail_imgs=dd['imgs'] or detail_imgs
|
||
except Exception: pass
|
||
rec.update(M=M, kogl=detail_kogl, imgs=detail_imgs, video=video, audio=audio, avn=d['avn'])
|
||
else:
|
||
rec.update(L='페이지', M=1, kogl=kogl, imgs=imgs, video=video, audio=audio)
|
||
res[r]=rec
|
||
kt=rec.get('kogl',{}).get('types',[])
|
||
print('r%d %-16s %s M=%s kogl=%s img=%d v=%s'%(r,lab[:16],rec['L'],rec.get('M'),kt,len(rec.get('imgs',[])),video),flush=True)
|
||
except Exception as e:
|
||
rec.update(L='ERR',err=str(e)[:30]); res[r]=rec; print('r%d EXC %s'%(r,str(e)[:30]),flush=True)
|
||
b.close()
|
||
json.dump({str(k):v for k,v in res.items()},open('_cy4.json','w',encoding='utf-8'),ensure_ascii=False)
|
||
from collections import Counter
|
||
print('\nL분포',Counter(v.get('L') for v in res.values()))
|
||
print('공공누리검출',sum(1 for v in res.values() if v.get('kogl',{}).get('types')))
|
||
print('이미지후보행',sum(1 for v in res.values() if v.get('imgs')))
|