df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
81 lines
4.4 KiB
Python
81 lines
4.4 KiB
Python
# -*- coding: utf-8 -*-
|
|
# 청양 123+ L 종합 재판정: URL패턴(확정) + 라이브크롤(eminwon iframe/총게시물/board링크/외부리다이렉트)
|
|
import openpyxl, json, io, sys, re, time
|
|
from urllib.parse import urlparse
|
|
from playwright.sync_api import sync_playwright
|
|
sys.stdout=io.TextIOWrapper(sys.stdout.buffer,encoding='utf-8')
|
|
F=r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청남도\13.청양군\충청남도_청양군.xlsx'
|
|
wb=openpyxl.load_workbook(F); ws=wb.active
|
|
INT=('cheongyang.go.kr','cy.go.kr')
|
|
def isint(h): return any(x in (h or '') for x in INT)
|
|
rows=[]
|
|
for r in range(123,ws.max_row+1):
|
|
b=ws.cell(r,2).value
|
|
if b is None: continue
|
|
hl=ws.cell(r,11).hyperlink
|
|
u=(hl.target if hl else ws.cell(r,11).value) or ''
|
|
rows.append((r,b,str(ws.cell(r,6).value or ws.cell(r,7).value or ws.cell(r,5).value or '')[:18],str(u),ws.cell(r,12).value))
|
|
print('대상',len(rows),flush=True)
|
|
|
|
JS=r'''()=>{
|
|
const t=document.body?document.body.innerText:'';
|
|
const ifr=[...document.querySelectorAll('iframe')].map(f=>f.src||'');
|
|
const eminwon=ifr.some(s=>/eminwon|emwp|OfrNotAncmt|jsp\/ofr/i.test(s));
|
|
const total=/총\s*게시물|총\s*\d+\s*건|게시물\s*수|Total\s*\d|\bTotal\b/.test(t);
|
|
// 본문영역 게시판 링크(LNB 제외 위해 #contents/.contents/.sub_content 우선)
|
|
const scope=document.querySelector('#contents,.contents,.sub_contents,.sub_container,#container .content,main')||document.body;
|
|
const blinks=[...scope.querySelectorAll('a')].filter(a=>/selectBoardArticle|selectBoardList|nttId=|BBSMSTR|articleNo=|bbsSeq|dataSid|notiNo/i.test(a.href||'')).length;
|
|
const btable=!!scope.querySelector('table.bbs_list,table.board_list,.board_list,.bbsList,.p-table,ul.gallery_list,.board-list,table.basic_list');
|
|
const paging=!!scope.querySelector('.pagination,.paging,.board_paging,.p-pagination');
|
|
return {eminwon,total,blinks,btable,paging,extra_iframe:ifr.filter(s=>s&&!/eminwon|emwp/i.test(s)).slice(0,2)};
|
|
}'''
|
|
def nav(pg,u):
|
|
for _ in range(2):
|
|
try: pg.goto(u,wait_until='domcontentloaded',timeout=20000); pg.wait_for_timeout(1400); return True
|
|
except Exception: time.sleep(1)
|
|
return False
|
|
res={}
|
|
with sync_playwright() as p:
|
|
b=p.chromium.launch(); pg=b.new_page(viewport={'width':1300,'height':2200}); pg.set_default_timeout(20000)
|
|
for r,bn,lab,u,L in rows:
|
|
v='?'; why=''; conf='low'
|
|
if not u.startswith('http'): res[r]=(L,'rel',L,'keep'); continue
|
|
host=urlparse(u).netloc.lower()
|
|
if not isint(host):
|
|
res[r]=('사이트','ext:'+host,L,'high');
|
|
pass
|
|
elif re.search(r'selectBoardList|/cop/bbs/BBSMSTR|selectBbsNttList',u):
|
|
res[r]=('게시판','bbs-url',L,'high')
|
|
else:
|
|
# 크롤필요(/prog/list.do, /kor/.do 등)
|
|
try:
|
|
if nav(pg,u):
|
|
fh=urlparse(pg.url).netloc.lower()
|
|
d=pg.evaluate(JS)
|
|
if not isint(fh):
|
|
v='사이트'; why='redirect:'+fh; conf='high'
|
|
elif d['eminwon']:
|
|
v='게시판'; why='eminwon-iframe'; conf='high'
|
|
elif d['blinks']>=3 or (d['total'] and (d['btable'] or d['blinks']>=1)):
|
|
v='게시판'; why='board(bl%d,tot%d,tbl%d)'%(d['blinks'],d['total'],d['btable']); conf='high'
|
|
elif re.search(r'/prog/.+/list\.do',u) and d['total']:
|
|
v='게시판'; why='prog-total'; conf='med'
|
|
else:
|
|
v='페이지'; why='no-board-sig(bl%d,tot%d,tbl%d,if%s)'%(d['blinks'],d['total'],d['btable'],d['extra_iframe']); conf='med'
|
|
else:
|
|
v=L; why='ERR-keep'; conf='low'
|
|
except Exception as e:
|
|
v=L; why='EXC:'+str(e)[:20]; conf='low'
|
|
res[r]=(v,why,L,conf)
|
|
nv,nw,oL,nc=res[r]
|
|
flag=' <<<DIFF' if nv!=oL else ''
|
|
print('r%d B%s %-18s %s->%s [%s/%s]%s'%(r,bn,lab,oL,nv,nw[:26],nc,flag),flush=True)
|
|
b.close()
|
|
json.dump({str(k):v for k,v in res.items()},open('_cy2_Lfull.json','w',encoding='utf-8'),ensure_ascii=False,indent=0)
|
|
# 요약
|
|
from collections import Counter
|
|
diffs=[(r,res[r]) for r in res if res[r][0]!=res[r][2] and res[r][0] in ('게시판','페이지','사이트')]
|
|
print('\n총 불일치',len(diffs))
|
|
print('전이:',Counter((v[2],v[0]) for v in res.values() if v[0]!=v[2]))
|
|
print('confidence별 불일치:',Counter(res[r][3] for r,_ in diffs))
|