df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
78 lines
3.6 KiB
Python
78 lines
3.6 KiB
Python
# -*- coding: utf-8 -*-
|
|
# 청양 123+ L 라이브검토: 각 행 K(url) 크롤해 게시판/페이지/사이트 실제판별 → 현재L과 비교
|
|
import openpyxl, json, io, sys, re, time
|
|
from urllib.parse import urlparse
|
|
from playwright.sync_api import sync_playwright
|
|
sys.stdout=io.TextIOWrapper(sys.stdout.buffer,encoding='utf-8')
|
|
F=r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청남도\13.청양군\충청남도_청양군.xlsx'
|
|
wb=openpyxl.load_workbook(F); ws=wb.active
|
|
INTERNAL=('cheongyang.go.kr','cy.go.kr')
|
|
def is_internal(host):
|
|
return any(h in (host or '') for h in INTERNAL)
|
|
rows=[]
|
|
for r in range(123,ws.max_row+1):
|
|
b=ws.cell(r,2).value
|
|
if b is None: continue
|
|
url=ws.cell(r,11).value; L=ws.cell(r,12).value
|
|
lab=ws.cell(r,6).value or ws.cell(r,7).value or ws.cell(r,5).value
|
|
# 하이퍼링크 우선
|
|
hl=ws.cell(r,11).hyperlink
|
|
u=hl.target if hl else url
|
|
rows.append((r,b,str(lab or '')[:20],str(u or ''),L))
|
|
print('대상',len(rows),flush=True)
|
|
|
|
# 게시판 시그널 JS
|
|
JS=r'''()=>{
|
|
const t=document.body?document.body.innerText:'';
|
|
const has=(re)=>re.test(t);
|
|
let sig=0, why=[];
|
|
// 총 게시물 / 게시물 수
|
|
if(/총\s*게시물|총\s*\d+\s*건|게시물\s*\d+|Total\s*:/.test(t)){sig+=2;why.push('총건수');}
|
|
// 게시판 테이블/리스트
|
|
const bt=document.querySelector('table.bbs_list,table.board_list,.bbs_list,.board_list,.bbsList,table.p-table,.board-list,.boardList,ul.board_list');
|
|
if(bt){sig+=2;why.push('board_list');}
|
|
// 등록일/작성일 헤더
|
|
if(document.querySelector('th') && /등록일|작성일|조회수|작성자/.test(t)){sig+=1;why.push('등록일헤더');}
|
|
// view 링크 다수
|
|
const vl=[...document.querySelectorAll('a')].filter(a=>/selectBoardArticle|nttId=|articleNo=|boardArticle|view\.do.*[?&](no|idx|seq)=/i.test(a.href||'')).length;
|
|
if(vl>=3){sig+=2;why.push('view'+vl);}
|
|
// 페이징
|
|
if(document.querySelector('.pagination,.paging,.page_wrap,.board_paging,nav.pagination')){sig+=1;why.push('paging');}
|
|
return {sig,why:why.join(','),hasIframe:!!document.querySelector('iframe')};
|
|
}'''
|
|
def nav(pg,u):
|
|
for _ in range(2):
|
|
try: pg.goto(u,wait_until='domcontentloaded',timeout=20000); pg.wait_for_timeout(1200); return True
|
|
except Exception: time.sleep(1)
|
|
return False
|
|
res={}
|
|
with sync_playwright() as p:
|
|
b=p.chromium.launch(); pg=b.new_page(viewport={'width':1280,'height':2200}); pg.set_default_timeout(20000)
|
|
for r,bn,lab,u,L in rows:
|
|
verdict='?'; why=''
|
|
if not u.startswith('http'):
|
|
res[r]=('?','no-url',L); continue
|
|
host=urlparse(u).netloc.lower()
|
|
try:
|
|
if nav(pg,u):
|
|
d=pg.evaluate(JS)
|
|
why=d['why']
|
|
final=urlparse(pg.url).netloc.lower()
|
|
if not is_internal(final) and final:
|
|
verdict='사이트'; why='ext:'+final
|
|
elif d['sig']>=3:
|
|
verdict='게시판'
|
|
else:
|
|
verdict='페이지'
|
|
else:
|
|
verdict='ERR'
|
|
except Exception as e:
|
|
verdict='ERR'; why=str(e)[:30]
|
|
res[r]=(verdict,why,L)
|
|
flag='' if verdict==L else ' <<<DIFF'
|
|
print('r%d B%s %-18s 현재=%s 판정=%s [%s]%s'%(r,bn,lab,L,verdict,why[:24],flag),flush=True)
|
|
b.close()
|
|
json.dump({str(k):v for k,v in res.items()},open('_cy2_Lverdict.json','w',encoding='utf-8'),ensure_ascii=False,indent=0)
|
|
diff=[(r,res[r]) for r in res if res[r][0] in ('게시판','페이지','사이트') and res[r][0]!=res[r][2]]
|
|
print('\n=== 불일치 %d건 ==='%len(diff))
|