DB_JOB/작업파일/완료/광역_사이트맵/충청북도/_cy2_med2.py

# -*- coding: utf-8 -*-
# 청양 med 35건 + 캘리브레이션: 검증된 디텍터(총게시물+article-view+board table+등록일+eminwon)
import openpyxl, json, io, sys, re, time
from urllib.parse import urlparse
from playwright.sync_api import sync_playwright
sys.stdout=io.TextIOWrapper(sys.stdout.buffer,encoding='utf-8')
F=r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청남도\13.청양군\충청남도_청양군.xlsx'
wb=openpyxl.load_workbook(F); ws=wb.active
INT=('cheongyang.go.kr','cy.go.kr')
def isint(h): return any(x in (h or '') for x in INT)
full=json.load(open('_cy2_Lfull.json',encoding='utf-8'))
med=sorted(int(r) for r,v in full.items() if v[3]=='med' and v[0]!=v[2] and v[0] in ('게시판','페이지','사이트'))
def info(r):
    hl=ws.cell(r,11).hyperlink
    u=(hl.target if hl else ws.cell(r,11).value) or ''
    return ws.cell(r,2).value, str(ws.cell(r,6).value or ws.cell(r,7).value or ws.cell(r,5).value or '')[:18], str(u), ws.cell(r,12).value
# 캘리브레이션 (정답 라벨)
CAL=[('CAL새소식','https://www.cheongyang.go.kr/kor/sub04_01_01.do','게시판'),
     ('CAL보도자료','https://www.cheongyang.go.kr/kor/sub04_01_09.do','게시판'),
     ('CAL자매결연','http://sports.cheongyang.go.kr/sports.do','페이지/사이트'),
     ('CAL가맹점등록','https://www.cheongyang.go.kr/kor/sub06_02_13_01.do','페이지')]

JS=r'''()=>{
  const t=document.body?document.body.innerText:'';
  let sig=0, w=[];
  if(/총\s*게시물|총\s*\d+\s*건|게시물\s*\d+\s*건|게시물\s*수|Total\s*:/.test(t)){sig+=2;w.push('총건수');}
  if(document.querySelector('table.bbs_list,table.board_list,.board_list,.bbsList,.p-table,ul.gallery_list,.board-list,table.basic_list,.bbs_wrap,.board_wrap,table.tbl_list')){sig+=2;w.push('btable');}
  if(document.querySelector('th')&&/등록일|작성일|조회수|작성자/.test(t)){sig+=1;w.push('등록일hdr');}
  const av=[...document.querySelectorAll('a')].filter(a=>/selectBoardArticle|nttId=|articleNo=|artclView|boardArticle|\/view\.do.*[?&](no|idx|seq|articleSeq|nttSn)=/i.test(a.href||'')).length;
  if(av>=3){sig+=2;w.push('view'+av);} else if(av>=1){w.push('v'+av);}
  if(document.querySelector('.pagination,.paging,.board_paging,.p-pagination,.page_num,.pgn')){sig+=1;w.push('paging');}
  const ifr=[...document.querySelectorAll('iframe')].map(f=>f.src||'');
  if(ifr.some(s=>/eminwon|emwp|OfrNotAncmt|jsp\/ofr/i.test(s))){sig+=3;w.push('eminwon');}
  const extifr=ifr.filter(s=>{try{return !/cheongyang\.go\.kr|cy\.go\.kr/.test(new URL(s,location.href).hostname);}catch(e){return false;}});
  return {sig,w:w.join(','),extifr:extifr.slice(0,2)};
}'''
def nav(pg,u):
    for _ in range(2):
        try: pg.goto(u,wait_until='networkidle',timeout=30000); pg.wait_for_timeout(2000); return True
        except Exception:
            try: pg.goto(u,wait_until='domcontentloaded',timeout=20000); pg.wait_for_timeout(2500); return True
            except Exception: time.sleep(1)
    return False
def judge(pg,u,oL):
    if not u.startswith('http'): return oL,'rel'
    if not nav(pg,u): return oL,'ERR'
    fh=urlparse(pg.url).netloc.lower()
    d=pg.evaluate(JS)
    if not isint(fh): return '사이트','redirect:'+fh
    if d['sig']>=3: return '게시판','sig%d[%s]'%(d['sig'],d['w'])
    if d['extifr']: return '사이트','extiframe:%s'%d['extifr']
    return '페이지','sig%d[%s]'%(d['sig'],d['w'])
res={}
with sync_playwright() as p:
    b=p.chromium.launch(); pg=b.new_page(viewport={'width':1300,'height':2400}); pg.set_default_timeout(30000)
    print('=== 캘리브레이션 ===',flush=True)
    for name,u,gold in CAL:
        v,w=judge(pg,u,'?'); print('%s 정답=%s 판정=%s [%s]'%(name,gold,v,w[:40]),flush=True)
    print('\n=== med 35 ===',flush=True)
    for r in med:
        bn,lab,u,oL=info(r)
        v,w=judge(pg,u,oL)
        res[r]=(oL,v,w)
        print('r%d B%s %-18s %s=>%s [%s]%s'%(r,bn,lab,oL,v,w[:40],'  CONFIRM' if v!=oL else '  유지'),flush=True)
    b.close()
json.dump({str(k):v for k,v in res.items()},open('_cy2_med2.json','w',encoding='utf-8'),ensure_ascii=False,indent=0)
from collections import Counter
print('\n전이:',Counter((o,v) for o,v,_ in res.values() if o!=v),'| 유지',sum(1 for o,v,_ in res.values() if o==v))