# -*- coding: utf-8 -*- # 청양 검토필요 35건(med) 행별 정밀확정: 본문영역 보드신호(nav제외), 외부리다이렉트, eminwon import openpyxl, json, io, sys, re, time from urllib.parse import urlparse from playwright.sync_api import sync_playwright sys.stdout=io.TextIOWrapper(sys.stdout.buffer,encoding='utf-8') F=r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청남도\13.청양군\충청남도_청양군.xlsx' wb=openpyxl.load_workbook(F); ws=wb.active INT=('cheongyang.go.kr','cy.go.kr') def isint(h): return any(x in (h or '') for x in INT) full=json.load(open('_cy2_Lfull.json',encoding='utf-8')) med=[] for rs,v in full.items(): r=int(rs); nv,why,oL,conf=v if conf=='med' and nv!=oL and nv in ('게시판','페이지','사이트'): med.append(r) med.sort() def rowinfo(r): hl=ws.cell(r,11).hyperlink u=(hl.target if hl else ws.cell(r,11).value) or '' return ws.cell(r,2).value, str(ws.cell(r,6).value or ws.cell(r,7).value or ws.cell(r,5).value or '')[:18], str(u), ws.cell(r,12).value print('med 대상',len(med),flush=True) JS=r'''()=>{ // 본문영역 한정(nav/lnb/header/footer 제외) let main=document.querySelector('#contents,.contents,.sub_contents,.sub_container,.contents_area,#content,.content_area,main'); if(!main) main=document.body; // nav/lnb 요소 제거 사본 const clone=main.cloneNode(true); clone.querySelectorAll('nav,.lnb,.snb,.gnb,.depth2_ul,.depth3_ul,.lnb_depth2__list,.lnb_depth3__list,header,footer,.location,.breadcrumb').forEach(e=>e.remove()); const t=clone.innerText||''; const total = /총\s*게시물|총\s*\d+\s*건|게시물\s*수\s*:?\s*\d|Total\s*:?\s*\d/.test(t); // 게시판 리스트: 링크+날짜가 같은 행/아이템에 있는 개수 let datelink=0; clone.querySelectorAll('tr,li,.item,.board_item').forEach(el=>{ const hasA=el.querySelector('a'); const hasDate=/20\d\d[-.\/]\s?\d\d?[-.\/]\s?\d\d?/.test(el.innerText||''); if(hasA&&hasDate) datelink++; }); const btable=!!clone.querySelector('table.bbs_list,table.board_list,.board_list,.bbsList,.p-table,ul.gallery_list,.board-list,table.basic_list,.bbs_wrap,.board_wrap'); const paging=!!clone.querySelector('.pagination,.paging,.board_paging,.p-pagination,.page_num'); const ifr=[...document.querySelectorAll('iframe')].map(f=>f.src||''); const eminwon=ifr.some(s=>/eminwon|emwp|OfrNotAncmt|jsp\/ofr/i.test(s)); const extifr=ifr.filter(s=>s&&!isInt(s)); function isInt(s){try{const h=new URL(s,location.href).hostname;return /cheongyang\.go\.kr|cy\.go\.kr/.test(h);}catch(e){return true;}} return {total,datelink,btable,paging,eminwon,extifr:extifr.slice(0,2),tlen:t.length}; }''' def nav(pg,u): for _ in range(2): try: pg.goto(u,wait_until='networkidle',timeout=30000); pg.wait_for_timeout(2200); return True except Exception: try: pg.goto(u,wait_until='domcontentloaded',timeout=20000); pg.wait_for_timeout(2500); return True except Exception: time.sleep(1) return False res={} with sync_playwright() as p: b=p.chromium.launch(); pg=b.new_page(viewport={'width':1300,'height':2400}); pg.set_default_timeout(30000) for r in med: bn,lab,u,oL=rowinfo(r) verdict=oL; why='' try: if u.startswith('http') and nav(pg,u): fh=urlparse(pg.url).netloc.lower() d=pg.evaluate(JS) board = d['eminwon'] or d['total'] or d['datelink']>=3 or (d['btable'] and d['paging']) if not isint(fh): verdict='사이트'; why='redirect:'+fh elif board: verdict='게시판'; why='board(emw%s,tot%s,dl%d,tbl%s,pg%s)'%(d['eminwon'],d['total'],d['datelink'],d['btable'],d['paging']) else: verdict='페이지'; why='page(dl%d,extifr%s,tlen%d)'%(d['datelink'],d['extifr'],d['tlen']) else: why='ERR' except Exception as e: why='EXC:'+str(e)[:30] res[r]=(oL,verdict,why) flag=' ==CONFIRM-DIFF' if verdict!=oL else ' (유지)' print('r%d B%s %-18s 현재=%s 확정=%s [%s]%s'%(r,bn,lab,oL,verdict,why[:40],flag),flush=True) b.close() json.dump({str(k):v for k,v in res.items()},open('_cy2_med.json','w',encoding='utf-8'),ensure_ascii=False,indent=0) from collections import Counter print('\n확정 전이:',Counter((o,v) for o,v,_ in res.values() if o!=v)) print('유지(현재값그대로):',sum(1 for o,v,_ in res.values() if o==v))