# -*- coding: utf-8 -*-
# 청양 175~끝 L/M/O(P/Q)/N재수집 크롤. 결과 _cy4.json
import openpyxl, json, io, sys, re, time
from urllib.parse import urlparse, urljoin
from playwright.sync_api import sync_playwright
sys.stdout=io.TextIOWrapper(sys.stdout.buffer,encoding='utf-8')
F=r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청남도\13.청양군\충청남도_청양군.xlsx'
ws=openpyxl.load_workbook(F).active
INT=('cheongyang.go.kr','cy.go.kr')
def isint(h): return any(x in (h or '') for x in INT)
rows=[]
for r in range(175,ws.max_row+1):
    hl=ws.cell(r,11).hyperlink; u=(hl.target if hl else ws.cell(r,11).value) or ''
    lab=str(ws.cell(r,8).value or ws.cell(r,7).value or ws.cell(r,6).value or ws.cell(r,5).value or '')
    rows.append((r,str(u),lab))
print('대상',len(rows),flush=True)

JS_PAGE=r'''()=>{
 const t=document.body?document.body.innerText:'';
 const ifr=[...document.querySelectorAll('iframe')].map(f=>f.src||'');
 const eminwon=ifr.some(s=>/eminwon|emwp|OfrNotAncmt|jsp\/ofr/i.test(s));
 const totM=(t.match(/총\s*게시물\s*[:：]?\s*([\d,]+)/)||t.match(/총\s*([\d,]+)\s*건/)||t.match(/게시물\s*수\s*[:：]?\s*([\d,]+)/));
 const av=[...document.querySelectorAll('a')].filter(a=>/selectBoardArticle|nttId=|articleNo=|artclView|boardArticle|bbsSeq|dataSid/i.test(a.href||''));
 // 공공누리 마크(본문)
 function koglScan(root){
   const imgs=[...root.querySelectorAll('img')].filter(im=>/opentype0?(\d)/i.test(im.getAttribute('src')||''));
   const types=new Set(); let link=false;
   for(const im of imgs){
     const m=/opentype0?(\d)/i.exec(im.getAttribute('src')||''); if(m) types.add(parseInt(m[1]));
     for(let n=im;n;n=n.parentElement){ if(n.tagName==='A'&&n.getAttribute('href')&&n.getAttribute('href')!=='#'){link=true;break;} }
     let box=im.closest('div,p,td,li')||im.parentElement;
     for(let up=0;box&&up<3;up++){ if([...box.querySelectorAll('a')].some(a=>a.getAttribute('href')&&a.getAttribute('href')!=='#'&&/kogl/i.test(a.href||''))){link=true;break;} box=box.parentElement; }
   }
   // div.open_type 텍스트형
   if(root.querySelector('.open_type,.opentype,.kogl')){ if(!types.size) types.add(4); }
   return {types:[...types].sort(),link};
 }
 const main=document.querySelector('#contents,.contents,.sub_contents,.sub_container,#container,main')||document.body;
 const kogl=koglScan(main);
 // 본문 콘텐츠 이미지(N후보)
 const DEC=/\/common\/|move\.png|no[-_]?img|\/icon|ico_|bul_|btn_|bg_|banner|popup|logo|opentype0?\d|sns_|btn\.|arrow|prev|next|\.svg/i;
 const imgs=[...main.querySelectorAll('img')].filter(im=>{const s=im.getAttribute('src')||''; if(DEC.test(s))return false; const w=im.naturalWidth||im.width||0,h=im.naturalHeight||im.height||0; return (w>=250||h>=250);}).map(im=>im.src).slice(0,8);
 const video=!!document.querySelector('iframe[src*="youtube"],iframe[src*="youtu.be"],video,iframe[src*="player"]');
 const audio=!!document.querySelector('audio,a[href*=".mp3"],a[href*=".wav"],a[href*=".m4a"]');
 return {total:totM?totM[1].replace(/,/g,''):null, av:av.map(a=>a.href).slice(0,3), avn:av.length, eminwon, kogl, imgs, video, audio};
}'''
def nav(pg,u,wait='domcontentloaded'):
    for _ in range(2):
        try: pg.goto(u,wait_until=wait,timeout=22000); pg.wait_for_timeout(1300); return True
        except Exception: time.sleep(1)
    return False
res={}
with sync_playwright() as p:
    b=p.chromium.launch(); pg=b.new_page(viewport={'width':1300,'height':2400}); pg.set_default_timeout(22000)
    for r,u,lab in rows:
        rec={'lab':lab,'u':u}
        if not u.startswith('http'):
            rec.update(L='페이지',rel=True); res[r]=rec; continue
        host=urlparse(u).netloc.lower()
        if not isint(host):
            rec.update(L='사이트',ext=host); res[r]=rec
            print('r%d %-16s 사이트(ext)'%(r,lab[:16]),flush=True); continue
        try:
            if not nav(pg,u): rec.update(L='ERR'); res[r]=rec; print('r%d ERR'%r,flush=True); continue
            fh=urlparse(pg.url).netloc.lower()
            if not isint(fh): rec.update(L='사이트',ext=fh); res[r]=rec; print('r%d %s 사이트(redir)'%(r,lab[:14]),flush=True); continue
            d=pg.evaluate(JS_PAGE)
            urlboard=bool(re.search(r'selectBoardList|/cop/bbs/BBSMSTR|selectBbsNttList',u))
            isboard = urlboard or d['eminwon'] or d['avn']>=3 or (d['total'] is not None and d['avn']>=1)
            kogl=d['kogl']; imgs=d['imgs']; video=d['video']; audio=d['audio']
            if isboard:
                rec['L']='게시판'
                # M
                M=None
                if d['total'] is not None: M=int(d['total'])
                # 상세글에서 공공누리/이미지 재확인
                detail_kogl=kogl; detail_imgs=list(imgs)
                if d['av']:
                    try:
                        if nav(pg,d['av'][0]):
                            dd=pg.evaluate(JS_PAGE)
                            if dd['kogl']['types']: detail_kogl=dd['kogl']
                            detail_imgs=dd['imgs'] or detail_imgs
                    except Exception: pass
                rec.update(M=M, kogl=detail_kogl, imgs=detail_imgs, video=video, audio=audio, avn=d['avn'])
            else:
                rec.update(L='페이지', M=1, kogl=kogl, imgs=imgs, video=video, audio=audio)
            res[r]=rec
            kt=rec.get('kogl',{}).get('types',[])
            print('r%d %-16s %s M=%s kogl=%s img=%d v=%s'%(r,lab[:16],rec['L'],rec.get('M'),kt,len(rec.get('imgs',[])),video),flush=True)
        except Exception as e:
            rec.update(L='ERR',err=str(e)[:30]); res[r]=rec; print('r%d EXC %s'%(r,str(e)[:30]),flush=True)
    b.close()
json.dump({str(k):v for k,v in res.items()},open('_cy4.json','w',encoding='utf-8'),ensure_ascii=False)
from collections import Counter
print('\nL분포',Counter(v.get('L') for v in res.values()))
print('공공누리검출',sum(1 for v in res.values() if v.get('kogl',{}).get('types')))
print('이미지후보행',sum(1 for v in res.values() if v.get('imgs')))