df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
49 lines
2.3 KiB
Python
49 lines
2.3 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""komipo 전 게시판 M 재조사 — 'Total : N' 추출(throttle+retry). 첫글 제목도 수집해 중복뷰 검증."""
|
|
import re, time, json, warnings
|
|
import openpyxl
|
|
from playwright.sync_api import sync_playwright
|
|
warnings.filterwarnings('ignore')
|
|
FN='한국중부발전.xlsx'
|
|
UA='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120 Safari/537.36'
|
|
TOTAL=re.compile(r'(?:Total|총건수|총|전체)\s*[:\s]*([\d,]+)',re.I)
|
|
wb=openpyxl.load_workbook(FN); ws=wb.active
|
|
boards=[(r,str(ws.cell(r,11).value)) for r in range(3,ws.max_row+1) if ws.cell(r,12).value=='게시판' and ws.cell(r,11).value]
|
|
print('게시판',len(boards),flush=True)
|
|
res=[]
|
|
with sync_playwright() as p:
|
|
b=p.chromium.launch(); pg=b.new_page(user_agent=UA,viewport={'width':1280,'height':1800})
|
|
for idx,(r,u) in enumerate(boards):
|
|
M=None; first=''
|
|
for att in range(4):
|
|
try:
|
|
pg.goto(u,timeout=25000,wait_until='domcontentloaded'); pg.wait_for_timeout(1200)
|
|
d=pg.evaluate(r"""()=>{
|
|
const txt=document.body.innerText||'';
|
|
let firstRow=document.querySelector('table tbody tr td a, .board_list li a, tbody tr a');
|
|
return {txt, first:(firstRow?.textContent||'').trim().slice(0,30), rows:document.querySelectorAll('table tbody tr, .board_list li').length};
|
|
}""")
|
|
m=TOTAL.search(d['txt'])
|
|
M=int(m.group(1).replace(',','')) if m else d['rows']
|
|
first=d['first']
|
|
break
|
|
except Exception:
|
|
time.sleep(2.0+att)
|
|
if M is None: M=ws.cell(r,13).value
|
|
res.append({'r':r,'M':M,'first':first,'url':u})
|
|
ws.cell(r,13).value=M
|
|
# 빈게시판이면 N=없음 동기화
|
|
if M==0 and ws.cell(r,14).value not in ('없음',): pass
|
|
if (idx+1)%10==0: wb.save(FN); print(f' {idx+1}/{len(boards)}',flush=True)
|
|
time.sleep(0.8)
|
|
b.close()
|
|
wb.save(FN)
|
|
json.dump(res,open('_boardM.json','w',encoding='utf-8'),ensure_ascii=False,indent=1)
|
|
from collections import Counter
|
|
mc=Counter(x['M'] for x in res)
|
|
print('M분포(상위):',mc.most_common(10),flush=True)
|
|
# 동일 M 다수 = 중복뷰 의심
|
|
dup=[m for m,c in mc.items() if c>=3 and m]
|
|
print('의심 중복M:',dup,flush=True)
|
|
print('완료',flush=True)
|