df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
126 lines
5.2 KiB
Python
126 lines
5.2 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""공공기관 N 이미지 — 게시판 전용 재판정(상세글 렌더 + 크기필터).
|
||
|
||
매뉴얼: 게시판 N 이미지 = 상세글 표본의 본문 실사진(첨부PDF/뷰어 제외).
|
||
대상: plan_{기관}.json 에서 L==게시판 & 원본N('이미지' 포함) 행.
|
||
각 게시판: 목록에서 상세URL 추출→상위 4개 렌더→본문 콘텐츠이미지(≥170×110, 노이즈제외) 있으면 이미지.
|
||
결과를 엑셀 N에 반영(게시판만). 사용: python _공공기관_nboard.py [기관명 ...]
|
||
"""
|
||
import sys, os, re, json, time, warnings, importlib.util
|
||
import requests, openpyxl
|
||
from urllib.parse import urljoin
|
||
from bs4 import BeautifulSoup
|
||
warnings.filterwarnings('ignore')
|
||
|
||
OUTDIR = r'D:\01.프로젝트\DB수집\공공기관'
|
||
TEMP = r'D:\01.프로젝트\DB수집\_temp'
|
||
UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
|
||
spec = importlib.util.spec_from_file_location('p234', r'D:\01.프로젝트\DB수집\_스크립트\_공공기관_phase234.py')
|
||
P = importlib.util.module_from_spec(spec); spec.loader.exec_module(P)
|
||
NOISE = re.compile(r'(ico[_\-/]|/icon|logo|btn|bul[_\-]|bg[_\-]|banner|sns|blank|spacer|loading|arrow|/dot|line[_\.]|top_|foot|header|common|_icon|symbol|copyright|qr_|no_img|noimage|share|facebook|insta|twitter|naver|kakao|/skin/|/template/|synap|pdf|viewer|/editor/icon)', re.I)
|
||
|
||
JS_IMGS = """() => { const o=[]; for(const i of document.querySelectorAll('img')){const r=i.getBoundingClientRect(); o.push({src:i.currentSrc||i.src||'',nw:i.naturalWidth,nh:i.naturalHeight,rw:Math.round(r.width),rh:Math.round(r.height)});} return o; }"""
|
||
|
||
|
||
def has_content_img(items):
|
||
for it in items:
|
||
s = it.get('src', '')
|
||
if not s or NOISE.search(s):
|
||
continue
|
||
if it.get('nw', 0) >= 170 and it.get('nh', 0) >= 110 and it.get('rw', 0) >= 100 and it.get('rh', 0) >= 80:
|
||
return True
|
||
return False
|
||
|
||
|
||
def add_image(N):
|
||
parts = [p for p in (N or '').split(',') if p]
|
||
if '없음' in parts:
|
||
parts = [p for p in parts if p != '없음']
|
||
if '어문' not in parts:
|
||
parts = ['어문'] + parts
|
||
if '이미지' not in parts:
|
||
# 어문 다음에 이미지
|
||
out = []
|
||
for p in parts:
|
||
out.append(p)
|
||
if p == '어문':
|
||
out.append('이미지')
|
||
parts = out
|
||
return ','.join(parts)
|
||
|
||
|
||
def rm_image(N):
|
||
parts = [p for p in (N or '').split(',') if p and p != '이미지']
|
||
return ','.join(parts) if parts else '없음'
|
||
|
||
|
||
def run(name, pg):
|
||
pf = os.path.join(TEMP, f'nshot_{name}', f'plan_{name}.json')
|
||
if not os.path.exists(pf):
|
||
print(f'[{name}] plan없음 스킵'); return
|
||
plan = json.load(open(pf, encoding='utf-8'))
|
||
boards = [it for it in plan if it.get('L') == '게시판' and '이미지' in (it.get('N') or '')]
|
||
if not boards:
|
||
print(f'[{name}] 재판정 게시판 0'); return
|
||
xlsx = os.path.join(OUTDIR, f'{name}.xlsx')
|
||
wb = openpyxl.load_workbook(xlsx)
|
||
ws = wb.active
|
||
sess = P.make_session()
|
||
img_cnt = txt_cnt = 0
|
||
montshots = []
|
||
for it in boards:
|
||
r, url = it['row'], it['url']
|
||
try:
|
||
soup, _ = P.fetch(sess, url)
|
||
if not soup:
|
||
continue
|
||
body = P.get_body(soup, P.BODY_SEL)
|
||
details = P.extract_detail_urls(body, url, limit=4)
|
||
found = False
|
||
for du in details:
|
||
try:
|
||
pg.goto(du, timeout=20000, wait_until='domcontentloaded'); pg.wait_for_timeout(1500)
|
||
if has_content_img(pg.evaluate(JS_IMGS)):
|
||
found = True
|
||
sp = os.path.join(TEMP, f'nshot_{name}', f'bd_r{r}.png')
|
||
try:
|
||
pg.screenshot(path=sp); montshots.append((r, sp))
|
||
except Exception:
|
||
pass
|
||
break
|
||
except Exception:
|
||
continue
|
||
cur = ws.cell(r, 14).value or ''
|
||
if found:
|
||
ws.cell(r, 14).value = add_image(cur); img_cnt += 1
|
||
else:
|
||
ws.cell(r, 14).value = rm_image(cur); txt_cnt += 1
|
||
except Exception:
|
||
continue
|
||
wb.save(xlsx)
|
||
print(f'[{name}] 게시판재판정 {len(boards)}개 → 이미지 {img_cnt}·어문 {txt_cnt}')
|
||
return montshots
|
||
|
||
|
||
def main():
|
||
from playwright.sync_api import sync_playwright
|
||
probe = {r['name']: r for r in json.load(open(r'D:\01.프로젝트\DB수집\_스크립트\_공공기관_probe.json', encoding='utf-8'))}
|
||
order = sorted(probe.values(), key=lambda x: -int(x['num']))
|
||
only = sys.argv[1:]
|
||
if only:
|
||
order = [p for p in order if p['name'] in only or str(p['num']) in only]
|
||
with sync_playwright() as pw:
|
||
b = pw.chromium.launch()
|
||
pg = b.new_page(user_agent=UA, viewport={'width': 1280, 'height': 1600})
|
||
for p in order:
|
||
try:
|
||
run(p['name'], pg)
|
||
except Exception as e:
|
||
print(f"[{p['name']}] 실패: {e}")
|
||
b.close()
|
||
print('=== 게시판 N 재판정 완료 ===')
|
||
|
||
|
||
if __name__ == '__main__':
|
||
main()
|