df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
121 lines
4.7 KiB
Python
121 lines
4.7 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""공주시 227~414행 재검수용 증거 수집(쓰기 없음).
|
|
각 URL을 크롤링해 게시판 글수/관광지 썸네일수/외부링크/공공누리마크/#nav탭수/본문이미지·영상 신호를 수집.
|
|
검증용으로 226행까지 검수에서 바뀐 행 일부도 같이 수집.
|
|
출력: _스크립트/_evidence.json + 콘솔 요약
|
|
"""
|
|
import sys, json, re, warnings
|
|
from urllib.parse import urlparse
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
import requests
|
|
from bs4 import BeautifulSoup
|
|
import openpyxl
|
|
warnings.filterwarnings('ignore')
|
|
|
|
XLSX = r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청남도\2.공주시\충청남도_공주시.xlsx'
|
|
H = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120 Safari/537.36'}
|
|
COMMON_IMG = re.compile(r'(flag\.jpg|slogan|/common/|/template/|btn_|/btn|icon|blank|_mark\.png|sns|share|loading)', re.I)
|
|
|
|
|
|
def fetch(row, url):
|
|
try:
|
|
r = requests.get(url, headers=H, timeout=25, verify=False)
|
|
r.encoding = r.apparent_encoding or 'utf-8'
|
|
return row, url, r.status_code, r.text
|
|
except Exception as e:
|
|
return row, url, None, ('ERR:%s' % e)
|
|
|
|
|
|
def analyze(url, html):
|
|
ev = {}
|
|
host = urlparse(url).netloc
|
|
ev['host'] = host
|
|
ev['external'] = ('gongju.go.kr' not in host)
|
|
if not html or html.startswith('ERR:'):
|
|
ev['error'] = html
|
|
return ev
|
|
s = BeautifulSoup(html, 'html.parser')
|
|
# board total count
|
|
cnt_el = s.select_one('.program--count strong')
|
|
if cnt_el:
|
|
m = re.sub(r'[^0-9]', '', cnt_el.get_text())
|
|
ev['board_count'] = int(m) if m else None
|
|
ev['is_board'] = True
|
|
else:
|
|
ev['is_board'] = False
|
|
# tursmCn thumbnails (관광지)
|
|
tt = len(re.findall(r'/thumbnail/tursmCn/', html))
|
|
if tt:
|
|
ev['tursm_count'] = tt
|
|
# content images (static html, excludes template/common) — JS maps missed
|
|
cimgs = []
|
|
for im in s.find_all('img'):
|
|
src = im.get('src') or im.get('data-src') or ''
|
|
if src and not COMMON_IMG.search(src):
|
|
cimgs.append(src)
|
|
ev['content_img'] = len(cimgs)
|
|
ev['content_img_sample'] = cimgs[:4]
|
|
# video signals
|
|
low = html.lower()
|
|
ev['video'] = bool(re.search(r'youtube\.com/embed|youtu\.be/|player\.vimeo|<video|\.mp4|data-video', low))
|
|
# KOGL / 공공누리
|
|
kogl_hits = re.findall(r'(opentype0?[1-4]|kogl[_\-]?[1-4]?|공공누리)', html, re.I)
|
|
ev['kogl'] = bool(kogl_hits)
|
|
ev['kogl_sample'] = list(dict.fromkeys(kogl_hits))[:6]
|
|
# explicit opentype number
|
|
ot = re.findall(r'opentype0?([1-4])', html, re.I)
|
|
if ot:
|
|
ev['kogl_types'] = sorted(set(int(x) for x in ot))
|
|
# #nav tab count (규칙 A)
|
|
best = 0
|
|
for ul in s.find_all('ul'):
|
|
cls = ' '.join(ul.get('class') or []).lower()
|
|
if 'tab-ul' in cls:
|
|
anchors = [a for a in ul.find_all('a')
|
|
if (a.get('href') or '').strip().startswith('#') and a.get_text(strip=True)]
|
|
if len(anchors) >= 2:
|
|
best = max(best, len(anchors))
|
|
if best:
|
|
ev['nav_tabs'] = best
|
|
return ev
|
|
|
|
|
|
def main():
|
|
lo, hi = 227, 414
|
|
if len(sys.argv) > 2:
|
|
lo, hi = int(sys.argv[1]), int(sys.argv[2])
|
|
wb = openpyxl.load_workbook(XLSX)
|
|
ws = wb.active
|
|
targets = []
|
|
rowinfo = {}
|
|
for r in range(lo, hi + 1):
|
|
u = ws.cell(r, 11).value
|
|
if isinstance(u, str) and u.strip().startswith('http'):
|
|
targets.append((r, u.strip()))
|
|
rowinfo[r] = {
|
|
'D': ws.cell(r, 4).value, 'E': ws.cell(r, 5).value, 'F': ws.cell(r, 6).value,
|
|
'G': ws.cell(r, 7).value, 'K': u.strip(), 'L': ws.cell(r, 12).value,
|
|
'M': ws.cell(r, 13).value, 'N': ws.cell(r, 14).value, 'O': ws.cell(r, 15).value,
|
|
}
|
|
print('크롤 대상:', len(targets), '행', lo, '~', hi)
|
|
out = {}
|
|
done = 0
|
|
with ThreadPoolExecutor(max_workers=10) as ex:
|
|
futs = [ex.submit(fetch, r, u) for r, u in targets]
|
|
for f in as_completed(futs):
|
|
row, url, st, html = f.result()
|
|
ev = analyze(url, html)
|
|
ev['status'] = st
|
|
ev['row'] = row
|
|
out[row] = {**rowinfo[row], **ev}
|
|
done += 1
|
|
if done % 25 == 0:
|
|
print(' ...', done, '/', len(targets))
|
|
json.dump(out, open(r'D:\01.프로젝트\DB수집\_스크립트\_evidence.json', 'w', encoding='utf-8'),
|
|
ensure_ascii=False, indent=1)
|
|
print('저장: _evidence.json (', len(out), '행 )')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|