df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
62 lines
2.6 KiB
Python
62 lines
2.6 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""정읍 L(게시판/페이지) 판별 신호 캘리브레이션.
|
|
3~282(사용자검수=정답)의 내부 index.jeongeup 페이지/게시판을 크롤해 신호 수집 -> _ldetect.json
|
|
"""
|
|
import re, json, os, time, urllib.request
|
|
from collections import Counter
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import openpyxl
|
|
|
|
DIR = os.path.dirname(os.path.abspath(__file__))
|
|
HDR = {'User-Agent': 'Mozilla/5.0'}
|
|
|
|
def get(u):
|
|
for _ in range(2):
|
|
try:
|
|
return urllib.request.urlopen(urllib.request.Request(u.replace(' ', '%20'), headers=HDR), timeout=25).read().decode('utf-8', 'replace')
|
|
except Exception:
|
|
time.sleep(0.4)
|
|
return ''
|
|
|
|
def signals(h):
|
|
s = {}
|
|
s['nview'] = len(re.findall(r'view\.jeongeup\?[^"\']*boardId=BBS_\d+', h))
|
|
s['nlist'] = len(re.findall(r'list\.jeongeup\?[^"\']*boardId=BBS_\d+', h))
|
|
s['ndown'] = len(re.findall(r'download\.jeongeup', h))
|
|
s['nbbs'] = len(set(re.findall(r'boardId=(BBS_\d+)', h)))
|
|
s['total'] = bool(re.search(r'총\s*<?[^>]{0,15}?[\d,]+\s*</?[^>]{0,6}?건', h)) or ('게시물' in h)
|
|
s['paging'] = bool(re.search(r'class="[^"]*(?:paging|pagination|board_page)', h)) or bool(re.search(r'startPage=\d', h))
|
|
# 게시판 리스트 테이블/구조 클래스
|
|
s['bbslist'] = bool(re.search(r'class="[^"]*(?:bbs|board_?list|p-table|tbl_list|basic_list|board_basic|gallery_list|news_list|photo_list)', h, re.I))
|
|
s['writeb'] = bool(re.search(r'write\.jeongeup', h))
|
|
s['skin'] = ('class="bbs_skin"' in h) # AJAX 게시판 스켈레톤(=진짜 게시판)
|
|
hb = re.findall(r'<input[^>]*name="boardId"[^>]*value="(BBS_\d+)"', h)
|
|
vb = Counter(re.findall(r'view\.jeongeup\?[^"\']*boardId=(BBS_\d+)', h))
|
|
s['hidbid'] = hb[0] if hb else ''
|
|
s['vbid'] = vb.most_common(1)[0][0] if vb else ''
|
|
s['len'] = len(h)
|
|
return s
|
|
|
|
t = [p for p in __import__('glob').glob(os.path.join(DIR, '*.xlsx'))
|
|
if 'backup' not in p and not os.path.basename(p).startswith(('~$', '_'))][0]
|
|
ws = openpyxl.load_workbook(t).active
|
|
|
|
jobs = []
|
|
for r in range(3, 283):
|
|
L = ws.cell(r, 12).value
|
|
k = ws.cell(r, 11).value
|
|
if not k:
|
|
continue
|
|
k = str(k)
|
|
if L in ('페이지', '게시판') and k.startswith('http') and 'index.jeongeup' in k:
|
|
jobs.append((r, L, k))
|
|
|
|
pool = ThreadPoolExecutor(max_workers=8)
|
|
def work(job):
|
|
r, L, k = job
|
|
return {'row': r, 'L': L, 'k': k, **signals(get(k))}
|
|
|
|
out = list(pool.map(work, jobs))
|
|
json.dump(out, open(os.path.join(DIR, '_ldetect.json'), 'w', encoding='utf-8'), ensure_ascii=False, indent=0)
|
|
print('crawled', len(out))
|