df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
147 lines
6.4 KiB
Python
147 lines
6.4 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""정읍 r283~끝 L 재분류(게시판↔페이지) + 게시판 M·O 재작성. dry-run -> _l283.json.
|
|
게시판 감지: bbs_skin 스켈레톤 OR list.jeongeup링크 OR view링크>=3.
|
|
boardId/contentsSid를 hidden input/링크에서 추출 -> /board/list.jeongeup 페치 ->
|
|
실제 글표(tbody tr) 있으면 게시판, M=총N건 or 첫칸번호 최댓값 or 행수.
|
|
O: 상세글(view.jeongeup) 표본 img_opentype0N -> 유형(P=게시물); 상세없으면 list본문 footer(P=게시판).
|
|
"""
|
|
import re, json, os, glob, time, urllib.request
|
|
from collections import Counter
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import openpyxl
|
|
|
|
DIR = os.path.dirname(os.path.abspath(__file__))
|
|
DOMAIN = 'https://www.jeongeup.go.kr'
|
|
HDR = {'User-Agent': 'Mozilla/5.0'}
|
|
OPT = re.compile(r'img_open(?:type|code)(\d{1,2})\.(?:png|jpe?g|gif)', re.I)
|
|
START = 283
|
|
SAMPLE = 12
|
|
pool = ThreadPoolExecutor(max_workers=6)
|
|
|
|
def get(u):
|
|
for _ in range(3):
|
|
try:
|
|
return urllib.request.urlopen(urllib.request.Request(u.replace(' ', '%20'), headers=HDR), timeout=25).read().decode('utf-8', 'replace')
|
|
except Exception:
|
|
time.sleep(0.5)
|
|
return ''
|
|
|
|
def types_in(h):
|
|
ts = set(int(x) for x in OPT.findall(h) if x.isdigit() and 1 <= int(x) <= 4)
|
|
return set() if ts == {1, 2, 3, 4} else ts
|
|
|
|
def parse_list(h):
|
|
"""list.jeongeup HTML -> (rowcount, Mnum, view_links)."""
|
|
views = [v for v in re.findall(r'view\.jeongeup\?([^"\']+)', h) if 'boardId=' in v]
|
|
# tbody data rows: first <td> numeric
|
|
nums = []
|
|
for tr in re.findall(r'<tr[^>]*>(.*?)</tr>', h, re.S):
|
|
td = re.search(r'<td[^>]*>\s*([\d,]+)\s*</td>', tr)
|
|
if td:
|
|
nums.append(int(td.group(1).replace(',', '')))
|
|
tot = re.findall(r'총\s*<?[^>]{0,12}?([\d,]+)\s*<?[^>]{0,8}?\s*건', h)
|
|
M = None
|
|
if tot:
|
|
M = int(tot[0].replace(',', ''))
|
|
elif nums:
|
|
M = max(nums)
|
|
return len(nums), M, views
|
|
|
|
def board_info(idx_h, menuCd):
|
|
"""index page에서 boardId/contentsSid 추출 -> list 페치 -> 정보."""
|
|
bid = (re.findall(r'<input[^>]*name="boardId"[^>]*value="(BBS_\d+)"', idx_h)
|
|
or re.findall(r'list\.jeongeup\?[^"\']*boardId=(BBS_\d+)', idx_h)
|
|
or re.findall(r'view\.jeongeup\?[^"\']*boardId=(BBS_\d+)', idx_h))
|
|
if not bid:
|
|
# index page 자체가 정적 리스트일 수도
|
|
return parse_list_self(idx_h)
|
|
bid = Counter(bid).most_common(1)[0][0]
|
|
csid = re.findall(r'<input[^>]*name="contentsSid"[^>]*value="(\d+)"', idx_h)
|
|
csid = csid[0] if csid else ''
|
|
lu = '%s/board/list.jeongeup?menuCd=%s&boardId=%s%s&startPage=1' % (
|
|
DOMAIN, menuCd, bid, ('&contentsSid=' + csid if csid else ''))
|
|
lh = get(lu)
|
|
nrow, M, views = parse_list(lh)
|
|
return {'bid': bid, 'listlen': len(lh), 'nrow': nrow, 'M': M, 'views': views, 'list_h': lh}
|
|
|
|
def parse_list_self(h):
|
|
nrow, M, views = parse_list(h)
|
|
return {'bid': '', 'listlen': len(h), 'nrow': nrow, 'M': M, 'views': views, 'list_h': h}
|
|
|
|
def board_O(info):
|
|
views = info.get('views') or []
|
|
seen, vs = set(), []
|
|
bidf = info.get('bid', '')
|
|
for v in views:
|
|
if bidf and 'boardId=' + bidf not in v:
|
|
continue
|
|
key = re.search(r'dataSid=(\d+)', v)
|
|
key = key.group(1) if key else v
|
|
if key in seen:
|
|
continue
|
|
seen.add(key)
|
|
vs.append(v.replace('&', '&'))
|
|
if len(vs) >= SAMPLE:
|
|
break
|
|
if vs:
|
|
ts = set()
|
|
for r in pool.map(lambda v: types_in(get(DOMAIN + '/board/view.jeongeup?' + v)), vs):
|
|
ts |= r
|
|
if ts:
|
|
return ','.join('%d유형' % n for n in sorted(ts)), '게시물', 'Y'
|
|
# 상세 없거나 마크없음 -> list 본문 footer
|
|
lt = types_in(info.get('list_h', ''))
|
|
if lt:
|
|
return ','.join('%d유형' % n for n in sorted(lt)), '게시판', 'Y'
|
|
return '미부착', '', ''
|
|
|
|
t = [p for p in glob.glob(os.path.join(DIR, '*.xlsx'))
|
|
if 'backup' not in p and not os.path.basename(p).startswith(('~$', '_'))][0]
|
|
ws = openpyxl.load_workbook(t).active
|
|
|
|
out = []
|
|
for r in range(START, ws.max_row + 1):
|
|
L = ws.cell(r, 12).value
|
|
k = ws.cell(r, 11).value
|
|
if not k:
|
|
continue
|
|
k = str(k)
|
|
if L not in ('페이지', '게시판'):
|
|
continue
|
|
if not k.startswith('http') or 'index.jeongeup' not in k:
|
|
out.append({'row': r, 'curL': L, 'verdict': 'skip(비표준URL)', 'k': k})
|
|
continue
|
|
mc = re.search(r'menuCd=([A-Z0-9_]+)', k)
|
|
mc = mc.group(1) if mc else ''
|
|
h = get(k)
|
|
skin = ('class="bbs_skin"' in h)
|
|
nlist = len(re.findall(r'list\.jeongeup\?[^"\']*boardId=BBS_\d+', h))
|
|
nview = len(re.findall(r'view\.jeongeup\?[^"\']*boardId=BBS_\d+', h))
|
|
pred = skin or nlist >= 1 or nview >= 3
|
|
rec = {'row': r, 'curL': L, 'name': str(ws.cell(r, 5).value or ws.cell(r, 6).value or ws.cell(r, 7).value or ws.cell(r, 4).value),
|
|
'skin': skin, 'nlist': nlist, 'nview': nview, 'curM': ws.cell(r, 13).value, 'curO': ws.cell(r, 15).value}
|
|
if not pred:
|
|
rec['verdict'] = 'page(신호無)'
|
|
out.append(rec)
|
|
print('r%d [%s] page (skin0 list0 view%d)' % (r, L[:2], nview), flush=True)
|
|
continue
|
|
info = board_info(h, mc)
|
|
is_board = (info['nrow'] >= 1) or (info['listlen'] > 5000 and (info['M'] or 0) >= 0 and info['bid'])
|
|
rec.update({'bid': info['bid'], 'listlen': info['listlen'], 'nrow': info['nrow'], 'M_new': info['M']})
|
|
if not is_board:
|
|
rec['verdict'] = 'page(빈board/위젯 listlen=%d nrow=%d)' % (info['listlen'], info['nrow'])
|
|
out.append(rec)
|
|
print('r%d [%s] page? listlen=%d nrow=%d bid=%s' % (r, L[:2], info['listlen'], info['nrow'], info['bid']), flush=True)
|
|
continue
|
|
O, P, Q = board_O(info)
|
|
rec.update({'verdict': '게시판', 'O_new': O, 'P_new': P, 'Q_new': Q})
|
|
flag = ' <<RECLASS' if L != '게시판' else ''
|
|
print('r%d [%s] 게시판 M=%s O=%s nrow=%d%s' % (r, L[:2], info['M'], O, info['nrow'], flag), flush=True)
|
|
out.append(rec)
|
|
json.dump(out, open(os.path.join(DIR, '_l283.json'), 'w', encoding='utf-8'), ensure_ascii=False, indent=0)
|
|
|
|
json.dump(out, open(os.path.join(DIR, '_l283.json'), 'w', encoding='utf-8'), ensure_ascii=False, indent=0)
|
|
recl = [x for x in out if x.get('verdict') == '게시판' and x['curL'] != '게시판']
|
|
print('\nSCAN=%d 게시판판정=%d 신규reclass(페이지→게시판)=%d rows=%s'
|
|
% (len(out), sum(1 for x in out if x.get('verdict') == '게시판'), len(recl), [x['row'] for x in recl]))
|