df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
157 lines
6.6 KiB
Python
157 lines
6.6 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""정읍 r283~끝 L 재분류 + 게시판 M·O 재작성 (robust). dry-run -> _l283.json, --write 적용.
|
|
게시판 판정 = list.jeongeup가 유효 게시판페이지(p.bbs_total 존재)인 경우.
|
|
M = 총 게시물 수(bbs_total). 못구하면 기존 M 유지(절대 wipe 안함).
|
|
O = 상세글(view.jeongeup) 표본 img_opentype0N(P=게시물); 없으면 list본문(P=게시판); else 미부착.
|
|
유효 게시판 아님(bbs_total無=빈위젯/잘못된 board참조) + 현재 페이지 => 페이지 유지.
|
|
"""
|
|
import re, json, os, glob, time, sys, shutil, urllib.request
|
|
from collections import Counter
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import openpyxl
|
|
|
|
DIR = os.path.dirname(os.path.abspath(__file__))
|
|
DOMAIN = 'https://www.jeongeup.go.kr'
|
|
HDR = {'User-Agent': 'Mozilla/5.0'}
|
|
OPT = re.compile(r'img_open(?:type|code)(\d{1,2})\.(?:png|jpe?g|gif)', re.I)
|
|
TOTAL = re.compile(r'class="bbs_total"[^>]*>[^<]*<strong>\s*([\d,]+)')
|
|
START = 283
|
|
SAMPLE = 12
|
|
WRITE = '--write' in sys.argv
|
|
pool = ThreadPoolExecutor(max_workers=6)
|
|
|
|
def get(u, minlen=0, tries=4):
|
|
for _ in range(tries):
|
|
try:
|
|
r = urllib.request.urlopen(urllib.request.Request(u.replace(' ', '%20'), headers=HDR), timeout=25).read().decode('utf-8', 'replace')
|
|
if len(r) >= minlen:
|
|
return r
|
|
except Exception:
|
|
pass
|
|
time.sleep(0.6)
|
|
return r if 'r' in dir() else ''
|
|
|
|
def types_in(h):
|
|
ts = set(int(x) for x in OPT.findall(h) if x.isdigit() and 1 <= int(x) <= 4)
|
|
return set() if ts == {1, 2, 3, 4} else ts
|
|
|
|
def total_of(h):
|
|
m = TOTAL.search(h)
|
|
return int(m.group(1).replace(',', '')) if m else None
|
|
|
|
def views_of(h, bid=''):
|
|
vs = [v for v in re.findall(r'view\.jeongeup\?([^"\']+)', h) if 'boardId=' in v]
|
|
if bid:
|
|
vs = [v for v in vs if 'boardId=' + bid in v]
|
|
out, seen = [], set()
|
|
for v in vs:
|
|
key = re.search(r'dataSid=(\d+)', v)
|
|
key = key.group(1) if key else v
|
|
if key in seen:
|
|
continue
|
|
seen.add(key)
|
|
out.append(v.replace('&', '&'))
|
|
return out
|
|
|
|
def board_probe(k, menuCd):
|
|
"""returns dict(valid, M, src_h, bid)."""
|
|
h = get(k, 3000)
|
|
M = total_of(h)
|
|
if M is not None: # index page 자체가 리스트
|
|
bid = (re.findall(r'name="boardId"[^>]*value="(BBS_\d+)"', h) or
|
|
re.findall(r'boardId=(BBS_\d+)', h) or [''])[0]
|
|
return {'valid': True, 'M': M, 'src': h, 'bid': bid}
|
|
bid = (re.findall(r'name="boardId"[^>]*value="(BBS_\d+)"', h) or
|
|
re.findall(r'list\.jeongeup\?[^"\']*boardId=(BBS_\d+)', h) or
|
|
re.findall(r'view\.jeongeup\?[^"\']*boardId=(BBS_\d+)', h))
|
|
if not bid:
|
|
return {'valid': False, 'M': None, 'src': h, 'bid': ''}
|
|
bid = Counter(bid).most_common(1)[0][0]
|
|
csid = (re.findall(r'name="contentsSid"[^>]*value="(\d+)"', h) or [''])[0]
|
|
lu = '%s/board/list.jeongeup?menuCd=%s&boardId=%s%s&startPage=1' % (
|
|
DOMAIN, menuCd, bid, ('&contentsSid=' + csid if csid else ''))
|
|
lh = get(lu, 3000)
|
|
M = total_of(lh)
|
|
return {'valid': M is not None, 'M': M, 'src': lh, 'bid': bid}
|
|
|
|
def board_O(src, bid):
|
|
vs = views_of(src, bid)[:SAMPLE]
|
|
if vs:
|
|
ts = set()
|
|
for r in pool.map(lambda v: types_in(get(DOMAIN + '/board/view.jeongeup?' + v)), vs):
|
|
ts |= r
|
|
if ts:
|
|
return ','.join('%d유형' % n for n in sorted(ts)), '게시물', 'Y'
|
|
lt = types_in(src)
|
|
if lt:
|
|
return ','.join('%d유형' % n for n in sorted(lt)), '게시판', 'Y'
|
|
return '미부착', '', ''
|
|
|
|
t = [p for p in glob.glob(os.path.join(DIR, '*.xlsx'))
|
|
if 'backup' not in p and not os.path.basename(p).startswith(('~$', '_'))][0]
|
|
wb = openpyxl.load_workbook(t)
|
|
ws = wb.active
|
|
|
|
out = []
|
|
for r in range(START, ws.max_row + 1):
|
|
L = ws.cell(r, 12).value
|
|
k = ws.cell(r, 11).value
|
|
if not k or L not in ('페이지', '게시판'):
|
|
continue
|
|
k = str(k)
|
|
name = str(ws.cell(r, 5).value or ws.cell(r, 6).value or ws.cell(r, 7).value or ws.cell(r, 4).value)
|
|
if 'index.jeongeup' not in k or not k.startswith('http'):
|
|
out.append({'row': r, 'curL': L, 'name': name, 'verdict': 'skip', 'k': k})
|
|
continue
|
|
mc = re.search(r'menuCd=([A-Z0-9_]+)', k)
|
|
mc = mc.group(1) if mc else ''
|
|
pr = board_probe(k, mc)
|
|
rec = {'row': r, 'curL': L, 'name': name, 'curM': ws.cell(r, 13).value,
|
|
'curO': ws.cell(r, 15).value, 'curP': ws.cell(r, 16).value, 'curQ': ws.cell(r, 17).value,
|
|
'bid': pr['bid']}
|
|
if pr['valid']:
|
|
O, P, Q = board_O(pr['src'], pr['bid'])
|
|
rec.update({'verdict': '게시판', 'M_new': pr['M'], 'O_new': O, 'P_new': P, 'Q_new': Q})
|
|
tag = ' <<RECLASS' if L != '게시판' else ''
|
|
print('r%d [%s] 게시판 M=%s O=%s%s | %s' % (r, L[:2], pr['M'], O, tag, name[:18]), flush=True)
|
|
else:
|
|
rec['verdict'] = 'page'
|
|
if L == '게시판':
|
|
rec['verdict'] = 'board_keep(검증실패)' # 기존 게시판인데 list검증 실패 -> 보존
|
|
print('r%d [%s] page (bbs_total無 bid=%s) | %s' % (r, L[:2], pr['bid'], name[:18]), flush=True)
|
|
out.append(rec)
|
|
json.dump(out, open(os.path.join(DIR, '_l283.json'), 'w', encoding='utf-8'), ensure_ascii=False, indent=0)
|
|
|
|
json.dump(out, open(os.path.join(DIR, '_l283.json'), 'w', encoding='utf-8'), ensure_ascii=False, indent=0)
|
|
recl = [x for x in out if x.get('verdict') == '게시판' and x['curL'] != '게시판']
|
|
print('\nSCAN=%d 게시판=%d reclass(P→B)=%d %s'
|
|
% (len(out), sum(1 for x in out if x.get('verdict') == '게시판'), len(recl),
|
|
[(x['row'], x['name'][:10], x.get('M_new')) for x in recl]))
|
|
|
|
if WRITE:
|
|
bak = t.replace('.xlsx', '_backup_L재분류전.xlsx')
|
|
if not os.path.exists(bak):
|
|
shutil.copy(t, bak)
|
|
# 동시편집 가드: URL이 그대로인지 확인하며 행 매칭
|
|
nb = openpyxl.load_workbook(t)
|
|
nws = nb.active
|
|
applied = 0
|
|
for x in out:
|
|
if x.get('verdict') != '게시판':
|
|
continue
|
|
r = x['row']
|
|
if str(nws.cell(r, 11).value) != str(ws.cell(r, 11).value):
|
|
print(' SKIP r%d URL 변경됨(동시편집)' % r)
|
|
continue
|
|
nws.cell(r, 12).value = '게시판'
|
|
if x.get('M_new') is not None:
|
|
nws.cell(r, 13).value = x['M_new']
|
|
nws.cell(r, 15).value = x['O_new']
|
|
nws.cell(r, 16).value = x['P_new'] or None
|
|
nws.cell(r, 17).value = x['Q_new'] or None
|
|
applied += 1
|
|
nb.save(t)
|
|
print('APPLIED %d boards -> %s (백업 %s)' % (applied, os.path.basename(t), os.path.basename(bak)))
|
|
else:
|
|
print('DRY-RUN (--write 로 적용)')
|