df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
85 lines
3.6 KiB
Python
85 lines
3.6 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""정읍 사전정보공표목록(BBS_0000171) 3단 카테고리 트리 크롤 -> _gongpyo.json.
|
|
cat1(col5)>cat2(col3)>cat3(col4 leaf). 각 leaf M=총게시물 + view링크(O용)."""
|
|
import re, json, os, time, urllib.request
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
HDR = {'User-Agent': 'Mozilla/5.0'}
|
|
DIR = os.path.dirname(os.path.abspath(__file__))
|
|
BID = 'BBS_0000171'
|
|
MENU = 'DOM_000000104003008000'
|
|
BASE = 'https://www.jeongeup.go.kr/board/list.jeongeup?boardId=%s&menuCd=%s' % (BID, MENU)
|
|
TOTAL = re.compile(r'class="bbs_total"[^>]*>[^<]*<strong>\s*([\d,]+)')
|
|
pool = ThreadPoolExecutor(max_workers=6)
|
|
|
|
def get(u, minlen=3000):
|
|
for _ in range(4):
|
|
try:
|
|
r = urllib.request.urlopen(urllib.request.Request(u.replace(' ', '%20'), headers=HDR), timeout=25).read().decode('utf-8', 'replace')
|
|
if len(r) >= minlen:
|
|
return r
|
|
except Exception:
|
|
pass
|
|
time.sleep(0.5)
|
|
return r if 'r' in dir() else ''
|
|
|
|
def tab_items(h, tabid):
|
|
"""div#<tabid> 안의 li>a 에서 (c1,c2,c3,label) 추출."""
|
|
m = re.search(r'id="%s"[^>]*>(.*?)</div>\s*(?:<div|$)' % tabid, h, re.S)
|
|
seg = m.group(1) if m else ''
|
|
out = []
|
|
for a in re.finditer(r'<a href="([^"]*)"[^>]*>(.*?)</a>', seg, re.S):
|
|
href, lab = a.group(1), re.sub(r'<[^>]+>', '', a.group(2)).strip()
|
|
c1 = re.search(r'categoryCode1=([^&#]+)', href)
|
|
c2 = re.search(r'categoryCode2=([^&#]+)', href)
|
|
c3 = re.search(r'categoryCode3=([^&#]+)', href)
|
|
out.append({'c1': c1.group(1) if c1 else '', 'c2': c2.group(1) if c2 else '',
|
|
'c3': c3.group(1) if c3 else '', 'label': lab})
|
|
return out
|
|
|
|
def url_of(c1, c2, c3):
|
|
return '%s&categoryCode1=%s&categoryCode2=%s&categoryCode3=%s' % (BASE, c1, c2, c3)
|
|
|
|
# 1) cat1 목록 (아무 fetch에나 들어있음)
|
|
h0 = get(url_of('A', 'A_01', 'A_01_001'))
|
|
cat1 = tab_items(h0, 'categoryCode1_tab')
|
|
print('cat1:', [(c['c1'], c['label']) for c in cat1])
|
|
|
|
tree = []
|
|
for c1 in cat1:
|
|
# cat1 선택 -> cat2 목록
|
|
hc1 = get(url_of(c1['c1'], c1['c1'] + '_01', c1['c1'] + '_01_001'))
|
|
cat2 = tab_items(hc1, 'categoryCode2_tab')
|
|
if not cat2:
|
|
cat2 = [{'c1': c1['c1'], 'c2': c1['c1'] + '_01', 'c3': '', 'label': ''}]
|
|
for c2 in cat2:
|
|
hc2 = get(url_of(c2['c1'], c2['c2'], c2['c2'] + '_001'))
|
|
cat3 = tab_items(hc2, 'categoryCode3_tab')
|
|
if not cat3:
|
|
cat3 = [{'c1': c2['c1'], 'c2': c2['c2'], 'c3': c2['c2'] + '_001', 'label': ''}]
|
|
for c3 in cat3:
|
|
tree.append({'c1': c1['c1'], 'c1_label': c1['label'],
|
|
'c2': c2['c2'], 'c2_label': c2['label'],
|
|
'c3': c3['c3'], 'c3_label': c3['label']})
|
|
print(' cat1=%s(%s) cat2=%d leaves so far=%d' % (c1['c1'], c1['label'], len(cat2), len(tree)), flush=True)
|
|
|
|
print('TOTAL leaves:', len(tree))
|
|
|
|
# 2) 각 leaf M + view링크
|
|
def leaf_data(node):
|
|
u = url_of(node['c1'], node['c2'], node['c3'])
|
|
h = get(u)
|
|
m = TOTAL.search(h)
|
|
M = int(m.group(1).replace(',', '')) if m else None
|
|
views = [v for v in re.findall(r'view\.jeongeup\?([^"\']+)', h) if 'boardId=' + BID in v]
|
|
node['M'] = M
|
|
node['url'] = u
|
|
node['nview'] = len(views)
|
|
node['views'] = views[:8]
|
|
return node
|
|
|
|
tree = list(pool.map(leaf_data, tree))
|
|
json.dump(tree, open(os.path.join(DIR, '_gongpyo.json'), 'w', encoding='utf-8'), ensure_ascii=False, indent=0)
|
|
tot = sum(x['M'] or 0 for x in tree)
|
|
print('leaves=%d M합=%d M>0 leaves=%d M=None=%d'
|
|
% (len(tree), tot, sum(1 for x in tree if x['M']), sum(1 for x in tree if x['M'] is None)))
|