df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
62 lines
2.7 KiB
Python
62 lines
2.7 KiB
Python
"""Unified crawl of all 정읍시 rows.
|
|
For each menuCd URL: capture bbs_total count (board vs page), redirect-loop flag.
|
|
Aggregate left-nav (ul.dep2/3/4) anchor target=_blank across ALL pages -> menuCd map.
|
|
Outputs _report.json for review before writing the xlsx."""
|
|
import urllib.request, urllib.error, ssl, re, io, sys, json
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from _navmap import NavParser
|
|
|
|
ctx = ssl.create_default_context(); ctx.check_hostname = False; ctx.verify_mode = ssl.CERT_NONE
|
|
BASE = 'https://www.jeongeup.go.kr/index.jeongeup?menuCd='
|
|
|
|
def fetch(url):
|
|
req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0'})
|
|
return urllib.request.urlopen(req, timeout=25, context=ctx).read().decode('utf-8', 'replace')
|
|
|
|
TOTAL_RE = re.compile(r'bbs_total[^>]*>[^<]*<strong>\s*([\d,]+)\s*</strong>')
|
|
|
|
def crawl_one(item):
|
|
r, code, url = item
|
|
out = {'r': r, 'code': code, 'count': None, 'err': None, 'nav': []}
|
|
try:
|
|
h = fetch(url)
|
|
except urllib.error.HTTPError as e:
|
|
out['err'] = 'redirect_loop' if e.code in (301, 302) else 'http%d' % e.code
|
|
return out
|
|
except Exception as e:
|
|
out['err'] = type(e).__name__
|
|
return out
|
|
m = TOTAL_RE.search(h)
|
|
if m:
|
|
out['count'] = int(m.group(1).replace(',', ''))
|
|
p = NavParser(); p.feed(h)
|
|
out['nav'] = [(a['code'], a['blank']) for a in p.results if a['code']]
|
|
return out
|
|
|
|
if __name__ == '__main__':
|
|
data = json.load(io.open('_dump.json', encoding='utf-8'))
|
|
items = []
|
|
for row in data:
|
|
k = row.get('K', '')
|
|
mc = re.search(r'menuCd=(DOM_\d+)', k)
|
|
if mc and 'jeongeup.go.kr/index.jeongeup' in k:
|
|
items.append((row['r'], mc.group(1), k))
|
|
results = []
|
|
with ThreadPoolExecutor(max_workers=8) as ex:
|
|
for res in ex.map(crawl_one, items):
|
|
results.append(res)
|
|
# aggregate nav target map across all pages
|
|
navmap = {}
|
|
for res in results:
|
|
for code, blank in res['nav']:
|
|
navmap.setdefault(code, set()).add(blank)
|
|
navblank = {c: (True in v) for c, v in navmap.items()}
|
|
report = {'pages': [{'r': x['r'], 'code': x['code'], 'count': x['count'], 'err': x['err']} for x in results],
|
|
'navblank': navblank}
|
|
json.dump(report, io.open('_report.json', 'w', encoding='utf-8'), ensure_ascii=False, indent=0)
|
|
errs = [x for x in results if x['err']]
|
|
counts = [x for x in results if x['count'] is not None]
|
|
print('crawled:', len(results), '| with bbs_total:', len(counts), '| errors:', len(errs))
|
|
print('distinct nav menuCd:', len(navblank), '| nav target=_blank:', sum(1 for v in navblank.values() if v))
|
|
print('errors:', [(x['r'], x['err']) for x in errs])
|