df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
101 lines
4.6 KiB
Python
101 lines
4.6 KiB
Python
import sys, io, re, ssl, json, urllib.request
|
|
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
ctx = ssl.create_default_context(); ctx.check_hostname=False; ctx.verify_mode=ssl.CERT_NONE
|
|
BASE = 'https://www.jangsu.go.kr/index.jangsu?menuCd='
|
|
def fetch(cd):
|
|
u = BASE + cd
|
|
return urllib.request.urlopen(urllib.request.Request(u, headers={'User-Agent':'Mozilla/5.0'}), context=ctx, timeout=25).read().decode('utf-8','replace')
|
|
|
|
COMMON = ('kogl.or.kr','elis.go.kr','jeonbuk.work.go.kr','/index.jangsu')
|
|
|
|
def content_seg(h):
|
|
m = re.search(r'id="contents"(.*?)(?:<footer|id="footer|<!-- //CONTENTS)', h, re.S)
|
|
seg = m.group(1) if m else h
|
|
return re.sub(r'<script.*?</script>', '', seg, flags=re.S)
|
|
|
|
def analyze(cd):
|
|
try:
|
|
h = fetch(cd)
|
|
except Exception as e:
|
|
return {'err': str(e)[:50]}
|
|
seg = content_seg(h)
|
|
txt = re.sub(r'\s+', ' ', re.sub(r'<[^>]+>', ' ', seg)).strip()
|
|
# 탭
|
|
taps = []
|
|
tm = re.search(r'<ul class="taps[^"]*">(.*?)</ul>', h, re.S)
|
|
if tm:
|
|
for am in re.finditer(r'menuCd=(DOM_000000\d{12})"[^>]*>(?:<span>)?\s*([^<]+)', tm.group(1)):
|
|
taps.append((am.group(1), am.group(2).strip()))
|
|
# 외부 사이트
|
|
exts = re.findall(r'href="(https?://[^"]+)"', seg)
|
|
real_ext = [e for e in exts if not any(c in e for c in COMMON)]
|
|
is_site = bool(real_ext) and bool(re.search(r'해당\s*사이트로\s*이동|홈페이지\s*바로가기|버튼을\s*클릭', txt))
|
|
# 게시판
|
|
is_board = 'bbs_list' in seg
|
|
mcount = None
|
|
bm = re.search(r'전체\s*([\d,]+)\s*건', txt)
|
|
if bm: mcount = int(bm.group(1).replace(',', ''))
|
|
# 공공누리
|
|
om = re.search(r'공공누리\s*제?\s*(\d)\s*유형', txt)
|
|
otype = om.group(1) + '유형' if om else '미부착'
|
|
# 이미지(본문 실이미지 — 데코 제외)
|
|
imgs = re.findall(r'<img[^>]+src="([^"]+)"', seg)
|
|
deco = re.compile(r'opentype|kogl|btn_|icon|blank|move|/template/|/images/|no_img', re.I)
|
|
realimg = [s for s in imgs if not deco.search(s)]
|
|
if is_site:
|
|
L = '사이트'; K = real_ext[0]; M=None; N=None; O='미부착'
|
|
elif is_board:
|
|
L = '게시판'; K = BASE+cd; M = mcount if mcount is not None else 0
|
|
N = '어문' if M and M>0 else '없음'; O = otype
|
|
else:
|
|
L = '페이지'; K = BASE+cd; M = 1
|
|
N = '어문,이미지' if len(realimg)>=2 else '어문'; O = otype
|
|
return {'taps':taps,'L':L,'K':K,'M':M,'N':N,'O':O,'site':is_site,'board':is_board}
|
|
|
|
tree = json.load(open(r'D:\01.프로젝트\DB수집\작업파일\_스크립트\_jangsu_tree.json', encoding='utf-8'))
|
|
대, 중, 소 = tree['대'], tree['중'], [tuple(x) for x in tree['소']]
|
|
|
|
# 1차: 모든 소분류 분석
|
|
print(f'소분류 {len(소)}개 크롤 시작...', file=sys.stderr)
|
|
res = {}
|
|
def run(cd_full):
|
|
return cd_full, analyze(cd_full)
|
|
with ThreadPoolExecutor(max_workers=8) as ex:
|
|
for cd_full, r in ex.map(run, [x[3] for x in 소]):
|
|
res[cd_full] = r
|
|
|
|
# 2차: 탭 자식 분석 (소분류에 탭 있으면)
|
|
tab_cds = set()
|
|
for cd, r in res.items():
|
|
if r.get('taps'):
|
|
for tcd, _ in r['taps']:
|
|
tab_cds.add(tcd)
|
|
tab_res = {}
|
|
with ThreadPoolExecutor(max_workers=8) as ex:
|
|
for cd_full, r in ex.map(run, sorted(tab_cds)):
|
|
tab_res[cd_full] = r
|
|
print(f'소분류 {len(res)} + 탭 {len(tab_res)} 분석 완료', file=sys.stderr)
|
|
|
|
# 행 빌드
|
|
rows = [] # dict: 대,중,소,세부(G),K,L,M,N,O
|
|
errs = []
|
|
for d, e, f, cd, lab in 소:
|
|
r = res.get(cd, {})
|
|
if r.get('err'): errs.append((cd, lab, r['err']))
|
|
Dl, El = 대.get(d, ''), 중.get(d+e, '')
|
|
taps = r.get('taps') or []
|
|
if taps:
|
|
for tcd, tlab in taps:
|
|
tr = tab_res.get(tcd, {})
|
|
rows.append({'D':Dl,'E':El,'F':lab,'G':tlab,'K':tr.get('K', BASE+tcd),
|
|
'L':tr.get('L','페이지'),'M':tr.get('M',1),'N':tr.get('N','어문'),'O':tr.get('O','미부착')})
|
|
else:
|
|
rows.append({'D':Dl,'E':El,'F':lab,'G':None,'K':r.get('K', BASE+cd),
|
|
'L':r.get('L','페이지'),'M':r.get('M',1),'N':r.get('N','어문'),'O':r.get('O','미부착')})
|
|
|
|
json.dump({'rows':rows,'errs':errs}, open(r'D:\01.프로젝트\DB수집\작업파일\_스크립트\_jangsu_rows.json','w',encoding='utf-8'), ensure_ascii=False, indent=1)
|
|
from collections import Counter
|
|
print(f'총 행 {len(rows)} | L분포 {dict(Counter(x["L"] for x in rows))} | 탭전개 소분류 {sum(1 for r in res.values() if r.get("taps"))} | 에러 {len(errs)}')
|
|
print('saved _jangsu_rows.json')
|