df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
51 lines
2.8 KiB
Python
51 lines
2.8 KiB
Python
# -*- coding: utf-8 -*-
|
|
# NHIS head-gnb 4단 트리(대>중>소>세부) 추출 → url2path 저장. depth4-ul 조상추적.
|
|
import urllib.request,ssl,http.cookiejar,re,json,os
|
|
from bs4 import BeautifulSoup
|
|
from urllib.parse import urljoin,urlparse,parse_qs
|
|
ctx=ssl.create_default_context(); ctx.check_hostname=False; ctx.verify_mode=ssl.CERT_NONE
|
|
cj=http.cookiejar.CookieJar(); op=urllib.request.build_opener(urllib.request.HTTPSHandler(context=ctx),urllib.request.HTTPCookieProcessor(cj))
|
|
op.addheaders=[('User-Agent','Mozilla/5.0')]
|
|
BASE='https://www.nhis.or.kr'
|
|
def norm(u):
|
|
if not u or str(u).startswith(('javascript','#')): return None
|
|
full=u if u.startswith('http') else urljoin(BASE+'/',u)
|
|
p=urlparse(full); q=parse_qs(p.query)
|
|
items=sorted((k,','.join(sorted(v))) for k,v in q.items())
|
|
return (p.netloc+p.path+('?'+'&'.join(f'{k}={v}' for k,v in items) if items else '')).rstrip('/')
|
|
def txt(el): return re.sub(r'\s+',' ',el.get_text(' ',strip=True)).strip() if el else ''
|
|
h=op.open(BASE+'/nhis/healthin/wbhacf03110m01.do',timeout=25).read().decode('utf-8','ignore')
|
|
s=BeautifulSoup(h,'html.parser')
|
|
gnb=s.select_one('nav.head-gnb ul.gnb') or s.select_one('ul.gnb')
|
|
url2path={}
|
|
def add(path,u):
|
|
nu=norm(u)
|
|
if nu and nu not in url2path: url2path[nu]=[x for x in path if x]
|
|
# 대/중/소 (sub-ul 직계 a.subm)
|
|
for li1 in gnb.find_all('li',class_='li',recursive=False):
|
|
btn=li1.find('button',class_='section') or li1.find('button')
|
|
d1=re.sub(r'\s*열기\s*$','',(btn.get('title') or txt(btn)).strip()) if btn else ''
|
|
if not d1: continue
|
|
for panel in li1.select('div.submenu-wrap div.submenu-in'):
|
|
h3a=panel.select_one('h3.sub-tit a')
|
|
if not h3a: continue
|
|
d2=txt(h3a) or (h3a.get('title') or ''); add([d1,d2],h3a.get('href'))
|
|
subul=panel.select_one('ul.sub-ul')
|
|
if not subul: continue
|
|
for li3 in subul.find_all('li',recursive=False):
|
|
a3=li3.find('a',class_='subm',recursive=False) or li3.find('a',href=True,recursive=False)
|
|
d3=(txt(a3) or (a3.get('title') if a3 else '')) if a3 else ''
|
|
if a3: add([d1,d2,d3],a3.get('href'))
|
|
# depth4-wrap 세부
|
|
for ul4 in li3.select('div.depth4-wrap ul.depth4-ul'):
|
|
for a4 in ul4.find_all('a',href=True):
|
|
d4=txt(a4) or (a4.get('title') or '')
|
|
add([d1,d2,d3,d4],a4.get('href'))
|
|
from collections import Counter
|
|
print('총 url',len(url2path),'깊이분포',dict(Counter(len(p) for p in url2path.values())))
|
|
print('평가정보 4단:')
|
|
for u,p in url2path.items():
|
|
if p and '평가정보' in p[-1]: print(' ',p,u)
|
|
json.dump(url2path,open(os.path.join(os.path.dirname(os.path.abspath(__file__)),'_nhis_d4map.json'),'w',encoding='utf-8'),ensure_ascii=False,indent=1)
|
|
print('saved _nhis_d4map.json')
|