df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
43 lines
2.2 KiB
Python
43 lines
2.2 KiB
Python
# -*- coding: utf-8 -*-
|
|
# 영동 트리: .sitemap>dl>dt(대) / dd>b>a(중) / dd>ul>li>a(소) / li>ul.s_4th_ul>li>a(세부). 레거시SSL. _blank→사이트힌트
|
|
import sys,io,json,urllib.request,ssl
|
|
from bs4 import BeautifulSoup
|
|
sys.stdout=io.TextIOWrapper(sys.stdout.buffer,encoding='utf-8')
|
|
ctx=ssl.create_default_context(); ctx.check_hostname=False; ctx.verify_mode=ssl.CERT_NONE
|
|
try: ctx.set_ciphers('DEFAULT@SECLEVEL=1')
|
|
except: pass
|
|
def get(u):
|
|
req=urllib.request.Request(u,headers={'User-Agent':'Mozilla/5.0'})
|
|
return urllib.request.urlopen(req,timeout=25,context=ctx).read().decode('utf-8','ignore')
|
|
def txt(a): return a.get_text(strip=True) if a else ''
|
|
def khint(a):
|
|
if not a: return '페이지'
|
|
tg=(a.get('target') or ''); ti=(a.get('title') or ''); href=a.get('href') or ''
|
|
if tg=='_blank' or '새창' in ti or '새창' in txt(a): return '사이트'
|
|
if href.startswith('http') and 'yd21.go.kr' not in href: return '사이트'
|
|
return '페이지'
|
|
h=get('https://www.yd21.go.kr/kr/html/guide/0701.html')
|
|
s=BeautifulSoup(h,'html.parser')
|
|
sm=s.select_one('.sitemap') or s
|
|
tree=[]
|
|
for dl in sm.find_all('dl'):
|
|
dt=dl.find('dt'); d1=txt(dt)
|
|
for dd in dl.find_all('dd', recursive=False) or dl.find_all('dd'):
|
|
b=dd.find('b'); a2=(b.find('a') if b else None) or dd.find('a'); d2=txt(a2)
|
|
ul=None
|
|
for u in dd.find_all('ul', recursive=False): ul=u; break
|
|
if not ul:
|
|
tree.append([d1,d2,'','',a2.get('href') if a2 else '',khint(a2)]); continue
|
|
for li in ul.find_all('li', recursive=False):
|
|
a3=li.find('a', recursive=False) or li.find('a'); d3=txt(a3)
|
|
sub4=li.find('ul', class_='s_4th_ul')
|
|
if not sub4:
|
|
tree.append([d1,d2,d3,'',a3.get('href'),khint(a3)])
|
|
else:
|
|
for li4 in sub4.find_all('li', recursive=False):
|
|
a4=li4.find('a'); tree.append([d1,d2,d3,txt(a4),a4.get('href'),khint(a4)])
|
|
json.dump(tree,open('_yd_tree.json','w',encoding='utf-8'),ensure_ascii=False)
|
|
from collections import Counter
|
|
print('영동 트리노드',len(tree),'| khint',dict(Counter(t[5] for t in tree)))
|
|
for t in tree[:5]: print(' ',t[0],'>',t[1],'>',t[2],'>',t[3],'|',(t[4] or '')[:32],'|',t[5])
|