df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
61 lines
2.7 KiB
Python
61 lines
2.7 KiB
Python
# -*- coding: utf-8 -*-
|
|
# 진천 트리: div.sitemap 재귀(li>a + div>ul>li>a 중첩). 플레이스홀더 제외. _blank→사이트힌트. href '/home/' 접두.
|
|
import sys,io,json,urllib.request,ssl
|
|
from bs4 import BeautifulSoup
|
|
sys.stdout=io.TextIOWrapper(sys.stdout.buffer,encoding='utf-8')
|
|
ctx=ssl.create_default_context(); ctx.check_hostname=False; ctx.verify_mode=ssl.CERT_NONE
|
|
def get(u):
|
|
req=urllib.request.Request(u,headers={'User-Agent':'Mozilla/5.0'})
|
|
return urllib.request.urlopen(req,timeout=25,context=ctx).read().decode('utf-8','ignore')
|
|
def txt(a): return a.get_text(strip=True) if a else ''
|
|
def fix(href):
|
|
if not href: return ''
|
|
if href.startswith('http'): return href
|
|
if href.startswith('/'): return href
|
|
if href.startswith('sub.do') or href.startswith('intro.do'): return '/home/'+href
|
|
return href
|
|
def khint(a):
|
|
if not a: return '페이지'
|
|
tg=(a.get('target') or ''); ti=(a.get('title') or ''); href=a.get('href') or ''
|
|
if tg=='_blank' or '새창' in ti or '새창' in txt(a): return '사이트'
|
|
if href.startswith('http') and 'jincheon.go.kr' not in href: return '사이트'
|
|
return '페이지'
|
|
def is_placeholder(a):
|
|
if not a: return True
|
|
lab=txt(a); oc=(a.get('onclick') or '')+(a.get('href') or '')
|
|
return lab in ('','메뉴명이 없습니다') or 'javascript:alert' in oc or "준비중" in oc
|
|
h=get('https://www.jincheon.go.kr/home/sub.do?menukey=445')
|
|
s=BeautifulSoup(h,'html.parser')
|
|
root=s.select_one('div.sitemap')
|
|
tree=[]
|
|
def child_ul(li):
|
|
d=li.find('div', recursive=False)
|
|
return d.find('ul', recursive=False) if d else None
|
|
def walk(lis, prefix):
|
|
for li in lis:
|
|
a=li.find('a', recursive=False)
|
|
if not a: continue
|
|
if is_placeholder(a):
|
|
continue
|
|
path=prefix+[txt(a)]
|
|
cul=child_ul(li)
|
|
real=[]
|
|
if cul:
|
|
for cli in cul.find_all('li', recursive=False):
|
|
ca=cli.find('a', recursive=False)
|
|
if ca and not is_placeholder(ca): real.append(cli)
|
|
if real:
|
|
walk(real, path)
|
|
else:
|
|
d1=path[0] if len(path)>0 else ''
|
|
d2=path[1] if len(path)>1 else ''
|
|
d3=path[2] if len(path)>2 else ''
|
|
d4=path[3] if len(path)>3 else ''
|
|
tree.append([d1,d2,d3,d4,fix(a.get('href')),khint(a)])
|
|
topul=root.find('ul', recursive=False) or root.find('ul')
|
|
walk(topul.find_all('li', recursive=False), [])
|
|
json.dump(tree,open('_jn_tree.json','w',encoding='utf-8'),ensure_ascii=False)
|
|
from collections import Counter
|
|
print('진천 트리노드',len(tree),'| khint',dict(Counter(t[5] for t in tree)))
|
|
for t in tree[:6]: print(' ',t[0],'>',t[1],'>',t[2],'>',t[3],'|',(t[4] or '')[:32],'|',t[5])
|