df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
114 lines
5.4 KiB
Python
114 lines
5.4 KiB
Python
# -*- coding: utf-8 -*-
|
||
# 트리(JSON: [[d1,d2,d3,d4,url],...]) → classify+본문탭전개 → nodes.json
|
||
# 사용: python _cb_nodes.py <host> <tree.json> <out_nodes.json>
|
||
import sys,io,json,re,time,urllib.request,ssl
|
||
from bs4 import BeautifulSoup
|
||
sys.stdout=io.TextIOWrapper(sys.stdout.buffer,encoding='utf-8')
|
||
ctx=ssl.create_default_context(); ctx.check_hostname=False; ctx.verify_mode=ssl.CERT_NONE
|
||
try: ctx.set_ciphers('DEFAULT@SECLEVEL=1')
|
||
except Exception: pass
|
||
HOST=sys.argv[1]; TREE=sys.argv[2]; OUT=sys.argv[3]
|
||
BASE='https://'+HOST
|
||
def absu(u):
|
||
if not u: return u
|
||
if u.startswith('http'): return u
|
||
if u.startswith('/'): return BASE+u
|
||
return BASE+'/'+u
|
||
_cache={}
|
||
def get(u):
|
||
if u in _cache: return _cache[u]
|
||
h=''
|
||
for _ in range(2):
|
||
try:
|
||
req=urllib.request.Request(u,headers={'User-Agent':'Mozilla/5.0'})
|
||
h=urllib.request.urlopen(req,timeout=25,context=ctx).read().decode('utf-8','ignore'); break
|
||
except Exception: time.sleep(1)
|
||
_cache[u]=h; return h
|
||
def kind(u):
|
||
if not u: return 'none'
|
||
if u.startswith('http') and HOST not in u: return '사이트'
|
||
if 'eminwon' in u: return '사이트'
|
||
if re.search(r'selectBbsNttList|selectOthbcInfoList|selectBoardList|/cop/bbs/|BBSMSTR|BbsNtt',u): return '게시판'
|
||
return '페이지'
|
||
def board_M(html):
|
||
m=re.search(r'게시물[^\d]{0,15}([\d,]+)', html)
|
||
if m:
|
||
try: return int(m.group(1).replace(',',''))
|
||
except: return 0
|
||
return 0
|
||
def has_opentype(html):
|
||
mm=re.findall(r'img_opentype0?(\d)', html); return sorted(set(int(x) for x in mm)) if mm else []
|
||
def board_signal(html):
|
||
# 내용기반 게시판 판별: 총게시물 카운트 + 등록일/작성일/조회수 동시존재
|
||
a=bool(re.search(r'총\s*게시물|게시물\s*[:：]?\s*\d|총\s*\d+\s*건',html))
|
||
b=bool(re.search(r'등록일|작성일|조회수',html))
|
||
return a and b
|
||
def page_tabs(html, self_url):
|
||
s=BeautifulSoup(html,'html.parser')
|
||
cont=s.select_one('#contents,.contents,.sub_content,.sub_contents,#content,.content_in') or s
|
||
selfk=re.search(r'key=(\d+)',self_url); selfk=selfk.group(1) if selfk else None
|
||
if not selfk: return []
|
||
for ul in cont.find_all('ul'):
|
||
cls=' '.join(ul.get('class') or [])
|
||
if not re.search(r'tab',cls,re.I): continue
|
||
anchors=ul.find_all('a')
|
||
if len(anchors)<2: continue
|
||
keys=[re.search(r'key=(\d+)',a.get('href') or '') for a in anchors]
|
||
keys=[k.group(1) for k in keys if k]
|
||
if selfk not in keys: continue
|
||
if len(keys)<len(anchors)-1: continue
|
||
out=[]
|
||
for a in anchors:
|
||
lab=a.get_text(strip=True); href=a.get('href'); k=re.search(r'key=(\d+)',href or '')
|
||
if not lab or not k or k.group(1)==selfk: continue
|
||
out.append((lab,href))
|
||
if out: return out
|
||
return []
|
||
def classify(u):
|
||
u=absu(u); k=kind(u); M=None; O=[]; det=None
|
||
if k=='게시판':
|
||
hh=get(u); M=board_M(hh); O=has_opentype(hh)
|
||
ss=BeautifulSoup(hh,'html.parser')
|
||
for a in ss.find_all('a'):
|
||
href=a.get('href') or ''
|
||
if re.search(r'NttView|nttNo=|addBbsNttView|selectBoardArticle|nttId=',href): det=absu(href); break
|
||
elif k=='페이지':
|
||
hh=get(u); O=has_opentype(hh)
|
||
if board_signal(hh): # 내용기반 게시판 승격
|
||
k='게시판'; M=board_M(hh)
|
||
ss=BeautifulSoup(hh,'html.parser')
|
||
for a in ss.find_all('a'):
|
||
href=a.get('href') or ''
|
||
if re.search(r'NttView|nttNo=|addBbsNttView|selectBoardArticle|nttId=|mode=view|articleNo=|idx=|seq=',href): det=absu(href); break
|
||
return k,M,O,det
|
||
tree=json.load(open(TREE,encoding='utf-8'))
|
||
print('트리노드',len(tree),flush=True)
|
||
nodes=[]; menu_urls=set(absu(t[4]) for t in tree if t[4]); seen_tab=set()
|
||
for i,row in enumerate(tree):
|
||
d1,d2,d3,d4,url = row[0],row[1],row[2],row[3],row[4]
|
||
khint = row[5] if len(row)>5 else None
|
||
u=absu(url); k,M,O,det=classify(url)
|
||
if khint and khint!='페이지' and k=='페이지': # URL로 페이지 판정됐지만 트리힌트가 게시판/사이트면 힌트 우선
|
||
k=khint
|
||
if k=='게시판':
|
||
hh=get(u); M=board_M(hh); O=has_opentype(hh)
|
||
ss=BeautifulSoup(hh,'html.parser')
|
||
for a in ss.find_all('a'):
|
||
href=a.get('href') or ''
|
||
if re.search(r'NttView|nttNo=|addBbsNttView|selectBoardArticle|nttId=|view',href): det=absu(href); break
|
||
if k=='none' and not u: continue
|
||
nodes.append({'d':[d1,d2,d3,d4,''],'url':u,'kind':k,'M':M,'O':O,'det':det})
|
||
if k=='페이지' and re.search(r'(contents|sub)\.do',u):
|
||
for lab,href in page_tabs(get(u),u):
|
||
tu=absu(href)
|
||
if tu in menu_urls or tu in seen_tab: continue
|
||
seen_tab.add(tu); tk,tM,tO,tdet=classify(href)
|
||
dd=[d1,d2,d3,d4,'']; depth=max((idx for idx in range(4) if dd[idx]),default=0)
|
||
child=dd[:]
|
||
child[4 if depth>=3 else depth+1]=lab
|
||
nodes.append({'d':child,'url':tu,'kind':tk,'M':tM,'O':tO,'det':tdet,'tab':True})
|
||
if i%25==0: print('%d/%d'%(i+1,len(tree)),flush=True)
|
||
json.dump(nodes,open(OUT,'w',encoding='utf-8'),ensure_ascii=False)
|
||
from collections import Counter
|
||
print('총행',len(nodes),'| L',dict(Counter(n['kind'] for n in nodes)),'| 탭',sum(1 for n in nodes if n.get('tab')),'| O',sum(1 for n in nodes if n['O']))
|