df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
98 lines
3.9 KiB
Python
98 lines
3.9 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""전력거래소: ①base GNB(ul#head_menu_all) 재추출 ②본문 ul#depth4_menu_ul 전수스캔
|
|
③member-set dedup·self탭 노드에 children 전개 → _gnb\2.json."""
|
|
import sys, io, os, json, re
|
|
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
|
|
from playwright.sync_api import sync_playwright
|
|
|
|
GNB = r"D:\01.프로젝트\DB수집\작업파일\공공기관2\_gnb\2.json"
|
|
|
|
def midkey(h):
|
|
m = re.search(r'mid=([a-z0-9]+)', h or '')
|
|
return m.group(1) if m else (h or '')
|
|
|
|
BASE_JS = """
|
|
() => {
|
|
const root=document.querySelector('#head_menu_all')||document.querySelector('#head_menu');
|
|
const rows=[];
|
|
function walk(ul,depth){[...ul.children].filter(li=>li.tagName==='LI').forEach(li=>{
|
|
let a=li.querySelector(':scope>a,:scope>div>a,:scope>span>a,:scope>button');
|
|
let l=a?(a.textContent||'').trim().replace(/\\s+/g,' '):'';
|
|
let h=a&&a.tagName==='A'?a.href:'';
|
|
if(l)rows.push({depth,label:l,href:h});
|
|
li.querySelectorAll(':scope>ul,:scope>div>ul').forEach(s=>walk(s,depth+1));});}
|
|
walk(root,1); return rows;
|
|
}
|
|
"""
|
|
D4_JS = """
|
|
() => { const ul=document.querySelector('#depth4_menu_ul,ul#depth4_menu_ul'); if(!ul) return [];
|
|
return [...ul.querySelectorAll(':scope>li>a')].map(a=>({label:(a.textContent||'').trim().replace(/\\s+/g,' '),href:a.href})); }
|
|
"""
|
|
|
|
with sync_playwright() as p:
|
|
br=p.chromium.launch(headless=True)
|
|
pg=br.new_context(ignore_https_errors=True,viewport={'width':1600,'height':1000}).new_page()
|
|
# ① base GNB
|
|
pg.goto('https://www.kpx.or.kr/main/',wait_until='domcontentloaded',timeout=30000); pg.wait_for_timeout(2500)
|
|
base=pg.evaluate(BASE_JS)
|
|
print('base GNB',len(base),'행')
|
|
# 고유 kpx 페이지
|
|
seen=set(); targets=[]
|
|
for r in base:
|
|
h=r.get('href') or ''
|
|
if 'kpx.or.kr' in h and 'mid=' in h:
|
|
k=midkey(h)
|
|
if k not in seen: seen.add(k); targets.append((k,h))
|
|
# ② depth4 스캔
|
|
page_groups={}
|
|
for i,(k,h) in enumerate(targets):
|
|
try:
|
|
pg.goto(h,wait_until='domcontentloaded',timeout=20000); pg.wait_for_timeout(600)
|
|
d4=pg.evaluate(D4_JS)
|
|
except Exception: d4=[]
|
|
if len(d4)>=2: page_groups[k]=d4
|
|
if (i+1)%30==0: print(f' ...{i+1}/{len(targets)} (탭 {len(page_groups)})')
|
|
br.close()
|
|
|
|
print(f'depth4 페이지 {len(page_groups)}개')
|
|
# ③ member-set dedup
|
|
uniq={} # frozenset(member mids) -> items
|
|
for k,items in page_groups.items():
|
|
key=frozenset(midkey(it['href']) for it in items)
|
|
if key not in uniq: uniq[key]=items
|
|
print(f'고유 그룹 {len(uniq)}개')
|
|
|
|
# base 행 mid 집합
|
|
base_mids={midkey(r.get('href','')) for r in base}
|
|
# 각 고유그룹의 부모 결정: 멤버 중 base에 있는 노드(self탭) → 그 노드; 없으면 page_groups에서 base에 있는 페이지
|
|
def parent_mid_for(memberset, items):
|
|
inb=[m for m in memberset if m in base_mids]
|
|
if inb: return inb[0] # self탭(멤버가 GNB노드)
|
|
# fallback: 이 그룹을 보여준 페이지 중 base에 있는 것
|
|
for k in page_groups:
|
|
if frozenset(midkey(it['href']) for it in page_groups[k])==memberset and k in base_mids:
|
|
return k
|
|
return None
|
|
|
|
# 부모mid -> 전개할 items
|
|
expand={}
|
|
for key,items in uniq.items():
|
|
pm=parent_mid_for(key,items)
|
|
if pm and pm not in expand:
|
|
expand[pm]=items
|
|
|
|
new=[]; done=set()
|
|
for r in base:
|
|
new.append(r)
|
|
k=midkey(r.get('href',''))
|
|
if k in expand and k not in done:
|
|
done.add(k)
|
|
for it in expand[k]:
|
|
new.append({'depth':r['depth']+1,'label':it['label'],'href':it['href']})
|
|
|
|
data={'num':2,'name':'한국전력거래소','base':'https://www.kpx.or.kr','gnb':new}
|
|
json.dump(data,open(GNB,'w',encoding='utf-8'),ensure_ascii=False,indent=1)
|
|
print(f'전개: {len(expand)}그룹 → 총 {len(new)}행 (base {len(base)})')
|
|
for pm,items in expand.items():
|
|
print(f' [{pm}] +{len(items)}: {", ".join(x["label"][:12] for x in items)}')
|