df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
108 lines
5.1 KiB
Python
108 lines
5.1 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""Python Playwright로 공공기관2 정크사이트 GNB 권위트리 추출 → _gnb\{num}.json.
|
|
컨테이너 자동탐지(계층 앵커 최다) + 정크라벨 필터 + 스플래시 진입.
|
|
사용: python _gnb_extract.py [num ...]"""
|
|
import sys, io, os, json, re, warnings
|
|
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
|
|
warnings.filterwarnings('ignore')
|
|
from playwright.sync_api import sync_playwright
|
|
|
|
GNB_DIR = r"D:\01.프로젝트\DB수집\작업파일\공공기관2\_gnb"
|
|
|
|
# num: (name, base, entry_url, container_sel(None=auto))
|
|
SITES = {
|
|
1: ("한국전기안전공사", "https://www.kesco.or.kr", "https://www.kesco.or.kr/main/main.do", "ul#gnb"),
|
|
6: ("한국중부발전", "https://www.komipo.co.kr", "https://www.komipo.co.kr/kor/main/main.do", "#gnbmenu"),
|
|
7: ("한국지능정보사회진흥원", "https://www.nia.or.kr", "https://www.nia.or.kr/site/nia_kor/main.do", ".gnb_area"),
|
|
8: ("한국지식재산보호원", "https://www.koipa.re.kr", "https://www.koipa.re.kr/home/main.do", "#gnb"),
|
|
10: ("한국지역난방공사", "https://www.kdhc.co.kr", "https://www.kdhc.co.kr/", "#gnb"),
|
|
11: ("한국직업능력연구원", "https://www.krivet.re.kr", "https://www.krivet.re.kr/", "#k_gnb"),
|
|
13: ("한국청소년상담복지개발원", "https://www.kyci.or.kr", "https://www.kyci.or.kr/userSite/index.asp", ".mb_gnb"),
|
|
15: ("한국청소년활동진흥원", "https://www.kywa.or.kr", "https://www.kywa.or.kr/main/main.jsp", ".gnb ul.one_depth"),
|
|
16: ("한국체육산업개발", "https://www.ksponco.or.kr", "https://www.ksponco.or.kr/", "#gnb"),
|
|
}
|
|
|
|
JUNK = re.compile(r'바로가기|닫기|열기|검색|전체\s*메뉴|메뉴\s*보기|메인메뉴|서브메뉴|본문\s*바로|주\s*메뉴|TOP|위로|맨위|로그인|로그아웃|회원가입|마이페이지|언어|english|日本|中文|sitemap|즐겨찾기|새창|팝업|로고|이동$|^\s*$', re.I)
|
|
|
|
EXTRACT_JS = """
|
|
(containerSel) => {
|
|
const cont = containerSel ? document.querySelector(containerSel) : null;
|
|
if(!cont) return {err:'no container', rows:[]};
|
|
function labelA(li){
|
|
for(const a of li.querySelectorAll('a, button')){
|
|
let p=a.parentElement, inside=false;
|
|
while(p&&p!==li){ if(p.tagName==='UL'){inside=true;break;} p=p.parentElement; }
|
|
if(inside) continue;
|
|
const t=(a.textContent||'').trim();
|
|
if(t) return a; // 빈 텍스트(아이콘링크) 건너뜀
|
|
}
|
|
return null;
|
|
}
|
|
function topUls(el){
|
|
return [...el.querySelectorAll('ul')].filter(ul=>{
|
|
let p=ul.parentElement;
|
|
while(p&&p!==el){ if(p.tagName==='UL') return false; p=p.parentElement; }
|
|
return true;
|
|
});
|
|
}
|
|
const rows=[];
|
|
function walk(ul, depth){
|
|
[...ul.children].filter(li=>li.tagName==='LI').forEach(li=>{
|
|
const a=labelA(li);
|
|
if(a){
|
|
const label=(a.textContent||'').trim().replace(/\\s+/g,' ');
|
|
const href=a.tagName==='A'?(a.href||''):'';
|
|
if(label) rows.push({depth, label, href});
|
|
}
|
|
topUls(li).forEach(sub=>walk(sub, depth+1));
|
|
});
|
|
}
|
|
const starts = cont.tagName==='UL' ? [cont] : topUls(cont);
|
|
starts.forEach(u=>walk(u, 1));
|
|
return {sel: containerSel, rows};
|
|
}
|
|
"""
|
|
|
|
def clean_rows(rows):
|
|
out=[]
|
|
for r in rows:
|
|
lab=r['label']
|
|
if JUNK.search(lab): continue
|
|
if len(lab)>60: continue
|
|
h=r['href'] or ''
|
|
if h.startswith('javascript') or h.endswith('#') or h=='':
|
|
# 링크없는 카테고리노드는 유지(라벨만), 단 정크 아닐때
|
|
pass
|
|
out.append({'depth':min(r['depth'],5), 'label':lab, 'href':h})
|
|
# depth 정규화: 최소깊이를 1로
|
|
if out:
|
|
md=min(r['depth'] for r in out)
|
|
if md>1:
|
|
for r in out: r['depth']-=(md-1)
|
|
return out
|
|
|
|
def run(nums):
|
|
with sync_playwright() as p:
|
|
br=p.chromium.launch(headless=True)
|
|
ctx=br.new_context(ignore_https_errors=True, viewport={'width':1920,'height':1080}, user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120 Safari/537.36')
|
|
pg=ctx.new_page()
|
|
for num in nums:
|
|
name, base, url, sel = SITES[num]
|
|
try:
|
|
pg.goto(url, wait_until='domcontentloaded', timeout=30000)
|
|
pg.wait_for_timeout(2500)
|
|
res = pg.evaluate(EXTRACT_JS, sel)
|
|
rows = clean_rows(res.get('rows', []))
|
|
d1 = [r['label'] for r in rows if r['depth']==1]
|
|
data={'num':num,'name':name,'base':base,'gnb':rows}
|
|
with open(os.path.join(GNB_DIR, f'{num}.json'),'w',encoding='utf-8') as f:
|
|
json.dump(data,f,ensure_ascii=False,indent=1)
|
|
print(f"[{num}.{name}] sel={res.get('sel')} rows={len(rows)} d1={len(d1)} :: {d1[:12]}")
|
|
except Exception as e:
|
|
print(f"[{num}.{name}] 실패: {type(e).__name__}: {e}")
|
|
br.close()
|
|
|
|
if __name__=='__main__':
|
|
nums=[int(x) for x in sys.argv[1:]] or list(SITES.keys())
|
|
run(nums)
|