# -*- coding: utf-8 -*- """공공기관3 GNB 권위트리 추출 (메가메뉴 펼침 + 자동 컨테이너 선택) → _gnb/{num}.json 사용: python _gnb_extract.py [num ...]""" import sys, io, os, json, re, warnings sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8') warnings.filterwarnings('ignore') from playwright.sync_api import sync_playwright GNB_DIR = r"D:\01.프로젝트\DB수집\작업파일\공공기관3\_gnb" os.makedirs(GNB_DIR, exist_ok=True) # num: (name, base, entry_url, container_sel(None=auto), toggle_sel(None)) SITES = { 1: ("한국원자력환경공단", "https://www.korad.or.kr", "https://www.korad.or.kr/", None, 'a:has-text("전체 항목 보기")'), 2: ("한국의료분쟁조정중재원", "https://www.k-medi.or.kr", "https://www.k-medi.or.kr/", ".gnb", None), 3: ("한국의약품안전관리원", "https://www.drugsafe.or.kr", "https://www.drugsafe.or.kr/", ".allmenu", None), 4: ("한국인터넷진흥원", "https://www.kisa.or.kr", "https://www.kisa.or.kr/", "#gnb", None), 5: ("한국임업진흥원", "https://www.kofpi.or.kr", "https://www.kofpi.or.kr/", None, None), 6: ("한국자산관리공사", "https://www.kamco.or.kr", "https://www.kamco.or.kr/main.do", "header nav", None), 7: ("한국잡월드", "https://www.koreajobworld.or.kr", "https://www.koreajobworld.or.kr/", None, None), 8: ("한국장기조직기증원", "https://www.koda1458.kr", "https://www.koda1458.kr/", ".header-menu", 'button:has-text("전체메뉴")'), 9: ("한국장애인개발원", "https://www.koddi.or.kr", "https://www.koddi.or.kr/", ".gnb", None), 10: ("한국장애인고용공단", "https://www.kead.or.kr", "https://www.kead.or.kr/", None, None), 11: ("한국장학재단", "https://www.kosaf.go.kr", "https://www.kosaf.go.kr/ko/main.do", None, None), 12: ("한국재정정보원", "https://www.fis.kr", "https://www.fis.kr/", None, None), 13: ("한국저작권보호원", "https://www.kcopa.or.kr", "https://www.kcopa.or.kr/", None, None), } JUNK = re.compile(r'^(바로가기|닫기|열기|검색|전체\s*메뉴|전체메뉴|메뉴\s*보기|메뉴|메인메뉴|서브메뉴|본문\s*바로|주\s*메뉴|TOP|위로|맨위|로그인|로그아웃|회원가입|마이페이지|언어선택|언어|english|eng|日本|中文|sitemap|사이트맵|즐겨찾기|새창|팝업|로고|이동|home|홈)\s*$', re.I) # 자동 컨테이너 후보 CAND_SELS = ["#gnb","ul#gnb","#gnbmenu",".gnb",".gnb_area",".gnb_wrap","nav .gnb", ".allmenu",".all_menu","#allMenu",".total_menu",".totalmenu",".menu_all", "#header nav ul","header nav","#nav",".nav","ul.depth1","ul.dep1",".gnb ul", ".lnb_all",".navi",".navbar","#snb",".menu_wrap",".header_menu","#headerWrap nav"] EXTRACT_JS = r""" (args) => { const {sels, candSels} = args; function topUls(el){ return [...el.querySelectorAll('ul')].filter(ul=>{ let p=ul.parentElement; while(p&&p!==el){ if(p.tagName==='UL') return false; p=p.parentElement; } return true; }); } function labelA(li){ for(const a of li.querySelectorAll('a, button')){ let p=a.parentElement, inside=false; while(p&&p!==li){ if(p.tagName==='UL'){inside=true;break;} p=p.parentElement; } if(inside) continue; const t=(a.textContent||'').trim(); if(t) return a; } return null; } function extractFrom(cont){ const rows=[]; function walk(ul, depth){ [...ul.children].filter(li=>li.tagName==='LI').forEach(li=>{ const a=labelA(li); if(a){ const label=(a.textContent||'').trim().replace(/\s+/g,' '); const href=a.tagName==='A'?(a.href||''):''; if(label) rows.push({depth, label, href}); } topUls(li).forEach(sub=>walk(sub, depth+1)); }); } (cont.tagName==='UL'?[cont]:topUls(cont)).forEach(u=>walk(u,1)); return rows; } // 컨테이너 선택: 지정 sels 우선, 없으면 후보 중 행수 최다 let chosen=null, chosenSel=null, best=-1; const trySels = sels && sels.length ? sels : candSels; for(const sel of trySels){ document.querySelectorAll(sel).forEach(el=>{ const r=extractFrom(el); const score=r.length*10 + Math.max(...r.map(x=>x.depth),0); if(r.length>=3 && score>best){ best=score; chosen=el; chosenSel=sel; } }); if(chosen && sels && sels.length) break; } if(!chosen) return {err:'no container', rows:[], sel:null}; return {sel: chosenSel, rows: extractFrom(chosen)}; } """ def clean_rows(rows): out=[] for r in rows: lab=r['label'] if JUNK.search(lab): continue if len(lab)>60: continue out.append({'depth':min(r['depth'],5),'label':lab,'href':r['href'] or ''}) if out: md=min(r['depth'] for r in out) if md>1: for r in out: r['depth']-=(md-1) return out def run(nums): with sync_playwright() as p: br=p.chromium.launch(headless=True) ctx=br.new_context(ignore_https_errors=True, viewport={'width':1920,'height':1080}, user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120 Safari/537.36') for num in nums: name, base, url, sel, toggle = SITES[num] pg=ctx.new_page() try: pg.goto(url, wait_until='domcontentloaded', timeout=40000) pg.wait_for_timeout(2500) # 전체메뉴 토글 시도 (있으면 펼침) for tsel in ([toggle] if toggle else ['button:has-text("전체메뉴")','a:has-text("전체메뉴")','.btn-allmenu','.allmenu_btn','#allmenuBtn','.gnb_all','.btn_total','button[class*=all]','button[class*=menu]']): try: el=pg.query_selector(tsel) if el: el.click(timeout=2000); pg.wait_for_timeout(1200); break except Exception: pass # 메가메뉴 hover로 펼침(top li들에 마우스) try: tops=pg.query_selector_all((sel or 'nav')+' > ul > li, '+(sel or 'nav')+' li') for el in tops[:12]: try: el.hover(timeout=500); pg.wait_for_timeout(120) except Exception: pass except Exception: pass res=pg.evaluate(EXTRACT_JS, {'sels':[sel] if sel else [], 'candSels':CAND_SELS}) rows=clean_rows(res.get('rows',[])) d1=[r['label'] for r in rows if r['depth']==1] data={'num':num,'name':name,'base':base,'sel':res.get('sel'),'gnb':rows} json.dump(data, open(os.path.join(GNB_DIR,f'{num}.json'),'w',encoding='utf-8'), ensure_ascii=False, indent=1) print(f"[{num:2}.{name}] sel={res.get('sel')} rows={len(rows)} d1={len(d1)} :: {d1[:10]}") except Exception as e: print(f"[{num:2}.{name}] 실패: {type(e).__name__}: {e}") pg.close() br.close() if __name__=='__main__': nums=[int(x) for x in sys.argv[1:]] or list(SITES.keys()) run(nums)