DB_JOB/작업파일/공공기관3/_gnb_extract.py

# -*- coding: utf-8 -*-
"""공공기관3 GNB 권위트리 추출 (메가메뉴 펼침 + 자동 컨테이너 선택) → _gnb/{num}.json
사용: python _gnb_extract.py [num ...]"""
import sys, io, os, json, re, warnings
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
warnings.filterwarnings('ignore')
from playwright.sync_api import sync_playwright

GNB_DIR = r"D:\01.프로젝트\DB수집\작업파일\공공기관3\_gnb"
os.makedirs(GNB_DIR, exist_ok=True)

# num: (name, base, entry_url, container_sel(None=auto), toggle_sel(None))
SITES = {
    1:  ("한국원자력환경공단",     "https://www.korad.or.kr", "https://www.korad.or.kr/", None, 'a:has-text("전체 항목 보기")'),
    2:  ("한국의료분쟁조정중재원", "https://www.k-medi.or.kr", "https://www.k-medi.or.kr/", ".gnb", None),
    3:  ("한국의약품안전관리원",   "https://www.drugsafe.or.kr", "https://www.drugsafe.or.kr/", ".allmenu", None),
    4:  ("한국인터넷진흥원",       "https://www.kisa.or.kr", "https://www.kisa.or.kr/", "#gnb", None),
    5:  ("한국임업진흥원",         "https://www.kofpi.or.kr", "https://www.kofpi.or.kr/", None, None),
    6:  ("한국자산관리공사",       "https://www.kamco.or.kr", "https://www.kamco.or.kr/main.do", "header nav", None),
    7:  ("한국잡월드",             "https://www.koreajobworld.or.kr", "https://www.koreajobworld.or.kr/", None, None),
    8:  ("한국장기조직기증원",     "https://www.koda1458.kr", "https://www.koda1458.kr/", ".header-menu", 'button:has-text("전체메뉴")'),
    9:  ("한국장애인개발원",       "https://www.koddi.or.kr", "https://www.koddi.or.kr/", ".gnb", None),
    10: ("한국장애인고용공단",     "https://www.kead.or.kr", "https://www.kead.or.kr/", None, None),
    11: ("한국장학재단",           "https://www.kosaf.go.kr", "https://www.kosaf.go.kr/ko/main.do", None, None),
    12: ("한국재정정보원",         "https://www.fis.kr", "https://www.fis.kr/", None, None),
    13: ("한국저작권보호원",       "https://www.kcopa.or.kr", "https://www.kcopa.or.kr/", None, None),
}

JUNK = re.compile(r'^(바로가기|닫기|열기|검색|전체\s*메뉴|전체메뉴|메뉴\s*보기|메뉴|메인메뉴|서브메뉴|본문\s*바로|주\s*메뉴|TOP|위로|맨위|로그인|로그아웃|회원가입|마이페이지|언어선택|언어|english|eng|日本|中文|sitemap|사이트맵|즐겨찾기|새창|팝업|로고|이동|home|홈)\s*$', re.I)

# 자동 컨테이너 후보
CAND_SELS = ["#gnb","ul#gnb","#gnbmenu",".gnb",".gnb_area",".gnb_wrap","nav .gnb",
             ".allmenu",".all_menu","#allMenu",".total_menu",".totalmenu",".menu_all",
             "#header nav ul","header nav","#nav",".nav","ul.depth1","ul.dep1",".gnb ul",
             ".lnb_all",".navi",".navbar","#snb",".menu_wrap",".header_menu","#headerWrap nav"]

EXTRACT_JS = r"""
(args) => {
  const {sels, candSels} = args;
  function topUls(el){
    return [...el.querySelectorAll('ul')].filter(ul=>{
      let p=ul.parentElement;
      while(p&&p!==el){ if(p.tagName==='UL') return false; p=p.parentElement; }
      return true; });
  }
  function labelA(li){
    for(const a of li.querySelectorAll('a, button')){
      let p=a.parentElement, inside=false;
      while(p&&p!==li){ if(p.tagName==='UL'){inside=true;break;} p=p.parentElement; }
      if(inside) continue;
      const t=(a.textContent||'').trim();
      if(t) return a;
    }
    return null;
  }
  function extractFrom(cont){
    const rows=[];
    function walk(ul, depth){
      [...ul.children].filter(li=>li.tagName==='LI').forEach(li=>{
        const a=labelA(li);
        if(a){
          const label=(a.textContent||'').trim().replace(/\s+/g,' ');
          const href=a.tagName==='A'?(a.href||''):'';
          if(label) rows.push({depth, label, href});
        }
        topUls(li).forEach(sub=>walk(sub, depth+1));
      });
    }
    (cont.tagName==='UL'?[cont]:topUls(cont)).forEach(u=>walk(u,1));
    return rows;
  }
  // 컨테이너 선택: 지정 sels 우선, 없으면 후보 중 행수 최다
  let chosen=null, chosenSel=null, best=-1;
  const trySels = sels && sels.length ? sels : candSels;
  for(const sel of trySels){
    document.querySelectorAll(sel).forEach(el=>{
      const r=extractFrom(el);
      const score=r.length*10 + Math.max(...r.map(x=>x.depth),0);
      if(r.length>=3 && score>best){ best=score; chosen=el; chosenSel=sel; }
    });
    if(chosen && sels && sels.length) break;
  }
  if(!chosen) return {err:'no container', rows:[], sel:null};
  return {sel: chosenSel, rows: extractFrom(chosen)};
}
"""

def clean_rows(rows):
    out=[]
    for r in rows:
        lab=r['label']
        if JUNK.search(lab): continue
        if len(lab)>60: continue
        out.append({'depth':min(r['depth'],5),'label':lab,'href':r['href'] or ''})
    if out:
        md=min(r['depth'] for r in out)
        if md>1:
            for r in out: r['depth']-=(md-1)
    return out

def run(nums):
    with sync_playwright() as p:
        br=p.chromium.launch(headless=True)
        ctx=br.new_context(ignore_https_errors=True, viewport={'width':1920,'height':1080},
                           user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120 Safari/537.36')
        for num in nums:
            name, base, url, sel, toggle = SITES[num]
            pg=ctx.new_page()
            try:
                pg.goto(url, wait_until='domcontentloaded', timeout=40000)
                pg.wait_for_timeout(2500)
                # 전체메뉴 토글 시도 (있으면 펼침)
                for tsel in ([toggle] if toggle else
                             ['button:has-text("전체메뉴")','a:has-text("전체메뉴")','.btn-allmenu','.allmenu_btn','#allmenuBtn','.gnb_all','.btn_total','button[class*=all]','button[class*=menu]']):
                    try:
                        el=pg.query_selector(tsel)
                        if el:
                            el.click(timeout=2000); pg.wait_for_timeout(1200); break
                    except Exception:
                        pass
                # 메가메뉴 hover로 펼침(top li들에 마우스)
                try:
                    tops=pg.query_selector_all((sel or 'nav')+' > ul > li, '+(sel or 'nav')+' li')
                    for el in tops[:12]:
                        try: el.hover(timeout=500); pg.wait_for_timeout(120)
                        except Exception: pass
                except Exception: pass
                res=pg.evaluate(EXTRACT_JS, {'sels':[sel] if sel else [], 'candSels':CAND_SELS})
                rows=clean_rows(res.get('rows',[]))
                d1=[r['label'] for r in rows if r['depth']==1]
                data={'num':num,'name':name,'base':base,'sel':res.get('sel'),'gnb':rows}
                json.dump(data, open(os.path.join(GNB_DIR,f'{num}.json'),'w',encoding='utf-8'), ensure_ascii=False, indent=1)
                print(f"[{num:2}.{name}] sel={res.get('sel')} rows={len(rows)} d1={len(d1)} :: {d1[:10]}")
            except Exception as e:
                print(f"[{num:2}.{name}] 실패: {type(e).__name__}: {e}")
            pg.close()
        br.close()

if __name__=='__main__':
    nums=[int(x) for x in sys.argv[1:]] or list(SITES.keys())
    run(nums)