DB_JOB/작업파일/완료/공공기관/_nhis_d4_rebuild.py

# -*- coding: utf-8 -*-
# NHIS(23) head-gnb 4단(대>중>소>세부) 완전트리로 시트 재구성. depth4-ul(m-gnb) 확장.
# 기존 L/M/N/O/K는 URL로 보존, 신규 세부는 L크롤(게시판판별)+N어문+O미부착. 비GNB행은 꼬리유지.
import urllib.request,ssl,http.cookiejar,re,glob,os,sys,time,shutil
from bs4 import BeautifulSoup
from urllib.parse import urljoin,urlparse,parse_qs
from copy import copy
import openpyxl
from openpyxl.styles import PatternFill
ctx=ssl.create_default_context(); ctx.check_hostname=False; ctx.verify_mode=ssl.CERT_NONE
cj=http.cookiejar.CookieJar(); op=urllib.request.build_opener(urllib.request.HTTPSHandler(context=ctx),urllib.request.HTTPCookieProcessor(cj))
op.addheaders=[('User-Agent','Mozilla/5.0')]
BASE='https://www.nhis.or.kr'; BLUE=PatternFill(fill_type='solid',fgColor='FFBDD7EE'); MAXC=27
APPLY='--apply' in sys.argv
def fetch(u): return op.open(u,timeout=25).read().decode('utf-8','ignore')
def norm(u):
    if not u or str(u).startswith(('javascript','#')): return None
    full=u if str(u).startswith('http') else urljoin(BASE+'/',str(u))
    p=urlparse(full); q=parse_qs(p.query)
    it=sorted((k,','.join(sorted(v))) for k,v in q.items())
    return (p.netloc+p.path+('?'+'&'.join(f'{k}={v}' for k,v in it) if it else '')).rstrip('/')
def full(u): return u if str(u).startswith('http') else urljoin(BASE+'/',str(u))
def txt(el): return re.sub(r'\s+',' ',el.get_text(' ',strip=True)).strip() if el else ''
H=fetch(BASE+'/nhis/healthin/wbhacf03110m01.do'); s=BeautifulSoup(H,'html.parser')
gnb=s.select_one('nav.head-gnb ul.gnb') or s.select_one('ul.gnb')
# 1) head-gnb 순서대로 (대,중,소,url)
gnb_rows=[]   # (d1,d2,d3 or None, url)
for li1 in gnb.find_all('li',class_='li',recursive=False):
    btn=li1.find('button',class_='section') or li1.find('button')
    d1=re.sub(r'\s*열기\s*$','',(btn.get('title') or txt(btn)).strip()) if btn else ''
    if not d1: continue
    for panel in li1.select('div.submenu-wrap div.submenu-in'):
        h3a=panel.select_one('h3.sub-tit a')
        if not h3a: continue
        d2=txt(h3a) or h3a.get('title') or ''
        subul=panel.select_one('ul.sub-ul')
        lis=subul.find_all('li',recursive=False) if subul else []
        if not lis:
            if norm(h3a.get('href')): gnb_rows.append((d1,d2,None,full(h3a.get('href'))))
        else:
            for li3 in lis:
                a3=li3.find('a',href=True)
                if a3 and norm(a3.get('href')): gnb_rows.append((d1,d2,txt(a3) or a3.get('title'),full(a3.get('href'))))
# 2) depth4 groups: 첫자식url -> [(name,fullurl)]
d4=[]; seen=set()
for ul in s.select('ul.depth4-ul'):
    li=ul.find_parent('li'); b=li.find(['button','a']) if li else None; lab=txt(b)
    kids=[(txt(a),full(a.get('href'))) for a in ul.find_all('a',href=True) if txt(a)]
    if not kids: continue
    key=(lab,tuple(norm(k[1]) for k in kids))
    if key in seen: continue
    seen.add(key); d4.append((lab,kids))
d4_by_first={norm(kids[0][1]):kids for lab,kids in d4}      # 첫자식 url로 그룹 찾기
d4_labels={lab:kids for lab,kids in d4}
# 3) 시트 로드
d=[x for x in glob.glob('23.*') if os.path.isdir(x)][0]
xp=[p for p in glob.glob(os.path.join(d,'*.xlsx')) if not os.path.basename(p).startswith(('_','~')) and 'conflict' not in os.path.basename(p) and 'backup' not in p][0]
mt=os.path.getmtime(xp); wb=openpyxl.load_workbook(xp); ws=wb.active
def eff(r,c):
    v=ws.cell(r,c).value
    if v is not None: return v
    for mr in ws.merged_cells.ranges:
        if mr.min_col==c and mr.min_row<=r<=mr.max_row: return ws.cell(mr.min_row,c).value
data=[r for r in range(3,ws.max_row+1) if any(ws.cell(r,c).value not in (None,'') for c in (2,4,5,6,7,11))]
# 기존 url -> {L,M,N,O,row, hl}
exist={}
for r in data:
    nu=norm(ws.cell(r,11).value)
    if nu and nu not in exist:
        exist[nu]={'L':ws.cell(r,12).value,'M':ws.cell(r,13).value,'N':ws.cell(r,14).value,'O':ws.cell(r,15).value,
                   'S':ws.cell(r,19).value,'r':r,'url':ws.cell(r,11).value}
gnb_urls=set(norm(u) for *_,u in gnb_rows)
for lab,kids in d4:
    for _,ku in kids: gnb_urls.add(norm(ku))
# 4) 새 행 빌드 (head-gnb 순서, 소에 depth4면 세부전개)
def Lof(u):
    if 'nhis.or.kr' not in u: return '사이트'
    if re.search(r'(bbs|board|Notice|wbhaea023|NoticeList)',u,re.I): return '게시판'
    return '페이지'  # NHIS retrieve*는 대개 서비스페이지(M/N은 검수영역)
def crawlLMNO(u):
    L=Lof(u)
    if L=='사이트': return ('사이트',None,None,None,'외부링크')
    M=0 if L=='게시판' else 1
    return (L,M,'어문','미부착',None)
rows=[]; newcount=0
def emit(d1,d2,d3,d4lab,url,is_new):
    global newcount
    rec={'D':d1,'E':d2,'F':d3,7:d4lab,'url':url,'mark':set()}
    nu=norm(url)
    if nu in exist:
        e=exist[nu]; rec['L']=e['L']; rec['M']=e['M']; rec['N']=e['N']; rec['O']=e['O']; rec['S']=e['S']
        hl=ws.cell(e['r'],11).hyperlink; rec['hl']=hl.target if hl else url
    else:
        L,M,N,O,S=crawlLMNO(url); rec['L']=L;rec['M']=M;rec['N']=N;rec['O']=O;rec['S']=S; rec['hl']=url
        newcount+=1; rec['mark']=set(range(4,16))
    rows.append(rec)
done_groups=set()
for d1,d2,d3,url in gnb_rows:
    nu=norm(url)
    grp=d4_by_first.get(nu) or (d4_labels.get(d3) if d3 else None)
    if grp and id(grp) not in done_groups:
        done_groups.add(id(grp))
        for i,(kn,ku) in enumerate(grp):
            emit(d1,d2,d3,kn,ku, norm(ku) not in exist)
    elif grp and id(grp) in done_groups:
        continue
    else:
        emit(d1,d2,d3,None,url, nu not in exist)
print('GNB행 %d, depth4그룹 %d, 신규세부 %d'%(len(gnb_rows),len(d4),newcount))
# 5) 비GNB 꼬리(기존행 중 url이 트리에 없는 것) 유지
tail=[r for r in data if norm(ws.cell(r,11).value) not in gnb_urls]
print('비GNB 꼬리행 %d'%len(tail))
if not APPLY:
    print('DRY. 총 예상행 %d (GNB확장 %d + 꼬리 %d)'%(len(rows)+len(tail),len(rows),len(tail)))
    sys.exit(0)
# 신규행 L 게시판 정밀화(크롤)
for rec in rows:
    if rec.get('L')=='게시판' and 4 in rec['mark']:
        pass
# 6) 재구성 (기존 스타일 템플릿 + 꼬리 보존)
tpl={c:copy(ws.cell(3,c)._style) for c in range(1,MAXC+1)}
tailrecs=[]
for r in tail:
    rec={'D':eff(r,4),'E':eff(r,5),'F':ws.cell(r,6).value,7:ws.cell(r,7).value,'url':ws.cell(r,11).value,'mark':set()}
    rec['L']=ws.cell(r,12).value;rec['M']=ws.cell(r,13).value;rec['N']=ws.cell(r,14).value;rec['O']=ws.cell(r,15).value;rec['S']=ws.cell(r,19).value
    hl=ws.cell(r,11).hyperlink; rec['hl']=hl.target if hl else rec['url']; tailrecs.append(rec)
allrecs=rows+tailrecs
for mr in list(ws.merged_cells.ranges):
    if mr.min_row>=3: ws.unmerge_cells(str(mr))
for r in range(3,ws.max_row+1):
    for c in range(1,MAXC+1):
        cell=ws.cell(r,c); cell.value=None; cell.hyperlink=None; cell.fill=PatternFill(fill_type=None)
n=len(allrecs)
for i,rc in enumerate(allrecs):
    r=3+i
    for c in range(1,MAXC+1): ws.cell(r,c)._style=copy(tpl[c])
    ws.cell(r,2).value=i+1; ws.cell(r,3).value='국민건강보험공단'
    ws.cell(r,4).value=rc['D']; ws.cell(r,5).value=rc['E']; ws.cell(r,6).value=rc['F']; ws.cell(r,7).value=rc.get(7)
    kc=ws.cell(r,11); kc.value=rc['url']; kc.hyperlink=rc['hl'] if rc['url'] else None
    ws.cell(r,12).value=rc.get('L'); ws.cell(r,13).value=rc.get('M'); ws.cell(r,14).value=rc.get('N'); ws.cell(r,15).value=rc.get('O'); ws.cell(r,19).value=rc.get('S')
    for c in rc.get('mark',()): ws.cell(r,c).fill=BLUE
    ws.row_dimensions[r].height=15
def rem(col,sc):
    r=3
    while r<3+n:
        v=ws.cell(r,col).value
        if v is None or v=='': r+=1; continue
        r2=r
        while r2+1<3+n and ws.cell(r2+1,col).value==v and all(ws.cell(r2+1,s).value==ws.cell(r,s).value for s in sc): r2+=1
        if r2>r:
            for rr in range(r+1,r2+1): ws.cell(rr,col).value=None
            ws.merge_cells(start_row=r,start_column=col,end_row=r2,end_column=col)
        r=r2+1
rem(7,[4,5,6]); rem(6,[4,5]); rem(5,[4]); rem(4,[])
if os.path.getmtime(xp)!=mt: print('ABORT'); sys.exit(1)
shutil.copy(xp, os.path.join(d,'_backup',os.path.basename(xp).replace('.xlsx','_backup_depth4전개전.xlsx')))
wb.save(xp)
bs=[ws.cell(r,2).value for r in range(3,3+n)]
print('saved %d행 B연속%s'%(n,bs==list(range(1,n+1))))