DB_JOB/작업파일/완료/공공기관/_gugak_prog_expand.py

# -*- coding: utf-8 -*-
# 국악방송(27) 프로그램(tv/radio_program_main_sub.jsp)의 div.program_menu_wrap ul.menu_list 자식 전개.
# E=프로그램(카테고리) → F=서브메뉴(프로그램소개/공지/선곡/방송보기 등). 라디오 중복(idx) dedup. javascript링크 제외.
# 이름기반 L/M/N: 소개/진행자=페이지·어문,이미지 / 방송보기=게시판·어문,영상 / 그외=게시판·어문(M=distinct bIdx).
# 사용: _gugak_prog_expand.py [--apply]
import openpyxl, glob, os, re, sys, shutil, time, urllib.request, ssl, http.cookiejar
from bs4 import BeautifulSoup
from copy import copy
from openpyxl.styles import PatternFill
ctx=ssl.create_default_context(); ctx.check_hostname=False; ctx.verify_mode=ssl.CERT_NONE
cj=http.cookiejar.CookieJar(); op=urllib.request.build_opener(urllib.request.HTTPSHandler(context=ctx),urllib.request.HTTPCookieProcessor(cj))
op.addheaders=[('User-Agent','Mozilla/5.0')]
def g(u):
    try: return op.open(u,timeout=20).read().decode('utf-8','replace')
    except Exception: return ''
BASE='http://www.igbf.kr'; BLUE=PatternFill(fill_type='solid',fgColor='FFBDD7EE'); MAXC=27
APPLY='--apply' in sys.argv
DIR=os.path.dirname(os.path.abspath(__file__))
d=[x for x in glob.glob(os.path.join(DIR,'27.*')) if os.path.isdir(x)][0]
xp=[p for p in glob.glob(os.path.join(d,'*.xlsx')) if not os.path.basename(p).startswith(('_','~')) and 'conflict' not in os.path.basename(p) and 'backup' not in p][0]
mt=os.path.getmtime(xp); wb=openpyxl.load_workbook(xp); ws=wb.active
def eff(r,c):
    v=ws.cell(r,c).value
    if v is not None: return v
    for mr in ws.merged_cells.ranges:
        if mr.min_col==c and mr.min_row<=r<=mr.max_row: return ws.cell(mr.min_row,c).value
    return None
data=[r for r in range(3,ws.max_row+1) if any(ws.cell(r,c).value not in (None,'') for c in (2,4,5,6,7,11))]
def fixdom(u):  # new.igbf.kr(404) → www.igbf.kr(작동), https→http
    u=str(u).replace('https://new.igbf.kr','http://www.igbf.kr').replace('http://new.igbf.kr','http://www.igbf.kr')
    return u
def menu_items(u):
    h=g(fixdom(u));
    if not h: return None
    s=BeautifulSoup(h,'html.parser')
    wrap=s.select_one('div.program_menu_wrap ul.menu_list')
    if not wrap: return []
    out=[]
    for a in wrap.find_all('a',href=True):
        href=a['href']; t=re.sub(r'\s+',' ',a.get_text()).strip()
        if not t or href.startswith('javascript') or href.strip()=='#': continue
        full=href if href.startswith('http') else BASE+href
        out.append((t,full))
    return out
def classify(name,url):
    pg='페이지' if re.search(r'(소개|진행자)',name) else '게시판'
    if pg=='페이지': return ('페이지',1,'어문,이미지')
    if re.search(r'(방송보기|다시보기|동영상|영상)',name):
        return ('게시판',None,'어문,영상')
    # 게시판 M=distinct bIdx (페이징無 가정, 있으면 last page*per 추정 생략)
    h=g(url); n=len(set(re.findall(r'bIdx=(\d+)',h)))
    Nv='어문,이미지' if re.search(r'(사진|갤러리|포토)',name) else '어문'
    return ('게시판', n if n else 0, Nv)
# 프로그램 행 식별
progs=[r for r in data if re.search(r'(tv|radio)_program_main_sub\.jsp',str(ws.cell(r,11).value or ''))]
def idxof(u):
    m=re.search(r'idx=(RD\d+)',str(u)); return m.group(1) if m else None
print('프로그램 행 %d개'%len(progs))
# 라디오 중복(idx) dedup: 같은 idx 두번째+는 삭제대상
seen_idx=set(); dup_rows=set()
for r in progs:
    ix=idxof(ws.cell(r,11).value)
    if ix in seen_idx: dup_rows.add(r)
    else: seen_idx.add(ix)
print('중복 프로그램행(삭제) %d개:'%len(dup_rows), sorted(dup_rows))
# 각 프로그램 menu 수집
plan={}
for r in progs:
    if r in dup_rows: continue
    its=menu_items(ws.cell(r,11).value)
    plan[r]=its
    print('  r%d [%s] menu=%s'%(r,eff(r,5), [t for t,_ in (its or [])] if its else its))
    time.sleep(0.05)
if not APPLY:
    tot=sum(len(v) for v in plan.values() if v)
    print('\nDRY. 전개대상 프로그램 %d, 신규자식 합 %d, 중복삭제 %d'%(len([v for v in plan.values() if v]),tot,len(dup_rows)))
    sys.exit(0)
# 빌드: data 순회, 프로그램행은 (dedup아니면) E카테고리+F자식들로 치환, 그외 유지
snap={(r,c):copy(ws.cell(r,c)._style) for r in data for c in range(1,MAXC+1)}
def rec_of(r):
    rec={'src':r,'D':eff(r,4),'E':eff(r,5),'F':ws.cell(r,6).value,'mark':set()}
    for c in range(7,MAXC+1): rec[c]=ws.cell(r,c).value
    rec['url']=ws.cell(r,11).value; hl=ws.cell(r,11).hyperlink; rec['hl']=hl.target if hl else rec['url']
    return rec
out=[]
for r in data:
    if r in dup_rows: continue
    if r in plan and plan[r]:
        its=plan[r]
        for i,(t,u) in enumerate(its):
            L,M,Nv=classify(t,u)
            ch={'src':r,'D':eff(r,4),'E':eff(r,5),'F':t,'mark':set(range(4,16)),'url':u,'hl':u}
            for c in range(7,MAXC+1): ch[c]=None
            ch[12]=L; ch[13]=M; ch[14]=Nv; ch[15]='미부착'
            out.append(ch)
    else:
        out.append(rec_of(r))
# write rebuild
for mr in list(ws.merged_cells.ranges):
    if mr.min_row>=3: ws.unmerge_cells(str(mr))
n=len(out)
for i,rc in enumerate(out):
    r=3+i; src=rc['src']
    for c in range(1,MAXC+1): ws.cell(r,c)._style=copy(snap[(src,c)])
    ws.cell(r,2).value=i+1; ws.cell(r,3).value=ws.cell(src,3).value
    ws.cell(r,4).value=rc['D']; ws.cell(r,5).value=rc['E']; ws.cell(r,6).value=rc['F']
    for c in range(7,MAXC+1): ws.cell(r,c).value=rc.get(c)
    kc=ws.cell(r,11); kc.value=rc['url']; kc.hyperlink=rc['hl'] if rc['url'] else None
    for c in rc.get('mark',()): ws.cell(r,c).fill=BLUE
for r in range(3+n,ws.max_row+1):
    for c in range(1,MAXC+1):
        cell=ws.cell(r,c); cell.value=None; cell.hyperlink=None; cell.fill=PatternFill(fill_type=None)
def rem(col,sc):
    r=3
    while r<3+n:
        v=ws.cell(r,col).value
        if v is None or v=='': r+=1; continue
        r2=r
        while r2+1<3+n and ws.cell(r2+1,col).value==v and all(ws.cell(r2+1,s).value==ws.cell(r,s).value for s in sc): r2+=1
        if r2>r:
            for rr in range(r+1,r2+1): ws.cell(rr,col).value=None
            ws.merge_cells(start_row=r,start_column=col,end_row=r2,end_column=col)
        r=r2+1
rem(7,[4,5,6]); rem(6,[4,5]); rem(5,[4]); rem(4,[])
if os.path.getmtime(xp)!=mt: print('ABORT changed'); sys.exit(1)
shutil.copy(xp, os.path.join(d,'_backup',os.path.basename(xp).replace('.xlsx','_backup_프로그램전개전.xlsx')))
wb.save(xp)
bs=[ws.cell(r,2).value for r in range(3,3+n)]
print('saved %d행 B연속%s'%(n,bs==list(range(1,n+1))))