97 lines
3.7 KiB
Python
97 lines
3.7 KiB
Python
# coding: utf-8
|
|
import openpyxl, requests, re, io, json
|
|
H={'User-Agent':'Mozilla/5.0','Referer':'https://www.kisa.or.kr/','X-Requested-With':'XMLHttpRequest'}
|
|
wb=openpyxl.load_workbook('한국인터넷진흥원.xlsx'); ws=wb.active
|
|
|
|
def seq_of(u):
|
|
m=re.search(r'kisa\.or\.kr/(\d+)',u or ''); return m.group(1) if m else None
|
|
|
|
# effective D..J per row (expand merges)
|
|
maxr=ws.max_row
|
|
eff={} # row -> [D..J]
|
|
for r in range(3,maxr+1):
|
|
eff[r]=[ws.cell(r,c).value for c in range(4,11)]
|
|
for mr in ws.merged_cells.ranges:
|
|
if mr.min_col>=4 and mr.max_col<=10:
|
|
v=ws.cell(mr.min_row,mr.min_col).value
|
|
for rr in range(mr.min_row,mr.max_row+1):
|
|
eff[rr][mr.min_col-4]=v
|
|
|
|
rowseq={}; sheet_seqs=set()
|
|
leaf_col={}; leaf_label={}
|
|
for r in range(3,maxr+1):
|
|
k=ws.cell(r,11).value; s=seq_of(k); rowseq[r]=s
|
|
if s: sheet_seqs.add(s)
|
|
# deepest non-empty col index (4..10)
|
|
lc=4; lab=None
|
|
for ci,v in enumerate(eff[r]):
|
|
if v not in (None,''): lc=4+ci; lab=v
|
|
leaf_col[r]=lc; leaf_label[r]=lab
|
|
|
|
def genuinely_new(tabseq):
|
|
if tabseq in sheet_seqs: return False
|
|
for ss in sheet_seqs:
|
|
if ss.startswith(tabseq) and len(ss)>len(tabseq): return False # tab is parent of existing
|
|
return True
|
|
|
|
def tablist(seq):
|
|
try:
|
|
r=requests.get('https://www.kisa.or.kr/selectTabList.do',headers=H,
|
|
params={'menu_seq':seq,'lang_type':'KO'},timeout=20)
|
|
if r.text.strip().startswith('['): return r.json()
|
|
except: return None
|
|
return None
|
|
|
|
def totalnum(seq):
|
|
try:
|
|
h=requests.get('https://www.kisa.or.kr/%s'%seq,headers=H,timeout=20).content.decode('utf-8','replace')
|
|
m=re.search(r'var\s+totalNum\s*=\s*(\d+)',h); return int(m.group(1)) if m else None
|
|
except: return None
|
|
|
|
seen=set(); plan=[]
|
|
for r in range(14,maxr+1):
|
|
s=rowseq[r]
|
|
if not s: continue
|
|
data=tablist(s)
|
|
if not data: continue
|
|
key=tuple(str(d.get('menu_seq')) for d in data)
|
|
if key in seen: continue
|
|
seen.add(key)
|
|
tabs=[(str(d.get('menu_seq')),d.get('menu_name'),(d.get('content_type') or '')) for d in data]
|
|
new_tabs=[t for t in tabs if t[0].isdigit() and genuinely_new(t[0])]
|
|
if not new_tabs: continue # SKIP (LNB or no children)
|
|
# EXPAND. children = tabs in order; self = the one matching s
|
|
children=[]
|
|
for (ts,tn,tc) in tabs:
|
|
is_self = (ts==s)
|
|
if tc=='URL' or not ts.isdigit():
|
|
# external URL tab
|
|
url=ts if ts.startswith('http') else None
|
|
children.append({'seq':ts,'name':tn,'ctype':'URL','self':is_self,
|
|
'url':url,'L':'사이트','M':None})
|
|
continue
|
|
if tc in ('POST','PHOTO'):
|
|
L='게시판'; M=totalnum(ts) if not is_self else (ws.cell(r,13).value)
|
|
else:
|
|
L='페이지'; M=1
|
|
children.append({'seq':ts,'name':tn,'ctype':tc,'self':is_self,
|
|
'url':'https://www.kisa.or.kr/%s'%ts,'L':L,'M':M})
|
|
plan.append({'leaf_row':r,'leaf_col':leaf_col[r],'leaf_label':leaf_label[r],
|
|
'self_seq':s,'children':children})
|
|
|
|
json.dump(plan,io.open('_plan.json','w',encoding='utf-8'),ensure_ascii=False,indent=1)
|
|
# summary
|
|
out=io.open('_plan_sum.txt','w',encoding='utf-8')
|
|
tot_new=0
|
|
for p in plan:
|
|
nnew=sum(1 for c in p['children'] if not c['self'])
|
|
tot_new+=nnew
|
|
out.write('LEAF r%d col%d %r -> %d children (+%d new)\n'%(
|
|
p['leaf_row'],p['leaf_col'],p['leaf_label'],len(p['children']),nnew))
|
|
for c in p['children']:
|
|
tag='SELF' if c['self'] else 'NEW '
|
|
out.write(' %s %-9s %-40s L=%s M=%s\n'%(tag,c['seq'],c['name'],c['L'],c['M']))
|
|
out.write('\nTOTAL expand groups=%d total new rows=%d\n'%(len(plan),tot_new))
|
|
out.close()
|
|
print(open('_plan_sum.txt',encoding='utf-8').read())
|