61 lines
2.2 KiB
Python
61 lines
2.2 KiB
Python
# coding: utf-8
|
|
import requests, re, io, openpyxl
|
|
H={'User-Agent':'Mozilla/5.0','Referer':'https://www.kisa.or.kr/','X-Requested-With':'XMLHttpRequest'}
|
|
out=io.open('_group_out.txt','w',encoding='utf-8')
|
|
def log(*a): out.write(' '.join(str(x) for x in a)+'\n')
|
|
wb=openpyxl.load_workbook('한국인터넷진흥원.xlsx'); ws=wb.active
|
|
|
|
def seq_of(url):
|
|
if not url: return None
|
|
m=re.search(r'kisa\.or\.kr/(\d+)',url); return m.group(1) if m else None
|
|
|
|
# build sheet: row -> (D..J labels, K, seq)
|
|
rows={}
|
|
sheet_seqs=set()
|
|
for r in range(3,ws.max_row+1):
|
|
labels=[ws.cell(r,c).value for c in range(4,11)] # D..J
|
|
k=ws.cell(r,11).value
|
|
seq=seq_of(k)
|
|
# leaf label = last non-empty in D..J
|
|
leaf=None
|
|
for v in labels:
|
|
if v not in (None,''): leaf=v
|
|
rows[r]={'labels':labels,'k':k,'seq':seq,'leaf':leaf}
|
|
if seq: sheet_seqs.add(seq)
|
|
|
|
def tablist(seq):
|
|
try:
|
|
r=requests.get('https://www.kisa.or.kr/selectTabList.do',headers=H,
|
|
params={'menu_seq':seq,'lang_type':'KO'},timeout=20)
|
|
if r.text.strip().startswith('['): return r.json()
|
|
except: return None
|
|
return None
|
|
|
|
groups={} # frozenset(seqs) -> info
|
|
for r in range(14, ws.max_row+1):
|
|
seq=rows[r]['seq']
|
|
if not seq: continue
|
|
data=tablist(seq)
|
|
if not data: continue
|
|
tabseqs=tuple(str(d.get('menu_seq')) for d in data)
|
|
key=frozenset(s for s in tabseqs if s.isdigit())
|
|
if key in groups:
|
|
groups[key]['rows'].append(r); continue
|
|
# new seqs (digit tabs not in sheet)
|
|
new=[(str(d.get('menu_seq')),d.get('menu_name'),d.get('content_type')) for d in data
|
|
if str(d.get('menu_seq')).isdigit() and str(d.get('menu_seq')) not in sheet_seqs]
|
|
groups[key]={'rows':[r],'tabs':data,'new':new,'rep':r,'rep_seq':seq,'leaf':rows[r]['leaf']}
|
|
|
|
log('TOTAL distinct tab-groups:',len(groups))
|
|
log('='*70)
|
|
gi=0
|
|
for key,g in groups.items():
|
|
gi+=1
|
|
rep=g['rep']
|
|
log('GROUP %d | rep row %d (seq %s) leaf=%r | rows=%s | NEW=%d'%(
|
|
gi,rep,g['rep_seq'],g['leaf'],g['rows'],len(g['new'])))
|
|
if g['new']:
|
|
for s,n,ct in g['new']:
|
|
log(' NEW seq=%s name=%r ctype=%s'%(s,n,ct))
|
|
out.close(); print('done', len(groups),'groups')
|