df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
104 lines
5.1 KiB
Python
104 lines
5.1 KiB
Python
# -*- coding: utf-8 -*-
|
|
import openpyxl, json, time, re, ssl, urllib.request, sys
|
|
from urllib.parse import urljoin, urlparse
|
|
from bs4 import BeautifulSoup
|
|
ctx=ssl.create_default_context(); ctx.check_hostname=False; ctx.verify_mode=ssl.CERT_NONE
|
|
HDR={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/124 Safari/537.36'}
|
|
BASE='https://www.komipo.co.kr'
|
|
def fetch(u):
|
|
return urllib.request.urlopen(urllib.request.Request(u,headers=HDR),timeout=25,context=ctx).read()
|
|
|
|
UI=re.compile(r'banner|/common/|icon|btn|logo|bullet|sprite|blank|no_img|arrow|/dot|/bg|popup|/sns',re.I)
|
|
DIAG=re.compile(r'조직도|체계도|순서도|흐름도|구성도|개념도|체계$|절차도|한눈|로드맵|추진체계|다이어그램|chart|graph')
|
|
|
|
def classify(url, self_label):
|
|
full=urljoin(BASE+'/', url) if url.startswith('/') else url
|
|
host=urlparse(full).netloc.lower()
|
|
rec={'url':full}
|
|
if host not in ('www.komipo.co.kr','komipo.co.kr'):
|
|
rec.update(L='사이트',M=None,N='',O=''); return rec
|
|
if '/board/' in full and 'boardMain' in full:
|
|
try:
|
|
soup=BeautifulSoup(fetch(full),'html.parser')
|
|
em=soup.select_one('div.total em.count') or soup.select_one('.total .count')
|
|
M=int(re.sub(r'[^0-9]','',em.get_text())) if em else 0
|
|
except Exception as e:
|
|
M=0; print(' board err',full,str(e)[:40],file=sys.stderr)
|
|
# N: list thumbnails
|
|
N='어문'
|
|
try:
|
|
lst=soup.select_one('#content') or soup
|
|
thumbs=[im for im in lst.find_all('img') if not UI.search(im.get('src','')) and 'getImage' in im.get('src','')]
|
|
if M>0 and thumbs: N='어문,이미지'
|
|
if M==0: N='없음'
|
|
except: pass
|
|
rec.update(L='게시판',M=M,N=N,O='미부착'); return rec
|
|
# page
|
|
try:
|
|
soup=BeautifulSoup(fetch(full),'html.parser')
|
|
cont=soup.select_one('#content') or soup.select_one('div.contents')
|
|
imgs=cont.find_all('img') if cont else []
|
|
real=[]
|
|
for im in imgs:
|
|
src=im.get('src','');alt=im.get('alt','')
|
|
if UI.search(src): continue
|
|
if 'getImage' in src and ('코로나' in alt or '정규직' in alt): continue # alio popups
|
|
real.append((src,alt))
|
|
diag_only = real and all(DIAG.search(a or '') for s,a in real)
|
|
N='어문,이미지' if (real and not diag_only) else '어문'
|
|
rec.update(L='페이지',M=1,N=N,O='미부착',_imgs=[a or s.split('/')[-1] for s,a in real][:6]); return rec
|
|
except Exception as e:
|
|
print(' page err',full,str(e)[:40],file=sys.stderr)
|
|
rec.update(L='페이지',M=1,N='어문',O='미부착'); return rec
|
|
|
|
# load found tabs + leaf info
|
|
found=json.load(open('tabscan.json',encoding='utf-8'))
|
|
wb=openpyxl.load_workbook('한국중부발전.xlsx');ws=wb.active
|
|
# effective D-J
|
|
eff={}
|
|
for r in range(3,ws.max_row+1):
|
|
for c in range(4,11): eff[(r,c)]=ws.cell(r,c).value
|
|
for mr in ws.merged_cells.ranges:
|
|
if mr.min_col>=4 and mr.max_col<=10 and mr.min_row>=3:
|
|
top=ws.cell(mr.min_row,mr.min_col).value
|
|
for rr in range(mr.min_row,mr.max_row+1): eff[(rr,mr.min_col)]=top
|
|
# existing urls (normalized no query) -> for sibling dup check
|
|
existing={}
|
|
for r in range(3,ws.max_row+1):
|
|
k=ws.cell(r,11).value
|
|
if k: existing.setdefault(str(k).split('?')[0],r)
|
|
|
|
plan=[]
|
|
for r_s,v in sorted(found.items(),key=lambda x:int(x[0])):
|
|
r=int(r_s)
|
|
cols={c:eff[(r,c)] for c in range(4,11) if eff[(r,c)] not in (None,'')}
|
|
lc=max(cols); leaf=cols[lc]
|
|
tabs=v['tabs']
|
|
selfi=next((i for i,t in enumerate(tabs) if t['current']),0)
|
|
self_label=tabs[selfi]['label']
|
|
mode='sibling' if self_label.strip()==str(leaf).strip() else 'child'
|
|
parent_vals={c:eff[(r,c)] for c in range(4,11)}
|
|
exp={'row':r,'b':ws.cell(r,2).value,'leaf':leaf,'lc':lc,'self':self_label,'mode':mode,'parent_path':[eff[(r,c)] for c in range(4,11)],'parent_K':ws.cell(r,11).value,'parent_LMNO':[ws.cell(r,c).value for c in (12,13,14,15)],'newrows':[]}
|
|
for i,t in enumerate(tabs):
|
|
full=urljoin(BASE+'/',t['href']) if t['href'].startswith('/') else t['href']
|
|
is_self=(i==selfi)
|
|
nk=str(full).split('?')[0]
|
|
if mode=='sibling' and not is_self and nk in existing:
|
|
continue # already in sheet
|
|
if mode=='sibling' and is_self:
|
|
continue # anchor already there
|
|
if is_self and mode=='child':
|
|
cl={'L':exp['parent_LMNO'][0],'M':exp['parent_LMNO'][1],'N':exp['parent_LMNO'][2],'O':exp['parent_LMNO'][3],'url':full}
|
|
else:
|
|
cl=classify(t['href'],self_label); time.sleep(0.7)
|
|
exp['newrows'].append({'label':t['label'],'is_self':is_self,**cl})
|
|
plan.append(exp)
|
|
json.dump(plan,open('plan.json','w',encoding='utf-8'),ensure_ascii=False,indent=1)
|
|
# print summary
|
|
for e in plan:
|
|
print(f"r{e['row']} b{e['b']} [{e['mode']}] leaf='{e['leaf']}' self='{e['self']}'")
|
|
for nr in e['newrows']:
|
|
tag='(self)' if nr['is_self'] else ''
|
|
print(f" +{nr['label']!r}{tag} L={nr.get('L')} M={nr.get('M')} N={nr.get('N')} {nr.get('_imgs','')}")
|
|
print('\n총 신규/재구성 행:', sum(len(e['newrows']) for e in plan))
|