df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
160 lines
6.2 KiB
Python
160 lines
6.2 KiB
Python
import requests, sys, io, re, json
|
|
from bs4 import BeautifulSoup
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import openpyxl, urllib3
|
|
urllib3.disable_warnings()
|
|
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
|
|
H={'User-Agent':'Mozilla/5.0'}
|
|
CMSBASE='https://www.asan.go.kr/main/cms/'
|
|
|
|
wb=openpyxl.load_workbook('충청남도_아산시.xlsx'); ws=wb.active
|
|
scan={o['row']:o for o in json.load(open('_scan_out.json',encoding='utf-8'))}
|
|
|
|
def no_of(u):
|
|
m=re.search(r'[?&]no=(\d+)',str(u or '')); return m.group(1) if m else None
|
|
def norm_url(u):
|
|
if not u: return None
|
|
u=str(u).strip().rstrip('/')
|
|
u=re.sub(r'^https?://','',u).replace('www.','')
|
|
return u.lower()
|
|
def resolve(href):
|
|
href=str(href).strip()
|
|
if href.startswith('?'): return CMSBASE+href
|
|
if href.startswith('/'): return 'https://www.asan.go.kr'+href
|
|
return href
|
|
|
|
# existing keys -> set of rows
|
|
no2rows={}; url2rows={}
|
|
for r in range(3,271):
|
|
u=ws.cell(r,11).value
|
|
if not u: continue
|
|
n=no_of(u); no2rows.setdefault(n,[]).append(r) if n else None
|
|
url2rows.setdefault(norm_url(u),[]).append(r)
|
|
|
|
def classify_html(url):
|
|
"""return (L,M) for a fetched url"""
|
|
try:
|
|
resp=requests.get(url,headers=H,timeout=25,verify=False,allow_redirects=True)
|
|
final=resp.url; resp.encoding=resp.apparent_encoding or 'utf-8'
|
|
s=BeautifulSoup(resp.text,'html.parser')
|
|
dom=re.sub(r'^https?://','',final).split('/')[0]
|
|
if 'asan.go.kr' not in dom:
|
|
return ('사이트','', dom)
|
|
# media portal
|
|
if dom=='media.asan.go.kr':
|
|
return ('게시판',0, dom)
|
|
cnt=None
|
|
for t in s.select('table'):
|
|
cap=t.find('caption')
|
|
if cap:
|
|
m=re.search(r':\s*([\d,]+)\s*/\s*\d+',cap.get_text())
|
|
if m: cnt=int(m.group(1).replace(',','')); break
|
|
bbstbl=False
|
|
for t in s.select('table'):
|
|
ths=[th.get_text(strip=True) for th in t.select('th')]
|
|
if '제목' in ths and ('번호' in ths or '작성일' in ths or '등록일' in ths or '게시일자' in ths):
|
|
bbstbl=True; break
|
|
# seip_sechul paginated table
|
|
seip = 'seip_sechul' in final and bool(s.select('.paging a, .pagination a'))
|
|
if cnt is not None: return ('게시판',cnt,dom)
|
|
if bbstbl or seip: return ('게시판',0,dom)
|
|
return ('페이지',1,dom)
|
|
except Exception as e:
|
|
return ('ERR',str(e)[:50],None)
|
|
|
|
# ---- existing rows final L/M ----
|
|
INFLATED={'어문,이미지,영상','어문,이미지,영상,오디오','어문,영상'}
|
|
existing_plan={}
|
|
def fix_existing(r):
|
|
o=scan.get(r,{})
|
|
url=ws.cell(r,11).value
|
|
newL=o.get('newL'); cnt=o.get('cnt')
|
|
dom=o.get('final_dom') or ''
|
|
# refine with special domains
|
|
if o.get('newL')=='ERR':
|
|
# external sites that errored -> 사이트 if not asan, else keep page
|
|
newL='사이트' if 'asan.go.kr' not in (dom or norm_url(url) or '') else '페이지'
|
|
if dom=='media.asan.go.kr': newL='게시판'; cnt=None
|
|
# seip
|
|
if url and 'seip_sechul' in str(url): newL='게시판'; cnt=None
|
|
# M
|
|
if newL=='게시판': M = cnt if cnt is not None else 0
|
|
elif newL=='페이지': M=1
|
|
else: M=''
|
|
return r,newL,M
|
|
for r in range(16,271):
|
|
if ws.cell(r,11).value: existing_plan[r]=fix_existing(r)
|
|
|
|
# ---- expansion groups (URL+no dedup) ----
|
|
groups=[] # each: {row, col, children:[(label,url,L,M,N,O,site)]}
|
|
def deepest_col(r):
|
|
for c in range(10,4,-1): # J..E (cols10..5)
|
|
if ws.cell(r,c).value not in (None,''): return c
|
|
return 5
|
|
def is_container(r,col):
|
|
# cell merged vertically across >1 row => it's a parent/category container
|
|
for mr in ws.merged_cells.ranges:
|
|
if mr.min_col==col and mr.min_row<=r<=mr.max_row and mr.max_row>mr.min_row:
|
|
return True
|
|
return False
|
|
new_children_urls=set()
|
|
plan_groups=[]
|
|
for r in range(16,271):
|
|
o=scan.get(r)
|
|
if not o or not o.get('ptab'): continue
|
|
cur_no=no_of(ws.cell(r,11).value)
|
|
incl=[] # children to add
|
|
for txt,href in o['ptab']:
|
|
tu=resolve(href); tn=no_of(tu); tnorm=norm_url(tu)
|
|
rows_no=no2rows.get(tn,[]) if tn else []
|
|
rows_url=url2rows.get(tnorm,[])
|
|
other=[x for x in set(rows_no+rows_url) if x!=r]
|
|
if other: # exists elsewhere -> sibling nav, skip
|
|
continue
|
|
incl.append((txt.strip(),tu,tn))
|
|
# skip container(대분류/category) rows: their tabs belong to a child menu item
|
|
if is_container(r,deepest_col(r)):
|
|
continue
|
|
# only expand if >=2 genuine children (self + new)
|
|
if len(incl)>=2:
|
|
plan_groups.append({'row':r,'col':deepest_col(r),'incl':incl})
|
|
for t,u,n in incl: new_children_urls.add(u)
|
|
|
|
# classify all new child urls
|
|
uniq=sorted(new_children_urls)
|
|
clsmap={}
|
|
with ThreadPoolExecutor(max_workers=12) as ex:
|
|
res=list(ex.map(lambda u:(u,classify_html(u)), uniq))
|
|
for u,lm in res: clsmap[u]=lm
|
|
|
|
# attach
|
|
for g in plan_groups:
|
|
kids=[]
|
|
for t,u,n in g['incl']:
|
|
L,M,dom=clsmap[u]
|
|
if L=='ERR': L='사이트'; M='' # external/unreachable -> site
|
|
N='' if L=='사이트' else '어문'
|
|
O='' if L=='사이트' else '미부착'
|
|
site='외부링크' if L=='사이트' else ''
|
|
kids.append({'label':t,'url':u,'L':L,'M':M,'N':N,'O':O,'S':site})
|
|
g['kids']=kids
|
|
|
|
json.dump({'existing':{str(k):v for k,v in existing_plan.items()},'groups':plan_groups},
|
|
open('_plan_out.json','w',encoding='utf-8'),ensure_ascii=False,indent=1)
|
|
|
|
# summary
|
|
from collections import Counter
|
|
cL=Counter(v[1] for v in existing_plan.values())
|
|
print('=== EXISTING rows 16-270 final L ===',dict(cL))
|
|
print(f'\n=== EXPANSION GROUPS: {len(plan_groups)} ===')
|
|
tot_new=0
|
|
for g in plan_groups:
|
|
r=g['row']; lbl=ws.cell(r,g['col']).value
|
|
nk=len(g['kids']); tot_new+=nk-1 # -1 because self row reused
|
|
from openpyxl.utils import get_column_letter
|
|
print(f" r{r} [{get_column_letter(g['col'])}={lbl}] -> {nk} children (+{nk-1} new):")
|
|
print(' '+' | '.join(f"{k['label']}({k['L']}{k['M']})" for k in g['kids']))
|
|
print(f'\nTOTAL new rows added: {tot_new}')
|
|
print('current data rows 3-270 =',sum(1 for r in range(3,271) if ws.cell(r,11).value))
|
|
print('after rebuild approx =', sum(1 for r in range(3,271) if ws.cell(r,11).value)+tot_new)
|