df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
425 lines
19 KiB
Python
425 lines
19 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""보령시 전면 재수집(복사본 베이스).
|
|
- 본문 = #contents (기존 #txt는 전역검색 포함→게시판 오판)
|
|
- 인페이지탭 = ul.depth_tab → 매뉴얼 1-5b/1-5c로 leaf+1 컬럼 전개(논산식)
|
|
· 탭이 페이지/게시판 섞여도 각각 개별 행, 게시판은 총건수, 페이지는 #grap 인페이지탭수 or 1
|
|
- 검출 로직(L/M/N/O/P/Q)·미디어·공공누리 = 공주시 _phase234.py 이식
|
|
- 순번3 사전정보공표 = 사전 정의된 16 leaf 블록 삽입(brm1/dept 필터)
|
|
출력: 충청남도_보령시.xlsx 재작성(헤더·스타일·병합·순번·하이퍼링크).
|
|
"""
|
|
import re, time, warnings, json
|
|
from copy import copy
|
|
from urllib.parse import urljoin, urlparse
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
import requests, openpyxl
|
|
from bs4 import BeautifulSoup
|
|
from openpyxl.worksheet.hyperlink import Hyperlink
|
|
warnings.filterwarnings('ignore')
|
|
|
|
DIR = r'작업파일/광역_사이트맵/충청남도/6.보령시/'
|
|
SRC = DIR + '보령시 복사본.xlsx'
|
|
OUT = DIR + '충청남도_보령시.xlsx'
|
|
DOMAIN = 'brcn.go.kr'
|
|
H = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120 Safari/537.36'}
|
|
ADV_URL = 'https://www.brcn.go.kr/prog/openInformation/advanceInformation/kor/sub01_01_03/list.do'
|
|
|
|
# ---------------- fetch ----------------
|
|
_cache = {}
|
|
def fetch(url, timeout=15):
|
|
if url in _cache: return _cache[url]
|
|
s = None
|
|
try:
|
|
r = requests.get(url, headers=H, timeout=timeout, verify=False)
|
|
if r.status_code == 200:
|
|
s = BeautifulSoup(r.content, 'html.parser')
|
|
except Exception:
|
|
s = None
|
|
_cache[url] = s
|
|
return s
|
|
|
|
def body_of(soup):
|
|
return soup.select_one('#contents') or soup.select_one('#container') or soup
|
|
|
|
# ---------------- board / page detection ----------------
|
|
TOTAL_PATS = [re.compile(r'총\s*게시물\s*[:\s]*([\d,]+)'),
|
|
re.compile(r'총\s*([\d,]+)\s*건'),
|
|
re.compile(r'전체\s*([\d,]+)\s*건'),
|
|
re.compile(r'게시물\s*수\s*[:\s]*([\d,]+)')]
|
|
def get_total(body):
|
|
bt = body.select_one('.board_total, .total, .b_total')
|
|
if bt:
|
|
m = re.search(r'([\d,]+)', bt.get_text())
|
|
if m: return int(m.group(1).replace(',', ''))
|
|
txt = body.get_text(' ', strip=True)
|
|
for p in TOTAL_PATS:
|
|
m = p.search(txt)
|
|
if m: return int(m.group(1).replace(',', ''))
|
|
return None
|
|
|
|
def anchor_tab_count(body):
|
|
best = 0
|
|
for ul in body.select('ul.tab-ul, ul.type2, ul.type3, ul.tab_type, .tab_wrap ul, .tabBox ul'):
|
|
las = ul.find_all('a')
|
|
if len(las) >= 2 and all((a.get('href') or '').strip().startswith('#') for a in las):
|
|
best = max(best, len(las))
|
|
return best
|
|
|
|
def is_board_url(u):
|
|
return bool(u) and ('selectBoardList' in u or '/list.do' in u or 'BBSMSTR' in u or 'bbsId=' in u) and '/viewer/' not in u
|
|
|
|
def detect_form(body, url):
|
|
# 직원안내·전화번호안내·부서안내 등 deptOn 디렉토리 = 게시물 아님 → 페이지(논산 컨벤션 M=1)
|
|
if 'deptOn' in url:
|
|
return '페이지', 1
|
|
total = get_total(body)
|
|
has_paging = bool(body.select('.pagination, .paging'))
|
|
is_bbs = ('BBSMSTR' in url or 'selectBoardList' in url or 'bbsId=' in url)
|
|
# 진짜 게시판: 총건수/페이징이 있거나 BBSMSTR/selectBoardList 보드
|
|
if (total is not None) or has_paging or is_bbs:
|
|
return '게시판', (total if total is not None else 0)
|
|
# 그 외(디렉토리성 list.do 포함) = 페이지
|
|
return '페이지', (anchor_tab_count(body) or 1)
|
|
|
|
# ---------------- media / kogl (공주시 이식) ----------------
|
|
KOGL_IMG = re.compile(r'(?:new_)?img_opentype(\d{2})\.png', re.I)
|
|
KOGL_LINK = re.compile(r'kogl\.or\.kr/info/licenseType(\d)', re.I)
|
|
YT = re.compile(r'(?:youtube\.com|youtu\.be)', re.I)
|
|
VIDEXT = re.compile(r'\.(mp4|webm|mov|avi)(?:\?|$)', re.I)
|
|
def detect_media(body):
|
|
has_text = len(body.get_text(strip=True)) > 30
|
|
has_image = False
|
|
for img in body.find_all('img'):
|
|
src = img.get('src', '')
|
|
if not src or KOGL_IMG.search(src): continue
|
|
if re.search(r'(btn_|icon|ico_|bullet|blank|spacer|move\.|no[-_]?img|arrow)', src, re.I): continue
|
|
has_image = True; break
|
|
has_video = bool(body.find_all('video'))
|
|
if not has_video:
|
|
for f in body.find_all('iframe'):
|
|
if YT.search(f.get('src', '')): has_video = True; break
|
|
if not has_video and VIDEXT.search(str(body)): has_video = True
|
|
return has_image, has_video, has_text
|
|
def n_string(t, i, v):
|
|
p = []
|
|
if t: p.append('어문')
|
|
if i: p.append('이미지')
|
|
if v: p.append('영상')
|
|
return ','.join(p) if p else '없음'
|
|
def img_anchor(img):
|
|
p = img.parent
|
|
while p is not None:
|
|
if p.name == 'a':
|
|
h = p.get('href', '')
|
|
return bool(h) and not h.startswith('#') and not h.lower().startswith('javascript:')
|
|
p = p.parent
|
|
return False
|
|
def detect_kogl(body):
|
|
types = set(); qy = qn = False
|
|
for a in body.find_all('a', href=True):
|
|
m = KOGL_LINK.search(a['href'])
|
|
if m: types.add(int(m.group(1))); qy = True
|
|
for img in body.find_all('img'):
|
|
m = KOGL_IMG.search(img.get('src', ''))
|
|
if m:
|
|
types.add(int(m.group(1)))
|
|
if img_anchor(img): qy = True
|
|
else: qn = True
|
|
return types, ('Y' if qy else ('N' if qn else None))
|
|
|
|
DETAIL_FN = re.compile(r"fn[_a-zA-Z]*\(\s*['\"]?(\d+)")
|
|
def detail_urls(body, list_url, limit=4):
|
|
urls, seen = [], set()
|
|
tb = body.select_one('table tbody') or body.find('table')
|
|
if not tb: return urls
|
|
for tr in tb.find_all('tr')[:limit*3]:
|
|
a = tr.find('a', href=True)
|
|
if a:
|
|
h = a['href']
|
|
if 'view' in h.lower() and ('nttId' in h or 'Seq' in h or 'idx' in h.lower()):
|
|
full = urljoin(list_url, h)
|
|
if full not in seen:
|
|
seen.add(full); urls.append(full)
|
|
if len(urls) >= limit: break
|
|
return urls
|
|
|
|
def classify(url):
|
|
"""return dict L,M,N,O,P,Q (게시판은 상세 추적해 미디어/공공누리)."""
|
|
s = fetch(url)
|
|
if s is None:
|
|
# 별도 시스템(서브도메인/결제앱 등) — 자동수집 불가, 페이지로 처리
|
|
return {'L': '페이지', 'M': 1, 'N': '어문', 'O': '미부착', 'P': '', 'Q': '', 'note': '별도 시스템(자동수집 불가)'}
|
|
body = body_of(s)
|
|
form, count = detect_form(body, url)
|
|
has_i, has_v, has_t = detect_media(body)
|
|
types, q = detect_kogl(body)
|
|
P = '게시판' if types else ''
|
|
qflags = [q] if q else []
|
|
if form == '게시판' and count and count > 0:
|
|
for du in detail_urls(body, url, limit=2):
|
|
ds = fetch(du, 10)
|
|
if not ds: continue
|
|
db = body_of(ds)
|
|
di, dv, dt = detect_media(db)
|
|
has_i |= di; has_v |= dv; has_t |= dt
|
|
dtypes, dq = detect_kogl(db)
|
|
if dtypes and not types and not P: P = '게시물'
|
|
types |= dtypes
|
|
if dq: qflags.append(dq)
|
|
N = n_string(has_t, has_i, has_v)
|
|
if form == '게시판' and count == 0:
|
|
N = '없음' # 빈 게시판
|
|
if not types:
|
|
O = '미부착'; P = ''; Q = ''
|
|
else:
|
|
O = ','.join(f'{n}유형' for n in sorted(types))
|
|
P = P or '게시판'
|
|
Q = 'Y' if 'Y' in qflags else 'N'
|
|
return {'L': form, 'M': count if form == '게시판' else (count or 1),
|
|
'N': N, 'O': O, 'P': P, 'Q': Q, 'note': ''}
|
|
|
|
# ---------------- 사전정보공표 블록 (순번3) ----------------
|
|
def adv_block():
|
|
cat = [('일반공공행정','brm1=A',117),('공공질서및안전','brm1=B',26),('통신','brm1=C',7),
|
|
('산업중소기업','brm1=D',33),('보건','brm1=E',82),('사회복지','brm1=F',69),
|
|
('문화체육관광','brm1=G',35),('수송및교통','brm1=H',38),('농림해양수산','brm1=I',39),
|
|
('교육','brm1=J',11),('환경보호','brm1=K',45),('지역개발','brm1=L',28)]
|
|
dep = [('시청','dept=1',405),('직속기관 사업소','dept=2',109),('시의회','dept=3',16),('읍면동','dept=4',0)]
|
|
recs = []
|
|
for i,(h,qs,c) in enumerate(cat):
|
|
recs.append({'F':'사전정보공표' if i==0 else None,'G':'정보분류별' if i==0 else None,'H':h,
|
|
'url':f'{ADV_URL}?{qs}&siteCode=kor&mno=sub01_01_03','L':'게시판','M':c,'N':'어문','O':'미부착','P':'','Q':''})
|
|
for i,(h,qs,c) in enumerate(dep):
|
|
recs.append({'F':None,'G':'공표부서' if i==0 else None,'H':h,
|
|
'url':f'{ADV_URL}?{qs}&siteCode=kor&mno=sub01_01_03','L':'게시판','M':c,'N':'어문','O':'미부착','P':'','Q':''})
|
|
return recs
|
|
|
|
# ---------------- 메뉴 읽기 + 전개 ----------------
|
|
def norm(href, base):
|
|
if not href: return None
|
|
href = href.strip()
|
|
if href.startswith('#') or href.lower().startswith('javascript:') or href == '': return None
|
|
return urljoin(base, href)
|
|
|
|
def get_tabs(body, url):
|
|
dt = body.select_one('ul.depth_tab')
|
|
if not dt: return []
|
|
tabs = []
|
|
for li in dt.find_all('li'):
|
|
a = li.find('a')
|
|
if not a: continue
|
|
h = norm(a.get('href'), url)
|
|
if not h: continue
|
|
tabs.append({'label': li.get_text(strip=True), 'href': h})
|
|
return tabs
|
|
|
|
def main():
|
|
wb = openpyxl.load_workbook(SRC)
|
|
ws = wb.active
|
|
# 메뉴 행 수집 (복사본)
|
|
menus = []
|
|
for r in range(3, ws.max_row + 1):
|
|
D, E, F = ws.cell(r,4).value, ws.cell(r,5).value, ws.cell(r,6).value
|
|
url = ws.cell(r,11).value
|
|
menus.append({'r':r,'D':D,'E':E,'F':F,'url':url})
|
|
all_urls = set(m['url'] for m in menus if m['url'])
|
|
print('메뉴 행:', len(menus))
|
|
|
|
# --- 병렬 사전 fetch: 모든 메뉴 URL (탭 탐지용) ---
|
|
t0 = time.time()
|
|
menu_urls = [m['url'] for m in menus if m['url'] and isinstance(m['url'],str) and DOMAIN in m['url'] and m['F']!='사전정보공표']
|
|
print('메뉴 URL 병렬 fetch:', len(menu_urls))
|
|
with ThreadPoolExecutor(max_workers=16) as ex:
|
|
list(ex.map(lambda u: fetch(u), set(menu_urls)))
|
|
print(f' 메뉴 fetch 완료 ({time.time()-t0:.0f}s)')
|
|
|
|
# --- 스켈레톤 빌드(캐시 사용) + classify 대상 URL 수집 ---
|
|
skel = [] # (rec_partial, classify_url_or_None)
|
|
def leafcol(m):
|
|
if m['F'] is not None: return 'F'
|
|
if m['E'] is not None: return 'E'
|
|
return 'D'
|
|
for m in menus:
|
|
url = m['url']
|
|
# 빈 junk 행(URL·D·E·F 전부 없음) 스킵
|
|
if (not url or not isinstance(url,str)) and not any([m['D'],m['E'],m['F']]):
|
|
continue
|
|
base = {'D':m['D'],'E':m['E'],'F':m['F'],'G':None,'H':None}
|
|
if m['F'] == '사전정보공표':
|
|
for j,b in enumerate(adv_block()):
|
|
rec = {'D':None,'E':None,'F':None,'G':b['G'],'H':b['H'],'url':b['url'],
|
|
'L':b['L'],'M':b['M'],'N':b['N'],'O':b['O'],'P':b['P'],'Q':b['Q'],'newseq':(j==0)}
|
|
if j==0: rec.update({'D':m['D'],'E':m['E'],'F':'사전정보공표'})
|
|
skel.append((rec, None))
|
|
continue
|
|
if not url or not isinstance(url,str) or DOMAIN not in url:
|
|
rec = dict(base); rec.update({'url':url,'L':'사이트','M':'','N':'','O':'','P':'','Q':'','newseq':True})
|
|
skel.append((rec, None)); continue
|
|
s = _cache.get(url)
|
|
tabs = get_tabs(body_of(s), url) if s else []
|
|
realtabs = [t for t in tabs if DOMAIN in t['href']]
|
|
expand = len(realtabs) >= 2 and not all(t['href'] in all_urls for t in realtabs)
|
|
lc = leafcol(m); sub = {'F':'G','E':'F','D':'E'}[lc]
|
|
if expand:
|
|
for j,t in enumerate(realtabs):
|
|
if j==0:
|
|
rec = dict(base); rec['url']=t['href']; rec[sub]=t['label']; rec['newseq']=True
|
|
else:
|
|
rec = {'D':None,'E':None,'F':None,'G':None,'H':None,'url':t['href'],'newseq':False}
|
|
rec[sub]=t['label']
|
|
skel.append((rec, t['href']))
|
|
else:
|
|
rec = dict(base); rec['url']=url; rec['newseq']=True
|
|
skel.append((rec, url))
|
|
|
|
# --- 병렬 classify ---
|
|
todo = sorted(set(u for _,u in skel if u))
|
|
print('classify 대상 URL:', len(todo))
|
|
cls = {}
|
|
done=[0]
|
|
def cl(u):
|
|
r = classify(u); done[0]+=1
|
|
if done[0]%60==0: print(f' classify {done[0]}/{len(todo)} ({time.time()-t0:.0f}s)')
|
|
return u, r
|
|
with ThreadPoolExecutor(max_workers=16) as ex:
|
|
for u,r in ex.map(cl, todo):
|
|
cls[u] = r
|
|
print(f' classify 완료 ({time.time()-t0:.0f}s)')
|
|
|
|
records = []
|
|
for rec, u in skel:
|
|
if u and u in cls:
|
|
rec.update(cls[u])
|
|
records.append(rec)
|
|
|
|
print(f'총 레코드: {len(records)} ({time.time()-t0:.0f}s)')
|
|
json.dump([{k:v for k,v in r.items() if k!="newseq"} for r in records],
|
|
open(DIR+'_records.json','w',encoding='utf-8'), ensure_ascii=False)
|
|
write_excel(wb, ws, records)
|
|
return
|
|
|
|
# (이하 구버전 순차 경로 — 미사용)
|
|
for idx, m in enumerate(menus):
|
|
url = m['url']
|
|
base = {'D':m['D'],'E':m['E'],'F':m['F'],'G':None,'H':None}
|
|
if m['F'] == '사전정보공표':
|
|
for j,b in enumerate(adv_block()):
|
|
rec = dict(base)
|
|
if j==0:
|
|
rec['url']=b['url']
|
|
else:
|
|
rec={'D':None,'E':None,'F':None,'G':b['G'],'H':b['H'],'url':b['url']}
|
|
rec.update({'G':b['G'] if j==0 else rec.get('G'),'H':b['H'],
|
|
'L':b['L'],'M':b['M'],'N':b['N'],'O':b['O'],'P':b['P'],'Q':b['Q'],'newseq':(j==0)})
|
|
# 첫 행은 F 유지
|
|
if j==0: rec['F']='사전정보공표'
|
|
records.append(rec)
|
|
continue
|
|
# 외부
|
|
if not url or not isinstance(url,str) or DOMAIN not in url:
|
|
rec = dict(base); rec.update({'url':url,'L':'사이트','M':'','N':'','O':'','P':'','Q':'','newseq':True})
|
|
records.append(rec); continue
|
|
# fetch + depth_tab
|
|
s = fetch(url)
|
|
tabs = get_tabs(body_of(s), url) if s else []
|
|
realtabs = [t for t in tabs if DOMAIN in t['href']]
|
|
# 전개 조건: 2+ 탭, 모두 사이트맵에 이미 있으면 스킵
|
|
expand = len(realtabs) >= 2 and not all(t['href'] in all_urls for t in realtabs)
|
|
lc = leafcol(m)
|
|
sub = {'F':'G','E':'F','D':'E'}[lc] # leaf+1 컬럼
|
|
if expand:
|
|
for j,t in enumerate(realtabs):
|
|
if j==0:
|
|
rec = dict(base); rec['url']=t['href']; rec[sub]=t['label']; rec['newseq']=True
|
|
else:
|
|
rec = {'D':None,'E':None,'F':None,'G':None,'H':None,'url':t['href'],'newseq':False}
|
|
rec[sub]=t['label']
|
|
c = classify(t['href'])
|
|
rec.update(c); records.append(rec)
|
|
else:
|
|
rec = dict(base); rec['url']=url; rec['newseq']=True
|
|
rec.update(classify(url)); records.append(rec)
|
|
if (idx+1) % 40 == 0:
|
|
print(f' 메뉴 {idx+1}/{len(menus)} 레코드 {len(records)} ({time.time()-t0:.0f}s)')
|
|
|
|
print(f'총 레코드: {len(records)} ({time.time()-t0:.0f}s)')
|
|
json.dump([{k:v for k,v in r.items() if k!="newseq"} for r in records],
|
|
open(DIR+'_records.json','w',encoding='utf-8'), ensure_ascii=False)
|
|
write_excel(wb, ws, records)
|
|
|
|
def write_excel(wb, ws, records):
|
|
# 템플릿 스타일 (복사본 row3)
|
|
TPL = 3
|
|
tpl_style = {c: copy(ws.cell(TPL,c)._style) for c in range(1,28)}
|
|
tpl_h = ws.row_dimensions[TPL].height
|
|
# 기존 데이터행/병합 제거
|
|
for mr in list(ws.merged_cells.ranges):
|
|
if mr.min_row >= 3:
|
|
ws.unmerge_cells(str(mr))
|
|
if ws.max_row >= 3:
|
|
ws.delete_rows(3, ws.max_row-2)
|
|
# 기입
|
|
seq = 0
|
|
for i, rec in enumerate(records):
|
|
r = 3 + i
|
|
for c in range(1,28):
|
|
ws.cell(r,c)._style = copy(tpl_style[c])
|
|
if tpl_h: ws.row_dimensions[r].height = tpl_h
|
|
if rec.get('newseq'):
|
|
seq += 1
|
|
ws.cell(r,2).value = seq
|
|
ws.cell(r,3).value = '보령시'
|
|
for col,key in [(4,'D'),(5,'E'),(6,'F'),(7,'G'),(8,'H')]:
|
|
v = rec.get(key)
|
|
if v is not None: ws.cell(r,col).value = v
|
|
url = rec.get('url')
|
|
if url:
|
|
kc = ws.cell(r,11); kc.value = url
|
|
kc.hyperlink = Hyperlink(ref=kc.coordinate, target=str(url))
|
|
for col,key in [(12,'L'),(13,'M'),(14,'N'),(15,'O'),(16,'P'),(17,'Q')]:
|
|
v = rec.get(key)
|
|
if v not in (None,''): ws.cell(r,col).value = v
|
|
note = rec.get('note')
|
|
if note: ws.cell(r,19).value = note
|
|
last = 3 + len(records) - 1
|
|
# 병합 F→E→D
|
|
def merge_col(col):
|
|
run_start = 3; prev = ws.cell(3,col).value
|
|
# 그룹키: D는 자체, E는 D+E, F는 D+E+F (상위 동일구간 안에서만)
|
|
for r in range(4, last+2):
|
|
cur = ws.cell(r,col).value if r<=last else '\x00END'
|
|
# 상위 컬럼 경계에서도 끊기
|
|
boundary = False
|
|
if col>=5 and r<=last and ws.cell(r,4).value is not None: boundary=True # 새 D 시작
|
|
if col>=6 and r<=last and ws.cell(r,5).value is not None: boundary=True # 새 E 시작
|
|
if cur is not None or r>last or boundary:
|
|
if r-1 > run_start and prev is not None:
|
|
ws.merge_cells(start_row=run_start,start_column=col,end_row=r-1,end_column=col)
|
|
run_start = r; prev = cur
|
|
# F 먼저, E, D 순
|
|
for col in (6,5,4):
|
|
# 연속 동일값 구간 병합(빈칸은 위와 동일그룹으로 흡수)
|
|
r = 3
|
|
while r <= last:
|
|
v = ws.cell(r,col).value
|
|
if v is None: r+=1; continue
|
|
r2 = r+1
|
|
while r2 <= last and ws.cell(r2,col).value is None:
|
|
# 상위 컬럼이 새로 시작하면 끊기
|
|
if col>=5 and ws.cell(r2,4).value is not None: break
|
|
if col>=6 and ws.cell(r2,5).value is not None: break
|
|
r2 += 1
|
|
if r2-1 > r:
|
|
ws.merge_cells(start_row=r,start_column=col,end_row=r2-1,end_column=col)
|
|
r = r2
|
|
wb.save(OUT)
|
|
print(f'저장: {OUT} 데이터행 {len(records)} (3~{last})')
|
|
# 통계
|
|
from collections import Counter
|
|
Lc = Counter(r.get('L','') for r in records)
|
|
print('L 분포:', dict(Lc))
|
|
att = sum(1 for r in records if r.get('O') and '유형' in str(r.get('O')))
|
|
print('공공누리 부착행:', att, '| 새 순번:', sum(1 for r in records if r.get('newseq')))
|
|
|
|
if __name__ == '__main__':
|
|
main()
|