# -*- coding: utf-8 -*- """보령시 전면 재수집(복사본 베이스). - 본문 = #contents (기존 #txt는 전역검색 포함→게시판 오판) - 인페이지탭 = ul.depth_tab → 매뉴얼 1-5b/1-5c로 leaf+1 컬럼 전개(논산식) · 탭이 페이지/게시판 섞여도 각각 개별 행, 게시판은 총건수, 페이지는 #grap 인페이지탭수 or 1 - 검출 로직(L/M/N/O/P/Q)·미디어·공공누리 = 공주시 _phase234.py 이식 - 순번3 사전정보공표 = 사전 정의된 16 leaf 블록 삽입(brm1/dept 필터) 출력: 충청남도_보령시.xlsx 재작성(헤더·스타일·병합·순번·하이퍼링크). """ import re, time, warnings, json from copy import copy from urllib.parse import urljoin, urlparse from concurrent.futures import ThreadPoolExecutor, as_completed import requests, openpyxl from bs4 import BeautifulSoup from openpyxl.worksheet.hyperlink import Hyperlink warnings.filterwarnings('ignore') DIR = r'작업파일/광역_사이트맵/충청남도/6.보령시/' SRC = DIR + '보령시 복사본.xlsx' OUT = DIR + '충청남도_보령시.xlsx' DOMAIN = 'brcn.go.kr' H = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120 Safari/537.36'} ADV_URL = 'https://www.brcn.go.kr/prog/openInformation/advanceInformation/kor/sub01_01_03/list.do' # ---------------- fetch ---------------- _cache = {} def fetch(url, timeout=15): if url in _cache: return _cache[url] s = None try: r = requests.get(url, headers=H, timeout=timeout, verify=False) if r.status_code == 200: s = BeautifulSoup(r.content, 'html.parser') except Exception: s = None _cache[url] = s return s def body_of(soup): return soup.select_one('#contents') or soup.select_one('#container') or soup # ---------------- board / page detection ---------------- TOTAL_PATS = [re.compile(r'총\s*게시물\s*[:\s]*([\d,]+)'), re.compile(r'총\s*([\d,]+)\s*건'), re.compile(r'전체\s*([\d,]+)\s*건'), re.compile(r'게시물\s*수\s*[:\s]*([\d,]+)')] def get_total(body): bt = body.select_one('.board_total, .total, .b_total') if bt: m = re.search(r'([\d,]+)', bt.get_text()) if m: return int(m.group(1).replace(',', '')) txt = body.get_text(' ', strip=True) for p in TOTAL_PATS: m = p.search(txt) if m: return int(m.group(1).replace(',', '')) return None def anchor_tab_count(body): best = 0 for ul in body.select('ul.tab-ul, ul.type2, ul.type3, ul.tab_type, .tab_wrap ul, .tabBox ul'): las = ul.find_all('a') if len(las) >= 2 and all((a.get('href') or '').strip().startswith('#') for a in las): best = max(best, len(las)) return best def is_board_url(u): return bool(u) and ('selectBoardList' in u or '/list.do' in u or 'BBSMSTR' in u or 'bbsId=' in u) and '/viewer/' not in u def detect_form(body, url): # 직원안내·전화번호안내·부서안내 등 deptOn 디렉토리 = 게시물 아님 → 페이지(논산 컨벤션 M=1) if 'deptOn' in url: return '페이지', 1 total = get_total(body) has_paging = bool(body.select('.pagination, .paging')) is_bbs = ('BBSMSTR' in url or 'selectBoardList' in url or 'bbsId=' in url) # 진짜 게시판: 총건수/페이징이 있거나 BBSMSTR/selectBoardList 보드 if (total is not None) or has_paging or is_bbs: return '게시판', (total if total is not None else 0) # 그 외(디렉토리성 list.do 포함) = 페이지 return '페이지', (anchor_tab_count(body) or 1) # ---------------- media / kogl (공주시 이식) ---------------- KOGL_IMG = re.compile(r'(?:new_)?img_opentype(\d{2})\.png', re.I) KOGL_LINK = re.compile(r'kogl\.or\.kr/info/licenseType(\d)', re.I) YT = re.compile(r'(?:youtube\.com|youtu\.be)', re.I) VIDEXT = re.compile(r'\.(mp4|webm|mov|avi)(?:\?|$)', re.I) def detect_media(body): has_text = len(body.get_text(strip=True)) > 30 has_image = False for img in body.find_all('img'): src = img.get('src', '') if not src or KOGL_IMG.search(src): continue if re.search(r'(btn_|icon|ico_|bullet|blank|spacer|move\.|no[-_]?img|arrow)', src, re.I): continue has_image = True; break has_video = bool(body.find_all('video')) if not has_video: for f in body.find_all('iframe'): if YT.search(f.get('src', '')): has_video = True; break if not has_video and VIDEXT.search(str(body)): has_video = True return has_image, has_video, has_text def n_string(t, i, v): p = [] if t: p.append('어문') if i: p.append('이미지') if v: p.append('영상') return ','.join(p) if p else '없음' def img_anchor(img): p = img.parent while p is not None: if p.name == 'a': h = p.get('href', '') return bool(h) and not h.startswith('#') and not h.lower().startswith('javascript:') p = p.parent return False def detect_kogl(body): types = set(); qy = qn = False for a in body.find_all('a', href=True): m = KOGL_LINK.search(a['href']) if m: types.add(int(m.group(1))); qy = True for img in body.find_all('img'): m = KOGL_IMG.search(img.get('src', '')) if m: types.add(int(m.group(1))) if img_anchor(img): qy = True else: qn = True return types, ('Y' if qy else ('N' if qn else None)) DETAIL_FN = re.compile(r"fn[_a-zA-Z]*\(\s*['\"]?(\d+)") def detail_urls(body, list_url, limit=4): urls, seen = [], set() tb = body.select_one('table tbody') or body.find('table') if not tb: return urls for tr in tb.find_all('tr')[:limit*3]: a = tr.find('a', href=True) if a: h = a['href'] if 'view' in h.lower() and ('nttId' in h or 'Seq' in h or 'idx' in h.lower()): full = urljoin(list_url, h) if full not in seen: seen.add(full); urls.append(full) if len(urls) >= limit: break return urls def classify(url): """return dict L,M,N,O,P,Q (게시판은 상세 추적해 미디어/공공누리).""" s = fetch(url) if s is None: # 별도 시스템(서브도메인/결제앱 등) — 자동수집 불가, 페이지로 처리 return {'L': '페이지', 'M': 1, 'N': '어문', 'O': '미부착', 'P': '', 'Q': '', 'note': '별도 시스템(자동수집 불가)'} body = body_of(s) form, count = detect_form(body, url) has_i, has_v, has_t = detect_media(body) types, q = detect_kogl(body) P = '게시판' if types else '' qflags = [q] if q else [] if form == '게시판' and count and count > 0: for du in detail_urls(body, url, limit=2): ds = fetch(du, 10) if not ds: continue db = body_of(ds) di, dv, dt = detect_media(db) has_i |= di; has_v |= dv; has_t |= dt dtypes, dq = detect_kogl(db) if dtypes and not types and not P: P = '게시물' types |= dtypes if dq: qflags.append(dq) N = n_string(has_t, has_i, has_v) if form == '게시판' and count == 0: N = '없음' # 빈 게시판 if not types: O = '미부착'; P = ''; Q = '' else: O = ','.join(f'{n}유형' for n in sorted(types)) P = P or '게시판' Q = 'Y' if 'Y' in qflags else 'N' return {'L': form, 'M': count if form == '게시판' else (count or 1), 'N': N, 'O': O, 'P': P, 'Q': Q, 'note': ''} # ---------------- 사전정보공표 블록 (순번3) ---------------- def adv_block(): cat = [('일반공공행정','brm1=A',117),('공공질서및안전','brm1=B',26),('통신','brm1=C',7), ('산업중소기업','brm1=D',33),('보건','brm1=E',82),('사회복지','brm1=F',69), ('문화체육관광','brm1=G',35),('수송및교통','brm1=H',38),('농림해양수산','brm1=I',39), ('교육','brm1=J',11),('환경보호','brm1=K',45),('지역개발','brm1=L',28)] dep = [('시청','dept=1',405),('직속기관 사업소','dept=2',109),('시의회','dept=3',16),('읍면동','dept=4',0)] recs = [] for i,(h,qs,c) in enumerate(cat): recs.append({'F':'사전정보공표' if i==0 else None,'G':'정보분류별' if i==0 else None,'H':h, 'url':f'{ADV_URL}?{qs}&siteCode=kor&mno=sub01_01_03','L':'게시판','M':c,'N':'어문','O':'미부착','P':'','Q':''}) for i,(h,qs,c) in enumerate(dep): recs.append({'F':None,'G':'공표부서' if i==0 else None,'H':h, 'url':f'{ADV_URL}?{qs}&siteCode=kor&mno=sub01_01_03','L':'게시판','M':c,'N':'어문','O':'미부착','P':'','Q':''}) return recs # ---------------- 메뉴 읽기 + 전개 ---------------- def norm(href, base): if not href: return None href = href.strip() if href.startswith('#') or href.lower().startswith('javascript:') or href == '': return None return urljoin(base, href) def get_tabs(body, url): dt = body.select_one('ul.depth_tab') if not dt: return [] tabs = [] for li in dt.find_all('li'): a = li.find('a') if not a: continue h = norm(a.get('href'), url) if not h: continue tabs.append({'label': li.get_text(strip=True), 'href': h}) return tabs def main(): wb = openpyxl.load_workbook(SRC) ws = wb.active # 메뉴 행 수집 (복사본) menus = [] for r in range(3, ws.max_row + 1): D, E, F = ws.cell(r,4).value, ws.cell(r,5).value, ws.cell(r,6).value url = ws.cell(r,11).value menus.append({'r':r,'D':D,'E':E,'F':F,'url':url}) all_urls = set(m['url'] for m in menus if m['url']) print('메뉴 행:', len(menus)) # --- 병렬 사전 fetch: 모든 메뉴 URL (탭 탐지용) --- t0 = time.time() menu_urls = [m['url'] for m in menus if m['url'] and isinstance(m['url'],str) and DOMAIN in m['url'] and m['F']!='사전정보공표'] print('메뉴 URL 병렬 fetch:', len(menu_urls)) with ThreadPoolExecutor(max_workers=16) as ex: list(ex.map(lambda u: fetch(u), set(menu_urls))) print(f' 메뉴 fetch 완료 ({time.time()-t0:.0f}s)') # --- 스켈레톤 빌드(캐시 사용) + classify 대상 URL 수집 --- skel = [] # (rec_partial, classify_url_or_None) def leafcol(m): if m['F'] is not None: return 'F' if m['E'] is not None: return 'E' return 'D' for m in menus: url = m['url'] # 빈 junk 행(URL·D·E·F 전부 없음) 스킵 if (not url or not isinstance(url,str)) and not any([m['D'],m['E'],m['F']]): continue base = {'D':m['D'],'E':m['E'],'F':m['F'],'G':None,'H':None} if m['F'] == '사전정보공표': for j,b in enumerate(adv_block()): rec = {'D':None,'E':None,'F':None,'G':b['G'],'H':b['H'],'url':b['url'], 'L':b['L'],'M':b['M'],'N':b['N'],'O':b['O'],'P':b['P'],'Q':b['Q'],'newseq':(j==0)} if j==0: rec.update({'D':m['D'],'E':m['E'],'F':'사전정보공표'}) skel.append((rec, None)) continue if not url or not isinstance(url,str) or DOMAIN not in url: rec = dict(base); rec.update({'url':url,'L':'사이트','M':'','N':'','O':'','P':'','Q':'','newseq':True}) skel.append((rec, None)); continue s = _cache.get(url) tabs = get_tabs(body_of(s), url) if s else [] realtabs = [t for t in tabs if DOMAIN in t['href']] expand = len(realtabs) >= 2 and not all(t['href'] in all_urls for t in realtabs) lc = leafcol(m); sub = {'F':'G','E':'F','D':'E'}[lc] if expand: for j,t in enumerate(realtabs): if j==0: rec = dict(base); rec['url']=t['href']; rec[sub]=t['label']; rec['newseq']=True else: rec = {'D':None,'E':None,'F':None,'G':None,'H':None,'url':t['href'],'newseq':False} rec[sub]=t['label'] skel.append((rec, t['href'])) else: rec = dict(base); rec['url']=url; rec['newseq']=True skel.append((rec, url)) # --- 병렬 classify --- todo = sorted(set(u for _,u in skel if u)) print('classify 대상 URL:', len(todo)) cls = {} done=[0] def cl(u): r = classify(u); done[0]+=1 if done[0]%60==0: print(f' classify {done[0]}/{len(todo)} ({time.time()-t0:.0f}s)') return u, r with ThreadPoolExecutor(max_workers=16) as ex: for u,r in ex.map(cl, todo): cls[u] = r print(f' classify 완료 ({time.time()-t0:.0f}s)') records = [] for rec, u in skel: if u and u in cls: rec.update(cls[u]) records.append(rec) print(f'총 레코드: {len(records)} ({time.time()-t0:.0f}s)') json.dump([{k:v for k,v in r.items() if k!="newseq"} for r in records], open(DIR+'_records.json','w',encoding='utf-8'), ensure_ascii=False) write_excel(wb, ws, records) return # (이하 구버전 순차 경로 — 미사용) for idx, m in enumerate(menus): url = m['url'] base = {'D':m['D'],'E':m['E'],'F':m['F'],'G':None,'H':None} if m['F'] == '사전정보공표': for j,b in enumerate(adv_block()): rec = dict(base) if j==0: rec['url']=b['url'] else: rec={'D':None,'E':None,'F':None,'G':b['G'],'H':b['H'],'url':b['url']} rec.update({'G':b['G'] if j==0 else rec.get('G'),'H':b['H'], 'L':b['L'],'M':b['M'],'N':b['N'],'O':b['O'],'P':b['P'],'Q':b['Q'],'newseq':(j==0)}) # 첫 행은 F 유지 if j==0: rec['F']='사전정보공표' records.append(rec) continue # 외부 if not url or not isinstance(url,str) or DOMAIN not in url: rec = dict(base); rec.update({'url':url,'L':'사이트','M':'','N':'','O':'','P':'','Q':'','newseq':True}) records.append(rec); continue # fetch + depth_tab s = fetch(url) tabs = get_tabs(body_of(s), url) if s else [] realtabs = [t for t in tabs if DOMAIN in t['href']] # 전개 조건: 2+ 탭, 모두 사이트맵에 이미 있으면 스킵 expand = len(realtabs) >= 2 and not all(t['href'] in all_urls for t in realtabs) lc = leafcol(m) sub = {'F':'G','E':'F','D':'E'}[lc] # leaf+1 컬럼 if expand: for j,t in enumerate(realtabs): if j==0: rec = dict(base); rec['url']=t['href']; rec[sub]=t['label']; rec['newseq']=True else: rec = {'D':None,'E':None,'F':None,'G':None,'H':None,'url':t['href'],'newseq':False} rec[sub]=t['label'] c = classify(t['href']) rec.update(c); records.append(rec) else: rec = dict(base); rec['url']=url; rec['newseq']=True rec.update(classify(url)); records.append(rec) if (idx+1) % 40 == 0: print(f' 메뉴 {idx+1}/{len(menus)} 레코드 {len(records)} ({time.time()-t0:.0f}s)') print(f'총 레코드: {len(records)} ({time.time()-t0:.0f}s)') json.dump([{k:v for k,v in r.items() if k!="newseq"} for r in records], open(DIR+'_records.json','w',encoding='utf-8'), ensure_ascii=False) write_excel(wb, ws, records) def write_excel(wb, ws, records): # 템플릿 스타일 (복사본 row3) TPL = 3 tpl_style = {c: copy(ws.cell(TPL,c)._style) for c in range(1,28)} tpl_h = ws.row_dimensions[TPL].height # 기존 데이터행/병합 제거 for mr in list(ws.merged_cells.ranges): if mr.min_row >= 3: ws.unmerge_cells(str(mr)) if ws.max_row >= 3: ws.delete_rows(3, ws.max_row-2) # 기입 seq = 0 for i, rec in enumerate(records): r = 3 + i for c in range(1,28): ws.cell(r,c)._style = copy(tpl_style[c]) if tpl_h: ws.row_dimensions[r].height = tpl_h if rec.get('newseq'): seq += 1 ws.cell(r,2).value = seq ws.cell(r,3).value = '보령시' for col,key in [(4,'D'),(5,'E'),(6,'F'),(7,'G'),(8,'H')]: v = rec.get(key) if v is not None: ws.cell(r,col).value = v url = rec.get('url') if url: kc = ws.cell(r,11); kc.value = url kc.hyperlink = Hyperlink(ref=kc.coordinate, target=str(url)) for col,key in [(12,'L'),(13,'M'),(14,'N'),(15,'O'),(16,'P'),(17,'Q')]: v = rec.get(key) if v not in (None,''): ws.cell(r,col).value = v note = rec.get('note') if note: ws.cell(r,19).value = note last = 3 + len(records) - 1 # 병합 F→E→D def merge_col(col): run_start = 3; prev = ws.cell(3,col).value # 그룹키: D는 자체, E는 D+E, F는 D+E+F (상위 동일구간 안에서만) for r in range(4, last+2): cur = ws.cell(r,col).value if r<=last else '\x00END' # 상위 컬럼 경계에서도 끊기 boundary = False if col>=5 and r<=last and ws.cell(r,4).value is not None: boundary=True # 새 D 시작 if col>=6 and r<=last and ws.cell(r,5).value is not None: boundary=True # 새 E 시작 if cur is not None or r>last or boundary: if r-1 > run_start and prev is not None: ws.merge_cells(start_row=run_start,start_column=col,end_row=r-1,end_column=col) run_start = r; prev = cur # F 먼저, E, D 순 for col in (6,5,4): # 연속 동일값 구간 병합(빈칸은 위와 동일그룹으로 흡수) r = 3 while r <= last: v = ws.cell(r,col).value if v is None: r+=1; continue r2 = r+1 while r2 <= last and ws.cell(r2,col).value is None: # 상위 컬럼이 새로 시작하면 끊기 if col>=5 and ws.cell(r2,4).value is not None: break if col>=6 and ws.cell(r2,5).value is not None: break r2 += 1 if r2-1 > r: ws.merge_cells(start_row=r,start_column=col,end_row=r2-1,end_column=col) r = r2 wb.save(OUT) print(f'저장: {OUT} 데이터행 {len(records)} (3~{last})') # 통계 from collections import Counter Lc = Counter(r.get('L','') for r in records) print('L 분포:', dict(Lc)) att = sum(1 for r in records if r.get('O') and '유형' in str(r.get('O'))) print('공공누리 부착행:', att, '| 새 순번:', sum(1 for r in records if r.get('newseq'))) if __name__ == '__main__': main()