df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
243 lines
9.0 KiB
Python
243 lines
9.0 KiB
Python
"""공주시 Phase 1 — 사이트맵 → 엑셀 (D~K + 병합·정렬·스타일).
|
|
|
|
매뉴얼: D:\\01.프로젝트\\DB수집\\사이트맵_수집_매뉴얼.md
|
|
"""
|
|
import re
|
|
import shutil
|
|
import warnings
|
|
from copy import copy
|
|
from urllib.parse import urljoin
|
|
|
|
import openpyxl
|
|
import requests
|
|
from bs4 import BeautifulSoup
|
|
from openpyxl.styles import Alignment, Font
|
|
|
|
warnings.filterwarnings('ignore')
|
|
|
|
INSTITUTION = '공주시'
|
|
BASE = 'https://www.gongju.go.kr'
|
|
SITEMAP_URL = 'https://www.gongju.go.kr/kr/sitemap.do'
|
|
SHEET_NAME = '02_공주시'
|
|
TEMPLATE = r'D:\01.프로젝트\DB수집\자료_취합_예시.xlsx'
|
|
OUTPUT = r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청남도\2.공주시\충청남도_공주시.xlsx'
|
|
|
|
UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
|
|
H = {'User-Agent': UA}
|
|
|
|
ONCLICK_PAT = re.compile(r"location\.href\s*=\s*encodeURI\(\s*['\"]([^'\"]+)['\"]\s*\)")
|
|
|
|
|
|
def extract_href(a):
|
|
"""공주시는 href가 #enURI..., 실제 URL은 onclick의 encodeURI 인자."""
|
|
href = (a.get('href') or '').strip()
|
|
if href and not href.startswith('#') and not href.startswith('javascript:'):
|
|
return href
|
|
onclick = a.get('onclick') or ''
|
|
m = ONCLICK_PAT.search(onclick)
|
|
if m:
|
|
return m.group(1)
|
|
return ''
|
|
|
|
|
|
def abs_url(href):
|
|
if not href:
|
|
return ''
|
|
href = href.strip()
|
|
if href.startswith(('javascript:', '#')):
|
|
return ''
|
|
if href.startswith(('http://', 'https://')):
|
|
return href
|
|
return urljoin(BASE + '/', href)
|
|
|
|
|
|
def is_external(url):
|
|
return url.startswith(('http://', 'https://')) and 'gongju.go.kr' not in url
|
|
|
|
|
|
def walk_ul(ul, depth, base_path, out):
|
|
for li in ul.find_all('li', recursive=False):
|
|
a = li.find('a', recursive=False)
|
|
if not a:
|
|
continue
|
|
text = a.get_text(strip=True).replace('\xa0', '').strip()
|
|
href = extract_href(a)
|
|
path = base_path[:depth] + [(text, href)]
|
|
nested = li.find('ul', recursive=False)
|
|
if nested:
|
|
out.append({'path': list(path), 'href': href})
|
|
walk_ul(nested, depth + 1, path, out)
|
|
else:
|
|
out.append({'path': list(path), 'href': href})
|
|
|
|
|
|
def main():
|
|
print('[1] 사이트맵 가져오기')
|
|
r = requests.get(SITEMAP_URL, headers=H, timeout=20, verify=False)
|
|
r.encoding = r.apparent_encoding
|
|
soup = BeautifulSoup(r.text, 'html.parser')
|
|
dls = soup.select('.sitemap dl')
|
|
# 헤더 메뉴 dl 같은 노이즈가 섞일 수 있어 dt에 a 있고 dd 있는 것만
|
|
dls = [dl for dl in dls if dl.find('dt') and dl.find('dd')]
|
|
print(f' 대분류 dl: {len(dls)}개')
|
|
|
|
raw_rows = []
|
|
for dl in dls:
|
|
dt = dl.find('dt')
|
|
dt_a = dt.find('a') if dt else None
|
|
menu_name = dt_a.get_text(strip=True) if dt_a else ''
|
|
for dd in dl.find_all('dd', recursive=False):
|
|
b = dd.find('b')
|
|
b_a = b.find('a') if b else None
|
|
mid_name = b_a.get_text(strip=True) if b_a else ''
|
|
mid_href = extract_href(b_a) if b_a else ''
|
|
nested = dd.find('ul', recursive=False)
|
|
if nested:
|
|
tmp = []
|
|
walk_ul(nested, 0, [], tmp)
|
|
for item in tmp:
|
|
p = item['path']
|
|
row = {'D': menu_name, 'E': mid_name, 'href': item['href'],
|
|
'F': '', 'G': '', 'H': '', 'I': '', 'J': ''}
|
|
for di, (t, _) in enumerate(p):
|
|
col = 'FGHIJ'[di] if di < 5 else 'J'
|
|
row[col] = t
|
|
raw_rows.append(row)
|
|
else:
|
|
raw_rows.append({'D': menu_name, 'E': mid_name, 'href': mid_href,
|
|
'F': '', 'G': '', 'H': '', 'I': '', 'J': ''})
|
|
|
|
print(f'[2] 원본 행: {len(raw_rows)}')
|
|
|
|
# 부모-자식 URL 중복 제거
|
|
final_rows = []
|
|
i = 0
|
|
removed = 0
|
|
while i < len(raw_rows):
|
|
row = raw_rows[i]
|
|
if (i + 1 < len(raw_rows)
|
|
and row.get('G', '') == ''
|
|
and raw_rows[i + 1].get('D') == row.get('D')
|
|
and raw_rows[i + 1].get('E') == row.get('E')
|
|
and raw_rows[i + 1].get('F') == row.get('F')
|
|
and raw_rows[i + 1].get('G', '') != ''
|
|
and raw_rows[i + 1].get('href') == row.get('href')):
|
|
removed += 1
|
|
i += 1
|
|
continue
|
|
final_rows.append(row)
|
|
i += 1
|
|
print(f'[3] 부모-자식 URL 중복 삭제: {removed}개 → 최종 {len(final_rows)}행')
|
|
|
|
print('[4] 엑셀 템플릿 복사')
|
|
shutil.copy(TEMPLATE, OUTPUT)
|
|
wb = openpyxl.load_workbook(OUTPUT)
|
|
ws = wb.active
|
|
ws.title = SHEET_NAME
|
|
|
|
HEADER = {'B1:R1', 'S1:W1', 'Y1:AA1'}
|
|
for rng in [str(m) for m in ws.merged_cells.ranges if str(m) not in HEADER]:
|
|
ws.unmerge_cells(rng)
|
|
for row in ws.iter_rows(min_row=3, max_row=ws.max_row, min_col=1, max_col=ws.max_column):
|
|
for cell in row:
|
|
cell.value = None
|
|
|
|
START = 3
|
|
template_r = 3
|
|
cur_max = ws.max_row
|
|
for idx, item in enumerate(final_rows, start=START):
|
|
if idx > cur_max:
|
|
for c in range(1, ws.max_column + 1):
|
|
srcc = ws.cell(template_r, c)
|
|
tgt = ws.cell(idx, c)
|
|
if srcc.has_style:
|
|
tgt.font = copy(srcc.font)
|
|
tgt.fill = copy(srcc.fill)
|
|
tgt.border = copy(srcc.border)
|
|
tgt.alignment = copy(srcc.alignment)
|
|
tgt.number_format = srcc.number_format
|
|
tgt.protection = copy(srcc.protection)
|
|
url = abs_url(item.get('href', ''))
|
|
ws.cell(idx, 2).value = idx - 2
|
|
ws.cell(idx, 3).value = INSTITUTION
|
|
ws.cell(idx, 4).value = item.get('D', '')
|
|
ws.cell(idx, 5).value = item.get('E', '')
|
|
ws.cell(idx, 6).value = item.get('F', '')
|
|
ws.cell(idx, 7).value = item.get('G', '')
|
|
ws.cell(idx, 8).value = item.get('H', '')
|
|
ws.cell(idx, 9).value = item.get('I', '')
|
|
ws.cell(idx, 10).value = item.get('J', '')
|
|
ws.cell(idx, 11).value = url
|
|
if is_external(url):
|
|
ws.cell(idx, 19).value = '외부링크'
|
|
|
|
END = START + len(final_rows) - 1
|
|
print(f'[5] 데이터 기입: {START}~{END}')
|
|
|
|
center = Alignment(horizontal='center', vertical='center', wrap_text=True)
|
|
left = Alignment(horizontal='left', vertical='center', wrap_text=False)
|
|
|
|
def merge_runs(col_letter, col_idx, group_cols=()):
|
|
runs = []
|
|
cur_val = ws.cell(START, col_idx).value
|
|
cur_grp = tuple(ws.cell(START, g).value for g in group_cols)
|
|
run_start = START
|
|
for r in range(START + 1, END + 1):
|
|
v = ws.cell(r, col_idx).value
|
|
g = tuple(ws.cell(r, gg).value for gg in group_cols)
|
|
if v == cur_val and g == cur_grp:
|
|
continue
|
|
if cur_val not in (None, '') and r - 1 > run_start:
|
|
runs.append((run_start, r - 1))
|
|
cur_val, cur_grp, run_start = v, g, r
|
|
if cur_val not in (None, '') and END > run_start:
|
|
runs.append((run_start, END))
|
|
for s, e in runs:
|
|
ws.merge_cells(f'{col_letter}{s}:{col_letter}{e}')
|
|
ws.cell(s, col_idx).alignment = center
|
|
return len(runs)
|
|
|
|
# 병합 순서 F→E→D (D를 먼저 병합하면 2행부터 D=None이 되어 E 그룹키가 깨짐)
|
|
n_f = merge_runs('F', 6, group_cols=(4, 5))
|
|
n_e = merge_runs('E', 5, group_cols=(4,))
|
|
n_d = merge_runs('D', 4)
|
|
for r in range(START, END + 1):
|
|
for c in (4, 5, 6, 7):
|
|
if ws.cell(r, c).value is not None:
|
|
ws.cell(r, c).alignment = center
|
|
print(f'[6] 병합 — D:{n_d} E:{n_e} F:{n_f}')
|
|
|
|
for r in range(1, END + 1):
|
|
ws.row_dimensions[r].height = 15
|
|
|
|
link_n = 0
|
|
for r in range(START, END + 1):
|
|
cell = ws.cell(r, 11)
|
|
u = cell.value
|
|
if u and isinstance(u, str) and u.startswith(('http://', 'https://')):
|
|
cell.hyperlink = u
|
|
old = cell.font
|
|
cell.font = Font(name=old.name or '맑은 고딕', size=old.size or 11,
|
|
bold=old.bold, italic=old.italic,
|
|
color='0000FF', underline='single')
|
|
cell.alignment = left
|
|
link_n += 1
|
|
print(f'[7] 행 높이 15 + K 하이퍼링크 {link_n}개 (좌측 정렬)')
|
|
|
|
wb.save(OUTPUT)
|
|
print(f'[8] 저장: {OUTPUT}')
|
|
|
|
# 통계
|
|
print('\n=== 통계 ===')
|
|
from collections import Counter
|
|
cnt = Counter(item.get('D') for item in final_rows)
|
|
print(f'대분류별 행수:')
|
|
for k, v in cnt.items():
|
|
print(f' {k}: {v}')
|
|
ext_n = sum(1 for it in final_rows if is_external(abs_url(it.get('href', ''))))
|
|
print(f'외부링크: {ext_n}')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|