df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
661 lines
27 KiB
Python
661 lines
27 KiB
Python
"""충청북도 11개 시·군 Phase 1 일괄 처리.
|
|
|
|
매뉴얼: D:\\01.프로젝트\\DB수집\\사이트맵_수집_매뉴얼.md
|
|
출력: 각 폴더의 {기관명}.xlsx
|
|
"""
|
|
import os
|
|
import re
|
|
import shutil
|
|
import ssl
|
|
import sys
|
|
import warnings
|
|
from copy import copy
|
|
from urllib.parse import urljoin
|
|
|
|
import openpyxl
|
|
import requests
|
|
from requests.adapters import HTTPAdapter
|
|
from urllib3.util.ssl_ import create_urllib3_context
|
|
from bs4 import BeautifulSoup
|
|
from openpyxl.styles import Alignment, Font
|
|
|
|
warnings.filterwarnings('ignore')
|
|
|
|
UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
|
|
H = {'User-Agent': UA}
|
|
TEMPLATE = r'D:\01.프로젝트\DB수집\자료_취합_예시.xlsx'
|
|
|
|
|
|
class WeakSSLAdapter(HTTPAdapter):
|
|
"""레거시 SSL 핸드셰이크 허용 (영동군 등)."""
|
|
def init_poolmanager(self, *args, **kwargs):
|
|
ctx = create_urllib3_context()
|
|
ctx.set_ciphers('DEFAULT@SECLEVEL=0')
|
|
ctx.options |= 0x4 # OP_LEGACY_SERVER_CONNECT
|
|
ctx.check_hostname = False
|
|
ctx.verify_mode = ssl.CERT_NONE
|
|
kwargs['ssl_context'] = ctx
|
|
return super().init_poolmanager(*args, **kwargs)
|
|
|
|
|
|
def make_session(weak_ssl=False):
|
|
s = requests.Session()
|
|
s.headers.update(H)
|
|
if weak_ssl:
|
|
s.mount('https://', WeakSSLAdapter())
|
|
return s
|
|
|
|
|
|
def fetch_html(url, session=None, timeout=20, force_encoding=None):
|
|
s = session or requests.Session()
|
|
if not session:
|
|
s.headers.update(H)
|
|
r = s.get(url, timeout=timeout, verify=False, allow_redirects=True)
|
|
if force_encoding:
|
|
r.encoding = force_encoding
|
|
else:
|
|
# 메타 태그에서 charset 시도 → 없으면 apparent_encoding
|
|
meta_charset = re.search(rb'<meta[^>]*charset=["\']?\s*([\w-]+)', r.content[:4096], re.I)
|
|
if meta_charset:
|
|
r.encoding = meta_charset.group(1).decode('ascii', errors='ignore')
|
|
else:
|
|
r.encoding = r.apparent_encoding
|
|
return r.text
|
|
|
|
|
|
def clean_text(s):
|
|
return re.sub(r'\s+', ' ', s or '').strip().replace('\xa0', '')
|
|
|
|
|
|
def extract_href(a):
|
|
if a is None:
|
|
return ''
|
|
href = (a.get('href') or '').strip()
|
|
if not href or href.startswith('#') or href.lower().startswith('javascript:'):
|
|
return ''
|
|
return href
|
|
|
|
|
|
ENCODE_URI_PAT = re.compile(r"""(?:location\.href|window\.location(?:\.href)?)\s*=\s*encodeURI\(\s*['"]([^'"]+)['"]\s*\)""")
|
|
ONCLICK_HREF_PAT = re.compile(r"""(?:location\.href|window\.open)\s*\(?\s*['"]([^'"]+)['"]""")
|
|
|
|
|
|
def extract_href_with_onclick(a):
|
|
if a is None:
|
|
return ''
|
|
href = (a.get('href') or '').strip()
|
|
if href and not href.startswith('#') and not href.lower().startswith('javascript:'):
|
|
return href
|
|
onclick = a.get('onclick', '')
|
|
if onclick:
|
|
m = ENCODE_URI_PAT.search(onclick)
|
|
if m:
|
|
return m.group(1)
|
|
m = ONCLICK_HREF_PAT.search(onclick)
|
|
if m:
|
|
return m.group(1)
|
|
return ''
|
|
|
|
|
|
# ================================================================
|
|
# 파서들
|
|
# ================================================================
|
|
|
|
|
|
def parse_depth1_chungbuk(soup, base):
|
|
"""충북 e-Gov depth1 형: div.depth1 > ul.depth1_list > li.depth1_item > a.depth1_text + div.depth2 > [div.depth2_content|depth2_wrap] > ul.depth2_list."""
|
|
rows = []
|
|
container = soup.select_one('div.depth.depth1, div.depth1')
|
|
if not container:
|
|
return rows
|
|
top_ul = container.find('ul', class_=re.compile(r'depth1?_list'), recursive=False)
|
|
if not top_ul:
|
|
return rows
|
|
|
|
def walk_depth(ul, level, base_path, out):
|
|
for li in ul.find_all('li', recursive=False):
|
|
a = li.find('a', class_=re.compile(rf'depth{level}_text'), recursive=False) or li.find('a', recursive=False)
|
|
if not a:
|
|
continue
|
|
text = clean_text(a.get_text())
|
|
href = extract_href(a)
|
|
path = base_path + [(text, href)]
|
|
# Find next depth container — 다양한 wrapper 클래스 지원: _content / _wrap / 없음
|
|
next_div = li.find('div', class_=re.compile(rf'depth\s+depth{level+1}\b'), recursive=False) or \
|
|
li.find('div', class_=re.compile(rf'\bdepth{level+1}\b'), recursive=False)
|
|
if next_div:
|
|
# wrapper 가 있을 수도, 없을 수도
|
|
next_ul = next_div.find('ul', class_=re.compile(rf'depth{level+1}_list'), recursive=True)
|
|
if next_ul:
|
|
# 같은 깊이의 첫번째 ul.depth_list (자손 검색이지만 보통 1번 wrapper 안)
|
|
out.append({'path': list(path), 'href': href})
|
|
walk_depth(next_ul, level + 1, path, out)
|
|
continue
|
|
out.append({'path': list(path), 'href': href})
|
|
|
|
tmp = []
|
|
walk_depth(top_ul, 1, [], tmp)
|
|
cols = 'DEFGHIJ'
|
|
for item in tmp:
|
|
p = item['path']
|
|
row = {c: '' for c in 'DEFGHIJ'}
|
|
row['href'] = item['href']
|
|
for i, (t, _) in enumerate(p):
|
|
col = cols[i] if i < len(cols) else 'J'
|
|
row[col] = t
|
|
rows.append(row)
|
|
return rows
|
|
|
|
|
|
def parse_danyang_ld(soup, base):
|
|
"""단양군: ul#menu_sitemap.ld1 > li.cd1 > a.l1 + div.lb1 > ul.ld2 > li.cd2 > a.l2 + div.lb2 > ul.ld3 > ...
|
|
|
|
'menutype_empty' (메인 등) 은 스킵.
|
|
"""
|
|
rows = []
|
|
container = soup.select_one('ul#menu_sitemap')
|
|
if not container:
|
|
return rows
|
|
|
|
def walk(ul, level, base_path, out):
|
|
for li in ul.find_all('li', recursive=False):
|
|
a = li.find('a', class_=re.compile(rf'\bl{level}\b'), recursive=False) or li.find('a', recursive=False)
|
|
if not a:
|
|
continue
|
|
a_cls = ' '.join(a.get('class', []))
|
|
if 'menutype_empty' in a_cls:
|
|
continue # 메인 같은 빈 항목
|
|
text = clean_text(a.get_text())
|
|
href = extract_href(a)
|
|
path = base_path + [(text, href)]
|
|
next_div = li.find('div', class_=re.compile(rf'\blb{level}\b'), recursive=False)
|
|
if next_div:
|
|
next_ul = next_div.find('ul', class_=re.compile(rf'\bld{level+1}\b'), recursive=False)
|
|
if next_ul:
|
|
out.append({'path': list(path), 'href': href})
|
|
walk(next_ul, level + 1, path, out)
|
|
continue
|
|
out.append({'path': list(path), 'href': href})
|
|
|
|
tmp = []
|
|
walk(container, 1, [], tmp)
|
|
cols = 'DEFGHIJ'
|
|
for item in tmp:
|
|
p = item['path']
|
|
row = {c: '' for c in cols}
|
|
row['href'] = item['href']
|
|
for i, (t, _) in enumerate(p):
|
|
col = cols[i] if i < len(cols) else 'J'
|
|
row[col] = t
|
|
rows.append(row)
|
|
return rows
|
|
|
|
|
|
def parse_dl_dt_dd(soup, base, use_onclick=False):
|
|
"""계룡시·홍성군·영동군형: div.sitemap > dl > dt + dd > b > a + ul > li > a (+ ul > li > a)."""
|
|
rows = []
|
|
sm = soup.select_one('div.sitemap[class*=type2]') or soup.select_one('div.sitemap[class*=type1]') or soup.select_one('div.sitemap')
|
|
if not sm:
|
|
return rows
|
|
href_fn = extract_href_with_onclick if use_onclick else extract_href
|
|
|
|
def walk(ul, depth, base_path, out):
|
|
for li in ul.find_all('li', recursive=False):
|
|
a = li.find('a', recursive=False)
|
|
if not a:
|
|
continue
|
|
text = clean_text(a.get_text())
|
|
href = href_fn(a)
|
|
path = base_path[:depth] + [(text, href)]
|
|
nested = li.find('ul', recursive=False)
|
|
if nested:
|
|
out.append({'path': list(path), 'href': href})
|
|
walk(nested, depth + 1, path, out)
|
|
else:
|
|
out.append({'path': list(path), 'href': href})
|
|
|
|
for dl in sm.find_all('dl', recursive=False):
|
|
dt = dl.find('dt')
|
|
dt_a = dt.find('a') if dt else None
|
|
D = clean_text(dt_a.get_text() if dt_a else (dt.get_text() if dt else ''))
|
|
for dd in dl.find_all('dd', recursive=False):
|
|
b = dd.find('b')
|
|
b_a = b.find('a') if b else None
|
|
E = clean_text(b_a.get_text()) if b_a else ''
|
|
E_href = href_fn(b_a) if b_a else ''
|
|
nested = dd.find('ul', recursive=False)
|
|
if nested:
|
|
tmp = []
|
|
walk(nested, 0, [], tmp)
|
|
for item in tmp:
|
|
p = item['path']
|
|
row = {'D': D, 'E': E, 'href': item['href'],
|
|
'F': '', 'G': '', 'H': '', 'I': '', 'J': ''}
|
|
for di, (t, _) in enumerate(p):
|
|
col = 'FGHIJ'[di] if di < 5 else 'J'
|
|
row[col] = t
|
|
rows.append(row)
|
|
else:
|
|
rows.append({'D': D, 'E': E, 'href': E_href,
|
|
'F': '', 'G': '', 'H': '', 'I': '', 'J': ''})
|
|
return rows
|
|
|
|
|
|
def parse_yesan_depth(soup, base):
|
|
"""예산·청양·증평형: ul.depth1_ul > li > a (D, .th_1st/.th1_lnk) + [div.item >]? ul.depth2_ul > li > a (E) + ul.depth3_ul > li > a (F).
|
|
|
|
div.item wrapper 있을수도 없을수도 — 둘 다 지원.
|
|
a 클래스: .th_1st (예산), .th1_lnk (증평) — 첫 번째 a 가져옴.
|
|
"""
|
|
rows = []
|
|
container = soup.select_one('ul.depth1_ul')
|
|
if not container:
|
|
return rows
|
|
for top_li in container.find_all('li', recursive=False):
|
|
a1 = top_li.find('a', class_=re.compile(r'th[_]?1?(?:_1st|_lnk)?'), recursive=False) or \
|
|
top_li.find('a', recursive=False)
|
|
if not a1:
|
|
continue
|
|
D = clean_text(a1.get_text())
|
|
# div.item wrapper 있을 수도 없을 수도
|
|
item = top_li.find('div', class_='item', recursive=False)
|
|
d2_ul = (item.find('ul', class_='depth2_ul') if item else None) or \
|
|
top_li.find('ul', class_='depth2_ul', recursive=False)
|
|
if not d2_ul:
|
|
continue
|
|
for d2_li in d2_ul.find_all('li', recursive=False):
|
|
d2_a = d2_li.find('a', recursive=False)
|
|
if not d2_a:
|
|
continue
|
|
E = clean_text(d2_a.get_text())
|
|
E_href = extract_href(d2_a)
|
|
d3_ul = d2_li.find('ul', class_='depth3_ul', recursive=False)
|
|
if not d3_ul:
|
|
rows.append({'D': D, 'E': E, 'href': E_href,
|
|
'F': '', 'G': '', 'H': '', 'I': '', 'J': ''})
|
|
continue
|
|
rows.append({'D': D, 'E': E, 'href': E_href,
|
|
'F': '', 'G': '', 'H': '', 'I': '', 'J': ''})
|
|
for d3_li in d3_ul.find_all('li', recursive=False):
|
|
d3_a = d3_li.find('a', recursive=False)
|
|
if not d3_a:
|
|
continue
|
|
F = clean_text(d3_a.get_text())
|
|
rows.append({'D': D, 'E': E, 'F': F, 'href': extract_href(d3_a),
|
|
'G': '', 'H': '', 'I': '', 'J': ''})
|
|
return rows
|
|
|
|
|
|
def parse_jincheon_recursive(soup, base):
|
|
"""진천군: div.sitemap > ul > li > a + div > ul > li > a + div > ul > ... 재귀."""
|
|
rows = []
|
|
container = soup.select_one('div.sitemap')
|
|
if not container:
|
|
return rows
|
|
top_ul = container.find('ul', recursive=False)
|
|
if not top_ul:
|
|
return rows
|
|
|
|
def walk(ul, base_path, out):
|
|
for li in ul.find_all('li', recursive=False):
|
|
a = li.find('a', recursive=False)
|
|
if not a:
|
|
continue
|
|
text = clean_text(a.get_text())
|
|
if not text or text == '메뉴명이 없습니다.':
|
|
continue
|
|
href = extract_href(a)
|
|
path = base_path + [(text, href)]
|
|
next_div = li.find('div', recursive=False)
|
|
if next_div:
|
|
next_ul = next_div.find('ul', recursive=False)
|
|
if next_ul:
|
|
out.append({'path': list(path), 'href': href})
|
|
walk(next_ul, path, out)
|
|
continue
|
|
out.append({'path': list(path), 'href': href})
|
|
|
|
tmp = []
|
|
walk(top_ul, [], tmp)
|
|
cols = 'DEFGHIJ'
|
|
for item in tmp:
|
|
p = item['path']
|
|
row = {c: '' for c in cols}
|
|
row['href'] = item['href']
|
|
for i, (t, _) in enumerate(p):
|
|
col = cols[i] if i < len(cols) else 'J'
|
|
row[col] = t
|
|
rows.append(row)
|
|
return rows
|
|
|
|
|
|
def parse_cheongju_sitemap(soup, base):
|
|
"""청주시: div#sitemap > div.site_map_col > div.sitemap_box > h3 > a (D) + ul.sm2depth > li > a (E) + ul.sm3depth > li > a (F).
|
|
|
|
별도의 '인트로' sitemap_box 는 D만 있고 ul.sm2depth 단순한 외부 링크 묶음 — 그대로 둠.
|
|
"""
|
|
rows = []
|
|
container = soup.select_one('div#sitemap')
|
|
if not container:
|
|
return rows
|
|
for box in container.select('div.sitemap_box'):
|
|
h3 = box.find('h3')
|
|
h3_a = h3.find('a') if h3 else None
|
|
D = clean_text(h3_a.get_text() if h3_a else (h3.get_text() if h3 else ''))
|
|
D_href = extract_href(h3_a) if h3_a else ''
|
|
sm2 = box.find('ul', class_='sm2depth')
|
|
if not sm2:
|
|
rows.append({'D': D, 'href': D_href, 'E': '', 'F': '', 'G': '', 'H': '', 'I': '', 'J': ''})
|
|
continue
|
|
for li2 in sm2.find_all('li', recursive=False):
|
|
a2 = li2.find('a', recursive=False)
|
|
if not a2:
|
|
continue
|
|
E = clean_text(a2.get_text())
|
|
E_href = extract_href(a2)
|
|
sm3 = li2.find('ul', class_='sm3depth', recursive=False)
|
|
if not sm3:
|
|
rows.append({'D': D, 'E': E, 'href': E_href,
|
|
'F': '', 'G': '', 'H': '', 'I': '', 'J': ''})
|
|
continue
|
|
rows.append({'D': D, 'E': E, 'href': E_href,
|
|
'F': '', 'G': '', 'H': '', 'I': '', 'J': ''})
|
|
for li3 in sm3.find_all('li', recursive=False):
|
|
a3 = li3.find('a', recursive=False)
|
|
if not a3:
|
|
continue
|
|
F = clean_text(a3.get_text())
|
|
rows.append({'D': D, 'E': E, 'F': F, 'href': extract_href(a3),
|
|
'G': '', 'H': '', 'I': '', 'J': ''})
|
|
return rows
|
|
|
|
|
|
def parse_chungju_sitemap(soup, base):
|
|
"""충주시: div#sitemap > div.site_map_col > div.sitemap_box > h3.h0 > a (D) + ul > li > a.h4 (E) + ul.bu > li > a (F)."""
|
|
rows = []
|
|
container = soup.select_one('div#sitemap')
|
|
if not container:
|
|
return rows
|
|
for box in container.select('div.sitemap_box'):
|
|
h3 = box.find('h3')
|
|
h3_a = h3.find('a') if h3 else None
|
|
D = clean_text(h3_a.get_text() if h3_a else (h3.get_text() if h3 else ''))
|
|
D_href = extract_href(h3_a) if h3_a else ''
|
|
e_ul = box.find('ul', recursive=False)
|
|
if not e_ul:
|
|
rows.append({'D': D, 'href': D_href, 'E': '', 'F': '', 'G': '', 'H': '', 'I': '', 'J': ''})
|
|
continue
|
|
for e_li in e_ul.find_all('li', recursive=False):
|
|
a_E = e_li.find('a', class_='h4', recursive=False) or e_li.find('a', recursive=False)
|
|
if not a_E:
|
|
continue
|
|
E = clean_text(a_E.get_text())
|
|
E_href = extract_href(a_E)
|
|
f_ul = e_li.find('ul', class_='bu', recursive=False)
|
|
if not f_ul:
|
|
rows.append({'D': D, 'E': E, 'href': E_href,
|
|
'F': '', 'G': '', 'H': '', 'I': '', 'J': ''})
|
|
continue
|
|
rows.append({'D': D, 'E': E, 'href': E_href,
|
|
'F': '', 'G': '', 'H': '', 'I': '', 'J': ''})
|
|
for f_li in f_ul.find_all('li', recursive=False):
|
|
f_a = f_li.find('a', recursive=False)
|
|
if not f_a:
|
|
continue
|
|
F = clean_text(f_a.get_text())
|
|
rows.append({'D': D, 'E': E, 'F': F, 'href': extract_href(f_a),
|
|
'G': '', 'H': '', 'I': '', 'J': ''})
|
|
return rows
|
|
|
|
|
|
# ================================================================
|
|
# 사이트 설정
|
|
# ================================================================
|
|
|
|
SITES = [
|
|
{
|
|
'idx': 1, 'name': '괴산군', 'base': 'https://www.goesan.go.kr',
|
|
'sitemap': 'https://www.goesan.go.kr/www/sitemap.do?key=28',
|
|
'sheet': '01_괴산군', 'parser': parse_depth1_chungbuk,
|
|
'domain': 'goesan.go.kr', 'folder': r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청북도\1.괴산군',
|
|
'weak_ssl': False,
|
|
},
|
|
{
|
|
'idx': 2, 'name': '단양군', 'base': 'https://www.danyang.go.kr',
|
|
'sitemap': 'https://www.danyang.go.kr/dy21/98',
|
|
'sheet': '02_단양군', 'parser': parse_danyang_ld,
|
|
'domain': 'danyang.go.kr', 'folder': r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청북도\2.단양군',
|
|
'weak_ssl': False,
|
|
},
|
|
{
|
|
# 보은군 — www.boeun.go.kr DNS 차단, apex boeun.go.kr 는 정상(2026-05-30 재확인)
|
|
'idx': 3, 'name': '보은군', 'base': 'https://boeun.go.kr',
|
|
'sitemap': 'https://boeun.go.kr/www/sitemap.do?key=1323',
|
|
'sheet': '03_보은군', 'parser': parse_depth1_chungbuk,
|
|
'domain': 'boeun.go.kr', 'folder': r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청북도\3.보은군',
|
|
'weak_ssl': False,
|
|
},
|
|
{
|
|
'idx': 4, 'name': '영동군', 'base': 'https://www.yd21.go.kr',
|
|
'sitemap': 'https://www.yd21.go.kr/kr/html/guide/0701.html',
|
|
'sheet': '04_영동군', 'parser': parse_dl_dt_dd,
|
|
'domain': 'yd21.go.kr', 'folder': r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청북도\4.영동군',
|
|
'weak_ssl': True,
|
|
},
|
|
{
|
|
'idx': 5, 'name': '옥천군', 'base': 'https://www.oc.go.kr',
|
|
'sitemap': 'https://www.oc.go.kr/www/sub.do?key=121',
|
|
'sheet': '05_옥천군', 'parser': parse_depth1_chungbuk,
|
|
'domain': 'oc.go.kr', 'folder': r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청북도\5.옥천군',
|
|
'weak_ssl': False,
|
|
},
|
|
{
|
|
'idx': 6, 'name': '음성군', 'base': 'https://www.eumseong.go.kr',
|
|
'sitemap': 'https://www.eumseong.go.kr/www/sub.do?key=722',
|
|
'sheet': '06_음성군', 'parser': parse_depth1_chungbuk,
|
|
'domain': 'eumseong.go.kr', 'folder': r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청북도\6.음성군',
|
|
'weak_ssl': False,
|
|
},
|
|
{
|
|
'idx': 7, 'name': '제천시', 'base': 'https://www.jecheon.go.kr',
|
|
'sitemap': 'https://www.jecheon.go.kr/www/sitemap.do?key=553',
|
|
'sheet': '07_제천시', 'parser': parse_depth1_chungbuk,
|
|
'domain': 'jecheon.go.kr', 'folder': r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청북도\7.제천시',
|
|
'weak_ssl': False,
|
|
},
|
|
{
|
|
'idx': 8, 'name': '증평군', 'base': 'https://www.jp.go.kr',
|
|
'sitemap': 'https://www.jp.go.kr/kor/sitemap_11.do',
|
|
'sheet': '08_증평군', 'parser': parse_yesan_depth,
|
|
'domain': 'jp.go.kr', 'folder': r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청북도\8.증평군',
|
|
'weak_ssl': False,
|
|
},
|
|
{
|
|
'idx': 9, 'name': '진천군', 'base': 'https://www.jincheon.go.kr',
|
|
'sitemap': 'https://www.jincheon.go.kr/home/sub.do?menukey=445',
|
|
'sheet': '09_진천군', 'parser': parse_jincheon_recursive,
|
|
'domain': 'jincheon.go.kr', 'folder': r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청북도\9.진천군',
|
|
'weak_ssl': False,
|
|
},
|
|
{
|
|
'idx': 10, 'name': '청주시', 'base': 'https://www.cheongju.go.kr',
|
|
'sitemap': 'https://www.cheongju.go.kr/www/sitemap.do?key=589',
|
|
'sheet': '10_청주시', 'parser': parse_cheongju_sitemap,
|
|
'domain': 'cheongju.go.kr', 'folder': r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청북도\10.청주시',
|
|
'weak_ssl': False,
|
|
},
|
|
{
|
|
'idx': 11, 'name': '충주시', 'base': 'https://www.chungju.go.kr',
|
|
'sitemap': 'https://www.chungju.go.kr/www/sub.do?key=692',
|
|
'sheet': '11_충주시', 'parser': parse_chungju_sitemap,
|
|
'domain': 'chungju.go.kr', 'folder': r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청북도\11.충주시',
|
|
'weak_ssl': False,
|
|
},
|
|
]
|
|
|
|
|
|
# ================================================================
|
|
# 엑셀 생성 (공통)
|
|
# ================================================================
|
|
|
|
|
|
def write_excel(site, raw_rows):
|
|
name = site['name']
|
|
base = site['base']
|
|
domain = site['domain']
|
|
output = f"{site['folder']}\\{os.path.basename(os.path.dirname(site['folder']))}_{name}.xlsx"
|
|
|
|
def abs_url(href):
|
|
if not href:
|
|
return ''
|
|
href = href.strip()
|
|
if href.startswith(('javascript:', '#')):
|
|
return ''
|
|
if href.startswith(('http://', 'https://')):
|
|
return href
|
|
return urljoin(base + '/', href)
|
|
|
|
def is_external(url):
|
|
return url.startswith(('http://', 'https://')) and domain not in url
|
|
|
|
# 부모-자식 URL 중복 제거
|
|
final_rows = []
|
|
i = 0
|
|
removed = 0
|
|
while i < len(raw_rows):
|
|
row = raw_rows[i]
|
|
if (i + 1 < len(raw_rows)
|
|
and row.get('G', '') == ''
|
|
and raw_rows[i + 1].get('D') == row.get('D')
|
|
and raw_rows[i + 1].get('E') == row.get('E')
|
|
and raw_rows[i + 1].get('F') == row.get('F')
|
|
and raw_rows[i + 1].get('G', '') != ''
|
|
and raw_rows[i + 1].get('href') == row.get('href')):
|
|
removed += 1
|
|
i += 1
|
|
continue
|
|
final_rows.append(row)
|
|
i += 1
|
|
|
|
print(f' [{name}] 원본 {len(raw_rows)} → 중복 {removed} → 최종 {len(final_rows)}')
|
|
if not final_rows:
|
|
print(f' [{name}] !! 행 0개 — 파서 점검 필요. 엑셀 생성 스킵.')
|
|
return False
|
|
|
|
shutil.copy(TEMPLATE, output)
|
|
wb = openpyxl.load_workbook(output)
|
|
ws = wb.active
|
|
ws.title = site['sheet']
|
|
|
|
HEADER = {'B1:R1', 'S1:W1', 'Y1:AA1'}
|
|
for rng in [str(m) for m in ws.merged_cells.ranges if str(m) not in HEADER]:
|
|
ws.unmerge_cells(rng)
|
|
for row in ws.iter_rows(min_row=3, max_row=ws.max_row, min_col=1, max_col=ws.max_column):
|
|
for cell in row:
|
|
cell.value = None
|
|
|
|
START = 3
|
|
template_r = 3
|
|
cur_max = ws.max_row
|
|
for idx, item in enumerate(final_rows, start=START):
|
|
if idx > cur_max:
|
|
for c in range(1, ws.max_column + 1):
|
|
srcc = ws.cell(template_r, c)
|
|
tgt = ws.cell(idx, c)
|
|
if srcc.has_style:
|
|
tgt.font = copy(srcc.font)
|
|
tgt.fill = copy(srcc.fill)
|
|
tgt.border = copy(srcc.border)
|
|
tgt.alignment = copy(srcc.alignment)
|
|
tgt.number_format = srcc.number_format
|
|
tgt.protection = copy(srcc.protection)
|
|
url = abs_url(item.get('href', ''))
|
|
ws.cell(idx, 2).value = idx - 2
|
|
ws.cell(idx, 3).value = name
|
|
ws.cell(idx, 4).value = item.get('D', '')
|
|
ws.cell(idx, 5).value = item.get('E', '')
|
|
ws.cell(idx, 6).value = item.get('F', '')
|
|
ws.cell(idx, 7).value = item.get('G', '')
|
|
ws.cell(idx, 8).value = item.get('H', '')
|
|
ws.cell(idx, 9).value = item.get('I', '')
|
|
ws.cell(idx, 10).value = item.get('J', '')
|
|
ws.cell(idx, 11).value = url
|
|
if is_external(url):
|
|
ws.cell(idx, 19).value = '외부링크'
|
|
|
|
END = START + len(final_rows) - 1
|
|
center = Alignment(horizontal='center', vertical='center', wrap_text=True)
|
|
left = Alignment(horizontal='left', vertical='center', wrap_text=False)
|
|
|
|
def merge_runs(col_letter, col_idx, group_cols=()):
|
|
runs = []
|
|
cur_val = ws.cell(START, col_idx).value
|
|
cur_grp = tuple(ws.cell(START, g).value for g in group_cols)
|
|
run_start = START
|
|
for r in range(START + 1, END + 1):
|
|
v = ws.cell(r, col_idx).value
|
|
g = tuple(ws.cell(r, gg).value for gg in group_cols)
|
|
if v == cur_val and g == cur_grp:
|
|
continue
|
|
if cur_val not in (None, '') and r - 1 > run_start:
|
|
runs.append((run_start, r - 1))
|
|
cur_val, cur_grp, run_start = v, g, r
|
|
if cur_val not in (None, '') and END > run_start:
|
|
runs.append((run_start, END))
|
|
for s, e in runs:
|
|
ws.merge_cells(f'{col_letter}{s}:{col_letter}{e}')
|
|
ws.cell(s, col_idx).alignment = center
|
|
return len(runs)
|
|
|
|
# 병합 순서 F→E→D (D를 먼저 병합하면 2행부터 D=None이 되어 E 그룹키가 깨짐)
|
|
n_f = merge_runs('F', 6, group_cols=(4, 5))
|
|
n_e = merge_runs('E', 5, group_cols=(4,))
|
|
n_d = merge_runs('D', 4)
|
|
for r in range(START, END + 1):
|
|
for c in (4, 5, 6, 7):
|
|
if ws.cell(r, c).value is not None:
|
|
ws.cell(r, c).alignment = center
|
|
|
|
for r in range(1, END + 1):
|
|
ws.row_dimensions[r].height = 15
|
|
|
|
link_n = 0
|
|
for r in range(START, END + 1):
|
|
cell = ws.cell(r, 11)
|
|
u = cell.value
|
|
if u and isinstance(u, str) and u.startswith(('http://', 'https://')):
|
|
cell.hyperlink = u
|
|
old = cell.font
|
|
cell.font = Font(name=old.name or '맑은 고딕', size=old.size or 11,
|
|
bold=old.bold, italic=old.italic,
|
|
color='0000FF', underline='single')
|
|
cell.alignment = left
|
|
link_n += 1
|
|
|
|
wb.save(output)
|
|
ext_n = sum(1 for r in range(START, END + 1) if ws.cell(r, 19).value == '외부링크')
|
|
print(f' [{name}] 병합 D:{n_d} E:{n_e} F:{n_f} | 외부링크 {ext_n} | K링크 {link_n} → {output}')
|
|
return True
|
|
|
|
|
|
def main():
|
|
targets = sys.argv[1:] if len(sys.argv) > 1 else [s['name'] for s in SITES]
|
|
for site in SITES:
|
|
if site['name'] not in targets:
|
|
continue
|
|
print(f"\n{'='*60}\n[{site['idx']}.{site['name']}] {site['sitemap']}\n{'='*60}")
|
|
try:
|
|
sess = make_session(weak_ssl=site.get('weak_ssl', False))
|
|
html = fetch_html(site['sitemap'], session=sess)
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
raw_rows = site['parser'](soup, site['base'])
|
|
write_excel(site, raw_rows)
|
|
except Exception as e:
|
|
print(f' [{site["name"]}] !! 실패: {e}')
|
|
import traceback
|
|
traceback.print_exc()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|