df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
708 lines
28 KiB
Python
708 lines
28 KiB
Python
"""전북특별자치도 14개 시·군 + 제주특별자치도 2개 시 Phase 1 일괄 처리.
|
|
|
|
매뉴얼: D:\\01.프로젝트\\DB수집\\사이트맵_수집_매뉴얼.md
|
|
출력: 각 폴더의 {기관명}.xlsx (D~K열)
|
|
|
|
파서 매핑은 _probe_jeonbuk*.py 탐색 결과 기반.
|
|
"""
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import ssl
|
|
import sys
|
|
import warnings
|
|
from copy import copy
|
|
from urllib.parse import urljoin
|
|
|
|
import openpyxl
|
|
import requests
|
|
from requests.adapters import HTTPAdapter
|
|
from urllib3.util.ssl_ import create_urllib3_context
|
|
from bs4 import BeautifulSoup
|
|
from openpyxl.styles import Alignment, Font
|
|
|
|
warnings.filterwarnings('ignore')
|
|
|
|
UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
|
|
H = {'User-Agent': UA}
|
|
ROOT = r'D:\01.프로젝트\DB수집'
|
|
TEMPLATE = ROOT + r'\자료_취합_예시.xlsx'
|
|
|
|
|
|
class WeakSSLAdapter(HTTPAdapter):
|
|
def init_poolmanager(self, *args, **kwargs):
|
|
ctx = create_urllib3_context()
|
|
ctx.set_ciphers('DEFAULT@SECLEVEL=0')
|
|
ctx.options |= 0x4
|
|
ctx.check_hostname = False
|
|
ctx.verify_mode = ssl.CERT_NONE
|
|
kwargs['ssl_context'] = ctx
|
|
return super().init_poolmanager(*args, **kwargs)
|
|
|
|
|
|
def make_session(weak_ssl=False):
|
|
s = requests.Session()
|
|
s.headers.update(H)
|
|
if weak_ssl:
|
|
s.mount('https://', WeakSSLAdapter())
|
|
return s
|
|
|
|
|
|
def fetch_html(url, session=None, timeout=25):
|
|
s = session or requests.Session()
|
|
if not session:
|
|
s.headers.update(H)
|
|
r = s.get(url, timeout=timeout, verify=False, allow_redirects=True)
|
|
meta = re.search(rb'<meta[^>]*charset=["\']?\s*([\w-]+)', r.content[:4096], re.I)
|
|
r.encoding = meta.group(1).decode('ascii', errors='ignore') if meta else r.apparent_encoding
|
|
return r.text
|
|
|
|
|
|
def clean_text(s):
|
|
return re.sub(r'\s+', ' ', s or '').strip().replace('\xa0', '').lstrip('-').strip()
|
|
|
|
|
|
def extract_href(a):
|
|
if a is None:
|
|
return ''
|
|
href = (a.get('href') or '').strip()
|
|
if not href or href.startswith('#') or href.lower().startswith('javascript:'):
|
|
return ''
|
|
return href
|
|
|
|
|
|
COLS = 'DEFGHIJ'
|
|
|
|
|
|
def rows_from_paths(tmp):
|
|
"""[{'path':[(text,href)...], 'href':..}] → D~J dict 행 리스트."""
|
|
rows = []
|
|
for item in tmp:
|
|
p = item['path']
|
|
row = {c: '' for c in COLS}
|
|
row['href'] = item['href']
|
|
for i, (t, _) in enumerate(p):
|
|
row[COLS[i] if i < len(COLS) else 'J'] = t
|
|
rows.append(row)
|
|
return rows
|
|
|
|
|
|
def walk_ul(ul, base_path, out, recursive_li=True):
|
|
"""ul > li > a (+ 중첩 ul) 재귀. 망가진 마크업(li 안 li)도 허용."""
|
|
for li in ul.find_all('li', recursive=False):
|
|
a = li.find('a', recursive=False)
|
|
if not a:
|
|
# div>a 형태(전주) 허용
|
|
d = li.find('div', recursive=False)
|
|
a = d.find('a', recursive=False) if d else None
|
|
if not a:
|
|
continue
|
|
text = clean_text(a.get_text())
|
|
href = extract_href(a)
|
|
path = base_path + [(text, href)]
|
|
if not text:
|
|
continue
|
|
# 자식 ul (정상) 또는 li 직접 중첩(망가진 마크업)
|
|
child_uls = li.find_all('ul', recursive=False)
|
|
child_lis = [c for c in li.find_all('li', recursive=False)]
|
|
if child_uls:
|
|
out.append({'path': list(path), 'href': href})
|
|
for cul in child_uls:
|
|
walk_ul(cul, path, out)
|
|
elif recursive_li and child_lis:
|
|
out.append({'path': list(path), 'href': href})
|
|
for cli in child_lis:
|
|
# cli를 단일 li로 감싼 가짜 ul처럼 처리
|
|
sub = a.find_parent() # not used
|
|
_walk_single_li(cli, path, out)
|
|
else:
|
|
out.append({'path': list(path), 'href': href})
|
|
|
|
|
|
def _walk_single_li(li, base_path, out):
|
|
a = li.find('a', recursive=False)
|
|
if not a:
|
|
d = li.find('div', recursive=False)
|
|
a = d.find('a', recursive=False) if d else None
|
|
if not a:
|
|
return
|
|
text = clean_text(a.get_text())
|
|
href = extract_href(a)
|
|
path = base_path + [(text, href)]
|
|
if not text:
|
|
return
|
|
child_uls = li.find_all('ul', recursive=False)
|
|
child_lis = li.find_all('li', recursive=False)
|
|
if child_uls:
|
|
out.append({'path': list(path), 'href': href})
|
|
for cul in child_uls:
|
|
walk_ul(cul, path, out)
|
|
elif child_lis:
|
|
out.append({'path': list(path), 'href': href})
|
|
for cli in child_lis:
|
|
_walk_single_li(cli, path, out)
|
|
else:
|
|
out.append({'path': list(path), 'href': href})
|
|
|
|
|
|
# ================================================================
|
|
# 파서들
|
|
# ================================================================
|
|
|
|
def parse_menu_div(soup, base):
|
|
"""고창/김제/임실: div.sitemap > div.menuN > h4>a (D) + div > ul > li>a (E) + ul (F) 재귀."""
|
|
rows = []
|
|
sm = soup.select_one('div.sitemap')
|
|
if not sm:
|
|
return rows
|
|
for block in sm.find_all('div', recursive=False):
|
|
h4 = block.find('h4')
|
|
D = clean_text(h4.get_text()) if h4 else ''
|
|
if not D:
|
|
continue
|
|
inner = block.find('div', recursive=False)
|
|
ul = inner.find('ul', recursive=False) if inner else block.find('ul', recursive=False)
|
|
if not ul:
|
|
rows.append({'D': D, 'href': '', **{c: '' for c in 'EFGHIJ'}})
|
|
continue
|
|
tmp = []
|
|
walk_ul(ul, [], tmp)
|
|
for r in rows_from_paths(tmp):
|
|
rows.append({'D': D, 'E': r.get('D', ''), 'F': r.get('E', ''),
|
|
'G': r.get('F', ''), 'H': r.get('G', ''), 'I': r.get('H', ''),
|
|
'J': r.get('I', ''), 'href': r.get('href', '')})
|
|
return rows
|
|
|
|
|
|
def parse_jeongeup(soup, base):
|
|
"""정읍: div.sitemap > div.st_mapNN > p.tit>a (D) + ul > li > b>a (E) + ul > li>a (F)."""
|
|
rows = []
|
|
sm = soup.select_one('div.sitemap')
|
|
if not sm:
|
|
return rows
|
|
for block in sm.find_all('div', recursive=False):
|
|
ptit = block.find('p', class_='tit')
|
|
D = clean_text(ptit.get_text()) if ptit else ''
|
|
if not D:
|
|
continue
|
|
ul = block.find('ul', recursive=False)
|
|
if not ul:
|
|
rows.append({'D': D, 'href': '', **{c: '' for c in 'EFGHIJ'}})
|
|
continue
|
|
for li in ul.find_all('li', recursive=False):
|
|
b = li.find('b', recursive=False)
|
|
b_a = b.find('a') if b else li.find('a', recursive=False)
|
|
E = clean_text(b_a.get_text()) if b_a else ''
|
|
E_href = extract_href(b_a) if b_a else ''
|
|
sub = li.find('ul', recursive=False)
|
|
if not sub:
|
|
rows.append({'D': D, 'E': E, 'href': E_href, **{c: '' for c in 'FGHIJ'}})
|
|
continue
|
|
rows.append({'D': D, 'E': E, 'href': E_href, **{c: '' for c in 'FGHIJ'}})
|
|
tmp = []
|
|
walk_ul(sub, [], tmp)
|
|
for r in rows_from_paths(tmp):
|
|
rows.append({'D': D, 'E': E, 'F': r.get('D', ''), 'G': r.get('E', ''),
|
|
'H': r.get('F', ''), 'I': r.get('G', ''), 'J': r.get('H', ''),
|
|
'href': r.get('href', '')})
|
|
return rows
|
|
|
|
|
|
def parse_group_sitemap(soup, base):
|
|
"""남원/익산: div.sitemap_group (여러개) > h4.title (D) + ul.sitemap_2dep > li>a (E) + ul (F) 재귀."""
|
|
rows = []
|
|
groups = soup.select('div.sitemap_group')
|
|
for g in groups:
|
|
h4 = g.find('h4', class_='title') or g.find('h4')
|
|
D = clean_text(h4.get_text()) if h4 else ''
|
|
if not D:
|
|
continue
|
|
ul = g.find('ul', class_='sitemap_2dep') or g.find('ul', recursive=False)
|
|
if not ul:
|
|
rows.append({'D': D, 'href': '', **{c: '' for c in 'EFGHIJ'}})
|
|
continue
|
|
tmp = []
|
|
walk_ul(ul, [], tmp)
|
|
for r in rows_from_paths(tmp):
|
|
rows.append({'D': D, 'E': r.get('D', ''), 'F': r.get('E', ''),
|
|
'G': r.get('F', ''), 'H': r.get('G', ''), 'I': r.get('H', ''),
|
|
'J': r.get('I', ''), 'href': r.get('href', '')})
|
|
return rows
|
|
|
|
|
|
def parse_namwon(soup, base):
|
|
"""남원: div.sitemap > (h4 (D) + ul (E/F 재귀)) 형제 반복."""
|
|
rows = []
|
|
sm = soup.select_one('div.sitemap')
|
|
if not sm:
|
|
return rows
|
|
curD = ''
|
|
for child in sm.find_all(['h4', 'ul'], recursive=False):
|
|
if child.name == 'h4':
|
|
curD = clean_text(child.get_text())
|
|
elif child.name == 'ul' and curD:
|
|
tmp = []
|
|
walk_ul(child, [], tmp)
|
|
for r in rows_from_paths(tmp):
|
|
rows.append({'D': curD, 'E': r.get('D', ''), 'F': r.get('E', ''),
|
|
'G': r.get('F', ''), 'H': r.get('G', ''), 'I': r.get('H', ''),
|
|
'J': r.get('I', ''), 'href': r.get('href', '')})
|
|
return rows
|
|
|
|
|
|
def parse_muju(soup, base):
|
|
"""무주: div#sitemap > div.sitemapN > (h4.sNN>a (D) + ul>li>a (E)) 형제 반복."""
|
|
rows = []
|
|
cont = soup.select_one('div#sitemap')
|
|
if not cont:
|
|
return rows
|
|
for box in cont.find_all('div', recursive=False):
|
|
curD = ''
|
|
for child in box.find_all(['h4', 'ul'], recursive=False):
|
|
if child.name == 'h4':
|
|
a = child.find('a')
|
|
curD = clean_text(a.get_text() if a else child.get_text())
|
|
elif child.name == 'ul' and curD:
|
|
for li in child.find_all('li', recursive=False):
|
|
a = li.find('a', recursive=False)
|
|
if not a:
|
|
continue
|
|
E = clean_text(a.get_text())
|
|
rows.append({'D': curD, 'E': E, 'href': extract_href(a),
|
|
**{c: '' for c in 'FGHIJ'}})
|
|
return rows
|
|
|
|
|
|
def parse_buan(soup, base):
|
|
"""부안: nav#onmenu > ul > li > div.depth_box > div.depth_boxcon > strong (D) + ul>li>a (E) + ul (F)."""
|
|
rows = []
|
|
nav = soup.select_one('nav#onmenu')
|
|
if not nav:
|
|
return rows
|
|
top = nav.find('ul')
|
|
if not top:
|
|
return rows
|
|
for li in top.find_all('li', recursive=False):
|
|
box = li.find('div', class_='depth_boxcon')
|
|
if not box:
|
|
continue
|
|
strong = box.find('strong')
|
|
a0 = li.find('a', recursive=False)
|
|
D = clean_text(strong.get_text()) if strong else clean_text(a0.get_text() if a0 else '')
|
|
if not D:
|
|
continue
|
|
ul = box.find('ul')
|
|
if not ul:
|
|
continue
|
|
tmp = []
|
|
walk_ul(ul, [], tmp)
|
|
for r in rows_from_paths(tmp):
|
|
rows.append({'D': D, 'E': r.get('D', ''), 'F': r.get('E', ''),
|
|
'G': r.get('F', ''), 'H': r.get('G', ''), 'I': r.get('H', ''),
|
|
'J': r.get('I', ''), 'href': r.get('href', '')})
|
|
return rows
|
|
|
|
|
|
def parse_sunchang(soup, base):
|
|
"""순창: ul.gnb > li > a (D) + div.box ul.gnb_2dep > li>a (E) + ul.gnb_3dep (F) + ul.gnb_4dep (G)."""
|
|
rows = []
|
|
gnb = soup.select_one('ul.gnb')
|
|
if not gnb:
|
|
return rows
|
|
for li in gnb.find_all('li', recursive=False):
|
|
a0 = li.find('a', recursive=False)
|
|
D = clean_text(a0.get_text()) if a0 else ''
|
|
if not D:
|
|
continue
|
|
ul2 = li.find('ul', class_='gnb_2dep')
|
|
if not ul2:
|
|
rows.append({'D': D, 'href': extract_href(a0), **{c: '' for c in 'EFGHIJ'}})
|
|
continue
|
|
for li2 in ul2.find_all('li', recursive=False):
|
|
a2 = li2.find('a', recursive=False)
|
|
if not a2:
|
|
continue
|
|
E = clean_text(a2.get_text())
|
|
E_href = extract_href(a2)
|
|
ul3 = li2.find('ul', class_='gnb_3dep')
|
|
if not ul3:
|
|
rows.append({'D': D, 'E': E, 'href': E_href, **{c: '' for c in 'FGHIJ'}})
|
|
continue
|
|
rows.append({'D': D, 'E': E, 'href': E_href, **{c: '' for c in 'FGHIJ'}})
|
|
for li3 in ul3.find_all('li', recursive=False):
|
|
a3 = li3.find('a', recursive=False)
|
|
if not a3:
|
|
continue
|
|
F = clean_text(a3.get_text())
|
|
F_href = extract_href(a3)
|
|
ul4 = li3.find('ul', class_='gnb_4dep')
|
|
if not ul4:
|
|
rows.append({'D': D, 'E': E, 'F': F, 'href': F_href, **{c: '' for c in 'GHIJ'}})
|
|
continue
|
|
rows.append({'D': D, 'E': E, 'F': F, 'href': F_href, **{c: '' for c in 'GHIJ'}})
|
|
for li4 in ul4.find_all('li', recursive=False):
|
|
a4 = li4.find('a', recursive=False)
|
|
if not a4:
|
|
continue
|
|
rows.append({'D': D, 'E': E, 'F': F, 'G': clean_text(a4.get_text()),
|
|
'href': extract_href(a4), **{c: '' for c in 'HIJ'}})
|
|
return rows
|
|
|
|
|
|
def parse_jangsu(soup, base):
|
|
"""장수: div.sitemap > ul.siteMapList > li.sml_1depth > a.sml_1depthBtn (D) + ul.sml_2depthList > li>a (E) + ul 재귀."""
|
|
rows = []
|
|
ul = soup.select_one('ul.siteMapList')
|
|
if not ul:
|
|
return rows
|
|
for li in ul.find_all('li', recursive=False):
|
|
a0 = li.find('a', recursive=False)
|
|
D = clean_text(a0.get_text()) if a0 else ''
|
|
if not D:
|
|
continue
|
|
ul2 = li.find('ul', recursive=False)
|
|
if not ul2:
|
|
rows.append({'D': D, 'href': extract_href(a0), **{c: '' for c in 'EFGHIJ'}})
|
|
continue
|
|
tmp = []
|
|
walk_ul(ul2, [], tmp)
|
|
for r in rows_from_paths(tmp):
|
|
rows.append({'D': D, 'E': r.get('D', ''), 'F': r.get('E', ''),
|
|
'G': r.get('F', ''), 'H': r.get('G', ''), 'I': r.get('H', ''),
|
|
'J': r.get('I', ''), 'href': r.get('href', '')})
|
|
return rows
|
|
|
|
|
|
def parse_jeonju(soup, base):
|
|
"""전주: div.sitemap_Warp (여러개) > h4.title_h4 (D) + ul > li > div>a (E) + ul/li (F) 망가진 마크업."""
|
|
rows = []
|
|
warps = soup.select('div.sitemap_Warp')
|
|
for w in warps:
|
|
h4 = w.find('h4', class_='title_h4') or w.find('h4')
|
|
D = clean_text(h4.get_text()) if h4 else ''
|
|
if not D:
|
|
continue
|
|
ul = w.find('ul', recursive=False)
|
|
if not ul:
|
|
continue
|
|
tmp = []
|
|
walk_ul(ul, [], tmp)
|
|
for r in rows_from_paths(tmp):
|
|
rows.append({'D': D, 'E': r.get('D', ''), 'F': r.get('E', ''),
|
|
'G': r.get('F', ''), 'H': r.get('G', ''), 'I': r.get('H', ''),
|
|
'J': r.get('I', ''), 'href': r.get('href', '')})
|
|
return rows
|
|
|
|
|
|
def parse_jinan(soup, base):
|
|
"""진안: div.sitemap > dl > dt (D) + dd > ul > li>a (E) + ul (F) 재귀."""
|
|
rows = []
|
|
sm = soup.select_one('div.sitemap')
|
|
if not sm:
|
|
return rows
|
|
for dl in sm.find_all('dl', recursive=False):
|
|
dt = dl.find('dt')
|
|
D = clean_text(dt.get_text()) if dt else ''
|
|
if not D:
|
|
continue
|
|
dd = dl.find('dd')
|
|
ul = dd.find('ul', recursive=False) if dd else None
|
|
if not ul:
|
|
rows.append({'D': D, 'href': '', **{c: '' for c in 'EFGHIJ'}})
|
|
continue
|
|
tmp = []
|
|
walk_ul(ul, [], tmp)
|
|
for r in rows_from_paths(tmp):
|
|
rows.append({'D': D, 'E': r.get('D', ''), 'F': r.get('E', ''),
|
|
'G': r.get('F', ''), 'H': r.get('G', ''), 'I': r.get('H', ''),
|
|
'J': r.get('I', ''), 'href': r.get('href', '')})
|
|
return rows
|
|
|
|
|
|
def parse_box_heading(soup, base):
|
|
"""서귀포/제주시: div.sitemap > div.sitemapBox|sitemap_menu > h3|h4 (D) + ul > li>a (E)."""
|
|
rows = []
|
|
sm = soup.select_one('div.sitemap')
|
|
if not sm:
|
|
return rows
|
|
for box in sm.find_all('div', recursive=False):
|
|
h = box.find(['h3', 'h4'])
|
|
D = clean_text(h.get_text()) if h else ''
|
|
if not D:
|
|
continue
|
|
ul = box.find('ul')
|
|
if not ul:
|
|
rows.append({'D': D, 'href': '', **{c: '' for c in 'EFGHIJ'}})
|
|
continue
|
|
tmp = []
|
|
walk_ul(ul, [], tmp)
|
|
for r in rows_from_paths(tmp):
|
|
rows.append({'D': D, 'E': r.get('D', ''), 'F': r.get('E', ''),
|
|
'G': r.get('F', ''), 'H': r.get('G', ''), 'I': r.get('H', ''),
|
|
'J': r.get('I', ''), 'href': r.get('href', '')})
|
|
return rows
|
|
|
|
|
|
def parse_wanju_json(soup, base):
|
|
"""완주: Playwright로 추출한 _wanju_menu.json 로드."""
|
|
rows = []
|
|
data = json.load(open(os.path.join(os.path.dirname(os.path.abspath(__file__)), '_wanju_menu.json'), encoding='utf-8'))
|
|
for r in data:
|
|
rows.append({'D': r.get('D', ''), 'E': r.get('E', ''), 'F': r.get('F', ''),
|
|
'href': r.get('href', ''), 'G': '', 'H': '', 'I': '', 'J': ''})
|
|
return rows
|
|
|
|
|
|
# ================================================================
|
|
# 사이트 설정
|
|
# ================================================================
|
|
def jb(i, name, folder, base, sitemap, parser, domain, weak=False, fetch_kind='html'):
|
|
return {'idx': i, 'name': name, 'base': base, 'sitemap': sitemap,
|
|
'sheet': f'{i:02d}_{name}', 'parser': parser, 'domain': domain,
|
|
'folder': fr'{ROOT}\작업파일\광역_사이트맵\전북특별자치도\{i}.{name}', 'weak_ssl': weak,
|
|
'fetch_kind': fetch_kind}
|
|
|
|
|
|
def jj(i, name, base, sitemap, parser, domain):
|
|
return {'idx': i, 'name': name, 'base': base, 'sitemap': sitemap,
|
|
'sheet': f'{i:02d}_{name}', 'parser': parser, 'domain': domain,
|
|
'folder': fr'{ROOT}\작업파일\광역_사이트맵\제주특별자치도\{i}.{name}', 'weak_ssl': False,
|
|
'fetch_kind': 'html'}
|
|
|
|
|
|
SITES = [
|
|
jb(1, '고창군', '', 'https://www.gochang.go.kr',
|
|
'https://www.gochang.go.kr/index.gochang?menuCd=DOM_000000106006000000', parse_menu_div, 'gochang.go.kr'),
|
|
jb(2, '군산시', '', 'https://www.gunsan.go.kr',
|
|
'https://www.gunsan.go.kr/main', parse_gunsan if False else None, 'gunsan.go.kr'),
|
|
jb(3, '김제시', '', 'https://www.gimje.go.kr',
|
|
'https://www.gimje.go.kr/index.gimje?menuCd=DOM_000000107002000000', parse_menu_div, 'gimje.go.kr'),
|
|
jb(4, '남원시', '', 'https://www.namwon.go.kr',
|
|
'https://www.namwon.go.kr/index.do?menuUid=ff8080818f2717db018f277767500088', parse_namwon, 'namwon.go.kr'),
|
|
jb(5, '무주군', '', 'https://www.muju.go.kr',
|
|
'https://www.muju.go.kr/index.9is?contentUid=ff8080816db80238016dc8e98fa10ef6', parse_muju, 'muju.go.kr'),
|
|
jb(6, '부안군', '', 'https://www.buan.go.kr',
|
|
'https://www.buan.go.kr/index.buan?contentsSid=1', parse_buan, 'buan.go.kr'),
|
|
jb(7, '순창군', '', 'https://www.sunchang.go.kr',
|
|
'https://www.sunchang.go.kr/', parse_sunchang, 'sunchang.go.kr'),
|
|
jb(8, '완주군', '', 'https://www.wanju.go.kr',
|
|
'JSON', parse_wanju_json, 'wanju.go.kr', fetch_kind='json'),
|
|
jb(9, '익산시', '', 'https://www.iksan.go.kr',
|
|
'https://www.iksan.go.kr/index.do?menuUid=ff80808199f0d11c019a041b8e35174a', parse_group_sitemap, 'iksan.go.kr'),
|
|
jb(10, '임실군', '', 'https://www.imsil.go.kr',
|
|
'https://www.imsil.go.kr/index.imsil?menuCd=DOM_000000107003000000', parse_menu_div, 'imsil.go.kr'),
|
|
jb(11, '장수군', '', 'https://www.jangsu.go.kr',
|
|
'https://www.jangsu.go.kr/index.jangsu?menuCd=DOM_000000107001000000', parse_jangsu, 'jangsu.go.kr'),
|
|
jb(12, '전주시', '', 'https://www.jeonju.go.kr',
|
|
'https://www.jeonju.go.kr/index.9is?contentUid=ff8080818c7c2e8e018c7fd67b9a03b6', parse_jeonju, 'jeonju.go.kr', fetch_kind='html5'),
|
|
jb(13, '정읍시', '', 'https://www.jeongeup.go.kr',
|
|
'https://www.jeongeup.go.kr/index.jeongeup?menuCd=DOM_000000106002000000', parse_jeongeup, 'jeongeup.go.kr'),
|
|
jb(14, '진안군', '', 'https://www.jinan.go.kr',
|
|
'https://www.jinan.go.kr/index.jinan?menuCd=DOM_000000110002000000', parse_jinan, 'jinan.go.kr'),
|
|
jj(1, '서귀포시', 'https://www.seogwipo.go.kr',
|
|
'https://www.seogwipo.go.kr/help/sitemap.htm', parse_box_heading, 'seogwipo.go.kr'),
|
|
jj(2, '제주시', 'https://www.jejusi.go.kr',
|
|
'https://www.jejusi.go.kr/guide/sitemap.do', parse_box_heading, 'jejusi.go.kr'),
|
|
]
|
|
|
|
|
|
def parse_gunsan(soup, base):
|
|
"""군산: div#all_pcmenu > div.allmenubox > a.Bmenu (D) + div.allmw ul.sub_pcmenu > li>a (E) + ul.dep3 (F) 재귀."""
|
|
rows = []
|
|
pc = soup.select_one('div#all_pcmenu')
|
|
if not pc:
|
|
return rows
|
|
for box in pc.select('div.allmenubox'):
|
|
bm = box.find('a', class_='Bmenu') or box.find('a')
|
|
D = clean_text(bm.get_text()) if bm else ''
|
|
if not D:
|
|
continue
|
|
ul = box.find('ul', class_='sub_pcmenu')
|
|
if not ul:
|
|
rows.append({'D': D, 'href': extract_href(bm), **{c: '' for c in 'EFGHIJ'}})
|
|
continue
|
|
# ★ ul.dep4 = 모바일 아코디언 잔재(F형제 전체를 '- '접두로 복제). 진짜 자식 아님 → 제거.
|
|
# 안 지우면 각 F의 G자식으로 재귀돼 카르테시안 폭발(2026-05-31 군산 208행 버그 수정).
|
|
for d4 in ul.select('ul.dep4'):
|
|
d4.decompose()
|
|
tmp = []
|
|
walk_ul(ul, [], tmp)
|
|
for r in rows_from_paths(tmp):
|
|
rows.append({'D': D, 'E': r.get('D', ''), 'F': r.get('E', ''),
|
|
'G': r.get('F', ''), 'H': r.get('G', ''), 'I': r.get('H', ''),
|
|
'J': r.get('I', ''), 'href': r.get('href', '')})
|
|
return rows
|
|
|
|
|
|
# 군산 파서 바인딩(전방참조 해결)
|
|
for _s in SITES:
|
|
if _s['name'] == '군산시':
|
|
_s['parser'] = parse_gunsan
|
|
|
|
|
|
# ================================================================
|
|
# 엑셀 생성 (충북 스크립트와 동일 로직)
|
|
# ================================================================
|
|
def write_excel(site, raw_rows):
|
|
name = site['name']
|
|
base = site['base']
|
|
domain = site['domain']
|
|
output = f"{site['folder']}\\{os.path.basename(os.path.dirname(site['folder']))}_{name}.xlsx"
|
|
|
|
def abs_url(href):
|
|
if not href:
|
|
return ''
|
|
href = href.strip()
|
|
if href.startswith(('javascript:', '#')):
|
|
return ''
|
|
if href.startswith(('http://', 'https://')):
|
|
return href
|
|
return urljoin(base + '/', href)
|
|
|
|
def is_external(url):
|
|
return url.startswith(('http://', 'https://')) and domain not in url
|
|
|
|
final_rows = []
|
|
i = 0
|
|
removed = 0
|
|
while i < len(raw_rows):
|
|
row = raw_rows[i]
|
|
if (i + 1 < len(raw_rows)
|
|
and row.get('G', '') == ''
|
|
and raw_rows[i + 1].get('D') == row.get('D')
|
|
and raw_rows[i + 1].get('E') == row.get('E')
|
|
and raw_rows[i + 1].get('F') == row.get('F')
|
|
and raw_rows[i + 1].get('G', '') != ''
|
|
and raw_rows[i + 1].get('href') == row.get('href')):
|
|
removed += 1
|
|
i += 1
|
|
continue
|
|
final_rows.append(row)
|
|
i += 1
|
|
|
|
print(f' [{name}] 원본 {len(raw_rows)} → 중복 {removed} → 최종 {len(final_rows)}')
|
|
if not final_rows:
|
|
print(f' [{name}] !! 행 0개 — 파서 점검 필요. 엑셀 생성 스킵.')
|
|
return False
|
|
|
|
shutil.copy(TEMPLATE, output)
|
|
wb = openpyxl.load_workbook(output)
|
|
ws = wb.active
|
|
ws.title = site['sheet']
|
|
|
|
HEADER = {'B1:R1', 'S1:W1', 'Y1:AA1'}
|
|
for rng in [str(m) for m in ws.merged_cells.ranges if str(m) not in HEADER]:
|
|
ws.unmerge_cells(rng)
|
|
for row in ws.iter_rows(min_row=3, max_row=ws.max_row, min_col=1, max_col=ws.max_column):
|
|
for cell in row:
|
|
cell.value = None
|
|
|
|
START = 3
|
|
template_r = 3
|
|
cur_max = ws.max_row
|
|
for idx, item in enumerate(final_rows, start=START):
|
|
if idx > cur_max:
|
|
for c in range(1, ws.max_column + 1):
|
|
srcc = ws.cell(template_r, c)
|
|
tgt = ws.cell(idx, c)
|
|
if srcc.has_style:
|
|
tgt.font = copy(srcc.font)
|
|
tgt.fill = copy(srcc.fill)
|
|
tgt.border = copy(srcc.border)
|
|
tgt.alignment = copy(srcc.alignment)
|
|
tgt.number_format = srcc.number_format
|
|
tgt.protection = copy(srcc.protection)
|
|
url = abs_url(item.get('href', ''))
|
|
ws.cell(idx, 2).value = idx - 2
|
|
ws.cell(idx, 3).value = name
|
|
ws.cell(idx, 4).value = item.get('D', '')
|
|
ws.cell(idx, 5).value = item.get('E', '')
|
|
ws.cell(idx, 6).value = item.get('F', '')
|
|
ws.cell(idx, 7).value = item.get('G', '')
|
|
ws.cell(idx, 8).value = item.get('H', '')
|
|
ws.cell(idx, 9).value = item.get('I', '')
|
|
ws.cell(idx, 10).value = item.get('J', '')
|
|
ws.cell(idx, 11).value = url
|
|
if is_external(url):
|
|
ws.cell(idx, 19).value = '외부링크'
|
|
|
|
END = START + len(final_rows) - 1
|
|
center = Alignment(horizontal='center', vertical='center', wrap_text=True)
|
|
left = Alignment(horizontal='left', vertical='center', wrap_text=False)
|
|
|
|
def merge_runs(col_letter, col_idx, group_cols=()):
|
|
runs = []
|
|
cur_val = ws.cell(START, col_idx).value
|
|
cur_grp = tuple(ws.cell(START, g).value for g in group_cols)
|
|
run_start = START
|
|
for r in range(START + 1, END + 1):
|
|
v = ws.cell(r, col_idx).value
|
|
g = tuple(ws.cell(r, gg).value for gg in group_cols)
|
|
if v == cur_val and g == cur_grp:
|
|
continue
|
|
if cur_val not in (None, '') and r - 1 > run_start:
|
|
runs.append((run_start, r - 1))
|
|
cur_val, cur_grp, run_start = v, g, r
|
|
if cur_val not in (None, '') and END > run_start:
|
|
runs.append((run_start, END))
|
|
for s, e in runs:
|
|
ws.merge_cells(f'{col_letter}{s}:{col_letter}{e}')
|
|
ws.cell(s, col_idx).alignment = center
|
|
return len(runs)
|
|
|
|
n_f = merge_runs('F', 6, group_cols=(4, 5))
|
|
n_e = merge_runs('E', 5, group_cols=(4,))
|
|
n_d = merge_runs('D', 4)
|
|
for r in range(START, END + 1):
|
|
for c in (4, 5, 6, 7):
|
|
if ws.cell(r, c).value is not None:
|
|
ws.cell(r, c).alignment = center
|
|
|
|
for r in range(1, END + 1):
|
|
ws.row_dimensions[r].height = 15
|
|
|
|
link_n = 0
|
|
for r in range(START, END + 1):
|
|
cell = ws.cell(r, 11)
|
|
u = cell.value
|
|
if u and isinstance(u, str) and u.startswith(('http://', 'https://')):
|
|
cell.hyperlink = u
|
|
old = cell.font
|
|
cell.font = Font(name=old.name or '맑은 고딕', size=old.size or 11,
|
|
bold=old.bold, italic=old.italic,
|
|
color='0000FF', underline='single')
|
|
cell.alignment = left
|
|
link_n += 1
|
|
|
|
wb.save(output)
|
|
ext_n = sum(1 for r in range(START, END + 1) if ws.cell(r, 19).value == '외부링크')
|
|
print(f' [{name}] 병합 D:{n_d} E:{n_e} F:{n_f} | 외부링크 {ext_n} | K링크 {link_n} → {output}')
|
|
return True
|
|
|
|
|
|
def main():
|
|
targets = sys.argv[1:] if len(sys.argv) > 1 else [s['name'] for s in SITES]
|
|
for site in SITES:
|
|
if site['name'] not in targets:
|
|
continue
|
|
print(f"\n{'='*60}\n[{site['idx']}.{site['name']}] {site['sitemap']}\n{'='*60}")
|
|
try:
|
|
if site['fetch_kind'] == 'json':
|
|
raw_rows = site['parser'](None, site['base'])
|
|
else:
|
|
sess = make_session(weak_ssl=site.get('weak_ssl', False))
|
|
html = fetch_html(site['sitemap'], session=sess)
|
|
engine = 'html5lib' if site.get('fetch_kind') == 'html5' else 'html.parser'
|
|
soup = BeautifulSoup(html, engine)
|
|
raw_rows = site['parser'](soup, site['base'])
|
|
write_excel(site, raw_rows)
|
|
except Exception as e:
|
|
print(f' [{site["name"]}] !! 실패: {e}')
|
|
import traceback
|
|
traceback.print_exc()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|