hehihoho3/DB_JOB
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
abf80b206b
DB_JOB/_스크립트/_chungnam_phase1_all.py
hehihoho3 df16c98366 백업: DB수집 전체 스냅샷 (공공기관2 정리 전) 
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-18 18:15:40 +09:00

735 lines
32 KiB
Python
Raw Blame History

"""충청남도 11개 시·군 Phase 1 일괄 처리.
매뉴얼: D:\\01.프로젝트\\DB수집\\사이트맵_수집_매뉴얼.md
출력: 각 폴더의 {기관명}.xlsx
"""
import os
import re
import shutil
import sys
import warnings
from copy import copy
from urllib.parse import urljoin
import openpyxl
import requests
from bs4 import BeautifulSoup
from openpyxl.styles import Alignment, Font
warnings.filterwarnings('ignore')
UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
H = {'User-Agent': UA}
TEMPLATE = r'D:\01.프로젝트\DB수집\자료_취합_예시.xlsx'
# ================================================================
# 공통 유틸
# ================================================================
def fetch_html(url, timeout=20):
r = requests.get(url, headers=H, timeout=timeout, verify=False, allow_redirects=True)
r.encoding = r.apparent_encoding
return r.text
def clean_text(s):
return re.sub(r'\s+', ' ', s or '').strip().replace('\xa0', '')
def extract_href(a):
if a is None:
return ''
href = (a.get('href') or '').strip()
if not href or href.startswith('#') or href.lower().startswith('javascript:'):
# Try onclick encodeURI extraction
return ''
return href
ENCODE_URI_PAT = re.compile(r"""(?:location\.href|window\.location(?:\.href)?)\s*=\s*encodeURI\(\s*['"]([^'"]+)['"]\s*\)""")
ONCLICK_HREF_PAT = re.compile(r"""(?:location\.href|window\.open)\s*\(?\s*['"]([^'"]+)['"]""")
def extract_href_with_onclick(a):
"""Extract href; if href is dummy (#...), check onclick for encodeURI/location.href."""
if a is None:
return ''
href = (a.get('href') or '').strip()
if href and not href.startswith('#') and not href.lower().startswith('javascript:'):
return href
# Try onclick
onclick = a.get('onclick', '')
if onclick:
m = ENCODE_URI_PAT.search(onclick)
if m:
return m.group(1)
m = ONCLICK_HREF_PAT.search(onclick)
if m:
return m.group(1)
return ''
# ================================================================
# 사이트별 파서 (각각 raw_rows 리스트 반환)
# raw_rows: [{'D': str, 'E': str, 'F': str, 'G': str, 'H': str, 'I': str, 'J': str, 'href': str}, ...]
# ================================================================
def parse_eGov_type1(soup, base):
"""논산시·아산시 형: div.sitemap.type1 > [div.s_1th + div.inner > div.s_2th + ul...]."""
sitemap = soup.select_one('div.sitemap.type1') or soup.select_one('div.sitemap')
rows = []
if not sitemap:
return rows
current_D = ''
def walk(ul, depth, base_path, out):
for li in ul.find_all('li', recursive=False):
a = li.find('a', recursive=False)
if not a:
continue
text = clean_text(a.get_text())
href = extract_href(a)
path = base_path[:depth] + [(text, href)]
nested = li.find('ul', recursive=False)
if nested:
out.append({'path': list(path), 'href': href})
walk(nested, depth + 1, path, out)
else:
out.append({'path': list(path), 'href': href})
for child in sitemap.find_all('div', recursive=False):
cls = child.get('class', [])
if 's_1th' in cls:
a = child.find('a')
current_D = clean_text(a.get_text()) if a else ''
elif 'inner' in cls:
s2 = child.find('div', class_='s_2th')
mid_a = s2.find('a') if s2 else None
mid_name = clean_text(mid_a.get_text()) if mid_a else ''
mid_href = extract_href(mid_a) if mid_a else ''
uls = child.find_all('ul', recursive=False)
if not uls:
rows.append({'D': current_D, 'E': mid_name, 'href': mid_href,
'F': '', 'G': '', 'H': '', 'I': '', 'J': ''})
continue
for ul in uls:
tmp = []
walk(ul, 0, [], tmp)
for item in tmp:
p = item['path']
row = {'D': current_D, 'E': mid_name, 'href': item['href'],
'F': '', 'G': '', 'H': '', 'I': '', 'J': ''}
for di, (t, _) in enumerate(p):
col = 'FGHIJ'[di] if di < 5 else 'J'
row[col] = t
rows.append(row)
return rows
def parse_amThum(soup, base):
"""당진시·태안군 형: div.amThum > h2.siteNN + div.sitemap_grep > ul.sitemap_list > li > a.first + ul > li > a (+ ul > li > a)."""
rows = []
for amthum in soup.select('div.amThum'):
h2 = amthum.find(['h2', 'h3'], class_=re.compile(r'site\d+'))
D = clean_text(h2.find('span').get_text()) if h2 and h2.find('span') else (clean_text(h2.get_text()) if h2 else '')
grep = amthum.find('div', class_='sitemap_grep') or amthum
for sl in grep.find_all('ul', class_='sitemap_list'):
# Each ul.sitemap_list contains li > a.first + ul > li > a
for top_li in sl.find_all('li', recursive=False):
a_first = top_li.find('a', class_='first', recursive=False)
if not a_first:
continue
E = clean_text(a_first.get_text())
E_href = extract_href(a_first)
inner_ul = top_li.find('ul', recursive=False)
if not inner_ul:
rows.append({'D': D, 'E': E, 'href': E_href,
'F': '', 'G': '', 'H': '', 'I': '', 'J': ''})
continue
for mid_li in inner_ul.find_all('li', recursive=False):
mid_a = mid_li.find('a', recursive=False)
if not mid_a:
continue
F = clean_text(mid_a.get_text())
F_href = extract_href(mid_a)
deeper = mid_li.find('ul', recursive=False)
if not deeper:
rows.append({'D': D, 'E': E, 'F': F, 'href': F_href,
'G': '', 'H': '', 'I': '', 'J': ''})
continue
rows.append({'D': D, 'E': E, 'F': F, 'href': F_href,
'G': '', 'H': '', 'I': '', 'J': ''})
for deep_li in deeper.find_all('li', recursive=False):
deep_a = deep_li.find('a', recursive=False)
if not deep_a:
continue
G = clean_text(deep_a.get_text())
rows.append({'D': D, 'E': E, 'F': F, 'G': G,
'href': extract_href(deep_a),
'H': '', 'I': '', 'J': ''})
return rows
def parse_ul_sitemap_h4(soup, base):
"""보령시·서천군 형: ul.sitemap > li (대분류) > h4.siteNN > span + ul > li > h5 > a + ul > li.list > a."""
rows = []
container = soup.select_one('ul.sitemap') or soup.select_one('div#contents ul.sitemap')
if not container:
return rows
for top_li in container.find_all('li', recursive=False):
h4 = top_li.find(['h4', 'h3'], recursive=False)
D = ''
if h4:
span = h4.find('span')
D = clean_text(span.get_text() if span else h4.get_text())
# Each direct ul under top_li is a sub-group
for sub_ul in top_li.find_all('ul', recursive=False):
for sub_li in sub_ul.find_all('li', recursive=False):
h5 = sub_li.find(['h5', 'h6'], recursive=False)
if h5:
h5_a = h5.find('a')
E = clean_text(h5_a.get_text()) if h5_a else clean_text(h5.get_text())
E_href = extract_href(h5_a) if h5_a else ''
else:
a = sub_li.find('a', recursive=False)
E = clean_text(a.get_text()) if a else ''
E_href = extract_href(a) if a else ''
deeper = sub_li.find('ul', recursive=False)
if not deeper:
rows.append({'D': D, 'E': E, 'href': E_href,
'F': '', 'G': '', 'H': '', 'I': '', 'J': ''})
continue
rows.append({'D': D, 'E': E, 'href': E_href,
'F': '', 'G': '', 'H': '', 'I': '', 'J': ''})
for leaf_li in deeper.find_all('li', recursive=False):
leaf_a = leaf_li.find('a', recursive=False)
if not leaf_a:
continue
F = clean_text(leaf_a.get_text())
rows.append({'D': D, 'E': E, 'F': F, 'href': extract_href(leaf_a),
'G': '', 'H': '', 'I': '', 'J': ''})
# 4-level (rare)
sub_ul2 = leaf_li.find('ul', recursive=False)
if sub_ul2:
for ll2 in sub_ul2.find_all('li', recursive=False):
la2 = ll2.find('a', recursive=False)
if la2:
G = clean_text(la2.get_text())
rows.append({'D': D, 'E': E, 'F': F, 'G': G,
'href': extract_href(la2),
'H': '', 'I': '', 'J': ''})
return rows
def parse_dl_dt_dd(soup, base, use_onclick=False):
"""홍성군·예산군·천안시·금산군 dl 형: div.sitemap (옵션 .type2) > dl > dt + dd > b > a + ul > li > a (+ ul > li > a).
use_onclick=True 인 경우 onclick의 encodeURI/location.href에서 URL을 추출.
"""
rows = []
sm = soup.select_one('div.sitemap[class*=type2]') or soup.select_one('div.sitemap')
if not sm:
return rows
href_fn = extract_href_with_onclick if use_onclick else extract_href
def walk(ul, depth, base_path, out):
for li in ul.find_all('li', recursive=False):
a = li.find('a', recursive=False)
if not a:
continue
text = clean_text(a.get_text())
href = href_fn(a)
path = base_path[:depth] + [(text, href)]
nested = li.find('ul', recursive=False)
if nested:
out.append({'path': list(path), 'href': href})
walk(nested, depth + 1, path, out)
else:
out.append({'path': list(path), 'href': href})
for dl in sm.find_all('dl', recursive=False):
dt = dl.find('dt')
dt_a = dt.find('a') if dt else None
D = clean_text(dt_a.get_text() if dt_a else (dt.get_text() if dt else ''))
for dd in dl.find_all('dd', recursive=False):
b = dd.find('b')
b_a = b.find('a') if b else None
E = clean_text(b_a.get_text()) if b_a else ''
E_href = href_fn(b_a) if b_a else ''
nested = dd.find('ul', recursive=False)
if nested:
tmp = []
walk(nested, 0, [], tmp)
for item in tmp:
p = item['path']
row = {'D': D, 'E': E, 'href': item['href'],
'F': '', 'G': '', 'H': '', 'I': '', 'J': ''}
for di, (t, _) in enumerate(p):
col = 'FGHIJ'[di] if di < 5 else 'J'
row[col] = t
rows.append(row)
else:
rows.append({'D': D, 'E': E, 'href': E_href,
'F': '', 'G': '', 'H': '', 'I': '', 'J': ''})
return rows
def parse_seosan_top_menu(soup, base):
"""서산시: ul.top_menu > li.depth1 > a.depth1_ti (D) + div...div.depth2_wrap > ul.depth2 > li > a (E) + ul.depth3 > li > a (F)."""
rows = []
tm = soup.select_one('ul.top_menu')
if not tm:
return rows
for top_li in tm.find_all('li', class_='depth1', recursive=False):
a1 = top_li.find('a', class_='depth1_ti')
D = clean_text(a1.get_text()) if a1 else ''
depth2 = top_li.select_one('ul.depth2')
if not depth2:
continue
for d2_li in depth2.find_all('li', recursive=False):
d2_a = d2_li.find('a', recursive=False)
if not d2_a:
continue
E = clean_text(d2_a.get_text())
E_href = extract_href(d2_a)
d3 = d2_li.find('ul', class_='depth3', recursive=False)
if not d3:
rows.append({'D': D, 'E': E, 'href': E_href,
'F': '', 'G': '', 'H': '', 'I': '', 'J': ''})
continue
rows.append({'D': D, 'E': E, 'href': E_href,
'F': '', 'G': '', 'H': '', 'I': '', 'J': ''})
for d3_li in d3.find_all('li', recursive=False):
d3_a = d3_li.find('a', recursive=False)
if not d3_a:
continue
F = clean_text(d3_a.get_text())
rows.append({'D': D, 'E': E, 'F': F, 'href': extract_href(d3_a),
'G': '', 'H': '', 'I': '', 'J': ''})
return rows
def parse_yesan_depth(soup, base):
"""예산군: #gnb > ul.depth1_ul > li > a.th_1st + div.item > ul.depth2_ul > li > a + ul.depth3_ul > li > a."""
rows = []
container = soup.select_one('#gnb ul.depth1_ul') or soup.select_one('ul.depth1_ul')
if not container:
return rows
for top_li in container.find_all('li', recursive=False):
a1 = top_li.find('a', class_='th_1st', recursive=False)
D = clean_text(a1.get_text()) if a1 else ''
item = top_li.find('div', class_='item')
if not item:
continue
d2_ul = item.find('ul', class_='depth2_ul')
if not d2_ul:
continue
for d2_li in d2_ul.find_all('li', recursive=False):
d2_a = d2_li.find('a', recursive=False)
if not d2_a:
continue
E = clean_text(d2_a.get_text())
E_href = extract_href(d2_a)
d3_ul = d2_li.find('ul', class_='depth3_ul', recursive=False)
if not d3_ul:
rows.append({'D': D, 'E': E, 'href': E_href,
'F': '', 'G': '', 'H': '', 'I': '', 'J': ''})
continue
rows.append({'D': D, 'E': E, 'href': E_href,
'F': '', 'G': '', 'H': '', 'I': '', 'J': ''})
for d3_li in d3_ul.find_all('li', recursive=False):
d3_a = d3_li.find('a', recursive=False)
if not d3_a:
continue
F = clean_text(d3_a.get_text())
rows.append({'D': D, 'E': E, 'F': F, 'href': extract_href(d3_a),
'G': '', 'H': '', 'I': '', 'J': ''})
return rows
def parse_cheonan_depth(soup, base):
"""천안시: ul.depth1-ul > li > a.depth1-btn + div.depth1-content > div.layout > ul.depth2-ul > li > a.depth2-btn + div.depth2-content > ul.depth3-ul > li > a.depth3-btn."""
rows = []
container = soup.select_one('ul.depth1-ul')
if not container:
return rows
for top_li in container.find_all('li', recursive=False):
a1 = top_li.find('a', class_='depth1-btn', recursive=False)
D = clean_text(a1.get_text()) if a1 else ''
D_href = extract_href(a1) if a1 else ''
d2_ul = top_li.select_one('ul.depth2-ul')
if not d2_ul:
rows.append({'D': D, 'E': '', 'href': D_href,
'F': '', 'G': '', 'H': '', 'I': '', 'J': ''})
continue
for d2_li in d2_ul.find_all('li', recursive=False):
d2_a = d2_li.find('a', class_='depth2-btn', recursive=False)
if not d2_a:
continue
E = clean_text(d2_a.get_text())
E_href = extract_href(d2_a)
d3_ul = d2_li.select_one('ul.depth3-ul')
if not d3_ul:
rows.append({'D': D, 'E': E, 'href': E_href,
'F': '', 'G': '', 'H': '', 'I': '', 'J': ''})
continue
rows.append({'D': D, 'E': E, 'href': E_href,
'F': '', 'G': '', 'H': '', 'I': '', 'J': ''})
for d3_li in d3_ul.find_all('li', recursive=False):
d3_a = d3_li.find('a', class_='depth3-btn', recursive=False)
if not d3_a:
continue
F = clean_text(d3_a.get_text())
rows.append({'D': D, 'E': E, 'F': F, 'href': extract_href(d3_a),
'G': '', 'H': '', 'I': '', 'J': ''})
return rows
def parse_cheongyang_depth1(soup, base):
"""청양군: ul.depth1_ul > li > a.th_1st + div.item > ul.depth2_ul > li > a + ul.depth3_ul > li > a (예산군 형)."""
# 청양군은 예산군과 동일한 e-Gov GNB 구조 — yesan 파서 재사용
return parse_yesan_depth(soup, base)
def parse_asan_gnb(soup, base):
"""아산시: div#mGnb-anchor{n}.gnb-sub-list > ul > li > a.gnb-sub-trigger + ul.sub-ul > li > a.subm."""
rows = []
# 대분류 이름 — mobile-nav의 gnb-main-trigger 텍스트 + href(#mGnb-anchorN) 매핑
nav_main = soup.select_one('nav#mobile-nav')
main_categories = [] # [(D, anchor_id)]
if nav_main:
for trig in nav_main.find_all(['a', 'button'], class_='gnb-main-trigger'):
text = clean_text(trig.get_text())
target = trig.get('href') or trig.get('data-target') or ''
if target.startswith('#mGnb-anchor'):
main_categories.append((text, target.lstrip('#')))
# Fallback: sections without name mapping
if not main_categories:
for sec in soup.select('div[id^=mGnb-anchor]'):
main_categories.append((sec.get('id'), sec.get('id')))
for D, anchor_id in main_categories:
section = soup.find('div', id=anchor_id)
if not section:
continue
# section > ul > li > a.gnb-sub-trigger + ul.sub-ul > li > a.subm
for ul in section.find_all('ul', recursive=False):
for li in ul.find_all('li', recursive=False):
a_E = li.find('a', class_='gnb-sub-trigger', recursive=False) or li.find('a', recursive=False)
if not a_E:
continue
E = clean_text(a_E.get_text())
E_href = extract_href(a_E)
sub_ul = li.find('ul', class_='sub-ul', recursive=False) or li.find('ul', recursive=False)
if not sub_ul:
rows.append({'D': D, 'E': E, 'href': E_href,
'F': '', 'G': '', 'H': '', 'I': '', 'J': ''})
continue
rows.append({'D': D, 'E': E, 'href': E_href,
'F': '', 'G': '', 'H': '', 'I': '', 'J': ''})
for sub_li in sub_ul.find_all('li', recursive=False):
sub_a = sub_li.find('a', recursive=False)
if not sub_a:
continue
F = clean_text(sub_a.get_text())
rows.append({'D': D, 'E': E, 'F': F, 'href': extract_href(sub_a),
'G': '', 'H': '', 'I': '', 'J': ''})
return rows
def parse_buyeo_topmenu(soup, base):
"""부여군: ul#tm > li.th1 > a.th1_lnk (D) + div.summry > ul.th2 > li > a.th2_lnk (E) + ul.th3 > li > a (F)."""
rows = []
tm = soup.select_one('ul#tm')
if not tm:
return rows
for top_li in tm.find_all('li', class_=re.compile(r'th1?'), recursive=False):
a1 = top_li.find('a', class_='th1_lnk', recursive=False)
if not a1:
a1 = top_li.find('a', recursive=False)
if not a1:
continue
D = clean_text(a1.get_text())
D_href = extract_href(a1)
# Find ul.th2 inside div.summry
summry = top_li.find('div', class_=re.compile(r'summry'), recursive=False)
th2 = (summry.find('ul', class_='th2') if summry else None) or top_li.find('ul', class_='th2')
if not th2:
rows.append({'D': D, 'E': '', 'href': D_href,
'F': '', 'G': '', 'H': '', 'I': '', 'J': ''})
continue
for li2 in th2.find_all('li', recursive=False):
a2 = li2.find('a', class_='th2_lnk', recursive=False) or li2.find('a', recursive=False)
if not a2:
continue
E = clean_text(a2.get_text())
E_href = extract_href(a2)
th3 = li2.find('ul', class_='th3', recursive=False)
if not th3:
rows.append({'D': D, 'E': E, 'href': E_href,
'F': '', 'G': '', 'H': '', 'I': '', 'J': ''})
continue
rows.append({'D': D, 'E': E, 'href': E_href,
'F': '', 'G': '', 'H': '', 'I': '', 'J': ''})
for li3 in th3.find_all('li', recursive=False):
a3 = li3.find('a', recursive=False)
if not a3:
continue
F = clean_text(a3.get_text())
rows.append({'D': D, 'E': E, 'F': F, 'href': extract_href(a3),
'G': '', 'H': '', 'I': '', 'J': ''})
return rows
# ================================================================
# 사이트 설정
# ================================================================
SITES = [
{
'idx': 5, 'name': '당진시', 'base': 'https://www.dangjin.go.kr',
'sitemap': 'https://www.dangjin.go.kr/kor/sitemap_11.do',
'sheet': '05_당진시', 'parser': parse_amThum,
'domain': 'dangjin.go.kr',
'folder': r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청남도\5.당진시',
},
{
'idx': 6, 'name': '보령시', 'base': 'https://www.brcn.go.kr',
'sitemap': 'https://www.brcn.go.kr/kor/sitemap_11.do',
'sheet': '06_보령시', 'parser': parse_ul_sitemap_h4,
'domain': 'brcn.go.kr',
'folder': r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청남도\6.보령시',
},
{
'idx': 7, 'name': '부여군', 'base': 'https://www.buyeo.go.kr',
'sitemap': 'https://www.buyeo.go.kr/html/kr/',
'sheet': '07_부여군', 'parser': parse_buyeo_topmenu,
'domain': 'buyeo.go.kr',
'folder': r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청남도\7.부여군',
},
{
'idx': 8, 'name': '서산시', 'base': 'https://www.seosan.go.kr',
'sitemap': 'https://www.seosan.go.kr/www/index.do',
'sheet': '08_서산시', 'parser': parse_seosan_top_menu,
'domain': 'seosan.go.kr',
'folder': r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청남도\8.서산시',
},
{
'idx': 9, 'name': '서천군', 'base': 'https://www.seocheon.go.kr',
'sitemap': 'https://www.seocheon.go.kr/kor/sitemap_11.do',
'sheet': '09_서천군', 'parser': parse_ul_sitemap_h4,
'domain': 'seocheon.go.kr',
'folder': r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청남도\9.서천군',
},
{
'idx': 10, 'name': '아산시', 'base': 'https://www.asan.go.kr',
'sitemap': 'https://www.asan.go.kr/main/',
'sheet': '10_아산시', 'parser': parse_asan_gnb,
'domain': 'asan.go.kr',
'folder': r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청남도\10.아산시',
},
{
'idx': 11, 'name': '예산군', 'base': 'https://www.yesan.go.kr',
'sitemap': 'https://www.yesan.go.kr/kor/sitemap.do',
'sheet': '11_예산군', 'parser': parse_yesan_depth,
'domain': 'yesan.go.kr',
'folder': r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청남도\11.예산군',
},
{
'idx': 12, 'name': '천안시', 'base': 'https://www.cheonan.go.kr',
'sitemap': 'https://www.cheonan.go.kr/kor/sitemap.do',
'sheet': '12_천안시', 'parser': parse_cheonan_depth,
'domain': 'cheonan.go.kr',
'folder': r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청남도\12.천안시',
},
{
'idx': 13, 'name': '청양군', 'base': 'https://www.cheongyang.go.kr',
'sitemap': 'https://www.cheongyang.go.kr/kor/sitemap_11.do',
'sheet': '13_청양군', 'parser': parse_cheongyang_depth1,
'domain': 'cheongyang.go.kr',
'folder': r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청남도\13.청양군',
},
{
'idx': 14, 'name': '태안군', 'base': 'https://www.taean.go.kr',
'sitemap': 'https://www.taean.go.kr/kor/sitemap_11.do',
'sheet': '14_태안군', 'parser': parse_amThum,
'domain': 'taean.go.kr',
'folder': r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청남도\14.태안군',
},
{
'idx': 15, 'name': '홍성군', 'base': 'https://www.hongseong.go.kr',
'sitemap': 'https://www.hongseong.go.kr/kor/sitemap.do',
'sheet': '15_홍성군',
'parser': lambda soup, base: parse_dl_dt_dd(soup, base, use_onclick=True),
'domain': 'hongseong.go.kr',
'folder': r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청남도\15.홍성군',
},
]
# ================================================================
# 엑셀 생성 (공통)
# ================================================================
def write_excel(site, raw_rows):
name = site['name']
base = site['base']
domain = site['domain']
output = f"{site['folder']}\\{os.path.basename(os.path.dirname(site['folder']))}_{name}.xlsx"
def abs_url(href):
if not href:
return ''
href = href.strip()
if href.startswith(('javascript:', '#')):
return ''
if href.startswith(('http://', 'https://')):
return href
return urljoin(base + '/', href)
def is_external(url):
return url.startswith(('http://', 'https://')) and domain not in url
# 부모-자식 URL 중복 제거
final_rows = []
i = 0
removed = 0
while i < len(raw_rows):
row = raw_rows[i]
if (i + 1 < len(raw_rows)
and row.get('G', '') == ''
and raw_rows[i + 1].get('D') == row.get('D')
and raw_rows[i + 1].get('E') == row.get('E')
and raw_rows[i + 1].get('F') == row.get('F')
and raw_rows[i + 1].get('G', '') != ''
and raw_rows[i + 1].get('href') == row.get('href')):
removed += 1
i += 1
continue
final_rows.append(row)
i += 1
print(f' [{name}] 원본 {len(raw_rows)} → 중복 {removed} → 최종 {len(final_rows)}')
if not final_rows:
print(f' [{name}] !! 행 0개 — 파서 점검 필요. 엑셀 생성 스킵.')
return False
shutil.copy(TEMPLATE, output)
wb = openpyxl.load_workbook(output)
ws = wb.active
ws.title = site['sheet']
HEADER = {'B1:R1', 'S1:W1', 'Y1:AA1'}
for rng in [str(m) for m in ws.merged_cells.ranges if str(m) not in HEADER]:
ws.unmerge_cells(rng)
for row in ws.iter_rows(min_row=3, max_row=ws.max_row, min_col=1, max_col=ws.max_column):
for cell in row:
cell.value = None
START = 3
template_r = 3
cur_max = ws.max_row
for idx, item in enumerate(final_rows, start=START):
if idx > cur_max:
for c in range(1, ws.max_column + 1):
srcc = ws.cell(template_r, c)
tgt = ws.cell(idx, c)
if srcc.has_style:
tgt.font = copy(srcc.font)
tgt.fill = copy(srcc.fill)
tgt.border = copy(srcc.border)
tgt.alignment = copy(srcc.alignment)
tgt.number_format = srcc.number_format
tgt.protection = copy(srcc.protection)
url = abs_url(item.get('href', ''))
ws.cell(idx, 2).value = idx - 2
ws.cell(idx, 3).value = name
ws.cell(idx, 4).value = item.get('D', '')
ws.cell(idx, 5).value = item.get('E', '')
ws.cell(idx, 6).value = item.get('F', '')
ws.cell(idx, 7).value = item.get('G', '')
ws.cell(idx, 8).value = item.get('H', '')
ws.cell(idx, 9).value = item.get('I', '')
ws.cell(idx, 10).value = item.get('J', '')
ws.cell(idx, 11).value = url
if is_external(url):
ws.cell(idx, 19).value = '외부링크'
END = START + len(final_rows) - 1
center = Alignment(horizontal='center', vertical='center', wrap_text=True)
left = Alignment(horizontal='left', vertical='center', wrap_text=False)
def merge_runs(col_letter, col_idx, group_cols=()):
runs = []
cur_val = ws.cell(START, col_idx).value
cur_grp = tuple(ws.cell(START, g).value for g in group_cols)
run_start = START
for r in range(START + 1, END + 1):
v = ws.cell(r, col_idx).value
g = tuple(ws.cell(r, gg).value for gg in group_cols)
if v == cur_val and g == cur_grp:
continue
if cur_val not in (None, '') and r - 1 > run_start:
runs.append((run_start, r - 1))
cur_val, cur_grp, run_start = v, g, r
if cur_val not in (None, '') and END > run_start:
runs.append((run_start, END))
for s, e in runs:
ws.merge_cells(f'{col_letter}{s}:{col_letter}{e}')
ws.cell(s, col_idx).alignment = center
return len(runs)
# 병합 순서 F→E→D (D를 먼저 병합하면 2행부터 D=None이 되어 E 그룹키가 깨짐)
n_f = merge_runs('F', 6, group_cols=(4, 5))
n_e = merge_runs('E', 5, group_cols=(4,))
n_d = merge_runs('D', 4)
for r in range(START, END + 1):
for c in (4, 5, 6, 7):
if ws.cell(r, c).value is not None:
ws.cell(r, c).alignment = center
for r in range(1, END + 1):
ws.row_dimensions[r].height = 15
link_n = 0
for r in range(START, END + 1):
cell = ws.cell(r, 11)
u = cell.value
if u and isinstance(u, str) and u.startswith(('http://', 'https://')):
cell.hyperlink = u
old = cell.font
cell.font = Font(name=old.name or '맑은 고딕', size=old.size or 11,
bold=old.bold, italic=old.italic,
color='0000FF', underline='single')
cell.alignment = left
link_n += 1
wb.save(output)
ext_n = sum(1 for r in range(START, END + 1) if ws.cell(r, 19).value == '외부링크')
print(f' [{name}] 병합 D:{n_d} E:{n_e} F:{n_f} | 외부링크 {ext_n} | K링크 {link_n} → {output}')
return True
def main():
targets = sys.argv[1:] if len(sys.argv) > 1 else [s['name'] for s in SITES]
for site in SITES:
if site['name'] not in targets:
continue
print(f"\n{'='*60}\n[{site['idx']}.{site['name']}] {site['sitemap']}\n{'='*60}")
try:
html = fetch_html(site['sitemap'])
soup = BeautifulSoup(html, 'html.parser')
raw_rows = site['parser'](soup, site['base'])
write_excel(site, raw_rows)
except Exception as e:
print(f' [{site["name"]}] !! 실패: {e}')
import traceback
traceback.print_exc()
if __name__ == '__main__':
main()
Reference in New Issue View Git Blame Copy Permalink