df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
187 lines
7.3 KiB
Python
187 lines
7.3 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""공공기관 31곳 사이트맵 페이지 발견 + 구조 분석 (Phase 0 정찰)."""
|
|
import sys, io, re, warnings, json
|
|
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
|
|
from urllib.parse import urljoin, urlparse
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
import requests
|
|
from bs4 import BeautifulSoup
|
|
import openpyxl
|
|
|
|
warnings.filterwarnings('ignore')
|
|
UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
|
|
H = {'User-Agent': UA}
|
|
STATUS = r'D:\01.프로젝트\DB수집\작업파일\공공기관_작업현황.xlsx'
|
|
|
|
SITEMAP_WORDS = ['사이트맵', '사이트 맵', '전체메뉴', '전체 메뉴', 'sitemap', 'site map', 'allmenu', '전체보기']
|
|
COMMON_PATHS = [
|
|
'/sitemap', '/sitemap.do', '/sitemap.jsp', '/sitemap.html', '/sitemap.asp', '/sitemap.aspx',
|
|
'/siteMap.do', '/siteMap', '/site/main/sitemap', '/main/sitemap.do', '/cms/sitemap.do',
|
|
'/kor/sitemap.do', '/html/sitemap.html', '/contents/sitemap.do', '/intro/sitemap.do',
|
|
'/user/sitemap.do', '/web/sitemap.do', '/main/html/sitemap.html', '/sub/sitemap.do',
|
|
]
|
|
|
|
|
|
def fetch(url, timeout=15):
|
|
try:
|
|
r = requests.get(url, headers=H, timeout=timeout, verify=False, allow_redirects=True)
|
|
ct = r.headers.get('content-type', '')
|
|
if 'html' not in ct and 'xml' not in ct and r.status_code == 200 and 'text' not in ct:
|
|
return r.url, None
|
|
meta = re.search(rb'charset=["\']?\s*([\w-]+)', r.content[:4096], re.I)
|
|
r.encoding = meta.group(1).decode(errors='ignore') if meta else r.apparent_encoding
|
|
if r.status_code == 200:
|
|
return r.url, r.text
|
|
except Exception as e:
|
|
return None, f'ERR:{type(e).__name__}'
|
|
return None, f'HTTP{r.status_code}'
|
|
|
|
|
|
def find_sitemap_link(home_html, base):
|
|
soup = BeautifulSoup(home_html, 'html.parser')
|
|
found = []
|
|
for a in soup.find_all('a', href=True):
|
|
txt = a.get_text(strip=True).lower().replace(' ', '')
|
|
href = a['href']
|
|
title = (a.get('title', '') or '').lower()
|
|
for w in SITEMAP_WORDS:
|
|
ww = w.replace(' ', '')
|
|
if ww in txt or ww in href.lower() or ww in title.replace(' ', ''):
|
|
if not href.startswith(('javascript:', '#')):
|
|
found.append(urljoin(base + '/', href))
|
|
break
|
|
# dedup preserve order
|
|
seen = set(); out = []
|
|
for u in found:
|
|
if u not in seen:
|
|
seen.add(u); out.append(u)
|
|
return out
|
|
|
|
|
|
def analyze(html):
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
selectors = [
|
|
'.sitemap', '#sitemap', '.site_map', '#siteMap', '.siteMap', '.sitemap_wrap', '.sitemapWrap',
|
|
'.allMenu', '.allmenu', '#allMenu', '.all_menu', '.allMenuWrap', '#allmenu',
|
|
'.menu_all', '.gnb_all', '.totalMenu', '.totMenu', '.total_menu', '.full_menu',
|
|
'#contents .sitemap', 'div[class*=sitemap]', 'div[class*=siteMap]', 'div[class*=allMenu]',
|
|
'div[class*=allmenu]', 'div[id*=sitemap]', 'div[id*=siteMap]',
|
|
]
|
|
cands = []
|
|
for sel in selectors:
|
|
try:
|
|
for el in soup.select(sel):
|
|
ac = len(el.find_all('a'))
|
|
if ac >= 15:
|
|
cands.append((ac, sel, el))
|
|
except Exception:
|
|
pass
|
|
if not cands:
|
|
for d in soup.find_all(['div', 'section', 'main', 'nav']):
|
|
cls = ' '.join(d.get('class', [])).lower()
|
|
did = (d.get('id', '') or '').lower()
|
|
if any(x in cls + did for x in ['footer', 'header']):
|
|
continue
|
|
ac = len(d.find_all('a'))
|
|
if ac >= 40:
|
|
cands.append((ac, f'<{d.name} class="{cls}" id="{did}">', d))
|
|
cands.sort(key=lambda x: x[0], reverse=True)
|
|
if not cands:
|
|
return None
|
|
ac, sel, el = cands[0]
|
|
uls = el.find_all('ul', recursive=False)
|
|
dls = el.find_all('dl', recursive=False)
|
|
if not uls and not dls:
|
|
for ch in el.find_all(['div', 'section'], recursive=False):
|
|
uls += ch.find_all('ul', recursive=False)
|
|
dls += ch.find_all('dl', recursive=False)
|
|
pat = 'A(dl)' if dls else ('B(ul)' if uls else 'UNK')
|
|
# max nesting depth of ul
|
|
def depth(e, d=0):
|
|
subs = e.find_all('ul', recursive=False) + [u for c in e.find_all('li', recursive=False) for u in c.find_all('ul', recursive=False)]
|
|
return max([depth(s, d + 1) for s in subs], default=d)
|
|
return {'sel': sel, 'a': ac, 'ul': len(el.find_all('ul')), 'dl': len(el.find_all('dl')),
|
|
'li': len(el.find_all('li')), 'pat': pat, 'depth': depth(el)}
|
|
|
|
|
|
def load_sites():
|
|
wb = openpyxl.load_workbook(STATUS, data_only=True)
|
|
ws = wb['현황']
|
|
sites = []
|
|
for r in ws.iter_rows(min_row=2, values_only=True):
|
|
if r[0] == '공공기관' and r[1]:
|
|
num, name, url = r[1], r[2], r[11]
|
|
if not url:
|
|
continue
|
|
if not url.startswith('http'):
|
|
url = 'https://' + url
|
|
sites.append({'num': num, 'name': name, 'home': url})
|
|
return sites
|
|
|
|
|
|
def probe(site):
|
|
name = site['name']
|
|
home = site['home']
|
|
pr = urlparse(home)
|
|
base = f'{pr.scheme}://{pr.netloc}'
|
|
res = {'num': site['num'], 'name': name, 'home': home, 'base': base, 'sitemap': None, 'struct': None, 'tried': []}
|
|
u, html = fetch(home)
|
|
if not u or not html:
|
|
res['err'] = f'home fail: {html}'
|
|
return res
|
|
res['base'] = f"{urlparse(u).scheme}://{urlparse(u).netloc}"
|
|
base = res['base']
|
|
# 1) homepage sitemap link
|
|
links = find_sitemap_link(html, base)
|
|
# 2) common paths
|
|
cand_urls = links + [base + p for p in COMMON_PATHS]
|
|
best = None
|
|
for cu in cand_urls[:20]:
|
|
fu, fhtml = fetch(cu)
|
|
res['tried'].append(cu)
|
|
if fu and fhtml:
|
|
st = analyze(fhtml)
|
|
if st and st['a'] >= 20:
|
|
best = (fu, st)
|
|
break
|
|
if best:
|
|
res['sitemap'] = best[0]
|
|
res['struct'] = best[1]
|
|
else:
|
|
# fallback: analyze homepage itself (mega menu)
|
|
st = analyze(html)
|
|
res['struct'] = st
|
|
res['sitemap'] = None
|
|
return res
|
|
|
|
|
|
def main():
|
|
sites = load_sites()
|
|
only = sys.argv[1:]
|
|
if only:
|
|
sites = [s for s in sites if s['name'] in only or str(s['num']) in only]
|
|
print(f'정찰 대상 {len(sites)}곳\n')
|
|
results = []
|
|
with ThreadPoolExecutor(max_workers=6) as ex:
|
|
futs = {ex.submit(probe, s): s for s in sites}
|
|
for f in as_completed(futs):
|
|
results.append(f.result())
|
|
results.sort(key=lambda x: x['num'])
|
|
for r in results:
|
|
sm = r['sitemap'] or '(없음→메가메뉴)'
|
|
st = r['struct']
|
|
sts = f"a={st['a']} {st['pat']} d={st['depth']} sel={st['sel']}" if st else 'STRUCT없음'
|
|
err = r.get('err', '')
|
|
print(f"{r['num']:>2}. {r['name']}")
|
|
print(f" home={r['home']}")
|
|
print(f" sitemap={sm}")
|
|
print(f" {sts} {err}")
|
|
out = r'D:\01.프로젝트\DB수집\_스크립트\_공공기관_probe.json'
|
|
with open(out, 'w', encoding='utf-8') as f:
|
|
json.dump([{k: (v if k != 'struct' or v is None else v) for k, v in r.items() if k != 'tried'} for r in results], f, ensure_ascii=False, indent=1, default=str)
|
|
print(f'\n→ {out}')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|