# -*- coding: utf-8 -*- """공공기관 31곳 사이트맵 페이지 발견 + 구조 분석 (Phase 0 정찰).""" import sys, io, re, warnings, json sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8') from urllib.parse import urljoin, urlparse from concurrent.futures import ThreadPoolExecutor, as_completed import requests from bs4 import BeautifulSoup import openpyxl warnings.filterwarnings('ignore') UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36' H = {'User-Agent': UA} STATUS = r'D:\01.프로젝트\DB수집\작업파일\공공기관_작업현황.xlsx' SITEMAP_WORDS = ['사이트맵', '사이트 맵', '전체메뉴', '전체 메뉴', 'sitemap', 'site map', 'allmenu', '전체보기'] COMMON_PATHS = [ '/sitemap', '/sitemap.do', '/sitemap.jsp', '/sitemap.html', '/sitemap.asp', '/sitemap.aspx', '/siteMap.do', '/siteMap', '/site/main/sitemap', '/main/sitemap.do', '/cms/sitemap.do', '/kor/sitemap.do', '/html/sitemap.html', '/contents/sitemap.do', '/intro/sitemap.do', '/user/sitemap.do', '/web/sitemap.do', '/main/html/sitemap.html', '/sub/sitemap.do', ] def fetch(url, timeout=15): try: r = requests.get(url, headers=H, timeout=timeout, verify=False, allow_redirects=True) ct = r.headers.get('content-type', '') if 'html' not in ct and 'xml' not in ct and r.status_code == 200 and 'text' not in ct: return r.url, None meta = re.search(rb'charset=["\']?\s*([\w-]+)', r.content[:4096], re.I) r.encoding = meta.group(1).decode(errors='ignore') if meta else r.apparent_encoding if r.status_code == 200: return r.url, r.text except Exception as e: return None, f'ERR:{type(e).__name__}' return None, f'HTTP{r.status_code}' def find_sitemap_link(home_html, base): soup = BeautifulSoup(home_html, 'html.parser') found = [] for a in soup.find_all('a', href=True): txt = a.get_text(strip=True).lower().replace(' ', '') href = a['href'] title = (a.get('title', '') or '').lower() for w in SITEMAP_WORDS: ww = w.replace(' ', '') if ww in txt or ww in href.lower() or ww in title.replace(' ', ''): if not href.startswith(('javascript:', '#')): found.append(urljoin(base + '/', href)) break # dedup preserve order seen = set(); out = [] for u in found: if u not in seen: seen.add(u); out.append(u) return out def analyze(html): soup = BeautifulSoup(html, 'html.parser') selectors = [ '.sitemap', '#sitemap', '.site_map', '#siteMap', '.siteMap', '.sitemap_wrap', '.sitemapWrap', '.allMenu', '.allmenu', '#allMenu', '.all_menu', '.allMenuWrap', '#allmenu', '.menu_all', '.gnb_all', '.totalMenu', '.totMenu', '.total_menu', '.full_menu', '#contents .sitemap', 'div[class*=sitemap]', 'div[class*=siteMap]', 'div[class*=allMenu]', 'div[class*=allmenu]', 'div[id*=sitemap]', 'div[id*=siteMap]', ] cands = [] for sel in selectors: try: for el in soup.select(sel): ac = len(el.find_all('a')) if ac >= 15: cands.append((ac, sel, el)) except Exception: pass if not cands: for d in soup.find_all(['div', 'section', 'main', 'nav']): cls = ' '.join(d.get('class', [])).lower() did = (d.get('id', '') or '').lower() if any(x in cls + did for x in ['footer', 'header']): continue ac = len(d.find_all('a')) if ac >= 40: cands.append((ac, f'<{d.name} class="{cls}" id="{did}">', d)) cands.sort(key=lambda x: x[0], reverse=True) if not cands: return None ac, sel, el = cands[0] uls = el.find_all('ul', recursive=False) dls = el.find_all('dl', recursive=False) if not uls and not dls: for ch in el.find_all(['div', 'section'], recursive=False): uls += ch.find_all('ul', recursive=False) dls += ch.find_all('dl', recursive=False) pat = 'A(dl)' if dls else ('B(ul)' if uls else 'UNK') # max nesting depth of ul def depth(e, d=0): subs = e.find_all('ul', recursive=False) + [u for c in e.find_all('li', recursive=False) for u in c.find_all('ul', recursive=False)] return max([depth(s, d + 1) for s in subs], default=d) return {'sel': sel, 'a': ac, 'ul': len(el.find_all('ul')), 'dl': len(el.find_all('dl')), 'li': len(el.find_all('li')), 'pat': pat, 'depth': depth(el)} def load_sites(): wb = openpyxl.load_workbook(STATUS, data_only=True) ws = wb['현황'] sites = [] for r in ws.iter_rows(min_row=2, values_only=True): if r[0] == '공공기관' and r[1]: num, name, url = r[1], r[2], r[11] if not url: continue if not url.startswith('http'): url = 'https://' + url sites.append({'num': num, 'name': name, 'home': url}) return sites def probe(site): name = site['name'] home = site['home'] pr = urlparse(home) base = f'{pr.scheme}://{pr.netloc}' res = {'num': site['num'], 'name': name, 'home': home, 'base': base, 'sitemap': None, 'struct': None, 'tried': []} u, html = fetch(home) if not u or not html: res['err'] = f'home fail: {html}' return res res['base'] = f"{urlparse(u).scheme}://{urlparse(u).netloc}" base = res['base'] # 1) homepage sitemap link links = find_sitemap_link(html, base) # 2) common paths cand_urls = links + [base + p for p in COMMON_PATHS] best = None for cu in cand_urls[:20]: fu, fhtml = fetch(cu) res['tried'].append(cu) if fu and fhtml: st = analyze(fhtml) if st and st['a'] >= 20: best = (fu, st) break if best: res['sitemap'] = best[0] res['struct'] = best[1] else: # fallback: analyze homepage itself (mega menu) st = analyze(html) res['struct'] = st res['sitemap'] = None return res def main(): sites = load_sites() only = sys.argv[1:] if only: sites = [s for s in sites if s['name'] in only or str(s['num']) in only] print(f'정찰 대상 {len(sites)}곳\n') results = [] with ThreadPoolExecutor(max_workers=6) as ex: futs = {ex.submit(probe, s): s for s in sites} for f in as_completed(futs): results.append(f.result()) results.sort(key=lambda x: x['num']) for r in results: sm = r['sitemap'] or '(없음→메가메뉴)' st = r['struct'] sts = f"a={st['a']} {st['pat']} d={st['depth']} sel={st['sel']}" if st else 'STRUCT없음' err = r.get('err', '') print(f"{r['num']:>2}. {r['name']}") print(f" home={r['home']}") print(f" sitemap={sm}") print(f" {sts} {err}") out = r'D:\01.프로젝트\DB수집\_스크립트\_공공기관_probe.json' with open(out, 'w', encoding='utf-8') as f: json.dump([{k: (v if k != 'struct' or v is None else v) for k, v in r.items() if k != 'tried'} for r in results], f, ensure_ascii=False, indent=1, default=str) print(f'\n→ {out}') if __name__ == '__main__': main()