df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
104 lines
3.9 KiB
Python
104 lines
3.9 KiB
Python
"""각 사이트의 사이트맵 URL 탐색."""
|
|
import sys, io, urllib.request, urllib.error, ssl, re
|
|
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
|
|
|
|
ctx = ssl.create_default_context()
|
|
ctx.check_hostname = False
|
|
ctx.verify_mode = ssl.CERT_NONE
|
|
|
|
HEADERS = {
|
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36',
|
|
'Accept-Language': 'ko-KR,ko;q=0.9,en;q=0.8',
|
|
}
|
|
|
|
def fetch(url, timeout=15):
|
|
try:
|
|
req = urllib.request.Request(url, headers=HEADERS)
|
|
with urllib.request.urlopen(req, timeout=timeout, context=ctx) as r:
|
|
data = r.read()
|
|
ct = r.headers.get('Content-Type', '')
|
|
charset = 'utf-8'
|
|
m = re.search(r'charset=([^\s;]+)', ct)
|
|
if m:
|
|
charset = m.group(1)
|
|
try:
|
|
return r.getcode(), data.decode(charset, errors='replace'), r.geturl()
|
|
except:
|
|
return r.getcode(), data.decode('utf-8', errors='replace'), r.geturl()
|
|
except urllib.error.HTTPError as e:
|
|
return e.code, '', url
|
|
except Exception as e:
|
|
return 0, f'ERR: {e}', url
|
|
|
|
# 사이트별 메인 + 후보 사이트맵 패턴
|
|
SITES = {
|
|
'인천광역시': 'https://www.incheon.go.kr',
|
|
'전라남도': 'https://www.jeonnam.go.kr',
|
|
'고용노동부': 'https://www.moel.go.kr',
|
|
'과학기술정보통신부': 'https://www.msit.go.kr',
|
|
'교육부': 'https://www.moe.go.kr',
|
|
'보건복지부': 'https://www.mohw.go.kr',
|
|
'성평등가족부': 'https://www.mogef.go.kr',
|
|
'외교부': 'https://www.mofa.go.kr',
|
|
}
|
|
|
|
# 1차 — 메인 페이지에서 'sitemap' 링크 찾기
|
|
import re
|
|
SITEMAP_KEY = re.compile(r'href=["\']([^"\']+)["\'][^>]*>[^<]*?(?:사이트맵|Sitemap|SITEMAP|site\s*map)', re.IGNORECASE)
|
|
SITEMAP_HREF = re.compile(r'href=["\']([^"\']*sitemap[^"\']*)["\']', re.IGNORECASE)
|
|
|
|
results = {}
|
|
for name, base in SITES.items():
|
|
print(f'=== {name} ===')
|
|
print(f' base: {base}')
|
|
candidates = set()
|
|
for entry in ['/', '/main', '/index.do', '/index.jsp']:
|
|
url = base + entry
|
|
code, html, final = fetch(url)
|
|
print(f' {entry} -> HTTP {code} (final: {final})')
|
|
if code == 200:
|
|
# 텍스트 기반 검색
|
|
for m in SITEMAP_KEY.findall(html):
|
|
candidates.add(m)
|
|
for m in SITEMAP_HREF.findall(html):
|
|
candidates.add(m)
|
|
break
|
|
# 일반 후보
|
|
common = ['/sitemap.do', '/sitemap.html', '/html/sub06/sitemap.html',
|
|
'/kr/html/guide/0701.html', '/kr/sitemap.do',
|
|
'/kor/sub05_01.do', '/main/contents.do?menuNo=00000']
|
|
for c in common:
|
|
candidates.add(c)
|
|
|
|
# 각 후보 시도
|
|
print(f' 후보: {len(candidates)}개')
|
|
found = []
|
|
for c in candidates:
|
|
if c.startswith('http'):
|
|
test_url = c
|
|
elif c.startswith('//'):
|
|
test_url = 'https:' + c
|
|
elif c.startswith('/'):
|
|
test_url = base + c
|
|
elif c.startswith('javascript:') or c.startswith('#'):
|
|
continue
|
|
else:
|
|
test_url = base + '/' + c
|
|
if 'sitemap' not in test_url.lower() and '0701' not in test_url:
|
|
continue
|
|
code, html, final = fetch(test_url)
|
|
if code == 200 and len(html) > 1000:
|
|
# 사이트맵스러운지 확인 (dl/dt나 ul.depth, .sitemap 클래스 등)
|
|
looks = ('class="sitemap"' in html.lower() or 'sitemap_wrap' in html.lower()
|
|
or html.count('<dl') > 2 or html.count('<dt') > 3)
|
|
print(f' {test_url} -> HTTP {code}, sitemap-like={looks}, len={len(html)}')
|
|
if looks:
|
|
found.append(test_url)
|
|
results[name] = found
|
|
print()
|
|
|
|
import json
|
|
with open(r'D:\01.프로젝트\DB수집\2주차\검토_리포트\sitemap_urls.json', 'w', encoding='utf-8') as f:
|
|
json.dump(results, f, ensure_ascii=False, indent=2)
|
|
print('saved sitemap_urls.json')
|