87 lines
3.3 KiB
Python
87 lines
3.3 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""KISA 14행~끝 전수 sub_tab_menu 스캔. 읽기전용 보고."""
|
|
import sys, io, re, json, time
|
|
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
|
|
import requests, urllib3
|
|
from bs4 import BeautifulSoup
|
|
from urllib.parse import urljoin
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
import openpyxl
|
|
urllib3.disable_warnings()
|
|
|
|
XLSX = r"D:\01.프로젝트\DB수집\작업파일\공공기관3\4.한국인터넷진흥원\한국인터넷진흥원.xlsx"
|
|
BASE = "https://www.kisa.or.kr"
|
|
H = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36'}
|
|
|
|
wb = openpyxl.load_workbook(XLSX)
|
|
ws = wb.active
|
|
rows = []
|
|
for r in range(3, ws.max_row + 1):
|
|
b = ws.cell(r, 2).value
|
|
if b is None:
|
|
break
|
|
url = ws.cell(r, 11).value
|
|
rows.append((r, url, ws.cell(r,4).value, ws.cell(r,5).value, ws.cell(r,6).value, ws.cell(r,7).value))
|
|
|
|
# 동향분석(r14)부터 끝까지
|
|
START_ROW = 14
|
|
scope = [x for x in rows if x[0] >= START_ROW]
|
|
print(f"스캔 대상 {len(scope)}행 (행 {START_ROW}~{rows[-1][0]})")
|
|
|
|
sess = requests.Session(); sess.headers.update(H)
|
|
|
|
def fetch(url):
|
|
try:
|
|
rr = sess.get(url, timeout=15, verify=False, allow_redirects=True)
|
|
meta = re.search(rb'charset=["\']?\s*([\w-]+)', rr.content[:4096], re.I)
|
|
rr.encoding = meta.group(1).decode(errors='ignore') if meta else rr.apparent_encoding
|
|
if rr.status_code == 200:
|
|
return BeautifulSoup(rr.text, 'html.parser')
|
|
except Exception as e:
|
|
return None
|
|
return None
|
|
|
|
def scan(task):
|
|
r, url = task[0], task[1]
|
|
if not url or not isinstance(url, str) or not url.startswith('http') or 'kisa.or.kr' not in url:
|
|
return (r, url, None)
|
|
soup = fetch(url)
|
|
if soup is None:
|
|
return (r, url, 'FETCH_FAIL')
|
|
tabs_found = []
|
|
for ul in soup.select('ul.sub_tab_menu'):
|
|
tabs = []
|
|
for li in ul.find_all('li', recursive=False):
|
|
a = li.find('a')
|
|
if not a:
|
|
continue
|
|
href = a.get('href', '')
|
|
label = a.get_text(strip=True)
|
|
active = ('active' in (li.get('class') or [])) or ('active' in (a.get('class') or []))
|
|
tabs.append({'label': label, 'href': urljoin(BASE, href) if href else '', 'active': active})
|
|
if tabs:
|
|
tabs_found.append(tabs)
|
|
return (r, url, tabs_found if tabs_found else [])
|
|
|
|
results = {}
|
|
with ThreadPoolExecutor(max_workers=8) as ex:
|
|
futs = [ex.submit(scan, t) for t in scope]
|
|
for f in as_completed(futs):
|
|
r, url, res = f.result()
|
|
results[r] = (url, res)
|
|
|
|
out = []
|
|
for r, url, d, e, fcol, g in scope:
|
|
url2, res = results.get(r, (url, None))
|
|
leaf = g or fcol or e or d
|
|
if res == 'FETCH_FAIL':
|
|
print(f"r{r} [{leaf}] {url} -> FETCH_FAIL")
|
|
elif res:
|
|
for ti, tabs in enumerate(res):
|
|
labels = [('*' if t['active'] else '')+t['label'] for t in tabs]
|
|
print(f"r{r} [{leaf}] {url} -> sub_tab_menu({len(tabs)}): {labels}")
|
|
out.append({'row': r, 'leaf': leaf, 'url': url, 'tabs': tabs})
|
|
|
|
print("\n=== sub_tab_menu 보유 행 수:", len(out))
|
|
json.dump(out, open(r"D:\01.프로젝트\DB수집\작업파일\공공기관3\_temp\_kisa_tabs.json", 'w', encoding='utf-8'), ensure_ascii=False, indent=2)
|