df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
92 lines
3.4 KiB
Python
92 lines
3.4 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""미지 탭 클래스 발견: 표본 페이지 본문에서 '같은 본문에 2+ 실제링크를 가진 UL'의
|
|
class 를 빈도순 집계한다. 사이트 전역 nav(거의 모든 페이지에 동일 링크집합) 제외 목적.
|
|
|
|
사용: python -X utf8 _tab_discover.py <엑셀> <도메인> [표본수=40]
|
|
"""
|
|
import sys, warnings, ssl
|
|
from collections import Counter
|
|
from urllib.parse import urlsplit
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
|
|
import openpyxl, requests
|
|
from requests.adapters import HTTPAdapter
|
|
from urllib3.util.ssl_ import create_urllib3_context
|
|
from bs4 import BeautifulSoup
|
|
|
|
warnings.filterwarnings('ignore')
|
|
H = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120 Safari/537.36'}
|
|
|
|
|
|
class WeakSSLAdapter(HTTPAdapter):
|
|
def init_poolmanager(self, *a, **k):
|
|
ctx = create_urllib3_context(); ctx.set_ciphers('DEFAULT@SECLEVEL=0')
|
|
ctx.options |= 0x4; ctx.check_hostname = False; ctx.verify_mode = ssl.CERT_NONE
|
|
k['ssl_context'] = ctx; return super().init_poolmanager(*a, **k)
|
|
|
|
|
|
S = requests.Session(); S.headers.update(H)
|
|
if '--weak-ssl' in sys.argv:
|
|
S.mount('https://', WeakSSLAdapter())
|
|
|
|
|
|
def norm(u):
|
|
s = urlsplit(u); return (s.path.rstrip('/')).lower()
|
|
|
|
|
|
def fetch(r, u):
|
|
try:
|
|
x = S.get(u, timeout=15, verify=False)
|
|
return r, u, x.content
|
|
except Exception:
|
|
return r, u, None
|
|
|
|
|
|
def main():
|
|
xlsx, domain = sys.argv[1], sys.argv[2]
|
|
nsamp = int([a for a in sys.argv[3:] if a.isdigit()][0]) if any(a.isdigit() for a in sys.argv[3:]) else 40
|
|
wb = openpyxl.load_workbook(xlsx, read_only=True); ws = wb.active
|
|
urls = []
|
|
for row in ws.iter_rows(min_row=3, min_col=11, max_col=11, values_only=True):
|
|
u = row[0]
|
|
if isinstance(u, str) and domain in u:
|
|
urls.append(u)
|
|
wb.close()
|
|
step = max(1, len(urls) // nsamp)
|
|
sample = urls[::step][:nsamp]
|
|
print(f'표본 {len(sample)}/{len(urls)} ({domain})')
|
|
|
|
# class별 등장 페이지 수 / 링크집합 다양성
|
|
cls_pages = Counter()
|
|
cls_linksets = {}
|
|
with ThreadPoolExecutor(max_workers=8) as ex:
|
|
futs = [ex.submit(fetch, i, u) for i, u in enumerate(sample)]
|
|
for f in as_completed(futs):
|
|
r, u, html = f.result()
|
|
if not html:
|
|
continue
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
for ul in soup.find_all(['ul', 'ol']):
|
|
links = []
|
|
for a in ul.find_all('a'):
|
|
t = a.get_text(strip=True)
|
|
h = (a.get('href') or '').strip()
|
|
if t and h and not h.startswith(('#', 'javascript:')):
|
|
links.append(norm(h))
|
|
if len(links) < 2:
|
|
continue
|
|
cls = ' '.join(ul.get('class') or []).strip() or '(no-class)'
|
|
cls_pages[cls] += 1
|
|
cls_linksets.setdefault(cls, set()).add(tuple(links))
|
|
|
|
print('\n=== UL/OL class별 (2+링크) — 등장페이지수 / 서로다른 링크집합수 ===')
|
|
print('(전역 nav = 등장多+집합1 / 본문탭 = 집합 다양) \n')
|
|
for cls, cnt in cls_pages.most_common(40):
|
|
nset = len(cls_linksets[cls])
|
|
flag = '★탭후보' if nset >= 3 and cnt >= 3 else ''
|
|
print(f' {cnt:3d}p / {nset:3d}집합 [{cls}] {flag}')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|