df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
82 lines
3.7 KiB
Python
82 lines
3.7 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""클래스명 무관 본문탭 탐지: 표본 페이지에서 '자기 URL을 포함하면서 2+실제링크 &
|
|
1+ 신규(엑셀에 없는) URL'을 가진 UL을 찾아 그 class 를 집계. 1-5b 조건2~4의 클래스 비의존 버전.
|
|
미지의 탭 클래스명을 발견하기 위함.
|
|
|
|
사용: python -X utf8 _tab_classfind.py <엑셀> <base> <도메인> [표본=50] [--weak-ssl]
|
|
"""
|
|
import sys, warnings, ssl
|
|
from collections import Counter
|
|
from urllib.parse import urlsplit, urljoin
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
import openpyxl, requests
|
|
from requests.adapters import HTTPAdapter
|
|
from urllib3.util.ssl_ import create_urllib3_context
|
|
from bs4 import BeautifulSoup
|
|
warnings.filterwarnings('ignore')
|
|
H={'User-Agent':'Mozilla/5.0 Chrome/120 Safari/537.36'}
|
|
class W(HTTPAdapter):
|
|
def init_poolmanager(self,*a,**k):
|
|
c=create_urllib3_context();c.set_ciphers('DEFAULT@SECLEVEL=0');c.options|=0x4
|
|
c.check_hostname=False;c.verify_mode=ssl.CERT_NONE;k['ssl_context']=c
|
|
return super().init_poolmanager(*a,**k)
|
|
S=requests.Session();S.headers.update(H)
|
|
if '--weak-ssl' in sys.argv: S.mount('https://',W())
|
|
|
|
def norm(u):
|
|
s=urlsplit(u);return urlsplit(urljoin('http://x/',s.path)).path.rstrip('/').lower()
|
|
def absu(h,base):
|
|
h=(h or '').strip()
|
|
if not h or h.startswith(('javascript:','#')): return ''
|
|
if h.startswith(('http://','https://')): return h
|
|
return urljoin(base.rstrip('/')+'/',h.lstrip('/'))
|
|
|
|
def main():
|
|
xlsx,base,domain=sys.argv[1],sys.argv[2],sys.argv[3]
|
|
nums=[a for a in sys.argv[4:] if a.isdigit()]
|
|
nsamp=int(nums[0]) if nums else 50
|
|
wb=openpyxl.load_workbook(xlsx,read_only=True);ws=wb.active
|
|
urls=[];existing=set()
|
|
for row in ws.iter_rows(min_row=3,min_col=11,max_col=11,values_only=True):
|
|
u=row[0]
|
|
if isinstance(u,str) and u.startswith('http'):
|
|
existing.add(norm(u))
|
|
if domain in u: urls.append(u)
|
|
wb.close()
|
|
step=max(1,len(urls)//nsamp);sample=urls[::step][:nsamp]
|
|
print(f'표본 {len(sample)}/{len(urls)} ({domain})')
|
|
cls_cnt=Counter();examples={}
|
|
def work(u):
|
|
try:
|
|
r=S.get(u,timeout=15,verify=False);return u,r.content
|
|
except Exception:return u,None
|
|
with ThreadPoolExecutor(max_workers=8) as ex:
|
|
for f in as_completed([ex.submit(work,u) for u in sample]):
|
|
u,html=f.result()
|
|
if not html:continue
|
|
cur=norm(u);soup=BeautifulSoup(html,'html.parser')
|
|
for ul in soup.find_all('ul'):
|
|
links=[]
|
|
ok=True
|
|
for a in ul.find_all('a'):
|
|
t=a.get_text(strip=True)
|
|
au=absu(a.get('href'),base)
|
|
if not t:continue
|
|
if not au: ok=False;break
|
|
links.append(au)
|
|
if not ok or len(links)<2:continue
|
|
norms=[norm(x) for x in links]
|
|
if cur not in norms:continue # 조건3: 자기 탭그룹
|
|
if sum(1 for n in norms if n not in existing)<1:continue # 조건4: 신규1+
|
|
cls=' '.join(ul.get('class') or []) or '(no-class)'
|
|
cls_cnt[cls]+=1
|
|
examples.setdefault(cls,(u,[ (a.get_text(strip=True), absu(a.get('href'),base)) for a in ul.find_all('a') if a.get_text(strip=True)][:6]))
|
|
print('\n=== 본문탭(자기URL포함+신규有) UL class 빈도 ===')
|
|
if not cls_cnt: print(' (없음 — 본문탭 진짜 없음 가능성)')
|
|
for cls,c in cls_cnt.most_common(15):
|
|
print(f' {c:3d}회 [{cls}]')
|
|
eu,el=examples[cls];print(f' 예: {eu}')
|
|
for t,h in el: print(f' - {t} -> {h}')
|
|
|
|
if __name__=='__main__':main()
|