df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
70 lines
3.1 KiB
Python
70 lines
3.1 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""벤더 콘텐츠탭 셀렉터 조사: 샘플 페이지에서 탭류 컨테이너를 위치(부모 컨텍스트)와 함께 덤프."""
|
|
import openpyxl, requests, sys, re
|
|
from bs4 import BeautifulSoup
|
|
from urllib.parse import urljoin, urlparse
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from collections import Counter
|
|
import urllib3
|
|
urllib3.disable_warnings()
|
|
sys.stdout.reconfigure(encoding='utf-8')
|
|
UA={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'}
|
|
NAV_ANCESTORS=('header','gnb','lnb','snb','nav','footer','top','aside','menu_wrap','allmenu','sitemap')
|
|
TABISH=re.compile(r'(tab|basic_tab|slave|grap|deth|nav_cont|intab)', re.I)
|
|
|
|
def is_in_nav(el):
|
|
p=el
|
|
for _ in range(8):
|
|
p=p.parent
|
|
if p is None: break
|
|
idc=(p.get('id') or '')+' '+' '.join(p.get('class') or [])
|
|
idc=idc.lower()
|
|
if any(k in idc for k in NAV_ANCESTORS): return True
|
|
return False
|
|
|
|
def main():
|
|
xlsx, n = sys.argv[1], int(sys.argv[2]) if len(sys.argv)>2 else 30
|
|
wb=openpyxl.load_workbook(xlsx); ws=wb.active
|
|
urls=[]
|
|
for r in range(3, ws.max_row+1):
|
|
c=ws.cell(r,11); L=ws.cell(r,12).value
|
|
u=c.hyperlink.target if c.hyperlink else (c.value if isinstance(c.value,str) and c.value.startswith('http') else None)
|
|
if u and L in ('페이지','게시판'): urls.append(u)
|
|
dom=Counter(urlparse(u).netloc.replace('www.','') for u in urls).most_common(1)[0][0]
|
|
urls=[u for u in urls if dom in urlparse(u).netloc]
|
|
seen=set(); samp=[]
|
|
for u in urls:
|
|
k=u.split('?')[0]
|
|
if k in seen: continue
|
|
seen.add(k); samp.append(u)
|
|
if len(samp)>=n: break
|
|
sess=requests.Session(); sess.headers.update(UA)
|
|
def fetch(u):
|
|
try:
|
|
rr=sess.get(u,timeout=20,verify=False); rr.encoding=rr.apparent_encoding or 'utf-8'; return u, rr.text
|
|
except: return u, None
|
|
sig=Counter()
|
|
with ThreadPoolExecutor(max_workers=12) as ex:
|
|
for fu in as_completed([ex.submit(fetch,u) for u in samp]):
|
|
u, html=fu.result()
|
|
if not html: continue
|
|
soup=BeautifulSoup(html,'html.parser')
|
|
for el in soup.find_all(['ul','div']):
|
|
idc=(el.get('id') or '')+' '+' '.join(el.get('class') or [])
|
|
if not TABISH.search(idc): continue
|
|
lis=[li for li in el.find_all('li') if li.find('a')]
|
|
if len(lis)<2: continue
|
|
anchors=[li.find('a') for li in lis]
|
|
hrefs=[a.get('href','') for a in anchors]
|
|
anchor_only=all(h.strip().startswith('#') or not h.strip() for h in hrefs)
|
|
kind='인페이지#' if anchor_only else '페이지/게시판링크'
|
|
where='NAV' if is_in_nav(el) else 'CONTENT'
|
|
tag=el.name
|
|
cls=re.sub(r'\d+','N','.'.join(el.get('class') or []) or ('#'+ (el.get('id') or '')))
|
|
sig[f"{where:7} {tag}.{cls[:24]:24} li={len(lis):>2} {kind}"]+=1
|
|
print(f'=== {xlsx.split(chr(92))[-1]} (도메인 {dom}, 샘플 {len(samp)}) ===')
|
|
for s,c in sig.most_common(25):
|
|
print(f' x{c:>2} {s}')
|
|
|
|
if __name__=='__main__': main()
|