# -*- coding: utf-8 -*- """ 범용 탭 스캐너 (읽기전용). 사용: python -X utf8 _tab_scan.py "" [--json out.json] [--workers 16] - 시트의 K열 내부 URL들을 동시 fetch - 각 페이지에서 '탭 컨테이너(ul/div)'를 구조 기반으로 탐지 - 탭 링크를 site/board/page/anchor 로 분류 - 탭 타깃이 시트에 없으면 '누락(missing)' 으로 집계 출력: 벤더추정 + 누락그룹 수 + 예상 추가행 수 + 그룹 상세 """ import openpyxl, requests, sys, re, json, argparse from bs4 import BeautifulSoup from urllib.parse import urljoin, urlparse from concurrent.futures import ThreadPoolExecutor, as_completed from collections import Counter import urllib3 urllib3.disable_warnings() sys.stdout.reconfigure(encoding='utf-8') UA = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'} TAB_KEYWORDS = ('tab', 'nav_', '_nav', 'depth', 'dep4', 'slave', 'sub_tab', 'subtab', 'lnbtab', 'snbtab') ON_CLASSES = ('on', 'active', 'select', 'current', 'sel', 'now') def norm_url(u): if not u: return '' u = u.split('#')[0].rstrip('/') u = re.sub(r'^https?://(www\.)?', '', u) return u.lower() def is_tab_container(ul): """ul이 탭 컨테이너인지 구조 기반 판정.""" lis = ul.find_all('li', recursive=False) if len(lis) < 2: # 일부 벤더는 li 비직속. 모든 li 중 a 가진 것 사용 lis = [li for li in ul.find_all('li') if li.find('a')] if len(lis) < 2: return None anchors = [li.find('a') for li in lis if li.find('a')] if len(anchors) < 2: return None cls = ' '.join(ul.get('class') or []).lower() kw = any(k in cls for k in TAB_KEYWORDS) has_on = any(any(o in ' '.join((li.get('class') or [])).lower() for o in ON_CLASSES) for li in lis) # 링크 경로 유사성: 같은 디렉토리 또는 공통 쿼리키 hrefs = [a.get('href', '') for a in anchors if a.get('href')] if len(hrefs) < 2: return None paths = [urlparse(h).path for h in hrefs] common_dir = len(set(p.rsplit('/', 1)[0] for p in paths)) == 1 qkeys = [tuple(sorted(re.findall(r'[?&](\w+)=', h))) for h in hrefs] common_q = len(set(qkeys)) == 1 and qkeys and qkeys[0] score = sum([kw, has_on, common_dir, bool(common_q)]) if (kw and (has_on or common_dir or common_q)) or (has_on and (common_dir or common_q)): return anchors return None def classify(a, href, domain): target = (a.get('target') or '') cls = ' '.join(a.get('class') or []).lower() host = urlparse(href).netloc.lower() h = href.split('#')[0] if not h or href.strip().startswith('#') or href.strip().lower().startswith('javascript'): return 'anchor' if target == '_blank' or 'link_3th' in cls or (host and domain not in host): return 'site' if 'selectboardlist' in h.lower() or '/bbs/' in h.lower() or 'bbsmstr' in h.lower(): return 'board' return 'page' def scan_xlsx(xlsx, workers=16, lo=3): class A: pass args = A(); args.xlsx = xlsx; args.workers = workers; args.lo = lo; args.json = None wb = openpyxl.load_workbook(args.xlsx) ws = wb.active # data rows rows = [] for r in range(args.lo, ws.max_row + 1): c = ws.cell(r, 11) url = c.hyperlink.target if c.hyperlink else (c.value if isinstance(c.value, str) and c.value.startswith('http') else None) L = ws.cell(r, 12).value F = ws.cell(r, 6).value; G = ws.cell(r, 7).value if url: rows.append({'r': r, 'url': url, 'L': L, 'F': F, 'G': G}) sheet_norm = set(norm_url(x['url']) for x in rows) # dominant domain hosts = Counter(urlparse(x['url']).netloc.replace('www.', '') for x in rows if urlparse(x['url']).netloc) domain = hosts.most_common(1)[0][0] if hosts else '' # internal content pages to scan targets = [x for x in rows if x['L'] in ('페이지', '게시판') and domain in urlparse(x['url']).netloc] uniq = {} for x in targets: uniq.setdefault(norm_url(x['url']), x) sess = requests.Session(); sess.headers.update(UA) def fetch(x): try: rr = sess.get(x['url'], timeout=20, verify=False) rr.encoding = rr.apparent_encoding or 'utf-8' return x, rr.text except Exception as e: return x, None groups = [] container_classes = Counter() with ThreadPoolExecutor(max_workers=args.workers) as ex: futs = [ex.submit(fetch, x) for x in uniq.values()] for fu in as_completed(futs): x, html = fu.result() if not html: continue soup = BeautifulSoup(html, 'html.parser') best = None for ul in soup.find_all('ul'): anchors = is_tab_container(ul) if not anchors: continue # build tabs tabs = [] for a in anchors: href = urljoin(x['url'], a.get('href', '').strip()) kind = classify(a, href, domain) tabs.append({'label': ' '.join(a.get_text().split())[:30], 'href': href, 'kind': kind, 'norm': norm_url(href)}) # only real page/board tabs count pb = [t for t in tabs if t['kind'] in ('page', 'board')] if len(pb) < 2: continue missing = [t for t in pb if t['norm'] not in sheet_norm] if not missing: continue cand = {'cls': ' '.join(ul.get('class') or []), 'n': len(pb), 'missing': len(missing), 'tabs': tabs} if best is None or cand['missing'] > best['missing']: best = cand if best: container_classes[best['cls']] += 1 groups.append({'r': x['r'], 'F': x['F'], 'url': x['url'], 'cls': best['cls'], 'tabs_n': best['n'], 'missing': best['missing']}) total_missing = sum(g['missing'] for g in groups) out = { 'xlsx': args.xlsx, 'domain': domain, 'data_rows': len(rows), 'scanned_pages': len(uniq), 'tab_groups': len(groups), 'est_new_rows': total_missing, 'container_classes': dict(container_classes.most_common()), 'groups': sorted(groups, key=lambda g: -g['missing']), } return out def main(): ap = argparse.ArgumentParser() ap.add_argument('xlsx') ap.add_argument('--json', default=None) ap.add_argument('--workers', type=int, default=16) ap.add_argument('--lo', type=int, default=3) args = ap.parse_args() out = scan_xlsx(args.xlsx, args.workers, args.lo) if args.json: json.dump(out, open(args.json, 'w', encoding='utf-8'), ensure_ascii=False, indent=1) print(json.dumps({k: v for k, v in out.items() if k != 'groups'}, ensure_ascii=False, indent=1)) print('--- top groups ---') for g in out['groups'][:15]: print(f" R{g['r']:>3} {str(g['F'])[:18]:18} cls='{g['cls']}' tabs={g['tabs_n']} missing={g['missing']}") if __name__ == '__main__': main()