df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
161 lines
6.9 KiB
Python
161 lines
6.9 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""
|
|
범용 탭 스캐너 (읽기전용).
|
|
사용: python -X utf8 _tab_scan.py "<xlsx경로>" [--json out.json] [--workers 16]
|
|
- 시트의 K열 내부 URL들을 동시 fetch
|
|
- 각 페이지에서 '탭 컨테이너(ul/div)'를 구조 기반으로 탐지
|
|
- 탭 링크를 site/board/page/anchor 로 분류
|
|
- 탭 타깃이 시트에 없으면 '누락(missing)' 으로 집계
|
|
출력: 벤더추정 + 누락그룹 수 + 예상 추가행 수 + 그룹 상세
|
|
"""
|
|
import openpyxl, requests, sys, re, json, argparse
|
|
from bs4 import BeautifulSoup
|
|
from urllib.parse import urljoin, urlparse
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from collections import Counter
|
|
import urllib3
|
|
urllib3.disable_warnings()
|
|
sys.stdout.reconfigure(encoding='utf-8')
|
|
|
|
UA = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'}
|
|
TAB_KEYWORDS = ('tab', 'nav_', '_nav', 'depth', 'dep4', 'slave', 'sub_tab', 'subtab', 'lnbtab', 'snbtab')
|
|
ON_CLASSES = ('on', 'active', 'select', 'current', 'sel', 'now')
|
|
|
|
def norm_url(u):
|
|
if not u: return ''
|
|
u = u.split('#')[0].rstrip('/')
|
|
u = re.sub(r'^https?://(www\.)?', '', u)
|
|
return u.lower()
|
|
|
|
def is_tab_container(ul):
|
|
"""ul이 탭 컨테이너인지 구조 기반 판정."""
|
|
lis = ul.find_all('li', recursive=False)
|
|
if len(lis) < 2:
|
|
# 일부 벤더는 li 비직속. 모든 li 중 a 가진 것 사용
|
|
lis = [li for li in ul.find_all('li') if li.find('a')]
|
|
if len(lis) < 2:
|
|
return None
|
|
anchors = [li.find('a') for li in lis if li.find('a')]
|
|
if len(anchors) < 2:
|
|
return None
|
|
cls = ' '.join(ul.get('class') or []).lower()
|
|
kw = any(k in cls for k in TAB_KEYWORDS)
|
|
has_on = any(any(o in ' '.join((li.get('class') or [])).lower() for o in ON_CLASSES) for li in lis)
|
|
# 링크 경로 유사성: 같은 디렉토리 또는 공통 쿼리키
|
|
hrefs = [a.get('href', '') for a in anchors if a.get('href')]
|
|
if len(hrefs) < 2:
|
|
return None
|
|
paths = [urlparse(h).path for h in hrefs]
|
|
common_dir = len(set(p.rsplit('/', 1)[0] for p in paths)) == 1
|
|
qkeys = [tuple(sorted(re.findall(r'[?&](\w+)=', h))) for h in hrefs]
|
|
common_q = len(set(qkeys)) == 1 and qkeys and qkeys[0]
|
|
score = sum([kw, has_on, common_dir, bool(common_q)])
|
|
if (kw and (has_on or common_dir or common_q)) or (has_on and (common_dir or common_q)):
|
|
return anchors
|
|
return None
|
|
|
|
def classify(a, href, domain):
|
|
target = (a.get('target') or '')
|
|
cls = ' '.join(a.get('class') or []).lower()
|
|
host = urlparse(href).netloc.lower()
|
|
h = href.split('#')[0]
|
|
if not h or href.strip().startswith('#') or href.strip().lower().startswith('javascript'):
|
|
return 'anchor'
|
|
if target == '_blank' or 'link_3th' in cls or (host and domain not in host):
|
|
return 'site'
|
|
if 'selectboardlist' in h.lower() or '/bbs/' in h.lower() or 'bbsmstr' in h.lower():
|
|
return 'board'
|
|
return 'page'
|
|
|
|
def scan_xlsx(xlsx, workers=16, lo=3):
|
|
class A: pass
|
|
args = A(); args.xlsx = xlsx; args.workers = workers; args.lo = lo; args.json = None
|
|
wb = openpyxl.load_workbook(args.xlsx)
|
|
ws = wb.active
|
|
# data rows
|
|
rows = []
|
|
for r in range(args.lo, ws.max_row + 1):
|
|
c = ws.cell(r, 11)
|
|
url = c.hyperlink.target if c.hyperlink else (c.value if isinstance(c.value, str) and c.value.startswith('http') else None)
|
|
L = ws.cell(r, 12).value
|
|
F = ws.cell(r, 6).value; G = ws.cell(r, 7).value
|
|
if url:
|
|
rows.append({'r': r, 'url': url, 'L': L, 'F': F, 'G': G})
|
|
sheet_norm = set(norm_url(x['url']) for x in rows)
|
|
# dominant domain
|
|
hosts = Counter(urlparse(x['url']).netloc.replace('www.', '') for x in rows if urlparse(x['url']).netloc)
|
|
domain = hosts.most_common(1)[0][0] if hosts else ''
|
|
# internal content pages to scan
|
|
targets = [x for x in rows if x['L'] in ('페이지', '게시판') and domain in urlparse(x['url']).netloc]
|
|
uniq = {}
|
|
for x in targets:
|
|
uniq.setdefault(norm_url(x['url']), x)
|
|
|
|
sess = requests.Session(); sess.headers.update(UA)
|
|
def fetch(x):
|
|
try:
|
|
rr = sess.get(x['url'], timeout=20, verify=False)
|
|
rr.encoding = rr.apparent_encoding or 'utf-8'
|
|
return x, rr.text
|
|
except Exception as e:
|
|
return x, None
|
|
|
|
groups = []
|
|
container_classes = Counter()
|
|
with ThreadPoolExecutor(max_workers=args.workers) as ex:
|
|
futs = [ex.submit(fetch, x) for x in uniq.values()]
|
|
for fu in as_completed(futs):
|
|
x, html = fu.result()
|
|
if not html: continue
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
best = None
|
|
for ul in soup.find_all('ul'):
|
|
anchors = is_tab_container(ul)
|
|
if not anchors: continue
|
|
# build tabs
|
|
tabs = []
|
|
for a in anchors:
|
|
href = urljoin(x['url'], a.get('href', '').strip())
|
|
kind = classify(a, href, domain)
|
|
tabs.append({'label': ' '.join(a.get_text().split())[:30], 'href': href, 'kind': kind, 'norm': norm_url(href)})
|
|
# only real page/board tabs count
|
|
pb = [t for t in tabs if t['kind'] in ('page', 'board')]
|
|
if len(pb) < 2: continue
|
|
missing = [t for t in pb if t['norm'] not in sheet_norm]
|
|
if not missing: continue
|
|
cand = {'cls': ' '.join(ul.get('class') or []), 'n': len(pb), 'missing': len(missing),
|
|
'tabs': tabs}
|
|
if best is None or cand['missing'] > best['missing']:
|
|
best = cand
|
|
if best:
|
|
container_classes[best['cls']] += 1
|
|
groups.append({'r': x['r'], 'F': x['F'], 'url': x['url'],
|
|
'cls': best['cls'], 'tabs_n': best['n'], 'missing': best['missing']})
|
|
total_missing = sum(g['missing'] for g in groups)
|
|
out = {
|
|
'xlsx': args.xlsx, 'domain': domain, 'data_rows': len(rows),
|
|
'scanned_pages': len(uniq), 'tab_groups': len(groups),
|
|
'est_new_rows': total_missing,
|
|
'container_classes': dict(container_classes.most_common()),
|
|
'groups': sorted(groups, key=lambda g: -g['missing']),
|
|
}
|
|
return out
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument('xlsx')
|
|
ap.add_argument('--json', default=None)
|
|
ap.add_argument('--workers', type=int, default=16)
|
|
ap.add_argument('--lo', type=int, default=3)
|
|
args = ap.parse_args()
|
|
out = scan_xlsx(args.xlsx, args.workers, args.lo)
|
|
if args.json:
|
|
json.dump(out, open(args.json, 'w', encoding='utf-8'), ensure_ascii=False, indent=1)
|
|
print(json.dumps({k: v for k, v in out.items() if k != 'groups'}, ensure_ascii=False, indent=1))
|
|
print('--- top groups ---')
|
|
for g in out['groups'][:15]:
|
|
print(f" R{g['r']:>3} {str(g['F'])[:18]:18} cls='{g['cls']}' tabs={g['tabs_n']} missing={g['missing']}")
|
|
|
|
if __name__ == '__main__':
|
|
main()
|