DB_JOB/작업파일/_스크립트/_tab_scan_cms.py

# -*- coding: utf-8 -*-
"""
CMS 프로파일 기반 탭 스캐너 (충남표준CMS 검증판).
범용탐지(_tab_scan)와 달리 '알려진 콘텐츠탭 셀렉터'만 봐서 내비 과탐 제거.
covered() 규칙: 부모필터 URL(?cate_b_cd=b01)이 자식(?cate_b_cd=b01&cate_cd=10)으로 시트에 있으면 커버로 간주.
"""
import openpyxl, requests, sys, re, json
from bs4 import BeautifulSoup
from urllib.parse import urljoin, urlparse
from concurrent.futures import ThreadPoolExecutor, as_completed
from collections import Counter
import urllib3
urllib3.disable_warnings()
sys.stdout.reconfigure(encoding='utf-8')
UA={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'}

# ---- 콘텐츠탭 셀렉터 카탈로그(여러 벤더 근거) ----
# 충남표준=tab-ul/basic_tab/slave_tab, 서산계=tab_menu/tab_button, 부여 사전공개=fiexd_tab
TAB_TOKENS = {'tab-ul', 'basic_tab', 'slave_tab', 'tab_menu', 'tab_button',
              'fiexd_tab', 'tabmenu', 'sub_tab', 'con_tab', 'contab', 'tab_wrap'}
BAD_TOKENS = {'table', 'tablewrap', 'response', 'grap', 'no-more-tables',
              'depth', 'side', 'location', 'gnb', 'lnb', 'snb', 'allmenu', 'footer'}
NAV_ANCESTORS=('header','gnb','lnb','snb','nav','footer','top','aside','allmenu','sitemap')

def norm_url(u):
    if not u: return ''
    u=u.split('#')[0].rstrip('/')
    return re.sub(r'^https?://(www\.)?','',u).lower()

def in_nav(el):
    p=el
    for _ in range(8):
        p=p.parent
        if p is None: break
        idc=((p.get('id') or '')+' '+' '.join(p.get('class') or [])).lower()
        if any(k in idc for k in NAV_ANCESTORS): return True
    return False

def is_cms_tab(el):
    toks=set(re.sub(r'\d+','',t).strip('_-') for t in (el.get('class') or []))
    toks={t for t in toks if t}
    if any(b in ' '.join(el.get('class') or []).lower() for b in BAD_TOKENS): return False
    # 토큰 정확매칭(tab-ul/basic_tab/slave_tab). tab-ul은 하이픈 보존 필요
    cls=' '.join(el.get('class') or [])
    if not any(re.search(r'(^|\s)'+re.escape(t)+r'(\d|\s|$)', cls+' ') for t in TAB_TOKENS):
        return False
    if in_nav(el): return False
    return True

def covered(t_norm, sheet_norm):
    if t_norm in sheet_norm: return True
    for s in sheet_norm:
        if s.startswith(t_norm+'&') or s.startswith(t_norm+'/') or s.startswith(t_norm+'?'):
            return True
    return False

def classify(a, href, domain):
    target=(a.get('target') or ''); cls=' '.join(a.get('class') or []).lower()
    host=urlparse(href).netloc.lower(); h=href.split('#')[0]
    if not h or href.strip().startswith('#') or href.strip().lower().startswith('javascript'): return 'anchor'
    if target=='_blank' or 'link_3th' in cls or (host and domain not in host): return 'site'
    if 'selectboardlist' in h.lower() or '/bbs/' in h.lower() or 'bbsmstr' in h.lower(): return 'board'
    return 'page'

def scan(xlsx, workers=16):
    wb=openpyxl.load_workbook(xlsx); ws=wb.active
    rows=[]
    for r in range(3, ws.max_row+1):
        c=ws.cell(r,11); L=ws.cell(r,12).value
        u=c.hyperlink.target if c.hyperlink else (c.value if isinstance(c.value,str) and c.value.startswith('http') else None)
        if u: rows.append({'r':r,'url':u,'L':L,'F':ws.cell(r,6).value})
    sheet_norm=set(norm_url(x['url']) for x in rows)
    dom=Counter(urlparse(x['url']).netloc.replace('www.','') for x in rows if urlparse(x['url']).netloc).most_common(1)[0][0]
    targets=[x for x in rows if x['L'] in ('페이지','게시판') and dom in urlparse(x['url']).netloc]
    uniq={}
    for x in targets: uniq.setdefault(norm_url(x['url']), x)
    sess=requests.Session(); sess.headers.update(UA)
    def fetch(x):
        try:
            rr=sess.get(x['url'],timeout=20,verify=False); rr.encoding=rr.apparent_encoding or 'utf-8'; return x,rr.text
        except: return x,None
    page_groups=[]; inpage_groups=[]
    with ThreadPoolExecutor(max_workers=workers) as ex:
        for fu in as_completed([ex.submit(fetch,x) for x in uniq.values()]):
            x,html=fu.result()
            if not html: continue
            soup=BeautifulSoup(html,'html.parser')
            for el in soup.find_all(['ul','div']):
                if not is_cms_tab(el): continue
                lis=[li for li in el.find_all('li') if li.find('a')]
                if len(lis)<2: continue
                tabs=[]
                for li in lis:
                    a=li.find('a'); href=urljoin(x['url'], a.get('href','').strip())
                    tabs.append({'label':' '.join(a.get_text().split())[:24],'href':href,
                                 'kind':classify(a,href,dom),'norm':norm_url(href)})
                pb=[t for t in tabs if t['kind'] in ('page','board')]
                anchors=[t for t in tabs if t['kind']=='anchor']
                if len(anchors)>=2 and len(pb)<2:
                    inpage_groups.append({'r':x['r'],'F':x['F'],'n':len(anchors)}); continue
                if len(pb)<2: continue
                missing=[t for t in pb if not covered(t['norm'], sheet_norm)]
                if missing:
                    page_groups.append({'r':x['r'],'F':x['F'],'cls':' '.join(el.get('class') or []),
                                        'tabs':len(pb),'missing':len(missing),
                                        'ex':[m['label'] for m in missing[:4]]})
    return {'xlsx':xlsx.split('\\')[-1].split('/')[-1],'dom':dom,'rows':len(rows),'pages':len(uniq),
            'page_tab_groups':len(page_groups),'est_new':sum(g['missing'] for g in page_groups),
            'inpage_groups':len(inpage_groups),
            'top':sorted(page_groups,key=lambda g:-g['missing'])[:12]}

if __name__=='__main__':
    out=scan(sys.argv[1])
    print(json.dumps({k:v for k,v in out.items() if k!='top'}, ensure_ascii=False, indent=1))
    for g in out['top']:
        print(f"  R{g['r']:>3} {str(g['F'])[:16]:16} cls='{g['cls'][:20]}' tabs={g['tabs']} missing={g['missing']}  ex={g['ex']}")