df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
43 lines
2.4 KiB
Python
43 lines
2.4 KiB
Python
# -*- coding: utf-8 -*-
|
|
import openpyxl, urllib.request, ssl, re, glob, os, sys, time, http.cookiejar
|
|
ctx=ssl.create_default_context(); ctx.check_hostname=False; ctx.verify_mode=ssl.CERT_NONE
|
|
cj=http.cookiejar.CookieJar()
|
|
op=urllib.request.build_opener(urllib.request.HTTPSHandler(context=ctx), urllib.request.HTTPCookieProcessor(cj))
|
|
op.addheaders=[('User-Agent','Mozilla/5.0')]
|
|
def fetch(u): return op.open(u,timeout=20).read().decode('utf-8','ignore')
|
|
NO=int(sys.argv[1])
|
|
BASE=os.path.dirname(os.path.abspath(__file__))
|
|
d=[x for x in glob.glob(os.path.join(BASE,'*')) if os.path.isdir(x) and os.path.basename(x).startswith(str(NO)+'.')][0]
|
|
xp=[p for p in glob.glob(os.path.join(d,'*.xlsx')) if not os.path.basename(p).startswith(('_','~')) and 'conflict' not in os.path.basename(p)][0]
|
|
ws=openpyxl.load_workbook(xp).active
|
|
def norm(u): return re.sub(r'\?.*','',str(u).split('#')[0]).rstrip('/').lower() if u else ''
|
|
rows=[(r,ws.cell(r,11).value) for r in range(3,ws.max_row+1) if ws.cell(r,2).value is not None and ws.cell(r,11).value]
|
|
host=re.match(r'(https?://[^/]+)', str(rows[0][1])).group(1)
|
|
sheetset=set(norm(u) for _,u in rows)
|
|
# tab widget: ul whose class contains tab and has >=2 <li><a> with hrefs
|
|
TABRE=re.compile(r'<ul[^>]*class="([^"]*(?:tab|Tab)[^"]*)"[^>]*>(.*?)</ul>', re.S)
|
|
seen=set(); groups=[]
|
|
n=0
|
|
for r,u in rows:
|
|
if 'http' in str(u) and host.split('//')[1] not in str(u): continue # internal only
|
|
try: html=fetch(str(u))
|
|
except: continue
|
|
n+=1
|
|
for m in TABRE.finditer(html):
|
|
cls=m.group(1); items=[]
|
|
for a in re.finditer(r'<a[^>]*href="([^"]*)"[^>]*>(.*?)</a>', m.group(2), re.S):
|
|
href=a.group(1); t=re.sub(r'<[^>]+>','',a.group(2)).strip()
|
|
if t and href and href not in ('#',) and 'javascript:void' not in href:
|
|
full=href if href.startswith('http') else host+('' if href.startswith('/') else '/')+href
|
|
items.append((t,full))
|
|
if len(items)>=2:
|
|
key=tuple(norm(h) for _,h in items)
|
|
if key in seen: continue
|
|
seen.add(key)
|
|
missing=[(t,h) for t,h in items if norm(h) not in sheetset]
|
|
if missing:
|
|
groups.append((r,cls,items,missing))
|
|
print('site %d: %d페이지 스캔, 탭위젯그룹(미수집탭보유) %d'%(NO,n,len(groups)))
|
|
for r,cls,items,missing in groups[:25]:
|
|
print(' r%d [%s] 미수집%d: %s'%(r,cls[:20],len(missing),' | '.join(t for t,_ in missing[:6])))
|