df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
51 lines
2.3 KiB
Python
51 lines
2.3 KiB
Python
# -*- coding: utf-8 -*-
|
|
import urllib.request, ssl, re, json, time
|
|
from urllib.parse import quote, urlsplit, urlunsplit
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
ctx=ssl.create_default_context(); ctx.check_hostname=False; ctx.verify_mode=ssl.CERT_NONE
|
|
def enc(u):
|
|
p=urlsplit(u); return urlunsplit((p.scheme,p.netloc,p.path,quote(p.query,safe='=&'),p.fragment))
|
|
def get(u):
|
|
for _ in range(3):
|
|
try:
|
|
req=urllib.request.Request(enc(u),headers={'User-Agent':'Mozilla/5.0'})
|
|
return urllib.request.urlopen(req,context=ctx,timeout=18).read().decode('utf-8','replace')
|
|
except Exception: time.sleep(0.6)
|
|
return None
|
|
|
|
rows={x['r']:x for x in json.load(open('_sgp_rows.json',encoding='utf-8'))}
|
|
# pages+boards >=18, seogwipo internal
|
|
targets=[x for x in rows.values() if x['r']>=18 and x['L'] in ('페이지','게시판') and x['url'] and 'seogwipo.go.kr' in x['url'] and '/tool/' not in x['url']]
|
|
|
|
def clean(t): return re.sub(r'\s+',' ',re.sub(r'<[^>]+>','',t)).strip()
|
|
|
|
def task(x):
|
|
h=get(x['url'])
|
|
if not h: return x['r'],{'fetch':False}
|
|
# find tab-menu uls (depth1 candidates): ul with class containing tab-menu OR id=sources
|
|
res=[]
|
|
for m in re.finditer(r'<ul[^>]*(?:id="sources"|class="[^"]*tab-menu[^"]*")[^>]*>(.*?)</ul>', h, re.S):
|
|
block=m.group(1)
|
|
lis=re.findall(r'<a[^>]+href="([^"]+)"[^>]*>(.*?)</a>', block, re.S)
|
|
lis=[(href,clean(t)) for href,t in lis if clean(t)]
|
|
if len(lis)>=2:
|
|
res.append(lis)
|
|
return x['r'],{'fetch':True,'tabgroups':res}
|
|
|
|
out={}
|
|
with ThreadPoolExecutor(max_workers=8) as ex:
|
|
for r,d in ex.map(task, targets):
|
|
out[r]=d
|
|
json.dump(out, open('_tabscan.json','w',encoding='utf-8'), ensure_ascii=False)
|
|
# report rows with tab groups
|
|
fails=[r for r,d in out.items() if not d.get('fetch')]
|
|
print('스캔 %d행 | fail %d'%(len(out),len(fails)), fails[:10])
|
|
print('=== 본문 탭그룹(ul tab-menu/sources >=2탭) 보유 행 ===')
|
|
for r in sorted(out):
|
|
tg=out[r].get('tabgroups',[])
|
|
if tg:
|
|
lbl=[m for m in rows[r]['menu'] if m]
|
|
# describe each group
|
|
desc=' || '.join('['+','.join(t for _,t in g[:6])[:60]+(' +%d'%(len(g)-6) if len(g)>6 else '')+']' for g in tg)
|
|
print(' r%d %s (%s) : %s'%(r,(lbl[-1] if lbl else '')[:16], rows[r]['L'], desc))
|