df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
83 lines
3.2 KiB
Python
83 lines
3.2 KiB
Python
import sys, io, json
|
|
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
|
|
import openpyxl, urllib.request, ssl, re
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
|
|
P = r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청남도\10.아산시\충청남도_아산시.xlsx'
|
|
wb = openpyxl.load_workbook(P)
|
|
ws = wb.active
|
|
|
|
ctx = ssl.create_default_context(); ctx.check_hostname=False; ctx.verify_mode=ssl.CERT_NONE
|
|
hdr={'User-Agent':'Mozilla/5.0'}
|
|
def fetch(url):
|
|
req=urllib.request.Request(url, headers=hdr)
|
|
with urllib.request.urlopen(req, context=ctx, timeout=25) as r:
|
|
return r.read().decode('utf-8','replace')
|
|
|
|
# collect target rows 37+: asan.go.kr pages, L in 페이지/게시판/None
|
|
targets=[]
|
|
for r in range(37, ws.max_row+1):
|
|
k = ws[f'K{r}'].value
|
|
l = ws[f'L{r}'].value
|
|
o = ws[f'O{r}'].value
|
|
if not k: continue
|
|
if 'asan.go.kr' not in str(k): continue
|
|
if l == '사이트': continue
|
|
targets.append((r, str(k), l, o))
|
|
|
|
print(f'targets: {len(targets)} rows', file=sys.stderr)
|
|
|
|
def work(t):
|
|
r,k,l,o = t
|
|
try:
|
|
html = fetch(k)
|
|
except Exception as e:
|
|
return (r,k,l,o,'ERR:'+str(e)[:60], None, None)
|
|
# opentype mark
|
|
mt = re.search(r'img_opentype_(\d)\.gif', html)
|
|
otype = mt.group(1) if mt else None
|
|
# in-page tab nav: ul with class containing openInfo / ng0X tab, listing pg= links
|
|
tabs=None
|
|
for um in re.finditer(r'<ul[^>]*class="([^"]*)"[^>]*>(.*?)</ul>', html, re.S):
|
|
cls=um.group(1); body=um.group(2)
|
|
if re.search(r'openInfo|ng0\d', cls):
|
|
items=re.findall(r'<a[^>]*href="([^"]*)"[^>]*>(.*?)</a>', body, re.S)
|
|
items=[(h, re.sub(r'<[^>]+>','',t).strip()) for h,t in items]
|
|
if len(items)>=2 and any('pg=' in h or 'no=' in h for h,_ in items):
|
|
tabs={'cls':cls.strip(),'items':items}
|
|
break
|
|
return (r,k,l,o,None,otype,tabs)
|
|
|
|
res=[]
|
|
with ThreadPoolExecutor(max_workers=10) as ex:
|
|
for out in ex.map(work, targets):
|
|
res.append(out)
|
|
|
|
# Report A: opentype marks where O is 미부착/empty (needs fixing)
|
|
print('\n=== A) img_opentype present but O=미부착/empty (FIX CANDIDATES) ===')
|
|
for r,k,l,o,err,otype,tabs in sorted(res):
|
|
if otype and (not o or str(o).strip()=='미부착'):
|
|
print(f'r{r}: TYPE={otype} L={l} O={o!r} K={k}')
|
|
|
|
print('\n=== A2) img_opentype present AND O already set (verify match) ===')
|
|
for r,k,l,o,err,otype,tabs in sorted(res):
|
|
if otype and o and str(o).strip()!='미부착':
|
|
print(f'r{r}: TYPE={otype} O={o!r} L={l} K={k}')
|
|
|
|
print('\n=== B) in-page openInfo/ngXX tabs found ===')
|
|
for r,k,l,o,err,otype,tabs in sorted(res):
|
|
if tabs:
|
|
print(f'r{r}: cls={tabs["cls"]} n={len(tabs["items"])} L={l} K={k}')
|
|
for h,t in tabs['items']:
|
|
print(f' {h} | {t}')
|
|
|
|
print('\n=== C) fetch errors ===')
|
|
for r,k,l,o,err,otype,tabs in sorted(res):
|
|
if err:
|
|
print(f'r{r}: {err} K={k}')
|
|
|
|
# dump json for later use
|
|
json.dump([{'r':r,'k':k,'l':l,'o':o,'err':err,'otype':otype,'tabs':tabs} for r,k,l,o,err,otype,tabs in res],
|
|
open(r'D:\01.프로젝트\DB수집\작업파일\_스크립트\_asan_crawl.json','w',encoding='utf-8'), ensure_ascii=False, indent=1)
|
|
print('\nsaved _asan_crawl.json', file=sys.stderr)
|