df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
80 lines
3.4 KiB
Python
80 lines
3.4 KiB
Python
import requests, sys, io, re, json
|
|
from bs4 import BeautifulSoup
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import openpyxl, urllib3
|
|
urllib3.disable_warnings()
|
|
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
|
|
H = {'User-Agent':'Mozilla/5.0'}
|
|
|
|
wb=openpyxl.load_workbook('충청남도_아산시.xlsx'); ws=wb.active
|
|
rows=[]
|
|
for r in range(16,271):
|
|
url=ws.cell(r,11).value; L=ws.cell(r,12).value; M=ws.cell(r,13).value
|
|
label=ws.cell(r,7).value or ws.cell(r,6).value or ws.cell(r,5).value or ws.cell(r,4).value
|
|
if url: rows.append((r,label,url,L,M))
|
|
|
|
def classify(item):
|
|
r,label,url,L,M=item
|
|
res={'row':r,'label':label,'url':url,'oldL':L}
|
|
try:
|
|
resp=requests.get(url,headers=H,timeout=25,verify=False,allow_redirects=True)
|
|
final=resp.url
|
|
resp.encoding=resp.apparent_encoding or 'utf-8'
|
|
s=BeautifulSoup(resp.text,'html.parser')
|
|
dom=re.sub(r'^https?://','',final).split('/')[0]
|
|
res['final_dom']=dom
|
|
# page_tab
|
|
ptabs=[(a.get_text(strip=True),a.get('href')) for a in s.select('div.page_tab ul li a') if a.get('href','').strip() not in ('','#')]
|
|
res['ptab']=ptabs
|
|
# board detection
|
|
cnt=None
|
|
# 1) lc06 caption [name:N/page]
|
|
for t in s.select('table'):
|
|
cap=t.find('caption')
|
|
if cap:
|
|
m=re.search(r':\s*([\d,]+)\s*/\s*\d+',cap.get_text())
|
|
if m: cnt=int(m.group(1).replace(',','')); break
|
|
# 2) total counters
|
|
if cnt is None:
|
|
txt=s.get_text(' ',strip=True)
|
|
m=re.search(r'(?:전체|총|Total)\s*([\d,]+)\s*건',txt)
|
|
if m: cnt=int(m.group(1).replace(',',''))
|
|
# signals
|
|
has_paging=bool(s.select('.paging a, .pagination a, .page_num a, nav.paging a'))
|
|
has_search=bool(s.select('input[name=searchKeyword], input[name=keyword], input[name=sword], input[type=search]'))
|
|
has_bbs_table=False
|
|
for t in s.select('table'):
|
|
ths=[th.get_text(strip=True) for th in t.select('th')]
|
|
if '제목' in ths and ('번호' in ths or '작성일' in ths or '등록일' in ths or '게시일자' in ths):
|
|
has_bbs_table=True; break
|
|
is_board = (cnt is not None) or has_bbs_table or (has_paging and has_search)
|
|
res['cnt']=cnt; res['paging']=has_paging; res['search']=has_search; res['bbstbl']=has_bbs_table
|
|
if 'asan.go.kr' not in dom:
|
|
res['newL']='사이트'
|
|
else:
|
|
res['newL']='게시판' if is_board else '페이지'
|
|
res['status']=resp.status_code
|
|
except Exception as e:
|
|
res['err']=str(e)[:80]; res['newL']='ERR'
|
|
return res
|
|
|
|
out=[]
|
|
with ThreadPoolExecutor(max_workers=12) as ex:
|
|
for res in ex.map(classify, rows):
|
|
out.append(res)
|
|
json.dump(out,open('_scan_out.json','w',encoding='utf-8'),ensure_ascii=False,indent=1)
|
|
|
|
# report
|
|
chg=[o for o in out if o.get('oldL')!=o.get('newL') and o.get('newL')!='ERR']
|
|
print(f'TOTAL {len(out)} | changes {len(chg)} | errors {len([o for o in out if o.get("newL")=="ERR"])}')
|
|
print('\n=== L CHANGES ===')
|
|
for o in chg:
|
|
print(f" r{o['row']} {o['label']}: {o['oldL']}→{o['newL']} cnt={o.get('cnt')}")
|
|
print('\n=== page_tab GROUPS ===')
|
|
for o in out:
|
|
if o.get('ptab'):
|
|
print(f" r{o['row']} {o['label']} ({len(o['ptab'])} tabs): {[t[0] for t in o['ptab']]}")
|
|
print('\n=== ERRORS ===')
|
|
for o in out:
|
|
if o.get('newL')=='ERR': print(f" r{o['row']} {o['label']}: {o.get('err')}")
|