df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
80 lines
3.5 KiB
Python
80 lines
3.5 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""전주시 신규 leaf phase2-4 분류 + 본문탭 전수 스캔. 기존 매칭행은 값 보존."""
|
|
import requests, warnings, sys, re, json, time
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
warnings.filterwarnings('ignore'); sys.stdout.reconfigure(encoding='utf-8', line_buffering=True)
|
|
from bs4 import BeautifulSoup
|
|
DIR=r'작업파일\광역_사이트맵\전북특별자치도\12.전주시'
|
|
BASE='https://www.jeonju.go.kr'
|
|
S=requests.Session(); S.headers['User-Agent']='Mozilla/5.0'
|
|
|
|
TAB_SELS=['.tab','ul.tab','.tab_menu','.tabmenu','.tab_wrap','.contab','.tab_con',
|
|
'.tab_list','.sub_tab','[role=tablist]','.tabContent','.tab_area']
|
|
|
|
def fetch(u):
|
|
r=S.get(u,timeout=25,verify=False); r.encoding='utf-8'
|
|
return BeautifulSoup(r.text,'html5lib')
|
|
|
|
def board_count(cu_):
|
|
try:
|
|
soup=fetch(BASE+'/index.9is?contentUid='+cu_)
|
|
if not soup.select_one('table.bbs_table'):
|
|
return None,soup
|
|
end=None
|
|
for a in soup.select('.paging a'):
|
|
if a.get_text(strip=True)=='끝': end=a.get('href')
|
|
if not end:
|
|
# single page: count numbered rows
|
|
tb=soup.select_one('table.bbs_table')
|
|
n=sum(1 for tr in tb.select('tbody tr') if (tr.find('td') and tr.find('td').get_text(strip=True).isdigit()))
|
|
return n,soup
|
|
m=re.search(r'page=(\d+)',end); last=int(m.group(1)) if m else 1
|
|
eu=BASE+end if end.startswith('/') else end
|
|
s2=fetch(eu); tb=s2.select_one('table.bbs_table')
|
|
lastn=sum(1 for tr in tb.select('tbody tr') if (tr.find('td') and tr.find('td').get_text(strip=True).isdigit())) if tb else 0
|
|
return (last-1)*10+lastn, soup
|
|
except Exception as e:
|
|
return -1,None
|
|
|
|
def classify(rec):
|
|
if 'match' in rec:
|
|
mt=rec['match']
|
|
rec.update(L=mt['L'],M=mt['M'],N=mt['N'],O=mt['O'],P=mt['P'],Q=mt['Q'],src='keep')
|
|
return rec
|
|
if rec['external']:
|
|
rec.update(L='사이트',M='',N='',O='',P='',Q='',src='ext'); return rec
|
|
cnt,soup=board_count(rec['cu'])
|
|
# tab scan
|
|
if soup is not None:
|
|
cont=soup.select_one('#content') or soup
|
|
tabs=[]
|
|
for sel in TAB_SELS:
|
|
for e in cont.select(sel):
|
|
txt=e.get_text(' ',strip=True)
|
|
if e.find('a') or e.find('li'): tabs.append((sel,txt[:60]))
|
|
rec['tabs']=tabs[:3]
|
|
if cnt is None:
|
|
rec.update(L='페이지',M=1,N='어문',O='미부착',P='',Q='',src='page')
|
|
elif cnt==-1:
|
|
rec.update(L='페이지',M=1,N='어문',O='미부착',P='',Q='',src='err')
|
|
else:
|
|
rec.update(L='게시판',M=cnt,N=('없음' if cnt==0 else '어문'),O='미부착',P='',Q='',src='board')
|
|
return rec
|
|
|
|
if __name__=='__main__':
|
|
rows=json.load(open(DIR+r'\_tree.json',encoding='utf-8'))
|
|
todo=[r for r in rows if 'match' not in r and not r['external']]
|
|
print('total leaves:',len(rows),'to-fetch:',len(todo))
|
|
done=0
|
|
with ThreadPoolExecutor(max_workers=12) as ex:
|
|
for r in ex.map(classify,rows):
|
|
done+=1
|
|
if done%50==0: print(' ..%d/%d'%(done,len(rows)))
|
|
json.dump(rows,open(DIR+r'\_rows.json','w',encoding='utf-8'),ensure_ascii=False,indent=1)
|
|
from collections import Counter
|
|
print('L분포:',Counter(r['L'] for r in rows))
|
|
tabbed=[r for r in rows if r.get('tabs')]
|
|
print('본문탭 발견 페이지:',len(tabbed))
|
|
for r in tabbed[:20]: print(' TAB',r['path'][-1],r['tabs'])
|
|
print('boards:',[(r['path'][-1],r['M']) for r in rows if r['L']=='게시판'][:40])
|