df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
77 lines
4.1 KiB
Python
77 lines
4.1 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""억제된 페이지→게시판 후보(아직 페이지인 행)를 빈게시판 신호로 재분석. 읽기전용."""
|
|
import sys, os, re, json, importlib.util, time
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
import openpyxl
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__)); TEMP = os.path.join(HERE,'..','_temp')
|
|
MODULES = ['_chungnam_phase234_all.py','_chungbuk_phase234_all.py','_jeonbuk_phase234_all.py']
|
|
def load():
|
|
sites,mods={},{}
|
|
for f in MODULES:
|
|
sp=importlib.util.spec_from_file_location(f[:-3],os.path.join(HERE,f));m=importlib.util.module_from_spec(sp);sp.loader.exec_module(m)
|
|
for k,v in m.SITES.items(): sites[k]=v;mods[k]=m
|
|
return sites,mods
|
|
EMPTY=re.compile(r'게시물이?\s*없|등록된?\s*(?:게시물|자료|글|내용)\s*가?\s*없|자료가\s*없|검색된\s*(?:게시물|결과)\s*가?\s*없|데이터가\s*없')
|
|
BOARDDOM='.board_list,.bbs_list,table.board_list,.board_view,.board_wrap,.bbs,.tbl_list,.board,.list_wrap,.gallery_list,.photo_list,table.board'
|
|
WRITE=re.compile(r'글쓰기|글등록|등록하기|write|글작성')
|
|
def mkfetch(M,cfg):
|
|
if hasattr(M,'make_session'):
|
|
s=M.make_session(weak_ssl=cfg.get('weak_ssl',False)); return lambda u:M.fetch(s,u)
|
|
return lambda u:M.fetch(u)
|
|
def analyze(M,url,bsel,fetch):
|
|
soup=None
|
|
for k in range(3):
|
|
soup=fetch(url)
|
|
if soup is not None: break
|
|
time.sleep(0.4*(k+1))
|
|
if soup is None: return {'v':'fail'}
|
|
b=M.get_body(soup,bsel)
|
|
txt=b.get_text(' ',strip=True)
|
|
search=len([i for i in b.find_all('input') if (i.get('type') or 'text').lower() in ('text','search')])>0
|
|
rows=[el for el in b.select('table tbody tr, .board_list li, ul.bbs_list li, .bbs_list li') if el.find('a',href=True)]
|
|
empty=bool(EMPTY.search(txt))
|
|
boarddom=bool(b.select(BOARDDOM))
|
|
paging=bool(b.select('.pagination,.paging,nav.paging,.page_nav,.paginate'))
|
|
write=bool(WRITE.search(txt))
|
|
if empty or (boarddom and (paging or write)):
|
|
v='empty_board'
|
|
else:
|
|
v='page'
|
|
return {'v':v,'search':search,'rows':len(rows),'empty':empty,'boarddom':boarddom,'paging':paging,'write':write,
|
|
'snip':txt[:160]}
|
|
def main():
|
|
sites,mods=load(); names=[a for a in sys.argv[1:] if not a.startswith('--')]
|
|
for n in names:
|
|
cfg=sites[n];M=mods[n];fetch=mkfetch(M,cfg)
|
|
wb=openpyxl.load_workbook(cfg['xlsx'],read_only=True);ws=wb.active
|
|
d=json.load(open(os.path.join(TEMP,f'_audit_{n}.json'),encoding='utf-8'))
|
|
cands=[]
|
|
for x in d['L_diffs']:
|
|
if x['oldL']=='페이지' and x['newL']=='게시판':
|
|
if ws.cell(x['r'],12).value=='페이지': # 아직 페이지(억제됨)
|
|
cands.append(x)
|
|
wb.close()
|
|
res=[]
|
|
with ThreadPoolExecutor(max_workers=6) as ex:
|
|
futs={ex.submit(analyze,M,x['url'],cfg['body_sel'],fetch):x for x in cands}
|
|
for f in as_completed(futs):
|
|
x=futs[f]
|
|
try:r=f.result()
|
|
except:r={'v':'fail'}
|
|
res.append((x,r))
|
|
eb=[(x,r) for x,r in res if r['v']=='empty_board']
|
|
pg=[(x,r) for x,r in res if r['v']=='page']
|
|
nf=len([1 for _,r in res if r['v']=='fail'])
|
|
print('\n==== %s: 억제후보%d → 빈게시판의심%d 페이지%d 실패%d'%(n,len(cands),len(eb),len(pg),nf))
|
|
print('--- 빈게시판 의심(empty_board) 예시 ---')
|
|
for x,r in sorted(eb,key=lambda t:t[0]['r'])[:8]:
|
|
print(f" r{x['r']} {(x['label'] or '')[:20]} | empty={r['empty']} boarddom={r['boarddom']} paging={r['paging']} | {x['url'][:62]}")
|
|
print(f" · {r['snip'][:110]}")
|
|
print('--- 순수 페이지 예시 ---')
|
|
for x,r in sorted(pg,key=lambda t:t[0]['r'])[:5]:
|
|
print(f" r{x['r']} {(x['label'] or '')[:20]} | boarddom={r['boarddom']} | {x['url'][:62]}")
|
|
print(f" · {r['snip'][:110]}")
|
|
json.dump([{**x,**r} for x,r in res],open(os.path.join(TEMP,f'_emptyboard_{n}.json'),'w',encoding='utf-8'),ensure_ascii=False,indent=1)
|
|
if __name__=='__main__': main()
|