df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
132 lines
4.5 KiB
Python
132 lines
4.5 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""공공기관 게시판 M(수량) 0 보정 — 총건수 추출 강화.
|
|
전략: ①총/전체/Total N건 정규식 ②목록행 글번호 최댓값(최신글=총건수) ③페이징 마지막페이지×페이지당행수.
|
|
대상: L==게시판 & M in (0,None,''). 못구하면 그대로(빈게시판 가능).
|
|
사용: python _공공기관_mfix.py [기관명 ...]
|
|
"""
|
|
import sys, os, re, json, importlib.util, warnings
|
|
import openpyxl
|
|
warnings.filterwarnings('ignore')
|
|
spec = importlib.util.spec_from_file_location('p234', r'D:\01.프로젝트\DB수집\_스크립트\_공공기관_phase234.py')
|
|
P = importlib.util.module_from_spec(spec); spec.loader.exec_module(P)
|
|
OUTDIR = r'D:\01.프로젝트\DB수집\공공기관'
|
|
TOTAL = re.compile(r'(?:총|전체|total|건수)\s*[:\-]?\s*([\d,]+)\s*(?:건|개|page|페이지|item|EA)?', re.I)
|
|
PAGE_PARAM = re.compile(r'(?:page|pageIndex|pageNo|cpage|curPage|nowPage|pageNum)\s*=\s*(\d+)', re.I)
|
|
|
|
|
|
def from_total(txt):
|
|
best = 0
|
|
for m in TOTAL.finditer(txt):
|
|
d = m.group(1).replace(',', '')
|
|
if d.isdigit():
|
|
best = max(best, int(d))
|
|
return best
|
|
|
|
|
|
def from_rownums(body):
|
|
"""목록 행의 글번호(보통 첫 칸 숫자) 최댓값 = 최신글번호 ≈ 총건수."""
|
|
nums = []
|
|
rows = body.select('table tbody tr') or body.select('table tr')
|
|
for tr in rows:
|
|
cells = tr.find_all(['td', 'th'])
|
|
if not cells:
|
|
continue
|
|
t = cells[0].get_text(strip=True).replace(',', '')
|
|
if re.fullmatch(r'\d{1,7}', t):
|
|
nums.append(int(t))
|
|
# ul형 게시판
|
|
if not nums:
|
|
for li in body.select('ul li, ol li'):
|
|
sp = li.find(['span', 'em', 'strong'])
|
|
if sp:
|
|
t = sp.get_text(strip=True).replace(',', '')
|
|
if re.fullmatch(r'\d{1,7}', t):
|
|
nums.append(int(t))
|
|
return max(nums) if nums else 0
|
|
|
|
|
|
def from_paging(body):
|
|
pages = []
|
|
for a in body.select('.pagination a, .paging a, .page_nav a, .paginate a, nav a'):
|
|
h = a.get('href', '') or ''
|
|
m = PAGE_PARAM.search(h)
|
|
if m:
|
|
pages.append(int(m.group(1)))
|
|
t = a.get_text(strip=True)
|
|
if t.isdigit():
|
|
pages.append(int(t))
|
|
if not pages:
|
|
return 0
|
|
last = max(pages)
|
|
# 페이지당 행수 추정
|
|
rows = len(body.select('table tbody tr')) or len(body.select('.board_list li, .bbs_list li'))
|
|
if rows and last:
|
|
return last * rows # 근사(상한)
|
|
return 0
|
|
|
|
|
|
def board_count(body):
|
|
txt = body.get_text(' ', strip=True)
|
|
c = from_total(txt)
|
|
if c > 0:
|
|
return c, '총건수'
|
|
c = from_rownums(body)
|
|
if c > 0:
|
|
return c, '글번호max'
|
|
c = from_paging(body)
|
|
if c > 0:
|
|
return c, '페이징근사'
|
|
return 0, '없음'
|
|
|
|
|
|
def run(name, sess):
|
|
xlsx = os.path.join(OUTDIR, f'{name}.xlsx')
|
|
wb = openpyxl.load_workbook(xlsx)
|
|
ws = wb.active
|
|
fixed = empty = 0
|
|
for r in range(3, ws.max_row + 1):
|
|
if ws.cell(r, 2).value is None:
|
|
break
|
|
L = ws.cell(r, 12).value
|
|
M = ws.cell(r, 13).value
|
|
if L != '게시판' or (M not in (0, None, '')):
|
|
continue
|
|
url = ws.cell(r, 11).value
|
|
if not url or not isinstance(url, str) or not url.startswith('http'):
|
|
continue
|
|
soup, _ = P.fetch(sess, url)
|
|
if not soup:
|
|
continue
|
|
body = P.get_body(soup, P.BODY_SEL)
|
|
c, how = board_count(body)
|
|
if c > 0:
|
|
ws.cell(r, 13).value = c; fixed += 1
|
|
else:
|
|
# 빈 게시판 → N=없음, O=미부착 (글없으면 저작물없음)
|
|
if ws.cell(r, 14).value:
|
|
ws.cell(r, 14).value = '없음'
|
|
empty += 1
|
|
wb.save(xlsx)
|
|
print(f'[{name}] M보정 {fixed}행 | 빈게시판처리 {empty}행')
|
|
return fixed, empty
|
|
|
|
|
|
def main():
|
|
probe = {r['name']: r for r in json.load(open(r'D:\01.프로젝트\DB수집\_스크립트\_공공기관_probe.json', encoding='utf-8'))}
|
|
order = sorted(probe.values(), key=lambda x: -int(x['num']))
|
|
only = sys.argv[1:]
|
|
if only:
|
|
order = [p for p in order if p['name'] in only or str(p['num']) in only]
|
|
sess = P.make_session()
|
|
tf = te = 0
|
|
for p in order:
|
|
try:
|
|
f, e = run(p['name'], sess); tf += f; te += e
|
|
except Exception as ex:
|
|
print(f"[{p['name']}] 실패: {ex}")
|
|
print(f'=== M보정 합계 {tf}행 | 빈게시판 {te}행 ===')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|