df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
115 lines
4.5 KiB
Python
115 lines
4.5 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""공공기관 접근실패(S='접근 실패') 행 재시도. 복구되면 L/M/N/O/P/Q 채우고 S 해제.
|
|
영상은 navfix 기준(채널링크 제외) 적용. 사용: python _공공기관_retry.py [기관명 ...]
|
|
"""
|
|
import sys, os, re, json, time, importlib.util, warnings
|
|
import openpyxl
|
|
warnings.filterwarnings('ignore')
|
|
spec = importlib.util.spec_from_file_location('p234', r'D:\01.프로젝트\DB수집\_스크립트\_공공기관_phase234.py')
|
|
P = importlib.util.module_from_spec(spec); spec.loader.exec_module(P)
|
|
specn = importlib.util.spec_from_file_location('nv', r'D:\01.프로젝트\DB수집\_스크립트\_공공기관_navfix.py')
|
|
NV = importlib.util.module_from_spec(specn); specn.loader.exec_module(NV)
|
|
OUTDIR = r'D:\01.프로젝트\DB수집\공공기관'
|
|
LOG = open(r'D:\01.프로젝트\DB수집\_스크립트\_retry_result.txt', 'w', encoding='utf-8')
|
|
|
|
|
|
def say(s):
|
|
LOG.write(s + '\n'); LOG.flush()
|
|
|
|
|
|
def fetch_retry(sess, url, tries=3):
|
|
for i in range(tries):
|
|
try:
|
|
r = sess.get(url, timeout=25, verify=False, allow_redirects=True)
|
|
meta = re.search(rb'charset=["\']?\s*([\w-]+)', r.content[:4096], re.I)
|
|
r.encoding = meta.group(1).decode(errors='ignore') if meta else r.apparent_encoding
|
|
if r.status_code == 200:
|
|
from bs4 import BeautifulSoup
|
|
return BeautifulSoup(r.text, 'html.parser'), r.url, r.status_code
|
|
last = r.status_code
|
|
except Exception as e:
|
|
last = type(e).__name__
|
|
time.sleep(1.2)
|
|
return None, None, last
|
|
|
|
|
|
def run(name, sess):
|
|
xlsx = os.path.join(OUTDIR, f'{name}.xlsx')
|
|
wb = openpyxl.load_workbook(xlsx)
|
|
ws = wb.active
|
|
targets = []
|
|
for r in range(3, ws.max_row + 1):
|
|
if ws.cell(r, 2).value is None:
|
|
break
|
|
if '접근 실패' in (ws.cell(r, 19).value or ''):
|
|
targets.append((r, ws.cell(r, 11).value))
|
|
if not targets:
|
|
return 0, 0
|
|
rec = dead = 0
|
|
for r, url in targets:
|
|
if not url or not isinstance(url, str) or not url.startswith('http'):
|
|
continue
|
|
soup, final, code = fetch_retry(sess, url)
|
|
if not soup:
|
|
dead += 1
|
|
continue
|
|
body = P.get_body(soup, P.BODY_SEL)
|
|
form, count = P.detect_form(body)
|
|
ws.cell(r, 12).value = form
|
|
ws.cell(r, 13).value = (count if form == '게시판' else 1)
|
|
has_img, _, _, has_txt = P.detect_media(body)
|
|
vid = NV.real_video(body); aud = NV.real_audio(body)
|
|
types, q = P.detect_kogl(body)
|
|
if form == '게시판':
|
|
for du in P.extract_detail_urls(body, final or url, limit=4):
|
|
ds, _ = P.fetch(sess, du)
|
|
if not ds:
|
|
continue
|
|
db = P.get_body(ds, P.BODY_SEL)
|
|
di, _, _, _ = P.detect_media(db)
|
|
has_img |= di; vid = vid or NV.real_video(db); aud = aud or NV.real_audio(db)
|
|
dt, dq = P.detect_kogl(db)
|
|
types |= dt
|
|
parts = ['어문'] if has_txt else []
|
|
if has_img:
|
|
parts.append('이미지')
|
|
if vid:
|
|
parts.append('영상')
|
|
if aud:
|
|
parts.append('오디오')
|
|
ws.cell(r, 14).value = ','.join(parts) if parts else '없음'
|
|
if types and types != {1, 2, 3, 4}:
|
|
ws.cell(r, 15).value = ','.join(f'{n}유형' for n in sorted(types))
|
|
ws.cell(r, 16).value = '게시판' if form == '게시판' else '게시물'
|
|
ws.cell(r, 17).value = 'Y' if q == 'Y' else 'N'
|
|
else:
|
|
ws.cell(r, 15).value = '미부착'
|
|
ws.cell(r, 19).value = None # 접근실패 해제
|
|
rec += 1
|
|
wb.save(xlsx)
|
|
say(f'[{name}] 재시도 {len(targets)}행 → 복구 {rec}·여전히실패 {dead}')
|
|
return rec, dead
|
|
|
|
|
|
def main():
|
|
probe = {r['name']: r for r in json.load(open(r'D:\01.프로젝트\DB수집\_스크립트\_공공기관_probe.json', encoding='utf-8'))}
|
|
order = sorted(probe.values(), key=lambda x: -int(x['num']))
|
|
only = sys.argv[1:]
|
|
if only:
|
|
order = [p for p in order if p['name'] in only or str(p['num']) in only]
|
|
sess = P.make_session()
|
|
tr = td = 0
|
|
for p in order:
|
|
try:
|
|
res = run(p['name'], sess)
|
|
if res:
|
|
tr += res[0]; td += res[1]
|
|
except Exception as e:
|
|
say(f"[{p['name']}] 실패: {e}")
|
|
say(f'=== 재시도 합계 복구 {tr}행·여전히실패 {td}행 ===')
|
|
LOG.close()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|