df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
103 lines
4.5 KiB
Python
103 lines
4.5 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""건설근로자공제회 2단(중분류) 메뉴 링크 보강.
|
|
megamenu div.menu_inner[data-ui-content] > h3.subtit a.titlink(goMenu) = 그 2단의 대표 URL.
|
|
button[data-ui-tab=동일키] = 2단 라벨. 라벨로 URL없음 행(E) 매칭해 K 채우고 L/M/N/O 산출.
|
|
"""
|
|
import os, re, importlib.util, warnings
|
|
from copy import copy
|
|
from urllib.parse import urljoin
|
|
import openpyxl
|
|
from bs4 import BeautifulSoup
|
|
from openpyxl.styles import Font, Alignment
|
|
warnings.filterwarnings('ignore')
|
|
spec = importlib.util.spec_from_file_location('p1', r'D:\01.프로젝트\DB수집\_스크립트\_공공기관_phase1.py')
|
|
m1 = importlib.util.module_from_spec(spec); spec.loader.exec_module(m1)
|
|
spec2 = importlib.util.spec_from_file_location('p234', r'D:\01.프로젝트\DB수집\_스크립트\_공공기관_phase234.py')
|
|
P = importlib.util.module_from_spec(spec2); spec2.loader.exec_module(P)
|
|
specn = importlib.util.spec_from_file_location('nv', r'D:\01.프로젝트\DB수집\_스크립트\_공공기관_navfix.py')
|
|
NV = importlib.util.module_from_spec(specn); specn.loader.exec_module(NV)
|
|
XLSX = r'D:\01.프로젝트\DB수집\공공기관\건설근로자공제회.xlsx'
|
|
BASE = 'https://www.cw.or.kr'
|
|
LOG = open(r'D:\01.프로젝트\DB수집\_스크립트\_cwlink_result.txt', 'w', encoding='utf-8')
|
|
|
|
|
|
def clean(s):
|
|
return re.sub(r'\s+', ' ', (s or '')).strip()
|
|
|
|
|
|
def build_map():
|
|
html = m1.render(BASE + '/', wait=4500)[1]
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
# 2단 라벨: button[data-ui-tab]
|
|
tab_label = {}
|
|
for b in soup.find_all('button'):
|
|
k = b.get('data-ui-tab')
|
|
if k and clean(b.get('title') or b.get_text()):
|
|
tab_label[k] = clean(b.get('title') or b.get_text())
|
|
# 2단 대표URL: menu_inner[data-ui-content] > h3.subtit a.titlink
|
|
label2url = {}
|
|
for inner in soup.select('div.menu_inner[data-ui-content], [data-ui-content]'):
|
|
k = inner.get('data-ui-content')
|
|
a = inner.select_one('h3.subtit a.titlink') or inner.select_one('a.titlink') or inner.select_one('a.menu_link')
|
|
if not a:
|
|
continue
|
|
u = m1.js_href(a) or (a.get('href') or '')
|
|
if u.startswith('/') or u.startswith('http'):
|
|
label = tab_label.get(k) or clean(a.get('title') or a.get_text())
|
|
if label:
|
|
label2url[label] = urljoin(BASE + '/', u)
|
|
return label2url
|
|
|
|
|
|
def main():
|
|
label2url = build_map()
|
|
LOG.write(f'megamenu 2단 라벨→URL {len(label2url)}개\n')
|
|
wb = openpyxl.load_workbook(XLSX)
|
|
ws = wb.active
|
|
sess = P.make_session()
|
|
left = Alignment(horizontal='left', vertical='center', wrap_text=False)
|
|
filled = 0
|
|
for r in range(3, ws.max_row + 1):
|
|
if ws.cell(r, 2).value is None:
|
|
break
|
|
if ws.cell(r, 11).value:
|
|
continue
|
|
# 라벨 = E(중분류) 우선, 없으면 D
|
|
label = clean(ws.cell(r, 5).value) or clean(ws.cell(r, 4).value)
|
|
u = label2url.get(label)
|
|
if not u:
|
|
continue
|
|
c = ws.cell(r, 11); c.value = u; c.hyperlink = u
|
|
old = c.font
|
|
c.font = Font(name=old.name or '맑은 고딕', size=old.size or 11, color='0000FF', underline='single')
|
|
c.alignment = left
|
|
# L/M/N/O 산출
|
|
soup, final = P.fetch(sess, u)
|
|
if soup:
|
|
body = P.get_body(soup, P.BODY_SEL)
|
|
form, count = P.detect_form(body)
|
|
ws.cell(r, 12).value = form
|
|
ws.cell(r, 13).value = count if form == '게시판' else 1
|
|
hi, _, _, ht = P.detect_media(body)
|
|
vid = NV.real_video(body); aud = NV.real_audio(body)
|
|
types, q = P.detect_kogl(body)
|
|
parts = (['어문'] if ht else []) + (['이미지'] if hi else []) + (['영상'] if vid else []) + (['오디오'] if aud else [])
|
|
ws.cell(r, 14).value = ','.join(parts) if parts else '없음'
|
|
if types and types != {1, 2, 3, 4}:
|
|
ws.cell(r, 15).value = ','.join(f'{n}유형' for n in sorted(types))
|
|
ws.cell(r, 16).value = '게시판' if form == '게시판' else '게시물'
|
|
ws.cell(r, 17).value = 'Y' if q == 'Y' else 'N'
|
|
else:
|
|
ws.cell(r, 15).value = '미부착'
|
|
if ws.cell(r, 19).value in ('URL 없음', '접근 실패'):
|
|
ws.cell(r, 19).value = None
|
|
filled += 1
|
|
LOG.write(f' r{r} {label} → {u}\n')
|
|
wb.save(XLSX)
|
|
LOG.write(f'=== 링크보강 {filled}행 ===\n')
|
|
LOG.close()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|