DB_JOB/작업파일/완료/광역_사이트맵/충청남도/1.계룡시/_phase234.py

"""계룡시 Phase 2~4 자동 채우기 (L/M/N/O/P/Q).

매뉴얼: D:\\01.프로젝트\\DB수집\\사이트맵_수집_매뉴얼.md
"""
import requests, warnings, re, time, openpyxl
from bs4 import BeautifulSoup
from urllib.parse import urljoin
from concurrent.futures import ThreadPoolExecutor, as_completed

warnings.filterwarnings('ignore')

XLSX = r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청남도\1.계룡시\충청남도_계룡시.xlsx'
UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
H = {'User-Agent': UA}


def fetch(url, timeout=12):
    try:
        r = requests.get(url, headers=H, timeout=timeout, verify=False)
        r.encoding = r.apparent_encoding
        if r.status_code == 200:
            return BeautifulSoup(r.text, 'html.parser')
    except Exception:
        pass
    return None


def get_body(soup):
    return soup.find(id='txt') or soup.find(id='contents') or soup.find('main') or soup


TOTAL_PAT = re.compile(r'총\s*(?:게시물\s*)?([\d,]+)\s*건')


def detect_form(body):
    has_paging = bool(body.select('.pagination'))
    text_inputs = [i for i in body.find_all('input') if i.get('type') == 'text']
    has_search = len(text_inputs) >= 1
    txt = body.get_text(' ', strip=True)
    m = TOTAL_PAT.search(txt)
    total = int(m.group(1).replace(',', '')) if m else None
    is_board = has_paging or has_search or (total is not None)
    if is_board:
        return '게시판', total if total is not None else 0
    return '페이지', 1


def extract_detail_urls(body, base_url, limit=5):
    urls, seen = [], set()
    for a in body.find_all('a', href=True):
        h = a['href']
        if not h or h.startswith('#'):
            continue
        if 'mode=V' in h or 'view.do' in h.lower() or 'bbtSn=' in h:
            full = urljoin(base_url, h)
            if full not in seen:
                seen.add(full)
                urls.append(full)
                if len(urls) >= limit:
                    break
    return urls


KOGL_IMG_PAT = re.compile(r'(?:new_)?img_opentype(\d{2})\.png', re.I)
YOUTUBE_PAT = re.compile(r'(?:youtube\.com|youtu\.be)', re.I)
VIDEO_EXT = re.compile(r'\.(mp4|webm|mov|avi)(?:\?|$)', re.I)


def detect_media(body):
    has_text = len(body.get_text(strip=True)) > 30
    has_image = False
    for img in body.find_all('img'):
        src = img.get('src', '')
        if KOGL_IMG_PAT.search(src):
            continue
        if not src:
            continue
        has_image = True
        break
    has_video = False
    for iframe in body.find_all('iframe'):
        if YOUTUBE_PAT.search(iframe.get('src', '')):
            has_video = True
            break
    if not has_video:
        for a in body.find_all('a', href=True):
            if YOUTUBE_PAT.search(a['href']):
                has_video = True
                break
    if not has_video and body.find_all('video'):
        has_video = True
    if not has_video and VIDEO_EXT.search(str(body)):
        has_video = True
    return has_image, has_video, has_text


def n_string(has_text, has_image, has_video):
    parts = []
    if has_text:
        parts.append('어문')
    if has_image:
        parts.append('이미지')
    if has_video:
        parts.append('영상')
    return ','.join(parts) if parts else '없음'


KOGL_LINK_PAT = re.compile(r'kogl\.or\.kr/info/licenseType(\d)', re.I)


def img_has_valid_anchor(img):
    p = img.parent
    while p is not None:
        if p.name == 'a':
            href = p.get('href', '')
            if href and not href.startswith('#') and not href.lower().startswith('javascript:'):
                return True
            return False
        p = p.parent
    return False


def detect_kogl(body):
    types = set()
    q_any_y = False
    q_any_n = False
    for a in body.find_all('a', href=True):
        m = KOGL_LINK_PAT.search(a['href'])
        if m:
            types.add(int(m.group(1)))
            q_any_y = True
    for img in body.find_all('img'):
        src = img.get('src', '')
        m = KOGL_IMG_PAT.search(src)
        if m:
            types.add(int(m.group(1)))
            if img_has_valid_anchor(img):
                q_any_y = True
            else:
                q_any_n = True
    for el in body.find_all(style=True):
        m = KOGL_IMG_PAT.search(el.get('style', ''))
        if m:
            types.add(int(m.group(1)))
            q_any_n = True
    if not types:
        return set(), None
    return types, ('Y' if q_any_y else 'N')


def process_row(url):
    out = {'L': '', 'M': '', 'N': '', 'O': '', 'P': '', 'Q': '', 'note': ''}
    soup = fetch(url)
    if soup is None:
        out['note'] = '접근 실패'
        return out
    body = get_body(soup)

    form, count = detect_form(body)
    out['L'] = form
    out['M'] = count if form == '게시판' else 1

    has_img, has_vid, has_txt = detect_media(body)
    types_main, q_main = detect_kogl(body)
    P = '게시판' if types_main else ''
    types_all = set(types_main)
    q_flags = []
    if q_main:
        q_flags.append(q_main)

    if form == '게시판':
        detail_urls = extract_detail_urls(body, url, limit=5)
        for du in detail_urls:
            d_soup = fetch(du, timeout=10)
            if not d_soup:
                continue
            d_body = get_body(d_soup)
            di, dv, dt = detect_media(d_body)
            has_img = has_img or di
            has_vid = has_vid or dv
            has_txt = has_txt or dt
            dt_types, dt_q = detect_kogl(d_body)
            if dt_types and not types_main and not P:
                P = '게시물'
            types_all |= dt_types
            if dt_q:
                q_flags.append(dt_q)

    out['N'] = n_string(has_txt, has_img, has_vid)

    if not types_all:
        out['O'] = '미부착'
    else:
        sorted_types = sorted(types_all)
        out['O'] = ','.join(f'{n}유형' for n in sorted_types)
        out['P'] = P if P else '게시판'
        out['Q'] = 'Y' if 'Y' in q_flags else 'N'

    return out


def main():
    print('[1] 엑셀 로드')
    wb = openpyxl.load_workbook(XLSX)
    ws = wb.active
    START, END = 3, ws.max_row

    tasks = []
    for r in range(START, END + 1):
        url = ws.cell(r, 11).value
        is_ext = (ws.cell(r, 19).value == '외부링크')
        tasks.append((r, url, is_ext))
    print(f'[2] 처리 대상: 총 {len(tasks)}행 (외부링크 {sum(1 for t in tasks if t[2])}개)')

    print('[3] 크롤링 시작 (병렬 10 worker)...')
    t0 = time.time()
    results = {}

    def worker(task):
        row, url, is_ext = task
        if is_ext:
            return row, {'L': '사이트', 'M': '', 'N': '', 'O': '', 'P': '', 'Q': '', 'note': ''}
        if not url or not isinstance(url, str):
            return row, {'L': '', 'M': '', 'N': '', 'O': '', 'P': '', 'Q': '', 'note': 'URL 없음'}
        return row, process_row(url)

    done = 0
    with ThreadPoolExecutor(max_workers=10) as ex:
        futs = [ex.submit(worker, t) for t in tasks]
        for fut in as_completed(futs):
            row, res = fut.result()
            results[row] = res
            done += 1
            if done % 30 == 0:
                print(f'   진행 {done}/{len(tasks)} ({time.time()-t0:.0f}s)')
    print(f'[4] 크롤링 완료 ({time.time()-t0:.0f}s)')

    print('[5] 엑셀 기입')
    for r in range(START, END + 1):
        res = results.get(r, {})
        if not res:
            continue
        if res.get('L'):
            ws.cell(r, 12).value = res['L']
        if res.get('M') != '':
            ws.cell(r, 13).value = res['M']
        if res.get('N'):
            ws.cell(r, 14).value = res['N']
        if res.get('O'):
            ws.cell(r, 15).value = res['O']
        if res.get('P'):
            ws.cell(r, 16).value = res['P']
        if res.get('Q'):
            ws.cell(r, 17).value = res['Q']
        if res.get('note'):
            existing = ws.cell(r, 19).value
            if not existing:
                ws.cell(r, 19).value = res['note']

    wb.save(XLSX)
    print(f'[6] 저장 완료: {XLSX}')

    forms = {}
    attach = {'미부착': 0, '부착': 0, '기타': 0}
    q_dist = {'Y': 0, 'N': 0, '': 0}
    for r, res in results.items():
        forms[res.get('L', '')] = forms.get(res.get('L', ''), 0) + 1
        o = res.get('O', '')
        if o == '미부착':
            attach['미부착'] += 1
        elif o and '유형' in o:
            attach['부착'] += 1
        else:
            attach['기타'] += 1
        q = res.get('Q', '')
        q_dist[q] = q_dist.get(q, 0) + 1

    print('\n=== L 분포 ===')
    for k, v in sorted(forms.items(), key=lambda x: -x[1]):
        print(f'  {k!r}: {v}')
    print('\n=== 공공누리 부착 ===')
    for k, v in attach.items():
        print(f'  {k}: {v}')
    print('\n=== Q 분포 ===')
    for k, v in q_dist.items():
        print(f'  {k!r}: {v}')


if __name__ == '__main__':
    main()