# -*- coding: utf-8 -*-
"""정읍 L(게시판/페이지) 판별 신호 캘리브레이션.
3~282(사용자검수=정답)의 내부 index.jeongeup 페이지/게시판을 크롤해 신호 수집 -> _ldetect.json
"""
import re, json, os, time, urllib.request
from collections import Counter
from concurrent.futures import ThreadPoolExecutor
import openpyxl

DIR = os.path.dirname(os.path.abspath(__file__))
HDR = {'User-Agent': 'Mozilla/5.0'}

def get(u):
    for _ in range(2):
        try:
            return urllib.request.urlopen(urllib.request.Request(u.replace(' ', '%20'), headers=HDR), timeout=25).read().decode('utf-8', 'replace')
        except Exception:
            time.sleep(0.4)
    return ''

def signals(h):
    s = {}
    s['nview'] = len(re.findall(r'view\.jeongeup\?[^"\']*boardId=BBS_\d+', h))
    s['nlist'] = len(re.findall(r'list\.jeongeup\?[^"\']*boardId=BBS_\d+', h))
    s['ndown'] = len(re.findall(r'download\.jeongeup', h))
    s['nbbs']  = len(set(re.findall(r'boardId=(BBS_\d+)', h)))
    s['total'] = bool(re.search(r'총\s*<?[^>]{0,15}?[\d,]+\s*</?[^>]{0,6}?건', h)) or ('게시물' in h)
    s['paging'] = bool(re.search(r'class="[^"]*(?:paging|pagination|board_page)', h)) or bool(re.search(r'startPage=\d', h))
    # 게시판 리스트 테이블/구조 클래스
    s['bbslist'] = bool(re.search(r'class="[^"]*(?:bbs|board_?list|p-table|tbl_list|basic_list|board_basic|gallery_list|news_list|photo_list)', h, re.I))
    s['writeb'] = bool(re.search(r'write\.jeongeup', h))
    s['skin'] = ('class="bbs_skin"' in h)                       # AJAX 게시판 스켈레톤(=진짜 게시판)
    hb = re.findall(r'<input[^>]*name="boardId"[^>]*value="(BBS_\d+)"', h)
    vb = Counter(re.findall(r'view\.jeongeup\?[^"\']*boardId=(BBS_\d+)', h))
    s['hidbid'] = hb[0] if hb else ''
    s['vbid'] = vb.most_common(1)[0][0] if vb else ''
    s['len'] = len(h)
    return s

t = [p for p in __import__('glob').glob(os.path.join(DIR, '*.xlsx'))
     if 'backup' not in p and not os.path.basename(p).startswith(('~$', '_'))][0]
ws = openpyxl.load_workbook(t).active

jobs = []
for r in range(3, 283):
    L = ws.cell(r, 12).value
    k = ws.cell(r, 11).value
    if not k:
        continue
    k = str(k)
    if L in ('페이지', '게시판') and k.startswith('http') and 'index.jeongeup' in k:
        jobs.append((r, L, k))

pool = ThreadPoolExecutor(max_workers=8)
def work(job):
    r, L, k = job
    return {'row': r, 'L': L, 'k': k, **signals(get(k))}

out = list(pool.map(work, jobs))
json.dump(out, open(os.path.join(DIR, '_ldetect.json'), 'w', encoding='utf-8'), ensure_ascii=False, indent=0)
print('crawled', len(out))