# -*- coding: utf-8 -*-
"""정읍 r283~끝 L 재분류(게시판↔페이지) + 게시판 M·O 재작성. dry-run -> _l283.json.
게시판 감지: bbs_skin 스켈레톤 OR list.jeongeup링크 OR view링크>=3.
boardId/contentsSid를 hidden input/링크에서 추출 -> /board/list.jeongeup 페치 ->
  실제 글표(tbody tr) 있으면 게시판, M=총N건 or 첫칸번호 최댓값 or 행수.
O: 상세글(view.jeongeup) 표본 img_opentype0N -> 유형(P=게시물); 상세없으면 list본문 footer(P=게시판).
"""
import re, json, os, glob, time, urllib.request
from collections import Counter
from concurrent.futures import ThreadPoolExecutor
import openpyxl

DIR = os.path.dirname(os.path.abspath(__file__))
DOMAIN = 'https://www.jeongeup.go.kr'
HDR = {'User-Agent': 'Mozilla/5.0'}
OPT = re.compile(r'img_open(?:type|code)(\d{1,2})\.(?:png|jpe?g|gif)', re.I)
START = 283
SAMPLE = 12
pool = ThreadPoolExecutor(max_workers=6)

def get(u):
    for _ in range(3):
        try:
            return urllib.request.urlopen(urllib.request.Request(u.replace(' ', '%20'), headers=HDR), timeout=25).read().decode('utf-8', 'replace')
        except Exception:
            time.sleep(0.5)
    return ''

def types_in(h):
    ts = set(int(x) for x in OPT.findall(h) if x.isdigit() and 1 <= int(x) <= 4)
    return set() if ts == {1, 2, 3, 4} else ts

def parse_list(h):
    """list.jeongeup HTML -> (rowcount, Mnum, view_links)."""
    views = [v for v in re.findall(r'view\.jeongeup\?([^"\']+)', h) if 'boardId=' in v]
    # tbody data rows: first <td> numeric
    nums = []
    for tr in re.findall(r'<tr[^>]*>(.*?)</tr>', h, re.S):
        td = re.search(r'<td[^>]*>\s*([\d,]+)\s*</td>', tr)
        if td:
            nums.append(int(td.group(1).replace(',', '')))
    tot = re.findall(r'총\s*<?[^>]{0,12}?([\d,]+)\s*<?[^>]{0,8}?\s*건', h)
    M = None
    if tot:
        M = int(tot[0].replace(',', ''))
    elif nums:
        M = max(nums)
    return len(nums), M, views

def board_info(idx_h, menuCd):
    """index page에서 boardId/contentsSid 추출 -> list 페치 -> 정보."""
    bid = (re.findall(r'<input[^>]*name="boardId"[^>]*value="(BBS_\d+)"', idx_h)
           or re.findall(r'list\.jeongeup\?[^"\']*boardId=(BBS_\d+)', idx_h)
           or re.findall(r'view\.jeongeup\?[^"\']*boardId=(BBS_\d+)', idx_h))
    if not bid:
        # index page 자체가 정적 리스트일 수도
        return parse_list_self(idx_h)
    bid = Counter(bid).most_common(1)[0][0]
    csid = re.findall(r'<input[^>]*name="contentsSid"[^>]*value="(\d+)"', idx_h)
    csid = csid[0] if csid else ''
    lu = '%s/board/list.jeongeup?menuCd=%s&boardId=%s%s&startPage=1' % (
        DOMAIN, menuCd, bid, ('&contentsSid=' + csid if csid else ''))
    lh = get(lu)
    nrow, M, views = parse_list(lh)
    return {'bid': bid, 'listlen': len(lh), 'nrow': nrow, 'M': M, 'views': views, 'list_h': lh}

def parse_list_self(h):
    nrow, M, views = parse_list(h)
    return {'bid': '', 'listlen': len(h), 'nrow': nrow, 'M': M, 'views': views, 'list_h': h}

def board_O(info):
    views = info.get('views') or []
    seen, vs = set(), []
    bidf = info.get('bid', '')
    for v in views:
        if bidf and 'boardId=' + bidf not in v:
            continue
        key = re.search(r'dataSid=(\d+)', v)
        key = key.group(1) if key else v
        if key in seen:
            continue
        seen.add(key)
        vs.append(v.replace('&amp;', '&'))
        if len(vs) >= SAMPLE:
            break
    if vs:
        ts = set()
        for r in pool.map(lambda v: types_in(get(DOMAIN + '/board/view.jeongeup?' + v)), vs):
            ts |= r
        if ts:
            return ','.join('%d유형' % n for n in sorted(ts)), '게시물', 'Y'
    # 상세 없거나 마크없음 -> list 본문 footer
    lt = types_in(info.get('list_h', ''))
    if lt:
        return ','.join('%d유형' % n for n in sorted(lt)), '게시판', 'Y'
    return '미부착', '', ''

t = [p for p in glob.glob(os.path.join(DIR, '*.xlsx'))
     if 'backup' not in p and not os.path.basename(p).startswith(('~$', '_'))][0]
ws = openpyxl.load_workbook(t).active

out = []
for r in range(START, ws.max_row + 1):
    L = ws.cell(r, 12).value
    k = ws.cell(r, 11).value
    if not k:
        continue
    k = str(k)
    if L not in ('페이지', '게시판'):
        continue
    if not k.startswith('http') or 'index.jeongeup' not in k:
        out.append({'row': r, 'curL': L, 'verdict': 'skip(비표준URL)', 'k': k})
        continue
    mc = re.search(r'menuCd=([A-Z0-9_]+)', k)
    mc = mc.group(1) if mc else ''
    h = get(k)
    skin = ('class="bbs_skin"' in h)
    nlist = len(re.findall(r'list\.jeongeup\?[^"\']*boardId=BBS_\d+', h))
    nview = len(re.findall(r'view\.jeongeup\?[^"\']*boardId=BBS_\d+', h))
    pred = skin or nlist >= 1 or nview >= 3
    rec = {'row': r, 'curL': L, 'name': str(ws.cell(r, 5).value or ws.cell(r, 6).value or ws.cell(r, 7).value or ws.cell(r, 4).value),
           'skin': skin, 'nlist': nlist, 'nview': nview, 'curM': ws.cell(r, 13).value, 'curO': ws.cell(r, 15).value}
    if not pred:
        rec['verdict'] = 'page(신호無)'
        out.append(rec)
        print('r%d [%s] page (skin0 list0 view%d)' % (r, L[:2], nview), flush=True)
        continue
    info = board_info(h, mc)
    is_board = (info['nrow'] >= 1) or (info['listlen'] > 5000 and (info['M'] or 0) >= 0 and info['bid'])
    rec.update({'bid': info['bid'], 'listlen': info['listlen'], 'nrow': info['nrow'], 'M_new': info['M']})
    if not is_board:
        rec['verdict'] = 'page(빈board/위젯 listlen=%d nrow=%d)' % (info['listlen'], info['nrow'])
        out.append(rec)
        print('r%d [%s] page? listlen=%d nrow=%d bid=%s' % (r, L[:2], info['listlen'], info['nrow'], info['bid']), flush=True)
        continue
    O, P, Q = board_O(info)
    rec.update({'verdict': '게시판', 'O_new': O, 'P_new': P, 'Q_new': Q})
    flag = ' <<RECLASS' if L != '게시판' else ''
    print('r%d [%s] 게시판 M=%s O=%s nrow=%d%s' % (r, L[:2], info['M'], O, info['nrow'], flag), flush=True)
    out.append(rec)
    json.dump(out, open(os.path.join(DIR, '_l283.json'), 'w', encoding='utf-8'), ensure_ascii=False, indent=0)

json.dump(out, open(os.path.join(DIR, '_l283.json'), 'w', encoding='utf-8'), ensure_ascii=False, indent=0)
recl = [x for x in out if x.get('verdict') == '게시판' and x['curL'] != '게시판']
print('\nSCAN=%d  게시판판정=%d  신규reclass(페이지→게시판)=%d rows=%s'
      % (len(out), sum(1 for x in out if x.get('verdict') == '게시판'), len(recl), [x['row'] for x in recl]))