import requests, io, re, openpyxl, json
from bs4 import BeautifulSoup
from urllib.parse import urljoin
hd={"User-Agent":"Mozilla/5.0"}
BASE="https://www.kofpi.or.kr"
out=io.open("_classify_out.txt","w",encoding="utf-8")

def norm(u):
    if not u: return ""
    u=str(u).strip()
    if u.startswith("/"): u=BASE+u
    return u

# existing sheet urls
wb=openpyxl.load_workbook("한국임업진흥원.xlsx")
ws=wb.active
sheet_urls={}
for r in range(3,ws.max_row+1):
    k=ws.cell(r,11).value
    if k: sheet_urls.setdefault(norm(k), []).append(r)

# tab groups (anchor_row, list of (label,href)) — from sweep, excluding pure '#' anchor boards
GROUPS={
 7:[("감사·윤리","/public/publicInfo_03.do"),("계약·회계","/public/publicInfo_03_02.do"),("기획·재정·홍보","/public/publicInfo_03_03.do"),("인사·총무·노무·법인","/public/publicInfo_03_04.do"),("임업관련","/public/publicInfo_03_05.do")],
 10:[("단기임산물 재배적지도","/public/dataOpen_04.do"),("임산물DB백과 API","/public/dataOpen_02.do"),("목재제품 DB백과 API","/public/dataOpen_03.do")],
 15:[("산림 관련 직종","/info/reForestInfo/job_01.do"),("산림분야 자격과정","/info/reForestInfo/job_02.do"),("전문임업인","/info/reForestInfo/job_03.do")],
 16:[("산촌 곳곳의 즐길거리","/info/reForestInfo/exper_01.do"),("산촌 인사이트","/info/reForestInfo/exper_02.do")],
 17:[("귀산촌인 창업 및 주택구입 자금","/info/rFarm_02_01.do"),("전문임업인 기반조성","/info/rFarm_02_02.do"),("산림소득분야 보조사업","/info/rFarm_02_03.do"),("임산물 소득원의 지원 대상 품목","/info/rFarm_02.do")],
 18:[("임산물정보","/info/imupStory/forestStory_01.do"),("산나물","/info/imupStory/forestStory_04_01.do"),("산약초","/info/imupStory/forestStory_05_01.do"),("임산물레시피","/info/imupStory/forestStory_02.do")],
 20:[("우리목재","/info/imupStory/wood_02.do"),("세계목재 300종","/info/imupStory/wood_03.do")],
 25:[("임가경제조사","/info/imupStory/statistics_01_01.do"),("임산물생산비조사","/info/imupStory/statistics_01_01_02.do"),("임산물소득조사","/info/imupStory/statistics_01_01_03.do"),("임산물생산조사","/info/imupStory/statistics_01_01_04.do"),("임업경영실태조사","/info/imupStory/statistics_01_01_05.do"),("산림산업조사","/info/imupStory/statistics_01_01_06.do")],
 26:[("목재이용실태","/info/imupStory/statistics_03.do"),("원목시장가격","/info/imupStory/statistics_04.do")],
 27:[("수출/수입절차","/info/exportInfo/exportGuide_01.do"),("무역서식 및 증명서 발급","/info/exportInfo/exportGuide_02.do"),("카드뉴스","/info/exportInfo/exportGuide_05.do")],
 28:[("공모중인 사업","/info/exportInfo/govSupport_02.do"),("임산물 수출지원 체계도","/info/exportInfo/govSupport_01.do"),("기관별 지원사업","/info/exportInfo/govSupport_03_01.do"),("내게 맞는 지원사업 찾기","/info/exportInfo/govSupport_04_01_01.do")],
 29:[("통합 자료실","/info/exportInfo/exportData_01.do"),("해외시장 보고서","/info/exportInfo/exportData_02.do"),("연구 보고서","/info/exportInfo/exportData_03.do"),("통계자료","/info/exportInfo/exportData_04.do"),("임산물수출유망 발굴업체홍보","/info/exportInfo/exportData_05.do"),("임업기계장비","/info/exportInfo/exportData_06.do")],
 30:[("유관기관","/info/exportInfo/exportSite_01.do"),("수출업체","/info/exportInfo/exportSite_02_01.do"),("관련사이트","/info/exportInfo/exportSite_02_02_01.do"),("국제박람회","/info/exportInfo/exportSite_03_01.do")],
 31:[("기술이전이란?","/info/tech/techComm_01.do"),("업무처리절차","/info/tech/techComm_02_01.do"),("기술이전현황","/info/tech/techComm_01_02.do"),("사업화 성공 사례","/info/tech/techComm_01_03.do"),("보유특허정보","/info/tech/techComm_01_01.do"),("기술자료집","/info/tech_06.do"),("특허·논문검색","/info/tech_05_01.do"),("관련양식","/info/tech/techComm_03_01.do")],
 53:[("시험·분석·조사·감정","/service/introduceInfo_01.do"),("목재제품규격ㆍ품질검사","/service/introduceInfo_04.do"),("KS인증","/service/introduceInfo_11.do"),("NET(신기술 지정)","/service/introduceInfo_15.do"),("임업기계 장비인증","/service/introduceInfo_34.do"),("목재제품 탄소저장량 표시제도","/service/introduceInfo_41.do"),("조달물자 전문기관검사","/service/introduceInfo_48.do")],
 55:[("특별관리임산물(산양삼) 분석","/service/introduceInfo_20.do"),("토양/식물체분석","/service/introduceInfo_23.do"),("산림토양조사","/service/introduceInfo_26.do")],
 70:[("숲푸드 소개","/service/ogFood_01.do"),("숲푸드 신청 절차/BI","/service/ogFood_02.do")],
 71:[("숲푸드 더자연산 소개","/service/kfFood_01.do"),("숲푸드 더자연산 지정절차","/service/kfFood_02.do")],
 89:[("윤리서약","/intro/intro_behavior_01_2.do"),("윤리헌장","/intro/intro_behavior_01.do"),("윤리강령","/intro/intro_behavior_01_1.do")],
 90:[("청탁금지법 준수 서약서","/intro/intro_13.do"),("청렴 서약서","/intro/intro_19.do")],
 93:[("이사장(원장)","/intro/intro_behavior_02.do"),("상임이사","/intro/intro_behavior_02_1.do"),("상임이사2","/intro/intro_behavior_02_2.do")],
 96:[("공정거래 자율준수 프로그램 소개","/intro/cp_01.do"),("기관장 의지 선언문","/intro/cp_02.do")],
}

# unique urls to classify
cache={}
def classify(u):
    if u in cache: return cache[u]
    try:
        r=requests.get(u,headers=hd,timeout=25)
        r.encoding=r.apparent_encoding
        s=BeautifulSoup(r.content,"html.parser")
        final=str(r.url)
        # board detection: list with date/number, pagination, total
        txt=s.get_text(" ",strip=True)
        tot=None
        for pat in [r"총\s*([\d,]+)\s*건",r"총\s*게시물\s*[:：]?\s*([\d,]+)",r"전체\s*([\d,]+)\s*건",r"Total\s*[:：]?\s*([\d,]+)"]:
            m=re.search(pat,txt)
            if m: tot=int(m.group(1).replace(",","")); break
        paging=bool(s.select(".paging a, .pagination a, [class*=paging] a, .board_paging a"))
        # board list table with rows that have number+title+date
        listrows=0
        for tr in s.select("table tbody tr"):
            tds=tr.find_all("td")
            if len(tds)>=3: listrows+=1
        is_board = (tot is not None) or paging or (listrows>=5 and bool(s.select("table")))
        # content images (>=140x100-ish heuristic by attrs not reliable; just count non-ui imgs in content)
        cont=s.select_one("#contents, #contens, #txt, #content, .sub_content, .contents")
        imgs=[]
        scope=cont if cont else s
        for img in scope.find_all("img"):
            src=(img.get("src") or "")
            if re.search(r"logo|footer|header|banner|btn|icon|bg|sns|family|arrow|blank|no_img|sprite|marker|common",src,re.I): continue
            alt=(img.get("alt") or "")
            imgs.append((src[-40:],alt[:20]))
        kogl=set()
        for img in s.find_all("img"):
            m=re.search(r"open(?:type|code)0*([1-4])\.jpg",(img.get('src') or ''))
            if m: kogl.add(int(m.group(1)))
        cache[u]={"final":final,"tot":tot,"paging":paging,"listrows":listrows,"board":is_board,"nimg":len(imgs),"imgs":imgs[:3],"kogl":sorted(kogl)}
    except Exception as e:
        cache[u]={"error":str(e)}
    return cache[u]

for ar,tabs in GROUPS.items():
    # anchor row leaf label
    leaf=None
    for c in range(10,3,-1):
        v=ws.cell(ar,c).value
        if v and str(v).strip(): leaf=str(v).strip(); break
    out.write(f"\n##### GROUP anchor r{ar} leaf='{leaf}'\n")
    for lbl,href in tabs:
        u=norm(href)
        info=classify(u)
        exrows=sheet_urls.get(u,[])
        out.write(f"  '{lbl}' {href}\n")
        out.write(f"      board={info.get('board')} tot={info.get('tot')} paging={info.get('paging')} listrows={info.get('listrows')} nimg={info.get('nimg')} kogl={info.get('kogl')} final={info.get('final','')[-50:]}\n")
        out.write(f"      in_sheet_rows={exrows}  imgs={info.get('imgs')}\n")
out.close()
print("done", len(cache), "urls")
