import requests, io, re, openpyxl
from bs4 import BeautifulSoup
from urllib.parse import urljoin
hd={"User-Agent":"Mozilla/5.0"}
out=io.open("_scan_out.txt","w",encoding="utf-8")
BASE="https://www.kofpi.or.kr"

def fetch(u):
    r=requests.get(u,headers=hd,timeout=25)
    r.encoding=r.apparent_encoding
    return r, BeautifulSoup(r.content,"html.parser")

# ---- classify the 5 tabs ----
tabs=[("감사·윤리","/public/publicInfo_03.do"),
      ("계약·회계","/public/publicInfo_03_02.do"),
      ("기획·재정·홍보","/public/publicInfo_03_03.do"),
      ("인사·총무·노무·법인","/public/publicInfo_03_04.do"),
      ("임업관련","/public/publicInfo_03_05.do")]
out.write("===== TAB CLASSIFICATION =====\n")
for lbl,path in tabs:
    u=BASE+path
    try:
        r,s=fetch(u)
        txt=s.get_text(" ",strip=True)
        # total count patterns
        tot=None
        for pat in [r"총\s*([\d,]+)\s*건", r"총\s*게시물\s*[:：]?\s*([\d,]+)", r"전체\s*[:：]?\s*([\d,]+)\s*건"]:
            m=re.search(pat,txt)
            if m: tot=m.group(1); break
        # table rows
        rows=s.select("table tbody tr")
        nrows=len([x for x in rows if x.find("td")])
        # pagination
        pg=bool(s.select(".paging, .pagination, .page, [class*=paging]"))
        # number column max
        nums=[]
        for td in s.select("table tbody tr td"):
            t=td.get_text(strip=True)
            if t.isdigit(): nums.append(int(t))
        maxnum=max(nums) if nums else None
        # KOGL marks
        kogl=set()
        for img in s.find_all("img"):
            src=(img.get("src") or "")
            m=re.search(r"open(?:type|code)0*([1-4])\.jpg",src)
            if m: kogl.add(int(m.group(1)))
        out.write(f"\n[{lbl}] {u}\n   total={tot} tbody_rows={nrows} maxnum={maxnum} paging={pg} kogl={sorted(kogl)}\n")
        # sample list item titles
        tits=[a.get_text(strip=True)[:30] for a in s.select("table tbody tr td a")][:4]
        out.write(f"   sample_titles={tits}\n")
    except Exception as e:
        out.write(f"\n[{lbl}] {u} ERROR {e}\n")

# ---- scan ALL rows 7..end K-urls for div.tab01 widget ----
out.write("\n\n===== TAB01 SWEEP rows 7..end =====\n")
wb=openpyxl.load_workbook("한국임업진흥원.xlsx")
ws=wb.active
seen=set()
for r in range(7,ws.max_row+1):
    k=ws.cell(r,11).value
    if not k or not str(k).startswith("http") or "kofpi.or.kr" not in str(k): continue
    k=str(k).strip()
    try:
        rr,s=fetch(k)
        td=s.select_one("div.tab01")
        if td:
            links=[(a.get_text(' ',strip=True), a.get('href')) for a in td.select("li a")]
            key=tuple(sorted(h for _,h in links))
            mark="DUP" if key in seen else "NEW"
            seen.add(key)
            out.write(f"r{r} {k}\n   [{mark}] tab01 class={td.get('class')} tabs={links}\n")
    except Exception as e:
        out.write(f"r{r} {k} ERR {e}\n")
out.write("\nDONE\n")
out.close()
print("done")