77 lines
2.9 KiB
Python
77 lines
2.9 KiB
Python
import requests, io, re, openpyxl
|
||
from bs4 import BeautifulSoup
|
||
from urllib.parse import urljoin
|
||
hd={"User-Agent":"Mozilla/5.0"}
|
||
out=io.open("_scan_out.txt","w",encoding="utf-8")
|
||
BASE="https://www.kofpi.or.kr"
|
||
|
||
def fetch(u):
|
||
r=requests.get(u,headers=hd,timeout=25)
|
||
r.encoding=r.apparent_encoding
|
||
return r, BeautifulSoup(r.content,"html.parser")
|
||
|
||
# ---- classify the 5 tabs ----
|
||
tabs=[("감사·윤리","/public/publicInfo_03.do"),
|
||
("계약·회계","/public/publicInfo_03_02.do"),
|
||
("기획·재정·홍보","/public/publicInfo_03_03.do"),
|
||
("인사·총무·노무·법인","/public/publicInfo_03_04.do"),
|
||
("임업관련","/public/publicInfo_03_05.do")]
|
||
out.write("===== TAB CLASSIFICATION =====\n")
|
||
for lbl,path in tabs:
|
||
u=BASE+path
|
||
try:
|
||
r,s=fetch(u)
|
||
txt=s.get_text(" ",strip=True)
|
||
# total count patterns
|
||
tot=None
|
||
for pat in [r"총\s*([\d,]+)\s*건", r"총\s*게시물\s*[:：]?\s*([\d,]+)", r"전체\s*[:：]?\s*([\d,]+)\s*건"]:
|
||
m=re.search(pat,txt)
|
||
if m: tot=m.group(1); break
|
||
# table rows
|
||
rows=s.select("table tbody tr")
|
||
nrows=len([x for x in rows if x.find("td")])
|
||
# pagination
|
||
pg=bool(s.select(".paging, .pagination, .page, [class*=paging]"))
|
||
# number column max
|
||
nums=[]
|
||
for td in s.select("table tbody tr td"):
|
||
t=td.get_text(strip=True)
|
||
if t.isdigit(): nums.append(int(t))
|
||
maxnum=max(nums) if nums else None
|
||
# KOGL marks
|
||
kogl=set()
|
||
for img in s.find_all("img"):
|
||
src=(img.get("src") or "")
|
||
m=re.search(r"open(?:type|code)0*([1-4])\.jpg",src)
|
||
if m: kogl.add(int(m.group(1)))
|
||
out.write(f"\n[{lbl}] {u}\n total={tot} tbody_rows={nrows} maxnum={maxnum} paging={pg} kogl={sorted(kogl)}\n")
|
||
# sample list item titles
|
||
tits=[a.get_text(strip=True)[:30] for a in s.select("table tbody tr td a")][:4]
|
||
out.write(f" sample_titles={tits}\n")
|
||
except Exception as e:
|
||
out.write(f"\n[{lbl}] {u} ERROR {e}\n")
|
||
|
||
# ---- scan ALL rows 7..end K-urls for div.tab01 widget ----
|
||
out.write("\n\n===== TAB01 SWEEP rows 7..end =====\n")
|
||
wb=openpyxl.load_workbook("한국임업진흥원.xlsx")
|
||
ws=wb.active
|
||
seen=set()
|
||
for r in range(7,ws.max_row+1):
|
||
k=ws.cell(r,11).value
|
||
if not k or not str(k).startswith("http") or "kofpi.or.kr" not in str(k): continue
|
||
k=str(k).strip()
|
||
try:
|
||
rr,s=fetch(k)
|
||
td=s.select_one("div.tab01")
|
||
if td:
|
||
links=[(a.get_text(' ',strip=True), a.get('href')) for a in td.select("li a")]
|
||
key=tuple(sorted(h for _,h in links))
|
||
mark="DUP" if key in seen else "NEW"
|
||
seen.add(key)
|
||
out.write(f"r{r} {k}\n [{mark}] tab01 class={td.get('class')} tabs={links}\n")
|
||
except Exception as e:
|
||
out.write(f"r{r} {k} ERR {e}\n")
|
||
out.write("\nDONE\n")
|
||
out.close()
|
||
print("done")
|