104 lines
8.0 KiB
Python
104 lines
8.0 KiB
Python
import requests, io, re, openpyxl, json
|
||
from bs4 import BeautifulSoup
|
||
from urllib.parse import urljoin
|
||
hd={"User-Agent":"Mozilla/5.0"}
|
||
BASE="https://www.kofpi.or.kr"
|
||
out=io.open("_classify_out.txt","w",encoding="utf-8")
|
||
|
||
def norm(u):
|
||
if not u: return ""
|
||
u=str(u).strip()
|
||
if u.startswith("/"): u=BASE+u
|
||
return u
|
||
|
||
# existing sheet urls
|
||
wb=openpyxl.load_workbook("한국임업진흥원.xlsx")
|
||
ws=wb.active
|
||
sheet_urls={}
|
||
for r in range(3,ws.max_row+1):
|
||
k=ws.cell(r,11).value
|
||
if k: sheet_urls.setdefault(norm(k), []).append(r)
|
||
|
||
# tab groups (anchor_row, list of (label,href)) — from sweep, excluding pure '#' anchor boards
|
||
GROUPS={
|
||
7:[("감사·윤리","/public/publicInfo_03.do"),("계약·회계","/public/publicInfo_03_02.do"),("기획·재정·홍보","/public/publicInfo_03_03.do"),("인사·총무·노무·법인","/public/publicInfo_03_04.do"),("임업관련","/public/publicInfo_03_05.do")],
|
||
10:[("단기임산물 재배적지도","/public/dataOpen_04.do"),("임산물DB백과 API","/public/dataOpen_02.do"),("목재제품 DB백과 API","/public/dataOpen_03.do")],
|
||
15:[("산림 관련 직종","/info/reForestInfo/job_01.do"),("산림분야 자격과정","/info/reForestInfo/job_02.do"),("전문임업인","/info/reForestInfo/job_03.do")],
|
||
16:[("산촌 곳곳의 즐길거리","/info/reForestInfo/exper_01.do"),("산촌 인사이트","/info/reForestInfo/exper_02.do")],
|
||
17:[("귀산촌인 창업 및 주택구입 자금","/info/rFarm_02_01.do"),("전문임업인 기반조성","/info/rFarm_02_02.do"),("산림소득분야 보조사업","/info/rFarm_02_03.do"),("임산물 소득원의 지원 대상 품목","/info/rFarm_02.do")],
|
||
18:[("임산물정보","/info/imupStory/forestStory_01.do"),("산나물","/info/imupStory/forestStory_04_01.do"),("산약초","/info/imupStory/forestStory_05_01.do"),("임산물레시피","/info/imupStory/forestStory_02.do")],
|
||
20:[("우리목재","/info/imupStory/wood_02.do"),("세계목재 300종","/info/imupStory/wood_03.do")],
|
||
25:[("임가경제조사","/info/imupStory/statistics_01_01.do"),("임산물생산비조사","/info/imupStory/statistics_01_01_02.do"),("임산물소득조사","/info/imupStory/statistics_01_01_03.do"),("임산물생산조사","/info/imupStory/statistics_01_01_04.do"),("임업경영실태조사","/info/imupStory/statistics_01_01_05.do"),("산림산업조사","/info/imupStory/statistics_01_01_06.do")],
|
||
26:[("목재이용실태","/info/imupStory/statistics_03.do"),("원목시장가격","/info/imupStory/statistics_04.do")],
|
||
27:[("수출/수입절차","/info/exportInfo/exportGuide_01.do"),("무역서식 및 증명서 발급","/info/exportInfo/exportGuide_02.do"),("카드뉴스","/info/exportInfo/exportGuide_05.do")],
|
||
28:[("공모중인 사업","/info/exportInfo/govSupport_02.do"),("임산물 수출지원 체계도","/info/exportInfo/govSupport_01.do"),("기관별 지원사업","/info/exportInfo/govSupport_03_01.do"),("내게 맞는 지원사업 찾기","/info/exportInfo/govSupport_04_01_01.do")],
|
||
29:[("통합 자료실","/info/exportInfo/exportData_01.do"),("해외시장 보고서","/info/exportInfo/exportData_02.do"),("연구 보고서","/info/exportInfo/exportData_03.do"),("통계자료","/info/exportInfo/exportData_04.do"),("임산물수출유망 발굴업체홍보","/info/exportInfo/exportData_05.do"),("임업기계장비","/info/exportInfo/exportData_06.do")],
|
||
30:[("유관기관","/info/exportInfo/exportSite_01.do"),("수출업체","/info/exportInfo/exportSite_02_01.do"),("관련사이트","/info/exportInfo/exportSite_02_02_01.do"),("국제박람회","/info/exportInfo/exportSite_03_01.do")],
|
||
31:[("기술이전이란?","/info/tech/techComm_01.do"),("업무처리절차","/info/tech/techComm_02_01.do"),("기술이전현황","/info/tech/techComm_01_02.do"),("사업화 성공 사례","/info/tech/techComm_01_03.do"),("보유특허정보","/info/tech/techComm_01_01.do"),("기술자료집","/info/tech_06.do"),("특허·논문검색","/info/tech_05_01.do"),("관련양식","/info/tech/techComm_03_01.do")],
|
||
53:[("시험·분석·조사·감정","/service/introduceInfo_01.do"),("목재제품규격ㆍ품질검사","/service/introduceInfo_04.do"),("KS인증","/service/introduceInfo_11.do"),("NET(신기술 지정)","/service/introduceInfo_15.do"),("임업기계 장비인증","/service/introduceInfo_34.do"),("목재제품 탄소저장량 표시제도","/service/introduceInfo_41.do"),("조달물자 전문기관검사","/service/introduceInfo_48.do")],
|
||
55:[("특별관리임산물(산양삼) 분석","/service/introduceInfo_20.do"),("토양/식물체분석","/service/introduceInfo_23.do"),("산림토양조사","/service/introduceInfo_26.do")],
|
||
70:[("숲푸드 소개","/service/ogFood_01.do"),("숲푸드 신청 절차/BI","/service/ogFood_02.do")],
|
||
71:[("숲푸드 더자연산 소개","/service/kfFood_01.do"),("숲푸드 더자연산 지정절차","/service/kfFood_02.do")],
|
||
89:[("윤리서약","/intro/intro_behavior_01_2.do"),("윤리헌장","/intro/intro_behavior_01.do"),("윤리강령","/intro/intro_behavior_01_1.do")],
|
||
90:[("청탁금지법 준수 서약서","/intro/intro_13.do"),("청렴 서약서","/intro/intro_19.do")],
|
||
93:[("이사장(원장)","/intro/intro_behavior_02.do"),("상임이사","/intro/intro_behavior_02_1.do"),("상임이사2","/intro/intro_behavior_02_2.do")],
|
||
96:[("공정거래 자율준수 프로그램 소개","/intro/cp_01.do"),("기관장 의지 선언문","/intro/cp_02.do")],
|
||
}
|
||
|
||
# unique urls to classify
|
||
cache={}
|
||
def classify(u):
|
||
if u in cache: return cache[u]
|
||
try:
|
||
r=requests.get(u,headers=hd,timeout=25)
|
||
r.encoding=r.apparent_encoding
|
||
s=BeautifulSoup(r.content,"html.parser")
|
||
final=str(r.url)
|
||
# board detection: list with date/number, pagination, total
|
||
txt=s.get_text(" ",strip=True)
|
||
tot=None
|
||
for pat in [r"총\s*([\d,]+)\s*건",r"총\s*게시물\s*[:：]?\s*([\d,]+)",r"전체\s*([\d,]+)\s*건",r"Total\s*[:：]?\s*([\d,]+)"]:
|
||
m=re.search(pat,txt)
|
||
if m: tot=int(m.group(1).replace(",","")); break
|
||
paging=bool(s.select(".paging a, .pagination a, [class*=paging] a, .board_paging a"))
|
||
# board list table with rows that have number+title+date
|
||
listrows=0
|
||
for tr in s.select("table tbody tr"):
|
||
tds=tr.find_all("td")
|
||
if len(tds)>=3: listrows+=1
|
||
is_board = (tot is not None) or paging or (listrows>=5 and bool(s.select("table")))
|
||
# content images (>=140x100-ish heuristic by attrs not reliable; just count non-ui imgs in content)
|
||
cont=s.select_one("#contents, #contens, #txt, #content, .sub_content, .contents")
|
||
imgs=[]
|
||
scope=cont if cont else s
|
||
for img in scope.find_all("img"):
|
||
src=(img.get("src") or "")
|
||
if re.search(r"logo|footer|header|banner|btn|icon|bg|sns|family|arrow|blank|no_img|sprite|marker|common",src,re.I): continue
|
||
alt=(img.get("alt") or "")
|
||
imgs.append((src[-40:],alt[:20]))
|
||
kogl=set()
|
||
for img in s.find_all("img"):
|
||
m=re.search(r"open(?:type|code)0*([1-4])\.jpg",(img.get('src') or ''))
|
||
if m: kogl.add(int(m.group(1)))
|
||
cache[u]={"final":final,"tot":tot,"paging":paging,"listrows":listrows,"board":is_board,"nimg":len(imgs),"imgs":imgs[:3],"kogl":sorted(kogl)}
|
||
except Exception as e:
|
||
cache[u]={"error":str(e)}
|
||
return cache[u]
|
||
|
||
for ar,tabs in GROUPS.items():
|
||
# anchor row leaf label
|
||
leaf=None
|
||
for c in range(10,3,-1):
|
||
v=ws.cell(ar,c).value
|
||
if v and str(v).strip(): leaf=str(v).strip(); break
|
||
out.write(f"\n##### GROUP anchor r{ar} leaf='{leaf}'\n")
|
||
for lbl,href in tabs:
|
||
u=norm(href)
|
||
info=classify(u)
|
||
exrows=sheet_urls.get(u,[])
|
||
out.write(f" '{lbl}' {href}\n")
|
||
out.write(f" board={info.get('board')} tot={info.get('tot')} paging={info.get('paging')} listrows={info.get('listrows')} nimg={info.get('nimg')} kogl={info.get('kogl')} final={info.get('final','')[-50:]}\n")
|
||
out.write(f" in_sheet_rows={exrows} imgs={info.get('imgs')}\n")
|
||
out.close()
|
||
print("done", len(cache), "urls")
|