48 lines
2.1 KiB
Python
48 lines
2.1 KiB
Python
# -*- coding: utf-8 -*-
|
|
import urllib.request, ssl, re, io, openpyxl
|
|
from bs4 import BeautifulSoup
|
|
ctx=ssl.create_default_context(); ctx.check_hostname=False; ctx.verify_mode=ssl.CERT_NONE
|
|
DECOR=re.compile(r'(logo|icon|btn|button|ebook|flowchart|banner|sprite|blank|no_?img|arrow|bullet|bg_|_bg|footer|header|sns|share|top_|quick|^er_)',re.I)
|
|
def get(u):
|
|
req=urllib.request.Request(u,headers={"User-Agent":"Mozilla/5.0"})
|
|
return urllib.request.urlopen(req,context=ctx,timeout=25).read()
|
|
def detect(u):
|
|
try: raw=get(u)
|
|
except Exception as e: return {"err":str(e)[:40]}
|
|
soup=BeautifulSoup(raw,"html.parser")
|
|
cont=None
|
|
for sel in [".contents","#contents","#content",".content"]:
|
|
el=soup.select_one(sel)
|
|
if el and len(el.get_text(strip=True))>10: cont=el;break
|
|
if cont is None: cont=soup.body
|
|
vids=len(cont.find_all("video"))
|
|
for ifr in cont.find_all("iframe"):
|
|
if re.search(r'youtube|youtu\.be|vimeo',ifr.get("src","") or ""): vids+=1
|
|
# content image candidates (non-decor)
|
|
cand=[]
|
|
for im in cont.find_all("img"):
|
|
s=(im.get("src") or im.get("data-src") or "")
|
|
fn=s.split("/")[-1].split("?")[0]
|
|
if not fn: continue
|
|
if DECOR.search(fn): continue
|
|
cand.append(fn[:45])
|
|
# board list thumbnails (for list.do) — look for gallery/thumb
|
|
thumb = bool(cont.select(".gallery img, .thumb img, .photo img, li img"))
|
|
return {"vid":vids,"imgcand":cand,"thumb":thumb,"textlen":len(cont.get_text(strip=True))}
|
|
wb=openpyxl.load_workbook("한국의료분쟁조정중재원.xlsx"); ws=wb.active
|
|
o=io.open("_ndetect_out.txt","w",encoding="utf-8")
|
|
for r in range(23,94):
|
|
K=ws.cell(r,11).value
|
|
if not K: continue
|
|
L=ws.cell(r,12).value; curN=ws.cell(r,14).value
|
|
eff=[ws.cell(r,c).value for c in range(4,11)]
|
|
# effective leaf label
|
|
lab=""
|
|
for mr in ws.merged_cells.ranges:
|
|
pass
|
|
path=ws.cell(r,14).value
|
|
d=detect(str(K))
|
|
o.write("r%d L=%s curN=%s | vid=%s thumb=%s imgcand=%s | %s\n"%(
|
|
r,L,curN,d.get("vid"),d.get("thumb"),d.get("imgcand",d.get("err")),str(K)[-55:]))
|
|
o.close();print("done")
|