b0acd1e705
- 공공기관2/3 작업본 + 오늘 제출 17곳 D~J 카테고리 셀병합 정상화 - 한국지역난방공사 옵션2(고아셀 F98 수정)+전행 높이17 - 제출_프리랜서2_2026-06-21.zip 생성(17개 xlsx, 2,468행) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
178 lines
7.5 KiB
Python
178 lines
7.5 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""한국원자력환경공단 전용 L/M/N/O 채우기.
|
|
- 단일 세션·완만한 페이싱(차단 회피)
|
|
- body = div.doc-body
|
|
- 게시판: table.bbs + span.bbs-result('총 게시물 : N 건') / td 번호열 최댓값
|
|
- 미디어: doc-body 내 이미지. menuResources 인포그래픽/도식 = 어문(이미지 아님), 진짜 사진(data 업로드 jpg with 사진성)만 이미지.
|
|
- KOGL: img_open(type|code)0N 마크
|
|
출력: L/M/N/O (P/Q는 마크없으면 빈칸). S(비고)는 건드리지 않음(외부링크 유지).
|
|
변경 셀 표시: N 파란색.
|
|
"""
|
|
import sys,io,re,time,warnings
|
|
sys.stdout=io.TextIOWrapper(sys.stdout.buffer,encoding='utf-8')
|
|
warnings.filterwarnings('ignore')
|
|
import requests
|
|
from bs4 import BeautifulSoup
|
|
from urllib.parse import urljoin
|
|
import openpyxl
|
|
from openpyxl.styles import PatternFill, Font
|
|
import urllib3;urllib3.disable_warnings()
|
|
|
|
P=r'D:\01.프로젝트\DB수집\작업파일\공공기관3\1.한국원자력환경공단\한국원자력환경공단.xlsx'
|
|
H={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|
'Accept':'text/html,application/xhtml+xml','Accept-Language':'ko-KR,ko;q=0.9'}
|
|
KOGL=re.compile(r'img_open(?:type|code)0*(\d{1,2})\.(?:png|jpe?g|gif)',re.I)
|
|
KOGL_LINK=re.compile(r'kogl\.or\.kr/info/licenseType(\d)',re.I)
|
|
YT=re.compile(r'(youtube\.com|youtu\.be|vimeo\.com)',re.I)
|
|
VIDEXT=re.compile(r'\.(mp4|webm|mov|m3u8)(\?|$)',re.I)
|
|
AUDEXT=re.compile(r'\.(mp3|wav|m4a|ogg)(\?|$|["\'&])',re.I)
|
|
NOISE=re.compile(r'(ico[_\-/]|/icon|logo|btn|bul[_\-]|bg[_\-/]|banner|sns|blank|spacer|loading|arrow|/dot|line[_\.]|top_|foot|header|common|btn_|_icon|symbol|copyright|qr_|movie_ico|no_img|noimage|share|facebook|insta|youtube_ic|blog|twitter|naver|kakao|navi\.png|nav_)',re.I)
|
|
PHOTO=re.compile(r'\.(jpe?g)(\?|$)',re.I) # 사진 후보 확장자
|
|
TOTAL=re.compile(r'(?:총\s*게시물|총|전체)\s*[:\-]?\s*(\d[\d,]*)\s*건?',re.I)
|
|
|
|
H['Referer']='https://www.korad.or.kr/'
|
|
H['Connection']='close'
|
|
s=requests.Session();s.headers.update(H)
|
|
def fetch(u,t=12):
|
|
for attempt in range(3):
|
|
try:
|
|
r=s.get(u,verify=False,timeout=(6,t))
|
|
if r.status_code==200:
|
|
return BeautifulSoup(r.content,'html.parser'),r.url
|
|
if r.status_code in (403,429,503):
|
|
time.sleep(1.0+attempt)
|
|
except Exception:
|
|
time.sleep(0.5+attempt*0.4)
|
|
return None,None
|
|
|
|
def get_body(soup):
|
|
for sel in ['div.doc-body','div.body-in','#content','#contents','.contents','.sub_content']:
|
|
e=soup.select_one(sel)
|
|
if e and len(e.get_text(strip=True))>20: return e
|
|
return soup
|
|
|
|
def detect_form_m(soup,body):
|
|
tb=soup.select_one('table.bbs')
|
|
res=soup.select_one('span.bbs-result')
|
|
gallery=soup.select_one('ul.bbs_gallery_pr')
|
|
movie=soup.select_one('ul.movieList, div.movie, div.search-results')
|
|
has_dt=bool(soup.select('div.dataTables_paginate, span.bbs-result, table.bbs tbody tr'))
|
|
if tb or res or has_dt or gallery or movie:
|
|
m=0; flag=''
|
|
if res:
|
|
mm=TOTAL.search(res.get_text(' ',strip=True))
|
|
if mm: m=int(mm.group(1).replace(',',''))
|
|
if not m and tb:
|
|
nums=[]
|
|
for tr in tb.select('tbody tr'):
|
|
tds=tr.find_all('td')
|
|
if tds:
|
|
t=tds[0].get_text(strip=True)
|
|
if t.isdigit(): nums.append(int(t))
|
|
if nums: m=max(nums)
|
|
if not m and (gallery or movie):
|
|
flag='AJAX목록 M브라우저확인'
|
|
kind='movie' if movie and not tb else ('gallery' if gallery and not tb else 'list')
|
|
return '게시판',m,flag,kind
|
|
return '페이지',1,'',''
|
|
|
|
def media(body,is_board=False):
|
|
txt=len(body.get_text(strip=True))>20
|
|
img=False
|
|
for im in body.find_all('img'):
|
|
src=im.get('src') or im.get('data-src') or ''
|
|
if not src or KOGL.search(src) or NOISE.search(src): continue
|
|
# menuResources 인포그래픽/도식 = 어문(이미지 아님). 진짜 사진만 이미지.
|
|
# 사진성: jpg + 캡션/사이즈 큰 업로드. png/gif menuResources = 도식 → 제외
|
|
if 'menuResources' in src and not PHOTO.search(src):
|
|
continue
|
|
img=True;break
|
|
vid=bool(body.find_all('video')) or bool(YT.search(str(body))) or bool(VIDEXT.search(str(body)))
|
|
aud=bool(body.find_all('audio')) or bool(AUDEXT.search(str(body)))
|
|
return txt,img,vid,aud
|
|
|
|
def kogl(body):
|
|
types=set();qy=False
|
|
for a in body.find_all('a',href=True):
|
|
m=KOGL_LINK.search(a['href'])
|
|
if m: types.add(int(m.group(1)));qy=True
|
|
for im in body.find_all('img'):
|
|
m=KOGL.search(im.get('src','') or '')
|
|
if m and 1<=int(m.group(1))<=4: types.add(int(m.group(1)))
|
|
if types=={1,2,3,4}: return set(),None
|
|
return types,('Y' if qy else 'N')
|
|
|
|
def nstr(t,i,v,a):
|
|
p=[]
|
|
if t:p.append('어문')
|
|
if i:p.append('이미지')
|
|
if v:p.append('영상')
|
|
if a:p.append('오디오')
|
|
return ','.join(p) if p else '없음'
|
|
|
|
def main():
|
|
wb=openpyxl.load_workbook(P);ws=wb.active
|
|
blue=PatternFill('solid',fgColor='FFBDD7EE',bgColor='FFBDD7EE')
|
|
rows=[]
|
|
for r in range(3,ws.max_row+1):
|
|
b=ws.cell(r,2).value
|
|
if b is None:continue
|
|
rows.append((r,ws.cell(r,11).value or '',ws.cell(r,19).value))
|
|
total=len(rows);done=0;t0=time.time()
|
|
for r,k,s19 in rows:
|
|
done+=1
|
|
if s19=='외부링크':
|
|
# 사이트: L=사이트, M~Q 빈칸
|
|
ws.cell(r,12).value='사이트'
|
|
for c in (13,14,15,16,17): ws.cell(r,c).value=None
|
|
continue
|
|
if 'korad.or.kr' not in k:
|
|
ws.cell(r,12).value='사이트'
|
|
for c in (13,14,15,16,17): ws.cell(r,c).value=None
|
|
continue
|
|
soup,final=fetch(k)
|
|
if soup is None:
|
|
ws.cell(r,19).value='접근실패-재시도필요' # S=비고
|
|
time.sleep(0.5);continue
|
|
body=get_body(soup)
|
|
form,m,flag,kind=detect_form_m(soup,body)
|
|
ws.cell(r,12).value=form
|
|
ws.cell(r,13).value=(m if form=='게시판' else 1)
|
|
if flag: ws.cell(r,19).value=flag # S=비고
|
|
is_board=(form=='게시판')
|
|
t,i,v,a=media(body,is_board)
|
|
types,qy=kogl(body)
|
|
if is_board:
|
|
# 게시판 상세는 JS링크라 정적 추적 불가 → 목록 위젯으로 미디어 판정
|
|
if kind=='gallery':
|
|
i=True
|
|
if kind=='movie':
|
|
v=True; i=True # 영상 게시판 썸네일+영상
|
|
gal=soup.select_one('ul.bbs_gallery_pr, ul.gallery, .photo_list, .gallery_list')
|
|
if gal:
|
|
for im in gal.find_all('img'):
|
|
src=im.get('src') or ''
|
|
if src and not NOISE.search(src): i=True;break
|
|
if m==0 and not flag:
|
|
nv='없음' # 빈 게시판
|
|
else:
|
|
nv=nstr(True,i,v,a)
|
|
else:
|
|
nv=nstr(t,i,v,a)
|
|
nc=ws.cell(r,14)
|
|
nc.value=nv; nc.fill=blue
|
|
if not types:
|
|
ws.cell(r,15).value='미부착'
|
|
else:
|
|
ws.cell(r,15).value=','.join(f'{n}유형' for n in sorted(types))
|
|
ws.cell(r,16).value=('게시판' if is_board else '게시물')
|
|
ws.cell(r,17).value=qy or 'N'
|
|
time.sleep(0.1)
|
|
if done%10==0:
|
|
wb.save(P);print(f' {done}/{total} ({int(time.time()-t0)}s)',flush=True)
|
|
wb.save(P)
|
|
print(f'DONE {done}/{total} ({int(time.time()-t0)}s)')
|
|
|
|
if __name__=='__main__':
|
|
main()
|