df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
39 lines
1.7 KiB
Python
39 lines
1.7 KiB
Python
import openpyxl, requests, sys, re
|
|
from bs4 import BeautifulSoup
|
|
import urllib3
|
|
urllib3.disable_warnings()
|
|
sys.stdout.reconfigure(encoding='utf-8')
|
|
p = r'D:\01.프로젝트\DB수집\작업파일\광역_사이트맵\충청남도\9.서천군\충청남도_서천군.xlsx'
|
|
wb = openpyxl.load_workbook(p); ws = wb.active
|
|
H = {'User-Agent':'Mozilla/5.0'}
|
|
def get(u):
|
|
r = requests.get(u, headers=H, timeout=20, verify=False); r.encoding=r.apparent_encoding or 'utf-8'; return r.text
|
|
# rows flagged with basic_tab from prior scan
|
|
rows = [17,22,25,35,48,57,66,70,78,94,97,102,104,107,134,135,136,168,170,171,176,183,193,199,200,217,219,222,233,235,236,237,250,268]
|
|
for r in rows:
|
|
cell = ws.cell(r,11)
|
|
url = cell.hyperlink.target if cell.hyperlink else cell.value
|
|
label = ws.cell(r,7).value or ws.cell(r,6).value or ''
|
|
L = ws.cell(r,12).value
|
|
try:
|
|
soup = BeautifulSoup(get(url),'html.parser')
|
|
except Exception as e:
|
|
print(f'R{r} ERR {e}'); continue
|
|
ul = soup.select_one('ul.basic_tab')
|
|
if not ul:
|
|
print(f'R{r} no basic_tab now'); continue
|
|
hrefs=[(' '.join(li.get_text().split())[:14], (li.find("a").get("href") if li.find("a") else "")) for li in ul.select('li')]
|
|
# classify
|
|
kinds=set()
|
|
for t,h in hrefs:
|
|
if not h or h.strip() in ('#','') or h.startswith('javascript') or h.startswith('#'):
|
|
kinds.add('anchor')
|
|
elif 'list.do' in h or 'cate_' in h or '?' in h:
|
|
kinds.add('filter')
|
|
else:
|
|
kinds.add('page')
|
|
has_slave = bool(soup.select_one('ul.slave_tab'))
|
|
print(f'R{r:>3} {str(label)[:18]:18} L={L} n={len(hrefs)} kinds={kinds} slave={has_slave}')
|
|
for t,h in hrefs[:6]:
|
|
print(f' - {t:14} {h[:55]}')
|