df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
99 lines
3.4 KiB
Python
99 lines
3.4 KiB
Python
"""Targeted probes for 청양군, 아산시, 부여군 to fix parsers."""
|
|
import warnings
|
|
import requests
|
|
from bs4 import BeautifulSoup
|
|
|
|
warnings.filterwarnings('ignore')
|
|
UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
|
|
H = {'User-Agent': UA}
|
|
|
|
|
|
def fetch(url):
|
|
r = requests.get(url, headers=H, timeout=15, verify=False, allow_redirects=True)
|
|
r.encoding = r.apparent_encoding
|
|
return r.text
|
|
|
|
|
|
# 청양군 — look at where the actual sitemap content lives
|
|
print('=== 청양군 ===')
|
|
html = fetch('https://www.cheongyang.go.kr/kor/sitemap_11.do')
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
# Look for #contents or main content
|
|
for sel in ['#contents', '.contents', 'main', '#txt', '#mainSection']:
|
|
el = soup.select_one(sel)
|
|
if el:
|
|
a_count = len(el.find_all('a'))
|
|
print(f' {sel}: a={a_count}')
|
|
# Look for the right sitemap container
|
|
print(' All elements with 200+ anchors:')
|
|
for el in soup.find_all(['div', 'ul', 'section']):
|
|
cls = ' '.join(el.get('class', []))
|
|
eid = el.get('id', '')
|
|
ac = len(el.find_all('a'))
|
|
if 200 <= ac <= 800 and (cls or eid):
|
|
print(f' {el.name}#{eid}.{cls[:60]} a={ac}')
|
|
|
|
|
|
print('\n=== 아산시 ===')
|
|
html = fetch('https://www.asan.go.kr/main/')
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
# Inspect mGnb-anchor1 structure deeply
|
|
sec = soup.find('div', id='mGnb-anchor1')
|
|
if sec:
|
|
# outline
|
|
def outline(el, d=0, max_lines=50, lines=None):
|
|
if lines is None: lines = []
|
|
if len(lines) >= max_lines: return lines
|
|
name = el.name
|
|
cls = ' '.join(el.get('class', []))
|
|
eid = el.get('id', '')
|
|
lbl = name
|
|
if eid: lbl += f'#{eid}'
|
|
if cls: lbl += '.' + cls.replace(' ', '.')
|
|
if name == 'a':
|
|
t = el.get_text(strip=True)[:30]
|
|
h = el.get('href', '')[:60]
|
|
lines.append(' ' * d + f'{lbl} "{t}" → {h}')
|
|
elif name == 'button':
|
|
t = el.get_text(strip=True)[:30]
|
|
lines.append(' ' * d + f'{lbl} BTN "{t}"')
|
|
else:
|
|
lines.append(' ' * d + lbl)
|
|
for c in el.find_all(recursive=False):
|
|
if c.name in ('script', 'style'): continue
|
|
outline(c, d+1, max_lines, lines)
|
|
if len(lines) >= max_lines: return lines
|
|
return lines
|
|
for line in outline(sec, max_lines=70):
|
|
print(' ', line)
|
|
|
|
|
|
print('\n=== 부여군 ===')
|
|
html = fetch('https://www.buyeo.go.kr/html/kr/')
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
# Inspect nav#gnb structure
|
|
sec = soup.select_one('nav#gnb')
|
|
if sec:
|
|
def outline(el, d=0, max_lines=80, lines=None):
|
|
if lines is None: lines = []
|
|
if len(lines) >= max_lines: return lines
|
|
name = el.name
|
|
cls = ' '.join(el.get('class', []))
|
|
eid = el.get('id', '')
|
|
lbl = name
|
|
if eid: lbl += f'#{eid}'
|
|
if cls: lbl += '.' + cls.replace(' ', '.')
|
|
if name == 'a':
|
|
t = el.get_text(strip=True)[:30]
|
|
h = el.get('href', '')[:60]
|
|
lines.append(' ' * d + f'{lbl} "{t}" → {h}')
|
|
else:
|
|
lines.append(' ' * d + lbl)
|
|
for c in el.find_all(recursive=False):
|
|
if c.name in ('script', 'style'): continue
|
|
outline(c, d+1, max_lines, lines)
|
|
if len(lines) >= max_lines: return lines
|
|
return lines
|
|
for line in outline(sec, max_lines=80):
|
|
print(' ', line)
|