df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
101 lines
3.9 KiB
Python
101 lines
3.9 KiB
Python
"""Parse left-nav tree (ul.dep2/dep3/dep4) anchors with ancestor tracking.
|
|
Build menuCd -> target_blank map restricted to the navigation tree only."""
|
|
import urllib.request, ssl, re, io, sys, json
|
|
from html.parser import HTMLParser
|
|
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
|
|
|
|
ctx = ssl.create_default_context(); ctx.check_hostname = False; ctx.verify_mode = ssl.CERT_NONE
|
|
|
|
def fetch(url):
|
|
req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0'})
|
|
return urllib.request.urlopen(req, timeout=25, context=ctx).read().decode('utf-8', 'replace')
|
|
|
|
class NavParser(HTMLParser):
|
|
def __init__(self):
|
|
super().__init__()
|
|
self.ul_stack = [] # stack of ul class strings
|
|
self.in_nav_depth = 0 # how many depN uls we are inside
|
|
self.cur_a = None # (menuCd, blank, href) while inside an <a> within nav
|
|
self.text = ''
|
|
self.results = [] # list of dict
|
|
|
|
def handle_starttag(self, tag, attrs):
|
|
d = dict(attrs)
|
|
if tag == 'ul':
|
|
cls = d.get('class', '') or ''
|
|
isnav = bool(re.match(r'dep[2-5]', cls.strip()))
|
|
self.ul_stack.append(isnav)
|
|
if isnav:
|
|
self.in_nav_depth += 1
|
|
elif tag == 'a' and self.in_nav_depth > 0:
|
|
href = d.get('href', '') or ''
|
|
mc = re.search(r'menuCd=(DOM_\d+)', href)
|
|
blank = (d.get('target', '') == '_blank')
|
|
self.cur_a = {'code': mc.group(1) if mc else None,
|
|
'href': href, 'blank': blank}
|
|
self.text = ''
|
|
|
|
def handle_data(self, data):
|
|
if self.cur_a is not None:
|
|
self.text += data
|
|
|
|
def handle_endtag(self, tag):
|
|
if tag == 'a' and self.cur_a is not None:
|
|
self.cur_a['label'] = self.text.strip()
|
|
self.results.append(self.cur_a)
|
|
self.cur_a = None
|
|
elif tag == 'ul' and self.ul_stack:
|
|
isnav = self.ul_stack.pop()
|
|
if isnav:
|
|
self.in_nav_depth -= 1
|
|
|
|
def build_map(htmls):
|
|
amap = {} # code -> set of blank values
|
|
labels = {}
|
|
for h in htmls:
|
|
p = NavParser(); p.feed(h)
|
|
for a in p.results:
|
|
if a['code']:
|
|
amap.setdefault(a['code'], set()).add(a['blank'])
|
|
labels[a['code']] = a['label']
|
|
return amap, labels
|
|
|
|
if __name__ == '__main__':
|
|
# fetch one representative page per top category to cover full nav tree
|
|
seeds = [
|
|
'DOM_000000101001001000', # 시민마당
|
|
'DOM_000000102001002000', # 종합민원
|
|
'DOM_000000103013019000', # 분야별정보
|
|
'DOM_000000104007011000', # 소통/정보공개
|
|
'DOM_000000105001001000', # 정읍은
|
|
'DOM_000000104001001000', # 정보공개
|
|
]
|
|
htmls = []
|
|
for s in seeds:
|
|
try:
|
|
htmls.append(fetch('https://www.jeongeup.go.kr/index.jeongeup?menuCd=' + s))
|
|
except Exception as e:
|
|
print('seed fail', s, e)
|
|
amap, labels = build_map(htmls)
|
|
print('distinct nav menuCd:', len(amap))
|
|
data = json.load(io.open('_dump.json', encoding='utf-8'))
|
|
miss = []
|
|
blank_rows = []
|
|
for row in data:
|
|
k = row.get('K', '')
|
|
mc = re.search(r'menuCd=(DOM_\d+)', k)
|
|
if not mc:
|
|
continue
|
|
code = mc.group(1)
|
|
if code not in amap:
|
|
miss.append((row['r'], row.get('F') or row.get('E') or row.get('G')))
|
|
continue
|
|
vals = amap[code]
|
|
if True in vals:
|
|
blank_rows.append((row['r'], row.get('L'), True in vals and False in vals, row.get('F') or row.get('E') or row.get('G')))
|
|
print('rows not found in nav:', len(miss))
|
|
for r, f in miss: print(' MISS', r, f)
|
|
print('rows whose nav anchor is target=_blank:', len(blank_rows))
|
|
for r, l, conflict, f in blank_rows:
|
|
print(' BLANK', r, 'L=' + str(l), 'conflict' if conflict else '', f)
|