# -*- coding: utf-8 -*-
import requests, urllib3, re, sys, io
from bs4 import BeautifulSoup
import openpyxl
urllib3.disable_warnings()
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
H={'User-Agent':'Mozilla/5.0'}
BASE='https://www.jejusi.go.kr'

def get(u):
    return requests.get(u, timeout=20, headers=H, verify=False, allow_redirects=True)

def norm(href):
    if not href: return ''
    href=href.split('#')[0]
    href=re.sub(r'[?&]_csrf=[0-9a-f\-]+','',href)
    href=href.rstrip('?&')
    if href.startswith('http'):
        if 'www.jejusi.go.kr' in href:
            href='/'+href.split('jejusi.go.kr/')[1]
        else:
            return href  # external
    return href

# 1) New /news/ LNB tree
print('===== NEW /news/ LNB (소통/시정소식 영역) =====')
r=get(BASE+'/news/communite/report.do')
soup=BeautifulSoup(r.content,'html.parser')
ul=soup.select_one('ul.depth1')
news_leaves=[]
if ul:
    for li in ul.find_all('li', recursive=False):
        a=li.find('a', recursive=False)
        if not a: continue
        top=a.get_text(strip=True); tophref=norm(a.get('href'))
        print(f'[{top}] {tophref}')
        sub=li.find('ul', recursive=False)
        if sub:
            for a2 in sub.find_all('a'):
                t2=a2.get_text(strip=True); h2=norm(a2.get('href'))
                tb=' (새창)' if a2.get('target')=='_blank' else ''
                print(f'    - {t2}  {h2}{tb}')
                news_leaves.append((top,t2,h2,bool(a2.get('target')=='_blank')))
else:
    print('!! ul.depth1 not found')

# 2) Excel current 시정소식/mainNews rows
print('\n===== EXCEL rows with /mainNews/ or /news/ =====')
wb=openpyxl.load_workbook('제주특별자치도_제주시.xlsx'); ws=wb.active
xl_paths=set()
for rr in range(3, ws.max_row+1):
    K=ws.cell(rr,11).value or ''
    djk=[ws.cell(rr,c).value for c in range(4,11)]
    djk=[str(x) for x in djk if x not in (None,'')]
    p=norm(K)
    xl_paths.add(p)
    if '/mainNews' in p or p.startswith('/news'):
        print(f'r{rr}', ' / '.join(djk), '||', p)

# 3) GNB internal leaves vs excel (path-based, no redirect resolve)
print('\n===== GNB internal leaves MISSING from excel (path compare) =====')
r=get(BASE+'/main.do')
soup=BeautifulSoup(r.content,'html.parser')
nav=soup.find('nav')
seen=set(); missing=[]
for a in nav.find_all('a'):
    h=norm(a.get('href')); t=a.get_text(strip=True)
    if not h.startswith('/'): continue   # internal only
    if not h.endswith('.do'): continue
    if h in seen: continue
    seen.add(h)
    if h not in xl_paths:
        missing.append((t,h))
print(f'GNB internal .do leaves: {len(seen)}, missing from excel: {len(missing)}')
for t,h in missing:
    print(f'   MISS {t}  {h}')