import requests, re, json, sys, time
from bs4 import BeautifulSoup
from urllib.parse import urljoin
sys.stdout.reconfigure(encoding='utf-8')
H={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'}
sess=requests.Session(); sess.headers.update(H)
UI=re.compile(r'logo|footer|header|/icons?/|ico_|btn|sprite|blank|no_?img|banner|flag|/common/|/template|sns|arrow|bg_|_bg|/img/comm|favicon|h1\.', re.I)
KOGL=re.compile(r'open(?:type|code)0*([1-5])|kogl|ccl', re.I)
ts=json.load(open(r'작업파일/공공기관2/12.한국철도공사/_tabscan.json',encoding='utf-8'))
BASE='https://info.korail.com/info/'
# collect all tab entries from expandable groups (leaf F, self-in-group)
EXPAND=[6,8,15,30,33,34,37,38,39,40,45,46,47,74,78,79,80,82,86,92,93,94,102,104,105,106]
def get(u):
    for _ in range(3):
        try: return sess.get(u,timeout=30)
        except: time.sleep(1.5)
    return None
out={}
seen=set()
for r in EXPAND:
    for t in ts[str(r)]['tabs']:
        href=t['href']
        if href.startswith('http') and 'korail.com' not in href:
            out[href]={'ext':1,'label':t['label']}; continue
        url=urljoin(BASE, href) if not href.startswith('http') else href
        key=url.split('key=')[-1]
        if url in seen: continue
        seen.add(url)
        rr=get(url)
        if not rr: out[url]={'err':1,'label':t['label']}; continue
        s=BeautifulSoup(rr.content,'html.parser')
        cont=s.select_one('#contents') or s.select_one('.contents') or s.select_one('.sub_content') or s.select_one('#container') or s
        cimg=[]; kogl=set()
        for im in cont.find_all('img'):
            src=im.get('src','') or ''
            if not src: continue
            if KOGL.search(src):
                m=KOGL.search(src); 
                if m.group(1): kogl.add(int(m.group(1)))
                continue
            if UI.search(src): continue
            cimg.append(src.split('?')[0])
        out[url]={'label':t['label'],'cimg':cimg,'kogl':sorted(kogl)}
        print(key, t['label'], 'cimg',len(cimg), 'kogl',sorted(kogl), cimg[:2])
        time.sleep(0.2)
json.dump(out, open(r'작업파일/공공기관2/12.한국철도공사/_childN.json','w',encoding='utf-8'),ensure_ascii=False,indent=0)
print('DONE', len(out))