df16c98366
공공기관2 작업 중. _temp 몽타주(재생성가능)는 제외. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
156 lines
11 KiB
Plaintext
156 lines
11 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 5,
|
|
"id": "ab0f3c3e",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"엑셀 파일을 읽어오는 중입니다...\n"
|
|
]
|
|
},
|
|
{
|
|
"ename": "FileNotFoundError",
|
|
"evalue": "[Errno 2] No such file or directory: 'C:\\\\Users\\\\hehih\\\\Downloads\\\\인천관역시\\\\인천광역시_홈페이지_사이트_r1.xlsx'",
|
|
"output_type": "error",
|
|
"traceback": [
|
|
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
|
"\u001b[0;31mFileNotFoundError\u001b[0m Traceback (most recent call last)",
|
|
"\u001b[0;32m/tmp/ipykernel_4190/3930924579.py\u001b[0m in \u001b[0;36m<cell line: 0>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 11\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"엑셀 파일을 읽어오는 중입니다...\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 12\u001b[0;31m \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_excel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfile_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msheet_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msheet_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mheader\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 13\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 14\u001b[0m \u001b[0;31m# 웹사이트 접속 차단 방지 설정\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
|
|
"\u001b[0;32m/usr/local/lib/python3.12/dist-packages/pandas/io/excel/_base.py\u001b[0m in \u001b[0;36mread_excel\u001b[0;34m(io, sheet_name, header, names, index_col, usecols, dtype, engine, converters, true_values, false_values, skiprows, nrows, na_values, keep_default_na, na_filter, verbose, parse_dates, date_parser, date_format, thousands, decimal, comment, skipfooter, storage_options, dtype_backend, engine_kwargs)\u001b[0m\n\u001b[1;32m 493\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mio\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mExcelFile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 494\u001b[0m \u001b[0mshould_close\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 495\u001b[0;31m io = ExcelFile(\n\u001b[0m\u001b[1;32m 496\u001b[0m \u001b[0mio\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 497\u001b[0m \u001b[0mstorage_options\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstorage_options\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
|
|
"\u001b[0;32m/usr/local/lib/python3.12/dist-packages/pandas/io/excel/_base.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, path_or_buffer, engine, storage_options, engine_kwargs)\u001b[0m\n\u001b[1;32m 1548\u001b[0m \u001b[0mext\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"xls\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1549\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1550\u001b[0;31m ext = inspect_excel_format(\n\u001b[0m\u001b[1;32m 1551\u001b[0m \u001b[0mcontent_or_path\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstorage_options\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstorage_options\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1552\u001b[0m )\n",
|
|
"\u001b[0;32m/usr/local/lib/python3.12/dist-packages/pandas/io/excel/_base.py\u001b[0m in \u001b[0;36minspect_excel_format\u001b[0;34m(content_or_path, storage_options)\u001b[0m\n\u001b[1;32m 1400\u001b[0m \u001b[0mcontent_or_path\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mBytesIO\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcontent_or_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1401\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1402\u001b[0;31m with get_handle(\n\u001b[0m\u001b[1;32m 1403\u001b[0m \u001b[0mcontent_or_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"rb\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstorage_options\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstorage_options\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mis_text\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1404\u001b[0m ) as handle:\n",
|
|
"\u001b[0;32m/usr/local/lib/python3.12/dist-packages/pandas/io/common.py\u001b[0m in \u001b[0;36mget_handle\u001b[0;34m(path_or_buf, mode, encoding, compression, memory_map, is_text, errors, storage_options)\u001b[0m\n\u001b[1;32m 880\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 881\u001b[0m \u001b[0;31m# Binary mode\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 882\u001b[0;31m \u001b[0mhandle\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mioargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 883\u001b[0m \u001b[0mhandles\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 884\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
|
|
"\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'C:\\\\Users\\\\hehih\\\\Downloads\\\\인천관역시\\\\인천광역시_홈페이지_사이트_r1.xlsx'"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"import pandas as pd\n",
|
|
"import requests\n",
|
|
"from bs4 import BeautifulSoup\n",
|
|
"import re\n",
|
|
"import time\n",
|
|
"\n",
|
|
"# 1. 파일 불러오기\n",
|
|
"file_name = r\"C:\\Users\\hehih\\Downloads\\인천관역시\\인천광역시_홈페이지_사이트_r1.xlsx\"\n",
|
|
"sheet_name = \"Sitemap\" \n",
|
|
"\n",
|
|
"print(\"엑셀 파일을 읽어오는 중입니다...\")\n",
|
|
"df = pd.read_excel(file_name, sheet_name=sheet_name, header=None)\n",
|
|
"\n",
|
|
"# 웹사이트 접속 차단 방지 설정\n",
|
|
"headers = {\n",
|
|
" 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'\n",
|
|
"}\n",
|
|
"\n",
|
|
"print(\"URL 순차 파싱을 시작합니다...\")\n",
|
|
"\n",
|
|
"# 2. 각 행을 순회하며 데이터 추출 및 입력\n",
|
|
"for index, row in df.iterrows():\n",
|
|
" if index < 2: # 안내문구 및 헤더 건너뛰기\n",
|
|
" continue\n",
|
|
" \n",
|
|
" url = row[8] # 8번 인덱스 = I열 (주소url)\n",
|
|
" \n",
|
|
" if pd.isna(url) or not str(url).startswith('http'):\n",
|
|
" continue\n",
|
|
" \n",
|
|
" print(f\"[{index+1}/{len(df)}] 파싱 중: {url}\")\n",
|
|
" \n",
|
|
" try:\n",
|
|
" response = requests.get(url, headers=headers, timeout=10)\n",
|
|
" \n",
|
|
" if response.status_code == 200:\n",
|
|
" soup = BeautifulSoup(response.text, 'html.parser')\n",
|
|
" \n",
|
|
" # 태그 찾기\n",
|
|
" target_div = soup.find('div', class_='board-page-total')\n",
|
|
" \n",
|
|
" # [조건문] 태그가 존재하는 경우 (게시판)\n",
|
|
" if target_div:\n",
|
|
" div_text = target_div.get_text()\n",
|
|
" match = re.search(r'전체\\s*([\\d,]+)\\s*건', div_text)\n",
|
|
" \n",
|
|
" if match:\n",
|
|
" total_count = match.group(1).replace(',', '')\n",
|
|
" df.at[index, 9] = \"게시판\" # J열\n",
|
|
" df.at[index, 10] = int(total_count) # K열\n",
|
|
" print(f\" => [게시판 확인] 건수: {total_count}건\")\n",
|
|
" else:\n",
|
|
" df.at[index, 9] = \"게시판(양식 다름)\"\n",
|
|
" df.at[index, 10] = \"-\"\n",
|
|
" print(\" => 태그는 있으나 건수 형식이 다릅니다.\")\n",
|
|
" \n",
|
|
" # [조건문 수정] 태그가 없는 경우 (일반 페이지 등)\n",
|
|
" else:\n",
|
|
" df.at[index, 9] = \"페이지\" # J열에 일반페이지 입력\n",
|
|
" df.at[index, 10] = \"\" # K열은 대시(-) 처리 (또는 0 입력 가능)\n",
|
|
" print(\" => [일반페이지] 'board-page-total' 태그 없음\")\n",
|
|
" \n",
|
|
" else:\n",
|
|
" df.at[index, 9] = \"접속실패\"\n",
|
|
" df.at[index, 10] = f\"에러({response.status_code})\"\n",
|
|
" print(f\" => 접속 실패 (상태코드: {response.status_code})\")\n",
|
|
" \n",
|
|
" except Exception as e:\n",
|
|
" df.at[index, 9] = \"접속오류\"\n",
|
|
" df.at[index, 10] = \"오류\"\n",
|
|
" print(f\" => 오류 발생: {e}\")\n",
|
|
" \n",
|
|
" # 서버 차단 방지를 위해 0.5초씩 휴식\n",
|
|
" time.sleep(0.5)\n",
|
|
"\n",
|
|
"# 3. 원래 양식을 유지하며 새 엑셀 파일로 저장\n",
|
|
"output_file = \"인천광역시_홈페이지_사이트_결과.xlsx\"\n",
|
|
"df.to_excel(output_file, index=False, header=False)\n",
|
|
"print(f\"\\n모든 작업이 완료되었습니다! 결과가 '{output_file}' 파일로 저장되었습니다.\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 2,
|
|
"id": "8d3096b0",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"['.config', 'sample_data']\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"import os\n",
|
|
"print(os.listdir('.')) # 현재 파이썬이 인식하는 폴더 안의 파일 목록 출력"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3 (ipykernel)",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.12.13"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|