{ "cells": [ { "cell_type": "markdown", "id": "e01539fe-9ee2-4604-b464-a9dd050bd2d7", "metadata": {}, "source": [ "## 中国戏考数据采集" ] }, { "cell_type": "code", "execution_count": null, "id": "b054c312-d5ee-46b7-a173-f00ac8c2c952", "metadata": { "tags": [] }, "outputs": [], "source": [ "import requests\n", "import time\n", "from bs4 import BeautifulSoup\n", "import pymongo\n", "import re\n", "\n", "#m_mongo = {}\n", "#requests.packages.urllib3.disable_warnings()\n", "#requests.packages.urllib3.util.ssl_.DEFAULT_CIPHERS += 'HIGH:!DH:!aNULL'\n", "#myclient = pymongo.MongoClient('mongodb://localhost:27017/')\n", "#mydb = myclient[\"gaokao\"]\n", "#mycol = mydb[\"news\"]\n", "url = 'data/京剧剧本 - 总目.html'\n", "a_url = 'https://scripts.xikao.com'\n", "#strhtml = requests.get(url)\n", "strhtml = 'data/京剧剧本 - 总目.html'\n", "#strhtml.encoding = 'utf8'\n", "soup = BeautifulSoup(strhtml,'html.parser')\n", "data = soup.select('#master-table')\n", "for item in data:\n", " print(item)\n", " print('\\br')" ] }, { "cell_type": "code", "execution_count": 139, "id": "1b3b3b53-d9c1-4e80-8e25-19165d2921e9", "metadata": { "execution": { "iopub.execute_input": "2024-04-04T15:44:59.306388Z", "iopub.status.busy": "2024-04-04T15:44:59.305906Z", "iopub.status.idle": "2024-04-04T15:44:59.370430Z", "shell.execute_reply": "2024-04-04T15:44:59.368843Z", "shell.execute_reply.started": "2024-04-04T15:44:59.306350Z" }, "tags": [] }, "outputs": [ { "ename": "FeatureNotFound", "evalue": "Couldn't find a tree builder with the features you requested: lxml. Do you need to install a parser library?", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mFeatureNotFound\u001b[0m Traceback (most recent call last)", "Cell \u001b[0;32mIn[139], line 5\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mre\u001b[39;00m\n\u001b[1;32m 4\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mopen\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdata/京剧剧本 - 总目.html\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;28;01mas\u001b[39;00m fp:\n\u001b[0;32m----> 5\u001b[0m soup \u001b[38;5;241m=\u001b[39m \u001b[43mBeautifulSoup\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfp\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mlxml\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 7\u001b[0m \u001b[38;5;66;03m#soup = BeautifulSoup(\"a web page\", 'html.parser')\u001b[39;00m\n\u001b[1;32m 8\u001b[0m data \u001b[38;5;241m=\u001b[39m soup\u001b[38;5;241m.\u001b[39mfind_all(\u001b[38;5;28mid\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mmaster-table\u001b[39m\u001b[38;5;124m'\u001b[39m)\n", "File \u001b[0;32m~/data/lib/python3.10/site-packages/bs4/__init__.py:250\u001b[0m, in \u001b[0;36mBeautifulSoup.__init__\u001b[0;34m(self, markup, features, builder, parse_only, from_encoding, exclude_encodings, element_classes, **kwargs)\u001b[0m\n\u001b[1;32m 248\u001b[0m builder_class \u001b[38;5;241m=\u001b[39m builder_registry\u001b[38;5;241m.\u001b[39mlookup(\u001b[38;5;241m*\u001b[39mfeatures)\n\u001b[1;32m 249\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m builder_class \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 250\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m FeatureNotFound(\n\u001b[1;32m 251\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCouldn\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mt find a tree builder with the features you \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 252\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrequested: \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m. Do you need to install a parser library?\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 253\u001b[0m \u001b[38;5;241m%\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(features))\n\u001b[1;32m 255\u001b[0m \u001b[38;5;66;03m# At this point either we have a TreeBuilder instance in\u001b[39;00m\n\u001b[1;32m 256\u001b[0m \u001b[38;5;66;03m# builder, or we have a builder_class that we can instantiate\u001b[39;00m\n\u001b[1;32m 257\u001b[0m \u001b[38;5;66;03m# with the remaining **kwargs.\u001b[39;00m\n\u001b[1;32m 258\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m builder \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n", "\u001b[0;31mFeatureNotFound\u001b[0m: Couldn't find a tree builder with the features you requested: lxml. Do you need to install a parser library?" ] } ], "source": [ "from bs4 import BeautifulSoup\n", "import re\n", "\n", "with open(\"data/京剧剧本 - 总目.html\") as fp:\n", " soup = BeautifulSoup(fp, 'lxml')\n", "\n", "#soup = BeautifulSoup(\"a web page\", 'html.parser')\n", "data = soup.find_all(id = 'master-table')\n", "for item in data:\n", " list1 = item.find_all(class_=re.compile(\"row_\"))\n", " #list1 = item.find_all(lambda tag: tag.get('class') == ['row_c'] or tag.get('class') == ['row_a'])\n", " #print(list1)\n", " for tr in list1:\n", " #print(tr)\n", " if tr.a:\n", " print(tr.a.get('href'),tr.b.text)\n", " #print('end')" ] }, { "cell_type": "code", "execution_count": null, "id": "fad890ec-010e-4afa-b6a6-5427b4f65f95", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.12" } }, "nbformat": 4, "nbformat_minor": 5 }