123 lines
6.4 KiB
Plaintext
123 lines
6.4 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "e01539fe-9ee2-4604-b464-a9dd050bd2d7",
|
|
"metadata": {},
|
|
"source": [
|
|
"## 中国戏考数据采集"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "b054c312-d5ee-46b7-a173-f00ac8c2c952",
|
|
"metadata": {
|
|
"tags": []
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"import requests\n",
|
|
"import time\n",
|
|
"from bs4 import BeautifulSoup\n",
|
|
"import pymongo\n",
|
|
"import re\n",
|
|
"\n",
|
|
"#m_mongo = {}\n",
|
|
"#requests.packages.urllib3.disable_warnings()\n",
|
|
"#requests.packages.urllib3.util.ssl_.DEFAULT_CIPHERS += 'HIGH:!DH:!aNULL'\n",
|
|
"#myclient = pymongo.MongoClient('mongodb://localhost:27017/')\n",
|
|
"#mydb = myclient[\"gaokao\"]\n",
|
|
"#mycol = mydb[\"news\"]\n",
|
|
"url = 'data/京剧剧本 - 总目.html'\n",
|
|
"a_url = 'https://scripts.xikao.com'\n",
|
|
"#strhtml = requests.get(url)\n",
|
|
"strhtml = 'data/京剧剧本 - 总目.html'\n",
|
|
"#strhtml.encoding = 'utf8'\n",
|
|
"soup = BeautifulSoup(strhtml,'html.parser')\n",
|
|
"data = soup.select('#master-table')\n",
|
|
"for item in data:\n",
|
|
" print(item)\n",
|
|
" print('\\br')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 139,
|
|
"id": "1b3b3b53-d9c1-4e80-8e25-19165d2921e9",
|
|
"metadata": {
|
|
"execution": {
|
|
"iopub.execute_input": "2024-04-04T15:44:59.306388Z",
|
|
"iopub.status.busy": "2024-04-04T15:44:59.305906Z",
|
|
"iopub.status.idle": "2024-04-04T15:44:59.370430Z",
|
|
"shell.execute_reply": "2024-04-04T15:44:59.368843Z",
|
|
"shell.execute_reply.started": "2024-04-04T15:44:59.306350Z"
|
|
},
|
|
"tags": []
|
|
},
|
|
"outputs": [
|
|
{
|
|
"ename": "FeatureNotFound",
|
|
"evalue": "Couldn't find a tree builder with the features you requested: lxml. Do you need to install a parser library?",
|
|
"output_type": "error",
|
|
"traceback": [
|
|
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
|
"\u001b[0;31mFeatureNotFound\u001b[0m Traceback (most recent call last)",
|
|
"Cell \u001b[0;32mIn[139], line 5\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mre\u001b[39;00m\n\u001b[1;32m 4\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mopen\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdata/京剧剧本 - 总目.html\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;28;01mas\u001b[39;00m fp:\n\u001b[0;32m----> 5\u001b[0m soup \u001b[38;5;241m=\u001b[39m \u001b[43mBeautifulSoup\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfp\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mlxml\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 7\u001b[0m \u001b[38;5;66;03m#soup = BeautifulSoup(\"<html>a web page</html>\", 'html.parser')\u001b[39;00m\n\u001b[1;32m 8\u001b[0m data \u001b[38;5;241m=\u001b[39m soup\u001b[38;5;241m.\u001b[39mfind_all(\u001b[38;5;28mid\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mmaster-table\u001b[39m\u001b[38;5;124m'\u001b[39m)\n",
|
|
"File \u001b[0;32m~/data/lib/python3.10/site-packages/bs4/__init__.py:250\u001b[0m, in \u001b[0;36mBeautifulSoup.__init__\u001b[0;34m(self, markup, features, builder, parse_only, from_encoding, exclude_encodings, element_classes, **kwargs)\u001b[0m\n\u001b[1;32m 248\u001b[0m builder_class \u001b[38;5;241m=\u001b[39m builder_registry\u001b[38;5;241m.\u001b[39mlookup(\u001b[38;5;241m*\u001b[39mfeatures)\n\u001b[1;32m 249\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m builder_class \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 250\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m FeatureNotFound(\n\u001b[1;32m 251\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCouldn\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mt find a tree builder with the features you \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 252\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrequested: \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m. Do you need to install a parser library?\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 253\u001b[0m \u001b[38;5;241m%\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(features))\n\u001b[1;32m 255\u001b[0m \u001b[38;5;66;03m# At this point either we have a TreeBuilder instance in\u001b[39;00m\n\u001b[1;32m 256\u001b[0m \u001b[38;5;66;03m# builder, or we have a builder_class that we can instantiate\u001b[39;00m\n\u001b[1;32m 257\u001b[0m \u001b[38;5;66;03m# with the remaining **kwargs.\u001b[39;00m\n\u001b[1;32m 258\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m builder \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
|
|
"\u001b[0;31mFeatureNotFound\u001b[0m: Couldn't find a tree builder with the features you requested: lxml. Do you need to install a parser library?"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"from bs4 import BeautifulSoup\n",
|
|
"import re\n",
|
|
"\n",
|
|
"with open(\"data/京剧剧本 - 总目.html\") as fp:\n",
|
|
" soup = BeautifulSoup(fp, 'lxml')\n",
|
|
"\n",
|
|
"#soup = BeautifulSoup(\"<html>a web page</html>\", 'html.parser')\n",
|
|
"data = soup.find_all(id = 'master-table')\n",
|
|
"for item in data:\n",
|
|
" list1 = item.find_all(class_=re.compile(\"row_\"))\n",
|
|
" #list1 = item.find_all(lambda tag: tag.get('class') == ['row_c'] or tag.get('class') == ['row_a'])\n",
|
|
" #print(list1)\n",
|
|
" for tr in list1:\n",
|
|
" #print(tr)\n",
|
|
" if tr.a:\n",
|
|
" print(tr.a.get('href'),tr.b.text)\n",
|
|
" #print('end')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "fad890ec-010e-4afa-b6a6-5427b4f65f95",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3 (ipykernel)",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.10.12"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|