diff --git a/数据采集.ipynb b/数据采集.ipynb index 613bef2..67d112d 100644 --- a/数据采集.ipynb +++ b/数据采集.ipynb @@ -43,62 +43,56 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "66a2d507-1232-4cb3-abbd-09fa7c4df315", + "execution_count": 139, + "id": "1b3b3b53-d9c1-4e80-8e25-19165d2921e9", "metadata": { + "execution": { + "iopub.execute_input": "2024-04-04T15:44:59.306388Z", + "iopub.status.busy": "2024-04-04T15:44:59.305906Z", + "iopub.status.idle": "2024-04-04T15:44:59.370430Z", + "shell.execute_reply": "2024-04-04T15:44:59.368843Z", + "shell.execute_reply.started": "2024-04-04T15:44:59.306350Z" + }, "tags": [] }, - "outputs": [], + "outputs": [ + { + "ename": "FeatureNotFound", + "evalue": "Couldn't find a tree builder with the features you requested: lxml. Do you need to install a parser library?", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mFeatureNotFound\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[139], line 5\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mre\u001b[39;00m\n\u001b[1;32m 4\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mopen\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdata/京剧剧本 - 总目.html\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;28;01mas\u001b[39;00m fp:\n\u001b[0;32m----> 5\u001b[0m soup \u001b[38;5;241m=\u001b[39m \u001b[43mBeautifulSoup\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfp\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mlxml\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 7\u001b[0m \u001b[38;5;66;03m#soup = BeautifulSoup(\"a web page\", 'html.parser')\u001b[39;00m\n\u001b[1;32m 8\u001b[0m data \u001b[38;5;241m=\u001b[39m soup\u001b[38;5;241m.\u001b[39mfind_all(\u001b[38;5;28mid\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mmaster-table\u001b[39m\u001b[38;5;124m'\u001b[39m)\n", + "File \u001b[0;32m~/data/lib/python3.10/site-packages/bs4/__init__.py:250\u001b[0m, in \u001b[0;36mBeautifulSoup.__init__\u001b[0;34m(self, markup, features, builder, parse_only, from_encoding, exclude_encodings, element_classes, **kwargs)\u001b[0m\n\u001b[1;32m 248\u001b[0m builder_class \u001b[38;5;241m=\u001b[39m builder_registry\u001b[38;5;241m.\u001b[39mlookup(\u001b[38;5;241m*\u001b[39mfeatures)\n\u001b[1;32m 249\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m builder_class \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 250\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m FeatureNotFound(\n\u001b[1;32m 251\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCouldn\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mt find a tree builder with the features you \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 252\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrequested: \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m. Do you need to install a parser library?\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 253\u001b[0m \u001b[38;5;241m%\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(features))\n\u001b[1;32m 255\u001b[0m \u001b[38;5;66;03m# At this point either we have a TreeBuilder instance in\u001b[39;00m\n\u001b[1;32m 256\u001b[0m \u001b[38;5;66;03m# builder, or we have a builder_class that we can instantiate\u001b[39;00m\n\u001b[1;32m 257\u001b[0m \u001b[38;5;66;03m# with the remaining **kwargs.\u001b[39;00m\n\u001b[1;32m 258\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m builder \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n", + "\u001b[0;31mFeatureNotFound\u001b[0m: Couldn't find a tree builder with the features you requested: lxml. Do you need to install a parser library?" + ] + } + ], "source": [ "from bs4 import BeautifulSoup\n", + "import re\n", "\n", "with open(\"data/京剧剧本 - 总目.html\") as fp:\n", - " soup = BeautifulSoup(fp, 'html.parser')\n", + " soup = BeautifulSoup(fp, 'lxml')\n", "\n", "#soup = BeautifulSoup(\"a web page\", 'html.parser')\n", "data = soup.find_all(id = 'master-table')\n", "for item in data:\n", - " list1 = item.find_all(attrs = {'class':'row_c'})\n", - " \n", - " #print(list1)\n", - " for tr in list1:\n", - " print(tr)\n", - " #if tr.b:\n", - " # print(tr.b.text)\n", - " # print('end')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5cc74584-aadc-488b-9426-228838d9c3db", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "from bs4 import BeautifulSoup\n", - "\n", - "with open(\"data/京剧剧本 - 总目.html\") as fp:\n", - " soup = BeautifulSoup(fp, 'html.parser')\n", - "\n", - "#soup = BeautifulSoup(\"a web page\", 'html.parser')\n", - "data = soup.find_all(id = 'master-table')\n", - "for item in data:\n", - " list1 = item.find_all(lambda tag: tag.get('class') == ['row_c'] or tag.get('class') == ['row_a'])\n", - " \n", + " list1 = item.find_all(class_=re.compile(\"row_\"))\n", + " #list1 = item.find_all(lambda tag: tag.get('class') == ['row_c'] or tag.get('class') == ['row_a'])\n", " #print(list1)\n", " for tr in list1:\n", " #print(tr)\n", " if tr.a:\n", " print(tr.a.get('href'),tr.b.text)\n", - " print('end')" + " #print('end')" ] }, { "cell_type": "code", "execution_count": null, - "id": "1b3b3b53-d9c1-4e80-8e25-19165d2921e9", + "id": "fad890ec-010e-4afa-b6a6-5427b4f65f95", "metadata": {}, "outputs": [], "source": []