jupyter/数据采集.ipynb

中国戏考数据采集

In [ ]:

import requests
import time
from bs4 import BeautifulSoup
import pymongo
import re

#m_mongo = {}
#requests.packages.urllib3.disable_warnings()
#requests.packages.urllib3.util.ssl_.DEFAULT_CIPHERS += 'HIGH:!DH:!aNULL'
#myclient = pymongo.MongoClient('mongodb://localhost:27017/')
#mydb = myclient["gaokao"]
#mycol = mydb["news"]
url = 'data/京剧剧本 - 总目.html'
a_url = 'https://scripts.xikao.com'
#strhtml = requests.get(url)
strhtml = 'data/京剧剧本 - 总目.html'
#strhtml.encoding = 'utf8'
soup = BeautifulSoup(strhtml,'html.parser')
data = soup.select('#master-table')
for item in data:
    print(item)
    print('\br')

In [139]:

from bs4 import BeautifulSoup
import re

with open("data/京剧剧本 - 总目.html") as fp:
    soup = BeautifulSoup(fp, 'lxml')

#soup = BeautifulSoup("<html>a web page</html>", 'html.parser')
data = soup.find_all(id = 'master-table')
for item in data:
    list1  = item.find_all(class_=re.compile("row_"))
    #list1  = item.find_all(lambda tag: tag.get('class') == ['row_c']  or tag.get('class') == ['row_a'])
    #print(list1)
    for tr in list1:
        #print(tr)
        if tr.a:
            print(tr.a.get('href'),tr.b.text)
            #print('end')

---------------------------------------------------------------------------
FeatureNotFound                           Traceback (most recent call last)
Cell In[139], line 5
      2 import re
      4 with open("data/京剧剧本 - 总目.html") as fp:
----> 5     soup = BeautifulSoup(fp, 'lxml')
      7 #soup = BeautifulSoup("<html>a web page</html>", 'html.parser')
      8 data = soup.find_all(id = 'master-table')

File ~/data/lib/python3.10/site-packages/bs4/__init__.py:250, in BeautifulSoup.__init__(self, markup, features, builder, parse_only, from_encoding, exclude_encodings, element_classes, **kwargs)
    248     builder_class = builder_registry.lookup(*features)
    249     if builder_class is None:
--> 250         raise FeatureNotFound(
    251             "Couldn't find a tree builder with the features you "
    252             "requested: %s. Do you need to install a parser library?"
    253             % ",".join(features))
    255 # At this point either we have a TreeBuilder instance in
    256 # builder, or we have a builder_class that we can instantiate
    257 # with the remaining **kwargs.
    258 if builder is None:

FeatureNotFound: Couldn't find a tree builder with the features you requested: lxml. Do you need to install a parser library?

In [ ]: