44 KiB
44 KiB
In [ ]:
import json
import re
m_xx = dict()
fl_name = 'teshe_2012.txt'
with open(fl_name,'r') as fl:
for line in fl:
if line.split():
l = line.split('\t',1)
#print(l[0]+'--'+re.sub('\n', '', l[1]))
m_len = len(l[0])
m_name = re.sub('\n', '', l[1])
if m_len == 2:
m_xx.setdefault(l[0],{})
ll = m_name.split('：',1)
m_xx[l[0]]['name'] = ll[1]
elif m_len == 4:
m_xx[l[0][:2]].setdefault(l[0],{})
m_xx[l[0][:2]][l[0]]['name'] = m_name
else:
m_xx[l[0][:2]][l[0][:4]].setdefault(l[0],{})
m_xx[l[0][:2]][l[0][:4]][l[0]]['code'] = l[0]
m_xx[l[0][:2]][l[0][:4]][l[0]]['name'] = m_name
#print(m_xx)
with open('teshe_2012.json','w') as fl1:
json.dump(m_xx,fl1,ensure_ascii=False) In [ ]:
import json
import re
m_teshe = dict()
m_new = dict()
fl_name = 'jiben_2012.json'
fl_name1 = 'teshe_2012.json'
with open(fl_name,'r') as fl:
m_new = json.load(fl)
for k,v in m_new.items():
for k1,v1 in v.items():
if k1 !='name':
for k2,v2 in m_new[k][k1].items():
if k2 !='name':
m_new[k][k1][k2]['lb'] = '基本'
with open(fl_name1,'r') as fl1:
m_teshe = json.load(fl1)
for k,v in m_teshe.items():
for k1,v1 in v.items():
if k1 !='name':
for k2,v2 in m_teshe[k][k1].items():
if k2 !='name':
m_teshe[k][k1][k2]['lb'] = '特设'
m_new[k][k1][k2] = m_teshe[k][k1][k2]
with open('hebing_2012.json','w') as fl2:
json.dump(m_new,fl2,ensure_ascii=False) In [ ]:
import json
import openpyxl
import pymongo
myclient = pymongo.MongoClient('mongodb://localhost:27017/')
mydb = myclient["gaokao"]
mycol = mydb["zhuanyemulu"]
m_xx = dict()
fl_name = '普通高等学校本科专业目录.xlsx'
wb = openpyxl.load_workbook(fl_name)
sheet = wb.active
for n in range(2,sheet.max_row+1):
m_code = sheet.cell(n,4).value
m_name = sheet.cell(n,5).value
m_k1 = m_code[:2]
m_k2 = m_code[:4]
m_xx.setdefault(m_k1,{})
m_xx[m_k1]['name'] = sheet.cell(n,2).value
m_xx[m_k1]['level'] = '门类'
m_xx[m_k1].setdefault(m_k2,{})
m_xx[m_k1][m_k2]['name'] = sheet.cell(n,3).value
m_xx[m_k1][m_k2]['level'] ='专业类'
m_xx[m_k1][m_k2].setdefault(m_code,{})
m_xx[m_k1][m_k2][m_code]['name'] = sheet.cell(n,5).value
m_xx[m_k1][m_k2][m_code]['shouyumenlei'] = sheet.cell(n,6).value
m_xx[m_k1][m_k2][m_code]['xiuyenianxian'] = sheet.cell(n,7).value
m_nf = sheet.cell(n,8).value
if m_nf.strip() != '':
m_xx[m_k1][m_k2][m_code]['zengshenianfen'] = sheet.cell(n,8).value
#print(m_xx)
'''
# 导入MongoDB数据库
for k,v in m_xx.items():
m_mongo = {}
m_mongo['code'] = k
for k1,v1 in v.items():
m_mongo[k1] = v1
mycol.insert_one(m_mongo)
'''
with open('2020.json','w') as fl2:
json.dump(m_xx,fl2,ensure_ascii=False)
In [1]:
import pymysql
import pymongo
myclient = pymongo.MongoClient('mongodb://localhost:27017/')
mydb = myclient["gaokao"]
mycol = mydb["college"]
m_xx = {}
m_mongo = {}
db = pymysql.connect("localhost","root","songyi","gaokao" )
cursor = db.cursor()
sql = 'SELECT college,SUM(num_dispense),min(num_min),max(RANK_min) FROM admission_2020 GROUP BY college'
cursor.execute(sql)
results = cursor.fetchall()
for result in results:
m_xx.clear()
m_code = result[0]
m_nian ='2020'
m_lb = 'z'
m_xx.setdefault(m_nian,{})
m_xx[m_nian].setdefault(m_lb,{})
m_xx[m_nian][m_lb] ['name']= '综合'
m_xx[m_nian][m_lb] ['dispense']= int(result[1])
m_xx[m_nian][m_lb] ['num_min']= result[2]
m_xx[m_nian][m_lb] ['rank_min']= result[3]
#print(m_code,m_xx)
m_mongo.clear()
#m_mongo['admission'] = m_xx
#m_mongo['discipline'] = x['discipline']
myquery = {'code':m_code}
m_new = {"$set":{'admission':m_xx}}
mycol.update_one(myquery,m_new)
In [2]:
import pymysql
import pymongo
myclient = pymongo.MongoClient('mongodb://localhost:27017/')
mydb = myclient["gaokao"]
mycol = mydb["college"]
m_lqrs = {} #录取人数
m_xx = {}
dict_lb ={'z':'综合','l':'理科','w':'文科'}
m_mongo = {}
db = pymysql.connect("localhost","root","songyi","gaokao" )
cursor = db.cursor()
sql = 'SELECT code,year,category,sum(num_act) FROM draft group BY code,year,category ORDER BY code,year,category'
cursor.execute(sql)
results = cursor.fetchall()
#生成录取人数字典
for result in results:
m_lqrs.setdefault(result[0],{})
m_lqrs[result[0]].setdefault(result[1],{})
m_lqrs[result[0]][result[1]][result[2]] = result[3]
#print(m_lqrs)
sql = 'SELECT college,SUM(num_dispense),min(num_min),max(RANK_min) FROM admission_2020 GROUP BY college'
cursor.execute(sql)
results = cursor.fetchall()
for result in results:
m_code = result[0]
m_nian ='2020'
m_lb = 'z'
m_xx.setdefault(m_code,{})
m_xx[m_code].setdefault(m_nian,{})
m_xx[m_code][m_nian].setdefault(m_lb,{})
m_xx[m_code][m_nian][m_lb] ['name']= '综合'
m_xx[m_code][m_nian][m_lb] ['dispense']= int(result[1])
m_xx[m_code][m_nian][m_lb] ['num_min']= result[2]
m_xx[m_code][m_nian][m_lb] ['rank_min']= result[3]
sql = 'SELECT college,nian,lb,MIN(num_min),max(rank_min) FROM admission where nian not IN ("2020") group BY college,nian,lb ORDER BY college,nian desc,lb'
cursor.execute(sql)
results = cursor.fetchall()
for result in results:
#m_xx.clear()
m_code = result[0]
m_nian = result[1]
m_lb = result[2]
m_xx.setdefault(m_code,{})
m_xx[m_code].setdefault(m_nian,{})
m_xx[m_code][m_nian].setdefault(m_lb,{})
m_xx[m_code][m_nian][m_lb] ['name']= dict_lb[m_lb]
if m_nian in m_lqrs[m_code]:
m_xx[m_code][m_nian][m_lb] ['dispense']= int(m_lqrs[m_code][m_nian][m_lb])
m_xx[m_code][m_nian][m_lb] ['num_min']= result[3]
m_xx[m_code][m_nian][m_lb] ['rank_min']= result[4]
#print(m_xx)
for k,v in m_xx.items():
#m_item ='admission.'+m_nian
m_mongo.clear()
myquery = {'code':k}
m_new = {"$set":{'admission':v}}
mycol.update_one(myquery,m_new)
#print(k,v)In [ ]:
import json
import openpyxl
import pymongo
myclient = pymongo.MongoClient('mongodb://localhost:27017/')
mydb = myclient["gaokao"]
mycol = mydb["document"]
m_mongo = {}
fl_name = '全国普通高等学校名单.xlsx'
m_title = fl_name.split('.')[0]
m_type = fl_name.split('.')[1]
#print(m_title,m_type)
wb = openpyxl.load_workbook(fl_name)
sheet = wb.active
m_head = []
m_body = []
for cell in sheet[2]:
m_head.append(cell.value)
#for n in range(2,sheet.max_row+1):
#print(m_head)
for n in range(3,sheet.max_row+1):
m_cell = []
for i in range(1,len(m_head)+1):
m_cell.append(sheet.cell(n,i).value)
m_body.append(m_cell)
m_mongo['title'] = m_title
m_mongo['head'] = m_head
m_mongo['body'] = m_body
m_mongo['beizhu'] = '截至2020年6月30日'
mycol.insert_one(m_mongo)
print('ok!')In [ ]:
import json
import openpyxl
import pymongo
myclient = pymongo.MongoClient('mongodb://localhost:27017/')
mydb = myclient["gaokao"]
mycol = mydb["gaoxiaomingdan"]
m_mongo = {}
fl_name = '全国普通高等学校名单.xlsx'
wb = openpyxl.load_workbook(fl_name)
sheet = wb.active
for n in range(3,sheet.max_row+1):
m_mongo ={}
m_mongo['name'] = sheet.cell(n,2).value
m_mongo['code'] = str(sheet.cell(n,3).value)
m_mongo['charge'] = sheet.cell(n,4).value
m_mongo['city'] = sheet.cell(n,5).value
m_mongo['grade'] = sheet.cell(n,6).value
m_mongo['note'] = sheet.cell(n,7).value
mycol.insert_one(m_mongo)
print('ok!')In [ ]:
import pymysql
import pymongo
import openpyxl
myclient = pymongo.MongoClient('mongodb://localhost:27017/')
mydb = myclient["gaokao"]
mycol = mydb["college"]
db = pymysql.connect("localhost","root","songyi","gaokao" )
cursor = db.cursor()
m_college = {}
sql = "select code,name from college"
cursor.execute(sql)
results = cursor.fetchall()
for result in results:
m_college[result[1]] =result[0]
fl_name = '全国普通高等学校名单.xlsx'
wb = openpyxl.load_workbook(fl_name)
sheet = wb.active
for n in range(3,sheet.max_row+1):
m_mongo ={}
if sheet.cell(n,2).value in m_college.keys():
m_mongo['code'] = m_college[sheet.cell(n,2).value]
m_mongo['name'] = sheet.cell(n,2).value
m_mongo['id_code'] = str(sheet.cell(n,3).value)
m_mongo['charge'] = sheet.cell(n,4).value
m_mongo['city'] = sheet.cell(n,5).value
m_mongo['grade'] = sheet.cell(n,6).value
m_mongo['note'] = sheet.cell(n,7).value
mycol.insert_one(m_mongo)
#print(m_mongo)In [45]:
import pymysql
import pymongo
import openpyxl
myclient = pymongo.MongoClient('mongodb://localhost:27017/')
mydb = myclient["gaokao"]
mycol = mydb["college"]
db = pymysql.connect("localhost","root","songyi","gaokao" )
cursor = db.cursor()
m_college = {}
sql = "select code,name from college"
cursor.execute(sql)
results = cursor.fetchall()
l_col = ['哈尔滨理工大学','山东大学','山东科技大学','青岛科技大学','济南大学','青岛理工大学','山东师范大学','山东财经大学','青岛大学']
for result in results:
m_code = result[0]
m_name = result[1]
myquery = {'code':m_code}
#x = mycol.find_one(myquery)
if not mycol.find_one(myquery) :
print(m_code,m_name)
#if mydoc.count ==0:
# print(m_name=' 不存在！')
Y007 山东财经大学 D687 中国传媒大学南广学院 D682 西安工业大学北方信息工程学院 Y002 山东科技大学 Y060 山东大学威海分校（走读） Y061 济南大学（走读） Y063 山东财经大学（走读）
In [43]:
import pymysql
import pymongo
import openpyxl
myclient = pymongo.MongoClient('mongodb://localhost:27017/')
mydb = myclient["gaokao"]
mycol = mydb["college"]
m_mongo = {"code":'Y022','name':'山东科技大学','charge':'山东省','city':'青岛市','grade':'本科','note':'校企合作，与青软实训教育科技股份有限公司'}
mycol.insert_one(m_mongo)
print(m_mongo['name'] + '导入成功！')
山东科技大学导入成功！
In [ ]:
import json
import pymysql
db = pymysql.connect("localhost","root","songyi","gaokao" )
cursor = db.cursor()
sql = "select code,name from college"
cursor.execute(sql)
results = cursor.fetchall()
for result in results:
m_xx = []
m_code = result[0]
sql1 = 'select xx,zy_dm,zdf,zdwc,pjf,nian,lb from digao_1719 where xx=%s order by nian'
cursor.execute(sql1,m_code)
ad_results = cursor.fetchall()
for ad_result in ad_results:
m_xx.append(tuple(ad_result))
sql = "insert into admission (college,speciality,num_min,rank_min,num_avg,nian,lb) values(%s,%s,%s,%s,%s,%s,%s)"
try:
cursor.executemany(sql,m_xx)
db.commit()
print(result[1]+"已添加！")
except:
# 如果发生错误则回滚
print("error!")
db.rollback()
db.close()
In [ ]:
import json
import pymysql
m_col = []
filename = './data/2020年未招生学校.json'
with open(filename,'r') as fl:
m_xx = json.load(fl)
for k,v in m_xx.items():
m_col.append((k,v['name']))
db = pymysql.connect("localhost","root","songyi","gaokao" )
cursor = db.cursor()
sql = "insert into college (code,name) values(%s,%s)"
try:
cursor.executemany(sql,m_col)
#db.commit()
print("ok!")
except:
# 如果发生错误则回滚
print("error!")
db.rollback()
db.close()In [ ]:
import json
import pymysql
db = pymysql.connect("localhost","root","songyi","gaokao" )
cursor = db.cursor()
sql = 'select xx, zy_dm,zdf from digao_1719 where xx="A001"'
cursor.execute(sql)
results = cursor.fetchall()
print(list(results))In [ ]:
import openpyxl
import pymysql
import json
m_xx = []
db = pymysql.connect("localhost","root","songyi","gaokao" )
cursor = db.cursor()
wb = openpyxl.load_workbook('./data/2020一分一段表.xlsx')
sheet = wb.active
i = 1
for n in range(4,sheet.max_row):
m_score = sheet.cell(n,1).value
m_num = sheet.cell(n,14).value
m_sum = sheet.cell(n,15).value
m_max = m_sum - m_num + 1
m_xx.append((m_score,m_num,m_max,m_sum,'z','2020'))
#print(m_xx)
sql = 'insert into fenduan(score,per_num,max_rank,min_rank,category,nian) values (%s,%s,%s,%s,%s,%s)'
try:
cursor.executemany(sql,m_xx)
db.commit()
print("ok!")
except:
# 如果发生错误则回滚
print("error!")
db.rollback()
db.close()
In [ ]:
import pymysql
import json
m_xx = {}
db = pymysql.connect("localhost","root","songyi","gaokao" )
cursor = db.cursor()
sql = 'SELECT * FROM fenduan AS a ORDER BY nian,category'
cursor.execute(sql)
results = cursor.fetchall()
for result in results:
m_year = result[6]
m_xx.setdefault(m_year,{})
m_xx[m_year].setdefault(result[5],{})
m_xx[m_year][result[5]].setdefault(result[1],{})
m_xx[m_year][result[5]][result[1]]['num_person'] = result[2]
m_xx[m_year][result[5]][result[1]]['max_rank'] = result[3]
m_xx[m_year][result[5]][result[1]]['min_rank'] = result[4]
fl_name = 'data/17-20年一分一段表.json'
with open(fl_name,'w') as fl:
json.dump(m_xx,fl,ensure_ascii=False)
print('ok!')
In [ ]:
import pymysql
import pymongo
myclient = pymongo.MongoClient('mongodb://localhost:27017/')
mydb = myclient["gaokao"]
mycol = mydb["fenduan"]
m_xx = {}
db = pymysql.connect("localhost","root","songyi","gaokao" )
cursor = db.cursor()
sql = 'SELECT * FROM fenduan AS a ORDER BY nian,category'
cursor.execute(sql)
results = cursor.fetchall()
for result in results:
m_year = result[6]
m_xx.setdefault(m_year,{})
m_xx[m_year].setdefault(result[5],{})
m_fenshu = str(result[1])
m_xx[m_year][result[5]].setdefault(m_fenshu,{})
m_xx[m_year][result[5]][m_fenshu]['num_person'] = result[2]
m_xx[m_year][result[5]][m_fenshu]['max_rank'] = result[3]
m_xx[m_year][result[5]][m_fenshu]['min_rank'] = result[4]
for k,v in m_xx.items():
m_mongo = {}
m_mongo.setdefault(k,{})
m_mongo[k] = v
x = mycol.insert_one(m_mongo)
#print('ok!')In [ ]:
import pymysql
import json
db = pymysql.connect("localhost","root","songyi","gaokao" )
cursor = db.cursor()
sql = 'SELECT a.college,sum(a.plan),SUM(a.num_dispense),MIN(a.num_min),max(a.rank_min) FROM admission_2020 AS a GROUP BY a.college'
cursor.execute(sql)
results = cursor.fetchall()
#print(results)
sql = "insert into admission_coll (college,num_plan,num_act,min_score,min_rank) values(%s,%s,%s,%s,%s)"
try:
cursor.executemany(sql,results)
db.commit()
print("ok!")
except:
# 如果发生错误则回滚
print("error!")
db.rollback()
db.close()In [ ]:
import pymysql
import pymongo
myclient = pymongo.MongoClient('mongodb://localhost:27017/')
mydb = myclient["gaokao"]
mycol = mydb["hz_luqu"]
m_xx = {}
m_mongo = {}
db = pymysql.connect("localhost","root","songyi","gaokao" )
cursor = db.cursor()
sql = 'SELECT a.code,b.name,a.year,a.num_plan,a.num_act,a.max_score,a.min_score,a.category FROM draft AS a \
left join college AS b ON a.code=b.code AND b.name is NOT NULL order BY a.code,a.year'
cursor.execute(sql)
results = cursor.fetchall()
for result in results:
m_code = result[0]
m_xx.setdefault(m_code,{})
m_xx[m_code]['code'] = result[0]
m_xx[m_code]['name'] = result[1]
m_xx[m_code].setdefault(result[2],{})
m_xx[m_code][result[2]].setdefault(result[7],{})
m_xx[m_code][result[2]][result[7]]['num_plan'] = result[3]
m_xx[m_code][result[2]][result[7]]['num_act'] = result[4]
m_xx[m_code][result[2]][result[7]]['max_score'] = result[5]
m_xx[m_code][result[2]][result[7]]['min_score'] = result[6]
for k,v in m_xx.items():
m_mongo[k] = v
x = mycol.insert_one(m_mongo[k])
print(m_mongo[k])
In [ ]:
import pymysql
import json
m_xx = {}
m_mongo = {}
db = pymysql.connect("localhost","root","songyi","gaokao" )
cursor = db.cursor()
sql = 'SELECT a.code,b.name,a.year,a.num_plan,a.num_act,a.max_score,a.min_score,a.category FROM draft AS a \
left join college AS b ON a.code=b.code AND b.name is NOT NULL order BY a.code,a.year'
cursor.execute(sql)
results = cursor.fetchall()
for result in results:
m_code = result[0]
m_xx.setdefault(m_code,{})
m_xx[m_code]['code'] = result[0]
m_xx[m_code]['name'] = result[1]
m_xx[m_code].setdefault(result[2],{})
m_xx[m_code][result[2]].setdefault(result[7],{})
m_xx[m_code][result[2]][result[7]]['num_plan'] = result[3]
m_xx[m_code][result[2]][result[7]]['num_act'] = result[4]
m_xx[m_code][result[2]][result[7]]['max_score'] = result[5]
m_xx[m_code][result[2]][result[7]]['min_score'] = result[6]
fl_name = 'data/15-19年高校录取汇总情况.json'
with open(fl_name,'w') as fl:
json.dump(m_xx,fl,ensure_ascii=False)
#print(json.dumps(m_xx,ensure_ascii=False))
#print(m_xx)
In [ ]:
In [ ]:
import requests
import time
from bs4 import BeautifulSoup
import pymongo
myclient = pymongo.MongoClient('mongodb://localhost:27017/')
mydb = myclient["gaokao"]
mycol = mydb["news"]
#web_xx = {}
n = 0
for i in range(1,11):
url = 'http://edu.sina.com.cn/other/roll.d.html?cat=80459&page={}&page_size=30'.format(str(i))
strhtml = requests.get(url)
strhtml.encoding = 'utf8'
soup = BeautifulSoup(strhtml.text,'lxml')
#data = strhtml.text
#data1 = data.split("\r")
data = soup.select('#Main > div.listBlk > ul > li > a')
#data = soup.select('#artibody')
for item in data:
web_xx.clear()
n += 1
m_num = 'sina'+str(n).rjust(6, '0')
web_xx.setdefault(m_num,{})
web_xx[m_num]['title'] = item.get_text()
c_url = item.get('href')
web_xx[m_num]['url'] = c_url
content = requests.get(c_url)
content.encoding = 'utf8'
soup_content = BeautifulSoup(content.text)
data1 = soup_content.select('#artibody')
for item1 in data1:
web_xx[m_num]['content'] = item1.get_text()
time.sleep(3)
mycol.insert_one(web_xx)
print(m_num+'入库成功！')In [ ]:
import requests
import time
from bs4 import BeautifulSoup
import pymongo
import re
m_mongo = {}
requests.packages.urllib3.disable_warnings()
requests.packages.urllib3.util.ssl_.DEFAULT_CIPHERS += 'HIGH:!DH:!aNULL'
myclient = pymongo.MongoClient('mongodb://localhost:27017/')
mydb = myclient["gaokao"]
mycol = mydb["syl_jianshexueke"]
url = 'http://daxue.eol.cn/syl.shtml'
strhtml = requests.get(url)
strhtml.encoding = 'utf8'
soup = BeautifulSoup(strhtml.text,'lxml')
data = soup.select('body > div.container.box > div.con > div:nth-child(4) > table > tbody > tr')
i = 0
for item in data:
m_mongo.clear()
data1 = item.select('td')
ss = re.sub('\n', '', data1[1].text)
#print(ss.split('、'))
m_mongo['college'] = data1[0].text
m_mongo['discipline'] = ss.split('、')
mycol.insert_one(m_mongo)
print(data1[0].text + '导入成功！')
In [ ]:
import requests
import time
from bs4 import BeautifulSoup
import pymongo
import re
m_mongo = {}
requests.packages.urllib3.disable_warnings()
requests.packages.urllib3.util.ssl_.DEFAULT_CIPHERS += 'HIGH:!DH:!aNULL'
myclient = pymongo.MongoClient('mongodb://localhost:27017/')
mydb = myclient["gaokao"]
mycol = mydb["syl_college"]
url = 'http://daxue.eol.cn/syl.shtml'
strhtml = requests.get(url)
strhtml.encoding = 'utf8'
soup = BeautifulSoup(strhtml.text,'lxml')
data = soup.select('body > div.container.box > div.con > div:nth-child(2) > table > tbody:nth-child(3) > tr')
m_name = []
#for item in data:
# m_mongo.clear()
for n in range(1,len(data)):
item = data[n].select('td')
ss = re.sub('\n', ' ', data[n].text)
m_name = m_name + ss.strip().split(' ')
for col in m_name:
m_mongo.clear()
m_mongo['name'] = col
m_mongo['level'] = 'B'
mycol.insert_one(m_mongo) In [ ]:
import pymongo
import re
myclient = pymongo.MongoClient('mongodb://localhost:27017/')
mydb = myclient["gaokao"]
mycol = mydb["college_syl"]
mycol1 = mydb["syl_college"]
mycol2 = mydb["syl_jianshexueke"]
m_mongo = {}
for x in mycol1.find({},{'_id':0}):
m_mongo.clear()
m_mongo['college'] = x['name']
m_mongo['level'] = x['level']
myquery = {'college':x['name']}
for x1 in mycol2.find(myquery,{'_id':0}):
m_mongo['discipline'] = x1['discipline']
mycol.insert_one(m_mongo)
In [ ]:
import requests
import time
from bs4 import BeautifulSoup
import pymongo
import re
pattern = re.compile(r'\d+')
m_mongo = {}
requests.packages.urllib3.disable_warnings()
requests.packages.urllib3.util.ssl_.DEFAULT_CIPHERS += 'HIGH:!DH:!aNULL'
myclient = pymongo.MongoClient('mongodb://localhost:27017/')
mydb = myclient["gaokao"]
mycol = mydb["syl_jianshexueke"]
url = 'http://daxue.eol.cn/syl.shtml'
strhtml = requests.get(url)
strhtml.encoding = 'utf8'
soup = BeautifulSoup(strhtml.text,'lxml')
data = soup.select('body > div.container.box > div.con > div:nth-child(2) > table > tbody > tr > td > a')
i = 0
for item in data:
m_url = item.get('href')
mm =re.search('\d+',m_url).group(0)
m_name = item.text
url1 = 'http://192.168.3.100:8050/render.html?https://gkcx.eol.cn/school/%s/introDetails' %mm
strhtml = requests.get(url1)
time.sleep(10)
soup = BeautifulSoup(strhtml.text,'lxml')
data1 = soup.select('#root > div > div > div > div > div > div > div.main > div > div.tuiji_left > div.intro_details_content')
print(url1,m_name)
print(data1)In [ ]:
import json
import openpyxl
import pymongo
myclient = pymongo.MongoClient('mongodb://localhost:27017/')
mydb = myclient["gaokao"]
mycol = mydb["college_211"]
m_mon = {}
fl_name = './data/985.xlsx'
wb = openpyxl.load_workbook(fl_name)
sheet = wb['211']
for n in range(1,sheet.max_row+1):
m_xx = []
m_mon.clear()
m_mon['college'] = sheet.cell(n,1).value
m_zy = sheet.cell(n,2).value
m_mon['discipline'] = m_zy.split('、')
mycol.insert_one(m_mon) In [ ]:
import pymongo
import re
myclient = pymongo.MongoClient('mongodb://localhost:27017/')
mydb = myclient["gaokao"]
mycol = mydb["college"]
mycol1 = mydb["college_985"]
mycol2 = mydb["college_syl"]
m_mongo = {}
for x in mycol2.find({},{'_id':0}):
m_mongo.clear()
m_mongo['level'] = x['level']
m_mongo['discipline'] = x['discipline']
myquery = {'name':x['college']}
m_new = {"$set":{'syl':m_mongo}}
mycol.update_one(myquery,m_new)
In [ ]:
import pymongo
import re
myclient = pymongo.MongoClient('mongodb://localhost:27017/')
mydb = myclient["gaokao"]
mycol = mydb["college"]
myquery = {'name':{"$regex":"\（"}}
for x in mycol.find(myquery,{'_id':0}):
#s = x['name'].replace('(','（').replace(')','）')
#myquery1 = {'id_code': x['id_code']}
#m_new = {"$set":{'name':s}}
#mycol.update_one(myquery1,m_new)
print(x)In [ ]:
import requests
import time
from bs4 import BeautifulSoup
import pymongo
import re
m_mongo = {}
requests.packages.urllib3.disable_warnings()
requests.packages.urllib3.util.ssl_.DEFAULT_CIPHERS += 'HIGH:!DH:!aNULL'
myclient = pymongo.MongoClient('mongodb://localhost:27017/')
mydb = myclient["gaokao"]
mycol = mydb["news"]
url = 'http://www.sdzk.cn/NewsList.aspx?BCID=2'
a_url = 'http://www.sdzk.cn/'
strhtml = requests.get(url)
strhtml.encoding = 'utf8'
soup = BeautifulSoup(strhtml.text,'lxml')
data = soup.select('#ctl00_ContentPlaceHolder1_RadAjaxPanel1 > ul > li > a')
m_name = []
for item in data:
m_mongo.clear()
news_url = a_url + item.get('href')
content = requests.get(news_url)
content.encoding = 'utf8'
soup_content = BeautifulSoup(content.text)
data1 = soup_content.select('#form1 > div.contain.laylist > div.fl.laylist-r > div > p')
print(item.text)
m_txt = ''
for item1 in data1:
m_txt += item1.text
m_txt += '\n'
m_mongo['url'] = item.get('href')
m_mongo['source'] = '山东省教育招生考试院'
m_mongo['title'] = item.text[:-10]
m_mongo['date'] = item.text[-10:]
m_mongo['content'] = m_txt
mycol.insert_one(m_mongo) In [ ]:
import requests
url = 'http://localhost:8050/render.html?url=http://gkcx.eol.cn/schoolhtm/schoolTemple/school31.htm&wait=5'
response = requests.get(url)
print(response.text)In [10]:
import requests
import time
from bs4 import BeautifulSoup
import pymongo
myclient = pymongo.MongoClient('mongodb://localhost:27017/')
mydb = myclient["gaokao"]
mycol = mydb["news"]
headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36'}
url = 'https://rdzs.ruc.edu.cn/cms/item/1642.html'
strhtml = requests.get(url,headers = headers)
strhtml.encoding = 'utf8'
soup = BeautifulSoup(strhtml.text,'lxml')
data = strhtml.text
#data1 = data.split("\r")
data = soup.select('body > div.y_tit_box > div > div > div > h1')
#for item1 in data:
# print(item1.get_text())
print(data)[<h1 class="y_tit">中国人民大学2020年强基计划热点问题解答</h1>]
In [ ]:
import pymongo
import re
myclient = pymongo.MongoClient('mongodb://localhost:27017/')
mydb = myclient["gaokao"]
mycol = mydb["news"]
mycol_new = mydb["news_new"]
m_mongo = {}
for x in mycol.find({},{'_id':0}):
for k,v in x.items():
if k[0:4] == 'sina':
m_mongo.clear()
m_mongo['url'] = v['url']
m_mongo['source'] = '新浪高考热讯'
m_mongo['title'] = v['title']
m_mongo['date'] = v['url'].split('/')[4]
m_mongo['content'] = re.sub('\n\n','\n',v['content'])
mycol_new.insert_one(m_mongo)
In [ ]:
import pymongo
import re
myclient = pymongo.MongoClient('mongodb://localhost:27017/')
mydb = myclient["gaokao"]
mycol = mydb["news"]
mycol_new = mydb["news_new"]
myquery = {'source':'山东省教育招生考试院'}
m_mongo = {}
for x in mycol.find(myquery):
m_mongo.clear()
m_mongo['url'] = 'http://www.sdzk.cn/' + x['url']
m_mongo['source'] = '山东省教育招生考试院'
m_mongo['title'] = x['title']
m_mongo['date'] = x['date']
m_mongo['content'] = re.sub('\n\n','\n',x['content'])
mycol_new.insert_one(m_mongo)
#print(m_mongo)In [ ]: