jupyter/高校资料管理.ipynb

重点高校信息管理

强基计划信息管理

录入强基计划信息

In [3]:

import requests
import time
from bs4 import BeautifulSoup
import pymongo


myclient = pymongo.MongoClient('mongodb://localhost:27017/')
mydb = myclient["gaokao"]
mycol = mydb["qiangji"]
m_content = ''
headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36'}
url = 'https://rdzs.ruc.edu.cn/cms/item/1642.html'
strhtml = requests.get(url,headers = headers)
strhtml.encoding = 'utf8'
soup = BeautifulSoup(strhtml.text,'lxml')
#data = strhtml.text
#data1 = data.split("\r")
data = soup.select('body > div.container.content > div > div > div.col-md-9.col-sm-8 > div')
for item1 in data:
    m_content +=item1.get_text()
#print(m_content)
m_title = soup.select('body > div.y_tit_box > div > div > div > h1')
m_name = '中国人民大学'
m_code = 'A002'
m_year = '2020'
myquery = {'code':m_code}
x = mycol.count_documents(myquery)
#print(x)
m_id = time.strftime("%Y%m%d%H%M%S", time.localtime())
if x > 0:    
    m_mg = {}  
    m_mg.setdefault(m_id,{})
    m_mg[m_id]['title'] = m_title[0].text
    m_mg[m_id]['content'] = m_content
    m_mg[m_id]['url'] = url
    mycol.update_one(myquery,{'$push':{m_year:m_mg}})
    print(x,s)
else:
    m_mg = {}
    m_mg['name'] = m_name
    m_mg['code'] = m_code
    m_mg.setdefault(m_year,[])
    m_mg1 = {}
    m_mg1.setdefault(m_id,{})
    m_mg1[m_id]['title'] = m_title[0].text
    m_mg1[m_id]['content'] = m_content
    m_mg1[m_id]['url'] = url
    m_mg[m_year].append(m_mg1)
    mycol.insert_one(m_mg) 

录入强基计划附件

In [73]:

import requests
import time
import pymongo
import os
from gridfs import GridFS

def upLoadFile(file_coll,file_name,data_link):    
    filter_condition = {"filename": os.path.basename(file_name), "url": data_link}
    gridfs_col = GridFS(mydb, collection=file_coll)
    file_ = "0"
    query = {"filename":""}
    query["filename"] = file_name
    if gridfs_col.exists(query):
        print('已经存在该文件')
    else:
        with open(file_name, 'rb') as file_r:
            file_data = file_r.read()
            file_ = gridfs_col.put(data=file_data, **filter_condition)  # 上传到gridfs

            #print(file_)
    return file_ 

myclient = pymongo.MongoClient('mongodb://localhost:27017/')
mydb = myclient["gaokao"]
mycol = mydb["qiangji"]
m_content = ''
url = 'https://www.bkzs.sdu.edu.cn/info/1036/1635.htm'
m_title = ''
m_name = '山东大学'
m_code = 'A422'
m_year = '2020'
myquery = {'code':m_code}
x = mycol.count_documents(myquery)
m_id = time.strftime("%Y%m%d%H%M%S", time.localtime())
m_dir = './data/tmp'
fls=os.listdir(m_dir)
l_fls =[]
for fl in fls:        
    full_path = m_dir+ '/' + fl
    l_fls.append(upLoadFile("document",full_path,""))
m_mg = {}
if x > 0:  
    m_mg.setdefault(m_id,{})
    m_mg[m_id]['title'] = m_title
    m_mg[m_id]['files'] = l_fls
    m_mg[m_id]['url'] = url
    mycol.update_one(myquery,{'$push':{m_year:m_mg}})
else:
    m_mg['name'] = m_name
    m_mg['code'] = m_code
    m_mg.setdefault(m_year,[])
    m_mg1 = {}
    m_mg1.setdefault(m_id,{})
    m_mg1[m_id]['title'] = m_title
    m_mg[m_id]['files'] = l_fls
    m_mg1[m_id]['url'] = url
    m_mg[m_year].append(m_mg1)
    mycol.insert_one(m_mg) 

1 gaokao.qiangji.2020

In [1]:

import time

print(time.strftime("%Y%m%d%H%M%S", time.localtime()))

20210224161356

In [ ]: