236 lines
6.7 KiB
Plaintext
236 lines
6.7 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"# 重点高校信息管理"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## 强基计划信息管理"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"### 录入强基计划信息"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 3,
|
|
"metadata": {
|
|
"execution": {
|
|
"iopub.execute_input": "2020-11-27T07:02:30.612429Z",
|
|
"iopub.status.busy": "2020-11-27T07:02:30.611524Z",
|
|
"iopub.status.idle": "2020-11-27T07:02:31.128055Z",
|
|
"shell.execute_reply": "2020-11-27T07:02:31.124461Z",
|
|
"shell.execute_reply.started": "2020-11-27T07:02:30.612326Z"
|
|
}
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"import requests\n",
|
|
"import time\n",
|
|
"from bs4 import BeautifulSoup\n",
|
|
"import pymongo\n",
|
|
"\n",
|
|
"\n",
|
|
"myclient = pymongo.MongoClient('mongodb://localhost:27017/')\n",
|
|
"mydb = myclient[\"gaokao\"]\n",
|
|
"mycol = mydb[\"qiangji\"]\n",
|
|
"m_content = ''\n",
|
|
"headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36'}\n",
|
|
"url = 'https://rdzs.ruc.edu.cn/cms/item/1642.html'\n",
|
|
"strhtml = requests.get(url,headers = headers)\n",
|
|
"strhtml.encoding = 'utf8'\n",
|
|
"soup = BeautifulSoup(strhtml.text,'lxml')\n",
|
|
"#data = strhtml.text\n",
|
|
"#data1 = data.split(\"\\r\")\n",
|
|
"data = soup.select('body > div.container.content > div > div > div.col-md-9.col-sm-8 > div')\n",
|
|
"for item1 in data:\n",
|
|
" m_content +=item1.get_text()\n",
|
|
"#print(m_content)\n",
|
|
"m_title = soup.select('body > div.y_tit_box > div > div > div > h1')\n",
|
|
"m_name = '中国人民大学'\n",
|
|
"m_code = 'A002'\n",
|
|
"m_year = '2020'\n",
|
|
"myquery = {'code':m_code}\n",
|
|
"x = mycol.count_documents(myquery)\n",
|
|
"#print(x)\n",
|
|
"m_id = time.strftime(\"%Y%m%d%H%M%S\", time.localtime())\n",
|
|
"if x > 0: \n",
|
|
" m_mg = {} \n",
|
|
" m_mg.setdefault(m_id,{})\n",
|
|
" m_mg[m_id]['title'] = m_title[0].text\n",
|
|
" m_mg[m_id]['content'] = m_content\n",
|
|
" m_mg[m_id]['url'] = url\n",
|
|
" mycol.update_one(myquery,{'$push':{m_year:m_mg}})\n",
|
|
" print(x,s)\n",
|
|
"else:\n",
|
|
" m_mg = {}\n",
|
|
" m_mg['name'] = m_name\n",
|
|
" m_mg['code'] = m_code\n",
|
|
" m_mg.setdefault(m_year,[])\n",
|
|
" m_mg1 = {}\n",
|
|
" m_mg1.setdefault(m_id,{})\n",
|
|
" m_mg1[m_id]['title'] = m_title[0].text\n",
|
|
" m_mg1[m_id]['content'] = m_content\n",
|
|
" m_mg1[m_id]['url'] = url\n",
|
|
" m_mg[m_year].append(m_mg1)\n",
|
|
" mycol.insert_one(m_mg) \n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"### 录入强基计划附件"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 73,
|
|
"metadata": {
|
|
"execution": {
|
|
"iopub.execute_input": "2020-11-26T10:35:49.756505Z",
|
|
"iopub.status.busy": "2020-11-26T10:35:49.755575Z",
|
|
"iopub.status.idle": "2020-11-26T10:35:50.001279Z",
|
|
"shell.execute_reply": "2020-11-26T10:35:49.998646Z",
|
|
"shell.execute_reply.started": "2020-11-26T10:35:49.756395Z"
|
|
}
|
|
},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"1 gaokao.qiangji.2020\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"import requests\n",
|
|
"import time\n",
|
|
"import pymongo\n",
|
|
"import os\n",
|
|
"from gridfs import GridFS\n",
|
|
"\n",
|
|
"def upLoadFile(file_coll,file_name,data_link): \n",
|
|
" filter_condition = {\"filename\": os.path.basename(file_name), \"url\": data_link}\n",
|
|
" gridfs_col = GridFS(mydb, collection=file_coll)\n",
|
|
" file_ = \"0\"\n",
|
|
" query = {\"filename\":\"\"}\n",
|
|
" query[\"filename\"] = file_name\n",
|
|
" if gridfs_col.exists(query):\n",
|
|
" print('已经存在该文件')\n",
|
|
" else:\n",
|
|
" with open(file_name, 'rb') as file_r:\n",
|
|
" file_data = file_r.read()\n",
|
|
" file_ = gridfs_col.put(data=file_data, **filter_condition) # 上传到gridfs\n",
|
|
"\n",
|
|
" #print(file_)\n",
|
|
" return file_ \n",
|
|
"\n",
|
|
"myclient = pymongo.MongoClient('mongodb://localhost:27017/')\n",
|
|
"mydb = myclient[\"gaokao\"]\n",
|
|
"mycol = mydb[\"qiangji\"]\n",
|
|
"m_content = ''\n",
|
|
"url = 'https://www.bkzs.sdu.edu.cn/info/1036/1635.htm'\n",
|
|
"m_title = ''\n",
|
|
"m_name = '山东大学'\n",
|
|
"m_code = 'A422'\n",
|
|
"m_year = '2020'\n",
|
|
"myquery = {'code':m_code}\n",
|
|
"x = mycol.count_documents(myquery)\n",
|
|
"m_id = time.strftime(\"%Y%m%d%H%M%S\", time.localtime())\n",
|
|
"m_dir = './data/tmp'\n",
|
|
"fls=os.listdir(m_dir)\n",
|
|
"l_fls =[]\n",
|
|
"for fl in fls: \n",
|
|
" full_path = m_dir+ '/' + fl\n",
|
|
" l_fls.append(upLoadFile(\"document\",full_path,\"\"))\n",
|
|
"m_mg = {}\n",
|
|
"if x > 0: \n",
|
|
" m_mg.setdefault(m_id,{})\n",
|
|
" m_mg[m_id]['title'] = m_title\n",
|
|
" m_mg[m_id]['files'] = l_fls\n",
|
|
" m_mg[m_id]['url'] = url\n",
|
|
" mycol.update_one(myquery,{'$push':{m_year:m_mg}})\n",
|
|
"else:\n",
|
|
" m_mg['name'] = m_name\n",
|
|
" m_mg['code'] = m_code\n",
|
|
" m_mg.setdefault(m_year,[])\n",
|
|
" m_mg1 = {}\n",
|
|
" m_mg1.setdefault(m_id,{})\n",
|
|
" m_mg1[m_id]['title'] = m_title\n",
|
|
" m_mg[m_id]['files'] = l_fls\n",
|
|
" m_mg1[m_id]['url'] = url\n",
|
|
" m_mg[m_year].append(m_mg1)\n",
|
|
" mycol.insert_one(m_mg) \n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 1,
|
|
"metadata": {
|
|
"execution": {
|
|
"iopub.execute_input": "2021-02-24T08:13:56.382398Z",
|
|
"iopub.status.busy": "2021-02-24T08:13:56.381289Z",
|
|
"iopub.status.idle": "2021-02-24T08:13:56.397601Z",
|
|
"shell.execute_reply": "2021-02-24T08:13:56.395607Z",
|
|
"shell.execute_reply.started": "2021-02-24T08:13:56.382135Z"
|
|
}
|
|
},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"20210224161356\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"import time\n",
|
|
"\n",
|
|
"print(time.strftime(\"%Y%m%d%H%M%S\", time.localtime()))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.8.10"
|
|
},
|
|
"toc-autonumbering": false
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 4
|
|
}
|