{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 重点高校信息管理"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 强基计划信息管理"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 录入强基计划信息"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T07:02:30.612429Z",
     "iopub.status.busy": "2020-11-27T07:02:30.611524Z",
     "iopub.status.idle": "2020-11-27T07:02:31.128055Z",
     "shell.execute_reply": "2020-11-27T07:02:31.124461Z",
     "shell.execute_reply.started": "2020-11-27T07:02:30.612326Z"
    }
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "import time\n",
    "from bs4 import BeautifulSoup\n",
    "import pymongo\n",
    "\n",
    "\n",
    "myclient = pymongo.MongoClient('mongodb://localhost:27017/')\n",
    "mydb = myclient[\"gaokao\"]\n",
    "mycol = mydb[\"qiangji\"]\n",
    "m_content = ''\n",
    "headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36'}\n",
    "url = 'https://rdzs.ruc.edu.cn/cms/item/1642.html'\n",
    "strhtml = requests.get(url,headers = headers)\n",
    "strhtml.encoding = 'utf8'\n",
    "soup = BeautifulSoup(strhtml.text,'lxml')\n",
    "#data = strhtml.text\n",
    "#data1 = data.split(\"\\r\")\n",
    "data = soup.select('body > div.container.content > div > div > div.col-md-9.col-sm-8 > div')\n",
    "for item1 in data:\n",
    "    m_content +=item1.get_text()\n",
    "#print(m_content)\n",
    "m_title = soup.select('body > div.y_tit_box > div > div > div > h1')\n",
    "m_name = '中国人民大学'\n",
    "m_code = 'A002'\n",
    "m_year = '2020'\n",
    "myquery = {'code':m_code}\n",
    "x = mycol.count_documents(myquery)\n",
    "#print(x)\n",
    "m_id = time.strftime(\"%Y%m%d%H%M%S\", time.localtime())\n",
    "if x > 0:    \n",
    "    m_mg = {}  \n",
    "    m_mg.setdefault(m_id,{})\n",
    "    m_mg[m_id]['title'] = m_title[0].text\n",
    "    m_mg[m_id]['content'] = m_content\n",
    "    m_mg[m_id]['url'] = url\n",
    "    mycol.update_one(myquery,{'$push':{m_year:m_mg}})\n",
    "    print(x,s)\n",
    "else:\n",
    "    m_mg = {}\n",
    "    m_mg['name'] = m_name\n",
    "    m_mg['code'] = m_code\n",
    "    m_mg.setdefault(m_year,[])\n",
    "    m_mg1 = {}\n",
    "    m_mg1.setdefault(m_id,{})\n",
    "    m_mg1[m_id]['title'] = m_title[0].text\n",
    "    m_mg1[m_id]['content'] = m_content\n",
    "    m_mg1[m_id]['url'] = url\n",
    "    m_mg[m_year].append(m_mg1)\n",
    "    mycol.insert_one(m_mg) \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 录入强基计划附件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T10:35:49.756505Z",
     "iopub.status.busy": "2020-11-26T10:35:49.755575Z",
     "iopub.status.idle": "2020-11-26T10:35:50.001279Z",
     "shell.execute_reply": "2020-11-26T10:35:49.998646Z",
     "shell.execute_reply.started": "2020-11-26T10:35:49.756395Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 gaokao.qiangji.2020\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "import time\n",
    "import pymongo\n",
    "import os\n",
    "from gridfs import GridFS\n",
    "\n",
    "def upLoadFile(file_coll,file_name,data_link):    \n",
    "    filter_condition = {\"filename\": os.path.basename(file_name), \"url\": data_link}\n",
    "    gridfs_col = GridFS(mydb, collection=file_coll)\n",
    "    file_ = \"0\"\n",
    "    query = {\"filename\":\"\"}\n",
    "    query[\"filename\"] = file_name\n",
    "    if gridfs_col.exists(query):\n",
    "        print('已经存在该文件')\n",
    "    else:\n",
    "        with open(file_name, 'rb') as file_r:\n",
    "            file_data = file_r.read()\n",
    "            file_ = gridfs_col.put(data=file_data, **filter_condition)  # 上传到gridfs\n",
    "\n",
    "            #print(file_)\n",
    "    return file_ \n",
    "\n",
    "myclient = pymongo.MongoClient('mongodb://localhost:27017/')\n",
    "mydb = myclient[\"gaokao\"]\n",
    "mycol = mydb[\"qiangji\"]\n",
    "m_content = ''\n",
    "url = 'https://www.bkzs.sdu.edu.cn/info/1036/1635.htm'\n",
    "m_title = ''\n",
    "m_name = '山东大学'\n",
    "m_code = 'A422'\n",
    "m_year = '2020'\n",
    "myquery = {'code':m_code}\n",
    "x = mycol.count_documents(myquery)\n",
    "m_id = time.strftime(\"%Y%m%d%H%M%S\", time.localtime())\n",
    "m_dir = './data/tmp'\n",
    "fls=os.listdir(m_dir)\n",
    "l_fls =[]\n",
    "for fl in fls:        \n",
    "    full_path = m_dir+ '/' + fl\n",
    "    l_fls.append(upLoadFile(\"document\",full_path,\"\"))\n",
    "m_mg = {}\n",
    "if x > 0:  \n",
    "    m_mg.setdefault(m_id,{})\n",
    "    m_mg[m_id]['title'] = m_title\n",
    "    m_mg[m_id]['files'] = l_fls\n",
    "    m_mg[m_id]['url'] = url\n",
    "    mycol.update_one(myquery,{'$push':{m_year:m_mg}})\n",
    "else:\n",
    "    m_mg['name'] = m_name\n",
    "    m_mg['code'] = m_code\n",
    "    m_mg.setdefault(m_year,[])\n",
    "    m_mg1 = {}\n",
    "    m_mg1.setdefault(m_id,{})\n",
    "    m_mg1[m_id]['title'] = m_title\n",
    "    m_mg[m_id]['files'] = l_fls\n",
    "    m_mg1[m_id]['url'] = url\n",
    "    m_mg[m_year].append(m_mg1)\n",
    "    mycol.insert_one(m_mg) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-02-24T08:13:56.382398Z",
     "iopub.status.busy": "2021-02-24T08:13:56.381289Z",
     "iopub.status.idle": "2021-02-24T08:13:56.397601Z",
     "shell.execute_reply": "2021-02-24T08:13:56.395607Z",
     "shell.execute_reply.started": "2021-02-24T08:13:56.382135Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20210224161356\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "print(time.strftime(\"%Y%m%d%H%M%S\", time.localtime()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "toc-autonumbering": false
 },
 "nbformat": 4,
 "nbformat_minor": 4
}