{ "cells": [ { "cell_type": "markdown", "metadata": { "tags": [] }, "source": [ "# 高考志愿管理" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 2020年高考录取信息导入MongoDB" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "tags": [] }, "outputs": [], "source": [ "import pymysql\n", "import pymongo\n", "import decimal\n", "\n", "myclient = pymongo.MongoClient('mongodb://localhost:27017/')\n", "mydb = myclient[\"gaokao\"]\n", "mycol = mydb[\"college\"]\n", "mycol1 = mydb[\"admission_2020\"]\n", "\n", "m_col = {}\n", "m_spe = {}\n", "m_xx = {}\n", "for x in mycol.find({\"code\":{'$exists': 'true'}},{\"_id\": 0, \"code\": 1, \"name\": 1}):\n", " m_col[x['code']] = x['name']\n", "\n", "\n", "db = pymysql.connect(host = \"localhost\",user = \"songyi\",password = \"yylzs\",database = \"gaokao\" )\n", "cursor = db.cursor()\n", "sql = 'SELECT a.code,a.college,a.name FROM speciality AS a WHERE a.nian=\"2020\"'\n", "cursor.execute(sql)\n", "results = cursor.fetchall()\n", "for result in results:\n", " m_spe.setdefault(result[1],{}) \n", " m_spe[result[1]][result[0]] = result[2]\n", "sql = 'SELECT * FROM admission_2020 AS a ORDER BY a.rank_min'\n", "cursor.execute(sql)\n", "results = cursor.fetchall()\n", "i = 0\n", "m_min = 0\n", "ii = 0\n", "for result in results:\n", " m_xx.clear()\n", " if result[6] == m_min:\n", " ii = ii\n", " i = i+1\n", " else:\n", " i = i+1\n", " ii = i\n", " m_min = result[6]\n", " m_xx['pos'] = ii\n", " m_xx['col_code'] = result[1]\n", " m_xx['col_name'] = m_col[result[1]]\n", " m_xx['spe_code'] = result[2]\n", " m_xx['spe_name'] = m_spe[result[1]][result[2]]\n", " m_xx['plan'] = result[3]\n", " m_xx['dispense'] = result[5]\n", " m_xx['num_min'] = result[6]\n", " m_xx['num_avg'] = int(result[7])\n", " m_xx['rank_min'] = result[8]\n", " m_xx['nian'] = '2020' \n", " mycol1.insert_one(m_xx)\n", "#print(m_spe)\n", "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 2021年全国高等学校信息更新" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### 2021年全国高等学校信息导出至json文件" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "tags": [] }, "outputs": [], "source": [ "import openpyxl\n", "import json\n", "\n", "filename = 'data/2021年全国普通高等学校名单.xlsx'\n", "wb = openpyxl.load_workbook(filename)\n", "sheet = wb.active\n", "data1 =list(sheet.values)\n", "del data1[0:3]\n", "i = 1\n", "dict1 = {}\n", "for items in data1:\n", " list1 = (items[1],str(items[2]),items[3],items[4],items[5],items[6])\n", " dict1[i] = list1\n", " i += 1 \n", "with open('data/2021年全国普通高等学校名单.json','w') as fl2:\n", " json.dump(dict1,fl2) \n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### 整理2021年全国高等学校信息" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "tags": [] }, "outputs": [], "source": [ "import openpyxl\n", "import json\n", "import pymongo\n", "\n", "\n", "filename = 'data/2021年全国普通高等学校名单.json'\n", "with open(filename,'r') as fl:\n", " m_xx = json.load(fl)\n", "\n", "myclient = pymongo.MongoClient('mongodb://localhost:27017/')\n", "mydb = myclient[\"gaokao\"]\n", "mycol = mydb[\"college_2021\"]\n", "\n", "m_col = {}\n", "\n", "\n", "for x in mycol.find({\"id_code\":{'$exists': 'true'}},{\"_id\": 0, \"id_code\": 1, \"name\": 1}):\n", " m_col[x['id_code']] = x['name']\n", "#print(m_col.keys())\n", "for k, v in m_xx.items():\n", " m_code = v[1]\n", " #print(m_code)\n", " if m_code not in m_col.keys():\n", " dict1 = {}\n", " dict1['id_code'] = v[1]\n", " dict1['name'] = v[0]\n", " dict1['charge'] = v[2]\n", " dict1['city'] = v[3]\n", " dict1['grade'] = v[4]\n", " dict1['note'] = v[5]\n", " mycol.insert_one(dict1) \n", " print(dict1['name'] + '导入成功！')\n", " #print(v[0],v[1])\n", " \n", "\n", "\n", " " ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 计算志愿分数概率" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import random\n", "\n", "array2 = []\n", "for i in range(300000):\n", " array1 = []\n", " s = 0\n", " for ii in range(60):\n", " m1 = random.randint(580,595)\n", " array1.append(m1)\n", " s = s + m1 \n", " m_avg = round(s/60,2)\n", " if m_avg == 582.8 and (580 in array1) and (595 in array1):\n", " #print(array1)\n", " array2.extend(array1)\n", " \n", "#print(array2)\n", "m_set = set(array2)\n", "for m in m_set:\n", " print(m,array2.count(m))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 导入山东大学录取明细" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import openpyxl\n", "import pymongo\n", "\n", "myclient = pymongo.MongoClient('mongodb://localhost:27017/')\n", "mydb = myclient[\"gaokao\"]\n", "mycol = mydb[\"admission_college\"]\n", "wb = openpyxl.load_workbook('./data/shandongdaxue.xlsx')\n", "sheet = wb.active\n", "#sheets = wb.sheetnames\n", "code = 'A422'\n", "name = '山东大学'\n", "new_col = []\n", "dict1 = {}\n", "\n", "new_code = []\n", "dict1['code'] = code\n", "dict1['name'] = name\n", "for n in range(1,sheet.max_row+1):\n", " nian = str(sheet.cell(n,1).value)\n", " dict2 = {}\n", " \n", " dict1.setdefault(nian,[])\n", " if sheet.cell(n,2).value =='理工':\n", " m_lb = 'l'\n", " elif sheet.cell(n,2).value =='文史':\n", " m_lb = 'w'\n", " else:\n", " m_lb = 'z' \n", " dict2['type'] = m_lb\n", " dict2['spe_name'] = sheet.cell(n,4).value\n", " dict2['max_score'] = sheet.cell(n,5).value\n", " dict2['min_score'] = sheet.cell(n,6).value\n", " dict2['avg_score'] = sheet.cell(n,7).value\n", " dict2['dispense'] = sheet.cell(n,8).value\n", " dict1[nian].append(dict2)\n", "mycol.insert_one(dict1)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 导入山东师范大学录取明细" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import openpyxl\n", "import pymongo\n", "\n", "myclient = pymongo.MongoClient('mongodb://localhost:27017/')\n", "mydb = myclient[\"gaokao\"]\n", "mycol = mydb[\"admission_college\"]\n", "wb = openpyxl.load_workbook('./data/中国海洋大学.xlsx')\n", "sheet = wb.active\n", "#sheets = wb.sheetnames\n", "code = 'A423'\n", "name = '中国海洋大学'\n", "new_col = []\n", "dict1 = {}\n", "\n", "new_code = []\n", "dict1['code'] = code\n", "dict1['name'] = name\n", "for n in range(1,sheet.max_row+1):\n", " nian = str(sheet.cell(n,1).value)\n", " dict2 = {}\n", " \n", " dict1.setdefault(nian,[])\n", " if sheet.cell(n,2).value =='理工':\n", " m_lb = 'l'\n", " elif sheet.cell(n,2).value =='文史':\n", " m_lb = 'w'\n", " else:\n", " m_lb = 'z'\n", " dict2['type'] = m_lb\n", " dict2['spe_name'] = sheet.cell(n,4).value\n", " dict2['max_score'] = sheet.cell(n,7).value\n", " dict2['min_score'] = sheet.cell(n,5).value\n", " dict2['avg_score'] = sheet.cell(n,6).value\n", " if sheet.cell(n,8).value:\n", " dict2['dispense'] = sheet.cell(n,8).value\n", " dict1[nian].append(dict2)\n", "mycol.insert_one(dict1)\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 按地区列示高校" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import pymongo\n", "\n", "myclient = pymongo.MongoClient('mongodb://localhost:27017/')\n", "mydb = myclient[\"gaokao\"]\n", "mycol = mydb[\"college\"]\n", "m_city = []\n", "m_xx = {}\n", "myquery = {'code':{'$exists': 'true'},'note':{'$not':{'$regex':'民办'}}}\n", "colleges = mycol.find(myquery,{ \"_id\": 0, \"name\": 1, \"code\": 1,'city':1 })\n", "for x in colleges:\n", " m_xx1 = []\n", " c = x['city']\n", " m_xx.setdefault(c,[])\n", " m_xx1.append(x['code'])\n", " m_xx1.append(x['name'])\n", " m_xx[c].append(m_xx1)\n", "#print(m_city)\n", "#按照原顺序对高校所在城市排序\n", "'''\n", "city = list(set(m_city))\n", "city.sort(key=m_city.index)\n", "for c in city:\n", "# m_xx['city'] = c\n", " m_xx.setdefault(c,[])\n", " for y in colleges:\n", " print(y['code'],y['name'])\n", " \n", "#m_xx \n", "''' \n", "for k,v in m_xx.items():\n", " print(k)\n", " for mm in v:\n", " print(mm[0],mm[1])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 显示学校2020年招生信息" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import pymongo\n", "\n", "myclient = pymongo.MongoClient('mongodb://localhost:27017/')\n", "mydb = myclient[\"gaokao\"]\n", "mycol = mydb[\"admission_2020\"]\n", "code = 'A422'\n", "m_city = []\n", "m_xx = {}\n", "myquery = {'col_code':code}\n", "colleges = mycol.find(myquery,{ \"_id\": 0 , \"col_code\":0,\"col_name\":0,'plan':0,'nian':0}).sort('pos')\n", "for x in colleges:\n", " print(x)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 2021年拟在山东招生普通高校专业（类）选考科目要求" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from os import mkdir\n", "from time import sleep\n", "from re import findall,sub,S\n", "from os.path import isdir,isfile\n", "from urllib.request import urlopen\n", "from urllib.parse import urlencode,quote\n", "from openpyxl import Workbook\n", "import ssl\n", "from bs4 import BeautifulSoup\n", "import pymongo\n", "ssl._create_default_https_context = ssl._create_unverified_context\n", "myclient = pymongo.MongoClient('mongodb://localhost:27017/')\n", "mydb = myclient[\"gaokao\"]\n", "mycol = mydb[\"xuankaokemu\"]\n", "list2 = []\n", "for x in mycol.find({},{ \"_id\": 0, \"code\": 1}):\n", " list2.append(x['code'])\n", "#m_xx = dict()\n", "start_url = 'https://xkkm.sdzk.cn/web/xx.html'\n", "with urlopen(start_url) as fp:\n", " content = fp.read().decode('utf8')\n", " \n", "pattern = (r'.*?.*?(.+?)'\n", " '.*?(.+?).*?(.+?)')\n", "\n", "for item in findall(pattern,content,S):\n", " if len(item[0]) > 5:\n", " continue\n", " \n", " shengfen,dm,mc = item\n", " print('学校代码：', dm)\n", " print('学校名称：', mc)\n", " m_xx = {}\n", " m_xx['code'] = dm\n", " m_xx['name'] = mc\n", " m_xx.setdefault('zhuanye',{})\n", " if dm in list2:\n", " continue\n", " url = r'https://xkkm.sdzk.cn/xkkm/queryXxInfor'\n", " data = urlencode({'dm':dm,'mc':quote(mc),'yzm':'ok'}).encode('ascii')\n", " with urlopen(url,data) as fp:\n", " xuexiao_content = fp.read().decode()\n", " soup = BeautifulSoup(xuexiao_content,'lxml')\n", " #data1 = soup.select('#ccc > div > table > tbody > tr > td:nth-child(5)')\n", " data = soup.select('#ccc > div > table > tbody > tr ')\n", " for data1 in data:\n", " list1 =[]\n", " m_xx1 = {}\n", " for item in data1.stripped_strings: \n", " list1.append(item)\n", " #del list1[0]\n", " #print('层次：',list1[1])\n", " #print('专业（类）名称：',list1[2])\n", " #print('选考科目范围：',list1[3])\n", " #print('类中所含专业：',list1[4:])\n", " code = list1[0]\n", " m_xx['zhuanye'].setdefault(code,{})\n", " \n", " m_xx['zhuanye'][code]['name'] = list1[2]\n", " m_xx['zhuanye'][code]['level'] = list1[1]\n", " m_xx['zhuanye'][code]['fanwei'] = list1[3]\n", " m_xx['zhuanye'][code]['suohanzhuanye'] = list1[4:]\n", " mycol.insert_one(m_xx) \n", " \n", " print('ok')\n", " \n", " # list1.clear\n", " \n", "\n", " sleep(5)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 2024年拟在山东招生普通高校专业（类）选考科目要求" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "tags": [] }, "outputs": [], "source": [ "from os import mkdir\n", "from time import sleep\n", "from re import findall,sub,S\n", "from os.path import isdir,isfile\n", "from urllib.request import urlopen\n", "from urllib.parse import urlencode,quote\n", "from openpyxl import Workbook\n", "import ssl\n", "from bs4 import BeautifulSoup\n", "import pymongo\n", "ssl._create_default_https_context = ssl._create_unverified_context\n", "myclient = pymongo.MongoClient('mongodb://localhost:27017/')\n", "mydb = myclient[\"gaokao\"]\n", "mycol = mydb[\"xuankaokemu2024\"]\n", "list2 = []\n", "for x in mycol.find({},{ \"_id\": 0, \"code\": 1}):\n", " list2.append(x['code'])\n", "#m_xx = dict()\n", "start_url = 'https://xkkm.sdzk.cn/web/xx.html'\n", "with urlopen(start_url) as fp:\n", " content = fp.read().decode('utf8')\n", " \n", "pattern = (r'.*?.*?(.+?)'\n", " '.*?(.+?).*?(.+?)')\n", "\n", "for item in findall(pattern,content,S):\n", " if len(item[0]) > 5:\n", " continue\n", " \n", " shengfen,dm,mc = item\n", " print('学校代码：', dm)\n", " print('学校名称：', mc)\n", " m_xx = {}\n", " m_xx['code'] = dm\n", " m_xx['name'] = mc\n", " m_xx.setdefault('zhuanye',{})\n", " #if dm in list2:\n", " # continue\n", " url = r'https://xkkm.sdzk.cn/xkkm/queryXxInfor'\n", " data = urlencode({'dm':dm,'mc':quote(mc),'yzm':'ok','nf':'2024'}).encode('ascii')\n", " with urlopen(url,data) as fp:\n", " xuexiao_content = fp.read().decode()\n", " soup = BeautifulSoup(xuexiao_content,'lxml')\n", " #data1 = soup.select('#ccc > div > table > tbody > tr > td:nth-child(5)')\n", " data = soup.select('#ccc > div > table > tbody > tr ')\n", " for data1 in data:\n", " list1 =[]\n", " m_xx1 = {}\n", " for item in data1.stripped_strings: \n", " list1.append(item)\n", " #del list1[0]\n", " #print('层次：',list1[1])\n", " #print('专业（类）名称：',list1[2])\n", " #print('选考科目范围：',list1[3])\n", " #print('类中所含专业：',list1[4:])\n", " code = list1[0]\n", " m_xx['zhuanye'].setdefault(code,{})\n", " \n", " m_xx['zhuanye'][code]['name'] = list1[2]\n", " m_xx['zhuanye'][code]['level'] = list1[1]\n", " m_xx['zhuanye'][code]['fanwei'] = list1[3]\n", " m_xx['zhuanye'][code]['suohanzhuanye'] = list1[4:]\n", " mycol.insert_one(m_xx) \n", " \n", " print('ok')\n", " \n", " # list1.clear\n", " \n", "\n", " sleep(5)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import pymongo\n", "\n", "myclient = pymongo.MongoClient('mongodb://localhost:27017/')\n", "mydb = myclient[\"gaokao\"]\n", "mycol = mydb[\"xuankaokemu\"]\n", "list2 = []\n", "for x in mycol.find({},{ \"_id\": 0, \"code\": 1}):\n", " list2.append(x['code'])\n", "print(list2)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 2022年高考选科数据分析" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### 2022年选科数据导入" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "tags": [] }, "outputs": [], "source": [ "import json\n", "import pymongo\n", "\n", "filename = 'data/2022招生计划（汇总）.json'\n", "with open(filename,'r') as fl:\n", " m_xx = json.load(fl)\n", "#print(m_xx)\n", "xuanke = []\n", "dict1 = {}\n", "dict2 = {}\n", "dict2['0'] = '不限'\n", "dict2['1'] = '物理和化学和生物'\n", "dict2['2'] = '思想政治和历史和地理'\n", "dict3 = {}\n", "for k, v in m_xx.items():\n", " col_code = k\n", " col_name = v['学校名称']\n", " for item in v['专业']:\n", " m_code = col_code+'-'+item['unit_code']\n", " m_name = item['unit_name']\n", " m_xuanke = item['xuanke'].strip()\n", " if m_xuanke not in xuanke:\n", " xuanke.append(m_xuanke)\n", " dict1[m_code] = [m_name,m_xuanke]\n", "fl_name = 'data/2022年选科分析表.json'\n", "with open(fl_name,'w') as fl:\n", " json.dump(dict1,fl)\n", "xuanke.remove('不限')\n", "xuanke.remove('物理和化学和生物')\n", "xuanke.remove('思想政治和历史和地理')\n", "print(xuanke)\n", "dict3['不限'] = [1,'0',['物理','化学','生物','思想政治','历史','地理']]\n", "dict3['物理和化学和生物'] = [2,'1',['物理','化学','生物']]\n", "dict3['思想政治和历史和地理'] = [3,'1',['思想政治','历史','地理']]\n", "i = 4\n", "for item in xuanke:\n", " #dict3[str(i)].setdault({})\n", " if '和' in item:\n", " mx = item.split('和')\n", " dict3[item] = [i,'1',mx]\n", " elif '或' in item:\n", " mx = item.split('或')\n", " dict3[item] = [i,'0',mx]\n", " else:\n", " dict3[item] = [i,'0',[item]]\n", " i+=1\n", "fl_name = 'data/选科代码表.json'\n", "with open(fl_name,'w') as fl:\n", " json.dump(dict3,fl)\n", "print('ok!')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### 选科数据导入Mysql" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "tags": [] }, "outputs": [], "source": [ "import pymysql\n", "import json\n", "\n", "filename = 'data/选科代码表.json'\n", "with open(filename,'r') as fl:\n", " dmb = json.load(fl)\n", "\n", "filename = 'data/2022年选科分析表.json'\n", "with open(filename,'r') as fl:\n", " jhb = json.load(fl) \n", "list1 = []\n", "db = pymysql.connect(host = \"localhost\",user = \"root\",password = \"songyi\",database = \"gaokao\" )\n", "cursor = db.cursor()\n", "\n", "sql = 'select college,speciality,rank_min from admission_2022 order by rank_min'\n", "cursor.execute(sql)\n", "results = cursor.fetchall()\n", "for result in results:\n", " m_dm = result[0]+'-'+result[1]\n", " if m_dm in jhb.keys():\n", " m_xk = jhb[m_dm][1].strip()\n", " m_item = ','.join(dmb[m_xk][2])\n", " list1.append((result[0],result[1],str(dmb[m_xk][0]),m_item))\n", "\n", "sql = 'insert into zhiyuanfenxi (col_code,spe_code,zy_code,item) values (%s,%s,%s,%s)'\n", "try:\n", " cursor.executemany(sql,list1)\n", " db.commit()\n", " print(\"ok!\")\n", "except:\n", " # 如果发生错误则回滚\n", " print(\"error!\")\n", " db.rollback() \n", "\n", "db.close()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# 招生简章管理" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 各大学招生简章采集" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from time import sleep\n", "import ssl\n", "from bs4 import BeautifulSoup\n", "import pymongo\n", "import requests\n", "\n", "myclient = pymongo.MongoClient('mongodb://localhost:27017/')\n", "mydb = myclient[\"gaokao\"]\n", "mycol = mydb[\"zhaoshengjianzhang\"]\n", "headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36'}\n", "#url = 'https://gaokao.chsi.com.cn/zsgs/zhangcheng/listVerifedZszc.do?method=index&ssdm=11&yxls=&yxlx=&xlcc=bk'\n", "\n", "sf = [11,12,13,14,15,21,22,23,31,32,33,34,35,36,37,41,42,43,44,45,46,50,51,52,53,54,61,62,63,64,65]\n", "for i in sf:\n", " url = f'https://gaokao.chsi.com.cn/zsgs/zhangcheng/listVerifedZszc.do?method=index&ssdm={str(i)}&yxls=&yxlx=&xlcc=bk'\n", " strhtml = requests.get(url,headers = headers)\n", " strhtml.encoding = 'utf8'\n", " soup = BeautifulSoup(strhtml.text,'lxml')\n", " data = soup.select('body > div.width1000 > table:nth-child(5) > tbody > tr> td > a')\n", " \n", " for item in data:\n", " dict1 = {}\n", " dict1['name'] = item.text.strip()\n", " dict1['url'] = item.get('href')\n", " if item.get('style') =='color:gray':\n", " dict1['bz'] = 0\n", " else:\n", " dict1['bz'] = 1\n", " mycol.insert_one(dict1) \n", "\n", "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 采集单个学校招生简章" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from time import sleep\n", "from re import findall,sub,S\n", "import ssl\n", "from bs4 import BeautifulSoup\n", "import pymongo\n", "import requests\n", "\n", "myclient = pymongo.MongoClient('mongodb://localhost:27017/')\n", "mydb = myclient[\"gaokao\"]\n", "mycol = mydb[\"zhaoshengjianzhang\"]\n", "headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36'}\n", "url = 'https://gaokao.chsi.com.cn/zsgs/zhangcheng/listZszc--schId-5.dhtml'\n", "strhtml = requests.get(url,headers = headers)\n", "strhtml.encoding = 'utf8'\n", "soup = BeautifulSoup(strhtml.text,'lxml')\n", "#data = strhtml.text\n", "#data1 = data.split(\"\\r\")\n", "#print(soup)\n", "data = soup.select('body > div.width1000.gery > div > div.right > table > tr > td > a')\n", "for item in data:\n", " url1 = item.get('href')\n", "url1 = 'https://gaokao.chsi.com.cn/' + url1\n", "strhtml = requests.get(url1,headers = headers)\n", "strhtml.encoding = 'utf8'\n", "soup = BeautifulSoup(strhtml.text,'lxml')\n", "data = soup.select('body > div.width1000.border.gery > div>p')\n", "nr = ''\n", "for item in data:\n", " nr += item.text+'\\n'\n", "print(nr)\n", " \n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 第一次采集招生简章" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from time import sleep\n", "from re import findall,sub,S\n", "import ssl\n", "from bs4 import BeautifulSoup\n", "import pymongo\n", "import requests\n", "\n", "myclient = pymongo.MongoClient('mongodb://localhost:27017/')\n", "mydb = myclient[\"gaokao\"]\n", "mycol = mydb[\"zhaoshengjianzhang\"]\n", "for x in mycol.find({\"bz\": 1 },{ \"_id\": 0, \"name\": 1, \"url\": 1 }):\n", " url = 'https://gaokao.chsi.com.cn'+x['url']\n", " col_name = x['name']\n", " strhtml = requests.get(url,headers = headers)\n", " strhtml.encoding = 'utf8'\n", " soup = BeautifulSoup(strhtml.text,'lxml')\n", " #data = strhtml.text\n", " #data1 = data.split(\"\\r\")\n", " #print(soup)\n", " data = soup.select('body > div.width1000.gery > div > div.right > table > tr > td > a')\n", " for item in data:\n", " url1 = item.get('href')\n", " url1 = 'https://gaokao.chsi.com.cn/' + url1\n", " strhtml = requests.get(url1,headers = headers)\n", " strhtml.encoding = 'utf8'\n", " soup = BeautifulSoup(strhtml.text,'lxml')\n", " data = soup.select('body > div.width1000.border.gery > div>p')\n", " nr = ''\n", " for item in data:\n", " nr += item.text+'\\n' \n", " myquery = {'name':col_name}\n", " mycol.update_one(myquery,{'$push':{'content':nr}})\n", " sleep(5)\n", " " ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 检查新增的学校及招生简章" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from time import sleep\n", "from re import findall,sub,S\n", "import ssl\n", "from bs4 import BeautifulSoup\n", "import pymongo\n", "import requests\n", "\n", "myclient = pymongo.MongoClient('mongodb://localhost:27017/')\n", "mydb = myclient[\"gaokao\"]\n", "mycol = mydb[\"zhaoshengjianzhang\"]\n", "headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36'}\n", "l_name = []\n", "for x in mycol.find({},{ \"_id\": 0, \"name\": 1}):\n", " l_name.append(x['name'])\n", "sf = [11,12,13,14,15,21,22,23,31,32,33,34,35,36,37,41,42,43,44,45,46,50,51,52,53,54,61,62,63,64,65]\n", "n_url = []\n", "for i in sf:\n", " url = f'https://gaokao.chsi.com.cn/zsgs/zhangcheng/listVerifedZszc.do?method=index&ssdm={str(i)}&yxls=&yxlx=&xlcc=bk'\n", " strhtml = requests.get(url,headers = headers)\n", " strhtml.encoding = 'utf8'\n", " soup = BeautifulSoup(strhtml.text,'lxml')\n", " data = soup.select('body > div.width1000 > table:nth-child(5) > tbody > tr> td > a')\n", " \n", " for item in data:\n", " m_url = item.get('href')\n", " m_name = item.text.strip()\n", " if m_name not in l_name:\n", " url = 'https://gaokao.chsi.com.cn'+m_url\n", " col_name = m_name \n", " dict1 = {}\n", " dict1['name'] = m_name\n", " dict1['url'] = m_url\n", " dict1['bz'] = 0 \n", " mycol.insert_one(dict1) \n", " print(dict1)\n", " sleep(3)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 检查、新增招生简章" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from time import sleep\n", "from re import findall,sub,S\n", "import ssl\n", "from bs4 import BeautifulSoup\n", "import pymongo\n", "import requests\n", "\n", "myclient = pymongo.MongoClient('mongodb://localhost:27017/')\n", "mydb = myclient[\"gaokao\"]\n", "mycol = mydb[\"zhaoshengjianzhang\"]\n", "headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36'}\n", "l_name = []\n", "for x in mycol.find({\"bz\": 0 },{ \"_id\": 0, \"name\": 1, \"url\": 1 }):\n", " l_name.append(x['name'])\n", "sf = [11,12,13,14,15,21,22,23,31,32,33,34,35,36,37,41,42,43,44,45,46,50,51,52,53,54,61,62,63,64,65]\n", "n_name = []\n", "for i in sf:\n", " url = f'https://gaokao.chsi.com.cn/zsgs/zhangcheng/listVerifedZszc.do?method=index&ssdm={str(i)}&yxls=&yxlx=&xlcc=bk' \n", " strhtml = requests.get(url,headers = headers)\n", " strhtml.encoding = 'utf8'\n", " soup = BeautifulSoup(strhtml.text,'lxml')\n", " data = soup.select('body > div.width1000 > table:nth-child(5) > tbody > tr> td > a')\n", " \n", " for item in data:\n", " m_url = item.get('href')\n", " col_name = item.text.strip()\n", " #print(m_url)\n", " if (item.get('style') !='color:gray') and (col_name in l_name):\n", " url = 'https://gaokao.chsi.com.cn' + m_url\n", " \n", " strhtml = requests.get(url,headers = headers)\n", " strhtml.encoding = 'utf8'\n", " soup = BeautifulSoup(strhtml.text,'lxml')\n", " data = soup.select('body > div.width1000.gery > div > div.right > table > tr > td > a')\n", " for item in data:\n", " url1 = item.get('href')\n", " print(url1)\n", " url1 = 'https://gaokao.chsi.com.cn' + url1\n", " strhtml = requests.get(url1,headers = headers)\n", " strhtml.encoding = 'utf8'\n", " soup = BeautifulSoup(strhtml.text,'lxml')\n", " data = soup.select('body > div.width1000.border.gery > div>p')\n", " nr = ''\n", " for item in data:\n", " nr += item.text+'\\n' \n", " myquery = {'name':col_name}\n", " mycol.update_one(myquery,{'$set':{'bz':1}})\n", " mycol.update_one(myquery,{'$push':{'content':nr}})\n", " print(col_name)\n", " sleep(3)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" } }, "nbformat": 4, "nbformat_minor": 4 }