512song/jupyter
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

20230522

Browse Source
This commit is contained in:
512song committed 2023-05-22 17:33:01 +08:00
1 parent 2e7d800503
commit 91ae3fce79
3 files changed
+599 -129
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

体测单位/体质检测数据处理.ipynb
+245 -62
View File
@@ -350,9 +350,9 @@
" data_list.append((int(k),v['name'])) \n",
"conn = psycopg2.connect(database=\"mycrm\", user=\"postgres\", password=\"songyi\", host=\"localhost\", port=\"5432\")\n",
"cursor = conn.cursor()\n",
"sql = 'insert into tianjin_item (id,name) values %s'\n",
"sql = 'insert into shijiazhuang_item (id,name) values %s'\n",
"ex.execute_values(cursor, sql, data_list, page_size=10000)\n",
"conn.commit()\n",
"#conn.commit()\n",
"cursor.close()\n",
"conn.close()\n",
"print('ok!')"
@@ -381,7 +381,7 @@
"\n",
"conn = psycopg2.connect(database=\"mycrm\", user=\"postgres\", password=\"songyi\", host=\"localhost\", port=\"5432\")\n",
"cursor = conn.cursor()\n",
"filename = 'data/天津石化人员名单.json'\n",
"filename = 'data/result_石家庄.json'\n",
"with open(filename,'r') as fl:\n",
" dict1 = json.load(fl)\n",
"unit = set()\n",
@@ -399,9 +399,9 @@
"for k, v in dict2.items():\n",
" data_list.append((v,k)) \n",
" \n",
"sql = 'insert into tianjin_unit (id,name) values %s'\n",
"sql = 'insert into shijiazhuang_unit (id,name) values %s'\n",
"ex.execute_values(cursor, sql, data_list, page_size=10000)\n",
"conn.commit()\n",
"#conn.commit()\n",
"cursor.close()\n",
"conn.close()\n",
"print('ok!')"
@@ -457,7 +457,7 @@
" i+=1\n",
"sql = 'insert into tianjin_sub_unit (id,name,unit_id) values %s'\n",
"ex.execute_values(cursor, sql, data_list, page_size=10000)\n",
"conn.commit()\n",
"#conn.commit()\n",
"cursor.close()\n",
"conn.close()\n",
"print('ok!')"
@@ -484,7 +484,7 @@
"import csv\n",
"\n",
"dict1 = {}\n",
"filename = 'data/unit_134.csv'\n",
"filename = 'data/shijiazhuang_unit_20230517.csv'\n",
"with open(filename,'r',newline='') as csv_file:\n",
" fl = csv.reader(csv_file,delimiter=',')\n",
" header = next(fl)\n",
@@ -511,6 +511,36 @@
"print(len(dict1)) "
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "5e57329d-a0ef-47b2-9866-fe2a911ff01d",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"import json\n",
"import csv\n",
"\n",
"dict1 = {}\n",
"filename = 'data/shijiazhuang_unit_20230517.csv'\n",
"with open(filename,'r',newline='') as csv_file:\n",
" fl = csv.reader(csv_file,delimiter=',')\n",
" header = next(fl)\n",
" list1 = []\n",
" for line in fl:\n",
" #line = re.sub('[\\r\\n\\f ]{1,}', '', line)\n",
" list1.append({'id':int(line[0]),'name':line[1]})\n",
" dict1[int(line[0])] = line[1]\n",
"\n",
"\n",
"filename = 'data/石家庄石化部门.json'\n",
"with open(filename,'w') as fl:\n",
" json.dump(dict1, fl, ensure_ascii=False) \n",
"print(len(dict1)) "
]
},
{
"cell_type": "markdown",
"id": "53bf98fe-0ae9-402f-b8d8-495e8f31df3d",
@@ -538,15 +568,10 @@
"with open(filename,'r') as fl:\n",
" person = json.load(fl)\n",
" \n",
"filename = 'data/天津石化人员名单.json'\n",
"filename = 'data/石家庄石化部门.json'\n",
"with open(filename,'r') as fl:\n",
" dict1 = json.load(fl)\n",
"unit = set()\n",
"for k, v in dict1.items():\n",
" #if 'sub_unit' not in v.keys():\n",
" # print(k,v['name'])\n",
" unit.add(v['unit'])\n",
"print(unit) \n",
"\n",
"i = 1\n",
"dict2 = {}\n",
"for item in unit:\n",
@@ -591,13 +616,54 @@
" data_list.append((id,name,sex,birth,unit))\n",
"sql = 'insert into tianjin_person (id,name,sex,birth,unit_id) values %s'\n",
"ex.execute_values(cursor, sql, data_list, page_size=10000)\n",
"conn.commit()\n",
"#conn.commit()\n",
"cursor.close()\n",
"conn.close()\n",
"print('ok!') \n",
" "
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c2007c46-38c8-4a20-9917-b0b8743fe2a9",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"import json\n",
"import psycopg2\n",
"from psycopg2 import extras as ex\n",
"\n",
"conn = psycopg2.connect(database=\"mycrm\", user=\"postgres\", password=\"songyi\", host=\"localhost\", port=\"5432\")\n",
"cursor = conn.cursor()\n",
" \n",
"filename = 'data/石家庄石化部门.json'\n",
"with open(filename,'r') as fl:\n",
" dict1 = json.load(fl)\n",
"dict3 = {}\n",
"for k,v in dict1.items():\n",
" dict3[v] = int(k)\n",
"filename = 'data/result_石家庄.json'\n",
"with open(filename,'r') as fl:\n",
" dict2 = json.load(fl)\n",
"data_list = []\n",
"for k, v in dict2.items():\n",
" id = int(k)\n",
" name = dict2[k]['name']\n",
" sex = dict2[k]['sex']\n",
" birth = dict2[k]['birth'] \n",
" unit = dict3[dict2[k]['unit']]\n",
" data_list.append((id,name,sex,birth,unit))\n",
"sql = 'insert into shijiazhuang_person (id,name,sex,birth,unit_id) values %s'\n",
"ex.execute_values(cursor, sql, data_list, page_size=10000)\n",
"#conn.commit()\n",
"cursor.close()\n",
"conn.close()\n",
"print('ok!') "
]
},
{
"cell_type": "markdown",
"id": "fd95a45a-c6df-47c3-9941-2d0579b2c5d9",
@@ -607,28 +673,23 @@
]
},
{
"cell_type": "code",
"execution_count": 84,
"id": "360e61e9-7895-484b-a9d6-333bdefa1a86",
"cell_type": "markdown",
"id": "c989de3f-a8ab-47dc-9662-544e4b6a0e2f",
"metadata": {
"execution": {
"iopub.execute_input": "2023-05-01T04:48:05.728955Z",
"iopub.status.busy": "2023-05-01T04:48:05.728103Z",
"iopub.status.idle": "2023-05-01T04:48:08.066386Z",
"shell.execute_reply": "2023-05-01T04:48:08.065262Z",
"shell.execute_reply.started": "2023-05-01T04:48:05.728912Z"
},
"tags": []
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"ok!\n"
]
}
],
"source": [
"#### 二级部门导入"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "360e61e9-7895-484b-a9d6-333bdefa1a86",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"import json\n",
"import psycopg2\n",
@@ -685,13 +746,21 @@
" \n",
"sql = 'insert into tianjin_records (performance,score,avatar_id_id,item_id_id,unit_id_id,sub_unit_id_id) values %s'\n",
"ex.execute_values(cursor, sql, data_list, page_size=10000)\n",
"conn.commit()\n",
"#conn.commit()\n",
"cursor.close()\n",
"conn.close()\n",
"print('ok!') \n",
" "
]
},
{
"cell_type": "markdown",
"id": "4aeabbbc-5b51-4dd0-b428-83e2012e63c4",
"metadata": {},
"source": [
"#### 一级部门导入"
]
},
{
"cell_type": "code",
"execution_count": null,
@@ -712,47 +781,37 @@
"for k, v in dict1.items():\n",
" dict2[v['name']] = int(k)\n",
"\n",
"filename = 'data/天津石化部门.json'\n",
"with open(filename,'r') as fl:\n",
" dict1 = json.load(fl) \n",
"unit = {}\n",
"for item in dict1['unit']:\n",
" unit[item['name']] = item['id']\n",
"\n",
"sub_unit = {}\n",
"for item in dict1['sub_unit']:\n",
" sub_unit.setdefault(item['unit_id'],[])\n",
" sub_unit[item['unit_id']].append({'id':item['id'],'name':item['name']})\n",
" \n",
"filename = 'data/天津石化人员名单.json'\n",
"filename = 'data/石家庄石化部门.json'\n",
"with open(filename,'r') as fl:\n",
" dict1 = json.load(fl)\n",
"person = {}\n",
"for k, v in dict1.items():\n",
" person[k] = {'unit':v['unit'],'sub_unit':v['sub_unit']}\n",
"\n",
"dict3 = {}\n",
"for k,v in dict1.items():\n",
" dict3[v] = int(k)\n",
" \n",
"\n",
"conn = psycopg2.connect(database=\"mycrm\", user=\"postgres\", password=\"songyi\", host=\"localhost\", port=\"5432\")\n",
"cursor = conn.cursor()\n",
"data_list = []\n",
"filename = 'data/result_天津.json'\n",
"filename = 'data/result_石家庄.json'\n",
"with open(filename,'r') as fl:\n",
" dict1 = json.load(fl) \n",
"for k, v in dict1.items():\n",
" for item in v.keys():\n",
" if item in dict2.keys():\n",
" item_id = dict2[item]\n",
" if v[item]['得分'] == '\\\\N':\n",
" item_id = dict2[item] \n",
" if v[item]['得分'] == '':\n",
" score = None\n",
" else:\n",
" score = int(v[item]['得分'])\n",
" unit_name = person[k]['unit'] \n",
" sub_unit_name = person[k]['sub_unit']\n",
" unit_id = unit[unit_name]\n",
" for items in sub_unit[unit_id]:\n",
" if items['name'] ==sub_unit_name:\n",
" sub_id = items['id'] \n",
" print(unit_id,sub_id)"
" unit_name = v['unit']\n",
" unit_id = dict3[unit_name]\n",
" data_list.append((v[item]['成绩'],score,int(k),item_id,unit_id))\n",
"sql = 'insert into shijiazhuang_records (performance,score,avatar_id_id,item_id_id,unit_id_id) values %s'\n",
"ex.execute_values(cursor, sql, data_list, page_size=10000)\n",
"#conn.commit()\n",
"cursor.close()\n",
"conn.close()\n",
"print('ok!')"
]
},
{
@@ -781,6 +840,130 @@
" unit[item['name']] = item['id']\n",
"print(unit)"
]
},
{
"cell_type": "markdown",
"id": "bdb81e20-32ec-4e0e-b2ac-13dac2060068",
"metadata": {
"tags": []
},
"source": [
"# 体质报告在线管理"
]
},
{
"cell_type": "markdown",
"id": "1485fc89-4fc2-437d-a29e-b4a0aab12aa0",
"metadata": {},
"source": [
"## 新增体测单位"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "cf345bf0-f11b-459e-9b69-b2843836f6f3",
"metadata": {
"execution": {
"iopub.execute_input": "2023-05-19T08:30:41.861658Z",
"iopub.status.busy": "2023-05-19T08:30:41.860765Z",
"iopub.status.idle": "2023-05-19T08:30:42.006745Z",
"shell.execute_reply": "2023-05-19T08:30:42.006220Z",
"shell.execute_reply.started": "2023-05-19T08:30:41.861616Z"
},
"tags": []
},
"outputs": [],
"source": [
"import pymongo\n",
"\n",
"myclient = pymongo.MongoClient('mongodb://localhost:27017/')\n",
"mydb = myclient[\"baogao\"]\n",
"mycol = mydb[\"place\"]\n",
"\n",
"mydict = { \"id\": \"714309\",\"name\": \"中国石化石家庄炼化公司\", }\n",
" \n",
"x = mycol.insert_one(mydict) "
]
},
{
"cell_type": "markdown",
"id": "df2d18e2-d2e4-499e-876a-59d8550a8201",
"metadata": {},
"source": [
"## 新增报告信息"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "1acaf159-637e-4101-9881-f6bb79400136",
"metadata": {
"execution": {
"iopub.execute_input": "2023-05-19T09:08:35.287714Z",
"iopub.status.busy": "2023-05-19T09:08:35.286693Z",
"iopub.status.idle": "2023-05-19T09:08:35.304177Z",
"shell.execute_reply": "2023-05-19T09:08:35.303461Z",
"shell.execute_reply.started": "2023-05-19T09:08:35.287667Z"
},
"tags": []
},
"outputs": [],
"source": [
"import pymongo\n",
"\n",
"myclient = pymongo.MongoClient('mongodb://localhost:27017/')\n",
"mydb = myclient[\"baogao\"]\n",
"mycol = mydb[\"pdf\"]\n",
"\n",
"mydict = { \"place\": \"714309\",\"code\":\"4\",\"name\": \"朱维\", }\n",
" \n",
"x = mycol.insert_one(mydict) "
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "ad3078ac-ae53-4bb4-95bd-387631092bf1",
"metadata": {
"execution": {
"iopub.execute_input": "2023-05-20T04:06:02.667541Z",
"iopub.status.busy": "2023-05-20T04:06:02.667013Z",
"iopub.status.idle": "2023-05-20T04:06:02.681377Z",
"shell.execute_reply": "2023-05-20T04:06:02.680428Z",
"shell.execute_reply.started": "2023-05-20T04:06:02.667502Z"
},
"tags": []
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'_id': ObjectId('64673c9319e8481e9c0ad1a3'), 'place': '714309', 'code': '4', 'name': '朱维'}\n"
]
}
],
"source": [
"import pymongo\n",
"\n",
"myclient = pymongo.MongoClient('mongodb://localhost:27017/')\n",
"mydb = myclient[\"baogao\"]\n",
"mycol = mydb[\"pdf\"]\n",
"\n",
"mydict = { \"place\": \"714309\",\"code\":\"4\",\"name\": \"朱维\", }\n",
" \n",
"x = mycol.find_one(mydict) \n",
"print(x)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "cc0571b6-1da5-4bcc-96ad-5613cd1198e3",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {