512song/jupyter
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

20230504

Browse Source
This commit is contained in:
512song committed 2023-05-04 20:58:35 +08:00
1 parent 5ac233adcc
commit 89ae72f87e
2 files changed
+185 -66
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

体测单位/体质检测数据处理.ipynb
+151 -31
View File
@@ -215,29 +215,12 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 53, "execution_count": null,
"id": "b32fbbdd-eb0a-4f0d-8901-e39a727c3ff8", "id": "b32fbbdd-eb0a-4f0d-8901-e39a727c3ff8",
"metadata": { "metadata": {
"execution": {
"iopub.execute_input": "2023-04-30T01:48:23.869481Z",
"iopub.status.busy": "2023-04-30T01:48:23.868619Z",
"iopub.status.idle": "2023-04-30T01:48:23.882339Z",
"shell.execute_reply": "2023-04-30T01:48:23.881272Z",
"shell.execute_reply.started": "2023-04-30T01:48:23.869438Z"
},
"tags": [] "tags": []
}, },
"outputs": [ "outputs": [],
{
"name": "stdout",
"output_type": "stream",
"text": [
"M50~59\n",
"177.0,81.1\n",
"5\n"
]
}
],
"source": [ "source": [
"import json\n", "import json\n",
"import time\n", "import time\n",
@@ -480,6 +463,54 @@
"print('ok!')" "print('ok!')"
] ]
}, },
{
"cell_type": "markdown",
"id": "12917000-59e0-4f8a-beee-929c6d7b60a0",
"metadata": {},
"source": [
"### 生成部门JSON文件"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "5446269c-d324-4a16-b10e-f24d87178d77",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"import json\n",
"import csv\n",
"\n",
"dict1 = {}\n",
"filename = 'data/unit_134.csv'\n",
"with open(filename,'r',newline='') as csv_file:\n",
" fl = csv.reader(csv_file,delimiter=',')\n",
" header = next(fl)\n",
" list1 = []\n",
" for line in fl:\n",
" #line = re.sub('[\\r\\n\\f ]{1,}', '', line)\n",
" list1.append({'id':int(line[0]),'name':line[1]})\n",
"dict1['unit'] = list1\n",
"\n",
"filename = 'data/sub_unit_134.csv'\n",
"with open(filename,'r',newline='') as csv_file:\n",
" fl = csv.reader(csv_file,delimiter=',')\n",
" header = next(fl)\n",
" list1 = []\n",
" for line in fl:\n",
" #line = re.sub('[\\r\\n\\f ]{1,}', '', line)\n",
" list1.append({'id':int(line[0]),'name':line[1],'unit_id':int(line[2])})\n",
"dict1['sub_unit'] = list1\n",
"print(dict1)\n",
"\n",
"filename = 'data/天津石化部门.json'\n",
"with open(filename,'w') as fl:\n",
" json.dump(dict1, fl, ensure_ascii=False) \n",
"print(len(dict1)) "
]
},
{ {
"cell_type": "markdown", "cell_type": "markdown",
"id": "53bf98fe-0ae9-402f-b8d8-495e8f31df3d", "id": "53bf98fe-0ae9-402f-b8d8-495e8f31df3d",
@@ -577,15 +608,15 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 50, "execution_count": 84,
"id": "360e61e9-7895-484b-a9d6-333bdefa1a86", "id": "360e61e9-7895-484b-a9d6-333bdefa1a86",
"metadata": { "metadata": {
"execution": { "execution": {
"iopub.execute_input": "2023-04-29T14:29:07.198753Z", "iopub.execute_input": "2023-05-01T04:48:05.728955Z",
"iopub.status.busy": "2023-04-29T14:29:07.197888Z", "iopub.status.busy": "2023-05-01T04:48:05.728103Z",
"iopub.status.idle": "2023-04-29T14:29:08.464741Z", "iopub.status.idle": "2023-05-01T04:48:08.066386Z",
"shell.execute_reply": "2023-04-29T14:29:08.463607Z", "shell.execute_reply": "2023-05-01T04:48:08.065262Z",
"shell.execute_reply.started": "2023-04-29T14:29:07.198711Z" "shell.execute_reply.started": "2023-05-01T04:48:05.728912Z"
}, },
"tags": [] "tags": []
}, },
@@ -609,6 +640,27 @@
"dict2 = {}\n", "dict2 = {}\n",
"for k, v in dict1.items():\n", "for k, v in dict1.items():\n",
" dict2[v['name']] = int(k)\n", " dict2[v['name']] = int(k)\n",
"\n",
"filename = 'data/天津石化部门.json'\n",
"with open(filename,'r') as fl:\n",
" dict1 = json.load(fl) \n",
"unit = {}\n",
"for item in dict1['unit']:\n",
" unit[item['name']] = item['id']\n",
"\n",
"sub_unit = {}\n",
"for item in dict1['sub_unit']:\n",
" sub_unit.setdefault(item['unit_id'],[])\n",
" sub_unit[item['unit_id']].append({'id':item['id'],'name':item['name']})\n",
" \n",
"filename = 'data/天津石化人员名单.json'\n",
"with open(filename,'r') as fl:\n",
" dict1 = json.load(fl)\n",
"person = {}\n",
"for k, v in dict1.items():\n",
" person[k] = {'unit':v['unit'],'sub_unit':v['sub_unit']}\n",
"\n",
" \n",
"conn = psycopg2.connect(database=\"mycrm\", user=\"postgres\", password=\"songyi\", host=\"localhost\", port=\"5432\")\n", "conn = psycopg2.connect(database=\"mycrm\", user=\"postgres\", password=\"songyi\", host=\"localhost\", port=\"5432\")\n",
"cursor = conn.cursor()\n", "cursor = conn.cursor()\n",
"data_list = []\n", "data_list = []\n",
@@ -623,10 +675,15 @@
" score = None\n", " score = None\n",
" else:\n", " else:\n",
" score = int(v[item]['得分'])\n", " score = int(v[item]['得分'])\n",
" \n", " unit_name = person[k]['unit'] \n",
" data_list.append((v[item]['成绩'],score,int(k),item_id))\n", " sub_unit_name = person[k]['sub_unit']\n",
" unit_id = unit[unit_name]\n",
" for items in sub_unit[unit_id]:\n",
" if items['name'] ==sub_unit_name:\n",
" sub_id = items['id'] \n",
" data_list.append((v[item]['成绩'],score,int(k),item_id,unit_id,sub_id))\n",
" \n", " \n",
"sql = 'insert into tianjin_records (performance,score,avatar_id_id,item_id_id) values %s'\n", "sql = 'insert into tianjin_records (performance,score,avatar_id_id,item_id_id,unit_id_id,sub_unit_id_id) values %s'\n",
"ex.execute_values(cursor, sql, data_list, page_size=10000)\n", "ex.execute_values(cursor, sql, data_list, page_size=10000)\n",
"conn.commit()\n", "conn.commit()\n",
"cursor.close()\n", "cursor.close()\n",
@@ -645,13 +702,57 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"import json\n", "import json\n",
"import psycopg2\n",
"from psycopg2 import extras as ex\n",
"\n", "\n",
"filename = '../item.json'\n",
"with open(filename,'r') as fl:\n",
" dict1 = json.load(fl) \n",
"dict2 = {}\n",
"for k, v in dict1.items():\n",
" dict2[v['name']] = int(k)\n",
"\n",
"filename = 'data/天津石化部门.json'\n",
"with open(filename,'r') as fl:\n",
" dict1 = json.load(fl) \n",
"unit = {}\n",
"for item in dict1['unit']:\n",
" unit[item['name']] = item['id']\n",
"\n",
"sub_unit = {}\n",
"for item in dict1['sub_unit']:\n",
" sub_unit.setdefault(item['unit_id'],[])\n",
" sub_unit[item['unit_id']].append({'id':item['id'],'name':item['name']})\n",
" \n",
"filename = 'data/天津石化人员名单.json'\n", "filename = 'data/天津石化人员名单.json'\n",
"with open(filename,'r') as fl:\n", "with open(filename,'r') as fl:\n",
" dict1 = json.load(fl)\n", " dict1 = json.load(fl)\n",
"person = {}\n",
"for k, v in dict1.items():\n", "for k, v in dict1.items():\n",
" if 'num_id' not in v.keys():\n", " person[k] = {'unit':v['unit'],'sub_unit':v['sub_unit']}\n",
" print(k,v['name'])" "\n",
" \n",
"conn = psycopg2.connect(database=\"mycrm\", user=\"postgres\", password=\"songyi\", host=\"localhost\", port=\"5432\")\n",
"cursor = conn.cursor()\n",
"data_list = []\n",
"filename = 'data/result_天津.json'\n",
"with open(filename,'r') as fl:\n",
" dict1 = json.load(fl) \n",
"for k, v in dict1.items():\n",
" for item in v.keys():\n",
" if item in dict2.keys():\n",
" item_id = dict2[item]\n",
" if v[item]['得分'] == '\\\\N':\n",
" score = None\n",
" else:\n",
" score = int(v[item]['得分'])\n",
" unit_name = person[k]['unit'] \n",
" sub_unit_name = person[k]['sub_unit']\n",
" unit_id = unit[unit_name]\n",
" for items in sub_unit[unit_id]:\n",
" if items['name'] ==sub_unit_name:\n",
" sub_id = items['id'] \n",
" print(unit_id,sub_id)"
] ]
}, },
{ {
@@ -660,7 +761,26 @@
"id": "51dc35a5-dc07-40a4-9ea7-375ca7f983c3", "id": "51dc35a5-dc07-40a4-9ea7-375ca7f983c3",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [] "source": [
"import json\n",
"import psycopg2\n",
"from psycopg2 import extras as ex\n",
"\n",
"filename = '../item.json'\n",
"with open(filename,'r') as fl:\n",
" dict1 = json.load(fl) \n",
"dict2 = {}\n",
"for k, v in dict1.items():\n",
" dict2[v['name']] = int(k)\n",
"\n",
"filename = 'data/天津石化部门.json'\n",
"with open(filename,'r') as fl:\n",
" dict1 = json.load(fl) \n",
"unit = {}\n",
"for item in dict1['unit']:\n",
" unit[item['name']] = item['id']\n",
"print(unit)"
]
} }
], ],
"metadata": { "metadata": {
体测单位/燕山石化.ipynb
+34 -35
View File
@@ -10,15 +10,15 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 149, "execution_count": 153,
"id": "1b92b7c5-cb77-49da-95b7-34827ecf1d16", "id": "1b92b7c5-cb77-49da-95b7-34827ecf1d16",
"metadata": { "metadata": {
"execution": { "execution": {
"iopub.execute_input": "2023-04-28T23:44:56.066757Z", "iopub.execute_input": "2023-05-04T11:27:38.797630Z",
"iopub.status.busy": "2023-04-28T23:44:56.065886Z", "iopub.status.busy": "2023-05-04T11:27:38.796775Z",
"iopub.status.idle": "2023-04-28T23:44:58.568470Z", "iopub.status.idle": "2023-05-04T11:27:41.228231Z",
"shell.execute_reply": "2023-04-28T23:44:58.567666Z", "shell.execute_reply": "2023-05-04T11:27:41.227480Z",
"shell.execute_reply.started": "2023-04-28T23:44:56.066716Z" "shell.execute_reply.started": "2023-05-04T11:27:38.797589Z"
}, },
"tags": [] "tags": []
}, },
@@ -89,15 +89,15 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 150, "execution_count": 154,
"id": "71e1ec6a-6797-4496-969a-1d1461b9ea58", "id": "71e1ec6a-6797-4496-969a-1d1461b9ea58",
"metadata": { "metadata": {
"execution": { "execution": {
"iopub.execute_input": "2023-04-28T23:45:21.332726Z", "iopub.execute_input": "2023-05-04T11:27:59.843239Z",
"iopub.status.busy": "2023-04-28T23:45:21.331812Z", "iopub.status.busy": "2023-05-04T11:27:59.842790Z",
"iopub.status.idle": "2023-04-28T23:45:21.591418Z", "iopub.status.idle": "2023-05-04T11:28:00.142474Z",
"shell.execute_reply": "2023-04-28T23:45:21.590691Z", "shell.execute_reply": "2023-05-04T11:28:00.141539Z",
"shell.execute_reply.started": "2023-04-28T23:45:21.332693Z" "shell.execute_reply.started": "2023-05-04T11:27:59.843195Z"
}, },
"tags": [] "tags": []
}, },
@@ -106,10 +106,9 @@
"name": "stdout", "name": "stdout",
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"335\n", "466\n",
"123 孙剑 已测试！\n", "466\n",
"335\n", "4602\n"
"4136\n"
] ]
} }
], ],
@@ -132,7 +131,7 @@
"with open(filename,'r') as fl:\n", "with open(filename,'r') as fl:\n",
" dict1 = json.load(fl) \n", " dict1 = json.load(fl) \n",
"\n", "\n",
"filename = 'data/places_result_20230428.csv'\n", "filename = 'data/places_result_20230504.csv'\n",
"with open(filename,'r',newline='') as csv_file:\n", "with open(filename,'r',newline='') as csv_file:\n",
" fl = csv.reader(csv_file,delimiter=',')\n", " fl = csv.reader(csv_file,delimiter=',')\n",
" header = next(fl) \n", " header = next(fl) \n",
@@ -165,7 +164,7 @@
" else:\n", " else:\n",
" for k1,v1 in v.items():\n", " for k1,v1 in v.items():\n",
" dict2[k][k1] = v1\n", " dict2[k][k1] = v1\n",
"filename = 'data/result_燕山石化（20230428）.json'\n", "filename = 'data/result_燕山石化（20230504）.json'\n",
"with open(filename,'w') as fl:\n", "with open(filename,'w') as fl:\n",
" json.dump(re_ta, fl, ensure_ascii=False) \n", " json.dump(re_ta, fl, ensure_ascii=False) \n",
"print(len(re_ta))\n", "print(len(re_ta))\n",
@@ -185,15 +184,15 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 151, "execution_count": 156,
"id": "c24880ad-bbbd-4ad8-a894-cbcea389e822", "id": "c24880ad-bbbd-4ad8-a894-cbcea389e822",
"metadata": { "metadata": {
"execution": { "execution": {
"iopub.execute_input": "2023-04-28T23:46:14.648250Z", "iopub.execute_input": "2023-05-04T11:28:48.304876Z",
"iopub.status.busy": "2023-04-28T23:46:14.647361Z", "iopub.status.busy": "2023-05-04T11:28:48.304030Z",
"iopub.status.idle": "2023-04-28T23:46:14.803308Z", "iopub.status.idle": "2023-05-04T11:28:48.493153Z",
"shell.execute_reply": "2023-04-28T23:46:14.802604Z", "shell.execute_reply": "2023-05-04T11:28:48.492438Z",
"shell.execute_reply.started": "2023-04-28T23:46:14.648208Z" "shell.execute_reply.started": "2023-05-04T11:28:48.304834Z"
}, },
"tags": [] "tags": []
}, },
@@ -205,7 +204,7 @@
"items = ['身高','体重','肺活量','握力','坐位体前屈','纵跳','俯卧撑','一分钟仰卧起坐','单脚站立','选择反应时','台阶指数']\n", "items = ['身高','体重','肺活量','握力','坐位体前屈','纵跳','俯卧撑','一分钟仰卧起坐','单脚站立','选择反应时','台阶指数']\n",
"title = ['编号','姓名','性别','单位','车间','班组','身高','体重','肺活量','握力','坐位体前屈','纵跳','俯卧撑','一分钟仰卧起坐','单脚站立','选择反应时','台阶指数']\n", "title = ['编号','姓名','性别','单位','车间','班组','身高','体重','肺活量','握力','坐位体前屈','纵跳','俯卧撑','一分钟仰卧起坐','单脚站立','选择反应时','台阶指数']\n",
"\n", "\n",
"filename = 'data/result_燕山石化（20230428）.json'\n", "filename = 'data/result_燕山石化（20230504）.json'\n",
"with open(filename,'r') as fl:\n", "with open(filename,'r') as fl:\n",
" dict1 = json.load(fl)\n", " dict1 = json.load(fl)\n",
"\n", "\n",
@@ -230,7 +229,7 @@
" else:\n", " else:\n",
" list2.append('') \n", " list2.append('') \n",
" list1.append(list2)\n", " list1.append(list2)\n",
"filename = 'data/燕山石化体测情况表(20230428).xlsx'\n", "filename = 'data/燕山石化体测情况表(20230504).xlsx'\n",
"wb = openpyxl.Workbook()\n", "wb = openpyxl.Workbook()\n",
"sheet = wb.active\n", "sheet = wb.active\n",
"sheet.append(title)\n", "sheet.append(title)\n",
@@ -250,15 +249,15 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 152, "execution_count": 157,
"id": "68454877-f1fe-4eed-8a48-265e428f892e", "id": "68454877-f1fe-4eed-8a48-265e428f892e",
"metadata": { "metadata": {
"execution": { "execution": {
"iopub.execute_input": "2023-04-28T23:46:28.786698Z", "iopub.execute_input": "2023-05-04T11:29:06.618769Z",
"iopub.status.busy": "2023-04-28T23:46:28.786246Z", "iopub.status.busy": "2023-05-04T11:29:06.617928Z",
"iopub.status.idle": "2023-04-28T23:46:28.816358Z", "iopub.status.idle": "2023-05-04T11:29:06.651954Z",
"shell.execute_reply": "2023-04-28T23:46:28.815328Z", "shell.execute_reply": "2023-05-04T11:29:06.651052Z",
"shell.execute_reply.started": "2023-04-28T23:46:28.786667Z" "shell.execute_reply.started": "2023-05-04T11:29:06.618729Z"
}, },
"tags": [] "tags": []
}, },
@@ -267,7 +266,7 @@
"name": "stdout", "name": "stdout",
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"{'储运厂工会': 37, '炼油厂工会': 42, '化学品厂工会': 29, '高科公司工会': 69, '有机化工厂工会': 91, '物装中心工会': 3, '合成树脂厂': 10, '热电厂工会': 2, '合成橡胶厂工会': 27, '教育培训中心工会': 2, '烯烃厂工会': 11, '检验计量中心工会': 8, '机关工会': 2, '生产运行保障中心': 2}\n" "{'储运厂工会': 254, '合成树脂厂': 114, '检验计量中心工会': 13, '合成橡胶厂工会': 19, '炼油厂工会': 26, '化学品厂工会': 17, '烯烃厂工会': 12, '热电厂工会': 11}\n"
] ]
} }
], ],
@@ -275,7 +274,7 @@
"import json\n", "import json\n",
"import openpyxl\n", "import openpyxl\n",
"\n", "\n",
"filename = 'data/result_燕山石化（20230428）.json'\n", "filename = 'data/result_燕山石化（20230504）.json'\n",
"with open(filename,'r') as fl:\n", "with open(filename,'r') as fl:\n",
" dict1 = json.load(fl)\n", " dict1 = json.load(fl)\n",
"dict2 = {}\n", "dict2 = {}\n",
@@ -290,7 +289,7 @@
" list2 = []\n", " list2 = []\n",
" list2 = [k,v]\n", " list2 = [k,v]\n",
" list1.append(list2)\n", " list1.append(list2)\n",
"filename = 'data/燕山石化部门测试人数情况表(20230428).xlsx'\n", "filename = 'data/燕山石化部门测试人数情况表(20230504).xlsx'\n",
"wb = openpyxl.Workbook()\n", "wb = openpyxl.Workbook()\n",
"sheet = wb.active\n", "sheet = wb.active\n",
"sheet.append(title)\n", "sheet.append(title)\n",