512song/jupyter
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
cbc1f8e56bfbdb41e8c1fb36645851df655ee798
jupyter/数据处理.ipynb
T
512song@sina.com 4611fae329 20220501
2022-05-01 17:19:01 +08:00

34 KiB
Raw Blame History

正则表达式分割文本

In [ ]:
import json
import openpyxl
import re

file_name = 'data/药食材性味归经.xlsx'
wb = openpyxl.load_workbook(file_name)
sheet = wb.active
#sheets = wb.sheetnames

list1 = []
dict1 = {}
mo =r'[。入归].+经$'
mo1 = r'[二]'
mo2 = r'[，、；]'
for n in range(2,sheet.max_row):
    name = sheet.cell(n,1).value
    content = re.findall(mo,sheet.cell(n,2).value)
    if len(content) > 0:
        l = len(content[0])
        gj = re.sub(mo1, '', content[0][1:l-1])        
        list_gj = re.split(mo2,gj)
        dict1[sheet.cell(n,1).value ] = list_gj
        #dict1['guijing'] = list_gj
        
filename = './data/药食材归经.json'
with open(filename,'w') as fl:
    json.dump(dict1, fl,ensure_ascii=False) 

#print(list1)
In [ ]:
import re

s = '消化不良、胃下垂、急性胃炎、慢性胃炎、萎缩性胃炎、神经性呕吐、胆囊炎、胆石症、胆道蛔虫症、胸胁痛等'
ss = '健中和胃，消食止呕，理气疏郁，清热利胆。'
mo = '等$'
mo2 = r'[，、；。]'
s1 = re.sub(mo, '', s) 
list1 = re.split(mo2,s1)
if '' in list1:
    list1.remove('')
print(list1)

药膳归经明细文件生成

In [ ]:
import json

filename = 'data/药膳210927.json'
with open(filename,'r') as fl:
    dict1 = json.load(fl)
filename = 'data/药食材归经.json'
with open(filename,'r') as fl:
    dict2 = json.load(fl)
k_zy = dict2.keys()
dict3 = {}
list1 = []
for item in dict1:
    print(item['name'])
    list1 = []
    #print(i,item['zy'])
    for m_zy in item['zy']:
        if m_zy in k_zy:
            dict4 = {}
            print(m_zy,dict2[m_zy])
            dict4[m_zy] = dict2[m_zy]
            list1.append(dict4)
    dict3[item['name']] = list1
filename = './data/药膳药食材.json'
with open(filename,'w') as fl:
    json.dump(dict3, fl,ensure_ascii=False)

药膳归经权重生成

In [ ]:
import json
import openpyxl
import os,sys,shutil

filename = './data/药膳药食材.json'
with open(filename,'r') as fl:
    dict1 = json.load(fl)
list1 = []
for k, v in dict1.items():
    if len(v) > 0 :
        dict2 = {}
        #print(k,v)
        ys_name = k
        dict2.setdefault(ys_name,{})
        for item in v:
            for k1, v1 in item.items():
                for item1 in v1:
                    dict2[ys_name].setdefault(item1,0)
                    dict2[ys_name][item1] += 1
        list1.append(dict2)                    
    
dict_qz = {}
for item in list1:
    for k, v in item.items():
        qz = sorted(v.items(), key = lambda kv:(kv[1], kv[0]),reverse=True)
        dict_qz[k] = qz
#print(dict_qz)
qz_key = dict_qz.keys()
filename = 'data/药膳210927.json'
with open(filename,'r') as fl:
    dict1 = json.load(fl)
#print(dict1)
wb = openpyxl.Workbook()
sheet = wb.active
sheet['A1'] = '药膳名称'
sheet['B1'] = '来源'
sheet['C1'] = '配方'
sheet['D1'] = '做法'
sheet['E1'] = '功效'
sheet['F1'] = '中药成分'
sheet['G1'] = '归经权重'

i =2
for item in dict1:
    sheet[f'A{i}'] = item['name']
    sheet[f'B{i}'] = item['source']
    sheet[f'C{i}'] = item['pf']
    sheet[f'D{i}'] = item['zf']
    sheet[f'E{i}'] = item['gx']
    sheet[f'F{i}'] = ','.join(item['zy'])
    if item['name'] in qz_key:
        s = ''
        for m_gj in dict_qz[item['name']]:
            s = s+ m_gj[0] +'('+str(m_gj[1])+')'
        sheet[f'G{i}'] = s
    else:
        sheet[f'G{i}'] = '暂无归经'
    i += 1

wb.save('data/test4.xlsx')

穴位数据导入

简单导出简介、内容

In [ ]:
import json
import re
dict1 = {}
filename = 'file/zhongyi/xuewei.json'
with open(filename,'r',encoding='utf-8') as fl:
    for line in fl:
        line1 = json.loads(re.sub(r'\xa0','',line))
        list1 = line1['tables']
        list2 = line1['contents']
        dict2 = dict(zip(list1, list2))
        #print(line1['title'],dict1)
        #print(dict1.keys())
        #print(line1)
        dict1.setdefault(line1['title'][0],{})
        dict1[line1['title'][0]]['简介'] = line1['jj'] 
        dict1[line1['title'][0]]['内容'] = dict2
filename = './file/穴位1.json'
with open(filename,'w') as fl:
    json.dump(dict1, fl)

数据导入文件中

In [ ]:
import json
import re

dict1 = {}
filename = 'file/zhongyi/xuewei.json'
with open(filename,'r',encoding='utf-8') as fl:
    for line in fl:
        line1 = json.loads(re.sub(r'\xa0','',line))
        list1 = line1['tables']
        list2 = line1['contents']
        list3 = line1['jj']
        dict2 = dict(zip(list1, list2))
        dict1.setdefault(line1['title'][0],{})
        if len(list3) > 0:
            
            dict1[line1['title'][0]]['about'] = list3
        dict1[line1['title'][0]].setdefault('content',{})
        for k, v in dict2.items():
            dict1[line1['title'][0]]['content'][k] = v 
        #dict1[line1['title'][0]]['内容'] = dict2
filename = './file/穴位1.json'
with open(filename,'w') as fl:
    json.dump(dict1, fl)

穴位数据导入数据库中

In [ ]:
import json
import re
import pymongo

myclient = pymongo.MongoClient('mongodb://10.147.17.6:27017/')
#dblist = myclient.list_database_names()
mydb = myclient['dayi']
mycol = mydb["xuewei"]
db_list = []
filename = 'file/zhongyi/xuewei.json'
with open(filename,'r',encoding='utf-8') as fl:
    for line in fl:
        dict1 = {}
        line1 = json.loads(re.sub(r'\xa0','',line))
        list1 = line1['tables']
        list2 = line1['contents']
        list3 = line1['jj']
        dict2 = dict(zip(list1, list2))
        dict1['name'] = line1['title'][0]
        if len(list3) > 0:
            dict1.setdefault('about',{})
            for s in list3:
                item = s.strip().split(':')
                dict1['about'][item[0].strip()] = item[1].strip()
        dict1.setdefault('content',{})
        for k, v in dict2.items():
            dict1['content'][k] = v 
        #dict1[line1['title'][0]]['内容'] = dict2
        db_list.append(dict1)
x = mycol.insert_many(db_list)
print('ok!')

中医症状数据导入数据库中

In [ ]:
import json
import re
import pymongo

myclient = pymongo.MongoClient('mongodb://10.147.17.6:27017/')
mydb = myclient['dayi']
mycol = mydb["zhongyizhengzhuang"]
db_list = []
filename = 'file/zhongyi/zhongyizhengzhuang.json'
with open(filename,'r',encoding='utf-8') as fl:
    for line in fl:
        dict1 = {}
        line1 = json.loads(re.sub(r'\xa0','',line))
        list1 = line1['tables']
        list2 = line1['contents']
        list3 = line1['jj']
        dict2 = dict(zip(list1, list2))
        dict1['name'] = line1['title'][0]
        if len(list3) > 0:
            dict1.setdefault('about',{})
            for i in range(0,int(len(list3)/2)):
                dict1['about'][list3[2*i]] = list3[2*i+1]
        dict1.setdefault('content',{})
        for k, v in dict2.items():
            dict1['content'][k] = v 
        #dict1[line1['title'][0]]['内容'] = dict2
        db_list.append(dict1)
x = mycol.insert_many(db_list)
print('ok!')

疾病数据导入数据库中

In [ ]:
import json
import re
import pymongo

myclient = pymongo.MongoClient('mongodb://10.147.17.6:27017/')
mydb = myclient['dayi']
mycol = mydb["jibing"]
db_list = []
filename = 'file/zhongyi/jibing.json'
with open(filename,'r',encoding='utf-8') as fl:
    for line in fl:
        dict1 = {}
        line1 = json.loads(re.sub(r'\xa0','',line))
        list1 = line1['tables']
        list2 = line1['contents']
        list3 = line1['jj']
        dict2 = dict(zip(list1, list2))
        dict1['name'] = line1['title'][0]
        if len(list3) > 0:
            dict1.setdefault('about',{})
            for i in range(0,int(len(list3)/2)):
                dict1['about'][list3[2*i]] = list3[2*i+1]
        dict1.setdefault('content',{})
        for k, v in dict2.items():
            dict1['content'][k] = v 
        #dict1[line1['title'][0]]['内容'] = dict2
        db_list.append(dict1)
x = mycol.insert_many(db_list)
print('ok!')

术语数据导入数据库中

In [ ]:
import json
import re
import pymongo

myclient = pymongo.MongoClient('mongodb://10.147.17.6:27017/')
#dblist = myclient.list_database_names()
mydb = myclient['dayi']
mycol = mydb["shuyu"]
db_list = []
filename = 'file/zhongyi/shuyu.json'
with open(filename,'r',encoding='utf-8') as fl:
    for line in fl:
        dict1 = {}
        line1 = json.loads(re.sub(r'\xa0','',line))
        list1 = line1['tables']
        list2 = line1['contents']
        list3 = line1['jj']
        dict2 = dict(zip(list1, list2))
        dict1['name'] = line1['title'][0]
        if len(list3) > 0:
            dict1.setdefault('about',{})
            for s in list3:
                item = s.strip().split(':')
                dict1['about'][item[0]] = item[1]
        dict1.setdefault('content',{})
        for k, v in dict2.items():
            dict1['content'][k] = v 
        #dict1[line1['title'][0]]['内容'] = dict2
        db_list.append(dict1)
x = mycol.insert_many(db_list)
print('ok!')

西医症状数据导入数据库中

In [ ]:
import json
import re
import pymongo

myclient = pymongo.MongoClient('mongodb://10.147.17.6:27017/')
mydb = myclient['dayi']
mycol = mydb["xiyizhengzhuang"]
db_list = []
filename = 'file/zhongyi/xiyizhengzhuang.json'
with open(filename,'r',encoding='utf-8') as fl:
    for line in fl:
        dict1 = {}
        line1 = json.loads(re.sub(r'\xa0','',line))
        list1 = line1['tables']
        list2 = line1['contents']
        list3 = line1['jj']
        dict2 = dict(zip(list1, list2))
        dict1['name'] = line1['title'][0]
        if len(list3) > 0:
            dict1.setdefault('about',{})
            for i in range(0,int(len(list3)/2)):
                dict1['about'][list3[2*i]] = list3[2*i+1]
        dict1.setdefault('content',{})
        for k, v in dict2.items():
            dict1['content'][k] = v 
        #dict1[line1['title'][0]]['内容'] = dict2
        db_list.append(dict1)
x = mycol.insert_many(db_list)
print('ok!')

药剂数据导入数据库中

In [ ]:
import json
import re
import pymongo

myclient = pymongo.MongoClient('mongodb://10.147.17.6:27017/')
#dblist = myclient.list_database_names()
mydb = myclient['dayi']
mycol = mydb["yaoji"]
db_list = []
filename = 'file/zhongyi/yaoji.json'
with open(filename,'r',encoding='utf-8') as fl:
    for line in fl:
        dict1 = {}
        line1 = json.loads(re.sub(r'\xa0','',line))
        list1 = line1['tables']
        list2 = line1['contents']
        list3 = line1['jj']        
        dict2 = dict(zip(list1, list2))        
        if len(list3) > 0: 
            jj = {}
            for s in list3:
                item = s.strip().split(':')
                jj[item[0]] = item[1]        
        dict1['name'] = jj['名称']
        dict1['about'] = jj
        dict1.setdefault('content',{})
        for k, v in dict2.items():
            dict1['content'][k] = v 
        #dict1[line1['title'][0]]['内容'] = dict2
        db_list.append(dict1)
x = mycol.insert_many(db_list)
print('ok!')

药膳数据导入数据库中

In [ ]:
import json
import re
import pymongo

myclient = pymongo.MongoClient('mongodb://10.147.17.6:27017/')
#dblist = myclient.list_database_names()
mydb = myclient['dayi']
mycol = mydb["yaoshan"]
db_list = []
filename = 'file/zhongyi/yaoshan.json'
with open(filename,'r',encoding='utf-8') as fl:
    for line in fl:
        dict1 = {}
        line1 = json.loads(re.sub(r'\xa0','',line))
        list1 = line1['tables']
        list2 = line1['contents']
        list3 = line1['jj']
        dict2 = dict(zip(list1, list2))
        dict1['name'] = line1['title'][0]
        if len(list3) > 0:
            dict1.setdefault('about',{})
            for s in list3:
                item = s.strip().split(':')
                dict1['about'][item[0]] = item[1]
        dict1.setdefault('content',{})
        for k, v in dict2.items():
            dict1['content'][k] = v 
        #dict1[line1['title'][0]]['内容'] = dict2
        db_list.append(dict1)
x = mycol.insert_many(db_list)
print('ok!')

中草药数据导入数据库中

In [ ]:
import json
import re
import pymongo

myclient = pymongo.MongoClient('mongodb://10.147.17.6:27017/')
mydb = myclient['dayi']
mycol = mydb["zhongcaoyao"]
db_list = []
filename = 'file/zhongyi/zhongcaoyao.json'
with open(filename,'r',encoding='utf-8') as fl:
    for line in fl:
        dict1 = {}
        line1 = json.loads(re.sub(r'\xa0','',line))
        list1 = line1['tables']
        list2 = line1['contents']
        list3 = line1['jj']
        dict2 = dict(zip(list1, list2))
        dict1['name'] = line1['title'][0]
        if len(list3) > 0:
            dict1.setdefault('about',{})
            for i in range(0,int(len(list3)/2)):
                dict1['about'][list3[2*i]] = list3[2*i+1]
        dict1.setdefault('content',{})
        for k, v in dict2.items():
            dict1['content'][k] = v 
        #dict1[line1['title'][0]]['内容'] = dict2
        db_list.append(dict1)
x = mycol.insert_many(db_list)
print('ok!')

穴位隶属整理

In [ ]:
import json
import re
import pymongo

myclient = pymongo.MongoClient('mongodb://10.147.17.6:27017/')
mydb = myclient['dayi']
mycol = mydb["xuewei"]
dict1 = {}
list1 = []
for x in mycol.find({},{ "_id": 0,"name":1,"about.隶属":1 }):
    if 'about' in x.keys():
        m_ls = x['about']['隶属']
        dict1.setdefault(m_ls,[])
        dict1[m_ls].append(x['name'])
'''
filename = './file/穴位隶属.json'
with open(filename,'w') as fl:
    json.dump(dict1, fl)     
'''
for item in dict1.keys():
    print(item)

穴位功能、主治统计

In [ ]:
import json
import re
import pymongo

myclient = pymongo.MongoClient('mongodb://10.147.17.6:27017/')
mydb = myclient['dayi']
mycol = mydb["xuewei"]

dict1 = {}
mo = '等$'
mo2 = r'[，、；。]'

for x in mycol.find({},{ "_id": 0,"name":1,"about":1 }):
    dict1.setdefault(x['name'],{})
    if '主治' in x['about'].keys():
        #dict1[x['name']].setdefault('主治',[])
        s = x['about']['主治']
        s1 = re.sub(mo, '', s)
        list1 = re.split(mo2,s1)
        if '' in list1:
            list1.remove('')
        dict1[x['name']]['主治'] = list1
    if '功能' in x['about'].keys():
        #dict1[x['name']].setdefault('主治',[])
        s = x['about']['功能']
        s1 = re.sub(mo, '', s)
        list1 = re.split(mo2,s1)
        if '' in list1:
            list1.remove('')
        dict1[x['name']]['功能'] = list1
        

filename = './file/穴位主治功能统计.json'
with open(filename,'w') as fl:
    json.dump(dict1, fl)

穴位数据导出Excel表

In [ ]:
import json
import openpyxl

filename = 'file/穴位.json'
with open(filename,'r') as fl:
    dict1 = json.load(fl)
filename = 'file/穴位简要情况表.xlsx'
wb = openpyxl.Workbook()
sheet = wb.active
sheet['A1'] = '穴位名称'
sheet['B1'] = '隶属'
sheet['C1'] = '位置'
sheet['D1'] = '主治'
sheet['E1'] = '功能'
sheet['F1'] = '操作'
sheet['G1'] = '主要配伍'
i = 2
for k, v in dict1.items():
    sheet[f'A{i}'] = k
    sheet[f'B{i}'] = v['简介'][0].split(':')[1]
    sheet[f'C{i}'] = v['简介'][1].split(':')[1]
    sheet[f'D{i}'] = v['简介'][2].split(':')[1]
    sheet[f'E{i}'] = v['简介'][3].split(':')[1]
    sheet[f'F{i}'] = v['简介'][4].split(':')[1]
    sheet[f'G{i}'] = v['简介'][5].split(':')[1]      
    i += 1
wb.save(filename)   
print('ok!')

术语数据处理

In [ ]:
import json
import re
import pymongo

myclient = pymongo.MongoClient('mongodb://10.147.17.6:27017/')
mydb = myclient['dayi']
mycol = mydb["shuyu"]
dict1 = {}
list1 = []
for x in mycol.find({},{ "_id": 0,"name":1,"about":1 }):
    if '类别' in x['about'].keys():
        m_lb = x['about']['类别'].replace(' ','')
    else:
        m_lb = '无类别'
    dict1.setdefault(re.sub('\xa0+','',m_lb),[])
    dict1[m_lb].append(x['name'])
filename = './file/术语类别.json'
with open(filename,'w') as fl:
    json.dump(dict1, fl)

疾病数据处理

In [ ]:
import json
import re
import pymongo

myclient = pymongo.MongoClient('mongodb://10.147.17.6:27017/')
mydb = myclient['dayi']
mycol = mydb["jibing"]
dict1 = {}
list1 = []
for x in mycol.find({},{ "_id": 0,"name":1,"about":1 }):
    if '疾病分类' in x['about'].keys():
        m_lb = x['about']['疾病分类'].replace(' ','')
    else:
        m_lb = '无类别'
    dict1.setdefault(re.sub('\xa0+','',m_lb),[])
    dict1[m_lb].append(x['name'])
filename = './file/疾病类别.json'
with open(filename,'w') as fl:
    json.dump(dict1, fl)

中草药数据处理

In [ ]:
import json
import re
import pymongo

myclient = pymongo.MongoClient('mongodb://10.147.17.6:27017/')
mydb = myclient['dayi']
mycol = mydb["zhongcaoyao"]

dict1 = {}
mo = '等$'
mo2 = r'[，、；。]'

for x in mycol.find({},{ "_id": 0,"name":1,"about":1 }):
    dict1.setdefault(x['name'],{})
    if '别名' in x['about'].keys():
        #dict1[x['name']].setdefault('主治',[])
        s = x['about']['别名']
        s1 = re.sub(mo, '', s)
        list1 = re.split(mo2,s1)
        if '' in list1:
            list1.remove('')
        dict1[x['name']]['别名'] = list1
    if '功能' in x['about'].keys():
        #dict1[x['name']].setdefault('主治',[])
        s = x['about']['功能']
        s1 = re.sub(mo, '', s)
        list1 = re.split(mo2,s1)
        if '' in list1:
            list1.remove('')
        dict1[x['name']]['功能'] = list1
    if '主治' in x['about'].keys():
        #dict1[x['name']].setdefault('主治',[])
        dict1[x['name']]['主治'] = s = x['about']['主治']
        

filename = './file/中草药主治功能统计.json'
with open(filename,'w') as fl:
    json.dump(dict1, fl)

药剂数据处理

In [ ]:
import json
import re
import pymongo

myclient = pymongo.MongoClient('mongodb://10.147.17.6:27017/')
mydb = myclient['dayi']
mycol = mydb["yaoji"]

dict1 = {}
mo = '等$'
mo2 = r'[，、；。]'

for x in mycol.find({},{ "_id": 0,"name":1,"about":1 }):
    dict1.setdefault(x['name'],{})
    if '功用' in x['about'].keys():
        #dict1[x['name']].setdefault('主治',[])
        s = x['about']['功用']
        s1 = re.sub(mo, '', s)
        list1 = re.split(mo2,s1)
        if '' in list1:
            list1.remove('')
        dict1[x['name']]['功用'] = list1
    if '主治' in x['about'].keys():
        #dict1[x['name']].setdefault('主治',[])
        dict1[x['name']]['主治'] = s = x['about']['主治']
        

filename = './file/药剂主治功用统计.json'
with open(filename,'w') as fl:
    json.dump(dict1, fl)

药膳数据处理

In [ ]:
import json
import re
import pymongo

myclient = pymongo.MongoClient('mongodb://10.147.17.6:27017/')
mydb = myclient['dayi']
mycol = mydb["yaoshan"]

dict1 = {}
mo = '等$'
mo2 = r'[，、；。]'

for x in mycol.find({},{ "_id": 0,"name":1,"about":1 }):
    dict1.setdefault(x['name'],{})
    if '功效' in x['about'].keys():
        #dict1[x['name']].setdefault('主治',[])
        s = x['about']['功效']
        s1 = re.sub(mo, '', s)
        list1 = re.split(mo2,s1)
        if '' in list1:
            list1.remove('')
        dict1[x['name']]['功效'] = list1
    if '相关疾病' in x['about'].keys():
        #dict1[x['name']].setdefault('主治',[])
        s = x['about']['相关疾病']
        s1 = re.sub(mo, '', s)
        list1 = re.split(mo2,s1)
        if '' in list1:
            list1.remove('')
        dict1[x['name']]['相关疾病'] = list1    
        

filename = './file/药膳功能统计.json'
with open(filename,'w') as fl:
    json.dump(dict1, fl)

北海炼化体检数据提取

In [ ]:
import json
import openpyxl
import os,sys,shutil

file_name = 'file/中石化北海炼化2020年团体体检报告.xlsx'
wb = openpyxl.load_workbook(file_name)
sheet = wb.active
dict1 = {}
for n in range(1,sheet.max_row+1):
    bh = sheet.cell(n,2).value
    name = sheet.cell(n,3).value
    xb = sheet.cell(n,4).value
    nl = sheet.cell(n,5).value
    bz = sheet.cell(n,7).value
    dict1.setdefault(bh,{})
    dict1[bh]['姓名'] = name
    dict1[bh]['性别'] = xb
    dict1[bh]['年龄'] = nl
    dict1[bh].setdefault('病症',[])
    dict1[bh]['病症'].append(bz.strip())
wb.close()
#print(dict1)
    
        
filename = 'file/中石化北海炼化2020年体检人员情况表.xlsx'
wb = openpyxl.Workbook()
sheet = wb.active
sheet['A1'] = '体检编号'
sheet['B1'] = '姓名'
sheet['C1'] = '性别'
sheet['D1'] = '年龄'
sheet['E1'] = '异常名称'

i =2
for k, v in dict1.items():
    sheet[f'A{i}'] = k
    sheet[f'B{i}'] = v['姓名']
    sheet[f'C{i}'] = v['性别']
    sheet[f'D{i}'] = v['年龄']
    sheet[f'E{i}'] = ','.join(v['病症'])    
    i += 1

wb.save(filename)

excel数据读取

In [ ]:
import openpyxl
import json

filename = 'data/长岭全成绩.xlsx'
wb = openpyxl.load_workbook(filename)
sheet = wb.active
data1 =list(sheet.values)
del data1[0]
print(data1)
#for data in data1:
#    print(data)
In [ ]:
import openpyxl
import json
import pymongo

myclient = pymongo.MongoClient('mongodb://512song:songyi@192.168.3.100:27017/tice')
#dblist = myclient.list_database_names()
mydb = myclient['tice']
 
collist = mydb. list_collection_names()
print(collist)
In [ ]:
In [ ]:
import json
import pymongo

myclient = pymongo.MongoClient('mongodb://10.147.17.6:27017/')
#dblist = myclient.list_database_names()
mydb = myclient['dayi']
 
collist = mydb. list_collection_names()
print(collist)
Reference in new issue View Git Blame Copy Permalink