医药问答系统（一）python将excel或json数据处理后存入neo4j

1.将excel的数据存入neo4j

本程序使用Jupyter运行，若使用pycharm还需安装py2neo

pip install py2neo==5.0b1 注意版本，要不对应不了

文档：https://py2neo.org/v4/index.html

• 在第一个文件中
  • 用panda读取excel中的数据
  • 将取出的数据中的实体去重，另外保存到变量中，最终用于创建结点：例中data_extraction()
  • 将读取到的数据抽取需要的列，再次拼成dataframe，最终用于创建关系：例中relation_extraction()
• 在第二个文件中
  • 通过py2neo工具将这些结点和关系导入neo4j

# -*- coding: utf-8 -*-
import os
import pandas as pd

invoice_data = pd.read_excel('D:/BaiduNetdiskDownload/pandasDemo/Invoice_data_Demo.xls', header=0)
#print("取到的数据：", invoice_data)


#从原数据中将需要创建的实体(买方、卖方)节点抽取出来，将所有的数据全部保存到数组中
def data_extraction():
    """节点数据抽取"""

    # 取出所有买方名称到node_buy_key[]
    node_buy_key = []
    for i in range(0, len(invoice_data)):
        node_buy_key.append(invoice_data['购买方名称'][i])
    
    node_sell_key = []
    for i in range(0, len(invoice_data)):
        node_sell_key.append(invoice_data['销售方名称'][i])
        
    # 去除重复的买方/卖方名称
    node_buy_key = list(set(node_buy_key))
    node_sell_key = list(set(node_sell_key))

    #除了第一列，将所有数据按行取出存到node_list_value[]
    node_list_value = []
    for i in range(0, len(invoice_data)):
        for n in range(1, len(invoice_data.columns)):
            node_list_value.append(invoice_data[invoice_data.columns[n]][i])


    # set()去重,list()转化成列表
    node_list_value = list(set(node_list_value))
    # 将list中浮点及整数类型全部转成string类型
    node_list_value = [str(i) for i in node_list_value]

    #返回所有去重后的购买方名称，去重后的销售方名称，以及所有数据
    return node_buy_key, node_sell_key,node_list_value

#将原数据中需要用到的列抽取出来，并且再次拼成excel的样子
def relation_extraction():
    """联系数据抽取"""

    links_dict = {}
    sell_list = []  # 销售方列表
    money_list = []  # 交易额列表
    buy_list = []  # 购买方列表

    # 取列名--“金额”
    # print("*****", invoice_data.columns[19], "********")

    for i in range(0, len(invoice_data)):
        money_list.append(invoice_data[invoice_data.columns[19]][i])# 将所有金额依次导入
        sell_list.append(invoice_data[invoice_data.columns[10]][i])# 将所有销售方依次导入
        buy_list.append(invoice_data[invoice_data.columns[6]][i])# 将所有购买方依次导入


    # 将数据中int类型全部转成string
    sell_list = [str(i) for i in sell_list]
    buy_list = [str(i) for i in buy_list]
    money_list = [str(i) for i in money_list]
    
    # 整合数据，将三个list整合成一个dict，字典里面存储了多个数组的首地址
    links_dict['buy'] = buy_list
    links_dict['money'] = money_list
    links_dict['sell'] = sell_list

    # 将数据转成DataFrame---类似excel的格式
    df_data = pd.DataFrame(links_dict)
    return df_data

# 实例化
create_data = DataToNeo4j()

# 调用create_data对象的方法创建结点,传参时调用本文件的data_extraction方法
create_data.create_node(data_extraction()[0], data_extraction()[1])
create_data.create_relation(relation_extraction())

# -*- coding: utf-8 -*-
from py2neo import Node, Graph, Relationship,NodeMatcher

"""将实体列表和关系dataframe存入neo4j"""
class DataToNeo4j(object):


    def __init__(self):
        """建立连接"""
        link = Graph("http://localhost:7474", username="neo4j", password="123456")
        self.graph = link

        # 定义label即节点类型
        self.buy = 'buy'
        self.sell = 'sell'
        self.graph.delete_all()
        self.matcher = NodeMatcher(link)

        # 三引号是注释，官方小例子,帮助理解
        #Node()定义结点，Relationship()定义关系，create()创建结点或关系
        """
        node3 = Node('animal' , name = 'cat')
        node4 = Node('animal' , name = 'dog')  
        node2 = Node('Person' , name = 'Alice')
        node1 = Node('Person' , name = 'Bob')  
        r1 = Relationship(node2 , 'know' , node1)    
        r2 = Relationship(node1 , 'know' , node3) 
        r3 = Relationship(node2 , 'has' , node3) 
        r4 = Relationship(node4 , 'has' , node2)    
        self.graph.create(node1)
        self.graph.create(node2)
        self.graph.create(node3)
        self.graph.create(node4)
        self.graph.create(r1)
        self.graph.create(r2)
        self.graph.create(r3)
        self.graph.create(r4)
        """


    def create_node(self, node_buy_key,node_sell_key):
        """建立节点"""
        for name in node_buy_key:
            buy_node = Node(self.buy, name=name)
            self.graph.create(buy_node)
        for name in node_sell_key:
            sell_node = Node(self.sell, name=name)
            self.graph.create(sell_node)
            
        

    def create_relation(self, df_data):
        """建立联系"""      
        m = 0
        for m in range(0, len(df_data)):
            try:    
                rel = Relationship(self.matcher.match(self.buy).where("_.name=" + "'" + df_data['buy'][m] + "'").first(),
                                   df_data['money'][m], self.matcher.match(self.sell).where("_.name=" + "'" + df_data['sell'][m] + "'").first())

                self.graph.create(rel)
            except AttributeError as e:
                print(e, m)
            

Invoice_data_Demo.xls
链接：https://pan.baidu.com/s/1lgSg47YWW-6sfC1T5D5P2w
提取码：uxg6

2.将json数据存入neo4j

本程序使用Jupyter运行

• 读取json文件
• 自定义函数read_nodes()通过循环判断
  • 如果是疾病的属性，记录在相应的列表中
  • 如果是可提取为实体的属性，不仅要记录在相应的列表中，还要记录实体间关系
  • 每一轮循环将所有属性拼接成一个字典，存入disease_infos[]
• 自定义函数create_diseases_nodes(disease_infos[])生成疾病类结点：Node()定义结点；create()在neo4j中创建结点
• 自定义函数create_node()生成其他所有类型的结点：Node()定义结点；create()在neo4j中创建结点
• 自定义函数create_relationship()创建关系：利用sql语句查找两结点并创建关系
  • 将关系列表中的两个实体中间用“###”隔开，再用set()去重
  • 将去重后的列表用split分开保存在两个变量中，拼接出sql语句
  • 用run()执行sql语句
    

#选材自开源项目(刘焕勇，中国科学院软件研究所)，数据集来自互联网爬虫数据
import os
import json
from py2neo import Graph,Node

class MedicalGraph:
    def __init__(self):
        cur_dir = 'D:\\BaiduNetdiskDownload\\QAMedicalKG'
        self.data_path = os.path.join(cur_dir, 'data/medical2.json')
        #建立连接
        self.g = Graph("http://localhost:7474", username="neo4j", password="123456")

        
    '''读取文件：分别记录所有结点，关系，以及疾病全部信息的字典'''
    def read_nodes(self):
        # 用于记录结点：共７类节点
        drugs = [] # 药品
        foods = [] #　食物
        checks = [] # 检查
        departments = [] #科室
        producers = [] #药品大类
        diseases = [] #记录疾病的名称
        symptoms = []#症状

        # 用于记录实体间关系
        rels_department = [] #　科室－科室关系
        rels_noteat = [] # 疾病－忌吃食物关系
        rels_doeat = [] # 疾病－宜吃食物关系
        rels_recommandeat = [] # 疾病－推荐吃食物关系
        rels_commonddrug = [] # 疾病－通用药品关系
        rels_recommanddrug = [] # 疾病－热门药品关系
        rels_check = [] # 疾病－检查关系
        rels_drug_producer = [] # 厂商－药物关系
        rels_symptom = [] #疾病症状关系
        rels_acompany = [] # 疾病并发关系
        rels_category = [] #　疾病与科室之间的关系

        #用于记录疾病的所有信息
        disease_infos = []
        
        count = 0
        
        
        for data in open(self.data_path):
            disease_dict = {}
            count += 1
            
            #读取json文件数据
            data_json = json.loads(data)
            
            #将 JSON 对象转换为 Python 字典
            disease = data_json['name']
            diseases.append(disease)
            #这部分用于记录疾病的属性
            disease_dict['name'] = disease
            disease_dict['desc'] = ''
            disease_dict['prevent'] = ''
            disease_dict['cause'] = ''
            disease_dict['easy_get'] = ''
            disease_dict['cure_department'] = ''
            disease_dict['cure_way'] = ''
            disease_dict['cure_lasttime'] = ''
            disease_dict['symptom'] = ''
            disease_dict['cured_prob'] = ''
            
            #如果该条json记录中存在下述字段：记录字段、关系
            if 'symptom' in data_json:
                symptoms += data_json['symptom']#记录字段
                for symptom in data_json['symptom']:
                    rels_symptom.append([disease, symptom])#记录关系

            if 'acompany' in data_json:
                for acompany in data_json['acompany']:
                    rels_acompany.append([disease, acompany])

            if 'desc' in data_json:
                disease_dict['desc'] = data_json['desc']

            if 'prevent' in data_json:
                disease_dict['prevent'] = data_json['prevent']

            if 'cause' in data_json:
                disease_dict['cause'] = data_json['cause']

            if 'get_prob' in data_json:
                disease_dict['get_prob'] = data_json['get_prob']

            if 'easy_get' in data_json:
                disease_dict['easy_get'] = data_json['easy_get']
            
            #如果对于一个疾病对应了门诊部，还要记录部门之间的关系
            if 'cure_department' in data_json:
                cure_department = data_json['cure_department']
                if len(cure_department) == 1:
                    rels_category.append([disease, cure_department[0]])
                if len(cure_department) == 2:
                    big = cure_department[0]
                    small = cure_department[1]
                    rels_department.append([small, big])
                    rels_category.append([disease, small])

                disease_dict['cure_department'] = cure_department
                departments += cure_department

            if 'cure_way' in data_json:
                disease_dict['cure_way'] = data_json['cure_way']

            if  'cure_lasttime' in data_json:
                disease_dict['cure_lasttime'] = data_json['cure_lasttime']

            if 'cured_prob' in data_json:
                disease_dict['cured_prob'] = data_json['cured_prob']

            if 'common_drug' in data_json:
                common_drug = data_json['common_drug']
                for drug in common_drug:
                    rels_commonddrug.append([disease, drug])
                drugs += common_drug

            if 'recommand_drug' in data_json:
                recommand_drug = data_json['recommand_drug']
                drugs += recommand_drug
                for drug in recommand_drug:
                    rels_recommanddrug.append([disease, drug])

            if 'not_eat' in data_json:
                not_eat = data_json['not_eat']
                for _not in not_eat:
                    rels_noteat.append([disease, _not])

                foods += not_eat
                do_eat = data_json['do_eat']
                for _do in do_eat:
                    rels_doeat.append([disease, _do])

                foods += do_eat
                recommand_eat = data_json['recommand_eat']

                for _recommand in recommand_eat:
                    rels_recommandeat.append([disease, _recommand])
                foods += recommand_eat

            if 'check' in data_json:
                check = data_json['check']
                for _check in check:
                    rels_check.append([disease, _check])
                checks += check
            if 'drug_detail' in data_json:
                drug_detail = data_json['drug_detail']
                producer = [i.split('(')[0] for i in drug_detail]
                rels_drug_producer += [[i.split('(')[0], i.split('(')[-1].replace(')', '')] for i in drug_detail]
                producers += producer
            disease_infos.append(disease_dict)
        return set(drugs), set(foods), set(checks), set(departments), set(producers), set(symptoms), set(diseases), disease_infos,\
               rels_check, rels_recommandeat, rels_noteat, rels_doeat, rels_department, rels_commonddrug, rels_drug_producer, rels_recommanddrug,\
               rels_symptom, rels_acompany, rels_category

    '''建立节点'''
    def create_node(self, label, nodes):
        count = 0
        for node_name in nodes:
            node = Node(label, name=node_name)
            self.g.create(node)
            count += 1
            #print(count, len(nodes))
        return

    '''创建知识图谱中心疾病的节点'''
    def create_diseases_nodes(self, disease_infos):
        count = 0
        #print(disease_infos)
        for disease_dict in disease_infos:
            node = Node("Disease", name=disease_dict['name'], desc=disease_dict['desc'],
                        prevent=disease_dict['prevent'] ,cause=disease_dict['cause'],
                        easy_get=disease_dict['easy_get'],cure_lasttime=disease_dict['cure_lasttime'],
                        cure_department=disease_dict['cure_department']
                        ,cure_way=disease_dict['cure_way'] , cured_prob=disease_dict['cured_prob'])
            self.g.create(node)
            count += 1
            #print(count)
        return

    '''创建知识图谱实体节点类型schema'''
    def create_graphnodes(self):
        #调用read_nodes()：因为该函数的返回值很多，所以需要一堆变量接收
        Drugs, Foods, Checks, Departments, Producers, Symptoms, Diseases, disease_infos,rels_check, rels_recommandeat, rels_noteat, rels_doeat, rels_department, rels_commonddrug, rels_drug_producer, rels_recommanddrug,rels_symptom, rels_acompany, rels_category = self.read_nodes()
        #创建疾病类结点
        self.create_diseases_nodes(disease_infos)
        #创建其他所有结点
        self.create_node('Drug', Drugs)
        self.create_node('Food', Foods)
        self.create_node('Check', Checks)
        self.create_node('Department', Departments)
        self.create_node('Producer', Producers)
        self.create_node('Symptom', Symptoms)
        return


    '''创建实体关系边'''
    def create_graphrels(self):
        #调用read_nodes()：因为该函数的返回值很多，所以需要一堆变量接收
        Drugs, Foods, Checks, Departments, Producers, Symptoms, Diseases, disease_infos, rels_check, rels_recommandeat, rels_noteat, rels_doeat, rels_department, rels_commonddrug, rels_drug_producer, rels_recommanddrug,rels_symptom, rels_acompany, rels_category = self.read_nodes()
        
        self.create_relationship('Disease', 'Food', rels_recommandeat, 'recommand_eat', '推荐食谱')
        self.create_relationship('Disease', 'Food', rels_noteat, 'no_eat', '忌吃')
        self.create_relationship('Disease', 'Food', rels_doeat, 'do_eat', '宜吃')
        self.create_relationship('Department', 'Department', rels_department, 'belongs_to', '属于')
        self.create_relationship('Disease', 'Drug', rels_commonddrug, 'common_drug', '常用药品')
        self.create_relationship('Producer', 'Drug', rels_drug_producer, 'drugs_of', '生产药品')
        self.create_relationship('Disease', 'Drug', rels_recommanddrug, 'recommand_drug', '好评药品')
        self.create_relationship('Disease', 'Check', rels_check, 'need_check', '诊断检查')
        self.create_relationship('Disease', 'Symptom', rels_symptom, 'has_symptom', '症状')
        self.create_relationship('Disease', 'Disease', rels_acompany, 'acompany_with', '并发症')
        self.create_relationship('Disease', 'Department', rels_category, 'belongs_to', '所属科室')

    '''创建实体关联边'''
    def create_relationship(self, start_node, end_node, edges, rel_type, rel_name):
        count = 0
        # 去重处理
        set_edges = []
        for edge in edges:
            #edge是关系，包含两实体，以“###为分隔符，将edge的每一字符分隔开”
            set_edges.append('###'.join(edge))
        #print(set_edges)
        
        all = len(set(set_edges))
        for edge in set(set_edges):
            edge = edge.split('###')
            p = edge[0]
            q = edge[1]
            query = "match(p:%s),(q:%s) where p.name='%s'and q.name='%s' create (p)-[rel:%s{name:'%s'}]->(q)" % (
                start_node, end_node, p, q, rel_type, rel_name)
            try:
                self.g.run(query)
                count += 1
#                 print(rel_type, count, all)
            except Exception as e:
                print(e)
        return



if __name__ == '__main__':
    handler = MedicalGraph()
    handler.create_graphnodes()
    handler.create_graphrels()

医药问答系统完整项目下载地址：
https://download.csdn.net/download/floracuu/15927225?spm=1001.2014.3001.5501

下一篇 医药问答系统（二）提取用户问题关键字以及识别问题类型