Flask项目手动实现mysql数据同步到Elasticsearch

一、新建索引和type

# 建立索引
from elasticsearch_dsl import DocType, Text, Integer, Keyword
from elasticsearch_dsl.connections import connections

connections.create_connection(hosts=["127.0.0.1:9200"])


class AbstractType(DocType):
    # 摘记的数据
    """
    'id_' :'id',
    'note_id_':'笔记本id',
    'content_':'内容',
    'star_':'星级,1:1颗星，2:2颗星，3:3颗星',
    'create_time_':'创建时间',
    'is_delete_':'0:未删除，1：已删除'
    """

    # 建立 索引和doc
    id_ = Keyword()  # 去重
    note_id_ = Text()
    content_ = Text(analyzer="ik_max_word")
    star_ = Integer()
    create_time_ = Text()
    is_delete_ = Text()

    class Index:
        name = 'abstract_index'  # 索引名称
        doc_type = 'note_abstract'  # type 类似数据库中的表[table]

二、将mysql上的数据按规则放入elasticsearch中

from common import db_helper, log_helper
from config import dict_config
from es_index import AbstractType

logger = log_helper.get_logger('db_helper')


class AbstractMesToEs():
    def get_abstract_data(self):  # 获取数据库该用户表的数据
        sql = 'select id_ from note_abstract order by create_time_ desc '
        result = db_helper.select(sql)
        result_list = [item[key] for item in result for key in item]
        if result_list:
            for res in result_list:
                wheres = [
                    ('id_', dict_config.where_equal, res)
                ]
                results = db_helper.find(table_name='note_abstract', wheres=wheres)
                if results:
                    data = results[0]
                    if data:
                        self.data_to_es(data)  # 调用 data_to_es方法向数据库中逐条插数据

    def data_to_es(self, item):  # 将数据写入到ES中
        abstract = AbstractType()  # 索引
        abstract.id_ = item['id_']  # 字段
        abstract.note_id_ = item['note_id_']
        abstract.content_ = item['content_']  
        abstract.star_ = item['star_']  
        abstract.create_time_ = item['create_time_']  
        abstract.is_delete_ = item['is_delete_']  
        # 保存
        try:
            abstract.save()
        except Exception as e:
            e_args_msgs = ''
            if e.args:
                for e_args_msg in e.args:
                    e_args_msgs += str(e_args_msg)
            logger.error('Failed to ElesticSearch:' + e_args_msgs)  # 将异常写入到日志中


if __name__ == "__main__":
    item = AbstractMesToEs()
    item.get_abstract_data()

三、使用elasticsearch开始搜索

from common import web_helper
from elasticsearch import Elasticsearch
from mysql_data_to_es import AbstractMesToEs

es = Elasticsearch()


# 搜索
def search_abstract():
    content_ = web_helper.get_param('content_')  # 搜索内容

    response = AbstractMesToEs().get_abstract_data()  # mysql数据同步到Elasticsearch
    if response == 1:
        body = {
            "query": {
                "match": {
                    "content_": content_
                }
            },
            "collapse": {
                "field": "id_.keyword"  # 指定去重字段
            }
        }
        res = es.search(index='abstract_index', body=body)  # 每个搜索index和doc_type在es_index.py配置
        return res['hits']['hits']