python es 批量写入

9 篇文章 0 订阅
from elasticsearch import Elasticsearch
from elasticsearch import helpers
from requests.auth import HTTPBasicAuth
from sqlalchemy import create_engine
import datetime


class SearchConfig:
  
    def __init__(self):
        self.address = 'http://****'
        # self.auth = [c['es']['username'], c['es']['password']]
        self.indexTikus ='new_pre_score'
        self.indexTikustype = 'pre_score'


class Search:
    config = None
    es = None

    def __init__(self, config=None):
        self.engine = create_engine(
            'mysql+pymysql://**u')
        if config is not None:
            self.config = config
            # self.es = Elasticsearch(self.config.address, http_auth=config.auth)
            self.es = Elasticsearch(self.config.address)

    def initAllIndex(self):
        # 创建题库预测索引
        _index_tiku_score_mappings = {
            'mappings': {
                '%s' % self.config.indexTikustype: {
                    'properties': {
                        'id': {
                            'type': 'integer'
                        },
                        'user_id': {
                            'type': 'integer'
                        },
                        'pre_score': {
                            'type': 'float'
                        },
                        'create_time': {
                            'type': 'integer'
                        },
                        'update_time': {
                            'type': 'integer'
                        },
                        'status': {
                            'type': 'integer'
                        },
                        'item_id': {
                            'type': 'integer'
                        }
                    }
                }
            }

        }
        self.es.indices.create(index=self.config.indexTikus, body=_index_tiku_score_mappings, ignore=400)

    def add_pre_score(self, id_, user_id, pre_score, create_time, update_time, status, item_id):
        try:
            index_type = '%s' % self.config.indexTikustype
            body = {'id': id_,
                    'user_id': user_id,
                    "pre_score": pre_score,
                    'create_time': create_time,
                    'update_time': update_time,
                    'status': status,
                    'item_id': item_id}
            res = self.es.index(index=self.config.indexTikus, doc_type=index_type, body=body, id=id_)
            if res['result'] == 'created' or res['result'] == 'updated':
                return True
        except:
            return False

    def get_pre_score(self, id):
        """
        根据id获取产品信息
        :param id:产品id
        :return:字典或None
        """
        try:
            indexType = '%s-type' % self.config.indexTikustype
            res = self.es.get(index=self.config.indexTikus, doc_type=indexType, id=id)
            print(res)
            if res['found'] == True and res['_source'] is not None:
                return res['_source']
        except:
            return None

    def save_score_to_es(self):
        with self.engine.connect() as conn:
            sql = """
            select `id`,`user_id`,`pre_score`,`create_time`,`update_time`,`status`,`item_id` from `t_user` 
                       """
            res = conn.execute(sql).fetchall()
        actions = []
        for i in res:
            action = {
                "_index": self.config.indexTikus,
                "_type": '%s' % self.config.indexTikustype,
                "_id": int(i[0]),
                "body": {'id': int(i[0]),
                         'user_id': i[1],
                         "pre_score": i[2],
                         'create_time': i[3],
                         'update_time': i[4],
                         'status': i[5],
                         'item_id': i[6]}

            }
            actions.append(action)
        helpers.bulk(self.es, actions)


config = SearchConfig()
search = Search(config)
search.initAllIndex()
search.add_pre_score(id_=100,update_time=1,user_id=1,status=1,create_time=1,pre_score=100,item_id=1)
res = search.get_pre_score(683918)
# # # print(res)
print(datetime.datetime.now())
search.save_score_to_es()
print(datetime.datetime.now())
  • 0
    点赞
  • 0
    收藏
    觉得还不错? 一键收藏
  • 0
    评论
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值