pyspark或python连接Elasticsearch

code学习社

已于 2024-03-14 10:30:08 修改

阅读量328

点赞数

分类专栏：大数据 spark 文章标签： python elasticsearch spark

于 2020-12-03 09:26:51 首次发布

本文链接：https://blog.csdn.net/qq_39142369/article/details/110518837

版权

大数据同时被 2 个专栏收录

2 篇文章 0 订阅

订阅专栏

spark

1 篇文章 0 订阅

订阅专栏

#
from pyspark import SparkConf

class EsSparkClient:
    def __init__(self,host,port,username=None,password=None):
        self.host = host
        self.port = port
        self.usename = usename
        self.password = password


    def Pyspark_Read(self,index,query=None):
        es_read_conf = {
            "es.nodes": self.host,
            "es.port": self.port,
            "es.net.http.auth.user": self.username,
            "es.net.http.auth.pass": self.password,
            "es.resource": index,
            "es.query": query
        }
        es_rdd = SparkConf.newAPIHadoopRDD(
            inputFormatClass="org.elasticsearch.hadoop.mr.EsInputFormat",
            keyClass="org.apache.hadoop.io.NullWritable",
            valueClass="org.elasticsearch.hadoop.mr.LinkedMapWritable",
            conf=es_read_conf)
        return es_rdd


    def Pyspark_Wrinfo(self,rdd,index,id=None):
        # 保存到ES
        es_write_conf = {
            "es.nodes": self.host,
            "es.port": self.port,
            "es.net.http.auth.user": self.username,
            "es.net.http.auth.pass": self.password,
            "es.resource": index,
            "es.mapping.id": id
        }
        rdd.saveAsNewAPIHadoopFile(
            path='-',
            outputFormatClass="org.elasticsearch.hadoop.mr.EsOutputFormat",
            keyClass="org.apache.hadoop.io.NullWritable",
            valueClass="org.elasticsearch.hadoop.mr.LinkedMapWritable",
            conf=es_write_conf)

python连接es

#
from elasticsearch import Elasticsearch

class EsClient:
    def __init__(self,host,port,username=None,password=None):
        self.host = host
        self.port = port
        self.usename = usename
        self.password = password
  

    def Es_Test(self):
        esClient = Elasticsearch([{"host": self.ip, "port": self.port}], http_auth=(self.username, self.password))
        return esClient