ElasticSearch-3

最新推荐文章于 2022-11-01 19:07:23 发布

syc0616

最新推荐文章于 2022-11-01 19:07:23 发布

阅读量103

点赞数

分类专栏： ElasticSearch

本文链接：https://blog.csdn.net/syc0616/article/details/118477671

版权

ElasticSearch 专栏收录该内容

3 篇文章 0 订阅

订阅专栏

Kibana

Kibana 是一个免费且开放的用户界面，能够让你对 Elasticsearch 数据进行可视化，并让你在 Elastic Stack 中进行导航。你可以进行各种操作，从跟踪查询负载，到理解请求如何流经你的整个应用，都能轻松完成

修改 config/kibana.yml 文件

# 默认端口

server.port: 5601

# ES 服务器的地址

elasticsearch.hosts: ["http://localhost:9200"]

# 索引名

kibana.index: ".kibana"

# 支持中文

i18n.locale: "zh-CN"

Windows 环境下执行 bin/kibana.bat 文件

通过浏览器访问 : http://localhost:5601

ElasticSearch集成

1.Spark Streaming 框架集成

<dependency>

<groupId>org.elasticsearch.client</groupId>

<artifactId>elasticsearch-rest-high-level-client</artifactId>

<version>7.8.0</version>

</dependency>



<dependency>

<groupId>org.apache.logging.log4j</groupId>

<artifactId>log4j-api</artifactId>

<version>2.8.2</version>

</dependency>

<dependency>

<groupId>org.apache.logging.log4j</groupId>

<artifactId>log4j-core</artifactId>

<version>2.8.2</version>

</dependency>























</dependencies>

</project>

Flink框架集成

数据精准一次性处理（Exactly-Once）

乱序数据，迟到数据

低延迟，高吞吐，准确性

容错性

Apache Flink 是一个框架和分布式处理引擎，用于对无界和有界数据流进行有状态计算。

<?xml version="1.0" encoding="UTF-8"?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0

http://maven.apache.org/xsd/maven-4.0.0.xsd">

<modelVersion>4.0.0</modelVersion>

<groupId>com.atguigu.es</groupId>

<artifactId>flink-elasticsearch</artifactId>

<version>1.0</version>

<properties>

<maven.compiler.source>8</maven.compiler.source>

<maven.compiler.target>8</maven.compiler.target>

</properties>

<dependencies>

<dependency>

<groupId>org.apache.flink</groupId>

<artifactId>flink-scala_2.12</artifactId>

<version>1.12.0</version>

</dependency>

<dependency>

<groupId>org.apache.flink</groupId>

<artifactId>flink-streaming-scala_2.12</artifactId>

<version>1.12.0</version>

</dependency>

<dependency>

<groupId>org.apache.flink</groupId>

<artifactId>flink-clients_2.12</artifactId>

<version>1.12.0</version>

</dependency>

<dependency>

<groupId>org.apache.flink</groupId>

<artifactId>flink-connector-elasticsearch7_2.11</artifactId>

<version>1.12.0</version>

</dependency>



<dependency>

<groupId>com.fasterxml.jackson.core</groupId>

<artifactId>jackson-core</artifactId>

<version>2.11.1</version>

</dependency>

</dependencies>

</project>

package com.atguigu.es;

import org.apache.flink.api.common.functions.RuntimeContext;

import org.apache.flink.streaming.api.datastream.DataStreamSource;

import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;

import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSinkFunctio n;

import org.apache.flink.streaming.connectors.elasticsearch.RequestIndexer;

import org.apache.flink.streaming.connectors.elasticsearch7.ElasticsearchSink;

import org.apache.http.HttpHost;

import org.elasticsearch.action.index.IndexRequest;

import org.elasticsearch.client.Requests;

import java.util.ArrayList;

import java.util.HashMap;

import java.util.List; import java.util.Map;

public class FlinkElasticsearchSinkTest {

public static void main(String[] args) throws Exception {

StreamExecutionEnvironment                         env                     = StreamExecutionEnvironment.getExecutionEnvironment();

DataStreamSource<String>        source =   env.socketTextStream("localhost",

9999);

List<HttpHost> httpHosts = new ArrayList<>(); httpHosts.add(new HttpHost("127.0.0.1", 9200, "http"));

//httpHosts.add(new HttpHost("10.2.3.1", 9200, "http"));

// use a ElasticsearchSink.Builder to create an ElasticsearchSink ElasticsearchSink.Builder<String>        esSinkBuilder       =       new

ElasticsearchSink.Builder<>(

httpHosts,

new ElasticsearchSinkFunction<String>() {

public IndexRequest createIndexRequest(String element) { Map<String, String> json = new HashMap<>(); json.put("data", element);

return Requests.indexRequest().index("my-index")//.type("my-type").source(json);

}

@Override

public                void process(String   element, RuntimeContext   ctx, RequestIndexer indexer) {

indexer.add(createIndexRequest(element));}

}

);

// configuration for the bulk requests; this instructs the sink to emit after every element, otherwise they would be buffered

esSinkBuilder.setBulkFlushMaxActions(1);

// provide a RestClientFactory for custom configuration on the internally created REST client

//   esSinkBuilder.setRestClientFactory(

//          restClientBuilder -> {

//              restClientBuilder.setDefaultHeaders(...)

//              restClientBuilder.setMaxRetryTimeoutMillis(...)

//              restClientBuilder.setPathPrefix(...)

//              restClientBuilder.setHttpClientConfigCallback(...)

//          }

//   );

source.addSink(esSinkBuilder.build()); env.execute("flink-es");

}

}