CentOS 7 搭建ElasticSearch 7.13.2 集群并与Springboot集成

CentOS 7 搭建ElasticSearch 7.13.2 集群并与Springboot集成

ElasticSearch单机版搭建教程 CentOS 7 搭建ELK 7.13.2 环境

一、基础设置

1.hosts

cat >> /etc/hosts <<EOF
192.168.63.17 node-1
192.168.63.19 node-2
EOF

2.设置hostname

• node-1

hostname node-1 && hostnamectl set-hostname node-1

• node-2

hostname node-2 && hostnamectl set-hostname node-2

3.防火墙配置

firewall-cmd --zone=public --add-port=9200/tcp --permanent
firewall-cmd --zone=public --add-port=9300/tcp --permanent
firewall-cmd --reload

4.selinux

sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
sed -i 's/SELINUX=permissive/SELINUX=disabled/g' /etc/selinux/config
setenforce 0
getenforce

二、软件安装和配置

1.安装

rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch
cat > /etc/yum.repos.d/elastic.repo <<EOF
[elasticsearch]
name=Elasticsearch repository for 7.x packages
baseurl=https://artifacts.elastic.co/packages/7.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=0
autorefresh=1
type=rpm-md

[kibana-7.x]
name=Kibana repository for 7.x packages
baseurl=https://artifacts.elastic.co/packages/7.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md

[logstash-7.x]
name=Elastic repository for 7.x packages
baseurl=https://artifacts.elastic.co/packages/7.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md

EOF
yum search elasticsearch --showduplicates
yum --disablerepo=\* --enablerepo=elasticsearch install -y elasticsearch

2.环境变量

elasticsearch 依赖JDK环境，推荐使用JDK 11，可使用自己的JDK版本，也可以使用ES内置的JDK；JDK的路径通过ES_JAVA_HOME环境变量指定。

cat >> /etc/profile << \EOF
# set embedded jdk
ES_JAVA_HOME=/usr/share/elasticsearch/jdk
ES_HOME=/usr/share/elasticsearch
PATH=$PATH:$ES_HOME/bin
export ES_JAVA_HOME ES_HOME PATH

EOF
source /etc/profile

3.数据目录

mkdir -p /home/data/elasticsearch/data
mkdir -p /home/data/elasticsearch/log
cd /home/data
chown elasticsearch:elasticsearch -R ./elasticsearch

4.xpack安全认证

• CA证书

CA证书生成在node-1中生成，分别复制到node-1和node-2的/etc/elasticsearch目录下

cd /usr/share/elasticsearch
./bin/elasticsearch-certutil ca
./bin/elasticsearch-certutil cert --ca elastic-stack-ca.p12

• 分发证书文件

cp ./elastic-certificates.p12 /etc/elasticsearch/
scp ./elastic-certificates.p12 root@node-2:/etc/elasticsearch/
chmod 644 /etc/elasticsearch/elastic-certificates.p12

• 追加配置

cat >> /etc/elasticsearch/elasticsearch.yml << \EOF
http.cors.enabled: true
http.cors.allow-credentials: true
http.cors.allow-origin: "*"
http.cors.allow-methods: OPTIONS, HEAD, GET, POST, PUT, DELETE
http.cors.allow-headers: X-Requested-With, X-Auth-Token, Content-Type, Content-Length, Authorization, Access-Control-Allow-Headers, Accept

xpack.security.enabled: true
xpack.security.audit.enabled: true
xpack.license.self_generated.type: basic
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.client_authentication: required
xpack.security.transport.ssl.verification_mode: certificate 
xpack.security.transport.ssl.keystore.path: /etc/elasticsearch/elastic-certificates.p12 
xpack.security.transport.ssl.truststore.path: /etc/elasticsearch/elastic-certificates.p12

EOF

• 设置密码

systemctl start elasticsearch.service
cd /usr/share/elasticsearch
./bin/elasticsearch-setup-passwords auto > password.txt

5.配置

集群模式下注释该配置项discovery.type: single-node

• node-1

vim /etc/elasticsearch/elasticsearch.yml

# ======================== Elasticsearch Configuration =========================
#
# NOTE: Elasticsearch comes with reasonable defaults for most settings.
#       Before you set out to tweak and tune the configuration, make sure you
#       understand what are you trying to accomplish and the consequences.
#
# The primary way of configuring a node is via this file. This template lists
# the most important settings you may want to configure for a production cluster.
#
# Please consult the documentation for further information on configuration options:
# https://www.elastic.co/guide/en/elasticsearch/reference/index.html
#
# ---------------------------------- Cluster -----------------------------------
#
# Use a descriptive name for your cluster:
#
cluster.name: my-application
#
# ------------------------------------ Node ------------------------------------
#
# Use a descriptive name for the node:
#
node.name: node-1
#
# Add custom attributes to the node:
#
#node.attr.rack: r1
#
# ----------------------------------- Paths ------------------------------------
#
# Path to directory where to store the data (separate multiple locations by comma):
#
#path.data: /var/lib/elasticsearch
path.data: /home/data/elasticsearch/data
#
# Path to log files:
#
#path.logs: /var/log/elasticsearch
path.logs: /home/data/elasticsearch/log
#
# ----------------------------------- Memory -----------------------------------
#
# Lock the memory on startup:
#
#bootstrap.memory_lock: true
#
# Make sure that the heap size is set to about half the memory available
# on the system and that the owner of the process is allowed to use this
# limit.
#
# Elasticsearch performs poorly when the system is swapping the memory.
#
# ---------------------------------- Network -----------------------------------
#
# By default Elasticsearch is only accessible on localhost. Set a different
# address here to expose this node on the network:
#
#network.host: 192.168.0.1
network.host: 0.0.0.0
#
# By default Elasticsearch listens for HTTP traffic on the first free port it
# finds starting at 9200. Set a specific HTTP port here:
#
http.port: 9200
#
# For more information, consult the network module documentation.
#
# --------------------------------- Discovery ----------------------------------
#
# Pass an initial list of hosts to perform discovery when this node is started:
# The default list of hosts is ["127.0.0.1", "[::1]"]
#
discovery.seed_hosts: ["node-1", "node-2"]
# discovery.seed_hosts: ["node-1"]
#
# Bootstrap the cluster using an initial set of master-eligible nodes:
#
#cluster.initial_master_nodes: ["node-1", "node-2"]
# discovery.type: single-node
cluster.initial_master_nodes: ["node-1"]
#
# For more information, consult the discovery and cluster formation module documentation.
#
# ---------------------------------- Various -----------------------------------
#
# Require explicit names when deleting indices:
#
#action.destructive_requires_name: true

http.cors.enabled: true
http.cors.allow-credentials: true
http.cors.allow-origin: "*"
http.cors.allow-methods: OPTIONS, HEAD, GET, POST, PUT, DELETE
http.cors.allow-headers: X-Requested-With, X-Auth-Token, Content-Type, Content-Length, Authorization, Access-Control-Allow-Headers, Accept

xpack.security.enabled: true
xpack.security.audit.enabled: true
xpack.license.self_generated.type: basic
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.client_authentication: required
xpack.security.transport.ssl.verification_mode: certificate 
xpack.security.transport.ssl.keystore.path: /etc/elasticsearch/elastic-certificates.p12 
xpack.security.transport.ssl.truststore.path: /etc/elasticsearch/elastic-certificates.p12

• node-2

vim /etc/elasticsearch/elasticsearch.yml

# ======================== Elasticsearch Configuration =========================
#
# NOTE: Elasticsearch comes with reasonable defaults for most settings.
#       Before you set out to tweak and tune the configuration, make sure you
#       understand what are you trying to accomplish and the consequences.
#
# The primary way of configuring a node is via this file. This template lists
# the most important settings you may want to configure for a production cluster.
#
# Please consult the documentation for further information on configuration options:
# https://www.elastic.co/guide/en/elasticsearch/reference/index.html
#
# ---------------------------------- Cluster -----------------------------------
#
# Use a descriptive name for your cluster:
#
cluster.name: my-application
#
# ------------------------------------ Node ------------------------------------
#
# Use a descriptive name for the node:
#
node.name: node-2
#
# Add custom attributes to the node:
#
#node.attr.rack: r1
#
# ----------------------------------- Paths ------------------------------------
#
# Path to directory where to store the data (separate multiple locations by comma):
#
#path.data: /var/lib/elasticsearch
path.data: /home/data/elasticsearch/data
#
# Path to log files:
#
#path.logs: /var/log/elasticsearch
path.logs: /home/data/elasticsearch/log
#
# ----------------------------------- Memory -----------------------------------
#
# Lock the memory on startup:
#
#bootstrap.memory_lock: true
#
# Make sure that the heap size is set to about half the memory available
# on the system and that the owner of the process is allowed to use this
# limit.
#
# Elasticsearch performs poorly when the system is swapping the memory.
#
# ---------------------------------- Network -----------------------------------
#
# By default Elasticsearch is only accessible on localhost. Set a different
# address here to expose this node on the network:
#
#network.host: 192.168.0.1
network.host: 0.0.0.0
#
# By default Elasticsearch listens for HTTP traffic on the first free port it
# finds starting at 9200. Set a specific HTTP port here:
#
http.port: 9200
#
# For more information, consult the network module documentation.
#
# --------------------------------- Discovery ----------------------------------
#
# Pass an initial list of hosts to perform discovery when this node is started:
# The default list of hosts is ["127.0.0.1", "[::1]"]
#
discovery.seed_hosts: ["node-1", "node-2"]
# discovery.seed_hosts: ["node-1"]
#
# Bootstrap the cluster using an initial set of master-eligible nodes:
#
#cluster.initial_master_nodes: ["node-1", "node-2"]
# discovery.type: single-node
cluster.initial_master_nodes: ["node-1"]
#
# For more information, consult the discovery and cluster formation module documentation.
#
# ---------------------------------- Various -----------------------------------
#
# Require explicit names when deleting indices:
#
#action.destructive_requires_name: true

http.cors.enabled: true
http.cors.allow-credentials: true
http.cors.allow-origin: "*"
http.cors.allow-methods: OPTIONS, HEAD, GET, POST, PUT, DELETE
http.cors.allow-headers: X-Requested-With, X-Auth-Token, Content-Type, Content-Length, Authorization, Access-Control-Allow-Headers, Accept

xpack.security.enabled: true
xpack.security.audit.enabled: true
xpack.license.self_generated.type: basic
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.client_authentication: required
xpack.security.transport.ssl.verification_mode: certificate 
xpack.security.transport.ssl.keystore.path: /etc/elasticsearch/elastic-certificates.p12 
xpack.security.transport.ssl.truststore.path: /etc/elasticsearch/elastic-certificates.p12

6.启动和关闭

systemctl start elasticsearch.service
systemctl status elasticsearch.service
systemctl stop elasticsearch.service

7.开机启动

systemctl enable elasticsearch.service

8.在线管理集群

cluster api

• 示例

# If no filters are given, the default is to select all nodes
GET /_nodes
# Explicitly select all nodes
GET /_nodes/_all
# Select just the local node
GET /_nodes/_local
# Select the elected master node
GET /_nodes/_master
# Select nodes by name, which can include wildcards
GET /_nodes/node_name_goes_here
GET /_nodes/node_name_goes_*
# Select nodes by address, which can include wildcards
GET /_nodes/10.0.0.3,10.0.0.4
GET /_nodes/10.0.0.*
# Select nodes by role
GET /_nodes/_all,master:false
GET /_nodes/data:true,ingest:true
GET /_nodes/coordinating_only:true
GET /_nodes/master:true,voting_only:false
# Select nodes by custom attribute (e.g. with something like `node.attr.rack: 2` in the configuration file)
GET /_nodes/rack:2
GET /_nodes/ra*:2
GET /_nodes/ra*:2*

三、连接

1.kibana

修改/etc/kibana/kibana.yml配置文件，将elasticsearch.hosts属性由elasticsearch.hosts: [“http://localhost:9200”]改为如下内容：

elasticsearch.hosts: ["http://node-1:9200", "http://node-2:9200"]

2.springboot连接es集群

• pom.xml

覆盖spring-boot-dependencies中的properties配置（通过dependencyManagement管理版本）。

    <properties>
        <java.version>1.8</java.version>
        <elasticsearch.version>7.13.2</elasticsearch.version>
    </properties>

        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-data-elasticsearch</artifactId>
        </dependency>

• application.yml

spring:
  elasticsearch:
    rest:
      uris: http://node-1:9200, http://node-2:9200
      username: elastic
      password: gAbEd5ViDusJgBF1p4GN
      connection-timeout: PT1S
      read-timeout: PT30S

代码示例

• Student.java

package com.example.es.entity;

import lombok.Data;
import org.springframework.data.annotation.Id;
import org.springframework.data.elasticsearch.annotations.DateFormat;
import org.springframework.data.elasticsearch.annotations.Document;
import org.springframework.data.elasticsearch.annotations.Field;
import org.springframework.data.elasticsearch.annotations.FieldType;

import java.util.Date;

/**
 * @author ：ouruyi
 * @version 1.0
 * @date ：Created in 2021/4/19 15:21
 * 功能描述：
 */
@Data
@Document(indexName = "student")
public class Student {
    @Id
    @Field(type = FieldType.Long)
    private Long id;
    @Field(type = FieldType.Keyword)
    private String name;
    @Field(type = FieldType.Integer)
    private Integer age;
    @Field(type = FieldType.Date, format = DateFormat.basic_date)
    private Date birthDate;
    @Field(type = FieldType.Text, analyzer = "ik_max_word", searchAnalyzer = "ik_smart")
    private String notes;

}

• StudentRepository.java

package com.example.es.repository;

import com.example.es.entity.Student;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.elasticsearch.annotations.Highlight;
import org.springframework.data.elasticsearch.annotations.HighlightField;
import org.springframework.data.elasticsearch.annotations.Query;
import org.springframework.data.elasticsearch.repository.ElasticsearchRepository;
import org.springframework.stereotype.Repository;

import java.util.List;

/**
 * @author ：ouruyi
 * @version 1.0
 * @date ：Created in 2021/4/19 15:41
 * 功能描述：
 */
@Repository
public interface StudentRepository extends ElasticsearchRepository<Student, Long> {

    /**
     * 通过名称查询
     *
     * @param name
     * @return
     */
    List<Student> findByName(String name);

    /**
     * 通过名称模糊查询
     *
     * @param name
     * @return
     */
    List<Student> findByNameLike(String name);

    /**
     * @param notes
     * @param pageable
     * @return
     */
    @Highlight(
            fields = {
                    @HighlightField(name = "notes")
            }
    )
    @Query("{\"bool\": {\"must\": {\"term\": {\"notes\": \"?0\"} } } }")
    Page<Student> findByNotes(String notes, Pageable pageable);

}

• EsManagerService.java

package com.example.es.service;

import com.example.es.entity.Student;
import org.springframework.data.domain.Page;

import java.util.List;
import java.util.Map;

/**
 * @author ：ouruyi
 * @version 1.0
 * @date ：Created in 2021/4/19 15:33
 * 功能描述：
 */
public interface EsManagerService {
    /**
     * 创建索引库、映射结构
     */
    void create();

    /**
     * 删除索引库
     */
    void drop();

    /**
     * 批量导入
     *
     * @param list
     */
    void saveAll(List<Student> list);

    /**
     * 清空索引库
     */
    void purge();

    /**
     * 查询所有
     *
     * @return
     */
    Iterable<Student> findAll();

    /**
     * 通过名称查询
     *
     * @return
     */
    List<Student> findByName(String name);

    /**
     * 通过名称模糊查询
     *
     * @param name
     * @return
     */
    List<Student> findByNameLike(String name);

    /**
     * 测试
     *
     * @param notes
     * @return
     */
    Page<Student> findByNotes(String notes);


}

• EsManagerServiceImpl.java

package com.example.es.service.impl;

import com.example.es.entity.Student;
import com.example.es.repository.StudentRepository;
import com.example.es.service.EsManagerService;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Sort;
import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate;
import org.springframework.data.elasticsearch.core.IndexOperations;
import org.springframework.stereotype.Service;

import javax.annotation.Resource;
import java.util.List;

/**
 * @author ：ouruyi
 * @version 1.0
 * @date ：Created in 2021/4/19 15:34
 * 功能描述：
 */
@Service("esManagerService")
public class EsManagerServiceImpl implements EsManagerService {

    @Resource
    private ElasticsearchRestTemplate elasticsearchRestTemplate;

    @Resource
    private StudentRepository studentRepository;


    @Override
    public void create() {
        IndexOperations indexOps = elasticsearchRestTemplate.indexOps(Student.class);
        indexOps.create();
        indexOps.putMapping(indexOps.createMapping(Student.class));
    }

    @Override
    public void drop() {
        IndexOperations indexOps = elasticsearchRestTemplate.indexOps(Student.class);
        indexOps.delete();
    }

    @Override
    public void saveAll(List<Student> list) {
        studentRepository.saveAll(list);
    }

    @Override
    public void purge() {
        studentRepository.deleteAll();
    }

    @Override
    public Iterable<Student> findAll() {
        return studentRepository.findAll(Sort.by(Sort.Direction.DESC, "age"));
    }

    @Override
    public List<Student> findByName(String name) {
        return studentRepository.findByName(name);
    }

    /**
     * @param name
     * @return
     */
    @Override
    public List<Student> findByNameLike(String name) {
        return studentRepository.findByNameLike(name);
    }

    /**
     * @param notes
     * @return
     */
    @Override
    public Page<Student> findByNotes(String notes) {
        final Pageable pageable = PageRequest.of(0, 2);
        return studentRepository.findByNotes(notes, pageable);
    }


}

• StudentController.java

package com.example.es.controller;

import com.example.es.entity.Student;
import com.example.es.service.EsManagerService;
import org.springframework.data.domain.Page;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;

import javax.annotation.Resource;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;

/**
 * @author ：ouruyi
 * @version 1.0
 * @date ：Created in 2021/4/19 16:07
 * 功能描述：
 */
@RestController
@RequestMapping("/student")
public class StudentController {

    @Resource
    private EsManagerService esManagerService;


    @RequestMapping("/create")
    public Boolean create() {
        try {
            esManagerService.create();
            return true;
        } catch (Exception e) {
            return false;
        }
    }

    @RequestMapping("/drop")
    public Boolean drop() {
        try {
            esManagerService.drop();
            return true;
        } catch (Exception e) {
            return false;
        }
    }

    @RequestMapping("/list")
    public Iterable<Student> findAll() {
        return esManagerService.findAll();
    }

    @RequestMapping("/findByName")
    public List<Student> findByName(@RequestParam(value = "name") String name) {
        return esManagerService.findByName(name);
    }

    @RequestMapping("/findByNameLike")
    public List<Student> findByNameLike(@RequestParam(value = "name") String name) {
        return esManagerService.findByNameLike(name);
    }

    @RequestMapping("/findByNotes")
    public Page<Student> findByNotes(@RequestParam(value = "notes") String notes) {
        return esManagerService.findByNotes(notes);
    }


    @RequestMapping("/purge")
    public Boolean purge() {
        try {
            esManagerService.purge();
            return true;
        } catch (Exception e) {
            return false;
        }
    }

    @RequestMapping("/saveAll")
    public Boolean saveAll() {
        try {
            List list = new ArrayList();
            mockData(list, 10);
            esManagerService.saveAll(list);
            return true;
        } catch (Exception e) {
            return false;
        }
    }


    private void mockData(List list, Integer nums) {
        for (int i = 0; i < nums; i++) {
            final Student student = new Student();
            long id = i + 1;
            student.setId(id);
            student.setName("张三");
            student.setAge(18);
            student.setBirthDate(new Date());
            student.setNotes("张三出生于2000年3月1日，毕业于合工大，就职于alibaba，是一名优秀的高材生！幼时家境贫寒，但经过自己多年的打拼，生活质量也在不断提高，是我辈之楷模！");
            list.add(student);
        }
    }


}