es

登录后复制

#!/usr/bin/env python3
# -*- encoding: utf-8 -*-
import os
import argparse
import socket
import subprocess
from time import sleep
from tqdm import tqdm

def run_command(command, check=True):
    result = subprocess.run(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    if check and result.returncode != 0:
        print(f"Error running command: {command}\n{result.stderr.decode().strip()}")
        exit(1)
    return result

def check_root():
    if os.geteuid() != 0:
        print("需要 root 用户执行.")
        exit(1)

def create_directories():
    if os.path.exists("/data/es"):
        run_command("rm -rf /data/es")
    os.makedirs("/data/es/data", exist_ok=True)
    os.makedirs("/data/es/logs", exist_ok=True)
    print("Created directories for Elasticsearch data and logs.")

def download_and_install_packages():
    urls = [
        "elasticsearch-8.3.1-x86_64.rpm",
        "kibana-8.3.1-x86_64.rpm"
    ]
    for url in tqdm(urls, desc="Downloading packages"):
        run_command(f"wget -P /tmp {url}")
    run_command("yum localinstall /tmp/elasticsearch-8.3.1-x86_64.rpm -y")
    run_command("yum localinstall /tmp/kibana-8.3.1-x86_64.rpm -y")
    print("Downloaded and installed Elasticsearch and Kibana.")

def adjust_system_parameters():
    with open("/etc/security/limits.conf", "r") as f:
        limits_conf = f.read()
    if "elasticsearch soft memlock unlimited" not in limits_conf:
        with open("/etc/security/limits.conf", "a") as f:
            f.write("elasticsearch soft memlock unlimited\n")
    if "elasticsearch hard memlock unlimited" not in limits_conf:
        with open("/etc/security/limits.conf", "a") as f:
            f.write("elasticsearch hard memlock unlimited\n")

    ulimit_value = int(subprocess.getoutput("ulimit -n"))
    if ulimit_value < 655360:
        with open("/etc/security/limits.conf", "a") as f:
            f.write("""
* soft nofile 102400
* hard nofile 102400
* soft nproc 102400
* hard nproc 102400
elasticsearch soft memlock unlimited
elasticsearch hard memlock unlimited
""")

    swappiness_set = False
    max_map_count_set = False
    if not subprocess.getoutput("grep 'vm.swappiness = 0' /etc/sysctl.conf"):
        with open("/etc/sysctl.conf", "a") as f:
            f.write("vm.swappiness = 0\n")
        swappiness_set = True
    if not subprocess.getoutput("grep 'vm.max_map_count = 655360' /etc/sysctl.conf"):
        with open("/etc/sysctl.conf", "a") as f:
            f.write("vm.max_map_count = 655360\n")
        max_map_count_set = True
    if swappiness_set or max_map_count_set:
        run_command("swapoff -a")
        run_command("sysctl -p")
    print("Adjusted system parameters.")

def adjust_jvm_heap_size():
    mem_total_kib = int(subprocess.getoutput("grep MemTotal /proc/meminfo | awk '{print $2}'"))
    mem_total_gib = mem_total_kib / 1024 / 1024
    heap_size = int(mem_total_gib / 5)

    with open("/etc/elasticsearch/jvm.options", "r") as file:
        lines = file.readlines()

    with open("/etc/elasticsearch/jvm.options", "w") as file:
        for line in lines:
            if line.startswith("-Xmx") or line.startswith("-Xms"):
                continue
            if "-XX:+UseG1GC" in line:
                continue
            file.write(line)
        file.write(f"-Xmx{heap_size}g\n")
        file.write(f"-Xms{heap_size}g\n")
        file.write("-XX:+UseG1GC\n")

    print(f"Adjusted JVM heap size to {heap_size}g.")

def adjust_elasticsearch_service():
    with open("/usr/lib/systemd/system/elasticsearch.service", "r") as file:
        lines = file.readlines()
    with open("/usr/lib/systemd/system/elasticsearch.service", "w") as file:
        for line in lines:
            if line.strip() == "[Service]":
                file.write(line)
                file.write("LimitMEMLOCK=infinity\n")
            else:
                file.write(line)
    print("Adjusted Elasticsearch service.")

def clean_old_certificates():
    cert_dir = "/etc/elasticsearch/certs"
    if os.path.exists(cert_dir):
        run_command(f"rm -rf {cert_dir}/*")
    run_command(f"rm -f /etc/elasticsearch/elasticsearch.keystore")
    print("已删除旧的证书和 keystore 文件.")

def generate_elasticsearch_config(ip_list, es_http_port, es_tcp_port, cluster_name):
    current_ip = socket.gethostbyname(socket.gethostname())
    initial_master_nodes = ",".join([f"\"{ip}\"" for ip in ip_list])
    seed_hosts = ",".join([f"\"{ip}:{es_tcp_port}\"" for ip in ip_list])

    config = f"""
path.data: /data/es/data
path.logs: /data/es/logs
cluster.name: {cluster_name}
node.name: {socket.gethostname()}
network.host: 0.0.0.0
network.publish_host: {current_ip}
http.port: {es_http_port}
transport.port: {es_tcp_port}
node.roles: ["master", "data"]
bootstrap.memory_lock: true
http.cors.enabled: true
http.cors.allow-origin: "*"
http.cors.allow-credentials: true
xpack.security.enabled: false
# xpack.security.enrollment.enabled: true
# xpack.security.http.ssl.enabled: false
# xpack.security.transport.ssl.enabled: true
# xpack.security.transport.ssl.verification_mode: none
# xpack.security.transport.ssl.keystore.path: /etc/elasticsearch/certs/transport.p12
# xpack.security.transport.ssl.keystore.password: changeit
# xpack.security.transport.ssl.truststore.path: /etc/elasticsearch/certs/transport.p12
# xpack.security.transport.ssl.truststore.password: changeit
cluster.initial_master_nodes: [{initial_master_nodes}]
indices.memory.index_buffer_size: 20%
indices.recovery.max_bytes_per_sec: 1g
cluster.fault_detection.leader_check.interval: 20s
discovery.cluster_formation_warning_timeout: 30s
cluster.publish.timeout: 90s
action.destructive_requires_name: true
discovery.seed_hosts: [{seed_hosts}]
"""
    with open("/etc/elasticsearch/elasticsearch.yml", "w") as f:
        f.write(config)
    print("Generated Elasticsearch configuration.")

def change_permissions():
    run_command("sudo chown -R elasticsearch:elasticsearch /data/es")
    run_command("sudo chown -R elasticsearch:elasticsearch /etc/elasticsearch")
    print("Changed permissions for Elasticsearch directories.")

def start_elasticsearch():
    run_command("sudo systemctl daemon-reload")
    run_command("sudo systemctl enable elasticsearch.service")
    run_command("systemctl start elasticsearch.service")
    print("Started Elasticsearch.")

def configure_slowlog(current_ip, es_http_port):
    slowlog_settings = """
{
    "index.search.slowlog.threshold.query.warn": "2s",
    "index.search.slowlog.threshold.query.info": "2s",
    "index.search.slowlog.threshold.query.debug": "1s",
    "index.search.slowlog.threshold.query.trace": "400ms",
    "index.search.slowlog.threshold.fetch.warn": "1s",
    "index.search.slowlog.threshold.fetch.info": "800ms",
    "index.search.slowlog.threshold.fetch.debug": "500ms",
    "index.search.slowlog.threshold.fetch.trace": "200ms",
    "index.indexing.slowlog.threshold.index.warn": "5s",
    "index.indexing.slowlog.threshold.index.info": "2s",
    "index.indexing.slowlog.threshold.index.debug": "1s",
    "index.indexing.slowlog.threshold.index.trace": "400ms"
}
"""
    run_command(f"curl -X PUT 'http://{current_ip}:{es_http_port}/_all/_settings' -H 'Content-Type: application/json' -d '{slowlog_settings}'")
    print("已配置慢日志设置.")

def create_kibana_user():
    run_command("/usr/share/elasticsearch/bin/elasticsearch-users useradd my_admin -p my_admin -r kibana_system")
    print("创建 Kibana 用户.")

def configure_kibana(kibana_ip, es_http_port, kibana_port):
    config = f"""
server.port: {kibana_port}
server.host: "{kibana_ip}"
elasticsearch.hosts: ["http://{kibana_ip}:{es_http_port}"]
elasticsearch.username: "my_admin"
elasticsearch.password: "my_admin"
elasticsearch.requestTimeout: 180000
i18n.locale: "zh-CN"
"""
    with open("/etc/kibana/kibana.yml", "w") as f:
        f.write(config)
    print("已配置 Kibana.")

def start_kibana():
    run_command("sudo systemctl daemon-reload")
    run_command("sudo systemctl enable kibana.service")
    run_command("systemctl start kibana.service")
    print("启动 Kibana.")

def main(ip_list, ports, cluster_name):
    es_http_port, es_tcp_port, kibana_port = ports
    first_node = ip_list[0]
    current_ip = socket.gethostbyname(socket.gethostname())

    check_root()

    create_directories()
    download_and_install_packages()

    adjust_system_parameters()
    adjust_jvm_heap_size()
    adjust_elasticsearch_service()

    clean_old_certificates()
    generate_elasticsearch_config(ip_list, es_http_port, es_tcp_port, cluster_name)

    change_permissions()
    start_elasticsearch()

    sleep(15)  # 等待 Elasticsearch 启动
    configure_slowlog(current_ip, es_http_port)
    create_kibana_user()

    if current_ip == first_node:
        configure_kibana(current_ip, es_http_port, kibana_port)
        start_kibana()

    # 检查服务是否成功启动
    sleep(10)
    if not run_command(f"ss -tnlp | egrep '{es_http_port}|{es_tcp_port}'", check=False):
        print("Elasticsearch 启动失败")
        exit(1)
    if current_ip == first_node and not run_command(f"ss -tnlp | grep '{kibana_port}'", check=False):
        print("Kibana 启动失败")
        exit(1)

    print("Elasticsearch 和 Kibana 安装并启动成功")

    # 检查 Elasticsearch 节点状态
    nodes_status = run_command("curl -X GET 'http://localhost:9200/_cat/nodes?v'", capture_output=True)
    print("Elasticsearch 节点状态:\n", nodes_status)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Install Elasticsearch and Kibana")
    parser.add_argument("-i", "--ips", required=True, help="Comma-separated list of IP addresses")
    parser.add_argument("-p", "--ports", required=True, help="Comma-separated list of ports for HTTP, TCP, and Kibana")
    parser.add_argument("-c", "--cluster", required=True, help="Cluster name")
    args = parser.parse_args()

    ip_list = args.ips.split(",")
    ports = list(map(int, args.ports.split(",")))
    cluster_name = args.cluster

    main(ip_list, ports, cluster_name)

• 1.
• 2.
• 3.
• 4.
• 5.
• 6.
• 7.
• 8.
• 9.
• 10.
• 11.
• 12.
• 13.
• 14.
• 15.
• 16.
• 17.
• 18.
• 19.
• 20.
• 21.
• 22.
• 23.
• 24.
• 25.
• 26.
• 27.
• 28.
• 29.
• 30.
• 31.
• 32.
• 33.
• 34.
• 35.
• 36.
• 37.
• 38.
• 39.
• 40.
• 41.
• 42.
• 43.
• 44.
• 45.
• 46.
• 47.
• 48.
• 49.
• 50.
• 51.
• 52.
• 53.
• 54.
• 55.
• 56.
• 57.
• 58.
• 59.
• 60.
• 61.
• 62.
• 63.
• 64.
• 65.
• 66.
• 67.
• 68.
• 69.
• 70.
• 71.
• 72.
• 73.
• 74.
• 75.
• 76.
• 77.
• 78.
• 79.
• 80.
• 81.
• 82.
• 83.
• 84.
• 85.
• 86.
• 87.
• 88.
• 89.
• 90.
• 91.
• 92.
• 93.
• 94.
• 95.
• 96.
• 97.
• 98.
• 99.
• 100.
• 101.
• 102.
• 103.
• 104.
• 105.
• 106.
• 107.
• 108.
• 109.
• 110.
• 111.
• 112.
• 113.
• 114.
• 115.
• 116.
• 117.
• 118.
• 119.
• 120.
• 121.
• 122.
• 123.
• 124.
• 125.
• 126.
• 127.
• 128.
• 129.
• 130.
• 131.
• 132.
• 133.
• 134.
• 135.
• 136.
• 137.
• 138.
• 139.
• 140.
• 141.
• 142.
• 143.
• 144.
• 145.
• 146.
• 147.
• 148.
• 149.
• 150.
• 151.
• 152.
• 153.
• 154.
• 155.
• 156.
• 157.
• 158.
• 159.
• 160.
• 161.
• 162.
• 163.
• 164.
• 165.
• 166.
• 167.
• 168.
• 169.
• 170.
• 171.
• 172.
• 173.
• 174.
• 175.
• 176.
• 177.
• 178.
• 179.
• 180.
• 181.
• 182.
• 183.
• 184.
• 185.
• 186.
• 187.
• 188.
• 189.
• 190.
• 191.
• 192.
• 193.
• 194.
• 195.
• 196.
• 197.
• 198.
• 199.
• 200.
• 201.
• 202.
• 203.
• 204.
• 205.
• 206.
• 207.
• 208.
• 209.
• 210.
• 211.
• 212.
• 213.
• 214.
• 215.
• 216.
• 217.
• 218.
• 219.
• 220.
• 221.
• 222.
• 223.
• 224.
• 225.
• 226.
• 227.
• 228.
• 229.
• 230.
• 231.
• 232.
• 233.
• 234.
• 235.
• 236.
• 237.
• 238.
• 239.
• 240.
• 241.
• 242.
• 243.
• 244.
• 245.
• 246.
• 247.
• 248.
• 249.
• 250.
• 251.
• 252.
• 253.
• 254.
• 255.
• 256.
• 257.
• 258.
• 259.
• 260.
• 261.
• 262.
• 263.
• 264.
• 265.
• 266.
• 267.

原创作者: jinyan2049 转载于: https://blog.51cto.com/jinyan2049/11643854