clickhouse在docker swarm中的部署

最新推荐文章于 2024-03-30 20:17:14 发布

吃了个奇怪的蘑菇

最新推荐文章于 2024-03-30 20:17:14 发布

阅读量503

点赞数

文章标签：数据库 docker

本文链接：https://blog.csdn.net/qq_35140572/article/details/119176493

版权

docker swarm部署clickhouse集群

简单使用docker swarm部署一个clickhouse集群

服务器初始化

服务器组内，安装docker，初始化docker swarm集群，将集群节点jion到swam集群中

挂载目录初始化(每台服务器都需要创建)

mkdir /data
mkdir -p /data/clickhouse/zookeeper/data
mkdir -p /data/clickhouse/zookeeper/log
mkdir -p /data/clickhouse/clickhouse-server/etc
mkdir -p /data/clickhouse/clickhouse-server/data
mkdir -p /data/clickhouse/clickhouse-server/log

挂载目录创建后需要根据启动的服务用户信息chmod目录所需的权限

docker swarm 配置及声明文件

stack 声明文件

clickhouse-stack.yml

version: '3.8'

services:
  #nginx 最终对外发布clickhouse集群代理
  nginx:
    image: nginx:latest
    networks:
      - clickhouse_network
    ports:
      - "18123:18123"
    dns: 8.8.8.8
    environment:
       TZ: Asia/Shanghai
    configs:
      - source: clickhouse_rute.conf
        target: /etc/nginx/conf.d/clickhouse_rute.conf
      - source: nginx.conf
        target: /etc/nginx/nginx.conf
    deploy:
      replicas: 1
      endpoint_mode: vip
  #zookeeper clickhouse集群协调
  zookeeper:
    image: zookeeper:3.5
    networks:
      - clickhouse_network
    dns: 8.8.8.8
    volumes:
      - "/data/clickhouse/zookeeper/data:/data"
      - "/data/clickhouse/zookeeper/log:/datalog"
    environment:
       TZ: Asia/Shanghai
    deploy:
      endpoint_mode: dnsrr
      placement:
        constraints:
          - node.hostname ==  clcikhouse01
  #clickhouse可视化web插件    
  tabix:
    image: spoonest/clickhouse-tabix-web-client
    networks:
      - clickhouse_network
    environment:
       TZ: Asia/Shanghai
    dns: 8.8.8.8
    ports:
      - "8090:80"
  #clickhouse节点01 
  ch01:
    image: yandex/clickhouse-server
    user: root
    privileged: true
    networks:
      - clickhouse_network
    dns: 8.8.8.8
    environment:
       TZ: Asia/Shanghai
    volumes:
      - /data/clickhouse/clickhouse-server/etc:/etc/clickhouse-server
      - /data/clickhouse/clickhouse-server/data:/var/lib/clickhouse
      - /data/clickhouse/clickhouse-server/log:/var/log/clickhouse-server/
    ulimits:
      nofile:
        soft: 262144
        hard: 262144
    deploy:
      replicas: 1
      endpoint_mode: dnsrr
      placement:
        constraints:
          - node.hostname == clcikhouse01
    depends_on:
      - "zookeeper"
  #clickhouse节点02
  ch02:
    image: yandex/clickhouse-server
    user: root
    privileged: true
    networks:
      - clickhouse_network
    dns: 8.8.8.8
    environment:
       TZ: Asia/Shanghai
    volumes:
      - /data/clickhouse/clickhouse-server/etc:/etc/clickhouse-server
      - /data/clickhouse/clickhouse-server/data:/var/lib/clickhouse
      - /data/clickhouse/clickhouse-server/log:/var/log/clickhouse-server/
    ulimits:
      nofile:
        soft: 262144
        hard: 262144
    deploy:
      replicas: 1
      endpoint_mode: dnsrr
      placement:
        constraints:
          - node.hostname == clcikhouse03
    depends_on:
      - "zookeeper"
      
configs:
  clickhouse_rute.conf:
    external: true
  nginx.conf:
    external: true
    
networks:
    clickhouse_network: 
      external: true

注意：由于CNM网络的VIP发布模式下，zookeeper获取的服务IP与docker 内置 DNS缓存IP不一致，此处使用dnsrr非对外公开路由进行service之间通信，对外使用nginx负载后发布服务

network详情

Name	clickhouse_network
ID	y9tulfwmc5t0jqxeq038lhmfu
Driver	overlay
Scope	swarm
Attachable	false
Internal	false
IPV4 Subnet - 10.0.2.0/24	IPV4 Gateway - 10.0.2.1
IPV4 IP range -	IPV4 Excluded Ips

根据自己实际情况创建跨宿主机通信网络.

configs详情

clickhouse_rute.conf：

upstream clickhouse {
          server ch01:8123;
          server ch02:8123;
}

server {
	listen       18123;
    location / {
            proxy_pass http://clickhouse;
            proxy_set_header Host $host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    }
}

nginx.conf:

user  nginx;
worker_processes  auto;

error_log  /var/log/nginx/error.log notice;
pid        /var/run/nginx.pid;


events {
    worker_connections  1024;
}


http {
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;

    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    access_log  /var/log/nginx/access.log  main;

    sendfile        on;
    #tcp_nopush     on;

    keepalive_timeout  65;

    #gzip  on;

    include /etc/nginx/conf.d/*.conf;
    client_max_body_size 50m;
}

clickhouse节点配置文件

users.xml简单配置：

<yandex>   
    <profiles>       
        <default>   
            <!-- 用于在单个服务器上运行查询的最大RAM -->
            <max_memory_usage>10000000000</max_memory_usage>     
            <!-- 使用未压缩数据块的缓存 -->
            <use_uncompressed_cache>0</use_uncompressed_cache>    
            <!-- 分布式查询处理过程中选择副本的方式 -->
            <!-- random   nearest_hostname:从错误最少的副本集合中选择    in_order:按照指定的顺序选择第一个活副本    first_or_random:如果第一个副本错误数较高，则从错误数最少的副本中随机选择一个副本  -->
            <load_balancing>random</load_balancing>
            <!-- 参数设置 -->
            <input_format_import_nested_json>1</input_format_import_nested_json>
            <allow_experimental_map_type>1</allow_experimental_map_type>
        </default>       
        <readonly>           
            <max_memory_usage>10000000000</max_memory_usage>           
            <use_uncompressed_cache>0</use_uncompressed_cache>           
            <load_balancing>random</load_balancing>            
            <readonly>1</readonly>       
        </readonly>   
    </profiles>
    <!-- 跟踪每个小时的资源消耗，而不限制使用量 -->
    <quotas>           
        <!-- Name of quota. -->       
        <default>           
            <interval>               
                <duration>3600</duration>                
                <queries>0</queries>               
                <errors>0</errors>               
                <result_rows>0</result_rows>               
                <read_rows>0</read_rows>               
                <execution_time>0</execution_time>           
            </interval>       
        </default>   
    </quotas>   
    <users>       
        <default>           
            <!--<password_sha256_hex>3210d55b042f88af7ab58530d1eaf2d36a6e5e2fb6ce7655e6b5e9f8d8712c73</password_sha256_hex>-->
			<password>3aO4rbXi</password>		
            <networks incl="networks" replace="replace">                
                <ip>::/0</ip>           
            </networks>           
            <profile>default</profile>      
            <quota>default</quota>
        </default>
        <!--<ck>   
            <password_sha256_hex>9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08</password_sha256_hex>  
            <networks incl="networks" replace="replace">      
                <ip>::/0</ip>   
            </networks>   
            <profile>readonly</profile>     
            <quota>default</quota>  
        </ck>-->
    </users>
</yandex>

config.xml简单配置：

<!-- This file was generated automatically.
     Do not edit it: it is likely to be discarded and generated again before it's read next time.
     Files used to generate this file:
       /etc/clickhouse-server/config.xml
       /etc/clickhouse-server/config.d/docker_related_config.xml      -->

<!--
  NOTE: User and query level settings are set up in "users.xml" file.
  If you have accidentally specified user-level settings here, server won't start.
  You can either move the settings to the right place inside "users.xml" file
   or add <skip_check_for_incorrect_settings>1</skip_check_for_incorrect_settings> here.
--><yandex>
    <logger>
        <level>debug</level>
        <log>/var/log/clickhouse-server/clickhouse-server.log</log>
        <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
        <size>1000M</size>
        <count>10</count>
    </logger>
    <http_port>8123</http_port>
    <tcp_port>9000</tcp_port>
    <interserver_http_port>9009</interserver_http_port>
    <listen_host>::</listen_host> 
    <max_connections>4096</max_connections>
    <keep_alive_timeout>3</keep_alive_timeout>
    <grpc>
        <enable_ssl>false</enable_ssl>
        <ssl_cert_file>/path/to/ssl_cert_file</ssl_cert_file>
        <ssl_key_file>/path/to/ssl_key_file</ssl_key_file>
        <ssl_require_client_auth>false</ssl_require_client_auth>
        <ssl_ca_cert_file>/path/to/ssl_ca_cert_file</ssl_ca_cert_file>
        <compression>deflate</compression>
        <compression_level>medium</compression_level>
        <max_send_message_size>-1</max_send_message_size>
        <max_receive_message_size>-1</max_receive_message_size>
        <verbose_logs>false</verbose_logs>
    </grpc>
    <max_concurrent_queries>100</max_concurrent_queries>
    <max_server_memory_usage>0</max_server_memory_usage>
    <max_thread_pool_size>10000</max_thread_pool_size
    <max_server_memory_usage_to_ram_ratio>0.9</max_server_memory_usage_to_ram_ratio>
    <total_memory_profiler_step>4194304</total_memory_profiler_step>
    <total_memory_tracker_sample_probability>0</total_memory_tracker_sample_probability>
    <uncompressed_cache_size>8589934592</uncompressed_cache_size>
    <mark_cache_size>5368709120</mark_cache_size>
    <path>/var/lib/clickhouse/</path>
    <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
    <user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
    <user_directories>
        <users_xml>
            <path>users.xml</path>
        </users_xml>
        <local_directory>
            <path>/var/lib/clickhouse/access/</path>
        </local_directory>
    </user_directories>
    <default_profile>default</default_profile>
    <custom_settings_prefixes/>
    <default_database>default</default_database>
    <mlock_executable>true</mlock_executable>
    <remap_executable>false</remap_executable>
	<include_from>/etc/clickhouse-server/metrika.xml</include_from>
    <remote_servers incl="clickhouse_remote_servers" />
	<zookeeper incl="zookeeper-servers" />
    <!-- 需要自己根据分片与副本规划自定义macros标签中内容的值 -->
	<macros>
		<layer>01</layer>
		<shard>01</shard>
		<replica>ch01</replica>
	</macros>
    <builtin_dictionaries_reload_interval>3600</builtin_dictionaries_reload_interval>
    <max_session_timeout>3600</max_session_timeout>
    <default_session_timeout>60</default_session_timeout>
    <query_log>
        <database>system</database>
        <table>query_log</table>
        <partition_by>toYYYYMM(event_date)</partition_by>
        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
    </query_log>
    <trace_log>
        <database>system</database>
        <table>trace_log</table>
        <partition_by>toYYYYMM(event_date)</partition_by>
        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
    </trace_log>
    <query_thread_log>
        <database>system</database>
        <table>query_thread_log</table>
        <partition_by>toYYYYMM(event_date)</partition_by>
        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
    </query_thread_log>
    <metric_log>
        <database>system</database>
        <table>metric_log</table>
        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
        <collect_interval_milliseconds>1000</collect_interval_milliseconds>
    </metric_log>
    <asynchronous_metric_log>
        <database>system</database>
        <table>asynchronous_metric_log</table>
        <flush_interval_milliseconds>60000</flush_interval_milliseconds>
    </asynchronous_metric_log>
    <opentelemetry_span_log>
        <engine>
            engine MergeTree
            partition by toYYYYMM(finish_date)
            order by (finish_date, finish_time_us, trace_id)
        </engine>
        <database>system</database>
        <table>opentelemetry_span_log</table>
        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
    </opentelemetry_span_log>
    <crash_log>
        <database>system</database>
        <table>crash_log</table>
        <partition_by/>
        <flush_interval_milliseconds>1000</flush_interval_milliseconds>
    </crash_log>
    <top_level_domains_lists>
    </top_level_domains_lists>
    <dictionaries_config>*_dictionary.xml</dictionaries_config>
    <distributed_ddl>
        <path>/clickhouse/task_queue/ddl</path>
    </distributed_ddl>
    <format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>

    <listen_host>0.0.0.0</listen_host>
    <listen_try>1</listen_try>

</yandex>

需根据实际情况参照官方文档，自定义上述配置文件

metrika.xml简单配置:

<yandex>
	<interserver_http_host>ch01</interserver_http_host>
    <!-- 集群配置 -->
    <clickhouse_remote_servers>
        <my_cluster>
            <!-- 数据分片1  -->
            <shard>
                <replica>
                    <host>ch01</host>
                    <port>9000</port>
                    <user>default</user>
                    <password>3aO4rbXi</password>
                </replica>
            </shard>
            <!-- 数据分片2  -->
            <shard>
                <replica>
                    <host>ch02</host>
                    <port>9000</port>
                    <user>default</user>
                    <password>3aO4rbXi</password>
                </replica>
            </shard>
        </my_cluster>
    </clickhouse_remote_servers>
    <!-- ZK  -->
    <zookeeper-servers>
        <node index="1">
            <host>zookeeper</host>
            <port>2181</port>
        </node>
    </zookeeper-servers>
    <networks>
        <ip>::/0</ip>
    </networks>
    <!-- 数据压缩算法  -->
    <clickhouse_compression>
        <case>
            <min_part_size>10000000000</min_part_size>
            <min_part_size_ratio>0.01</min_part_size_ratio>
            <method>lz4</method>
        </case>
    </clickhouse_compression>
</yandex>

将上面三个配置文件创建配置后放置与每个服务器节点 /data/clickhouse/clickhouse-server/etc 路径下

启动集群

docker stack deploy -c clickhouse-stack.yml clickhouse

吃了个奇怪的蘑菇

关注

0
点赞
踩
0

收藏

觉得还不错? 一键收藏
0
评论
clickhouse在docker swarm中的部署

docker swarm部署clickhouse集群简单使用docker swarm部署一个clickhouse集群服务器初始化服务器组内，安装docker，初始化docker swarm集群，将集群节点jion到swam集群中挂载目录初始化(每台服务器都需要创建)mkdir /datamkdir -p /data/clickhouse/zookeeper/datamkdir -p /data/clickhouse/zookeeper/logmkdir -p /data/clickh
复制链接

扫一扫