canal+es编排部署

#初始测试的方法及环境

docker network create elastic

docker pull docker.elastic.co/elasticsearch/elasticsearch:7.15.2

docker run --name es01-test --net elastic -p 127.0.0.1:9200:9200 -p 127.0.0.1:9300:9300 -e "discovery.type=single-node" docker.elastic.co/elasticsearch/elasticsearch:7.15.2

docker pull docker.elastic.co/kibana/kibana:7.15.2

docker run --name kib01-test --net elastic -p 192.168.100.198:5601:5601 -e "ELASTICSEARCH_HOSTS=http://es01-test:9200" docker.elastic.co/kibana/kibana:7.15.2

docker stop es01-test

docker stop kib01-test

docker network rm elastic

docker rm es01-test

docker rm kib01-test

[root@localhost middleware]#

[root@localhost middleware]# docker network ls

NETWORK ID          NAME                 DRIVER              SCOPE

de02d40dda76        bridge               bridge              local

2ac66367a612        elastic              bridge              local

ab2a5d8c1a18        es_default           bridge              local

f8e93156a04a        es_esnet             bridge              local

f97c1077f93e        host                 host                local

b4f27f924d6f        middleware_default   bridge              local

65c203f6cf73        none                 null                local

e28ac206dd24        run_default          bridge              local

#利用canal同步mysql数据库到elasticsearch

[root@localhost _data]# cat  mysql.cnf

[mysql]

#设置mysql客户端默认字符集

default-character-set=utf8

socket=/var/lib/mysql/mysql.sock

[mysqld]

log-bin=mysql-bin #开启

binlog-format=ROW #选择row模式

mysql -uroot -proot

# 创建账号

CREATE USER canal IDENTIFIED BY 'canal';

# 授予权限

GRANT SELECT, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'canal'@'%';

-- GRANT ALL PRIVILEGES ON *.* TO 'canal'@'%' ;

# 刷新并应用

FLUSH PRIVILEGES;

docker pull canal/canal-server

#启动镜像

docker run --name canal -d canal/canal-server

#找到文件位置后 exit退出容器 将容器内部文件copy到外部

[root@localhost _data]# cd /data/canal/

[root@localhost canal]# ls

[root@localhost canal]# docker cp canal:/home/admin/canal-server/conf/example/instance.properties .

[root@localhost canal]# docker cp canal:/home/admin/canal-server/conf/canal.properties .

#关闭容器

docker stop canal

#移除容器

docker rm canal

docker run --name canal-adapter -p 8081:8081 -v /data/canal-adapter/conf:/opt/canal-adapter/conf  -d slpcat/canal-adapter:v1.1.5

#复制canal-adapter初始配置文件

cd /data/canal-adapter/

docker cp canal-adapter:/opt/canal-adapter/conf  .

#关闭容器

docker stop canal-adapter

#移除容器

docker rm canal-adapter

#修改刚刚复制的instance文件

vi /data/canal/instance.properties

# 找到数据库信息,按需更改

canal.instance.master.address=内网IP:3306

canal.instance.dbUsername=canal

canal.instance.dbPassword=canal

#按照需要同步的表选择

canal.instance.filter.regex=db.table

#创建canal-adapter具名挂载目录

mkdir -p /data/canal-adapter/conf

touch application.yml

mkdir -p /data/canal-adapter/conf/es7

cd es7

touch test.yml

application.yml

server:

  port: 8081

spring:

  jackson:

    date-format: yyyy-MM-dd HH:mm:ss

    time-zone: GMT+8

    default-property-inclusion: non_null

canal.conf:

  mode: tcp

  flatMessage: true

  zookeeperHosts:

  syncBatchSize: 1000

  retries: 0

  timeout:

  accessKey:

  secretKey:

  consumerProperties:

    canal.tcp.server.host: canal-server地址:11111

    canal.tcp.batch.size: 500

    canal.tcp.username:

    canal.tcp.password:

  srcDataSources:

    defaultDS:

      url: jdbc:mysql://数据库IP:数据库端口/数据库名?useUnicode=true

      username: db_username

      password: db_password

  canalAdapters:

  - instance: example # canal instance Name or mq topic name

    groups:

    - groupId: g1

      outerAdapters:

      - name: logger

      - name: es7 # 该版本发现只能是es7/es6

        hosts: esip地址:9300 # 127.0.0.1:9200 for rest mode

        properties:

          mode: transport # transport # or rest

#test.yml文件

#文件内容要对应数据库和esindex

dataSourceKey: defaultDS

destination: example

groupId: g1

esMapping:

  _index: test

  _id: _id

  _type: _doc

  upsert: true

  sql: "SELECT a.id _id from t_test a "

  etlCondition: "where a.>={}"

  commitBatch: 3000

#编写容器编排文件

[root@localhost middleware]# cat docker-compose.yml

services:

  canal:

    container_name: canal

    image: canal/canal-server

    network_mode: host

    volumes:

    - /data/canal/canal.properties:/home/admin/canal-server/conf/canal.properties:rw

    - /data/canal/instance.properties:/home/admin/canal-server/conf/example/instance.properties:rw

  canal-adapter:

    container_name: canal-adapter

    image: slpcat/canal-adapter:v1.1.5

    network_mode: host

    volumes:

    - /data/canal-adapter/conf:/opt/canal-adapter/conf:rw

  elasticsearch:

    container_name: elasticsearch

    environment:

      discovery.type: single-node

    image: docker.elastic.co/elasticsearch/elasticsearch:7.15.2

    network_mode: host

    volumes:

    - /etc/localtime:/etc/localtime:ro

    - /data/elasticsearch/logs:/user/share/elasticsearch/logs:rw

    - /data/elasticsearch/config:/usr/share/elasticsearch/config:rw

    - /data/elasticsearch/data:/usr/share/elasticsearch/data:rw

  kinaba:

    container_name: kibana

    depends_on:

      elasticsearch:

        condition: service_started

    environment:

      ELASTICSEARCH_HOSTS: http://内网IP:9200

    image: docker.elastic.co/kibana/kibana:7.15.2

    network_mode: host

    volumes:

    - /etc/localtime:/etc/localtime:ro

    - /data/kibana/config/kibana.yml:/usr/share/kibana/config/kibana.yml:rw

version: '3'

#中间件的路径

[root@localhost middleware]# ls /data/

canal  canal-adapter  elasticsearch  kibana

[root@localhost middleware]# ls /data/canal

canal.properties  instance.properties

[root@localhost middleware]# ls /data/canal-adapter/

conf

[root@localhost middleware]# ls /data/canal-adapter/conf/

application.yml  es7

[root@localhost middleware]# ls /data/canal-adapter/conf/es7

test.yml

[root@localhost middleware]# ls /data/elasticsearch/

config/ data/   logs/   

[root@localhost middleware]# ls /data/elasticsearch/

config/ data/   logs/   

[root@localhost middleware]# ls /data/elasticsearch/

config  data  logs

[root@localhost middleware]# ls /data/elasticsearch/config/

elasticsearch.keystore  elasticsearch.yml  jvm.options  jvm.options.d  log4j2.file.properties  log4j2.properties  role_mapping.yml  roles.yml  users  users_roles

[root@localhost middleware]# ls /data/kibana/config/

kibana.yml

#elasticsearch配置说明

cat  /data/elasticsearch/config/elasticsearch.yml

cluster.name: "docker-cluster"

network.host: 0.0.0.0

http.cors.enabled: true

http.cors.allow-origin: "*"

http.cors.allow-headers: Authorization

xpack.security.enabled: true

xpack.security.transport.ssl.enabled: true

#kibana配置文件

ls /data/kibana/config/

kibana.yml

#kibana配置说明

cat /data/kibana/config/kibana.yml

server.name: kibana

server.host: "0.0.0.0"

elasticsearch.hosts: [ "http://内网IP:9200" ]

server.publicBaseUrl: "http://内网IP:5601"

monitoring.ui.container.elasticsearch.enabled: true

elasticsearch.username: "账号"

elasticsearch.password: "密码"

i18n.locale: "zh-CN"

#启动容器

docker-compose up -d

#设置账号密码

docker exec -it elasticsearch /bin/bash -c 'cd /bin'

bash-4.4# elasticsearch-setup-passwords interactive

Initiating the setup of passwords for reserved users elastic,apm_system,kibana,kibana_system,logstash_system,beats_system,remote_monitoring_user.

You will be prompted to enter passwords as the process progresses.

Please confirm that you would like to continue [y/N]y

Enter password for [elastic]:

Reenter password for [elastic]:

Passwords do not match.

Try again.

Enter password for [elastic]:

Reenter password for [elastic]:

Enter password for [apm_system]:

Reenter password for [apm_system]:

Enter password for [kibana_system]:

Reenter password for [kibana_system]:

Enter password for [logstash_system]:

Reenter password for [logstash_system]:

Enter password for [beats_system]:

Reenter password for [beats_system]:

Enter password for [remote_monitoring_user]:

Reenter password for [remote_monitoring_user]:

Changed password for user [apm_system]

Changed password for user [kibana_system]

Changed password for user [kibana]

Changed password for user [logstash_system]

Changed password for user [beats_system]

Changed password for user [remote_monitoring_user]

Changed password for user [elastic]

bash-4.4# exit

#修改配置后重启加载

docker-compose restart

#检查

[root@localhost middleware]# docker-compose ps

    Name                   Command               State   Ports

--------------------------------------------------------------

canal           /alidata/bin/main.sh /home ...   Up           

canal-adapter   sh -c ./bin/startup.sh           Up           

elasticsearch   /bin/tini -- /usr/local/bi ...   Up           

kibana          /bin/tini -- /usr/local/bi ...   Up           

[root@localhost middleware]# ss -tunlp|grep 9200

tcp    LISTEN     0      128    [::]:9200               [::]:*                   users:(("java",pid=141657,fd=307))

[root@localhost middleware]# ss -tunlp|grep 9300

tcp    LISTEN     0      128    [::]:9300               [::]:*                   users:(("java",pid=141657,fd=305))

[root@localhost middleware]# ss -tunlp|grep 11112

tcp    LISTEN     0      3         *:11112                 *:*                   users:(("java",pid=142356,fd=71))

[root@localhost middleware]# ss -tunlp|grep 11111

tcp    LISTEN     0      50        *:11111                 *:*                   users:(("java",pid=142356,fd=87))

[root@localhost middleware]# ss -tunlp|grep 11110

tcp    LISTEN     0      50        *:11110                 *:*                   users:(("java",pid=142356,fd=91))

#访问

http://IP:5601