mongodb抽取数据到ES,使用ELK内部插件无法获取数据,只能试试monstache抽取mongodb数据,但是monstache需要mongodb replica set 模式才能采集数据。
生成集群证书
mkdir -p /data/mongodb
openssl rand -base64 765 > /data/mongodb/key.file
chmod 400 /data/mongodb/key.file
mongodb 配置文件
#/data/mongodb/mongod.conf
systemLog:
destination: file
path: "/data/db/mongod.log"
logAppend: true
storage:
dbPath: /data/db/db
wiredTiger:
engineConfig:
cacheSizeGB: 0.25
processManagement:
fork: false
net:
bindIpAll: true
port: 27017
security:
keyFile: /data/db/key.file
authorization: enabled
replication:
replSetName: replicas
oplogSizeMB: 2048
#replicas set 启动服务
docker run -d --name r_db01 -p 37071:27017 -v /data/mongodb:/data/db mongo:4.2.8 mongod -f /data/db/mongod.conf
#登录主mongodb初始化rs
rs.initiate(
{
_id : "replicas",
members: [
{ _id : 0, host : "192.168.21.131:37071" },
{ _id : 1, host : "192.168.21.130:37071" }
]
}
)
rs.status()
#主节点执行
rs.isMaster()
rs.conf()
# 默认备份节点不能读写,可以设置
rs.slaveOk();
#Primary 创建用户
db.createUser({
user:'admin',
pwd:'123456',
roles:[{role:'root',db:'admin'}],
})
#主节点测试数据
db.t1.insert( { name : "test" } )
#增加仲裁节点
rs.addArb("192.168.21.133:37075")
rs.remove("192.168.21.133:37075")
############monstache.config.toml配置################
# ip地址注意要修改
mongo-url = "mongodb://admin:654321@192.168.21.131:37071/?authSource=admin"
# ip地址注意要修改
elasticsearch-urls = ["http://192.168.21.130:9200"]
# mydb指的是需要同步的数据库
direct-read-namespaces = ["test.*"]
#elasticsearch-user = "elastic"
#elasticsearch-password = "pwd"
elasticsearch-max-conns = 4
dropped-collections = true
dropped-databases = true
resume = true
resume-strategy = 0
verbose = true
cluster-name = 'my-es'
############monstache-compose文件
version: '3.3'
services:
monstache:
image: rwynn/monstache
restart: always
container_name: monstache
volumes:
- /data/monstache/config/monstache.config.toml:/app/monstache.config.toml
command: -f /app/monstache.config.toml