centos 7 ansible 安装 mongodb 集群

阅读本文需要知道ansible的基础知识

修改host和iptables,同步集群

ansible mongo -m copy -a 'src=mongo/hosts dest=/etc/hosts'
ansible mongo -m copy -a 'src=mongo/iptables dest=/etc/sysconfig/iptables'

iptables要设置端口互相可访问

mongos 端口 17088
config server 端口 17089
shard server 端口 17090,17091,17092

重载iptables服务

ansible mongo -m service -a 'name=iptables state=reloaded'

时间同步

ansible mongo -m yum -a 'name=ntp state=present'
ansible mongo -m shell -a 'ntpdate ntp1.aliyun.com'
ansible mongo -m lineinfile -a "path=/var/spool/cron/root line='1 */6 * * * ntpdate ntp1.aliyun.com > /dev/null 2>&1'"
ansible mongo -m service -a "name=ntpd state=stopped enabled=no"

JAVA安装

ansible mongo -m yum -a 'name=java* state=absent'
ansible mongo -m yum -a 'name=jdk* state=absent'
//# 需要jdk-8u162-linux-x64.rpm的地址
ansible mongo -m get_url -a "url=http://{{ip}}/jdk-8u162-linux-x64.rpm dest=/var/tmp"
ansible mongo -m yum -a 'name=/var/tmp/jdk-8u162-linux-x64.rpm state=present'
ansible mongo -m copy -a 'content="export JAVA_HOME=/usr/java/jdk1.8.0_162\nexport JRE_HOME=$JAVA_HOME/jre\nexport CLASSPATH=.:$JAVA_HOME/lib:$JRE_HOME/lib" dest=/etc/profile.d/java.sh'
ansible mongo -m replace -a 'path="/etc/profile.d/path.sh" regexp="(PATH.+)$" replace="\1:$JAVA_HOME/bin"'
ansible mongo -m shell -a "source /etc/profile"
ansible mongo -m shell -a "java -version"

jdbc安装

//# 需要mysql-connector-java的地址
ansible mongo -m get_url -a "url=http://{{ip}}/mysql-connector-java-5.1.46-bin.jar dest=/var/tmp"
ansible mongo -m file -a "path=/usr/share/java state=directory"
ansible mongo -m shell -a "cp /var/tmp/mysql-connector-java-5.1.46-bin.jar /usr/share/java/mysql-connector-java.jar"

系统优化

ansible vps -m shell -a 'echo never > /sys/kernel/mm/transparent_hugepage/enabled'
ansible vps -m shell -a 'echo never > /sys/kernel/mm/transparent_hugepage/defrag'

ansible vps -m lineinfile -a "path=/etc/rc.local regexp='transparent_hugepage/enabled' line='echo never > /sys/kernel/mm/transparent_hugepage/enabled'"
ansible vps -m lineinfile -a "path=/etc/rc.local regexp='transparent_hugepage/defrag' line='echo never > /sys/kernel/mm/transparent_hugepage/defrag'"

ansible vps -m blockinfile -a 'path=/etc/security/limits.conf block="* soft nofile 655360\n* hard nofile 655360\n* soft nproc 327680\n* hard nproc 327680"'

安装国内镜像库

ansible vps -m yum_repository -a 'file="mongodb" name="mongodb-org" state=absent'
ansible vps -m yum -a 'name=mongodb-org state=absent'
ansible vps -m yum -a 'name=mongodb-org-server state=absent'
ansible vps -m yum -a 'name=mongodb-org-tools state=absent'
ansible vps -m yum -a 'name=mongodb-org-mongos state=absent'
ansible vps -m yum -a 'name=mongodb-org-shell state=absent'
//安装4.0;
ansible vps -m yum_repository -a 'file="mongodb" name="mongodb-org" baseurl="https://mirrors.tuna.tsinghua.edu.cn/mongodb/yum/el$releasever-4.0/" gpgcheck=no enabled=yes state=present description="Mongodb china YUM repo"'
ansible vps -m yum -a 'name=mongodb-org state=present update_cache=yes'

生成keyfiles

openssl rand -base64 756 > vps/mongodb/mongoshard.key

创建配置服务器副本集

config server 配置文件

$ vi vps/mongodb/configsvr.yml.j2
systemLog:
  destination: file
  logAppend: true
  logRotate: reopen
  path: /cache1/mongodb/configsvr/log/configsrv.log
storage:
  dbPath: /cache1/mongodb/configsvr/data
  directoryPerDB: true
  journal:
    enabled: true
  wiredTiger:
    engineConfig:
      cacheSizeGB: 8
      directoryForIndexes: true
processManagement:
  fork: true
  pidFilePath: /var/run/mongodb/configsrv.pid
  timeZoneInfo: /usr/share/zoneinfo
net:
  port: 17088
  bindIp: localhost,{{ ansible_default_ipv4.address  }}
sharding:
  clusterRole: configsvr
replication:
  replSetName: confset

config server el7 服务启动文件

$ vi vps/mongodb/configsvr.service
[Unit]
Description=MongoDB Config Server
After=network.target
Documentation=https://docs.mongodb.org/manual
[Service]
User=mongod
Group=mongod
Environment="OPTIONS=-f /etc/mongoDB.d/configsvr.yml"
EnvironmentFile=-/etc/sysconfig/mongod
ExecStart=/usr/bin/numactl --interleave=all /usr/bin/mongod $OPTIONS
ExecStartPre=/usr/bin/mkdir -p /var/run/mongodb
ExecStartPre=/usr/bin/chown mongod:mongod /var/run/mongodb
ExecStartPre=/usr/bin/chmod 0755 /var/run/mongodb
PermissionsStartOnly=true
Restart=always
RestartSec=5
TimeoutStartSec=0
RestartPreventExitStatus=SIGKILL
PIDFile=/var/run/mongodb/configsrv.pid
Type=forking
LimitFSIZE=infinity
LimitCPU=infinity
LimitAS=infinity
LimitNOFILE=64000
LimitNPROC=64000
LimitMEMLOCK=infinity
TasksMax=infinity
TasksAccounting=false
[Install]
WantedBy=multi-user.target

创shard副本集

shard 配置文件

$ vi vps/mongodb/shardsvr.yml.j2
systemLog:
  destination: file
  logAppend: true
  path: /cache1/mongodb/shardsvr/log/shardsvr.log
storage:
  dbPath: /cache1/mongodb/shardsvr/data
  journal:
    enabled: true
processManagement:
  fork: true
  pidFilePath: /var/run/mongodb/shardsvr.pid
  timeZoneInfo: /usr/share/zoneinfo
net:
  port: 17089
  bindIp: localhost,{{ ansible_default_ipv4.address  }}
sharding:
  clusterRole: shardsvr
replication:
  replSetName: shardset

shard el7 服务启动文件

$ vi vps/mongodb/shardsvr.service
[Unit]
Description=MongoDB Shard Server
After=network.target
Documentation=https://docs.mongodb.org/manual
[Service]
User=mongod
Group=mongod
Environment="OPTIONS=-f /etc/mongoDB.d/shardsvr.yml"
EnvironmentFile=-/etc/sysconfig/mongod
ExecStart=/usr/bin/numactl --interleave=all /usr/bin/mongod $OPTIONS
ExecStartPre=/usr/bin/mkdir -p /var/run/mongodb
ExecStartPre=/usr/bin/chown mongod:mongod /var/run/mongodb
ExecStartPre=/usr/bin/chmod 0755 /var/run/mongodb
PermissionsStartOnly=true
Restart=always
RestartSec=5
TimeoutStartSec=infinity
RestartPreventExitStatus=SIGKILL
PIDFile=/var/run/mongodb/shardsvr.pid
Type=forking
LimitFSIZE=infinity
LimitCPU=infinity
LimitAS=infinity
LimitNOFILE=64000
LimitNPROC=64000
LimitMEMLOCK=infinity
TasksMax=infinity
TasksAccounting=false
[Install]
WantedBy=multi-user.target

配置路由服务器 mongos

mongos 配置文件

$ vi vps/mongodb/mongos.yml.j2
systemLog:
  destination: file
  logAppend: true
  path: /cache1/mongodb/mongos/log/mongos.log
processManagement:
  fork: true
  pidFilePath: /var/run/mongodb/mongos.pid
  timeZoneInfo: /usr/share/zoneinfo
net:
  port: 17090
  bindIp: localhost,{{ ansible_default_ipv4.address  }}
sharding:
  configDB: confset/vps138:17088,vps155:17088,vps156:17088

mongos el7 服务启动文件

$ vi vps/mongodb/mongos.service
[Unit]
Description=MongoDB Mongos Server
After=network.target
Documentation=https://docs.mongodb.org/manual
[Service]
User=mongod
Group=mongod
Environment="OPTIONS=-f /etc/mongoDB.d/mongos.yml"
EnvironmentFile=-/etc/sysconfig/mongod
ExecStart=/usr/bin/numactl --interleave=all /usr/bin/mongos $OPTIONS
ExecStartPre=/usr/bin/mkdir -p /var/run/mongodb
ExecStartPre=/usr/bin/chown mongod:mongod /var/run/mongodb
ExecStartPre=/usr/bin/chmod 0755 /var/run/mongodb
PermissionsStartOnly=true
Restart=always
RestartSec=5
TimeoutStartSec=infinity
RestartPreventExitStatus=SIGKILL
PIDFile=/var/run/mongodb/mongos.pid
Type=forking
LimitFSIZE=infinity
LimitCPU=infinity
LimitAS=infinity
LimitNOFILE=64000
LimitNPROC=64000
LimitMEMLOCK=infinity
TasksMax=infinity
TasksAccounting=false
[Install]
WantedBy=multi-user.target

ansible 配置

$ vi mongodb.yml

---
- hosts : vps
  tasks :
    - name: create group
      group:
        name: mongod
        state: present
    - name: create user
      user:
        name: mongod
        groups: mongod
        create_home: no
        shell: /sbin/nologin
        state: present
    - name: delete data and log dir
      file:
        path: /cache1/mongodb
        state: absent
        owner: mongod
        group: mongod
    - name: delete yml conf dir
      file:
        path: /etc/mongoDB.d
        state: absent
    - name: create configsvr data dir
      file:
        path: /cache1/mongodb/configsvr/data
        state: directory
        owner: mongod
        group: mongod
    - name: create configsvr log dir
      file:
        path: /cache1/mongodb/configsvr/log
        state: directory
        owner: mongod
        group: mongod
    - name: create shardsvr data dir
      file:
        path: /cache1/mongodb/shardsvr/data
        state: directory
        owner: mongod
        group: mongod
    - name: create shardsvr log dir
      file:
        path: /cache1/mongodb/shardsvr/log
        state: directory
        owner: mongod
        group: mongod
    - name: create mongos log dir
      file:
        path: /cache1/mongodb/mongos/log
        state: directory
        owner: mongod
        group: mongod
    - name: create run dir
      file:
        path: /var/run/mongodb
        state: directory
        owner: mongod
        group: mongod
    - name: create mongoDB.d dir
      file:
        path: /etc/mongoDB.d/
        state: directory
        owner: mongod
        group: mongod
    - name: configsvr config file
      template:
        src: mongodb/configsvr.yml.j2
        dest: /etc/mongoDB.d/configsvr.yml
        trim_blocks: yes
    - name: configsvr service
      copy:
        src: mongodb/configsvr.service
        dest: /etc/systemd/system/configsvr.service
        mode: 0644
    - name: shardsvr config file
      template:
        src: mongodb/shardsvr.yml.j2
        dest: /etc/mongoDB.d/shardsvr.yml
        trim_blocks: yes
    - name: shardsvr service
      copy:
        src: mongodb/shardsvr.service
        dest: /etc/systemd/system/shardsvr.service
        mode: 0644
    - name: mongos config file
      template:
        src: mongodb/mongos.yml.j2
        dest: /etc/mongoDB.d/mongos.yml
        trim_blocks: yes
    - name: mongos service
      copy:
        src: mongodb/mongos.service
        dest: /etc/systemd/system/mongos.service
        mode: 0644
    - name: keyfiles
      copy:
        src: mongodb/mongoshard.key
        dest: /etc/mongoDB.d/mongoshard.key
        mode: 0600
        owner: mongod
        group: mongod
    - name: just force systemd to reread configs
      systemd:
        daemon_reload: yes

上传服务

ansible-playbook vps/mongodb.yml

登录一台配置服务器,初始化:

ansible vps -m systemd -a 'name=configsvr state=started enabled=yes'
# 初始化
ansible vps138 -m shell -a "mongo --host vps138 --port 17088 --eval \"rs.initiate({_id : 'confset',members : [{_id : 0, host : 'vps138:17088' },{_id : 1, host : 'vps155:17088' },{_id : 2, host : 'vps156:17088' }]})\""
# 权限控制
ansible vps -m shell -a "mongo localhost:17088/admin --eval \"db.createUser({user: 'admin',pwd: 'admin',roles: [{ role: 'clusterManager', db: 'admin' },{ role: 'userAdminAnyDatabase', db: 'admin' }]})\""

登录一台shard,初始化:

ansible vps -m systemd -a 'name=shardsvr state=started enabled=yes'
# 初始化
ansible vps138 -m shell -a "mongo --host vps138 --port 17089 --eval \"rs.initiate({_id : 'shardset',members : [{_id : 0, host : 'vps138:17089' },{_id : 1, host : 'vps155:17089' },{_id : 2, host : 'vps156:17089' }]})\""
# 权限控制
ansible vps -m shell -a "mongo localhost:17089/admin --eval \"db.createUser({user: 'admin',pwd: 'admin',roles: [{ role: 'clusterManager', db: 'admin' },{ role: 'userAdminAnyDatabase', db: 'admin' }]})\""

登录一台mongos,关联分片服务器:

ansible vps -m systemd -a 'name=mongos state=started enabled=yes'
# 添加分片
ansible vps138 -m shell -a "mongo vps138:17090/admin --eval \"sh.addShard('shardset/vps138:17089,vps155:17089,vps156:17089')\""

停止:

#按顺序停止
ansible vps -m systemd -a 'name=mongos state=stopped'
ansible vps -m systemd -a 'name=configsvr state=stopped'
ansible vps -m systemd -a 'name=shardsvr state=stopped'

增加访问控制配置

# 配置服务器
ansible vps -m lineinfile -a "path=/etc/mongoDB.d/configsvr.yml regexp='keyFile' state=absent"
ansible vps -m lineinfile -a "path=/etc/mongoDB.d/configsvr.yml regexp='authorization' state=absent"
ansible vps -m lineinfile -a "path=/etc/mongoDB.d/configsvr.yml regexp='security' state=absent"
ansible vps -m lineinfile -a "path=/etc/mongoDB.d/configsvr.yml line='security:\n  authorization: enabled\n  keyFile: /etc/mongoDB.d/mongoshard.key' regexp='security' backrefs=no"
# 分片服务器
ansible vps -m lineinfile -a "path=/etc/mongoDB.d/shardsvr.yml regexp='keyFile' state=absent"
ansible vps -m lineinfile -a "path=/etc/mongoDB.d/shardsvr.yml regexp='authorization' state=absent"
ansible vps -m lineinfile -a "path=/etc/mongoDB.d/shardsvr.yml regexp='security' state=absent"
ansible vps -m lineinfile -a "path=/etc/mongoDB.d/shardsvr.yml line='security:\n  authorization: enabled\n  keyFile: /etc/mongoDB.d/mongoshard.key' regexp='security' backrefs=no"
# mongos
ansible vps -m lineinfile -a "path=/etc/mongoDB.d/mongos.yml regexp='keyFile' state=absent"
ansible vps -m lineinfile -a "path=/etc/mongoDB.d/mongos.yml regexp='security' state=absent"
ansible vps -m lineinfile -a "path=/etc/mongoDB.d/mongos.yml line='security:\n  keyFile: /etc/mongoDB.d/mongoshard.key' regexp='security' backrefs=no"

启动:

ansible vps -m systemd -a 'name=configsvr state=started enabled=yes'
ansible vps -m systemd -a 'name=shardsvr state=started enabled=yes'
ansible vps -m systemd -a 'name=mongos state=started enabled=yes'

测试

ansible mongo -m shell -a 'netstat -tnlp|grep mongo'