MongoDB 副本集+分片 架构部署



 Replica Set + Sharding  配置部署

准备工作

架构设计

 

需要三台机器(racdb,host8,host9)：

• 数据节点，sharding2片

• 每个sharding为Replic Set方式，冗余度为3（主、从、延迟从）

• mongos(路由节点)，数量为3

• config  server(配置服务器节点)，数量为3

• arbiter server(表决节点)，数量为6

 

Mongodb ReplicaSET+Sharding架构如下:

 

系统配置

软件环境：

OS：Oracle Linux 6.5

MongoDB:mongodb3.2.3

 

下载地址：

mongodb-linux-x86_64-rhel62-3.2.3.tgz

 

创建用户：

# groupadd mongod

# useradd -g mongod mongod

 

配置hosts：

# vi /etc/hosts

10.1.5.x1     racdb                    racdb.localdomain

10.1.5.x2     host8       host8.localdomain

10.1.5.x3    host9       host9.localdomain

 

解压文件：

# tar zxvf mongodb-linux-x86_64-rhel62-3.2.3.tgz

# mv mongodb-linux-x86_64-3.x.x/ /home/mongod/mongodb

 

安装部署

1.     自动部署脚本

在三个机器分别配置shard1、shard2、arbiter、config server、mongos需要的参数文件

 

机器1（racdb）

# 数据节点

mkdir -p /home/mongod/mongodb/shard11

cat > /home/mongod/mongodb/shard11/shard11.conf <<EOF

shardsvr=true

replSet=shard1

port=28017

dbpath=/home/mongod/mongodb/shard11

oplogSize=2048

logpath=/home/mongod/mongodb/shard11/shard11.log

logappend=true

fork=true

rest=true

httpinterface=true

nojournal = true

EOF

 

mkdir -p /home/mongod/mongodb/shard21

cat > /home/mongod/mongodb/shard21/shard21.conf <<EOF

shardsvr=true

replSet=shard2

port=28018

dbpath=/home/mongod/mongodb/shard21

oplogSize=2048

logpath=/home/mongod/mongodb/shard21/shard21.log

logappend=true

fork=true

rest=true

httpinterface=true

nojournal = true

EOF

 

#表决节点

mkdir -p /home/mongod/mongodb/arbiter1

cat  >  /home/mongod/mongodb/arbiter1/arbiter1.conf <<EOF

shardsvr=true

replSet=shard1

port=28031

dbpath=/home/mongod/mongodb/arbiter1

oplogSize=100

logpath=/home/mongod/mongodb/arbiter1/arbiter1.log

logappend=true

fork=true

rest=true

httpinterface=true

nojournal = true

EOF

 

mkdir -p /home/mongod/mongodb/arbiter2

cat > /home/mongod/mongodb/arbiter2/arbiter2.conf <<EOF

shardsvr=true

replSet=shard2

port=28032

dbpath=/home/mongod/mongodb/arbiter2

oplogSize=100

logpath=/home/mongod/mongodb/arbiter2/arbiter2.log

logappend=true

fork=true

rest=true

httpinterface=true

nojournal = true

EOF

 

#配置节点

mkdir -p /home/mongod/mongodb/config/

cat > /home/mongod/mongodb/config/config1.conf <<EOF

configsvr=true

dbpath=/home/mongod/mongodb/config/

port=20000

logpath=/home/mongod/mongodb/config/config1.log

logappend=true

fork=true

httpinterface=true

#nojournal=true

EOF

 

#路由节点

mkdir -p /home/mongod/mongodb/mongos

cat > /home/mongod/mongodb/mongos/mongos1.conf <<EOF

configdb=racdb:20000,host8:20000,host9:20000

port=28885

chunkSize=100

logpath=/home/mongod/mongodb/mongos/mongos1.log

logappend=true

fork=true

EOF

 

 

机器2（host8）

#数据节点

mkdir -p /home/mongod/mongodb/shard12

cat > /home/mongod/mongodb/shard12/shard12.conf <<EOF

shardsvr=true

replSet=shard1

port=28017

dbpath=/home/mongod/mongodb/shard12

oplogSize=2048

logpath=/home/mongod/mongodb/shard12/shard12.log

logappend=true

fork=true

rest=true

httpinterface=true

nojournal = true

EOF

 

 

mkdir -p /home/mongod/mongodb/shard22

cat > /home/mongod/mongodb/shard22/shard22.conf <<EOF

shardsvr=true

replSet=shard2

port=28018

dbpath=/home/mongod/mongodb/shard22

oplogSize=2048

logpath=/home/mongod/mongodb/shard22/shard22.log

logappend=true

fork=true

rest=true

httpinterface=true

nojournal = true

EOF

 

 

#表决节点

mkdir -p /home/mongod/mongodb/arbiter1

cat  >  /home/mongod/mongodb/arbiter1/arbiter1.conf<<EOF

shardsvr=true

replSet=shard1

port=28031

dbpath=/home/mongod/mongodb/arbiter1

oplogSize=100

logpath=/home/mongod/mongodb/arbiter1/arbiter1.log

logappend=true

fork=true

rest=true

httpinterface=true

nojournal = true

EOF

 

 

mkdir -p /home/mongod/mongodb/arbiter2

cat > /home/mongod/mongodb/arbiter2/arbiter2.conf <<EOF

shardsvr=true

replSet=shard2

port=28032

dbpath=/home/mongod/mongodb/arbiter2

oplogSize=100

logpath=/home/mongod/mongodb/arbiter2/arbiter2.log

logappend=true

fork=true

rest=true

httpinterface=true

nojournal = true

EOF

 

 

#配置节点

mkdir -p /home/mongod/mongodb/config/

cat > /home/mongod/mongodb/config/config2.conf <<EOF

configsvr=true

dbpath=/home/mongod/mongodb/config/

port=20000

logpath=/home/mongod/mongodb/config/config2.log

logappend=true

fork=true

#nojournal = true

EOF

 

 

#路由节点

mkdir -p /home/mongod/mongodb/mongos

cat >/home/mongod/mongodb/mongos/mongos2.conf <<EOF

configdb=racdb:20000,host8:20000,host9:20000

port=28885

chunkSize=100

logpath=/home/mongod/mongodb/mongos/mongos2.log

logappend=true

fork=true

EOF

 

 

机器3（host9）

#数据节点

mkdir -p /home/mongod/mongodb/shard13

cat > /home/mongod/mongodb/shard13/shard13.conf <<EOF

shardsvr=true

replSet=shard1

port=28017

dbpath=/home/mongod/mongodb/shard13

oplogSize=2048

logpath=/home/mongod/mongodb/shard13/shard13.log

logappend=true

fork=true

rest=true

httpinterface=true

nojournal = true

EOF

 

 

mkdir -p /home/mongod/mongodb/shard23

cat > /home/mongod/mongodb/shard23/shard23.conf <<EOF

shardsvr=true

replSet=shard2

port=28018

dbpath=/home/mongod/mongodb/shard23

oplogSize=2048

logpath=/home/mongod/mongodb/shard23/shard23.log

logappend=true

fork=true

rest=true

httpinterface=true

nojournal = true

EOF

 

 

#表决节点

mkdir -p /home/mongod/mongodb/arbiter1

cat  >  /home/mongod/mongodb/arbiter1/arbiter1.conf<<EOF

shardsvr=true

replSet=shard1

port=28031

dbpath=/home/mongod/mongodb/arbiter1

oplogSize=100

logpath=/home/mongod/mongodb/arbiter1/arbiter1.log

logappend=true

fork=true

rest=true

httpinterface=true

nojournal = true

EOF

 

mkdir -p /home/mongod/mongodb/arbiter2

cat > /home/mongod/mongodb/arbiter2/arbiter2.conf <<EOF

shardsvr=true

replSet=shard2

port=28032

dbpath=/home/mongod/mongodb/arbiter2

oplogSize=100

logpath=/home/mongod/mongodb/arbiter2/arbiter2.log

logappend=true

fork=true

rest=true

httpinterface=true

nojournal = true

EOF

 

 

#配置节点

mkdir -p /home/mongod/mongodb/config/

cat > /home/mongod/mongodb/config/config3.conf <<EOF

configsvr=true

dbpath=/home/mongod/mongodb/config/

port=20000

logpath=/home/mongod/mongodb/config/config3.log

logappend=true

fork=true

#nojournal = true

EOF

 

 

#路由节点

mkdir -p/home/mongod/mongodb/mongos

cat >/home/mongod/mongodb/mongos/mongos3.conf <<EOF

configdb=racdb:20000,host8:20000,host9:20000

port=28885

chunkSize=100

logpath=/home/mongod/mongodb/mongos/mongos3.log

logappend=true

fork=true

EOF

 

 

2. 启动服务

 

--在Server1启动shard1、shard2、arbiter1、arbiter2服务

# server 1

mongod --config /home/mongod/mongodb/shard11/shard11.conf

mongod --config /home/mongod/mongodb/shard21/shard21.conf

mongod --config /home/mongod/mongodb/arbiter1/arbiter1.conf

mongod --config /home/mongod/mongodb/arbiter2/arbiter2.conf

mongod --config /home/mongod/mongodb/config/config1.conf

 

--在Server2启动shard1、shard2、arbiter1、arbiter2服务

# server 2

mongod --config /home/mongod/mongodb/shard12/shard12.conf

mongod --config /home/mongod/mongodb/shard22/shard22.conf

mongod --config /home/mongod/mongodb/arbiter1/arbiter1.conf

mongod --config /home/mongod/mongodb/arbiter2/arbiter2.conf

mongod --config /home/mongod/mongodb/config/config2.conf

 

--在Server3启动shard1、shard2、arbiter1、arbiter2服务

# server 3

mongod --config /home/mongod/mongodb/shard13/shard13.conf

mongod --config /home/mongod/mongodb/shard23/shard23.conf

mongod --config /home/mongod/mongodb/arbiter1/arbiter1.conf

mongod --config /home/mongod/mongodb/arbiter2/arbiter2.conf

mongod --config /home/mongod/mongodb/config/config3.conf

 

然后在server1、server2、server3上分别启动mongos服务

# server1

mongos --config /home/mongod/mongodb/mongos/mongos1.conf

# server2

mongos --config /home/mongod/mongodb/mongos/mongos2.conf

# server2

mongos --config /home/mongod/mongodb/mongos/mongos3.conf

 

3.     配置分片副本集

任意登陆一个机器，比如登陆racdb，连接mongodb

 

设置第一个分片副本集

mongo racdb:28017/admin

 

#定义副本集配置

config = {_id: "shard1", members:[

                     {_id: 0, host:"racdb:28017"},

                     {_id: 1, host:"host8:28017"},

                     {_id: 2, host:"host9:28017",slaveDelay : 7200, priority:0},

                     {_id: 3, host:"racdb:28031", arbiterOnly : true},

                     {_id: 4, host:"host8:28031", arbiterOnly : true},

                     {_id: 5, host:"host9:28031", arbiterOnly : true}

              ]

};

 

#初始化副本集配置

rs.initiate(config);

 

 

 

设置第二个分片副本集

mongo racdb:28018/admin

 

#定义副本集配置

config = {_id: "shard2", members:[

                     {_id: 0, host:"racdb:28018"},

                     {_id: 1, host:"host8:28018"},

                     {_id: 2, host:"host9:28018",slaveDelay : 7200, priority:0},

                     {_id: 3, host:"racdb:28032", arbiterOnly : true},

                     {_id: 4, host:"host8:28032", arbiterOnly : true},

                     {_id: 5, host:"host9:28032", arbiterOnly : true}

              ]

};

 

#初始化副本集配置

rs.initiate(config);

 

 

# 添加分片 (可在任一机器登录)

mongo host9:28885/admin

 

#串联路由服务器与分配副本集1

db.runCommand({"addshard" :"shard1/racdb:28017,host8:28017"})

#串联路由服务器与分配副本集2

db.runCommand({"addshard" :"shard2/host8:28018,racdb:28018"})

 

因为host9:28017和host9:28018是延迟节点，所以不在副本集当中

#查看分片服务器的配置                

db.runCommand( { listshards : 1 } );

 

连接在mongos上，准备让指定的数据库、指定的集合分片生效

mongo racdb:28885/admin

#指定im_offline_msg库分片生效

db.runCommand( { "enablesharding": "im_offline_msg"})

#指定数据库里需要分片的集合和片键

db.runCommand( { shardcollection : " enablesharding.step",key: {_id: 1} } )

 

 

这里我们设置对im_offline_msg库的step 表进行分片，根据 id 自动分片到 shard1 ，shard2 上面去。这样设置是因为不是所有mongodb 的数据库和表都需要分片。

 

4.     测试分片结果

下面我们进一步对分片结果进行测试

#连接mongos服务器并对库表分片

[mongod@racdb ~]$ mongo racdb:28885/admin

MongoDB shell version: 3.2.3

connecting to: racdb:28885/admin

 

mongos> db.runCommand( { enablesharding:"testdb"});

{ "ok" : 1 }

mongos> db.runCommand( { shardcollection: "testdb.table1",key : {id: 1} } )

{ "collectionsharded" :"testdb.table1", "ok" : 1 }

 

#插入测试数据

for (var i = 1; i <= 10000; i++)

db.table1.save({id:i,"test1":"licz"});

部分参考：http://www.lanceyan.com/tech/arch/mongodb_shard1.html