mongoDB 2.0.0 shard implemention with noauth=true
彭建业
配置和1.8差不多，只是mongoDB2.0.0本身带来了一些新的特性，所以参数配置有点不一样。
以下是详细部署过程 :
产品部署注意事项 :
do not use hugepage
use ntp
use ext4/xfs
mount with option : noatime
Do not use NUMA
DELL服务器关闭NUMA : Node Interleaving 改为Enabled则安装对称内存配置时支持内存交错。如果为 Disabled（已禁用），系统支持非一体化内存体系结构 (NUMA)（非对称）内存配置。
不关闭numa会有警告信息 :
Mon Sep 26 14:07:01 File I/O errno:29 Illegal seek
Mon Sep 26 14:07:01 ** WARNING: cannot parse numa_maps
Mon Sep 26 14:07:01 ** WARNING: cannot parse numa_maps
Servers :
10.10.10.10
10.10.10.11
10.10.10.12
10.10.10.13
10.10.10.14
10.10.10.16
configdb :
10.10.10.14:6669 , 10.10.10.14:6670 , 10.10.10.16:6669
mongos :
10.10.10.16:6666
shard1 :
digoal001/10.10.10.10:6666(PRIMARY),10.10.10.11:6667(SECONDARY),10.10.10.12:6668(arbit)
shard2 :
digoal002/10.10.10.11:6666(PRIMARY),10.10.10.10:6667(SECONDARY),10.10.10.13:6668(arbit)
shard3 :
digoal003/10.10.10.12:6666(PRIMARY),10.10.10.13:6667(SECONDARY),10.10.10.10:6668(arbit)
shard4 :
digoal004/10.10.10.13:6666(PRIMARY),10.10.10.12:6667(SECONDARY),10.10.10.11:6668(arbit)
存储配置 :
# pvcreate /dev/sdb
# pvcreate /dev/sdc
# pvcreate /dev/sda4
# pvs
PV VG Fmt Attr PSize PFree
/dev/sda4 vgdata01 lvm2 a- 154.56G 0
/dev/sdb vgdata01 lvm2 a- 278.87G 0
/dev/sdc vgdata01 lvm2 a- 278.87G 0
# vgcreate vgdata01 /dev/sdb /dev/sdc /dev/sda4
# vgs
VG #PV #LV #SN Attr VSize VFree
vgdata01 3 0 0 wz--n- 712.30G 712.30G
# lvcreate -l 71391 -n lv01 vgdata01 /dev/sdb
# lvcreate -l 71391 -n lv02 vgdata01 /dev/sdc
# lvextend -l +19784 /dev/vgdata01/lv01 /dev/sda4
# lvextend -l +19784 /dev/vgdata01/lv02 /dev/sda4
# mkfs.ext4 /dev/mapper/vgdata01-lv01
# mkfs.ext4 /dev/mapper/vgdata01-lv02
# vi /etc/fstab
/dev/mapper/vgdata01-lv01 /mongodata/digoal/6666/data01 ext4 defaults,noatime 0 0
/dev/mapper/vgdata01-lv02 /mongodata/digoal/6667/data01 ext4 defaults,noatime 0 0
# mkdir -p /mongodata/digoal/6666/data01
# mkdir -p /mongodata/digoal/6667/data01
# mount -a
# mkdir /mongodata/digoal/6666/data01/mongodb_datafile
# mkdir /mongodata/digoal/6666/data01/mongodb_etc
# mkdir /mongodata/digoal/6666/data01/mongodb_run
# mkdir /mongodata/digoal/6667/data01/mongodb_datafile
# mkdir /mongodata/digoal/6667/data01/mongodb_etc
# mkdir /mongodata/digoal/6667/data01/mongodb_run
# mkdir -p /opt/mongodata/digoal/6668/data01/mongodb_datafile
# mkdir -p /opt/mongodata/digoal/6668/data01/mongodb_etc
# mkdir -p /opt/mongodata/digoal/6668/data01/mongodb_run
# chown -R mongo:mongo /mongodata
# chown -R mongo:mongo /opt/mongodata
用户配置 :
# useradd mongo
# passwd mongo
# passwd root
# chown -R mongo:mongo /mongodata
# su - mongo
$ vi .bash_profile
export PS1="$USER@`/bin/hostname -s`-> "
export MONGO_HOME=/opt/mongodb-linux-x86_64-2.0.0
export PATH=$MONGO_HOME/bin:$PATH:.
umask 022
alias rm='rm -i'
alias ll='ls -lh'
系统配置 :
# vi /etc/sysctl.conf
kernel.shmmni = 4096
kernel.sem = 50100 64128000 50100 1280
fs.file-max = 7672460
net.core.rmem_default = 1048576
net.core.rmem_max = 4194304
net.core.wmem_default = 262144
net.core.wmem_max = 1048576
net.ipv4.ip_local_port_range = 9000 65000
net.ipv4.tcp_tw_recycle = 1
net.ipv4.tcp_max_syn_backlog = 4096
net.ipv4.ip_conntrack_max = 655360
net.core.netdev_max_backlog = 10000
net.ipv4.ip_conntrack_max = 655360
fs.aio-max-nr = 1048576
net.ipv4.tcp_timestamps = 0
vm.overcommit_memory = 0
vm.zone_reclaim_mode = 0
# chkconfig cmirror off
# chkconfig ip6tables off
# chkconfig iscsi off
# chkconfig iscsid off
# chkconfig rhnsd off
# chkconfig ricci off
# chkconfig avahi-daemon off
# chkconfig modclusterd off
# vi /etc/security/limits.conf
* soft nofile 131072
* hard nofile 131072
* soft nproc 131072
* hard nproc 131072
* soft core unlimited
* hard core unlimited
* soft memlock 50000000
* hard memlock 50000000
# echo -e "\nsession required pam_limits.so" >>/etc/pam.d/login
# vi /etc/sysconfig/iptables
# vi /etc/sysconfig/ntpd
# vi /etc/ssh/sshd_config
# vi /etc/ssh/ssh_config
# vi /etc/resolv.conf
# vi /etc/hosts
127.0.0.1 localhost.localdomain localhost
10.10.10.10 db-10-10-10-10.sky-mobi.com.sh.nj db-10-10-10-10
10.10.10.11 db-10-10-10-11.sky-mobi.com.sh.nj db-10-10-10-11
10.10.10.12 db-10-10-10-12.sky-mobi.com.sh.nj db-10-10-10-12
10.10.10.13 db-10-10-10-13.sky-mobi.com.sh.nj db-10-10-10-13
10.10.10.14 db-10-10-10-14.sky-mobi.com.sh.nj db-10-10-10-14
10.10.10.16 db-10-10-10-16.sky-mobi.com.sh.nj db-10-10-10-16
# hostname
# vi /etc/sysconfig/network
# crontab -e
8 * * * * /usr/sbin/ntpdate asia.pool.ntp.org && /sbin/hwclock --systohc
数据库配置 :
# tar -zxvf mongodb-linux-x86_64-2.0.0.tgz
# tar -zxvf mongodb-src-r2.0.0.tar.gz
# chown -R mongo:mongo mongodb*
# mv mongodb-linux-x86_64-2.0.0 /opt/mongodb-linux-x86_64-2.0.0
# mkdir /var/log/mongo
# chown mongo:mongo /var/log/mongo
# 配置见附录
$ vi /mongodata/digoal/6666/data01/mongodb_etc/mongod_6666.conf
$ vi /mongodata/digoal/6667/data01/mongodb_etc/mongod_6667.conf
$ vi /opt/mongodata/digoal/6668/data01/mongodb_etc/mongod_6668.conf
启动数据库 :
10.10.10.10,
# su - mongo -c "/opt/mongodb-linux-x86_64-2.0.0/bin/mongod --shardsvr --replSet=digoal001 -f /mongodata/digoal/6666/data01/mongodb_etc/mongod_6666.conf"
# su - mongo -c "/opt/mongodb-linux-x86_64-2.0.0/bin/mongod --shardsvr --replSet=digoal002 -f /mongodata/digoal/6667/data01/mongodb_etc/mongod_6667.conf"
# su - mongo -c "/opt/mongodb-linux-x86_64-2.0.0/bin/mongod --shardsvr --replSet=digoal003 -f /opt/mongodata/digoal/6668/data01/mongodb_etc/mongod_6668.conf"
10.10.10.11,
# su - mongo -c "/opt/mongodb-linux-x86_64-2.0.0/bin/mongod --shardsvr --replSet=digoal002 -f /mongodata/digoal/6666/data01/mongodb_etc/mongod_6666.conf"
# su - mongo -c "/opt/mongodb-linux-x86_64-2.0.0/bin/mongod --shardsvr --replSet=digoal001 -f /mongodata/digoal/6667/data01/mongodb_etc/mongod_6667.conf"
# su - mongo -c "/opt/mongodb-linux-x86_64-2.0.0/bin/mongod --shardsvr --replSet=digoal004 -f /opt/mongodata/digoal/6668/data01/mongodb_etc/mongod_6668.conf"
10.10.10.12,
# su - mongo -c "/opt/mongodb-linux-x86_64-2.0.0/bin/mongod --shardsvr --replSet=digoal003 -f /mongodata/digoal/6666/data01/mongodb_etc/mongod_6666.conf"
# su - mongo -c "/opt/mongodb-linux-x86_64-2.0.0/bin/mongod --shardsvr --replSet=digoal004 -f /mongodata/digoal/6667/data01/mongodb_etc/mongod_6667.conf"
# su - mongo -c "/opt/mongodb-linux-x86_64-2.0.0/bin/mongod --shardsvr --replSet=digoal001 -f /opt/mongodata/digoal/6668/data01/mongodb_etc/mongod_6668.conf"
10.10.10.13,
# su - mongo -c "/opt/mongodb-linux-x86_64-2.0.0/bin/mongod --shardsvr --replSet=digoal004 -f /mongodata/digoal/6666/data01/mongodb_etc/mongod_6666.conf"
# su - mongo -c "/opt/mongodb-linux-x86_64-2.0.0/bin/mongod --shardsvr --replSet=digoal003 -f /mongodata/digoal/6667/data01/mongodb_etc/mongod_6667.conf"
# su - mongo -c "/opt/mongodb-linux-x86_64-2.0.0/bin/mongod --shardsvr --replSet=digoal002 -f /opt/mongodata/digoal/6668/data01/mongodb_etc/mongod_6668.conf"
加入自动启动
# vi /etc/rc.local
配置replicaSet
$ mongo 10.10.10.10:6666/admin
> cfg = {
_id : "digoal001",
members : [
{_id : 0, host : "10.10.10.10:6666", arbiterOnly : false, priority : 2},
{_id : 1, host : "10.10.10.11:6667", arbiterOnly : false, priority : 1},
{_id : 2, host : "10.10.10.12:6668", arbiterOnly : true, priority : 0}
]
}
> rs.initiate(cfg)
> rs.status()
$ mongo 10.10.10.11:6666/admin
> cfg = {
_id : "digoal002",
members : [
{_id : 0, host : "10.10.10.11:6666", arbiterOnly : false, priority : 2},
{_id : 1, host : "10.10.10.10:6667", arbiterOnly : false, priority : 1},
{_id : 2, host : "10.10.10.13:6668", arbiterOnly : true, priority : 0}
]
}
> rs.initiate(cfg)
> rs.status()
$ mongo 10.10.10.12:6666/admin
> cfg = {
_id : "digoal003",
members : [
{_id : 0, host : "10.10.10.12:6666", arbiterOnly : false, priority : 2},
{_id : 1, host : "10.10.10.13:6667", arbiterOnly : false, priority : 1},
{_id : 2, host : "10.10.10.10:6668", arbiterOnly : true, priority : 0}
]
}
> rs.initiate(cfg)
> rs.status()
$ mongo 10.10.10.13:6666/admin
> cfg = {
_id : "digoal004",
members : [
{_id : 0, host : "10.10.10.13:6666", arbiterOnly : false, priority : 2},
{_id : 1, host : "10.10.10.12:6667", arbiterOnly : false, priority : 1},
{_id : 2, host : "10.10.10.11:6668", arbiterOnly : true, priority : 0}
]
}
> rs.initiate(cfg)
> rs.status()
> db.addUser("","")
configsrv配置
configdb :
10.10.10.14:6669 , 10.10.10.14:6670 , 10.10.10.16:6669
10.10.10.14 :
# mkdir /data1/mongoconfigdbdata
# ln -s /data1/mongoconfigdbdata /mongodata
# mkdir -p /mongodata/digoal/6669/data01/mongodb_datafile
# mkdir -p /mongodata/digoal/6669/data01/mongodb_etc
# mkdir -p /mongodata/digoal/6669/data01/mongodb_run
# mkdir -p /opt/mongodata/digoal/6670/data01/mongodb_datafile
# mkdir -p /opt/mongodata/digoal/6670/data01/mongodb_etc
# mkdir -p /opt/mongodata/digoal/6670/data01/mongodb_run
# chown -R mongo:mongo /data1/mongoconfigdbdata
# chown -R mongo:mongo /opt/mongodata
10.10.10.16 :
# mkdir -p /opt/mongodata/digoal/6669/data01/mongodb_datafile
# mkdir -p /opt/mongodata/digoal/6669/data01/mongodb_etc
# mkdir -p /opt/mongodata/digoal/6669/data01/mongodb_run
# mkdir -p /opt/mongodata/digoal/6666/data01/mongodb_datafile
# mkdir -p /opt/mongodata/digoal/6666/data01/mongodb_etc
# mkdir -p /opt/mongodata/digoal/6666/data01/mongodb_run
# chown -R mongo:mongo /opt/mongodata
configsrv配置文件示例
logpath = /var/log/mongo/mongod_6669.log
logappend = true
fork = true
port = 6669
bind_ip = 0.0.0.0
dbpath = /opt/mongodata/digoal/6669/data01/mongodb_datafile
maxConns = 12000
pidfilepath = /opt/mongodata/digoal/6669/data01/mongodb_run/mongod_6669.pid
nounixsocket = true
directoryperdb = true
oplogSize = 31280
journal = true
journalCommitInterval = 40
profile = 1
slowms = 50
noauth = true
nohttpinterface = true
nssize = 2000
mongos配置文件
logpath = /var/log/mongo/mongod_6666.log
logappend = true
fork = true
port = 6666
bind_ip = 0.0.0.0
maxConns = 200
pidfilepath = /opt/mongodata/digoal/6666/data01/mongodb_run/mongod_6666.pid
启动configsrv,mongos
10.10.10.14 :
# su - mongo -c "/opt/mongodb-linux-x86_64-2.0.0/bin/mongod --configsvr -f /mongodata/digoal/6669/data01/mongodb_etc/mongod_6669.conf"
# su - mongo -c "/opt/mongodb-linux-x86_64-2.0.0/bin/mongod --configsvr -f /opt/mongodata/digoal/6670/data01/mongodb_etc/mongod_6670.conf"
10.10.10.16 :
su - mongo -c "/opt/mongodb-linux-x86_64-2.0.0/bin/mongod --configsvr -f /opt/mongodata/digoal/6669/data01/mongodb_etc/mongod_6669.conf"
su - mongo -c "/opt/mongodb-linux-x86_64-2.0.0/bin/mongos --configdb 10.10.10.14:6669,10.10.10.14:6670,10.10.10.16:6669 -f /opt/mongodata/digoal/6666/data01/mongodb_etc/mongod_6666.conf"
shard配置,连接到mongos
> db.adminCommand( { addShard : "digoal001/10.10.10.10:6666,10.10.10.11:6667,10.10.10.12:6668" } )
> db.adminCommand( { addShard : "digoal002/10.10.10.11:6666,10.10.10.10:6667,10.10.10.13:6668" } )
> db.adminCommand( { addShard : "digoal003/10.10.10.12:6666,10.10.10.13:6667,10.10.10.10:6668" } )
> db.adminCommand( { addShard : "digoal004/10.10.10.13:6666,10.10.10.12:6667,10.10.10.11:6668" } )
mongod.conf配置文件范例 :
# mongod_$PORT.conf
# where to log
logpath = /var/log/mongo/mongod_6666.log
logappend = true
# fork and run in background
fork = true
port = 6666
bind_ip = 0.0.0.0
dbpath = /mongodata/digoal/6666/data01/mongodb_datafile
maxConns = 12000
pidfilepath = /mongodata/digoal/6666/data01/mongodb_run/mongod_6666.pid
nounixsocket = true
directoryperdb = true
oplogSize = 31280
journal = true
journalCommitInterval = 40
profile = 1
slowms = 50
# Disables write-ahead journaling
# nojournal = true
# Enables periodic logging of CPU utilization and I/O wait
#cpu = true
# Turn on/off security. Off is currently the default
noauth = true
#auth = true
# Verbose logging output.
#verbose = true
# Inspect all client data for validity on receipt (useful for
# developing drivers)
#objcheck = true
# Enable db quota management
#quota = true
# Set oplogging level where n is
# 0=off (default)
# 1=W
# 2=R
# 3=both
# 7=W+some reads
#oplog = 0
# Ignore query hints
#nohints = true
# Disable the HTTP interface (Defaults to localhost:27018).
nohttpinterface = true
# Turns off server-side scripting. This will result in greatly limited
# functionality
#noscripting = true
# Turns off table scans. Any query that would do a table scan fails.
#notablescan = true
# Disable data file preallocation.
#noprealloc = true
# Specify .ns file size(MB) for new databases.
nssize = 2000
# Accout token for Mongo monitoring server.
#mms-token = <token>
# Server name for Mongo monitoring server.
#mms-name = <server-name>
# Ping interval for Mongo monitoring server.
#mms-interval = <seconds>
# Replication Options
# in replicated mongo databases, specify here whether this is a slave or master
#slave = true
#source = master.example.com
# Slave only: specify a single database to replicate
#only = master.example.com
# or
#master = true
#source = slave.example.com
由于使用了noauth=true,所以配置防火墙是很有必要的.以下是范例
iptables防火墙配置
-A RH-Firewall-1-INPUT -i lo -j ACCEPT
# mongoDB:mongod,configsrv,mongos
-A RH-Firewall-1-INPUT -m iprange --src-range 10.10.10.10-10.10.10.13 -j ACCEPT
-A RH-Firewall-1-INPUT -m iprange --src-range 10.10.10.14-10.10.10.16 -j ACCEPT
# mongoDB:appserver
-A RH-Firewall-1-INPUT -m iprange --src-range xxx.xxx.xxx.xxx-xxx.xxx.xxx.xxx -p tcp -m multiport --destination-ports 6666,6669:6670 -j ACCEPT
# mongoDB:reject Port
-A RH-Firewall-1-INPUT -m state --state NEW -p tcp -m multiport --destination-ports 6666,6669:6670 -j REJECT
最后是GRIDFS的注意事项 :
"chunks" collection should be sharded using the index "files_id: 1". The existing "files_id, n" index created by the drivers cannot be used to shard on "files_id" (this is a sharding limitation that will be fixed), so you have to create a separate index on just "files_id". The reason to use "files_id" is so that all chunks of a given file live on the same shard, which is safer and allows "filemd5" command to work (required by certain drivers).
类似资料：