CentOS 7.7安装Ceph nautilus
CentOS 7.7最小化安装
所有节点安装epel源
yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
所有节点初始化配置
nmtui配置网络连接，自动连接
# Kernel pid max
echo 4194303 > /proc/sys/kernel/pid_max
#关闭selinux
sed -i 's/SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config
setenforce 0
##停止firewall 禁止firewall开机启动
systemctl stop firewalld.service
systemctl disable firewalld.service
##host访问
vim /etc/hosts:
192.168.111.179 ceph1
192.168.111.180 ceph2
192.168.111.181 ceph3
hostnamectl set-hostname ceph1
hostnamectl set-hostname ceph2
hostnamectl set-hostname ceph3
## 安装ntp
yum install ntp
vi /etc/ntp.conf
## 修改启动ntp服务
##注释如下四项，然后选择内部的时钟服务器或者ceph管理节点
server 192.168.111.179
#server 0.centos.pool.ntp.org iburst
#server 1.centos.pool.ntp.org iburst
#server 2.centos.pool.ntp.org iburst
#server 3.centos.pool.ntp.org iburst
##启动
systemctl restart ntpd.service
systemctl enable ntpd.service
## 其他
yum install wget net-tools
所有添加Ceph源
cat << EOM > /etc/yum.repos.d/ceph.repo
[ceph-noarch]
name=Ceph noarch packages
baseurl=https://download.ceph.com/rpm-nautilus/el7/noarch
enabled=1
gpgcheck=1
type=rpm-md
gpgkey=https://download.ceph.com/keys/release.asc
priority=1
EOM
Update your repository and install ceph-deploy:
管理节点执行
生成密钥
##生成秘钥，免密码登陆
ssh-keygen
ssh-copy-id root@ceph1
ssh-copy-id root@ceph2
ssh-copy-id root@ceph3
安装配置 ceph-deploy
yum install -y https://download.ceph.com/rpm-nautilus/el7/noarch/ceph-deploy-2.0.1-0.noarch.rpm
#yum install ceph-deploy
创建集群
创建目录
mkdir my_cluster
cd my_cluster
生成ceph.conf和kering
ceph-deploy new ceph1 ceph2 ceph3 ##会生成ceph.conf和ceph.mon.keyring
[root@ceph1 my_cluster]# ceph-deploy new ceph1 ceph2 ceph3
Traceback (most recent call last):
File "/usr/bin/ceph-deploy", line 18, in <module>
from ceph_deploy.cli import main
File "/usr/lib/python2.7/site-packages/ceph_deploy/cli.py", line 1, in <module>
import pkg_resources
ImportError: No module named pkg_resources
[root@ceph1 my_cluster]# yum install python-pkg-resources python-setuptools
yum install python-pkg-resources python-setuptools
配置ceph.conf
cd my-cluster
vim ceph.conf:
[global]
fsid = 2595ba62-8ae1-difg-893a-892a675757c6
mon_initial_members = ceph1, ceph2, ceph3
mon_host = 172.28.7.239,172.28.7.240,172.28.7.241
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
##如果有两块可以连通的网卡，配置public_network和cluster_network，如果是测试环境没有，就不配置
#public_network = 172.28.7.0/22 #管理网络
#cluster_network = 172.28.11.0/22 #集群网络
mon_pg_warn_max_per_osd = 1000
osd_pool_default_size = 3
osd_pool_default_min_size = 2
mon_osd_backfillfull_ratio = 0.75
mon_osd_full_ratio = .85
mon_osd_nearfull_ratio = .70
osd_failsafe_full_ratio = 0.90
osd_deep_scrub_randomize_ratio = 0.01
[mgr]
mgr modules = dashboard
[osd]
osd_max_write_size = 1024
osd_recovery_op_priority = 1
osd_recovery_max_active = 1
osd_recovery_max_single_start = 1
osd_recovery_max_chunk = 1048576
osd_recovery_threads = 1
osd_max_backfills = 1
osd_scrub_begin_hour = 22
osd_scrub_end_hour = 7
osd_recovery_sleep = 0
osd_crush_update_on_start = false ##如果初始化的时候，把该值设置为true，然后重启所有osd。不然创建完pool会提示：100.000% pgs unknown100.000% pgs unknown。所有osd都加入，集群ok后，再统一把该值设置为false, 然后重启所有osd
注意：只要新创建了pool， 所有osd 设置：osd_crush_update_on_start = true 才会进行调整
安装ceph
# export CEPH_DEPLOY_REPO_URL=http://mirrors.163.com/ceph/rpm-nautilus/el7
# export CEPH_DEPLOY_GPG_URL=http://mirrors.163.com/ceph/keys/release.asc
export CEPH_DEPLOY_REPO_URL=http://mirrors.ustc.edu.cn/ceph/rpm-nautilus/el7
export CEPH_DEPLOY_GPG_URL=http://mirrors.ustc.edu.cn/ceph/keys/release.asc
export CEPH_DEPLOY_REPO_URL=http://mirrors.aliyun.com/ceph/rpm-nautilus/el7
export CEPH_DEPLOY_GPG_URL=http://mirrors.aliyun.com/ceph/keys/release.asc
export CEPH_DEPLOY_REPO_URL=https://download.ceph.com/rpm-nautilus/el7
export CEPH_DEPLOY_GPG_URL=https://download.ceph.com/keys/release.asc
# ceph-deploy install ceph1 ceph2 ceph3
或者
# ceph-deploy install --release nautilus ceph1 ceph2 ceph3
集群初始化
# cd my_cluster
# ceph-deploy mon create-initial
拷贝管理文件
根据实际情况，拷贝管理文件到设定的管理节点
ceph-deploy admin ceph1 ceph2 ceph3
配置Manager节点
ceph-deploy mgr create ceph1
扩展Manager节点
ceph-deploy mgr create ceph2
ceph-deploy mgr create ceph3
测试集群的健康状态
[root@ceph1 my_cluster]# ceph -s
cluster:
id: f75e1135-05c8-4765-9503-bb353722c879
health: HEALTH_WARN
clock skew detected on mon.ceph2, mon.ceph3
services:
mon: 3 daemons, quorum ceph1,ceph2,ceph3 (age 7m)
mgr: ceph1(active, since 24s), standbys: ceph2, ceph3
osd: 0 osds: 0 up, 0 in
data:
pools: 0 pools, 0 pgs
objects: 0 objects, 0 B
usage: 0 B used, 0 B / 0 B avail
pgs:
[root@ceph1 my_cluster]#
[root@ceph1 my_cluster]# ceph health
HEALTH_WARN clock skew detected on mon.ceph2, mon.ceph3
[root@ceph1 my_cluster]#
添加OSD
ceph-deploy osd create --data /dev/sdb ceph1
ceph-deploy osd create --data /dev/sdb ceph2
ceph-deploy osd create --data /dev/sdb ceph3
[root@ceph1 my_cluster]# ceph -s
cluster:
id: f75e1135-05c8-4765-9503-bb353722c879
health: HEALTH_OK
services:
mon: 3 daemons, quorum ceph1,ceph2,ceph3 (age 5m)
mgr: ceph1(active, since 6m), standbys: ceph2, ceph3
osd: 3 osds: 3 up (since 11s), 3 in (since 11s)
data:
pools: 0 pools, 0 pgs
objects: 0 objects, 0 B
usage: 3.0 GiB used, 432 GiB / 435 GiB avail
pgs:
[root@ceph1 my_cluster]# ceph osd tree
ID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF
-1 0 root default
0 hdd 0 osd.0 up 1.00000 1.00000
1 hdd 0 osd.1 up 1.00000 1.00000
2 hdd 0 osd.2 up 1.00000 1.00000
[root@ceph1 my_cluster]#
配置dashboard
1. Install the appropriate package from the below link
# yum install http://download.ceph.com/rpm-nautilus/el7/noarch/ceph-mgr-dashboard-14.2.1-0.el7.noarch.rpm
# yum install http://mirrors.163.com/ceph/rpm-nautilus/el7/noarch/ceph-mgr-dashboard-14.2.2-0.el7.noarch.rpm
Note: it will ask for few dependencies, install with yum/apt package manager and then execute the above command.
2. Enable the ceph mgr dashboard
# ceph mgr module enable dashboard --force
# ceph mgr module ls
3. Create self-signed certificate
# ceph dashboard create-self-signed-cert
Self-signed certificate created
4. Create a user for Dashboard
Example: [ceph dashboard ac-user-create (username) (password) administrator]
# ceph dashboard ac-user-create admin password administrator
{"username": "cent", "lastUpdate": 1560292901, "name": null, "roles": ["administrator"], "password": "$2b$12$w60gItcbKd6PULNYI9McmOBMiAzFoKJ9T9XGva8vC6dxIyqMsE4kK", "email": null}
# ceph mgr services
{
"dashboard": "https://ceph-mgr:8443/"
}
Note: Here you can access with IP address of ceph-mgr node, instead of hostname.
5. Make sure firewall port is open
# firewall-cmd --add-port=8443/tcp --permanent
# firewall-cmd --reload
6. Open the dashboard url in any browser
https://ceph-mgr:8443 or https://192.168.1.10:8443
7. Enter the username: cent and password: password
Here you go...
客户端上操作：
yum -y install centos-release-ceph-nautilus.noarch
yum -y install ceph-common
ceph 服务器上操作：
ceph auth get-or-create client.clt132 mon 'allow r' osd 'allow class-read object_prefix rbd_children,allow rwx pool=rbd'
# ceph auth get-or-create client.clt132 mon 'allow r' osd 'allow class-read object_prefix rbd_children,allow rwx pool=rbd'
[client.clt132]
key = AQCRVYldu2N4CBAAD5UiNpWnrE3GlHVLa12Miw==
ceph auth get-or-create client.clt132 | tee /etc/ceph/ceph.client.clt132.keyring
scp /etc/ceph/ceph.client.clt132.keyring 192.168.111.132:/etc/ceph/
scp /etc/ceph/ceph.conf 192.168.111.132:/etc/ceph/
客户端上操作：
rbd --image rbd_data1 info --name client.clt132
ceph -s --name client.clt132
rbd create rbd/rbd132 --size 1G --image-feature layering --name client.clt132
rbd --image rbd132 info --name client.clt132
rbd map rbd/rbd132 --name client.clt132
rbd showmapped --name client.clt132
mkfs.xfs /dev/rbd0
mount /dev/rbd0 /mnt/
df -h
umount /mnt/
rbd unmap rbd/rbd132 --name client.clt132
rbd map rbd/rbd_data1 --name client.clt132
mount /dev/rbd1 /mnt/
umount /mnt/
rbd unmap rbd/rbd_data1 --name client.clt132