centos 7.x 安装 yugabytedb 的优化设置

颜新

2023-12-01

os: centos 7.4.1708

yum

centos-release-scl
epel

# yum install -y centos-release-scl;
wget -P /etc/yum.repos.d/ http://mirrors.aliyun.com/repo/epel-7.repo;
yum clean all ; 
yum makecache ; 

yum install -y zip unzip dos2unix pigz iputils iftop git tree sysstat ntp \
make cmake cmake3 gcc gcc-c++ lrzsz mpstat dstat wget bison unixODBC \
e2fsprogs e2fsprogs-devel e4fsprogs e4fsprogs-devel xfsprogs \
smartmontools coreutils glib2 perl perl-devel perl-Pod-Markdown perl-ExtUtils* perl-Text-CSV* \
zlib zlib-devel openldap openldap-devel flex readline readline-devel \
systemtap systemtap-sdt-devel python python-devel tcl tcl-devel sgml-common docbook* stylesheets \
openjade sgml-tools libxslt libxslt-devel libxml2 libxml2-devel openssl openssl-devel \
pam pam-devel libicu libicu-devel uuid uuid-devel libuuid-devel jadetex bzip2 \
automake autoconf libtool hdparm numactl libtool-ltdl libtool-ltdl-devel \
lz4 lz4-devel lz4-static  wget curl \
devtoolset-7-binutils devtoolset-7-gcc devtoolset-7-gcc-c++ devtoolset-7-libstdc++-devel \
devtoolset-7-runtime libedit-devel llvm-toolset-7-clang  llvm-toolset-7-clang-libs  \
llvm-toolset-7-compiler-rt  llvm-toolset-7-libomp  llvm-toolset-7-llvm-libs  \
llvm-toolset-7-runtime llvm5.0 llvm5.0-devel llvm5.0-libs

python2 生命已经终结，后面考虑安装python3

# yum install -y python3 python3-libs python3-devel python3-pip python3-setuptools

# alternatives --set python /usr/bin/python3

另外参考<<centos 使用 scl 软件集>>

user

# groupadd -g 5000 yugabyte;
useradd -g yugabyte -u 5000 yugabyte -d /home/yugabyte;
usermod -g yugabyte yugabyte;
mkdir -p /var/lib/yugabytedb ; 
chown -R yugabyte:yugabyte /var/lib/yugabytedb;

yugabyte 用户的环境变量，先占位，后面修改

# vi /home/yugabyte/.bash_profile

export PGHOME=/usr/yugabytedb-
export PGDATA=/var/lib/yugabytedb 

export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${PGHOME}/lib
export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/lib:/usr/lib:/usr/local/lib
export PATH=${PGHOME}/bin:$PATH
export MANPATH=${PGHOME}/share/man:$MANPATH

export TMP=/tmp
export TEMP=/tmp
export TMPDIR=/tmp

rc.local

# vi /etc/rc.local

############################################
#禁用THP
############################################
if test -f /sys/kernel/mm/transparent_hugepage/enabled; then
   echo never > /sys/kernel/mm/transparent_hugepage/enabled
fi
if test -f /sys/kernel/mm/transparent_hugepage/defrag; then
   echo never > /sys/kernel/mm/transparent_hugepage/defrag
fi


############################################
#预读块大小
############################################
/usr/sbin/blockdev --setra 16384 /dev/sda


############################################
#设置 io scheduler
############################################
# noop 
# deadline
# cfq
############################################
echo deadline > /sys/block/sda/queue/scheduler


############################################
#设置 write through
############################################
# yum install hdparm
#
#set write type 
# 0:write through
# 1:write back
# hdparm -I /dev/sda |grep "Write cache"
# hdparm -W 0 /dev/sda
# hdparm -W 1 /dev/sda
############################################
hdparm -W 0 /dev/sda 


############################################
#清空 iptables
############################################
iptables -F

记得授予可执行权限

# chmod u+x /etc/rc.d/rc.local

sysctl.conf

# vi /etc/sysctl.conf

#共享内存设置,默认已经足够大
#kernel.shmall = 18446744073692774399    
#kernel.shmmax = 18446744073692774399    
kernel.shmmni = 819200 

kernel.sched_migration_cost_ns=50000
kernel.sched_autogroup_enabled=1

#信号量设置
#kernel.sem = 250 32000 100 128 #这个设置在很多场合都报错了,有公式计算
#SEMMSL SEMMNS SEMOPM SEMMNI
#SEMMSL = 50100
#SEMMNS = SEMMSL × SEMMNI = 50100 * 1280 = 64128000
#SEMOPM = SEMMSL = 50100
#SEMMNI = 1280
kernel.sem = 50100 64128000 50100 1280

#接收套接字缓冲区大小
net.core.rmem_default = 262144    
net.core.rmem_max = 4194304 
#发送套接字缓冲区大小
net.core.wmem_default = 262144    
net.core.wmem_max = 4194304 

fs.aio-max-nr = 1048576
fs.nr_open=20480000
fs.file-max = 7672460

net.ipv6.conf.all.disable_ipv6 = 1

net.ipv4.ip_local_port_range = 9000 65000
net.ipv4.tcp_keepalive_time = 60
net.ipv4.tcp_keepalive_probes = 9
net.ipv4.tcp_keepalive_intvl =60
net.ipv4.tcp_retries2 = 5
net.ipv4.tcp_mem = 8388608 12582912 16777216
net.ipv4.tcp_rmem = 8192 87380 16777216    
net.ipv4.tcp_wmem = 8192 65536 16777216  

#操作系统脏页，当大内存时需要调低
vm.dirty_ratio = 5 
vm.dirty_bytes=536870912
vm.dirty_background_ratio=1
vm.dirty_background_bytes = 134217728
vm.dirty_expire_centisecs = 3000 
vm.dirty_writeback_centisecs = 100

#内存分配方式,允许超分配
vm.overcommit_memory = 2
vm.overcommit_ratio = 90
vm.swappiness = 1

#大页设置,大内存建议启动
#vm.nr_hugepages = xxxx
#grep ^Hugepagesize /proc/meminfo

# sysctl -p

nptdate

yugabytedb 对时间的要求比较严格

# crontab -e

# 每1分钟
*/1 * * * * /usr/sbin/ntpdate ntp.sjtu.edu.cn >> /var/log/ntpdate.log 2>&1 &

limits.conf

https://docs.yugabyte.com/latest/deploy/manual-deployment/system-config/#ulimits

# vi /etc/security/limits.conf

*                -       core            unlimited
*                -       data            unlimited
*                -       fsize           unlimited
*                -       sigpending      119934
*                -       memlock         unlimited
*                -       rss             unlimited
*                -       nofile          1048576
*                -       msgqueue        819200
*                -       stack           unlimited
*                -       cpu             unlimited
*                -       nproc           unlimited 
*                -       locks           unlimited

有个坑需要注意下，里面默认设置了非root用户的最大进程数为4096

# cat /etc/security/limits.d/20-nproc.conf

# Default limit for number of user's processes to prevent
# accidental fork bombs.
# See rhbz #432903 for reasoning.

*          soft    nproc     4096
root       soft    nproc     unlimited

删除该文件，或者修改 4096 为更大值

# rm /etc/security/limits.d/20-nproc.conf

systemd ulimit

systemd 的资源限制稍有不同，主要是 centos 对资源限制做了调整。

# man systemd-system.conf

# vi /etc/systemd/system.conf 

DefaultLimitNOFILE=1024000
DefaultLimitNPROC=1024000
DefaultLimitMEMLOCK=infinity
DefaultLimitCORE=infinity
DefaultLimitSTACK=infinity
DefaultLimitAS=infinity
DefaultLimitFSIZE=infinity
DefaultLimitSIGPENDING=119934

# systemctl daemon-reload
# systemctl restart sshd.service

# cat /proc/YOUR-PID/limits

文件系统优化

xfs

# vi /etc/fstab

/dev/sdd1 /data nodev,discard,noatime,inode64,allocsize=16m,errors=remount-ro 0 0

ext4

# vi /etc/fstab

/dev/sdd1 /data defaults,noatime,nodiratime,nodelalloc,barrier=0 0 0

selinux disable

# vi /etc/selinux/config
SELINUX=DISABLED

或者

# vi /etc/sysconfig/selinux
SELINUX=DISABLED

# sed -i 's/SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config

firewalld disable

# systemctl stop firewalld.service;
systemctl disable firewalld.service;

grup

# vi /etc/default/grub
ipv6.disable=1 numa=off transparent_hugepage=never elevator=deadline

# grub2-mkconfig -o /boot/grub2/grub.cfg

备注：

确认numa 已经关闭

# grep -i numa /var/log/dmesg
# cat /proc/cmdline

# yum install -y numactl
# numastat
# numactl --hardware
# numactl --show