安装

优质

小牛编辑

133浏览

2023-12-01

ubuntu 18.04 上在线安装

本部分在 ubuntu 18.04 上在线安装，每个节点均可联网，且可下载 Google 的容器镜像的 RPM 包。

环境准备

1. 所有节点静态域名配置

// static ip addr
- machine01
network:
  version: 2
  renderer: NetworkManager
  ethernets:
    ens33:
      dhcp4: no
      addresses: [192.168.100.101/24]
      gateway4: 192.168.100.2
      nameservers:
        addresses: [192.168.100.2]

- machine02
network:
  version: 2
  renderer: NetworkManager
  ethernets:
    ens33:
      dhcp4: no
      addresses: [192.168.100.102/24]
      gateway4: 192.168.100.2
      nameservers:
        addresses: [192.168.100.2]

- machine03
network:
  version: 2
  renderer: NetworkManager
  ethernets:
    ens33:
      dhcp4: no
      addresses: [192.168.100.103/24]
      gateway4: 192.168.100.2
      nameservers:
        addresses: [192.168.100.2]

// config static ip
netplan apply

// set ip host mapping
sudo echo "192.168.1.128 machine01.example.com machine01" >> /etc/hosts
sudo echo "192.168.1.129 machine02.example.com machine02" >> /etc/hosts
sudo echo "192.168.1.130 machine03.example.com machine03" >> /etc/hosts

2. ping 测试

for i in 1 2 3 ; do ping machine0$i.example.com -c3 ; done

3. ssh 免密登录

// generate key
ssh-keygen

// sshd on all machines
apt-get install openssh-server -y
systemctl status ssh

// enable ssh root login
sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
systemctl restart ssh

// copy to other machines
for i in 1 2 3 ; do ssh-copy-id machine0$i.example.com ; done

// check connectivity
for i in 1 2 3 ; do ssh machine0$i.example.com 'date' ; done

4. Software install & setup

// install
for i in 1 2 3 ; do ssh machine0$i.example.com 'apt install vim tree -y' ; done

// disable swap

// docker
https://www.digitalocean.com/community/tutorials/how-to-install-and-use-docker-on-ubuntu-18-04
apt install apt-transport-https ca-certificates curl software-properties-common
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable"
apt update
apt-cache policy docker-ce
apt install docker-ce
systemctl status docker
systemctl enable docker

kubeadm 准备

apt-get update && sudo apt-get install -y apt-transport-https curl

curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -

cat <<EOF | sudo tee /etc/apt/sources.list.d/kubernetes.list
deb https://apt.kubernetes.io/ kubernetes-xenial main
EOF

apt-get update
apt-get install -y kubelet kubeadm kubectl
apt-mark hold kubelet kubeadm kubectl

systemctl daemon-reload
systemctl restart kubelet

安装

1. config.yml

apiVersion: kubeadm.k8s.io/v1beta2
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: abcdef.0123456789abcdef
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 1.2.3.4
  bindPort: 6443
nodeRegistration:
  criSocket: /var/run/dockershim.sock
  name: machine01.example.com
  taints:
  - effect: NoSchedule
    key: node-role.kubernetes.io/master
---
apiServer:
  timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta2
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns:
  type: CoreDNS
etcd:
  local:
    dataDir: /var/lib/etcd
imageRepository: k8s.gcr.io
kind: ClusterConfiguration
kubernetesVersion: v1.17.0
networking:
  dnsDomain: cluster.local
  serviceSubnet: 10.96.0.0/12
scheduler: {}

2. init control panel

# kubeadm init --control-plane-endpoint=control-plane.example.com --ignore-preflight-errors=NumCPU

...

Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

You can now join any number of control-plane nodes by copying certificate authorities
and service account keys on each node and then running the following as root:

  kubeadm join control-plane.example.com:6443 --token 887x5p.6uyb4cembh7926ok \
    --discovery-token-ca-cert-hash sha256:cb29759ded3490c7edc204ad8238cf973284e41d769e793ca49cebf14ee8996b \
    --control-plane

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join control-plane.example.com:6443 --token 887x5p.6uyb4cembh7926ok \
    --discovery-token-ca-cert-hash sha256:cb29759ded3490c7edc204ad8238cf973284e41d769e793ca49cebf14ee8996b

3. none root user run kubectl

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

4. pod networking

kubectl apply -f https://docs.projectcalico.org/v3.11/manifests/calico.yaml

5. check kube-system pods

# kubectl get pods --all-namespaces
NAMESPACE     NAME                                            READY   STATUS    RESTARTS   AGE
kube-system   calico-kube-controllers-5b644bc49c-m6wdh        1/1     Running   0          46m
kube-system   calico-node-5nqz7                               1/1     Running   0          46m
kube-system   coredns-6955765f44-f4wxq                        1/1     Running   0          56m
kube-system   coredns-6955765f44-rfdzc                        1/1     Running   0          56m
kube-system   etcd-machine01.example.com                      1/1     Running   0          57m
kube-system   kube-apiserver-machine01.example.com            1/1     Running   0          57m
kube-system   kube-controller-manager-machine01.example.com   1/1     Running   0          57m
kube-system   kube-proxy-ghm6k                                1/1     Running   0          56m
kube-system   kube-scheduler-machine01.example.com            1/1     Running   0          57m

6. join workers

kubeadm join control-plane.example.com:6443 --token 887x5p.6uyb4cembh7926ok \
    --discovery-token-ca-cert-hash sha256:cb29759ded3490c7edc204ad8238cf973284e41d769e793ca49cebf14ee8996b

7. check all nodes is ready

# kubectl get nodes
NAME                    STATUS   ROLES    AGE     VERSION
machine01.example.com   Ready    master   158m    v1.17.3
machine02.example.com   Ready    worker   6m44s   v1.17.3
machine03.example.com   Ready    worker   5m10s   v1.17.3

RHEL 7 上离线安装

本部分说明如何在 RHEL 7 上离线安装，每个节点都没有连接互联网。

环境准备

1. 静态域名配置（所有节点）

echo "192.168.122.11 machine01.example.com machine01" >> /etc/hosts
echo "192.168.122.12 machine02.example.com machine02" >> /etc/hosts

2. 节点互信免密登录

ssh-keygen
for i in machine01 machine02; do ssh-copy-id $i.example.com; done;

3. 查看主机名和操作系统版本

$ for i in 1 2 ; do ssh machine0$i.example.com 'hostname; cat /etc/redhat-release' ; done
machine01.example.com
CentOS Linux release 7.8.2003 (Core)
machine02.example.com
CentOS Linux release 7.8.2003 (Core)

4. 禁用 Swap（所有节点）

# blkid | grep swap
/dev/mapper/centos-swap: UUID="c53e4112-342c-429a-bb05-77e2f7e0462f" TYPE="swap"

# swapoff /dev/mapper/centos-swap

// To permanently disable swap, comment out the swap line
# vim /etc/fstab

# mount -a

# reboot

5. 确保每个节点二层 MAC 地址和 product_uuid 唯一

$ for i in 1 2 ; do ssh machine0$i.example.com 'ip link | grep link/ether' ; done
    link/ether 52:54:00:93:61:a8 brd ff:ff:ff:ff:ff:ff
    link/ether 52:54:00:dc:6e:20 brd ff:ff:ff:ff:ff:ff

$ for i in 1 2 ; do ssh machine0$i.example.com 'sudo cat /sys/class/dmi/id/product_uuid' ; done
F6BC8F4B-EA5C-4CDD-8773-94F3600947A7
F1E65C6B-21B0-4FF5-9DBD-5199F24DFEAA

6. 关闭 SElinux

// change from enforcing to disabled
vim /etc/selinux/config

// alternatives
setenforce 0
sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config

7. 管理节点打开端口

firewall-cmd --zone=public --add-port=6443/tcp --permanent
firewall-cmd --zone=public --add-port=2379-2380/tcp --permanent
firewall-cmd --zone=public --add-port=10250-10252/tcp --permanent
firewall-cmd --reload

8. 计算节点打开端口

firewall-cmd --zone=public --add-port=10250/tcp --permanent
firewall-cmd --zone=public --add-port=30000-32767/tcp --permanent
firewall-cmd --reload

9. 创建 Snapshot（本部分只有在使用 KVM 虚拟化情况下在 KVM 宿主机执行）

// create
qemu-img snapshot -c 20200505 /home/virt/machine01
qemu-img snapshot -c 20200505 /home/virt/machine02

// view exist snapshot
qemu-img snapshot -l /home/virt/machine01
qemu-img snapshot -l /home/virt/machine02

// rollback to snapshot(only in the time need to rollback)
qemu-img snapshot -a 20200505 /home/virt/machine01
qemu-img snapshot -a 20200505 /home/virt/machine02

安装 Docker

安装 Docker 18.06.2

yum install yum-utils device-mapper-persistent-data lvm2

yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo

yum install docker-ce-18.06.2.ce -y

mkdir /etc/docker

cat > /etc/docker/daemon.json <<EOF
{
  "exec-opts": ["native.cgroupdriver=systemd"],
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "100m"
  },
  "storage-driver": "overlay2",
  "storage-opts": [
    "overlay2.override_kernel_check=true"
  ]
}
EOF

mkdir -p /etc/systemd/system/docker.service.d

systemctl daemon-reload
systemctl restart docker
systemctl enable docker

安装 kubeadm, kubelet and kubectl

1. 解压 RPM 包

tar -xvf kubernets.tar.gz

2. 安装

cd kubernets
yum install `ls`

3. net.bridge.bridge-nf-call-iptables

cat <<EOF >  /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF

sysctl --system

4. 确保 br_netfilter 加载到内核

lsmod | grep br_netfilter

5. kubelet 开机器启动

systemctl enable --now kubelet

6. kubelet 启动

systemctl daemon-reload
systemctl restart kubelet

管理节点安装

1. 导入镜像

docker load -i k8s-v1.15.5.tar.gz
docker load -i calico-master.tar.gz

2. 安装

kubeadm init --pod-network-cidr=192.168.0.0/16

3. 拷贝配置文件到本地 HOME 目录

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

4. 安装网络插件

kubectl apply -f https://docs.projectcalico.org/v3.11/manifests/calico.yaml

5. 管理节点也可分配计算任务

kubectl taint nodes --all node-role.kubernetes.io/master-

6. 查看运行的容器

$ kubectl get pods --all-namespaces
NAMESPACE     NAME                                            READY   STATUS    RESTARTS   AGE
kube-system   calico-kube-controllers-846568ccc-bm9xf         1/1     Running   0          2m42s
kube-system   calico-node-2xf7s                               1/1     Running   0          2m42s
kube-system   coredns-5c98db65d4-5xgrf                        1/1     Running   0          10m
kube-system   coredns-5c98db65d4-md5hw                        1/1     Running   0          10m
kube-system   etcd-machine01.example.com                      1/1     Running   0          9m11s
kube-system   kube-apiserver-machine01.example.com            1/1     Running   0          9m8s
kube-system   kube-controller-manager-machine01.example.com   1/1     Running   0          9m
kube-system   kube-proxy-pkg8d                                1/1     Running   0          10m
kube-system   kube-scheduler-machine01.example.com            1/1     Running   0          9m

计算节点加入

1. 导入镜像

docker load -i k8s-v1.15.5.tar.gz
docker load -i calico-worker.tar.gz

2. 计算节点加入

kubeadm join 192.168.122.11:6443 --token q8fdva.fg35rk2ael190gv8 \
     --discovery-token-ca-cert-hash sha256:244e6bf4d6b9d5b7d1ee14a70f3ff05bf003917bcb3e053ff4995ee72c85e339

3. 返回管理节点查看所有节点

kubectl get nodes
NAME                    STATUS   ROLES    AGE   VERSION
machine01.example.com   Ready    master   34m   v1.15.5
machine02.example.com   Ready    <none>   15m   v1.15.5

集群添加节点

添加节点方法一

// 1. generate token
# kubeadm token generate
yfvb6j.0628gwwf3ec5bzu4

// 2. pring join commands
# kubeadm token create yfvb6j.0628gwwf3ec5bzu4 --ttl 2h --print-join-command
kubeadm join control-plane.example.com:6443 --token yfvb6j.0628gwwf3ec5bzu4     --discovery-token-ca-cert-hash sha256:cb29759ded3490c7edc204ad8238cf973284e41d769e793ca49cebf14ee8996b

// 3. copy kubeadm join commands to worker node implement the node join

添加节点方法二

// 1. generate token
$ kubeadm token create
n2kb3q.ctmc0wpfnt4cjtbl

// 2. generate join node commands
$ openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | \
    openssl dgst -sha256 -hex | sed 's/^.* //'
cb29759ded3490c7edc204ad8238cf973284e41d769e793ca49cebf14ee8996b

kubeadm join control-plane.example.com:6443 --token n2kb3q.ctmc0wpfnt4cjtbl \
    --discovery-token-ca-cert-hash sha256:cb29759ded3490c7edc204ad8238cf973284e41d769e793ca49cebf14ee8996b

// 3. copy kubeadm join commands to worker node implement the node join

已存在集群重新设定 iptables 规则 & 端口打开

1. iptables 规则删除

iptables-save | awk '/^[*]/ { print $1 }
                     /^:[A-Z]+ [^-]/ { print $1 " ACCEPT" ; }
                     /COMMIT/ { print $0; }' | iptables-restore

2. Master 节点端口打开

firewall-cmd --zone=public --add-port=6443/tcp --permanent
firewall-cmd --zone=public --add-port=2379-2380/tcp --permanent
firewall-cmd --zone=public --add-port=10250-10252/tcp --permanent
firewall-cmd --reload

3. Node 节点端口打开

firewall-cmd --zone=public --add-port=10250/tcp --permanent
firewall-cmd --zone=public --add-port=30000-32767/tcp --permanent
firewall-cmd --reload