AWS EKS搭建Redis集群

关飞翼

2023-12-01

部署redis集群

申请AWS EFS

mount -t efs fs-5f5bd3ba:/ /tmp
mkdir -p /tmp/{redis1,redis2,redis3,redis4,redis5,redis6}
umount /tmp

安装持久化存储

这里使用AWS EFS

文件ID fs-b035f655

redis-pv.yaml

apiVersion: v1
kind: PersistentVolume
metadata:
  name: "redis-pv1"
  namespace: "redis"
spec:
  capacity:
    storage: 200M
  volumeMode: Filesystem
  accessModes:
    - ReadWriteMany
  persistentVolumeReclaimPolicy: Delete
  storageClassName: efs-sc
  csi:
    driver: efs.csi.aws.com
    volumeHandle: "fs-5f5bd3ba:/redis1"
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: "redis-pv2"
  namespace: "redis"
spec:
  capacity:
    storage: 200M
  volumeMode: Filesystem
  accessModes:
    - ReadWriteMany
  persistentVolumeReclaimPolicy: Delete
  storageClassName: efs-sc
  csi:
    driver: efs.csi.aws.com
    volumeHandle: "fs-5f5bd3ba:/redis2"
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: "redis-pv3"
  namespace: "redis"
spec:
  capacity:
    storage: 200M
  volumeMode: Filesystem
  accessModes:
    - ReadWriteMany
  persistentVolumeReclaimPolicy: Delete
  storageClassName: efs-sc
  csi:
    driver: efs.csi.aws.com
    volumeHandle: "fs-5f5bd3ba:/redis3"
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: "redis-pv4"
  namespace: "redis"
spec:
  capacity:
    storage: 200M
  volumeMode: Filesystem
  accessModes:
    - ReadWriteMany
  persistentVolumeReclaimPolicy: Delete
  storageClassName: efs-sc
  csi:
    driver: efs.csi.aws.com
    volumeHandle: "fs-5f5bd3ba:/redis4"
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: "redis-pv5"
  namespace: "redis"
spec:
  capacity:
    storage: 200M
  volumeMode: Filesystem
  accessModes:
    - ReadWriteMany
  persistentVolumeReclaimPolicy: Delete
  storageClassName: efs-sc
  csi:
    driver: efs.csi.aws.com
    volumeHandle: "fs-5f5bd3ba:/redis5"
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: "redis-pv6"
  namespace: "redis"
spec:
  capacity:
    storage: 200M
  volumeMode: Filesystem
  accessModes:
    - ReadWriteMany
  persistentVolumeReclaimPolicy: Delete
  storageClassName: efs-sc
  csi:
    driver: efs.csi.aws.com
    volumeHandle: "fs-5f5bd3ba:/redis6"

创建configmap redis-conf文件

redis-configmap.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: redis-conf
  namespace: redis
  labels:
    k8s-app: redis
    kubernetes.io/cluster-service: "true"
data:
  redis.conf: |-
    appendonly yes
    requirepass wuerp	#设置密码
    cluster-enabled yes
    cluster-config-file /var/lib/redis/nodes.conf
    cluster-node-timeout 5000
    dir /var/lib/redis
    port 6379

创建redis集群节点

这是本文的核心内容,创建redis.yaml文件

apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
  name: redis-app
  namespace: redis
spec:
  serviceName: "redis-service"
  replicas: 6
  template:
    metadata:
      labels:
        app: redis
        appCluster: redis-cluster
    spec:
      containers:
      - name: redis
        image: "redis"
        command:
          - "redis-server"
        args:
          - "/etc/redis/redis.conf"
          - "--protected-mode"
          - "no"
        resources:
          requests:
            cpu: "100m"
            memory: "100Mi"
        ports:
            - name: redis
              containerPort: 6379
              protocol: "TCP"
            - name: cluster
              containerPort: 16379
              protocol: "TCP"
        volumeMounts:
          - name: "redis-conf"
            mountPath: "/etc/redis"
          - name: "redis-data"
            mountPath: "/var/lib/redis"
      volumes:
      - name: "redis-conf"
        configMap:
          name: "redis-conf"
          items:
            - key: "redis.conf"
              path: "redis.conf"
  volumeClaimTemplates:                     #进行pvc持久卷声明,
  - metadata:
      name: redis-data
    spec:
      accessModes:
      - ReadWriteMany
      storageClassName: efs-sc
      resources:
        requests:
          storage: 200M

创建headless service

Headless service是StatefulSet实现稳定网络标识的基础，我们需要提前创建。准备文件headless-service.yml如下：

apiVersion: v1
kind: Service
metadata:
  name: redis-service
  namespace: redis
  labels:
    app: redis
spec:
  ports:
  - name: redis-port
    port: 6379
  clusterIP: None
  selector:
    app: redis
    appCluster: redis-cluster

初始化redis集群

创建好6个Redis Pod后，我们还需要利用常用的Redis-tribe工具进行集群的初始化。

CentOS初始化

kubectl run -it centos --image=097364988641.dkr.ecr.cn-northwest-1.amazonaws.com.cn/centos:latest --restart=Never /bin/bash

更换CentOS的镜像源

cd /etc/yum.repos.d
rm -rf ./*
curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-8.repo
sed -i 's/releasever/releasever-stream/g' /etc/yum.repos.d/CentOS-Base.repo
yum makecache
yum -y install vim gcc wget make bind-utils ruby
wget https://download.redis.io/releases/redis-5.0.14.tar.gz
tar -xf redis-5.0.14.tar.gz
cd redis-5.0.14
make && make install

初始化集群

redis-cli --cluster create --cluster-replicas 1 \
`dig +short redis-app-0.redis-service.redis.svc.cluster.local`:6379 \
`dig +short redis-app-1.redis-service.redis.svc.cluster.local`:6379 \
`dig +short redis-app-2.redis-service.redis.svc.cluster.local`:6379 \
`dig +short redis-app-3.redis-service.redis.svc.cluster.local`:6379 \
`dig +short redis-app-4.redis-service.redis.svc.cluster.local`:6379 \
`dig +short redis-app-5.redis-service.redis.svc.cluster.local`:6379

root@k8s-node1 ~]# kubectl exec -it redis-app-2 /bin/bash
root@redis-app-2:/data# /usr/local/bin/redis-cli -c
127.0.0.1:6379> cluster info
cluster_state:ok
cluster_slots_assigned:16384
cluster_slots_ok:16384
cluster_slots_pfail:0
cluster_slots_fail:0
cluster_known_nodes:6
cluster_size:3
cluster_current_epoch:6
cluster_my_epoch:1
cluster_stats_messages_ping_sent:186
cluster_stats_messages_pong_sent:199
cluster_stats_messages_sent:385
cluster_stats_messages_ping_received:194
cluster_stats_messages_pong_received:186
cluster_stats_messages_meet_received:5
cluster_stats_messages_received:385

127.0.0.1:6379> cluster nodes
589b4f4f908a04f56d2ab9cd6fd0fd25ea14bb8f 10.42.1.13:6379@16379 master - 0 1550555011000 3 connected 10923-16383
e9f1f704ff7c8f060d6b39e23be9cd8e55cb2e46 10.42.1.14:6379@16379 slave 589b4f4f908a04f56d2ab9cd6fd0fd25ea14bb8f 0 1550555011512 6 connected
366abbba45d3200329a5c6305fbcec9e29b50c80 10.42.2.18:6379@16379 slave 4676f8913cdcd1e256db432531c80591ae6c5fc3 0 1550555010507 4 connected
505f3e126882c0c5115885e54f9b361bc7e74b97 10.42.0.15:6379@16379 master - 0 1550555011000 2 connected 5461-10922
cee3a27cc27635da54d94f16f6375cd4acfe6c30 10.42.0.16:6379@16379 slave 505f3e126882c0c5115885e54f9b361bc7e74b97 0 1550555011713 5 connected
4676f8913cdcd1e256db432531c80591ae6c5fc3 10.42.2.17:6379@16379 myself,master - 0 1550555010000 1 co

创建用于访问service

前面我们创建了用于实现statefulset的headless service,但该service没有cluster IP,因此不能用于外界访问.所以我们还需要创建一个service,专用于为Redis集群提供访问和负载均衡:

redis-svc.yaml

apiVersion: v1
kind: Service
metadata:
  name: redis-access
  namespace: redis
  labels:
    app: redis
  annotations:
    service.beta.kubernetes.io/aws-load-balancer-type: "nlb"
spec:
  ports:
  - name: redis-port
    protocol: "TCP"
    port: 6379
    targetPort: 6379
  type: LoadBalancer  
  selector:
    app: redis
    appCluster: redis-cluster

这里使用的是AWS alb负载均衡器创建访问

得到访问地址

[root@ip-172-29-140-52 redis-cluster]# kubectl get svc
NAME            TYPE           CLUSTER-IP       EXTERNAL-IP                                                                             PORT(S)          AGE
redis-access    LoadBalancer   10.100.184.249   xxxxxxxxxxxxxx   6379:30585/TCP   22h
redis-service   ClusterIP      None             <none>                                                                                  6379/TCP         23h

xxxxxxxxxxxxxx

测试远程连接

redis-cli -c -h xxxxxxxxxxxxxx    -p 6379
auth wuerp
cluster nodes