docker-compose redis 6.2 cluster (二) 随机分配三主三从

欧阳智志
2023-12-01

[docker redis 6.2 cluster (一) 随机分配三主三从]

ENV

Docker version 19.03.5, build 633a0ea838
docker-compose-1.22.0-2.fc29.noarch
compose file 3.7

code

code on github

创建目录和配置文件

创建配置文件并设置权限

basepath=$(pwd)
for port in $(seq 1 6); \
do \
mkdir -p ./node-${port}/conf
mkdir -p ./node-${port}/data
touch ./node-${port}/conf/redis.conf
sed -e "s/cluster-announce-ip 10.0.0.1/cluster-announce-ip 10.0.0.1${port}/" redis.conf > ./node-${port}/conf/redis.conf

chown -R redis-docker-root:redis-docker-root ${basepath}/node-${port}
chown -R redis-docker-user:redis-docker-root ${basepath}/node-${port}/data
done
ls
diff node-1/conf/redis.conf node-2/conf/redis.conf

使用 docker-compose 创建集群

version: '3.7'

x-image:
 &default-image
 redis:6.2
x-restart:
 &default-restart
 always

services:

 redis-cluster-1:
  image: *default-image
  container_name: redis-cluster-1
  restart: *default-restart
  working_dir: /data
  volumes:
  - /home/data/cluster/redis/compose/node-1/data:/data
  - /home/data/cluster/redis/compose/node-1/conf:/etc/redis/
  command: redis-server /etc/redis/redis.conf
  environment:
      - 'REDIS_NODES=redis-cluster-1 redis-cluster-2 redis-cluster-3 redis-cluster-4 redis-cluster-5 redis-cluster-6'
  ports:
      - 7001:6379
      - 17001:16379
  networks:
      redis-cluster-net:
          ipv4_address: 10.0.0.11

 redis-cluster-2:
  image: *default-image
  container_name: redis-cluster-2
  restart: *default-restart
  working_dir: /data
  volumes:
  - /home/data/cluster/redis/compose/node-2/data:/data
  - /home/data/cluster/redis/compose/node-2/conf:/etc/redis/
  working_dir: /data
  command: redis-server /etc/redis/redis.conf
  environment:
  - 'REDIS_NODES=redis-cluster-1 redis-cluster-2 redis-cluster-3 redis-cluster-4 redis-cluster-5 redis-cluster-6'
  ports:
      - 7002:6379
      - 17002:16379
  networks:
      redis-cluster-net:
          ipv4_address: 10.0.0.12

 redis-cluster-3:
  image: *default-image
  container_name: redis-cluster-3
  restart: *default-restart
  working_dir: /data
  volumes:
  - /home/data/cluster/redis/compose/node-3/data:/data
  - /home/data/cluster/redis/compose/node-3/conf:/etc/redis/
  command: redis-server /etc/redis/redis.conf
  environment:
  - 'REDIS_NODES=redis-cluster-1 redis-cluster-2 redis-cluster-3 redis-cluster-4 redis-cluster-5 redis-cluster-6'
  - 'CLUSTER_CONFIG_FILE=/etc/redis/redis.conf'
  ports:
      - 7003:6379
      - 17003:16379
  networks:
      redis-cluster-net:
          ipv4_address: 10.0.0.13

 redis-cluster-4:
  image: *default-image
  container_name: redis-cluster-4
  restart: *default-restart
  working_dir: /data
  volumes:
  - /home/data/cluster/redis/compose/node-4/data:/data
  - /home/data/cluster/redis/compose/node-4/conf:/etc/redis
  command: redis-server /etc/redis/redis.conf
  environment:
  - 'REDIS_NODES=redis-cluster-1 redis-cluster-2 redis-cluster-3 redis-cluster-4 redis-cluster-5 redis-cluster-6'
  ports:
      - 7004:6379
      - 17004:16379
  networks:
      redis-cluster-net:
          ipv4_address: 10.0.0.14
 redis-cluster-5:
  image: *default-image
  container_name: redis-cluster-5
  restart: *default-restart
  working_dir: /data
  volumes:
  - /home/data/cluster/redis/compose/node-5/data:/data
  - /home/data/cluster/redis/compose/node-5/conf:/etc/redis
  command: redis-server /etc/redis/redis.conf
  environment:
  - 'REDIS_NODES=redis-cluster-1 redis-cluster-2 redis-cluster-3 redis-cluster-4 redis-cluster-5 redis-cluster-6'
  ports:
      - 7005:6379
      - 17005:16379
  networks:
      redis-cluster-net:
          ipv4_address: 10.0.0.15

 redis-cluster-6:
  image: *default-image
  container_name: redis-cluster-6
  restart: *default-restart
  working_dir: /data
  volumes:
    - /home/data/cluster/redis/compose/node-6/data:/data
    - /home/data/cluster/redis/compose/node-6/conf:/etc/redis
  command: redis-server /etc/redis/redis.conf
  environment:
    - 'REDIS_NODES=redis-cluster-1 redis-cluster-2 redis-cluster-3 redis-cluster-4 redis-cluster-5 redis-cluster-6'
  ports:
      - 7006:6379
      - 17006:16379

  networks:
      redis-cluster-net:
          ipv4_address: 10.0.0.16


networks:
    redis-cluster-net:
        ipam:
            driver: default
            config:
                - subnet: "10.0.0.0/24"

执行 docker-compose 命令

docker-compose --project-name mycomposeprj up -d

遇到 docker-compose-not-accepting-extensions-starting-with-x-invalid-top-level, 升级 docker-compose

在 host 机器查看目录权限

[hostuser@host-machine]$ ls -l node-1/
total 8
drwxr-xr-x. 2 redis-docker-root redis-docker-root 4096 May 14 13:22 conf
drwxr-xr-x. 2 redis-docker-user redis-docker-root 4096 May 14 13:24 data
[hostuser@host-machine]$ ls -l node-1/conf/
total 92
-rw-r--r--. 1 redis-docker-root redis-docker-root 93940 May 14 13:22 redis.conf
[hostuser@host-machine]$ ls -l node-1/data/
total 8
-rw-r--r--. 1 redis-docker-user redis-docker-user   0 May 14 13:22 appendonly.aof
-rw-r--r--. 1 redis-docker-user redis-docker-user 175 May 14 13:24 dump.rdb
-rw-r--r--. 1 redis-docker-user redis-docker-user 760 May 14 14:02 nodes.conf

随便进入一个结点容器
docker exec -it redis-cluster-5 bash

在进入的容器中创建集群
redis-cli --cluster create 10.0.0.11:6379 10.0.0.12:6379 10.0.0.13:6379 10.0.0.14:6379 10.0.0.15:6379 10.0.0.16:6379 --cluster-replicas 1

[hostuser@host-machine]$ docker exec -it redis-cluster-5 bash
root@4eac5c1df626:/data# redis-cli --cluster create 10.0.0.11:6379 10.0.0.12:6379 10.0.0.13:6379 10.0.0.14:6379 10.0.0.15:6379 10.0.0.16:6379 --cluster-replicas 1
>>> Performing hash slots allocation on 6 nodes...
Master[0] -> Slots 0 - 5460
Master[1] -> Slots 5461 - 10922
Master[2] -> Slots 10923 - 16383
Adding replica 10.0.0.15:6379 to 10.0.0.11:6379
Adding replica 10.0.0.16:6379 to 10.0.0.12:6379
Adding replica 10.0.0.14:6379 to 10.0.0.13:6379
M: 1cfa486c1f49ccdaa5e3faffbf6e42d196a325d7 10.0.0.11:6379
   slots:[0-5460] (5461 slots) master
M: 23dae408127ddd4a0308f2bad76b6efa94e0aac3 10.0.0.12:6379
   slots:[5461-10922] (5462 slots) master
M: a2a514c416be4d2ad8e9ef5b8e16a11d40a8214a 10.0.0.13:6379
   slots:[10923-16383] (5461 slots) master
S: 8b3a54b02e29d8e7633272c9edd0761c0c93b7bd 10.0.0.14:6379
   replicates a2a514c416be4d2ad8e9ef5b8e16a11d40a8214a
S: b65d2c3ab466a0e81165024e64170ec756205a17 10.0.0.15:6379
   replicates 1cfa486c1f49ccdaa5e3faffbf6e42d196a325d7
S: 0342e4d71bd6a00ce808af5110c4c28129468abd 10.0.0.16:6379
   replicates 23dae408127ddd4a0308f2bad76b6efa94e0aac3
Can I set the above configuration? (type 'yes' to accept): yes
>>> Nodes configuration updated
>>> Assign a different config epoch to each node
>>> Sending CLUSTER MEET messages to join the cluster
Waiting for the cluster to join
....
>>> Performing Cluster Check (using node 10.0.0.11:6379)
M: 1cfa486c1f49ccdaa5e3faffbf6e42d196a325d7 10.0.0.11:6379
   slots:[0-5460] (5461 slots) master
   1 additional replica(s)
S: b65d2c3ab466a0e81165024e64170ec756205a17 10.0.0.15:6379
   slots: (0 slots) slave
   replicates 1cfa486c1f49ccdaa5e3faffbf6e42d196a325d7
M: a2a514c416be4d2ad8e9ef5b8e16a11d40a8214a 10.0.0.13:6379
   slots:[10923-16383] (5461 slots) master
   1 additional replica(s)
S: 8b3a54b02e29d8e7633272c9edd0761c0c93b7bd 10.0.0.14:6379
   slots: (0 slots) slave
   replicates a2a514c416be4d2ad8e9ef5b8e16a11d40a8214a
M: 23dae408127ddd4a0308f2bad76b6efa94e0aac3 10.0.0.12:6379
   slots:[5461-10922] (5462 slots) master
   1 additional replica(s)
S: 0342e4d71bd6a00ce808af5110c4c28129468abd 10.0.0.16:6379
   slots: (0 slots) slave
   replicates 23dae408127ddd4a0308f2bad76b6efa94e0aac3
[OK] All nodes agree about slots configuration.
>>> Check for open slots...
>>> Check slots coverage...
[OK] All 16384 slots covered.

11 主 15从,12主 16从,13主 14从

redis-cli -c 进入集群,并查看集群结点 cluster nodes

root@4eac5c1df626:/data# redis-cli -c
127.0.0.1:6379> cluster nodes
23dae408127ddd4a0308f2bad76b6efa94e0aac3 10.0.0.12:6379@16379 master - 0 1652505902918 2 connected 5461-10922
b65d2c3ab466a0e81165024e64170ec756205a17 10.0.0.15:6379@16379 myself,slave 1cfa486c1f49ccdaa5e3faffbf6e42d196a325d7 0 1652505901000 1 connected
1cfa486c1f49ccdaa5e3faffbf6e42d196a325d7 10.0.0.11:6379@16379 master - 0 1652505901915 1 connected 0-5460
a2a514c416be4d2ad8e9ef5b8e16a11d40a8214a 10.0.0.13:6379@16379 master - 0 1652505903922 3 connected 10923-16383
8b3a54b02e29d8e7633272c9edd0761c0c93b7bd 10.0.0.14:6379@16379 slave a2a514c416be4d2ad8e9ef5b8e16a11d40a8214a 0 1652505903000 3 connected
0342e4d71bd6a00ce808af5110c4c28129468abd 10.0.0.16:6379@16379 slave 23dae408127ddd4a0308f2bad76b6efa94e0aac3 0 1652505904926 2 connected

使用 Java 查看使用集群

import redis.clients.jedis.HostAndPort;
import redis.clients.jedis.JedisCluster;

import java.util.HashSet;
import java.util.Set;

public class Main {
    public static void main (String[] args) {
        Set<HostAndPort> jedisClusterNodes = new HashSet<HostAndPort>();
        jedisClusterNodes.add(new HostAndPort("localhost", 7003));
        JedisCluster jedis = new JedisCluster(jedisClusterNodes);
        System.out.println(jedis.getClusterNodes().keySet());
        jedis.set("planets", "Mars");
        System.out.println(jedis.get("planets"));
    }
}

在字符界面客户端 redis-cli 查看

127.0.0.1:6379> get planets
-> Redirected to slot [8813] located at 10.0.0.12:6379
"Mars"
10.0.0.12:6379> set planets Earth
OK
10.0.0.12:6379> get planets
"Earth"

关闭主结点 10.0.0.12,docker stop redis-cluster-2
可以看到从结点 10.0.0.16 变成主结点

root@4eac5c1df626:/data# redis-cli -c
127.0.0.1:6379> get planets
-> Redirected to slot [8813] located at 10.0.0.16:6379
"Earth"
10.0.0.16:6379> cluster nodes
23dae408127ddd4a0308f2bad76b6efa94e0aac3 10.0.0.12:6379@16379 master,fail - 1652508145885 1652508142000 2 connected
a2a514c416be4d2ad8e9ef5b8e16a11d40a8214a 10.0.0.13:6379@16379 master - 0 1652508217000 3 connected 10923-16383
0342e4d71bd6a00ce808af5110c4c28129468abd 10.0.0.16:6379@16379 myself,master - 0 1652508218000 7 connected 5461-10922
8b3a54b02e29d8e7633272c9edd0761c0c93b7bd 10.0.0.14:6379@16379 slave a2a514c416be4d2ad8e9ef5b8e16a11d40a8214a 0 1652508217200 3 connected
b65d2c3ab466a0e81165024e64170ec756205a17 10.0.0.15:6379@16379 slave 1cfa486c1f49ccdaa5e3faffbf6e42d196a325d7 0 1652508216197 1 connected
1cfa486c1f49ccdaa5e3faffbf6e42d196a325d7 10.0.0.11:6379@16379 master - 0 1652508218204 1 connected 0-5460

Ref

  1. redis docker-compose bitnami sample
  2. Redis Cluster基于Docker的集群搭建 jianshu
  3. redis集群搭建
  4. docker-compose.yml sample github
  5. docker-compose-does-not-accept-my-volumes-declaration stackoverflow
  6. volumes-type-is-a-required-property-when-running-docker-compose superuser
  7. Officle Doc compose-file
  8. github issue Volume must be a mapping, not an array #5092
  9. volume must be a mapping not a string
  10. how-do-i-mount-a-host-directory-as-a-volume-in-docker-compose stackoverflow
  11. run-redis-with-docker-compose
  12. Docker环境下秒建Redis集群 tencent
  13. 如何用docker部署redis cluster cnblogs
  14. Official Doc Use bridge networks
  15. https://newbedev.com/get-docker-compose-yml-file-location-from-running-container
  16. Official Doc Compose specification
  17. Official Doc Use volumes
  18. Official sample redis 6.2 configuration file
  19. building-redis-cluster-with-docker-compose
 类似资料: