SpringBoot redis 读写分离，优化lettuce客户端底层bug（-READONLY You can‘t write against a read only replica.）
扈高逸
Redis
安装单点
-
下载redis包
-
[root@master opt] wget https://download.redis.io/releases/redis-6.2.6.tar.gz tar -zxvf redis-6.2.6.tar.gz
-
-
编译安装
-
[root@master redis-6.2.6]pwd //进入redis目录 /opt/redis-6.2.6 [root@master redis-6.2.6]yum -y install gcc gcc-c++ //安装c语言编译器 [root@master redis-6.2.6]yum -y install make //安装make编译器 [root@master redis-6.2.6]make MALLOC=libc
-
-
配置环境变量
-
// redis的二进制文件放在src目录下 [root@master redis-6.2.6]cat /etc/profile.d/redis.sh //为了能直接使用redis命令 export PATH=/opt/redis-6.2.6/src:$PATH [root@master redis-6.2.6]source /etc/profile.d/redis.sh //使其生效
-
-
启动redis
-
[root@master redis-6.2.6]vim redis.conf daemonize yes //把no改为yes 启动： [root@master redis-6.2.6]redis-server /opt/redis-6.2.6/redis.conf
-
-
进入客户端的命令
-
redis-cli -p 6379 127.0.0.1:6379>
-
-
关闭redis服务
-
[root@master redis-6.2.6]redis-cli -p 6379 shutdown
-
-
Yml配置
-
spring: # redis 配置 redis: # 地址 host: 123.56.245.41 # 端口，默认为6379 port: 6379 # 数据库索引 database: 0 # 密码 password: # 连接超时时间 timeout: 10s lettuce: pool: # 连接池中的最小空闲连接 min-idle: 0 # 连接池中的最大空闲连接 max-idle: 8 # 连接池的最大数据库连接数 max-active: 8 # #连接池最大阻塞等待时间（使用负值表示没有限制） max-wait: -1ms
-
安装主从集群
-
删除redis.conf空行和注释行代码
-
sed -i '/^$/ d' redis.conf sed -i '/^#/ d' redis.conf
-
-
模拟集群，复制三个文件夹
-
mv redis-6.2.6 redis-6379 cp -r redis-6379 redis-6380 cp -r redis-6379 redis-6381
-
-
修改端口等配置
-
redis.conf中的port项
-
修改redis.conf的dir为绝对路径
-
修改redis.conf的pidfile项目
-
-
启动、停止
-
6379 master 6380 slave1 6381 slave2 -
/opt/redis-6379/src/redis-server /opt/redis-6379/redis.conf /opt/redis-6380/src/redis-server /opt/redis-6380/redis.conf /opt/redis-6381/src/redis-server /opt/redis-6381/redis.conf -
./src/redis-cli -p 6379 shutdown ./src/redis-cli -p 6380 shutdown ./src/redis-cli -p 6381 shutdown
-
-
配置主从
-
master
-
[root@master redis-6.2.6]mkdir logs bind 0.0.0.0 protected-mode no logfile "/opt/redis-6379/logs/redis.log"
-
-
slave1、slave2
-
[root@slave redis-6.2.6]vim redis.conf bind 0.0.0.0 protected-mode no replicaof 172.28.208.107 6379 //master的IP和端口号 pidfile "/opt/redis-6381/redis_6381.pid"
-
-
-
测试主从
-
master
-
./src/redis-cli -h 172.28.208.107 -p 6379 set a 1
-
-
slave
-
./src/redis-cli -h 172.28.208.107 -p 6380 get a
-
-
输入info replication命令、查看节点状态
-
停止redis主从
-
./src/redis-cli -h 172.28.208.107 -p 6379 shutdown ./src/redis-cli -h 172.28.208.107 -p 6380 shutdown ./src/redis-cli -h 172.28.208.107 -p 6381 shutdown
-
-
搭建哨兵
-
三台主机做同样的操作
-
sed -i '/^$/ d' sentinel.conf sed -i '/^#/ d' sentinel.conf -
sentinel monitor mymaster 172.28.208.107 6379 2
- 配置监听的主服务器，mymaster代表服务器的名称，⾃定义，172.28.208.107 代表监控的主服务器，6379代表端⼝，2代表只有两个或两个以上的哨兵认为主服务器不可⽤的时候，才会进⾏failover操作。
-
[root@master] vim sentinel.conf port ... #你的端口 bind 0.0.0.0 daemonize yes logfile "/opt/redis-6379/logs/sentinel.log" #改成当前目录地址，从节点没有logs目录先创建 pidfile "/opt/redis-6379/logs/redis-sentinel.pid" dir "/opt/redis-6379/tmp" sentinel monitor mymaster 123.56.245.41 6379 2 sentinel down-after-milliseconds mymaster 5000 sentinel parallel-syncs mymaster 1 sentinel failover-timeout mymaster 180000 sentinel announce-ip "123.56.245.41" sentinel announce-port #你的端口
-
-
启动哨兵
-
/opt/redis-6379/src/redis-sentinel /opt/redis-6379/sentinel.conf /opt/redis-6380/src/redis-sentinel /opt/redis-6380/sentinel.conf /opt/redis-6381/src/redis-sentinel /opt/redis-6381/sentinel.conf
-
-
查看哨兵
-
./src/redis-cli -h 172.28.208.107 -p 26379 ./src/redis-cli -h 172.28.208.107 -p 26380 ./src/redis-cli -h 172.28.208.107 -p 26381 -
登录到哨兵查看状态 info sentinel
-
172.28.208.107:26379> info sentinel # Sentinel sentinel_masters:1 sentinel_tilt:0 sentinel_running_scripts:0 sentinel_scripts_queue_length:0 sentinel_simulate_failure_flags:0 master0:name=mymaster,status=ok,address=172.28.208.107:6379,slaves=2,sentinels=3
-
-
-
停止哨兵
-
./src/redis-cli -h 172.28.208.107 -p 26379 shutdown ./src/redis-cli -h 172.28.208.107 -p 26380 shutdown ./src/redis-cli -h 172.28.208.107 -p 26381 shutdown
-
-
yml配置
-
spring: redis: password: sentinel: master: mymaster nodes: - 123.56.245.41:26379 - 123.56.245.41:26380 - 123.56.245.41:26381
-
-
模拟down机 ./src/redis-cli -h 172.28.208.107 -p 6379 shutdown
-
2531646:X 06 Sep 2022 15:50:24.654 # +sdown master mymaster 123.56.245.41 6379 2531646:X 06 Sep 2022 15:50:24.714 # +odown master mymaster 123.56.245.41 6379 #quorum 2/2 2531646:X 06 Sep 2022 15:50:24.714 # +new-epoch 1 2531646:X 06 Sep 2022 15:50:24.714 # +try-failover master mymaster 123.56.245.41 6379 2531646:X 06 Sep 2022 15:50:24.718 # +vote-for-leader 04764d28eaf8699c1a8233f4aa3e31802003f6b4 1 2531646:X 06 Sep 2022 15:50:24.734 # 9ec358f74618a36e1c27fb77e93f277ceaa192ed voted for 04764d28eaf8699c1a8233f4aa3e31802003f6b4 1 2531646:X 06 Sep 2022 15:50:24.736 # 367bf704da483eab74b71abf0c482eeccef43c08 voted for 04764d28eaf8699c1a8233f4aa3e31802003f6b4 1 2531646:X 06 Sep 2022 15:50:24.802 # +elected-leader master mymaster 123.56.245.41 6379 2531646:X 06 Sep 2022 15:50:24.802 # +failover-state-select-slave master mymaster 123.56.245.41 6379 2531646:X 06 Sep 2022 15:50:24.887 # +selected-slave slave 123.56.245.41:6381 123.56.245.41 6381 @ mymaster 123.56.245.41 6379 2531646:X 06 Sep 2022 15:50:24.887 * +failover-state-send-slaveof-noone slave 123.56.245.41:6381 123.56.245.41 6381 @ mymaster 123.56.245.41 6379 2531646:X 06 Sep 2022 15:50:24.949 * +failover-state-wait-promotion slave 123.56.245.41:6381 123.56.245.41 6381 @ mymaster 123.56.245.41 6379 2531646:X 06 Sep 2022 15:50:25.740 # +promoted-slave slave 123.56.245.41:6381 123.56.245.41 6381 @ mymaster 123.56.245.41 6379 2531646:X 06 Sep 2022 15:50:25.740 # +failover-state-reconf-slaves master mymaster 123.56.245.41 6379 2531646:X 06 Sep 2022 15:50:25.834 * +slave-reconf-sent slave 123.56.245.41:6380 123.56.245.41 6380 @ mymaster 123.56.245.41 6379 2531646:X 06 Sep 2022 15:50:26.783 * +slave-reconf-inprog slave 123.56.245.41:6380 123.56.245.41 6380 @ mymaster 123.56.245.41 6379 2531646:X 06 Sep 2022 15:50:26.783 * +slave-reconf-done slave 123.56.245.41:6380 123.56.245.41 6380 @ mymaster 123.56.245.41 6379 2531646:X 06 Sep 2022 15:50:26.880 # +failover-end master mymaster 123.56.245.41 6379 2531646:X 06 Sep 2022 15:50:26.880 # +switch-master mymaster 123.56.245.41 6379 123.56.245.41 6381 2531646:X 06 Sep 2022 15:50:26.880 * +slave slave 123.56.245.41:6380 123.56.245.41 6380 @ mymaster 123.56.245.41 6381 2531646:X 06 Sep 2022 15:50:26.880 * +slave slave 123.56.245.41:6379 123.56.245.41 6379 @ mymaster 123.56.245.41 6381 2531646:X 06 Sep 2022 15:50:31.897 # +sdown slave 123.56.245.41:6379 123.56.245.41 6379 @ mymaster 123.56.245.41 6381
-
SpringBoot配置哨兵模式读写分离
yml开启debug日志
logging:
pattern:
console: '%date{yyyy-MM-dd HH:mm:ss.SSS} | %highlight(%5level) [%green(%16.16thread)] %clr(%-50.50logger{49}){cyan} %4line -| %highlight(%msg%n)'
level:
root: info
io.lettuce.core: debug
org.springframework.data.redis: debug
观察日志，读写都在主节点
读写分离配置
@Bean
public RedisConnectionFactory lettuceConnectionFactory(RedisProperties redisProperties) {
RedisSentinelConfiguration redisSentinelConfiguration = new RedisSentinelConfiguration(
redisProperties.getSentinel().getMaster(), new HashSet<>(redisProperties.getSentinel().getNodes())
);
LettucePoolingClientConfiguration lettuceClientConfiguration = LettucePoolingClientConfiguration.builder()
// 读写分离，读任何节点、写主节点
.readFrom(ReadFrom.ANY)
.build();
return new LettuceConnectionFactory(redisSentinelConfiguration, lettuceClientConfiguration);
}
ReadFrom类详情配置：https://lettuce.io/core/6.1.4.RELEASE/api/index.html
-
Modifier and Type Field and Description static ReadFromANYSetting to read from any node.static ReadFromANY_REPLICASetting to read from any replica node.static ReadFromMASTERSetting to read from the upstream only.static ReadFromMASTER_PREFERREDSetting to read preferred from the upstream and fall back to a replica if the master is not available.static ReadFromNEARESTSetting to read from the nearest node.static ReadFromREPLICASetting to read from the replica only.static ReadFromREPLICA_PREFERREDSetting to read preferred from replica and fall back to upstream if no replica is not available.static ReadFromSLAVEDeprecated. renamed toREPLICA.static ReadFromSLAVE_PREFERREDDeprecated. Renamed toREPLICA_PREFERRED.static ReadFromUPSTREAMSetting to read from the upstream only.static ReadFromUPSTREAM_PREFERREDSetting to read preferred from the upstream and fall back to a replica if the upstream is not available.
观察日志，写在主节点、读分布在随机节点上
-
查看读写分离的读取节点
-
io.lettuce.core.masterslave.MasterSlaveConnectionProvider
-
public CompletableFuture<StatefulRedisConnection<K, V>> getConnectionAsync(MasterSlaveConnectionProvider.Intent intent) { if (this.debugEnabled) { logger.debug("getConnectionAsync(" + intent + ")"); } if (this.readFrom != null && intent == MasterSlaveConnectionProvider.Intent.READ) { List<RedisNodeDescription> selection = this.readFrom.select(new Nodes() { public List<RedisNodeDescription> getNodes() { return MasterSlaveConnectionProvider.this.knownNodes; } public Iterator<RedisNodeDescription> iterator() { return MasterSlaveConnectionProvider.this.knownNodes.iterator(); } }); if (selection.isEmpty()) { throw new RedisException(String.format("Cannot determine a node to read (Known nodes: %s) with setting %s", this.knownNodes, this.readFrom)); } else { try { Flux<StatefulRedisConnection<K, V>> connections = Flux.empty(); RedisNodeDescription node; for(Iterator var4 = selection.iterator(); var4.hasNext(); connections = connections.concatWith(Mono.fromFuture(this.getConnection(node)))) { node = (RedisNodeDescription)var4.next(); } return !OrderingReadFromAccessor.isOrderSensitive(this.readFrom) && selection.size() != 1 ? connections.filter(StatefulConnection::isOpen).collectList().map((it) -> { int index = ThreadLocalRandom.current().nextInt(it.size()); return (StatefulRedisConnection)it.get(index); }).switchIfEmpty(connections.next()).toFuture() : connections.filter(StatefulConnection::isOpen).next().switchIfEmpty(connections.next()).toFuture(); } catch (RuntimeException var6) { throw Exceptions.bubble(var6); } } } else { return this.getConnection(this.getMaster()); } } -
查看selection节点即可
-
-
springboot整合哨兵读写分离时发现lettuce客户端底层bug
public boolean getLockByLua(String key, String value,int expireTim){
String script = "if redis.call('setNx',KEYS[1],ARGV[1])==1 then return redis.call('expire',KEYS[1],ARGV[2]) else return 0 end ";
RedisScript<Long> redisScript = new DefaultRedisScript<>(script, Long.class);
Object result = redisTemplate.execute(redisScript, Collections.singletonList(key),value,expireTim);
return SUCCESS.equals(result);
}
redisTemplate.execute方法在执行lua脚本时，使用的是redis的evalsha方法，默认算读操作（io.lettuce.core.masterslave.ReadOnlyCommands.isReadOnlyCommand方法中有体现），所以被分配到了slave节点，而lua脚本一般读写并存操作，比如以上代码释放redis锁，而redis的slave节点不能进行write操作，所以报以下错误
2022-09-09 10:44:44.196 | ERROR [-nio-8081-exec-2] o.a.c.c.C.[.[localhost].[/].[dispatcherServlet] 175 -| Servlet.service() for servlet [dispatcherServlet] in context with path [] threw exception [Request processing failed; nested exception is org.springframework.data.redis.RedisSystemException: Error in execution; nested exception is io.lettuce.core.RedisCommandExecutionException: ERR Error running script (call to f_e9f69f2beb755be68b5e456ee2ce9aadfbc4ebf4): @user_script:1: @user_script: 1: -READONLY You can't write against a read only replica.] with root cause
io.lettuce.core.RedisCommandExecutionException: ERR Error running script (call to f_e9f69f2beb755be68b5e456ee2ce9aadfbc4ebf4): @user_script:1: @user_script: 1: -READONLY You can't write against a read only replica.
at io.lettuce.core.ExceptionFactory.createExecutionException(ExceptionFactory.java:135)
at io.lettuce.core.ExceptionFactory.createExecutionException(ExceptionFactory.java:108)
at io.lettuce.core.protocol.AsyncCommand.completeResult(AsyncCommand.java:120)
at io.lettuce.core.protocol.AsyncCommand.complete(AsyncCommand.java:111)
at io.lettuce.core.protocol.CommandHandler.complete(CommandHandler.java:654)
at io.lettuce.core.protocol.CommandHandler.decode(CommandHandler.java:614)
at io.lettuce.core.protocol.CommandHandler.channelRead(CommandHandler.java:565)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:714)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:748)
lettuce源码分析
//
// Source code recreated from a .class file by IntelliJ IDEA
// (powered by FernFlower decompiler)
//
package io.lettuce.core.masterslave;
import io.lettuce.core.protocol.CommandType;
import io.lettuce.core.protocol.ProtocolKeyword;
import java.util.Collections;
import java.util.EnumSet;
import java.util.Set;
class ReadOnlyCommands {
private static final Set<CommandType> READ_ONLY_COMMANDS = EnumSet.noneOf(CommandType.class);
ReadOnlyCommands() {
}
// 如果此操作在READ_ONLY_COMMANDS（在下面static静态块中初始化）中，算读操作
public static boolean isReadOnlyCommand(ProtocolKeyword protocolKeyword) {
return READ_ONLY_COMMANDS.contains(protocolKeyword);
}
public static Set<CommandType> getReadOnlyCommands() {
return Collections.unmodifiableSet(READ_ONLY_COMMANDS);
}
// READ_ONLY_COMMANDS集合通过静态块读取CommandName枚举类初始化的
static {
ReadOnlyCommands.CommandName[] var0 = ReadOnlyCommands.CommandName.values();
int var1 = var0.length;
for(int var2 = 0; var2 < var1; ++var2) {
ReadOnlyCommands.CommandName commandNames = var0[var2];
READ_ONLY_COMMANDS.add(CommandType.valueOf(commandNames.name()));
}
}
static enum CommandName {
ASKING,
BITCOUNT,
BITPOS,
CLIENT,
COMMAND,
DUMP,
ECHO,
// eval 和 evalsha是redis执行脚本的方法，
EVAL,
EVALSHA,
EXISTS,
GEODIST,
GEOPOS,
GEORADIUS,
GEORADIUS_RO,
GEORADIUSBYMEMBER,
GEORADIUSBYMEMBER_RO,
GEOHASH,
GET,
GETBIT,
GETRANGE,
HEXISTS,
HGET,
HGETALL,
HKEYS,
HLEN,
HMGET,
HSCAN,
HSTRLEN,
HVALS,
INFO,
KEYS,
LINDEX,
LLEN,
LRANGE,
MGET,
PFCOUNT,
PTTL,
RANDOMKEY,
READWRITE,
SCAN,
SCARD,
SCRIPT,
SDIFF,
SINTER,
SISMEMBER,
SMEMBERS,
SRANDMEMBER,
SSCAN,
STRLEN,
SUNION,
TIME,
TTL,
TYPE,
XINFO,
XLEN,
XPENDING,
XRANGE,
XREVRANGE,
XREAD,
ZCARD,
ZCOUNT,
ZLEXCOUNT,
ZRANGE,
ZRANGEBYLEX,
ZRANGEBYSCORE,
ZRANK,
ZREVRANGE,
ZREVRANGEBYLEX,
ZREVRANGEBYSCORE,
ZREVRANK,
ZSCAN,
ZSCORE;
private CommandName() {
}
}
}
如何解决
重写lettuce底层源码ReadOnlyCommands中CommandName枚举类，将 EVAL, EVALSHA从此类删除
- 找到你所要重写类，查看其中的路径；io.lettuce.core.masterslave.ReadOnlyCommands
- 在我们的 src 目录下新建一个同包名同类名的类；
- 将 jar 包中的重写方法所在类的所有代码复制到我们新建的同包名同类名的类中；
- 在CommandName枚举中删除EVAL, EVALSHA属性
- 程序会优先使用我们 src 下面的类，这样就覆盖了 jar 包的方法 。
类似资料：