RedisDB双机主从同步性能测试

安装redisDB

主节点

apt install redis-server

修改配置 /etc/redis/redis.conf

bind 0.0.0.0save ""   # 禁止RDB持久化
#save 900 1
#save 300 10
#save 60 10000appendonly no  # 禁止AOF持久化

重启服务

systemctl restart redis-server

从节点配置文件

bind 0.0.0.0save ""   # 禁止RDB持久化
#save 900 1
#save 300 10
#save 60 10000appendonly no  # 禁止AOF持久化# 从节点关键配置
slaveof 172.40.20.132 6379

主从验证

主从状态

主节点执行 redis-cli info replication

root@r750-132:~# redis-cli  info replication
# Replication
role:master
connected_slaves:1
slave0:ip=172.40.20.131,port=6379,state=online,offset=84,lag=0
master_replid:1e214902bfe781c96a43a65ff20eefa9a9e6a75d
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:84
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1
repl_backlog_histlen:84

从节点检查

redis-cli info replication
# Replication
role:slave
master_host:172.40.20.132
master_port:6379
master_link_status:up
master_last_io_seconds_ago:6
master_sync_in_progress:0
slave_repl_offset:168
slave_priority:100
slave_read_only:1
connected_slaves:0
master_replid:1e214902bfe781c96a43a65ff20eefa9a9e6a75d
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:168
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1
repl_backlog_histlen:168

数据同步

主节点插入数据

root@r750-132:~# redis-cli set name1 ckun
OK
root@r750-132:~# redis-cli get name1
"ckun"

从节点读取数据

root@r750-131:~# redis-cli get name1
"ckun"

测试程序

安装hiredis API

git clone https://github.com/redis/hiredis.git
cd hiredis/
make
make install
ldconfig

测试代码

#include <stdio.h>
#include <stdint.h>
#include <pthread.h>
#include <signal.h>
#include <unistd.h>
#include <stdlib.h>
#include <string.h>
#include <arpa/inet.h>
#include <sys/time.h>
#include <hiredis/hiredis.h>typedef struct {uint32_t orig_saddr;uint32_t orig_daddr;uint8_t  proto;uint16_t orig_sport;uint16_t orig_dport;uint32_t new_saddr;uint32_t new_daddr;uint16_t new_sport;uint16_t new_dport;
} session_t;#define CHANNEL "session_channel"
#define SESS_NUM 100000uint16_t random_port() {return rand() % 65535 + 1;
}char* session_to_string(session_t *session) {char *str = (char *)malloc(256);snprintf(str, 256,"{""\"orig_saddr\":%u,""\"orig_daddr\":%u,""\"proto\":%u,""\"orig_sport\":%u,""\"orig_dport\":%u,""\"new_saddr\":%u,""\"new_daddr\":%u,""\"new_sport\":%u,""\"new_dport\":%u""}",session->orig_saddr,session->orig_daddr,session->proto,session->orig_sport,session->orig_dport,session->new_saddr,session->new_daddr,session->new_sport,session->new_dport);return str;
}static int count = 0;
static struct timeval start_time, end_time;
// 订阅消息的回调函数
void subscribe_callback(redisContext *c, redisReply *reply) {if (reply == NULL) {printf("Received NULL or empty message\n");return;}switch (reply->type) {case REDIS_REPLY_ERROR:case REDIS_REPLY_STRING:case REDIS_REPLY_NIL:case REDIS_REPLY_INTEGER:case REDIS_REPLY_STATUS:break;case REDIS_REPLY_ARRAY:{int i = 0;// for (i = 0; i < reply->elements; ++i)// {//     redisReply* childReply = reply->element[i];//     if (childReply->type == REDIS_REPLY_STRING)//     {//         printf("value:%s\n", childReply->str);//     }// }// 增加计数i = reply->elements - 1;redisReply* childReply = reply->element[i];if ( childReply->str != NULL) {//printf("value: %s\n", childReply->str);count++;}}break;default:break;}if (count == 1) {gettimeofday(&start_time, NULL); // 记录开始一时间}// 如果接收了 100,000 条数据，计算时间if (count == SESS_NUM) {gettimeofday(&end_time, NULL);double elapsed_time = (end_time.tv_sec - start_time.tv_sec) +(end_time.tv_usec - start_time.tv_usec) / 1000000.0;printf("Received %d sessions in %.3f seconds\n", SESS_NUM, elapsed_time);exit(0);}
}// 订阅 Redis 频道并接收消息
void subscribe_to_redis(redisContext *c) {redisReply *reply;redisAppendCommand(c, "SUBSCRIBE %s", CHANNEL);while (1) {if (redisGetReply(c, (void **)&reply) == REDIS_OK) {// 调用回调函数处理数据subscribe_callback(c, reply);freeReplyObject(reply);}}
}// 连接 Redis 并插入数据
void insert_sessions_to_redis(redisContext *c, int set_no_pub) {redisReply *reply = NULL;session_t session;char key[256];srand(time(NULL));gettimeofday(&start_time, NULL); // 记录开始一时间int printsize = 1;for (int i = 0; i < SESS_NUM; i++) {// 构造五元组 keysession.orig_saddr = 0xC0A8000A + i;  // orig_saddr 递增session.orig_daddr = 0x0A000001;session.proto = 6;  // 假设是 TCP 协议session.orig_sport = 12345;session.orig_dport = 8080;// 为 session 生成随机数据session.new_saddr = 0xAC100001;session.new_daddr = 0xAC10000A;session.new_sport = random_port();session.new_dport = random_port();// 生成 Redis key (五元组)snprintf(key, sizeof(key), "SESS:%u:%u:%u:%u:%u", session.orig_saddr, session.orig_daddr, session.proto, session.orig_sport, session.orig_dport);// 将 session 转为字符串存入 Redischar *session_str = session_to_string(&session);if (printsize) {int valuesize = strlen(session_str);int keysize = strlen(key);printf("KeySize: %d, ValueSize: %d\n", keysize, valuesize);printsize = 0;}reply = redisCommand(c, "SET %s %s", key, session_str);free(session_str);if (reply != NULL) {freeReplyObject(reply);}// 发布消息到订阅频道if (set_no_pub == 0) {redisCommand(c, "PUBLISH %s %s", CHANNEL, key);  // 发布消息}}gettimeofday(&end_time, NULL);double elapsed_time = (end_time.tv_sec - start_time.tv_sec) +(end_time.tv_usec - start_time.tv_usec) / 1000000.0;printf("SET %d sessions in %.3f seconds\n", SESS_NUM, elapsed_time);
}int sub_session() {redisContext *c = redisConnect("127.0.0.1", 6379);if (c == NULL || c->err) {if (c) {printf("Redis connection error: %s\n", c->errstr);} else {printf("Redis connection error: can't allocate redis context\n");}return 1;}// 订阅 Redis 频道subscribe_to_redis(c);redisFree(c);return 0;
}int pub_session(int set_no_pub) {redisContext *c = redisConnect("127.0.0.1", 6379);if (c == NULL || c->err) {if (c) {printf("Redis connection error: %s\n", c->errstr);} else {printf("Redis connection error: can't allocate redis context\n");}return 1;}printf("Begin to insert data\n");// 插入数据insert_sessions_to_redis(c, set_no_pub);redisFree(c);return 0;
}int main(int argc, char* argv[])
{if (argc < 2) {printf("Usage: ./redistest pub / sub\n");return 0;}if(strlen(argv[1]) >= 3) {if(strcmp(argv[1], "pub") == 0) {return pub_session(0);} else if (strcmp(argv[1], "set_no_pub") == 0) {return pub_session(1);}   else if (strcmp(argv[1], "sub") == 0) {return sub_session();}} return 0;
}

Makefile

APP_tool = redistestSRCS-t = $(wildcard *.c) 
PKGCONF ?= pkg-configCFLAGS += -g -O2 $(shell $(PKGCONF) --cflags hiredis) 
LDFLAGS += $(shell $(PKGCONF) --libs hiredis) -lpthread all: $(APP_tool)$(APP_tool): $(SRCS-t)$(CC) $(CFLAGS) $(SRCS-t) -o $@ $(LDFLAGS).PHONY: clean
clean:rm $(APP_tool)

RedisDB命令

# 清空数据库
redis-cli FLUSHDB  # 找到 SESS:前缀的 KEY， 通过这些KEY可以查到到VALUE
ckun@r750-131:~$ redis-cli KEYS "SESS:*"1) "SESS:3232235530:167772161:6:12345:8080"2) "SESS:3232235537:167772161:6:12345:8080"3) "SESS:3232235538:167772161:6:12345:8080"4) "SESS:3232235539:167772161:6:12345:8080"5) "SESS:3232235531:167772161:6:12345:8080"6) "SESS:3232235532:167772161:6:12345:8080"7) "SESS:3232235533:167772161:6:12345:8080"8) "SESS:3232235536:167772161:6:12345:8080"9) "SESS:3232235535:167772161:6:12345:8080"
10) "SESS:3232235534:167772161:6:12345:8080"# 通过KEY 查询VALUE
ckun@r750-131:~$ redis-cli get "SESS:3232235530:167772161:6:12345:8080"
"{\"orig_saddr\":3232235530,\"orig_daddr\":167772161,\"proto\":6,\"orig_sport\":12345,\"orig_dport\":8080,\"new_saddr\":2886729729,\"new_daddr\":2886729738,\"new_sport\":3543,\"new_dport\":54296}"

测试

主写入10万条记录，这里以NAT会话作为写入条目，KEY 38字节，VALUE 177字节。
从订阅10万条记录，看看同步一共花费多少时间。

主设备写入10万条花了9.49秒

ckun@r750-132:~/ws/redistest$ ./redistest pub 
Begin to insert data
KeySize: 38, ValueSize: 177
SET 100000 sessions in 9.493 seconds

从设备同步时间也是一致的 （从订阅到第一个记录开始计时）

ckun@r750-131:~$ ./redistest sub
Received 100000 sessions in 9.493 seconds

平均一秒写入并同步一万条。

那只写入，不PUB看一下性能

ckun@r750-132:~/ws/redistest$ ./redistest set_no_pub
Begin to insert data
KeySize: 38, ValueSize: 177
SET 100000 sessions in 4.484 seconds

平均有20000多条了，看起来加上PUB是要多消耗一倍的计算资源。

单机写入Benchmark

ckun@r750-132:~/ws/redistest$ redis-benchmark -h  127.0.0.1 -p 6379 -c 8 -n 100000  -t set -d 100 -P 8 -q 
SET: 395288.56 requests per secondckun@r750-132:~/ws/redistest$ redis-benchmark -h  127.0.0.1 -p 6379 -c 8 -n 100000  -t set -d 100 -P 1 -q
SET: 134408.59 requests per secondckun@r750-132:~/ws/redistest$ redis-benchmark -h  127.0.0.1 -p 6379 -c 1 -n 100000  -t set -d 100 -P 1 -q 
SET: 18419.60 requests per second

单连接，一个请求写入一个记录，性能18000 req/s 。

小结

在当前Intel® Xeon® Gold 6346 CPU @ 3.10GHz 环境下，
测试程序单连接写入并且同步只有 10000 req/s 左右的写入性能。
如果关闭PUB，单纯写入会有20000 req/s左右性能。
通过benchmark验证，单链接也有18000 req/s左右写入性能。

在大容量大并发的NAT环境中，20W每秒的新建会话速度，每个会话得有两个表， 上下行各一个，总计40W个条目，40个线程并发写入才能满足。这明显不能满足性能需求。