- 联系我们
- duidaima.com 版权声明
- 闽ICP备2020021581号
-
闽公网安备 35020302035485号
闽公网安备 35020302035485号
3.实际使用场景是多样化的，如何选择合适的淘汰策略？
# 客户端命令方式配置和查看内存大小 127.0.0.1:6379> config get maxmemory "maxmemory" "0" 127.0.0.1:6379> config set maxmemory 100mb OK 127.0.0.1:6379> config get maxmemory "maxmemory" "104857600" #通过redis.conf 配置文件配置 127.0.0.1:6379> info # Server #... # 配置文件路径 config_file:/opt/homebrew/etc/redis.conf #... # 堆代码 duidaima.com # 修改内存大小 > vim /opt/homebrew/etc/redis.conf ############################## MEMORY MANAGEMENT ################################ # Set a memory usage limit to the specified amount of bytes. # When the memory limit is reached Redis will try to remove keys # according to the eviction policy selected (see maxmemory-policy). # #... maxmemory 100mb #...注：若`maxmemory=0`则表示不做内存限制，但是对于windows系统来说，32位系统默认可使用空间是3G，因为整个系统内存是4G，需要留1G给系统运行。且淘汰策略会自动设置为noeviction，即不开启淘汰策略，当使用空间达到3G的时候，新的内存请求会报错。
# 命令行配置方式 127.0.0.1:6379> CONFIG GET maxmemory-policy "maxmemory-policy" "noeviction" 127.0.0.1:6379> CONFIG SET maxmemory-policy volatile-lru OK 127.0.0.1:6379> CONFIG GET maxmemory-policy "maxmemory-policy" "volatile-lru" #redis.conf文件配置方式 # MAXMEMORY POLICY: how Redis will select what to remove when maxmemory # is reached. You can select one from the following behaviors: # # volatile-lru -> Evict using approximated LRU, only keys with an expire set. # allkeys-lru -> Evict any key using approximated LRU. # volatile-lfu -> Evict using approximated LFU, only keys with an expire set. # allkeys-lfu -> Evict any key using approximated LFU. # volatile-random -> Remove a random key having an expire set. # allkeys-random -> Remove a random key, any key. # volatile-ttl -> Remove the key with the nearest expire time (minor TTL) # noeviction -> Don't evict anything, just return an error on write operations. # # LRU means Least Recently Used # LFU means Least Frequently Used # # Both LRU, LFU and volatile-ttl are implemented using approximated # randomized algorithms. # The default is: # ... maxmemory-policy noevictionfreeMemoryIfNeeded逻辑处理
int freeMemoryIfNeeded(void) {
size_t mem_used, mem_tofree, mem_freed;
int slaves = listLength(server.slaves);
/* Remove the size of slaves output buffers and AOF buffer from the count of used memory.*/
// 计算出 Redis 目前占用的内存总数，但有两个方面的内存不会计算在内：
// 1）从服务器的输出缓冲区的内存
// 2）AOF 缓冲区的内存
mem_used = zmalloc_used_memory();
if (slaves) {
listIter li;
listNode *ln;
listRewind(server.slaves,&li);
while((ln = listNext(&li))) {
redisClient *slave = listNodeValue(ln);
unsigned long obuf_bytes = getClientOutputBufferMemoryUsage(slave);
if (obuf_bytes > mem_used)
mem_used = 0;
else
mem_used -= obuf_bytes;
}
}
if (server.aof_state != REDIS_AOF_OFF) {
mem_used -= sdslen(server.aof_buf);
mem_used -= aofRewriteBufferSize();
}
/* Check if we are over the memory limit. */
// 如果目前使用的内存大小比设置的 maxmemory 要小，那么无须执行进一步操作
if (mem_used <= server.maxmemory) return REDIS_OK;
// 如果占用内存比 maxmemory 要大，但是 maxmemory 策略为不淘汰，那么直接返回
if (server.maxmemory_policy == REDIS_MAXMEMORY_NO_EVICTION)
return REDIS_ERR; /* We need to free memory, but policy forbids. */
/* Compute how much memory we need to free. */
// 计算需要释放多少字节的内存
mem_tofree = mem_used - server.maxmemory;
// 初始化已释放内存的字节数为 0
mem_freed = 0;
// 根据 maxmemory 策略，
// 遍历字典，释放内存并记录被释放内存的字节数
while (mem_freed < mem_tofree) {
int j, k, keys_freed = 0;
// 遍历所有字典
for (j = 0; j < server.dbnum; j++) {
long bestval = 0; /* just to prevent warning */
sds bestkey = NULL;
dictEntry *de;
redisDb *db = server.db+j;
dict *dict;
if (server.maxmemory_policy == REDIS_MAXMEMORY_ALLKEYS_LRU ||
server.maxmemory_policy == REDIS_MAXMEMORY_ALLKEYS_RANDOM)
{
// 如果策略是 allkeys-lru 或者 allkeys-random
// 那么淘汰的目标为所有数据库键
dict = server.db[j].dict;
} else {
// 如果策略是 volatile-lru 、 volatile-random 或者 volatile-ttl
// 那么淘汰的目标为带过期时间的数据库键
dict = server.db[j].expires;
}
// 跳过空字典
if (dictSize(dict) == 0) continue;
/* volatile-random and allkeys-random policy */
// 如果使用的是随机策略，那么从目标字典中随机选出键
if (server.maxmemory_policy == REDIS_MAXMEMORY_ALLKEYS_RANDOM ||
server.maxmemory_policy == REDIS_MAXMEMORY_VOLATILE_RANDOM)
{
de = dictGetRandomKey(dict);
bestkey = dictGetKey(de);
}
/* volatile-lru and allkeys-lru policy */
// 如果使用的是 LRU 策略，
// 那么从一集 sample 键中选出 IDLE 时间最长的那个键
else if (server.maxmemory_policy == REDIS_MAXMEMORY_ALLKEYS_LRU ||
server.maxmemory_policy == REDIS_MAXMEMORY_VOLATILE_LRU)
{
struct evictionPoolEntry *pool = db->eviction_pool;
while(bestkey == NULL) {
evictionPoolPopulate(dict, db->dict, db->eviction_pool);
/* Go backward from best to worst element to evict. */
for (k = REDIS_EVICTION_POOL_SIZE-1; k >= 0; k--) {
if (pool[k].key == NULL) continue;
de = dictFind(dict,pool[k].key);
/* Remove the entry from the pool. */
sdsfree(pool[k].key);
/* Shift all elements on its right to left. */
memmove(pool+k,pool+k+1,
sizeof(pool[0])*(REDIS_EVICTION_POOL_SIZE-k-1));
/* Clear the element on the right which is empty since we shifted one position to the left. */
pool[REDIS_EVICTION_POOL_SIZE-1].key = NULL;
pool[REDIS_EVICTION_POOL_SIZE-1].idle = 0;
/* If the key exists, is our pick. Otherwise it is a ghost and we need to try the next element. */
if (de) {
bestkey = dictGetKey(de);
break;
} else {
/* Ghost... */
continue;
}
}
}
}
/* volatile-ttl */
// 策略为 volatile-ttl ，从一集 sample 键中选出过期时间距离当前时间最接近的键
else if (server.maxmemory_policy == REDIS_MAXMEMORY_VOLATILE_TTL) {
for (k = 0; k < server.maxmemory_samples; k++) {
sds thiskey;
long thisval;
de = dictGetRandomKey(dict);
thiskey = dictGetKey(de);
thisval = (long) dictGetVal(de);
/* Expire sooner (minor expire unix timestamp) is better candidate for deletion */
if (bestkey == NULL || thisval < bestval) {
bestkey = thiskey;
bestval = thisval;
}
}
}
/* Finally remove the selected key. */
// 删除被选中的键
if (bestkey) {
long long delta;
robj *keyobj = createStringObject(bestkey,sdslen(bestkey));
propagateExpire(db,keyobj);
// 计算删除键所释放的内存数量
delta = (long long) zmalloc_used_memory();
dbDelete(db,keyobj);
delta -= (long long) zmalloc_used_memory();
mem_freed += delta;
// 对淘汰键的计数器增一
server.stat_evictedkeys++;
notifyKeyspaceEvent(REDIS_NOTIFY_EVICTED, "evicted",
keyobj, db->id);
decrRefCount(keyobj);
keys_freed++;
/* When the memory to free starts to be big enough, we may */
/* start spending so much time here that is impossible to */
/* deliver data to the slaves fast enough, so we force the */
/* transmission here inside the loop. */
if (slaves) flushSlavesOutputBuffers();
}
}
if (!keys_freed) return REDIS_ERR; /* nothing to free... */
}
return REDIS_OK;
}
/* Redis maxmemory strategies */ #define REDIS_MAXMEMORY_VOLATILE_LRU 0 #define REDIS_MAXMEMORY_VOLATILE_TTL 1 #define REDIS_MAXMEMORY_VOLATILE_RANDOM 2 #define REDIS_MAXMEMORY_ALLKEYS_LRU 3 #define REDIS_MAXMEMORY_ALLKEYS_RANDOM 4 #define REDIS_MAXMEMORY_NO_EVICTION 5 #define REDIS_DEFAULT_MAXMEMORY_POLICY REDIS_MAXMEMORY_NO_EVICTION
noeviction( REDIS_MAXMEMORY_NO_EVICTION): Don't evict anything, just return an error on write operations -> 不开启淘汰策略，在不配置淘汰策略的情况下，maxmemory-policy默认等于该值。内存不足时，会抛出异常，写操作不可用。不同系统存在差异性-具体见⇑
# The counter decay time is the time, in minutes, that must elapse in order # for the key counter to be divided by two (or decremented if it has a value # less <= 10). # # The default value for the lfu-decay-time is 1. A special value of 0 means to # decay the counter every time it happens to be scanned. # lfu-decay-time 1若整体访问频率较为平衡，则可选择allkeys-random策略随机淘汰数据。
3.内存空间是有限的，除了过期策略，Redis还有什么其他保障？
# expire: t秒后过期 expire key seconds # pexpire: t毫秒后过期 pexpire key millseconds # expireat: 到达具体的时间戳时过期,精确到秒 expireat key timestamp # pexpireat: 到达具体的时间戳时过期，精确到毫秒 pexpire key millseconds这四个命令看似有差异，但在RedisDb底层，最终都会转换成pexpireat指令。内部由db.c/expireGenericCommand函数实现，对外由上面四个指令调用
//expire命令
void expireCommand(redisClient *c) {
expireGenericCommand(c,mstime(),UNIT_SECONDS);
}
//expireat命令
void expireatCommand(redisClient *c) {
expireGenericCommand(c,0,UNIT_SECONDS);
}
//pexpire命令
void pexpireCommand(redisClient *c) {
expireGenericCommand(c,mstime(),UNIT_MILLISECONDS);
}
//pexpireat命令
void pexpireatCommand(redisClient *c) {
expireGenericCommand(c,0,UNIT_MILLISECONDS);
}
/* This is the generic command implementation for EXPIRE, PEXPIRE, EXPIREAT
* and PEXPIREAT. Because the commad second argument may be relative or absolute
* the "basetime" argument is used to signal what the base time is (either 0
* for *AT variants of the command, or the current time for relative expires).
*/
void expireGenericCommand(redisClient *c, long long basetime, int unit) {
...
/* unix time in milliseconds when the key will expire. */
long long when;
...
//如果是秒转换为毫秒
if (unit == UNIT_SECONDS) when *= 1000;
when += basetime;
...
}
过期字典内部存储结构：key表示一个指向具体键的指针，value是long类型的毫秒精度的UNIX时间戳。
3.惰性删除：数据不做及时释放，待下一次接收到读写请求时，先进行过期检查，若已过期则直接删除。用内存存储空间换取CPU性能——即用空间换取时间。
| 删除方式 | 优点 | 缺点 |
|---|---|---|
| 定时删除 | 能及时释放内存空间，不会产生滞留数据 | 频繁生成和销毁定时器，非常损耗CPU性能,影响响应时间和指令吞吐量 |
| 定期删除 | 固定的频率进行过期检查，对CPU交友好 | 1.数据量比较大的情况下，会因为全局扫描而损耗CPU性能，且主线程的阻塞会导致其他请求响应延迟。2.未能及时释放内存空间。3.数据已过期，但定时器未执行时会导致数据不一致。 |
| 惰性删除 | 节约CPU性能 | 当某些数据长时间无请求访问时，会导致数据滞留，使内存无法释放，占用内存空间，甚至坑导致内存泄漏而引发服务不可用 |
# The range is between 1 and 500, however a value over 100 is usually not # a good idea. Most users should use the default of 10 and raise this up to # 100 only in environments where very low latency is required. hz 10
| 删除方式 | 优点 | 缺点 |
|---|---|---|
| Redis定期删除 | 避免了全局扫描，每次随机抽取数据量较少，性能较稳定，执行频率可配置；避免了惰性删除低频数据长时间滞留的问题 | 存在概率上某些数据一直没被抽取的情况，导致数据滞留 |
| Redis惰性删除 | 解决了定期删除可能导致的数据滞留现象，性能较高 | 低频数据长时间无法释放 |
2.3 过期时间命令EX｜PX，在主从同步时，因为同步需要时间，就会导致主从库实际过期时间出现偏差。比如主库设置过期时间60s，但同步全量花费了1分钟，那么在从库接收到命令并执行之后，就导致从库key的过期时间整体跨越了两分钟，而此时主库在一分钟之前数据就已经过期了。EXAT｜PXAT 命令来设置过期时间节点。这样可避免增量同步的发生。但需注意主从服务器时间一致。
