掌阅科技 钱文品 海量数据和⾼并发下的Redis优化实践

CodeWarrior

2019/07/08 发布于 编程 分类

GIAC2019 

文字内容
1. Redis
3. 5
4. Key Value # def get_user(user_id): user = redis.get(user_id) # if not user: user = db.get(user_id) redis.setex(user_id, ttl, user) # return user def save_user(user): redis.setex(user.id, ttl, user) db.save_async(user) # # codis redis-cluster
5. Key Value # config set maxmemory 20gb # config set maxmemory_policy allkeys-lru 1. no-eviction 2. volatile-xxx 3. allkeys-xxx key 4. xxx-random CEO 5. xxx-lru LRU 6. xxx-lfu LFU —
6. # user_state = json.parse(redis.get(user_id)) user_state.exp += delta_for(user_state, input_event) redis.set(user_id, jsonify(user_state)) # set "lock:$user_id" owner_id nx ex=5 5s owenr_id # # del_if_equals lock:$user_id owner_id if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("del", KEYS[1]) else return 0 end # redlock
7. # zset delay(5, task) # zadd(queue-key, now_ts+5, task_json) # task_json = zrevrangebyscore(queue-key, now_ts, 0, 0, 1) if task_json: grabbed_ok = zrem(queue-key, task_json) if grabbed_ok: process_task(task_json) # Lua
8. master-workers python celery master
9. multi-master(worker)
10. # Redis hset tasks name trigger_rule # hgetall tasks # set task_lock_${name} true nx ex=5 # watch set tasks_version $new_version get tasks_version
11. # UGC #1 5 zset hist_key="ugc:$user_id" zadd(hist_key, ts, uuid) # zremrangebyscore(hist_key, 0, now_ts - 3600) # count=zcard(hist_key) # expire(hist_key, 3600) # #
12. #
13. # zset zadd $service_key heartbeat_ts addr # zrange $service_key 0 -1 # vs ( ) zrem $service_key addr zremrangebyscore $service_key 0 now_ts - 10 # 10s # incr version:$service_key # incr gversion #
15. # 0 1 2 hset sign:$user_id 2019-01-01 1 hset sign:$user_id 2019-01-05 1 hset sign:$user_id 2019-01-10 2 ... #
16. # 00 01 10 11 # 28*2~31*2 bits 7 8 # 30G => 10G # # # dangerous
17. # # set # Redis HyperLogLog pfadd sign_uv_${day} user_id pfcount sign_uv_${day} # 12k # # 0.81% remove contains
18. # # user_id def get_user_state0(user_id): state = cache.get(user_id) if not state: state = db.get(user_id) or {} cache.set(user_id, state) return state def save_user_state0(user_id, state): cache.set(user_id, state) db.set_async(user_id, state) # db
19. # 99% # db set def get_user_state(user_id): exists = bloom_redis.exists(bloom_key, user_id) if not exists: return {} return get_user_state0(user_id) def save_user_state(user_id, state): save_user_state0(user_id, state) bloom_redis.add(bloom_key, user_id) # Redis # Redis Module # https://github.com/seomoz/pyreBloom https://github.com/RedisBloom/RedisBloom
20. Redis
21. msup