AWS Redis apparently not allowing KEY * command operation to fetch all keys

Question

I got the below exception while reading the all the keys from my AWS Redis cache. The write operation to cache was successful and I validated it with logs. To the best of my knowledge AWS does not allow KEY * operation because it can load a huge amount of data and thus can impact the performance.

redis.clients.jedis.exceptions.JedisDataException: ERR unknown command 'keys', with args beginning with: *

This code works on my local redis server but breaks on the AWS Redis.

package myintiative.work.config;

import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.redis.connection.RedisConnectionFactory;
import org.springframework.data.redis.connection.jedis.JedisClientConfiguration;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
import org.springframework.data.redis.serializer.StringRedisSerializer;


@Configuration
@Slf4j
public class RedisConfig {
    @Value("${redis.host}")
    private String host;

    @Value("${redis.port}")
    private int port;

    @Bean
    public JedisConnectionFactory jedisConnectionFactory() {
        RedisStandaloneConfiguration redisStandaloneConfiguration = new RedisStandaloneConfiguration(host, port);
        JedisClientConfiguration.JedisClientConfigurationBuilder jedisClientConfiguration = JedisClientConfiguration.builder();
        jedisClientConfiguration.usePooling();
        return new JedisConnectionFactory(redisStandaloneConfiguration, jedisClientConfiguration.build());
    }

    @Bean
    public RedisTemplate<String, String> redisTemplate(RedisConnectionFactory redisConnectionFactory) {
        RedisTemplate<String, String> redisTemplate = new RedisTemplate<>();
        redisTemplate.setConnectionFactory(redisConnectionFactory);

        redisTemplate.setKeySerializer(new StringRedisSerializer());
        redisTemplate.setValueSerializer(new StringRedisSerializer());
        redisTemplate.setHashKeySerializer(new StringRedisSerializer());
        redisTemplate.setHashValueSerializer(new StringRedisSerializer());

        redisTemplate.afterPropertiesSet();
        return redisTemplate;
    }
}

package myintiative.work.service;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.migration.mailconnectorsendgrid.dto.CustomerContactDTO;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.stereotype.Service;

import java.util.Set;
import java.util.List;
import java.util.stream.Collectors;

@Service
public class AWSRedisCacheService {

    @Autowired
    RedisTemplate<String, String> redisTemplate;
    ObjectMapper objectMapper;

    AWSRedisCacheService() {
        objectMapper = new ObjectMapper();
    }


    public void writeCache(List<CustomerContactDTO> list) {
        System.out.println("Total Number of Records="+list.size());
        System.out.println("WRITING THE CACHE!!!!!!!!!!");
        String serializedData;
        for (CustomerContactDTO c : list) {
            try {
                serializedData = objectMapper.writeValueAsString(c);
            } catch (JsonProcessingException e) {
                e.printStackTrace();
                return;
            }
            System.out.println("KEY++++++++++++" + c.getUserEmail());
            redisTemplate.opsForValue().set(c.getUserEmail(), serializedData);
            System.out.println("ADDED++++++++++++" + redisTemplate.opsForValue().get(c.getUserEmail()));
        }
        System.out.println("COMPLETED WRITING THE CACHE!!!!!!!!!!");
    }

    public CustomerContactDTO readCache(String key) {
        String retrievedData = redisTemplate.opsForValue().get(key);
        if (retrievedData == null) {
            System.out.println("No data found for " + key);
            return null;
        }
        CustomerContactDTO deserializedData = null;
        try {
            deserializedData = objectMapper.readValue(retrievedData, new TypeReference<>() {
            });
        } catch (JsonProcessingException e) {
            e.printStackTrace();
        }
        return deserializedData;

    }

    public void removeCacheEntry(String key) {
        redisTemplate.delete(key);
    }

    public void updateCache(CustomerContactDTO c) {
        String serializedData;
        try {
            serializedData = objectMapper.writeValueAsString(c);
        } catch (JsonProcessingException e) {
            e.printStackTrace();
            return;
        }
        redisTemplate.opsForValue().set(c.getUserEmail(), serializedData);
    }

    public List<String> readFirst50Entries() {
        System.out.println("READING THE CACHE!!!!!!!!!!");
        Set<String> keys = redisTemplate.keys("*");
        System.out.println("Total Entries:" + keys.size());
        List<String> keysList = keys.stream().limit(50).collect(Collectors.toList());
        System.out.println("EXITING READING THE CACHE!!!!!!!!!!");
        return keysList;
    }
}

I tried an alternative SCAN 0 MATCH "*" COUNT 1000 but unfortunately Spring redisTemplate does not support it directly and the implementation was getting just too complicated for me.

Answer 1

I solved it just now. The idea was same to go ahead and use the SCAN 0 MATCH "*" COUNT 1000 cmd itself. I created a dedicated custom scanner class

package myintiative.work.config;;

import org.springframework.data.redis.core.Cursor;
import org.springframework.data.redis.core.RedisCallback;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.data.redis.core.ScanOptions;
import org.springframework.stereotype.Component;
import java.util.HashSet;
import java.util.Set;

@Component
public class RedisKeyScanner {

    private final RedisTemplate<String, String> redisTemplate;

    public RedisKeyScanner(RedisTemplate<String, String> redisTemplate) {
        this.redisTemplate = redisTemplate;
    }

    public Set<String> scanKeys() {
        return redisTemplate.execute((RedisCallback<Set<String>>) connection -> {
            Set<String> keys = new HashSet<>();
            byte[] cursor;
            var scanResult = connection.scan(ScanOptions.scanOptions().match("*").count(1000).build());
            Cursor<byte[]> cursorObject = scanResult;
            do {
                cursor = cursorObject.next();
                keys.add(new String(cursor));
            } while (cursorObject.hasNext() && !new String(cursor).equals("0"));
            return keys;
        });
    }
}

This scanKeys() method was invoked in my service class in the following manner:

Set<String> keys = new RedisKeyScanner(redisTemplate).scanKeys();

This avoids all the risks of the KEY * command.