压缩Redis里的字符串大对象

背景

• Redis缓存的字符串过大时会有问题。不超过10KB最好，最大不能超过1MB。
• 有几个配置缓存，上千个flink任务调用，每个任务5分钟命中一次，大小在5KB到6MB不等，因此需要压缩。

第一种，使用gzip

/**
 * 使用gzip压缩字符串
 */publicstatic Stringcompress(String str){if(str== null|| str.length()==0){return str;}
    ByteArrayOutputStream out=newByteArrayOutputStream();
    GZIPOutputStream gzip= null;try{
        gzip=newGZIPOutputStream(out);
        gzip.write(str.getBytes());}catch(IOException e){
        e.printStackTrace();}finally{if(gzip!= null){try{
                gzip.close();}catch(IOException e){
                e.printStackTrace();}}}returnnewsun.misc.BASE64Encoder().encode(out.toByteArray());}/**
 * 使用gzip解压缩
 */publicstatic Stringuncompress(String compressedStr){if(compressedStr== null|| compressedStr.length()==0){return compressedStr;}
 
    ByteArrayOutputStream out=newByteArrayOutputStream();
    ByteArrayInputStream in= null;
    GZIPInputStream ginzip= null;byte[] compressed= null;
    String decompressed= null;try{
        compressed=newsun.misc.BASE64Decoder().decodeBuffer(compressedStr);
        in=newByteArrayInputStream(compressed);
        ginzip=newGZIPInputStream(in);byte[] buffer=newbyte[1024];int offset=-1;while((offset= ginzip.read(buffer))!=-1){
            out.write(buffer,0, offset);}
        decompressed= out.toString();}catch(IOException e){
        e.printStackTrace();}finally{if(ginzip!= null){try{
                ginzip.close();}catch(IOException e){}}if(in!= null){try{
                in.close();}catch(IOException e){}}if(out!= null){try{
                out.close();}catch(IOException e){}}}return decompressed;}

第二种，使用Zstd

<!-- https://mvnrepository.com/artifact/com.github.luben/zstd-jni --><dependency><groupId>com.github.luben</groupId><artifactId>zstd-jni</artifactId><version>1.4.5-6</version></dependency>

publicclassConfigCacheUtil{privatestatic ZstdDictCompress compressDict;privatestatic ZstdDictDecompress decompressDict;privatestaticfinal Integer LEVEL=5;publicstaticvoidtrain()throws IOException{// 初始化词典对象
        String dictContent= FileUtils.readFileToString(newFile("/Users/yangguang/vscode/text/cache.json"),
            StandardCharsets.UTF_8);byte[] dictBytes= dictContent.getBytes(StandardCharsets.UTF_8);
        compressDict=newZstdDictCompress(dictBytes, LEVEL);
        decompressDict=newZstdDictDecompress(dictBytes);}publicstaticvoidmain(String[] args)throws IOException{
        String read= FileUtils.readFileToString(newFile("/Users/yangguang/vscode/text/cache.json"));

        ConfigCacheUtil.testGzip(read);
        System.out.println("");

        ConfigCacheUtil.test(read.getBytes());
        System.out.println("");

        ConfigCacheUtil.testByTrain(read.getBytes());}publicstaticvoidtestGzip(String str){
        logger.info("初始数据: {}", str.length());// 压缩数据long compressBeginTime= System.currentTimeMillis();
        String compressed= ConfigCacheUtil.compress(str);long compressEndTime= System.currentTimeMillis();
        logger.info("压缩耗时: {}", compressEndTime- compressBeginTime);
        logger.info("数据大小: {}", compressed.length());// 解压数据long decompressBeginTime= System.currentTimeMillis();// 第 3 个参数不能小于解压后的字节数组的大小
        String decompressed= ConfigCacheUtil.uncompress(compressed);long decompressEndTime= System.currentTimeMillis();
        logger.info("解压耗时: {}", decompressEndTime- decompressBeginTime);
        logger.info("数据大小: {}", decompressed.length());}publicstaticvoidtest(byte[] bytes){
        logger.info("初始数据: {}", bytes.length);// 压缩数据long compressBeginTime= System.currentTimeMillis();byte[] compressed= Zstd.compress(bytes);long compressEndTime= System.currentTimeMillis();
        logger.info("压缩耗时: {}", compressEndTime- compressBeginTime);
        logger.info("数据大小: {}", compressed.length);// 解压数据long decompressBeginTime= System.currentTimeMillis();// 第 3 个参数不能小于解压后的字节数组的大小byte[] decompressed= Zstd.decompress(compressed,20*1024*1024*8);long decompressEndTime= System.currentTimeMillis();
        logger.info("解压耗时: {}", decompressEndTime- decompressBeginTime);
        logger.info("数据大小: {}", decompressed.length);}publicstaticvoidtestByTrain(byte[] bytes)throws IOException{
        ConfigCacheUtil.train();
        logger.info("初始数据: {}", bytes.length);// 压缩数据long compressBeginTime= System.currentTimeMillis();byte[] compressed= Zstd.compress(bytes, compressDict);long compressEndTime= System.currentTimeMillis();
        logger.info("压缩耗时: {}", compressEndTime- compressBeginTime);
        logger.info("数据大小: {}", compressed.length);// 解压数据long decompressBeginTime= System.currentTimeMillis();// 第 3 个参数不能小于解压后的字节数组的大小byte[] decompressed= Zstd.decompress(compressed, decompressDict,20*1024*1024*8);long decompressEndTime= System.currentTimeMillis();
        logger.info("解压耗时: {}", decompressEndTime- decompressBeginTime);
        logger.info("数据大小: {}", decompressed.length);

        compressDict.toString();}}

输出

5KB

2020-09-08 22:42:48 INFO  ConfigCacheUtil:157 - 初始数据: 5541
2020-09-08 22:42:48 INFO  ConfigCacheUtil:163 - 压缩耗时: 2
2020-09-08 22:42:48 INFO  ConfigCacheUtil:164 - 数据大小: 1236
2020-09-08 22:42:48 INFO  ConfigCacheUtil:171 - 解压耗时: 2
2020-09-08 22:42:48 INFO  ConfigCacheUtil:172 - 数据大小: 5541

2020-09-08 22:42:48 INFO  ConfigCacheUtil:176 - 初始数据: 5541
2020-09-08 22:42:48 INFO  ConfigCacheUtil:182 - 压缩耗时: 523
2020-09-08 22:42:48 INFO  ConfigCacheUtil:183 - 数据大小: 972
2020-09-08 22:42:48 INFO  ConfigCacheUtil:190 - 解压耗时: 85
2020-09-08 22:42:48 INFO  ConfigCacheUtil:191 - 数据大小: 5541

2020-09-08 22:42:48 INFO  ConfigCacheUtil:196 - 初始数据: 5541
2020-09-08 22:42:48 INFO  ConfigCacheUtil:202 - 压缩耗时: 1
2020-09-08 22:42:48 INFO  ConfigCacheUtil:203 - 数据大小: 919
2020-09-08 22:42:48 INFO  ConfigCacheUtil:210 - 解压耗时: 22
2020-09-08 22:42:48 INFO  ConfigCacheUtil:211 - 数据大小: 5541

6MB

2020-09-08 22:44:06 INFO  ConfigCacheUtil:158 - 初始数据: 5719269
2020-09-08 22:44:06 INFO  ConfigCacheUtil:164 - 压缩耗时: 129
2020-09-08 22:44:06 INFO  ConfigCacheUtil:165 - 数据大小: 330090
2020-09-08 22:44:06 INFO  ConfigCacheUtil:172 - 解压耗时: 69
2020-09-08 22:44:06 INFO  ConfigCacheUtil:173 - 数据大小: 5719269

2020-09-08 22:44:06 INFO  ConfigCacheUtil:177 - 初始数据: 5874139
2020-09-08 22:44:06 INFO  ConfigCacheUtil:183 - 压缩耗时: 265
2020-09-08 22:44:06 INFO  ConfigCacheUtil:184 - 数据大小: 201722
2020-09-08 22:44:06 INFO  ConfigCacheUtil:191 - 解压耗时: 81
2020-09-08 22:44:06 INFO  ConfigCacheUtil:192 - 数据大小: 5874139

2020-09-08 22:44:06 INFO  ConfigCacheUtil:197 - 初始数据: 5874139
2020-09-08 22:44:06 INFO  ConfigCacheUtil:203 - 压缩耗时: 42
2020-09-08 22:44:06 INFO  ConfigCacheUtil:204 - 数据大小: 115423
2020-09-08 22:44:07 INFO  ConfigCacheUtil:211 - 解压耗时: 49
2020-09-08 22:44:07 INFO  ConfigCacheUtil:212 - 数据大小: 5874139

总结

略