基于常用的压缩算法 bzip、deflate、lz4、snappy、minilzo、zstd提供压缩API接口,以实现对数据与文件的压缩。
基于lz4-1.9.3.tar.gz、libdeflate-1.18.tar.gz、bzip2-1.0.6.tar.gz、minilzo-2.10.tar.gz、zstd-1.4.10.tar.gz,封装接口,测试程序的编译/交叉编译脚本如下:
----X86系统编译 ./build.sh
----arm交叉编译 ./build.sh -host arm-linux-gnueabihf
#!/bin/bash
# 解析命令行参数
while [[ $# -gt 0 ]]; do
key="$1"
case $key in
-host)
host="$2"
shift # 将参数向前移动一个位置
;;
-target)
target="$2"
shift # 将参数向前移动一个位置
;;
*)
exit 1
;;
esac
shift # 将参数向前移动一个位置
done
# 判断是否传入了 host
if [ -z "$host" ]; then
wowcc=gcc
wowar=ar
wowranlib=ranlib
else
wowcc=$host-gcc
wowar=$host-ar
wowranlib=$host-ranlib
fi
project_path=$(pwd)
rm -rf $project_path/build/
cd $project_path/3rdparty/
rm -rf lz4-1.9.3
tar -xzvf lz4-1.9.3.tar.gz
cd lz4-1.9.3
make CC=$wowcc -j4 && make install PREFIX=$project_path/build
cd $project_path/3rdparty/
rm -rf libdeflate-1.18
tar xzvf libdeflate-1.18.tar.gz
cd libdeflate-1.18/
mkdir build && cd build
cmake ../ -DCMAKE_INSTALL_PREFIX=$project_path/build
make CC=$wowcc -j4 && make install;
cd $project_path/3rdparty/
rm -rf bzip2-1.0.6
tar xzvf bzip2-1.0.6.tar.gz
cd bzip2-1.0.6/
make CC=$wowcc AR=$wowar RANLIB=$wowranlib -j4 && make install PREFIX=$project_path/build
cd $project_path/3rdparty/
rm -rf snappy-c-master
unzip snappy-c-master.zip
cd snappy-c-master/
make CC=$wowcc
cp snappy.h $project_path/build/include
mv libsnappyc.so.1 $project_path/build/lib/libsnappyc.so
cd $project_path/3rdparty/
rm -rf minilzo-2.10
tar xzvf minilzo-2.10.tar.gz
cd minilzo-2.10
$wowcc -shared -fPIC -o libminilzo.so minilzo.c -lm -pthread
cp libminilzo.so $project_path/build/lib
cp minilzo.h lzoconf.h lzodefs.h $project_path/build/include
cd $project_path/3rdparty/
rm -rf zstd-1.4.10/
tar xzvf zstd-1.4.10.tar.gz
cd zstd-1.4.10/
make CC=$wowcc -j4 && make install PREFIX=$project_path/build
本项目未对其进行深入研究,未涉及如何提高使用库的压缩率或压缩速率,也未涉及多线程压缩等问题,此处只是实现基础应用来满足项目需求。 其bz2压缩算法的接口封装如下:
/*brief 使用bz2压缩buffer内容
*param : src_data:原数据缓存
*param : src_len:原数据缓存长度
*param : bz2_data:压缩数据缓存
*param : bz2_len:压缩缓存长度
*return: 成功返回 > 0 压缩长度 失败返回 <0
*/
int wow_bz2_compress_buffer(char* src_data,size_t src_len,char* bz2_data,size_t bz2_len);
/*brief 使用bz2解压缩buffer内容
*param : bz2_data:压缩数据缓存
*param : bz2_len:压缩缓存长度
*param : dest_data:解压缩数据缓存
*param : dest_len:解压缩缓存长度
*return: 成功返回 > 0 压缩长度 失败返回 <0
*/
int wow_bz2_decompress_buffer(char* bz2_data,size_t bz2_len,char* dest_data,size_t dest_len);
/*brief 使用bz2压缩文件
*param : src_file:原文件名称
*param : bz2_file:压缩文件名称
*return: 成功返回 0 失败返回 -1
*/
int wow_bz2_compress_file(const char* src_file,const char* bz2_file);
/*brief 使用bz2解压缩文件
*param : src_file:原文件名称
*param : bz2_file:压缩文件名称
*return: 成功返回 0 失败返回 -1
*/
int wow_bz2_decompress_file(const char* bz2_file,const char* dest_file);
其deflate压缩算法的接口封装如下:
/*brief 使用deflate压缩buffer内容
*param : src_data:原数据缓存
*param : src_len:原数据缓存长度
*param : deflate_data:压缩数据缓存
*param : deflate_len:压缩缓存长度
*return: 成功返回 > 0 压缩长度 失败返回 <0
*/
int wow_deflate_compress_buffer(char* src_data,size_t src_len,char* deflate_data,size_t deflate_len);
/*brief 使用deflate解压缩buffer内容
*param : deflate_data:压缩数据缓存
*param : deflate_len:压缩缓存长度
*param : dest_data:解压缩数据缓存
*param : dest_len:解压缩缓存长度
*return: 成功返回 > 0 压缩长度 失败返回 <0
*/
int wow_deflate_decompress_buffer(char* deflate_data,size_t deflate_len,char* dest_data,size_t dest_len);
/*brief 使用deflate-zlib压缩文件
*param : src_file:原文件名称
*param : deflate_file:压缩文件名称
*return: 成功返回 0 失败返回 -1
*/
int wow_zlib_compress_file(const char* src_file,const char* deflate_file);
/*brief 使用deflate-zlib解压缩文件
*param : src_file:原文件名称
*param : deflate_file:压缩文件名称
*return: 成功返回 0 失败返回 -1
*/
int wow_zlib_decompress_file(const char* deflate_file,const char* dest_file);
/*brief 使用deflate-gzip压缩文件
*param : src_file:原文件名称
*param : deflate_file:压缩文件名称
*return: 成功返回 0 失败返回 -1
*/
int wow_gzip_compress_file(const char* src_file,const char* deflate_file);
/*brief 使用deflate-gzip解压缩文件
*param : src_file:原文件名称
*param : deflate_file:压缩文件名称
*return: 成功返回 0 失败返回 -1
*/
int wow_gzip_decompress_file(const char* deflate_file,const char* dest_file);
其lz4压缩算法的接口封装如下:
/*brief 使用lz4压缩buffer内容
*param : src_data:原数据缓存
*param : src_len:原数据缓存长度
*param : lz4_data:压缩数据缓存
*param : lz4_len:压缩缓存长度
*return: 成功返回 > 0 压缩长度 失败返回 <0
*/
int wow_lz4_compress_buffer(char* src_data,size_t src_len,char* lz4_data,size_t lz4_len);
/*brief 使用lz4解压缩buffer内容
*param : lz4_data:压缩数据缓存
*param : lz4_len:压缩缓存长度
*param : dest_data:解压缩数据缓存
*param : dest_len:解压缩缓存长度
*return: 成功返回 > 0 压缩长度 失败返回 <0
*/
int wow_lz4_decompress_buffer(char* lz4_data,size_t lz4_len,char* dest_data,size_t dest_len);
/*brief 使用lz4压缩文件
*param : src_file:原文件名称
*param : lz4_file:压缩文件名称
*return: 成功返回 0 失败返回 -1
*/
int wow_lz4_compress_file(const char* src_file,const char* lz4_file);
/*brief 使用lz4解压缩文件
*param : src_file:原文件名称
*param : lz4_file:压缩文件名称
*return: 成功返回 0 失败返回 -1
*/
int wow_lz4_decompress_file(const char* lz4_file,const char* dest_file);
其minilzo压缩算法的接口封装如下:
/*brief 使用lzo压缩buffer内容
*param : src_data:原数据缓存
*param : src_len:原数据缓存长度
*param : lzo_data:压缩数据缓存
*param : lzo_len:压缩缓存长度
*return: 成功返回 > 0 压缩长度 失败返回 <0
*/
int wow_lzo_compress_buffer(char* src_data,size_t src_len,char* lzo_data,size_t lzo_len);
/*brief 使用lzo解压缩buffer内容
*param : lzo_data:压缩数据缓存
*param : lzo_len:压缩缓存长度
*param : dest_data:解压缩数据缓存
*param : dest_len:解压缩缓存长度
*return: 成功返回 > 0 压缩长度 失败返回 <0
*/
int wow_lzo_decompress_buffer(char* lzo_data,size_t lzo_len,char* dest_data,size_t dest_len);
其snappy压缩算法的接口封装如下:
/*brief 使用snappy压缩buffer内容
*param : src_data:原数据缓存
*param : src_len:原数据缓存长度
*param : snappy_data:压缩数据缓存
*param : snappy_len:压缩缓存长度
*return: 成功返回 > 0 压缩长度 失败返回 <0
*/
int wow_snappy_compress_buffer(char* src_data,size_t src_len,char* snappy_data,size_t snappy_len);
/*brief 使用snappy解压缩buffer内容
*param : snappy_data:压缩数据缓存
*param : snappy_len:压缩缓存长度
*param : dest_data:解压缩数据缓存
*param : dest_len:解压缩缓存长度
*return: 成功返回 > 0 压缩长度 失败返回 <0
*/
int wow_snappy_decompress_buffer(char* snappy_data,size_t snappy_len,char* dest_data,size_t dest_len);
/*brief 使用snappy压缩文件
*param : src_file:原文件名称
*param : snappy_file:压缩文件名称
*return: 成功返回 0 失败返回 -1
*/
int wow_snappy_compress_file(const char* src_file,const char* snappy_file);
/*brief 使用snappy解压缩文件
*param : src_file:原文件名称
*param : snappy_file:压缩文件名称
*return: 成功返回 0 失败返回 -1
*/
int wow_snappy_decompress_file(const char* snappy_file,const char* dest_file);
其zstd压缩算法的接口封装如下:
/*brief 使用zstd压缩buffer内容
*param : src_data:原数据缓存
*param : src_len:原数据缓存长度
*param : zstd_data:压缩数据缓存
*param : zstd_len:压缩缓存长度
*return: 成功返回 > 0 压缩长度 失败返回 <0
*/
int wow_zstd_compress_buffer(char* src_data,size_t src_len,char* zstd_data,size_t zstd_len);
/*brief 使用zstd解压缩buffer内容
*param : zstd_data:压缩数据缓存
*param : zstd_len:压缩缓存长度
*param : dest_data:解压缩数据缓存
*param : dest_len:解压缩缓存长度
*return: 成功返回 > 0 压缩长度 失败返回 <0
*/
int wow_zstd_decompress_buffer(char* zstd_data,size_t zstd_len,char* dest_data,size_t dest_len);
/*brief 使用zstd压缩文件
*param : src_file:原文件名称
*param : zstd_file:压缩文件名称
*return: 成功返回 0 失败返回 -1
*/
int wow_zstd_compress_file(const char* src_file,const char* zstd_file);
/*brief 使用zstd解压缩文件
*param : src_file:原文件名称
*param : zstd_file:压缩文件名称
*return: 成功返回 0 失败返回 -1
*/
int wow_zstd_decompress_file(const char* zstd_file,const char* dest_file);
测试代码如下:
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <time.h>
#include "utils/wow_check.h"
#include "utils/wow_utils.h"
#include "impl/wow_lz4_compress.h"
#include "impl/wow_deflate_compress.h"
#include "impl/wow_bz2_compress.h"
#include "impl/wow_snappy_compress.h"
#include "impl/wow_lzo_compress.h"
#include "impl/wow_zstd_compress.h"
int test_lz4_compress_buffer(void)
{
char* src = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Lorem ipsum dolor site amat.";
char* src_data = malloc(strlen(src));
int src_len = wow_lz4_compress_buffer(src,strlen(src),src_data,strlen(src));
CHECK_RET_VAL_P(src_len > 0,-1,"wow_lz4_compress_buffer failed!\n");
char* dest_data = malloc(strlen(src));
int dest_len = wow_lz4_decompress_buffer(src_data,src_len,dest_data,strlen(src));
CHECK_RET_VAL_P(dest_len > 0,-1,"wow_lz4_decompress_buffer failed!\n");
CHECK_RET_VAL_P(dest_len == strlen(src),-1,"decompress buffer len failed!\n");
CHECK_RET_VAL_P(!memcmp(src, dest_data, strlen(src)),-1,"decompress buffer data failed!\n");
return 0;
}
int test_lz4_compress_file(void)
{
const char* source_file = "input.txt";
const char* compressed_file = "compressed.lz4";
const char* decompressed_file = "lz4_decompressed.txt";
int ret = 0;
clock_t start_time = clock();
ret = wow_lz4_compress_file(source_file,compressed_file);
CHECK_RET_VAL_P(ret == 0,-1,"wow_lz4_compress_file failed!\n");
clock_t end_time = clock();
double elapsed_time = (double)(end_time - start_time) / CLOCKS_PER_SEC;
printf("LZ4 Compression completed in %.4f seconds\n", elapsed_time);
start_time = clock();
ret = wow_lz4_decompress_file(compressed_file,decompressed_file);
CHECK_RET_VAL_P(ret == 0,-1,"wow_lz4_decompress_file failed!\n");
end_time = clock();
elapsed_time = (double)(end_time - start_time) / CLOCKS_PER_SEC;
printf("LZ4 Decompression completed in %.4f seconds\n", elapsed_time);
return 0;
}
int test_deflate_compress_buffer(void)
{
char* src = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Lorem ipsum dolor site amat.";
char* src_data = malloc(strlen(src));
int src_len = wow_deflate_compress_buffer(src,strlen(src),src_data,strlen(src));
CHECK_RET_VAL_P(src_len > 0,-1,"wow_deflate_compress_buffer failed!\n");
char* dest_data = malloc(strlen(src));
int dest_len = wow_deflate_decompress_buffer(src_data,src_len,dest_data,strlen(src));
CHECK_RET_VAL_P(dest_len > 0,-1,"wow_deflate_decompress_buffer failed!\n");
CHECK_RET_VAL_P(dest_len == strlen(src),-1,"decompress buffer len failed!\n");
CHECK_RET_VAL_P(!memcmp(src, dest_data, strlen(src)),-1,"decompress buffer data failed!\n");
return 0;
}
int test_deflate_zlib_compress_file(void)
{
const char* source_file = "input.txt";
const char* compressed_file = "compressed.z";
const char* decompressed_file = "zlib_decompressed.txt";
int ret = 0;
clock_t start_time = clock();
ret = wow_zlib_compress_file(source_file,compressed_file);
CHECK_RET_VAL_P(ret == 0,-1,"wow_zlib_compress_file failed!\n");
clock_t end_time = clock();
double elapsed_time = (double)(end_time - start_time) / CLOCKS_PER_SEC;
printf("ZLIB Compression completed in %.4f seconds\n", elapsed_time);
start_time = clock();
ret = wow_zlib_decompress_file(compressed_file,decompressed_file);
CHECK_RET_VAL_P(ret == 0,-1,"wow_zlib_decompress_file failed!\n");
end_time = clock();
elapsed_time = (double)(end_time - start_time) / CLOCKS_PER_SEC;
printf("ZLIB Decompression completed in %.4f seconds\n", elapsed_time);
return 0;
}
int test_deflate_gzip_compress_file(void)
{
const char* source_file = "input.txt";
const char* compressed_file = "compressed.gz";
const char* decompressed_file = "gzip_decompressed.txt";
int ret = 0;
clock_t start_time = clock();
ret = wow_gzip_compress_file(source_file,compressed_file);
CHECK_RET_VAL_P(ret == 0,-1,"wow_gzip_compress_file failed!\n");
clock_t end_time = clock();
double elapsed_time = (double)(end_time - start_time) / CLOCKS_PER_SEC;
printf("GZIP Compression completed in %.4f seconds\n", elapsed_time);
start_time = clock();
ret = wow_gzip_decompress_file(compressed_file,decompressed_file);
CHECK_RET_VAL_P(ret == 0,-1,"wow_gzip_decompress_file failed!\n");
end_time = clock();
elapsed_time = (double)(end_time - start_time) / CLOCKS_PER_SEC;
printf("GZIP Decompression completed in %.4f seconds\n", elapsed_time);
return 0;
}
int test_bz2_compress_file(void)
{
const char* source_file = "input.txt";
const char* compressed_file = "compressed.bz2";
const char* decompressed_file = "bz2_decompressed.txt";
int ret = 0;
clock_t start_time = clock();
ret = wow_bz2_compress_file(source_file,compressed_file);
CHECK_RET_VAL_P(ret == 0,-1,"wow_gzip_compress_file failed!\n");
clock_t end_time = clock();
double elapsed_time = (double)(end_time - start_time) / CLOCKS_PER_SEC;
printf("BZ2 Compression completed in %.4f seconds\n", elapsed_time);
start_time = clock();
ret = wow_bz2_decompress_file(compressed_file,decompressed_file);
CHECK_RET_VAL_P(ret == 0,-1,"wow_gzip_decompress_file failed!\n");
end_time = clock();
elapsed_time = (double)(end_time - start_time) / CLOCKS_PER_SEC;
printf("BZ2 Decompression completed in %.4f seconds\n", elapsed_time);
return 0;
}
int test_snappy_compress_buffer(void)
{
char* src = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Lorem ipsum dolor site amat.";
char* src_data = malloc(strlen(src));
int src_len = wow_snappy_compress_buffer(src,strlen(src),src_data,strlen(src));
CHECK_RET_VAL_P(src_len > 0,-1,"wow_snappy_compress_buffer failed!\n");
char* dest_data = malloc(strlen(src));
int dest_len = wow_snappy_decompress_buffer(src_data,src_len,dest_data,strlen(src));
CHECK_RET_VAL_P(dest_len > 0,-1,"wow_snappy_decompress_buffer failed!\n");
CHECK_RET_VAL_P(dest_len == strlen(src),-1,"decompress buffer len failed!\n");
CHECK_RET_VAL_P(!memcmp(src, dest_data, strlen(src)),-1,"decompress buffer data failed!\n");
return 0;
}
int test_snappy_compress_file(void)
{
const char* source_file = "input.txt";
const char* compressed_file = "compressed.snp";
const char* decompressed_file = "snp_decompressed.txt";
int ret = 0;
clock_t start_time = clock();
ret = wow_snappy_compress_file(source_file,compressed_file);
CHECK_RET_VAL_P(ret == 0,-1,"wow_snappy_compress_file failed!\n");
clock_t end_time = clock();
double elapsed_time = (double)(end_time - start_time) / CLOCKS_PER_SEC;
printf("SNAPPY Compression completed in %.4f seconds\n", elapsed_time);
start_time = clock();
ret = wow_snappy_decompress_file(compressed_file,decompressed_file);
CHECK_RET_VAL_P(ret == 0,-1,"wow_snappy_decompress_file failed!\n");
end_time = clock();
elapsed_time = (double)(end_time - start_time) / CLOCKS_PER_SEC;
printf("SNAPPY Decompression completed in %.4f seconds\n", elapsed_time);
return 0;
}
int test_lzo_compress_buffer(void)
{
char* src = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Lorem ipsum dolor site amat.";
char* src_data = malloc(strlen(src));
int src_len = wow_lzo_compress_buffer(src,strlen(src),src_data,strlen(src));
CHECK_RET_VAL_P(src_len > 0,-1,"wow_lzo_compress_buffer failed!\n");
char* dest_data = malloc(strlen(src));
int dest_len = wow_lzo_decompress_buffer(src_data,src_len,dest_data,strlen(src));
CHECK_RET_VAL_P(dest_len > 0,-1,"wow_lzo_decompress_buffer failed!\n");
CHECK_RET_VAL_P(dest_len == strlen(src),-1,"decompress buffer len failed!\n");
CHECK_RET_VAL_P(!memcmp(src, dest_data, strlen(src)),-1,"decompress buffer data failed!\n");
return 0;
}
int test_zstd_compress_buffer(void)
{
char* src = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Lorem ipsum dolor site amat.";
char* src_data = malloc(strlen(src));
int src_len = wow_zstd_compress_buffer(src,strlen(src),src_data,strlen(src));
CHECK_RET_VAL_P(src_len > 0,-1,"wow_zstd_compress_buffer failed!\n");
char* dest_data = malloc(strlen(src));
int dest_len = wow_zstd_decompress_buffer(src_data,src_len,dest_data,strlen(src));
CHECK_RET_VAL_P(dest_len > 0,-1,"wow_zstd_decompress_buffer failed!\n");
CHECK_RET_VAL_P(dest_len == strlen(src),-1,"decompress buffer len failed!\n");
CHECK_RET_VAL_P(!memcmp(src, dest_data, strlen(src)),-1,"decompress buffer data failed!\n");
return 0;
}
int test_zstd_compress_file(void)
{
const char* source_file = "input.txt";
const char* compressed_file = "compressed.zsd";
const char* decompressed_file = "zsd_decompressed.txt";
int ret = 0;
clock_t start_time = clock();
ret = wow_zstd_compress_file(source_file,compressed_file);
CHECK_RET_VAL_P(ret == 0,-1,"wow_zstd_compress_file failed!\n");
clock_t end_time = clock();
double elapsed_time = (double)(end_time - start_time) / CLOCKS_PER_SEC;
printf("ZSTD Compression completed in %.4f seconds\n", elapsed_time);
start_time = clock();
ret = wow_zstd_decompress_file(compressed_file,decompressed_file);
CHECK_RET_VAL_P(ret == 0,-1,"wow_zstd_decompress_file failed!\n");
end_time = clock();
elapsed_time = (double)(end_time - start_time) / CLOCKS_PER_SEC;
printf("ZSTD Decompression completed in %.4f seconds\n", elapsed_time);
return 0;
}
int main()
{
test_lz4_compress_buffer();
test_lz4_compress_file();
test_deflate_compress_buffer();
test_deflate_zlib_compress_file();
test_deflate_gzip_compress_file();
test_bz2_compress_file();
test_snappy_compress_buffer();
test_snappy_compress_file();
test_lzo_compress_buffer();
test_zstd_compress_buffer();
test_zstd_compress_file();
return 0;
}
项目编译测试与结果展示: export LD_LIBRARY_PATH=$(pwd)/build/lib/ cd example ./wow_compress
We successfully compressed input.txt->compressed.lz4! Ratio: 0.41
LZ4 Compression completed in 0.0010 seconds
LZ4 Decompression completed in 0.0005 seconds
We successfully compressed input.txt->compressed.z! Ratio: 0.26
ZLIB Compression completed in 0.0027 seconds
ZLIB Decompression completed in 0.0009 seconds
We successfully compressed input.txt->compressed.gz! Ratio: 0.26
GZIP Compression completed in 0.0027 seconds
GZIP Decompression completed in 0.7958 seconds
We successfully compressed input.txt->compressed.bz2! Ratio: 0.22
BZ2 Compression completed in 0.0115 seconds
BZ2 Decompression completed in 0.0019 seconds
We successfully compressed input.txt->compressed.snp! Ratio: 0.42
SNAPPY Compression completed in 0.0003 seconds
SNAPPY Decompression completed in 0.0002 seconds
We successfully compressed input.txt->compressed.zsd! Ratio: 0.29
ZSTD Compression completed in 0.0004 seconds
ZSTD Decompression completed in 0.0003 seconds
总结: 目前嵌入式实时操作系统小型工程使用minilzo,移植简单,只是简单的数据操作使用方便。 目前嵌入式linux操作系统使用ZSTD,数据处理速度快,文件处理效率高效。 源码:wow-compress: 基于常用的压缩算法 bzip、deflate、lz4、snappy、minilzo、zstd实现对数据与文件的压缩,并提供基于压缩接口。https://gitee.com/wow-iot/wow-compress