不同的分块方式处理
基于内容的分块
CDC(content-defined chunking)算法是一种变长分块算法，它应用数据指纹(如Rabin指纹)将文件分割成长度大小不等的分块策略。与定长分块算法不同，它是基于文件内容进行数据块切分的，因此数据块大小是可变化的。算法执行过程中，CDC使用一个固定大小(如48字节)的滑动窗口对文件数据计算数据指纹。如果指纹满足某个条件，如当它的值模特定的整数等于预先设定的数时，则把窗口位置作为块的边界。CDC算法可能会出现病态现象，即指纹条件不能满足，块边界不能确定，导致数据块过大。实现中可以对数据块的大小进行限定，设定上下限，解决这种问题。CDC算法对文件内容变化不敏感，插入或删除数据只会影响到检少的数据块，其余数据块不受影响。CDC算法也是有缺陷的，数据块大小的确定比较困难，粒度太细则开销太大，粒度过粗则dedup效果不佳。如何两者之间权衡折衷，这是一个难点。Deduputil中CDC分块算法代码如下。
/*
* content-defined chunking:基于内容的分块
* 1. BLOCK_MIN_SIZE <= block_size <= BLOCK_MAX_SIZE
* 2. hash(block) % d == r
*/
static int file_chunk_cdc(int fd, int fd_ldata, int fd_bdata, unsigned int *pos, unsigned int *block_num,
block_id_t **metadata, hashtable *htable, char *last_block_buf, unsigned int *last_block_len)
{
char buf[BUF_MAX_SIZE] = {0};
char buf_bz[BUF_MAX_SIZE] = {0};
char block_buf[BLOCK_MAX_SIZE * 2] = {0};
char win_buf[BLOCK_WIN_SIZE + 1] = {0};
char md5_str[33] = {0};
char adler_pre_char;
unsigned char md5_checksum[32 + 1] = {0};
unsigned int bpos = 0;
unsigned int rwsize = 0, bzsize = 0;
unsigned int exp_rwsize = BUF_MAX_SIZE;
unsigned int head, tail;
unsigned int block_sz = 0, old_block_sz = 0;
unsigned int hkey = 0;
int ret = 0;
while(rwsize = read(fd, buf + bpos, exp_rwsize))
{
/* last chunk @@@@@@*/
if ((rwsize + bpos + block_sz) < BLOCK_MIN_SIZE)
break;
head = 0;
tail = bpos + rwsize;
/* avoid unnecessary computation and comparsion */
if (block_sz < (BLOCK_MIN_SIZE - BLOCK_WIN_SIZE))
old_block_sz = block_sz;
block_sz = ((block_sz + tail - head) > (BLOCK_MIN_SIZE - BLOCK_WIN_SIZE)) ?
BLOCK_MIN_SIZE - BLOCK_WIN_SIZE : block_sz + tail -head;
memcpy(block_buf + old_block_sz, buf + head, block_sz - old_block_sz);
head += (block_sz - old_block_sz);
}
while ((head + BLOCK_WIN_SIZE) <= tail)
{
memcpy(win_buf, buf + head, BLOCK_WIN_SIZE);
/*
* Firstly, i think rabinhash is the best. However, it's performance is very bad.
* After some testing, i found ELF_hash is better both on performance and dedup rate.
* So, EFL_hash is default. Now, adler_hash as default.
*/
if (g_rolling_hash)
{
hkey = (block_sz == (BLOCK_MIN_SIZE - BLOCK_WIN_SIZE)) ? adler32_checksum(win_buf, BLOCK_WIN_SIZE) :
adler32_rolling_checksum(hkey, BLOCK_WIN_SIZE, adler_pre_char, buf[head+BLOCK_WIN_SIZE-1]);
}
else
hkey = g_cdc_chunk_hashfunc(win_buf);
/* get a normal chunk */
if ((hkey % g_block_size) == CHUNK_CDC_R)
{
memcpy(block_buf + block_sz, buf + head, BLOCK_WIN_SIZE);
head += BLOCK_WIN_SIZE;
block_sz += BLOCK_WIN_SIZE;
if (block_sz >= BLOCK_MIN_SIZE)
{
/* compress block is -z flag is given */
if (g_bz)
{
bzsize = BUF_MAX_SIZE;
if (Z_OK != zlib_compress_block(block_buf, block_sz, buf_bz, &bzsize))
{
ret = -1;
goto _FILE_CHUNK_CDC_EXIT;
}
memcpy(block_buf, buf_bz, bzsize);
block_sz = bzsize;
}
md5(block_buf, block_sz, md5_checksum);
md5_2_str(md5_checksum);
if (0 != (ret = dedup_regfile_block_process(block_buf, block_sz,
md5_checksum, fd_ldata, fd_bdata, pos, block_num, metadata, htable)))
{
perror("dedup_reggile_block_process in file_chunk_cdc");
goto _FILE_CHUNK_CDC_EXIT;
}
block_sz = 0;
}
}
else
{
block_buf[block_sz++] = buf[head++];
/* get an abnormal chunk */
if (block_sz >= BLOCK_MAX_SIZE)
{
/* compress block if -z flag is given */
if (g_bz)
{
bzsize = BUF_MAX_SIZE;
if (Z_OK != zlib_compress_block(block_buf, block_sz, buf_bz, &bzsize))
{
ret = -1;
goto _FILE_CHUNK_CDC_EXIT;
}
memcpy(block_buf, buf_bz, bzsize);
block_sz = bzsize;
}
md5(block_buf, block_sz, md5_checksum);
md5_2_str(md5_checksum);
if (0 != (ret = dedup_regfile_block_process(block_buf, block_sz,
md5_checksum, fd_ldata, fd_bdata, pos, block_num, metadata, htable)))
{
perror("dedup_reggile_block_process in file_chunk_cdc");
goto _FILE_CHUNK_CDC_EXIT;
}
block_sz = 0;
}
}
/* avoid unnecessary computation and comparsion */
if (block_sz == 0)
{
block_sz = ((tail - head) > (BLOCK_MIN_SIZE - BLOCK_WIN_SIZE)) ?
BLOCK_MIN_SIZE - BLOCK_WIN_SIZE : tail - head;
memcpy(block_buf, buf + head, block_sz);
head = ((tail - head) > (BLOCK_MIN_SIZE - BLOCK_WIN_SIZE)) ?
head + (BLOCK_MIN_SIZE - BLOCK_WIN_SIZE) : tail;
}
adler_pre_char = buf[head -1];
}
/* read expected data from file to full up buf */
bpos = tail - head;
exp_rwsize = BUF_MAX_SIZE - bpos;
adler_pre_char = buf[head -1];
memmove(buf, buf + head, bpos);
}
/* last chunk */
*last_block_len = ((rwsize + bpos + block_sz) >= 0) ? rwsize + bpos + block_sz : 0;
if (*last_block_len > 0)
{
memcpy(last_block_buf, block_buf, block_sz);
memcpy(last_block_buf + block_sz, buf, rwsize + bpos);
}
_FILE_CHUNK_CDC_EXIT:
return ret;
}
滑动分块
/*
* slideing block chunking, performance is a big issue due to too many hash lookup.
* *滑动分块，性能是一个大问题，由于太多的哈希查找。
*/
static int file_chunk_sb(int fd, int fd_ldata, int fd_bdata, unsigned int *pos, unsigned int *block_num,
block_id_t **metadata, hashtable *htable, char *last_block_buf, unsigned int *last_block_len)
{
char buf[BUF_MAX_SIZE] = {0};
char buf_bz[BUF_MAX_SIZE] = {0};
char win_buf[BLOCK_MAX_SIZE * 2] = {0};
char block_buf[BLOCK_MAX_SIZE * 2] = {0};
char adler_pre_char;
unsigned char md5_checksum[32 + 1] = {0};
unsigned char md5_checksum1[32 + 1] = {0};
unsigned char crc_checksum[16] = {0};
unsigned int bpos = 0;
unsigned int slide_sz = 0;
unsigned int rwsize = 0, bzsize = 0, bzsize_f = 0;
unsigned int exp_rwsize = BUF_MAX_SIZE;
unsigned int head, tail;
unsigned int hkey = 0;
unsigned int bflag = 0;
int ret = 0;
while(rwsize = read(fd, buf + bpos, exp_rwsize))
{
/* last chunk */
if ((rwsize + bpos + slide_sz) < g_block_size)
break;
head = 0;
tail = bpos + rwsize;
while ((head + g_block_size) <= tail)
{
memcpy(win_buf, buf + head, g_block_size);
hkey = (slide_sz == 0) ? adler32_checksum(win_buf, g_block_size) :
adler32_rolling_checksum(hkey, g_block_size, adler_pre_char, buf[head+g_block_size-1]);
uint_2_str(hkey, crc_checksum);
/* bflag: 0, both CRC and MD5 are not idenitical
1, both CRC and MD5 are identical
2, CRC is identical and MD5 is not
*/
bflag = 0;
/* this block maybe is duplicate */
bzsize = g_block_size;
if (hash_exist(g_sb_htable_crc, crc_checksum))
{
bflag = 2;
/* compress block if -z flag is given */
if (g_bz)
{
bzsize = BUF_MAX_SIZE;
if (Z_OK != zlib_compress_block(win_buf, g_block_size, buf_bz, &bzsize))
{
ret = -1;
goto _FILE_CHUNK_SB_EXIT;
}
memcpy(win_buf, buf_bz, bzsize);
}
md5(win_buf, bzsize, md5_checksum);
md5_2_str(md5_checksum);
if (hash_exist(htable, md5_checksum))
{
/* insert fragment */
if (slide_sz != 0)
{
/* compress block if -z flag is given */
if (g_bz)
{
bzsize_f = BUF_MAX_SIZE;
if (Z_OK != zlib_compress_block(block_buf, slide_sz, buf_bz, &bzsize_f))
{
ret = -1;
goto _FILE_CHUNK_SB_EXIT;
}
memcpy(block_buf, buf_bz, bzsize_f);
slide_sz = bzsize_f;
}
md5(block_buf, slide_sz, md5_checksum1);
md5_2_str(md5_checksum1);
if (0 != (ret = dedup_regfile_block_process(block_buf, slide_sz, md5_checksum1,
fd_ldata, fd_bdata, pos, block_num, metadata, htable)))
{
perror("dedup_regfile_block_process in file_chunk_sb");
goto _FILE_CHUNK_SB_EXIT;
}
}
/* insert fixed-size block */
if (0 != (ret = dedup_regfile_block_process(win_buf, bzsize, md5_checksum,
fd_ldata, fd_bdata, pos, block_num, metadata, htable)))
{
perror("dedup_regfile_block_process in file_chunk_sb");
goto _FILE_CHUNK_SB_EXIT;
}
head += g_block_size;
slide_sz = 0;
bflag = 1;
}
}
/* this block is not duplicate */
if (bflag != 1)
{
block_buf[slide_sz++] = buf[head++];
if (slide_sz == g_block_size)
{
bzsize = g_block_size;
/* calculate checksum and check in */
hkey = adler32_checksum(block_buf, bzsize);
uint_2_str(hkey, crc_checksum);
hash_checkin(g_sb_htable_crc, crc_checksum);
/* compress block if -z flag is given */
if (g_bz)
{
bzsize = BUF_MAX_SIZE;
if (Z_OK != zlib_compress_block(block_buf, g_block_size, buf_bz, &bzsize))
{
ret = -1;
goto _FILE_CHUNK_SB_EXIT;
}
memcpy(block_buf, buf_bz, bzsize);
}
md5(block_buf, bzsize, md5_checksum);
md5_2_str(md5_checksum);
if (0 != (ret = dedup_regfile_block_process(block_buf, bzsize, md5_checksum,
fd_ldata, fd_bdata, pos, block_num, metadata, htable)))
{
perror("dedup_regfile_block_process in file_chunk_sb");
goto _FILE_CHUNK_SB_EXIT;
}
slide_sz = 0;
}
}
adler_pre_char = buf[head - 1];
}
/* read expected data from file to full up buf */
bpos = tail - head;
exp_rwsize = BUF_MAX_SIZE - bpos;
adler_pre_char = buf[head - 1];
memmove(buf, buf + head, bpos);
}
/* last chunk */
*last_block_len = ((rwsize + bpos + slide_sz) > 0) ? rwsize + bpos + slide_sz : 0;
if (*last_block_len > 0)
{
memcpy(last_block_buf, block_buf, slide_sz);
memcpy(last_block_buf + slide_sz, buf, rwsize + bpos);
}
_FILE_CHUNK_SB_EXIT:
lseek(fd, 0, SEEK_SET);
return ret;
}
固定分块
/*
* fixed-sized file chunking
*固定分块
*/
static int file_chunk_fsp(int fd, int fd_ldata, int fd_bdata, unsigned int *pos, unsigned int *block_num,
block_id_t **metadata, hashtable *htable, char *last_block_buf, unsigned int *last_block_len)
{
int ret = 0;
unsigned int rwsize, bzsize;
unsigned char md5_checksum[32 + 1] = {0};
char *buf = NULL, *buf_bz = NULL;//buf_bz 给压缩使用的
buf = (char *)malloc(g_block_size * 2);//4K*2
buf_bz = (char *)malloc(g_block_size * 2);//4K*2
if (buf == NULL || buf_bz == NULL)
{
perror("malloc in file_chunk_fsp");
return errno;
}
while (rwsize = read(fd, buf, g_block_size))//
{
/*if the last block 当最后一块不够一块时*/
if (rwsize != g_block_size)
break;
/* compress block if -z flag is given */
/*
if (g_bz) {
bzsize = g_block_size * 2;
if (Z_OK != zlib_compress_block(buf, rwsize, buf_bz, &bzsize)) {
ret = -1;
goto _FILE_CHUNK_FSP_EXIT;
}
memcpy(buf, buf_bz, bzsize);
rwsize = bzsize;
}
*/
/* calculate md5 */
md5(buf, rwsize, md5_checksum);
md5_2_str(md5_checksum);
if (0 != (ret = dedup_regfile_block_process(buf, rwsize, md5_checksum, fd_ldata,
fd_bdata, pos, block_num, metadata, htable)))
{
perror("dedup_regfile_block_process in file_chunk_fsp");
goto _FILE_CHUNK_FSP_EXIT;
}
}
*last_block_len = (rwsize > 0) ? rwsize : 0;
if ((*last_block_len)) memcpy(last_block_buf, buf, *last_block_len);
_FILE_CHUNK_FSP_EXIT:
if (buf) free(buf);
return ret;
}
扫一扫
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
