生产环境中需要获取原端和目标端的已由文件的差异情况,因为在docker中运行,要求尽量避免落地文件,所以采用比较暴力的比对方式:
for i in `echo -e "${source}"`
do
j=`echo -e "${local}"|grep ${i}$`
if [[ "${j}" == "" ]]
then
str="${str} ${i}"
fi
done
对3000个文件的比较需要耗时72秒以上
grep的执行效率明显堪忧,所以考虑用awk命令去查找差集,来优化
echo |awk -v source_list="${source}" -v local_list="${local}"
'{split(sourcelist,arr_source," ")
split(locallist,arr_local," ")
for(i in arr_source){
arr_diff[arr_source[i]]=arr_source[i]
}
for(i in arr_local){
if(arr_diff[arr_local[i]]!=""){
delete arr_diff[arr_local[i]]
}
}
for(i in arr_diff){
print arr_diff[i]
}
}'
1、awk中首先定义两个变量,接受shell中的变量
2、使用split函数将字符串拆分成数组
3、复制原端的列表数组,存入一个以键来存取的数组arr_diff(arr_diff["20190303abcd"]="20190303abcd")
4、遍历本地列表数组,如果文件名在arr_diff数组中出现,则用delete函数删除这个元素
5、输出差集
附上GNU下载的grep命令源码,下载地址:http://mirrors.ustc.edu.cn/gnu/
static int
grep (int fd, char const *file, struct stats *stats)
{
int nlines, i;
int not_text;
size_t residue, save;
char oldc;
char *beg;
char *lim;
char eol = eolbyte;
if (!reset (fd, file, stats))
return 0;
if (file && directories == RECURSE_DIRECTORIES
&& S_ISDIR (stats->stat.st_mode))
{
/* Close fd now, so that we don't open a lot of file descriptors
when we recurse deeply. */
if (close (fd) != 0)
error (0, errno, "%s", file);
return grepdir (file, stats) - 2;
}
totalcc = 0;
lastout = 0;
totalnl = 0;
outleft = max_count;
after_last_match = 0;
pending = 0;
nlines = 0;
residue = 0;
save = 0;
if (! fillbuf (save, stats))
{
if (! is_EISDIR (errno, file))
suppressible_error (filename, errno);
return 0;
}
not_text = (((binary_files == BINARY_BINARY_FILES && !out_quiet)
|| binary_files == WITHOUT_MATCH_BINARY_FILES)
&& memchr (bufbeg, eol ? '\0' : '\200', buflim - bufbeg));
if (not_text && binary_files == WITHOUT_MATCH_BINARY_FILES)
return 0;
done_on_match += not_text;
out_quiet += not_text;
for (;;)
{
lastnl = bufbeg;
if (lastout)
lastout = bufbeg;
beg = bufbeg + save;
/* no more data to scan (eof) except for maybe a residue -> break */
if (beg == buflim)
break;
/* Determine new residue (the length of an incomplete line at the end of
the buffer, 0 means there is no incomplete last line). */
oldc = beg[-1];
beg[-1] = eol;
for (lim = buflim; lim[-1] != eol; lim--)
continue;
beg[-1] = oldc;
if (lim == beg)
lim = beg - residue;
beg -= residue;
residue = buflim - lim;
if (beg < lim)
{
if (outleft)
nlines += grepbuf (beg, lim);
if (pending)
prpending (lim);
if((!outleft && !pending) || (nlines && done_on_match && !out_invert))
goto finish_grep;
}
/* The last OUT_BEFORE lines at the end of the buffer will be needed as
leading context if there is a matching line at the begin of the
next data. Make beg point to their begin. */
i = 0;
beg = lim;
while (i < out_before && beg > bufbeg && beg != lastout)
{
++i;
do
--beg;
while (beg[-1] != eol);
}
/* detect if leading context is discontinuous from last printed line. */
if (beg != lastout)
lastout = 0;
/* Handle some details and read more data to scan. */
save = residue + lim - beg;
if (out_byte)
totalcc = add_count (totalcc, buflim - bufbeg - save);
if (out_line)
nlscan (beg);
if (! fillbuf (save, stats))
{
if (! is_EISDIR (errno, file))
suppressible_error (filename, errno);
goto finish_grep;
}
}
if (residue)
{
*buflim++ = eol;
if (outleft)
nlines += grepbuf (bufbeg + save - residue, buflim);
if (pending)
prpending (buflim);
}
finish_grep:
done_on_match -= not_text;
out_quiet -= not_text;
if ((not_text & ~out_quiet) && nlines != 0)
printf (_("Binary file %s matches\n"), filename);
return nlines;
}
每一次grep都需要对管道的数据全部读取然后进行查找,而数组只需要寻址匹配的删除即可。