脚本技术要点:
1、切割文件
2、http接口写es
#!/bin/bash
es_server=127.0.0.1
#for i in `ls esCrash.data*`
for i in `ls *data* |grep -v "\-D\-"`
do
line_num=$(wc -l $i |awk '{print $1}')
if [ $line_num -eq 0 ];then continue;fi
if [ $line_num -le 5000 ];then
cp $i ${i}-D-1
continue
else
begin=1
end=5000
count=`expr $line_num / 5000 + 1`
for j in `seq 1 $count`
do
if sed -n "${begin},${end}p" ${i} |tail -n 1 |grep -q _index;then
end=`expr ${end} - 1`
fi
sed -n "${begin},${end}p" $i >${i}-D-$j
begin=`expr $end + 1`
end=`expr 5000 \* $j + 5000`
done
fi
done
for i in `ls *-D-* |grep -v 'index.sh'`
do
#echo >>${i}
date
echo "write $i to es $es_server ......"
curl -s -XPOST ${es_server}:9200/_bulk --data-binary "@$i";
echo
sleep 3
done
rm -rf *data*