批量导入千万数据到Redis集群处理方案，shell脚本处理（使用pipe处理）

从hive查询数据，使用shell脚本导入到Redis。

样例如下：

#!/bin/bash
/usr/local/bin/redis-cli -h 172.28.xx.xx -a xxxxx cluster nodes | grep master > info.txt
sed -i 's/@/:/g' info.txt
sed -i 's/-/:/g' info.txt
info=`cat info.txt | awk '{print $2":"$9}' | awk -F ':' '{print $1" "$2" "$4" "$5}'`
array=(${info// / })
len=${#array}
cnt=`expr $len / 4`
for((i=0;i<$cnt;i++));
 do
  a=`expr $i \* 4`
  b=`expr $a + 1`
  c=`expr $a + 2`
  d=`expr $a + 3`
  ip=${array[$a]}
  port=${array[$b]}
  slot1=${array[$c]}
  slot2=${array[$d]}
  hive -e "select concat('*3\n','$','3\n','set\n','$',length(key),'\n',key,'\n','$',length(value),'\n',value) from (select concat('key1:phone:', user_id) as key, concat('\"', pro_id,'\"') as value from dbName.table_name where crc16(concat('key1:phone:', user_id)) >= $slot1 and crc16(concat('key1:phone:', user_id)) <= $slot2)t;" | grep -v "^WARN" > $slot1"_"$slot2".data"
  unix2dos $slot1"_"$slot2".data"
  cat $slot1"_"$slot2".data" | /usr/local/bin/redis-cli -h $ip -p $port -a xxxxx--pipe
 done

改进版如下：

#!/bin/bash

cd /home/abc
param_date=`date -d '-2 days' +%Y%m%d`
/apprun/redis.20200206/bin/redis-cli -h 172.26.xxx.xx -a pwdpwd cluster nodes | grep master > info.txt
sed -i 's/@/:/g' info.txt
sed -i 's/-/:/g' info.txt
info=`cat info.txt | awk '{print $2":"$9}' | awk -F ':' '{print $1" "$2" "$4" "$5}'`
array=(${info// / })
len=${#array}
cnt=`expr $len / 4`
cd res
crc_slot=`hive -e "select crc16('recommend:hot:list');" | grep -v "WARN"`
for((i=0;i<$cnt;i++));
  do
    a=`expr $i \* 4`
    b=`expr $a + 1`
    c=`expr $a + 2`
    d=`expr $a + 3`
    ip=${array[$a]}
    port=${array[$b]}
    slot1=${array[$c]}
    slot2=${array[$d]}
    if (($crc_slot>=$slot1)) && (($crc_slot<=$slot2));then
      hive -e "
      set hive.execution.engine=mr;
      select concat('*3\n', '$', '3\n', 'set\n', '$', length(key), '\n', key, '\n', '$', length(value), '\n', value) from
      (select concat('recommend:hot:list') as key, concat('\"', concat_ws(',', collect_set(prod_id)), '\"') as value from
      (select * from abc_db.t_tbl_prodhot_score where dayid=${param_date} order by hot_score desc limit 300)t1)t;"  | grep -v "WARN" > hot.data
      unix2dos hot.data
      cat "related_"$slot1"_"$slot2.data | /usr/local/bin/redis-cli -h $ip -p $port -a xxxxxxx --pipe
    fi
  done