Sqoop从Mysql自动批量导入到HBase

通过shell脚本进行自动导入，由于数量较多，容易发生内存溢出问题，可以每10w条数据导入一次，进行自动循环导入

#!/bin/bash

minsql="select min(id) from IP_COPYRIGHT_SOFTWARE_LIST;"
minid=$(mysql -h 192.168.131.168 -P 3306 -u root -pDataadt123! -D IPTRADEMARK -s -e "${minsql}")
maxsql="select max(id) from IP_COPYRIGHT_SOFTWARE_LIST;"
maxid=$(mysql -h 192.168.131.168 -P 3306 -u root -pDataadt123! -D IPTRADEMARK -s -e "${maxsql}")
size=$(($maxid-$minid))
echo $minid
echo $maxid
echo $size
n=100000
quotient=`expr $size / $n`
remainder=`expr $size % $n`
echo $remainder
echo $quotient
if [ $remainder -ge 1 ] ;then
  quotient=$(($quotient+1))
fi
j=$minid
for((i=1; i <= $quotient; i++))
do
 sql="select count(id) from IP_COPYRIGHT_SOFTWARE_LIST where id>=${j} and id <${j}+100000;"
 lenth=$(mysql -h 192.168.131.168 -P 3306 -u root -pDataadt123! -D IPTRADEMARK -s -e "${sql}")
 if [ $lenth -ge 1 ] ;then
   sql="select * from IP_COPYRIGHT_SOFTWARE_LIST where id>=${j} and id <${j}+100000 and\$CONDITIONS"
   bin/sqoop import --connect jdbc:mysql://192.168.131.168:3306/IPTRADEMARK?useSSL=false --username root --password Dataadt123! --m 12 --query "${sql}" --driver com.mysql.jdbc.Driver --split-by id --hbase-table 'IP_COPYRIGHT_SOFTWARE_LIST' --column-family info --hbase-create-table --hbase-row-key ID
fi
j=$(($j+100000))
  if [ $i -eq $quotient-1 ] ;then
     sql="select * from IP_COPYRIGHT_SOFTWARE_LIST where id>=${j} and id <=${maxid} and\$CONDITIONS"
     bin/sqoop import --connect jdbc:mysql://192.168.131.168:3306/IPTRADEMARK?useSSL=false --username root --password Dataadt123! --m 12 --query "${sql}" --driver com.mysql.jdbc.Driver --split-by id --hbase-table 'IP_COPYRIGHT_SOFTWARE_LIST' --column-family info --hbase-create-table --hbase-row-key ID
  fi

    echo Sqoop import from: ${j} to: $(($j+100000)) success....................................

done


exit