rm -rf conf_$DATE > /dev/null 2>&1
mkdir conf_$DATE
for t in `cat ../../table-oracle-tohbase.list | grep -v ^#`
do
table=`echo $t | awk -F ',' '{print $1}'`
sh genconf.sh $table
done
USUAGE="Usuage: bash run_all_bulkload.sh"
source ../../../conf.properties
LOG_DIR=logs_$DATE
if [ ! -d $LOG_DIR ]; then
mkdir $LOG_DIR
fi
for t in `cat ../../table-oracle-tohbase.list | grep -v ^#`; do
table=`echo $t | awk -F ',' '{print $1}'`
echo "Bulkload "$table" ..."
nohup sh run_bulkload.sh $table > $LOG_DIR/$table.log 2>&1 &
done
#! /bin/bash
USUAGE="Usuage: bash run_bulkload.sh [TABLENAME] <-test>"
PROJECT_HOME=.
CLASSPATH=$PROJECT_HOME/lib/bulkload.jar
IS_TEST=1 #Default is false 闈炴祴璇?
if [ "$#" -lt "1" ]; then
echo $USUAGE
exit 1
elif [ "$#" -eq "1" ]; then
TABLENAME=$1
elif [ "$#" -eq "2" ]; then
TABLENAME=$1
IS_TEST=0
else
echo $USUAGE
exit 1
fi
source ../../../conf.properties
tablename=$TABLENAME"_"$DATE
CONF_FILE=conf_$DATE/$tablename.properties
source $CONF_FILE
# When use the test mode, judge whether the test parameters have been set
checktest=0
if [ "$IS_TEST" -eq "0" ]; then
if [ "$test_outputDir" = "" ]; then
echo "ERROR: You must set the test_outputDir in the conf file."
checktest=1
fi
if [ "$test_tableName" = "" ]; then
echo "ERROR: You must set the test_tableName in the conf file."
checktest=1
fi
fi
if [ "$checktest" -eq "1" ]; then
exit 1
fi
if [ "$IS_TEST" -eq "0" ];then
HFILE_DIR=$test_outputDir
HBASE_TABLE_NAME=$test_tableName
INDEX_TABLE_NAME=$test_indextablename
else
HFILE_DIR=$outputDir
HBASE_TABLE_NAME=$tableName
INDEX_TABLE_NAME=$indextablename
fi
#======= Check and disable the hbase table ===============
echo "Check whether the table exists"
if echo "list" | hbase shell | grep $tableName ; then
echo "The table has existed, disable and drop the table"
echo "disable '$tableName'; drop '$tableName'" | hbase shell
fi
echo "Start loading data into HBase at `date`"
start=`date +%s`
jars=`ls $PROJECT_HOME/lib`
for jar in $jars
do
CLASSPATH="$CLASSPATH:$PROJECT_HOME/lib/$jar"
done
CLASSPATH=/etc/hdfs1/conf:/etc/hadoop/conf:/etc/hbase/conf:$CLASSPATH
sudo -u hdfs hdfs dfs -rm -r $HFILE_DIR
if [ "$IS_TEST" -eq "0" ]; then
java -Djava.library.path=/usr/lib/hadoop/lib/native -cp $CLASSPATH com.transwarp.hbase.bulkload.ImportTextFile2HBase $CONF_FILE -test
else
java -Djava.library.path=/usr/lib/hadoop/lib/native -cp $CLASSPATH com.transwarp.hbase.bulkload.ImportTextFile2HBase $CONF_FILE
fi
sleep 60
sudo -u hdfs hdfs dfs -chmod -R 777 $HFILE_DIR
sleep 60
if [ "$INDEX_TABLE_NAME" != "" ]; then
/usr/lib/hbase/bin/hbase org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles $HFILE_DIR $INDEX_TABLE_NAME
else
/usr/lib/hbase/bin/hbase org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles $HFILE_DIR $HBASE_TABLE_NAME
fi
#echo "Delete the hfile output dir "
#sudo -u hdfs hdfs dfs -rmr $HFILE_DIR
end=`date +%s`
interval=`expr $end - $start`
echo "End loading data into HBase at `date`"
echo "Time used $interval seconds"