DataX分库分表脚本实现

#!/bin/bash
. ~/.bashrc

time=$(date "+%Y-%m-%d %H:%M:%S")
echo "${time}"
for ((i=0;i<100;i++));
do printf "%02d " $i;
    db_no=`printf "%02d\n" $i`
    nohup bash ./trans_query.sh ${db_no} > trans_query.out 2>&1 &
    sleep 6h
    time=$(date "+%Y-%m-%d %H:%M:%S")
    echo "${time}"
done
echo "trans_query_run.sh done"

#!/bin/bash
. ~/.bashrc
if [ $# -lt 1 ]
then
    echo "./loaddata db_no"
    exit
fi
db_no=$1
time=$(date "+%Y-%m-%d %H:%M:%S")
echo "${time}"

example="/home/work/datax/job/core/example"
tempdir="/home/work/datax/job/core"
basepath=$(cd `dirname $0`; pwd)
cd $basepath
ARR=(0 1 2 3 4 5 6 7 8 9)
for ID in ${ARR[*]}
do
    jsonfile="${tempdir}/trans_query_${db_no}_$ID"
    oneexample="${example}/trans_query_example"
    mo_val_str="${db_no}$ID"
    # 去掉串前面的所有0
    mo_val=$(echo -e $mo_val_str | sed -r 's/0*([0-9])/\1/')
    sed "s/DB_NO/${db_no}/g;s/MO_VAL/${mo_val}/g;s/TB_NO/$ID/g" ${oneexample} > ${jsonfile}
done
echo "${jsonfile} done"
ARR=(0 1 2 3 4 5 6 7 8 9)
#ARR=(1)
for ID in ${ARR[*]}
do
    echo "~/datax/log/trans_query_${db_no}_$ID.log"
    nohup /home/work/datax/bin/start_datax.sh /home/work/datax/job/core/trans_query_${db_no}_$ID > /home/work/datax/sh-load/sh-out/trans_query_${db_no}_$ID.out 2>&1 &
    #sleep 1s
done
echo "trans_query.sh done"

{
  "job": {
    "setting": {
      "speed": {
        "channel": 2000,
        "record": "150000"
      },
      "errorLimit": {
        "record": 0,
        "percentage": 0
      }
    },
    "content": [
      {
        "reader": {
          "name": "mysqlreader",
          "parameter": {
            "column": [
              "F_trans_id",
              "F_enabled",
              "F_trans_type",
              "F_payment_mode",
              "F_state"
            ],
            "splitPk": "F_trans_id",
            "where": "F_buyer_user_id%1000=MO_VAL or F_seller_user_id%1000=MO_VAL",
            "connection": [
              {
                "jdbcUrl": [
                  "jdbc:mysql://x.x.x.x:6000?useUnicode=true&characterEncoding=CP1252"
                ],
                "table": [
                  "dyp_db_00.t_trans_00_[0-9]",
                  "dyp_db_08.t_trans_08_[0-9]",
                  "dyp_db_16.t_trans_16_[0-9]",
                  "dyp_db_24.t_trans_24_[0-9]",
                  "dyp_db_32.t_trans_32_[0-9]",
                  "dyp_db_40.t_trans_40_[0-9]",
                  "dyp_db_48.t_trans_48_[0-9]",
                  "dyp_db_56.t_trans_56_[0-9]",
                  "dyp_db_64.t_trans_64_[0-9]",
                  "dyp_db_72.t_trans_72_[0-9]",
                  "dyp_db_80.t_trans_80_[0-9]",
                  "dyp_db_88.t_trans_88_[0-9]",
                  "dyp_db_96.t_trans_96_[0-9]"
                ]
              },
              {
                "jdbcUrl": [
                  "jdbc:mysql://x.x.x.x:6000?useUnicode=true&characterEncoding=CP1252"
                ],
                "table": [
                  "dyp_db_01.t_trans_01_[0-9]",
                  "dyp_db_09.t_trans_09_[0-9]",
                  "dyp_db_17.t_trans_17_[0-9]",
                  "dyp_db_25.t_trans_25_[0-9]",
                  "dyp_db_33.t_trans_33_[0-9]",
                  "dyp_db_41.t_trans_41_[0-9]",
                  "dyp_db_49.t_trans_49_[0-9]",
                  "dyp_db_57.t_trans_57_[0-9]",
                  "dyp_db_65.t_trans_65_[0-9]",
                  "dyp_db_73.t_trans_73_[0-9]",
                  "dyp_db_81.t_trans_81_[0-9]",
                  "dyp_db_89.t_trans_89_[0-9]",
                  "dyp_db_97.t_trans_97_[0-9]"
                ]
              }
            ],
            "username": "datax_r",
            "password": "******"
          }
        },
        "writer": {
          "name": "mysqlwriter",
          "parameter": {
            "writeMode": "insert",
            "username": "datax_w",
            "password": "******",
            "column": [
              "F_trans_id",
              "F_enabled",
              "F_trans_type",
              "F_payment_mode",
              "F_state"
            ],
            "preSql": [],
            "connection": [
              {
                "jdbcUrl": "jdbc:mysql://x.x.x.x:1111?useUnicode=true&characterEncoding=CP1252",
                "table": [
                  "dyp_db_DB_NO.t_trans_query_DB_NO_TB_NO"
                ]
              }
            ]
          }
        }
      }
    ]
  }
}