datax运维中遇到的问题

1、HDFS主从切换会导致Datax脚本无法连接HDFS

加以下参数解决

"defaultFS":"hdfs://nameservice1",
"hadoopConfig":{
  "dfs.nameservices": "nameservice1",
  "dfs.ha.namenodes.nameservice1": "namenode342,namenode411",
  "dfs.namenode.rpc-address.nameservice1.namenode342": "master1.zd.prod:8020",
  "dfs.namenode.rpc-address.nameservice1.namenode411": "master2.zd.prod:8020",
  "dfs.client.failover.proxy.provider.nameservice1": "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
},

 参数从以下地方截取

  <property>
    <name>dfs.nameservices</name>
    <value>nameservice1</value>
  </property>

  <property>
    <name>dfs.client.failover.proxy.provider.nameservice1</name>
   <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
  </property>

  <property>
    <name>dfs.ha.namenodes.nameservice1</name>
    <value>namenode342,namenode411</value>
  </property>

  <property>
    <name>dfs.namenode.rpc-address.nameservice1.namenode342</name>
    <value>master1.zd.prod:8020</value>
  </property>

  <property>
    <name>dfs.namenode.rpc-address.nameservice1.namenode411</name>
    <value>master2.zd.prod:8020</value>
  </property>