sqoop2 java 批量入库程序

2 篇文章 0 订阅
2 篇文章 0 订阅

1、需求,sqoop2写命令太麻烦,于是写了一个通用型的java 的sqoop2导入程序,很简单,有什么不足,望各位看官指出。

import org.apache.sqoop.client.SqoopClient;
import org.apache.sqoop.model.MConnector;
import org.apache.sqoop.model.MDriverConfig;
import org.apache.sqoop.model.MFromConfig;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MSubmission;
import org.apache.sqoop.model.MToConfig;
import org.apache.sqoop.validation.Status;
import org.mortbay.jetty.Connector;


public class SaveJob {
<span style="white-space:pre">	</span>/**
<span style="white-space:pre">	</span> * 
<span style="white-space:pre">	</span> * @param 1,table 表名
<span style="white-space:pre">	</span> *        2,date 导出日期
<span style="white-space:pre">	</span> *        3,sql sqoop通用语句
<span style="white-space:pre">	</span> *        4,pc  sqoop的分区列,尽量选用字符串,分区列的选择会影响作业的效率
<span style="white-space:pre">	</span> *        5,job sqoop作业名字
<span style="white-space:pre">	</span> *        6,dir 在hdfs的存储位置
<span style="white-space:pre">	</span> *        7,url sqoop的服务指向
<span style="white-space:pre">	</span> *        8,fromLink 作业数据源的linkID
<span style="white-space:pre">	</span> *        9,toLink   作业导向的linkID
<span style="white-space:pre">	</span> * @throws InterruptedException
<span style="white-space:pre">	</span> */
<span style="white-space:pre">	</span>
<span style="white-space:pre">	</span>public static void main(String[] args) throws InterruptedException {
<span style="white-space:pre">		</span>String dataBase=args[0];
<span style="white-space:pre">		</span>String table = args[1];
<span style="white-space:pre">		</span>String date = args[2];<span style="white-space:pre">	</span>
<span style="white-space:pre">		</span>String sql =args[3]+" and ${CONDITIONS}"; 
<span style="white-space:pre">		</span>String pc = args[4];<span style="white-space:pre">	</span>
        String jobName = table+"-"+date;<span style="white-space:pre">		</span>
<span style="white-space:pre">		</span>String url = "http://xx:12000/sqoop/";
<span style="white-space:pre">		</span>SqoopClient client = new SqoopClient(url);
<span style="white-space:pre">	</span>    String fromLink=args[5];
<span style="white-space:pre">	</span>    String toLink=args[6];
<span style="white-space:pre">		</span>String dir = args[7];
        long fromLinkId = Long.parseLong(fromLink);
<span style="white-space:pre">	</span>    long toLinkId = Long.parseLong(toLink);
<span style="white-space:pre">	</span>    Long jobId=null;
<span style="white-space:pre">	</span>    try{
<span style="white-space:pre">	</span>    System.out.println(sql);
<span style="white-space:pre">	</span>    long starTime=System.currentTimeMillis();
<span style="white-space:pre">		</span>MJob job = client.createJob(fromLinkId, toLinkId);
<span style="white-space:pre">		</span>job.setName(jobName);
<span style="white-space:pre">		</span>job.setCreationUser("BUffy");
<span style="white-space:pre">		</span>//配置数据源的设置
       <span style="white-space:pre">	</span>MFromConfig fromJobConfig = job.getFromJobConfig();
<span style="white-space:pre">		</span>fromJobConfig.getStringInput("fromJobConfig.sql").setValue(sql); 
<span style="white-space:pre">		</span>fromJobConfig.getStringInput("fromJobConfig.partitionColumn").setValue(pc);
<span style="white-space:pre">		</span>//配置数据存储的设置
<span style="white-space:pre">	</span>    MToConfig toJobConfig = job.getToJobConfig();
<span style="white-space:pre">		</span>toJobConfig.getStringInput("toJobConfig.outputDirectory").setValue(dir);
<span style="white-space:pre">		</span>Status status=client.saveJob(job);
<span style="white-space:pre">	</span>    jobId = job.getPersistenceId();<span style="white-space:pre">		</span>
<span style="white-space:pre">		</span>MSubmission submission = client.startJob(jobId);
<span style="white-space:pre">		</span>System.out.println("jobId: " + jobId);
<span style="white-space:pre">		</span>//监控sqoop作业的执行时间
<span style="white-space:pre">		</span>while(true){<span style="white-space:pre">	</span>
<span style="white-space:pre">			</span>long currenTime=System.currentTimeMillis();
<span style="white-space:pre">			</span>if(!client.getJobStatus(jobId).getStatus().isRunning()){<span style="white-space:pre">			</span>
<span style="white-space:pre">				</span>break;
<span style="white-space:pre">			</span>}
<span style="white-space:pre">			</span>System.out.println("Job "+jobName+" running"+((currenTime-starTime)/1000.0)+"s");
<span style="white-space:pre">			</span>Thread.sleep(3*1000L);;
<span style="white-space:pre">		</span>}
<span style="white-space:pre">		</span>long endTime=System.currentTimeMillis();
<span style="white-space:pre">		</span>System.out.println("JobName: "+jobName+"commint "+ ((endTime-starTime)/1000.0)+"s successed" );
<span style="white-space:pre">	</span>    } catch(Exception e){
<span style="white-space:pre">	</span>    <span style="white-space:pre">	</span>e.printStackTrace();
<span style="white-space:pre">	</span>    }finally{
<span style="white-space:pre">	</span>    <span style="white-space:pre">	</span>if(null!=jobId){
<span style="white-space:pre">	</span>    <span style="white-space:pre">		</span>client.deleteJob(jobId);//容错处理
<span style="white-space:pre">	</span>    <span style="white-space:pre">	</span>}    <span style="white-space:pre">	</span>
<span style="white-space:pre">	</span>    }
<span style="white-space:pre">	</span>    
<span style="white-space:pre">		</span>
<span style="white-space:pre">	</span>}


}


评论 1
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值