1、需求,sqoop2写命令太麻烦,于是写了一个通用型的java 的sqoop2导入程序,很简单,有什么不足,望各位看官指出。
import org.apache.sqoop.client.SqoopClient;
import org.apache.sqoop.model.MConnector;
import org.apache.sqoop.model.MDriverConfig;
import org.apache.sqoop.model.MFromConfig;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MSubmission;
import org.apache.sqoop.model.MToConfig;
import org.apache.sqoop.validation.Status;
import org.mortbay.jetty.Connector;
public class SaveJob {
<span style="white-space:pre"> </span>/**
<span style="white-space:pre"> </span> *
<span style="white-space:pre"> </span> * @param 1,table 表名
<span style="white-space:pre"> </span> * 2,date 导出日期
<span style="white-space:pre"> </span> * 3,sql sqoop通用语句
<span style="white-space:pre"> </span> * 4,pc sqoop的分区列,尽量选用字符串,分区列的选择会影响作业的效率
<span style="white-space:pre"> </span> * 5,job sqoop作业名字
<span style="white-space:pre"> </span> * 6,dir 在hdfs的存储位置
<span style="white-space:pre"> </span> * 7,url sqoop的服务指向
<span style="white-space:pre"> </span> * 8,fromLink 作业数据源的linkID
<span style="white-space:pre"> </span> * 9,toLink 作业导向的linkID
<span style="white-space:pre"> </span> * @throws InterruptedException
<span style="white-space:pre"> </span> */
<span style="white-space:pre"> </span>
<span style="white-space:pre"> </span>public static void main(String[] args) throws InterruptedException {
<span style="white-space:pre"> </span>String dataBase=args[0];
<span style="white-space:pre"> </span>String table = args[1];
<span style="white-space:pre"> </span>String date = args[2];<span style="white-space:pre"> </span>
<span style="white-space:pre"> </span>String sql =args[3]+" and ${CONDITIONS}";
<span style="white-space:pre"> </span>String pc = args[4];<span style="white-space:pre"> </span>
String jobName = table+"-"+date;<span style="white-space:pre"> </span>
<span style="white-space:pre"> </span>String url = "http://xx:12000/sqoop/";
<span style="white-space:pre"> </span>SqoopClient client = new SqoopClient(url);
<span style="white-space:pre"> </span> String fromLink=args[5];
<span style="white-space:pre"> </span> String toLink=args[6];
<span style="white-space:pre"> </span>String dir = args[7];
long fromLinkId = Long.parseLong(fromLink);
<span style="white-space:pre"> </span> long toLinkId = Long.parseLong(toLink);
<span style="white-space:pre"> </span> Long jobId=null;
<span style="white-space:pre"> </span> try{
<span style="white-space:pre"> </span> System.out.println(sql);
<span style="white-space:pre"> </span> long starTime=System.currentTimeMillis();
<span style="white-space:pre"> </span>MJob job = client.createJob(fromLinkId, toLinkId);
<span style="white-space:pre"> </span>job.setName(jobName);
<span style="white-space:pre"> </span>job.setCreationUser("BUffy");
<span style="white-space:pre"> </span>//配置数据源的设置
<span style="white-space:pre"> </span>MFromConfig fromJobConfig = job.getFromJobConfig();
<span style="white-space:pre"> </span>fromJobConfig.getStringInput("fromJobConfig.sql").setValue(sql);
<span style="white-space:pre"> </span>fromJobConfig.getStringInput("fromJobConfig.partitionColumn").setValue(pc);
<span style="white-space:pre"> </span>//配置数据存储的设置
<span style="white-space:pre"> </span> MToConfig toJobConfig = job.getToJobConfig();
<span style="white-space:pre"> </span>toJobConfig.getStringInput("toJobConfig.outputDirectory").setValue(dir);
<span style="white-space:pre"> </span>Status status=client.saveJob(job);
<span style="white-space:pre"> </span> jobId = job.getPersistenceId();<span style="white-space:pre"> </span>
<span style="white-space:pre"> </span>MSubmission submission = client.startJob(jobId);
<span style="white-space:pre"> </span>System.out.println("jobId: " + jobId);
<span style="white-space:pre"> </span>//监控sqoop作业的执行时间
<span style="white-space:pre"> </span>while(true){<span style="white-space:pre"> </span>
<span style="white-space:pre"> </span>long currenTime=System.currentTimeMillis();
<span style="white-space:pre"> </span>if(!client.getJobStatus(jobId).getStatus().isRunning()){<span style="white-space:pre"> </span>
<span style="white-space:pre"> </span>break;
<span style="white-space:pre"> </span>}
<span style="white-space:pre"> </span>System.out.println("Job "+jobName+" running"+((currenTime-starTime)/1000.0)+"s");
<span style="white-space:pre"> </span>Thread.sleep(3*1000L);;
<span style="white-space:pre"> </span>}
<span style="white-space:pre"> </span>long endTime=System.currentTimeMillis();
<span style="white-space:pre"> </span>System.out.println("JobName: "+jobName+"commint "+ ((endTime-starTime)/1000.0)+"s successed" );
<span style="white-space:pre"> </span> } catch(Exception e){
<span style="white-space:pre"> </span> <span style="white-space:pre"> </span>e.printStackTrace();
<span style="white-space:pre"> </span> }finally{
<span style="white-space:pre"> </span> <span style="white-space:pre"> </span>if(null!=jobId){
<span style="white-space:pre"> </span> <span style="white-space:pre"> </span>client.deleteJob(jobId);//容错处理
<span style="white-space:pre"> </span> <span style="white-space:pre"> </span>} <span style="white-space:pre"> </span>
<span style="white-space:pre"> </span> }
<span style="white-space:pre"> </span>
<span style="white-space:pre"> </span>
<span style="white-space:pre"> </span>}
}