hadoop集群安装与配置--sqoop服务调用

一、基于thrift生成java服务端代码和C#端代码

thrift接口代码：文件名称sqoopthrift.thrift

namespace java com.javabloger.gen.code
service SqoopThrift{   
string CreateTable(1:string host, 2:string database, 3:string userName, 4:string password, 5:string tableName, 6:string hiveTaleb)
string ImportDataInHive(1:string host, 2:string database, 3:string userName, 4:string password, 5:string tableName, 6:string hiveTaleb,7:string keyfield)
string ImportDataInHiveByWhere(1:string host, 2:string database, 3:string userName, 4:string password, 5:string tableName, 6:string hiveTaleb, 7:string where, 8:list<string> columnNames,9:string keyfield)
string ExportData(1:string host, 2:string database, 3:string userName, 4:string password, 5:string tableName, 6:string hiveTaleb)
string ExportDataUpdate(1:string host, 2:string database, 3:string userName, 4:string password, 5:string tableName, 6:string hiveTaleb,7:string keyfield)
}

thrift命令：thrift -gen java XXX.thrift              （注：基于windows下thrift0.90版本）

生成的java端thrift代码略:下载

thrift命令：thrift -gen csharp XXX.thrift

生成的C#端thrift代码略：下载

二、C#端调用sqoop服务demo

   var socket = new TSocket(sqoop服务在ip,sqoop服务的端口);
            transport = new TBufferedTransport(socket);
            var proto = new TBinaryProtocol(transport);
            sqoopClient = new SqoopThriftClient(proto);
 var feedback = "";
            try
            {
                if (!transport.IsOpen)
                    transport.Open();
                if (string.IsNullOrEmpty(hiveTaleb))
                    feedback = sqoopClient.ImportDataInHive(数据库ip, 数据库, 数据库用户, 数据库用户密码, 要导入的表名, 要导入的hive表名, 导入hive需要的分区字段);
                else
                    feedback = sqoopClient.ImportDataInHive(_host, _database, _userName, _password, tableName, hiveTaleb, keyfield);
                transport.Close();
            }
            catch (Exception ex)
            {
                transport.Close();               
            }

三、java端sqoop服务编写

采用java调用shell命令去调用sqoo的功能

1.调用shell的类，此类来源，有部分改变

package centos.shell;

import java.io.BufferedReader;  
import java.io.FileOutputStream;  
import java.io.IOException;   
import java.io.InputStreamReader;
import java.io.OutputStream;  
import java.io.OutputStreamWriter;  
import java.text.DateFormat;  
import java.text.SimpleDateFormat;  
import java.util.Date;  
  
public class JavaShellUtil {  
	private static String basePath = "/tmp/"; 
	private static  String executeShellLogFile;  
	public JavaShellUtil(String logPath){
		if(!logPath.isEmpty())
		basePath=logPath;
		executeShellLogFile = basePath + "executeShell"; 
	}
    
public String executeShell(String shellCommand) throws IOException {   
	StringBuffer stringBuffer = new StringBuffer();  
	BufferedReader bufferedReader = null;  
	BufferedReader errorReader = null;  
	DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:SS ");  
	DateFormat dateFormat1 = new SimpleDateFormat("yyyy-MM-dd-HH-mm-SS"); 
  
	try {  
		stringBuffer.append(dateFormat.format(new Date())).append("ready for cmd:").append(shellCommand).append(" \r\n");	  
		Process pid = null;  
		String[] cmd = {"/bin/bash", "-c", shellCommand};  	
		pid = Runtime.getRuntime().exec(cmd);  
		if (pid != null) {  
			stringBuffer.append("threadid:").append(pid.toString()).append("\r\n");  
			errorReader = new BufferedReader(new InputStreamReader(pid.getErrorStream()));  
			bufferedReader = new BufferedReader(new InputStreamReader(pid.getInputStream()));  
			pid.waitFor();  
		} else {  
			stringBuffer.append("no pid\r\n");  
		}  
		stringBuffer.append(dateFormat.format(new Date())).append("Shell is over\r\nfeed back\r\n");  
		String line = null;  
		while (bufferedReader != null &&  (line = bufferedReader.readLine()) != null) {  
		stringBuffer.append(line).append("\r\n");  
		}  
		while (errorReader != null &&  (line = errorReader.readLine()) != null) {  
			stringBuffer.append(line).append("\r\n");  
			}
	} catch (Exception ioe) {  
		stringBuffer.append("runing of exception\r\n").append(ioe.getMessage()).append("\r\n");  
	} finally {  
		if (bufferedReader != null) {  
			OutputStreamWriter outputStreamWriter = null;  
			try {  
				bufferedReader.close();  
				errorReader.close(); 
				OutputStream outputStream = new FileOutputStream(executeShellLogFile+dateFormat1.format(new Date())+".log");  
				outputStreamWriter = new OutputStreamWriter(outputStream, "UTF-8");  
				outputStreamWriter.write(stringBuffer.toString());  
			} catch (Exception e) {  
				e.printStackTrace();  
			} finally {  
				outputStreamWriter.close();  
			}  
		} 
	}  
	return stringBuffer.toString();  
	}  
}  

2.业务实现

package centos.shell;


import java.io.IOException;
import java.util.List;

import org.apache.thrift.TException;

public class SqoopServiceImp implements SqoopThrift.Iface {
	private JavaShellUtil cmdHelper;	
	private String shellpath="/usr/local/sqoop/";
	public SqoopServiceImp(String shellbasepath,String shellLogPath){
		if(!shellbasepath.isEmpty()){
			shellpath=shellbasepath;
		}
		cmdHelper=new JavaShellUtil(shellLogPath);
	}	
	@Override
	public String ExportData(String host, String database, String userName,
			String password, String tableName, String hiveTaleb)
			throws TException {
		System.out.print("begin ExportData "+tableName+"\n");
		String feedback="";
		String shellCommand=shellpath+"bin/sqoop export --connect 'jdbc:sqlserver://"+host
		+";username="+userName+";password="+password+";database="+database+"' --table "+tableName+" --export-dir /user/hive/warehouse/"+hiveTaleb
		+" --input-fields-terminated-by '\001'";
		try {
			feedback=cmdHelper.executeShell(shellCommand);
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		System.out.print("ExportData over\n");
		return feedback;
	}

	@Override
	public String ExportDataUpdate(String host, String database,
			String userName, String password, String tableName,
			String hiveTaleb, String keyfield) throws TException {
		System.out.print("begin ExportDataUpdate "+tableName+"\n");
		String feedback="";
		String shellCommand=shellpath+"bin/sqoop export --connect 'jdbc:sqlserver://"+host
		+";username="+userName+";password="+password+";database="+database+"' --table "+tableName
		+" --update-key '"+keyfield+"' --export-dir /user/hive/warehouse/"+hiveTaleb+" --input-fields-terminated-by '\001'";
		try {
			feedback=cmdHelper.executeShell(shellCommand);
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		System.out.print("ExportDataUpdate over\n");
		return feedback;
	}

	@Override
	public String CreateTable(String host, String database,
			String userName, String password, String tableName,
			String hiveTaleb)
			throws TException {
		System.out.print("begin create hive table "+tableName+"\n");
		String feedback="";		
		String shellCommand=shellpath+"bin/sqoop create-hive-table --connect 'jdbc:sqlserver://"+host
		+";username="+userName+";password="+password+";database="+database+"' --table "+tableName+" --hive-table "+hiveTaleb;
		try {
			//System.out.print("cmd is run \n");
			feedback=cmdHelper.executeShell(shellCommand);
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		System.out.print("create hive table over\n");
		return feedback;
	}

	@Override
	public String ImportDataInHive(String host, String database,
			String userName, String password, String tableName,
			String hiveTaleb, String keyfield) throws TException {
		System.out.print("begin ImportDataInHive "+tableName+"\n");
		String feedback="";
		String shellCommand=shellpath+"bin/sqoop import --connect 'jdbc:sqlserver://"+host
		+";username="+userName+";password="+password+";database="+database+"' --table "+tableName+" --hive-import --hive-table "+hiveTaleb
		+" --split-by '"+keyfield+"'";
		try {
			feedback=cmdHelper.executeShell(shellCommand);
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		System.out.print("ImportDataInHive over\n");
		return feedback;
	}

	@Override
	public String ImportDataInHiveByWhere(String host, String database,
			String userName, String password, String tableName,
			String hiveTaleb, String where, List<String> columnNames,
			String keyfield) throws TException {
		System.out.print("begin ImportDataInHiveByWhere "+tableName+"\n");
		String fileds="";
		for(int i=0;i<columnNames.size();i++){
			fileds=fileds+","+columnNames.get(i);
		}
		if(fileds.length()>1)
			fileds=fileds.substring(1, fileds.length()-1);
		String feedback="";
		String shellCommand=shellpath+"bin/sqoop import --connect 'jdbc:sqlserver://"+host
		+";username="+userName+";password="+password+";database="+database+"' --table "+tableName+" --where '"+where+"' --columns '"+fileds
		+"' --hive-import --hive-table "+hiveTaleb
		+" --split-by '"+keyfield+"'";
		try {
			feedback=cmdHelper.executeShell(shellCommand);
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		System.out.print("ImportDataInHiveByWhere over\n");
		return feedback;
	}
	

	

}

3.服务运行

import java.io.IOException;

import org.apache.thrift.protocol.TBinaryProtocol;
import org.apache.thrift.protocol.TBinaryProtocol.Factory;
import org.apache.thrift.server.TServer;
import org.apache.thrift.server.TThreadPoolServer;
import org.apache.thrift.server.TThreadPoolServer.Args;
import org.apache.thrift.transport.TServerSocket;
import org.apache.thrift.transport.TTransportException;

import centos.shell.SqoopServiceImp;
import centos.shell.SqoopThrift;



public class shelltest {

	/**
	 * @param args
	 * @throws IOException 
	 */
	public static void main(String[] args) throws IOException {	
		System.out.print("sqoop thrift server is start!\n");
		 startServer(args[0],args[1]);
			
	}
	public static void startServer(String shellpath,String logPath) {
        try {

            TServerSocket serverTransport = new TServerSocket(服务运行端口);

            SqoopThrift.Processor<SqoopServiceImp> process = new SqoopThrift.Processor<SqoopServiceImp>(new SqoopServiceImp(shellpath,logPath));

            Factory portFactory = new TBinaryProtocol.Factory(true, true);

            Args args = new Args(serverTransport);
            args.processor(process);
            args.protocolFactory(portFactory);

            TServer server = new TThreadPoolServer(args);
            server.serve();
        } catch (TTransportException e) {
            e.printStackTrace();
        }
    }


}

4.注意，此项目引用如下类库

libthrift-0.9.0.jar
log4j-1.2.16.jar
slf4j-log4j12-1.6.1.jar
slf4j-api-1.6.1.jar

5.服务启动

命令格式：java -jar 生成的jar的名称

若要后台运行格式为：java -jar 生成的jar的名称 &