java提交远程spark任务_java执行远程服务器上的shell脚本-spark-submit.sh

所需jar包，commons-io-2.6.jar， ganymed-ssh2-262.jar ,

import ch.ethz.ssh2.ChannelCondition;

import ch.ethz.ssh2.Connection;

import ch.ethz.ssh2.Session;

import ch.ethz.ssh2.StreamGobbler;

import java.io.BufferedReader;

import java.io.InputStream;

import java.io.InputStreamReader;

import java.io.PrintWriter;

public class DemoSparkRunShell1 {

/**

* 需要事先收到把文件wordcount_data.txt上传到hdfs的根目录/

* @param args

*/

public static void main(String[] args) {

String ip = "192.168.137.190";

String username = "root";

String password = "111111";

String sparkhome = "/opt/cloudera/parcels/SPARK2-2.1.0.cloudera2-1.cdh5.7.0.p0.171658/lib/spark2";

String jarpath = "/opt/cloudera/parcels/SPARK2-2.1.0.cloudera2-1.cdh5.7.0.p0.171658/lib/spark2/examples/jars/spark-examples_2.11-2.1.0.cloudera2.jar";

String mainclass = "--class org.apache.spark.examples.SparkPi ";

String shellcmd = sparkhome.trim() + "/bin/spark-submit jar " + jarpath.trim() + " " + mainclass.trim() + "--master spark://192.168.137.190:7077 ";

shellcmd ="/opt/cloudera/parcels/SPARK2-2.1.0.cloudera2-1.cdh5.7.0.p0.171658/lib/spark2/bin/spark-submit --class org.apache.spark.examples.SparkPi --master local /opt/cloudera/parcels/SPARK2-2.1.0.cloudera2-1.cdh5.7.0.p0.171658/lib/spark2/examples/jars/spark-examples_2.11-2.1.0.cloudera2.jar";

Connection connection = null;

Session session = null;

InputStream stdOut = null;

InputStream stdErr = null;

try {

connection = new Connection(ip);

connection.connect();//连接

boolean isAuthenticated = connection.authenticateWithPassword(username, password);

if (isAuthenticated) {

System.out.println("连接linux成功");

} else {

System.out.println("连接linux失败");

throw new Exception("连接linux失败");

}

session = connection.openSession();

session.requestPTY("bash"); //建立虚拟终端

session.startShell();//打开一个Shell

stdOut = new StreamGobbler(session.getStdout());

stdErr = new StreamGobbler(session.getStderr());

BufferedReader stdoutReader = new BufferedReader(new InputStreamReader(stdOut));

BufferedReader stderrReader = new BufferedReader(new InputStreamReader(stdErr));

PrintWriter out = new PrintWriter(session.getStdin()); //准备输入命令

out.println(shellcmd); //输入待执行命令

out.println("exit");

out.close();// 6. 关闭输入流

// 7. 等待，除非1.连接关闭；2.输出数据传送完毕；3.进程状态为退出；4.超时

session.waitForCondition(ChannelCondition.CLOSED | ChannelCondition.EOF | ChannelCondition.EXIT_STATUS, 1000 * 3600);

System.out.println("Here is the output from stdout:");

while (true) {

String line = stdoutReader.readLine();

if (line == null)

break;

System.out.println(line);

}

System.out.println("Here is the output from stderr:");

while (true) {

String line = stderrReader.readLine();

if (line == null)

break;

System.out.println(line);

}

int exitstatus = session.getExitStatus();

System.out.println("退出状态: " + exitstatus);

} catch (Exception e) {

e.printStackTrace();

} finally {

try {

if (stdErr != null) {

stdErr.close();

}

if (stdOut != null) {

stdOut.close();

}

} catch (Exception e) {

e.printStackTrace();

}

if (session != null) {

session.close();

}

if (connection != null) {

connection.close();

}

}

}

}