四 Spark Spark的Java操作实现

编译Spark源码
一 centos7 mav安装

wget  http://mirrors.hust.edu.cn/apache/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz
tar -zxvf apache-maven-3.3.9-bin.tar.gz
mv apache-maven-3.3.9 /usr/local/maven3

vi /etc/profile
export M2_HOME=/usr/local/maven3
export PATH=$PATH:$M2_HOME/bin

source /etc/profile

git clone https://github.com/apache/spark.git
cd spark
git checkout origin/branch-2.1
二编译
nohup build/mvn -DskipTests clean package  &
编译出来target/spark-parent_2.11-2.2.3-SNAPSHOT-tests.jar
查看jar包内容
[hdfs@kolla ~]$ jar vtf spark-parent_2.11-2.2.3-SNAPSHOT-tests.jar 

编译出来的包位于target目录下，pro-spark-example-2.2.0.jar
启动spark集群
[root@testceph ~]# cd /usr/local/spark-2.2.0-bin-hadoop2.6/sbin
[root@testceph sbin]# ./start-all.sh
[root@testceph sbin]# cd ../
[root@testceph bin]# ./spark-submit --help


三上传文件
[root@testceph ~]# hdfs dfs -ls /
Found 1 items
drwx-wx-wx   - root supergroup          0 2018-12-27 19:04 /tmp
[root@testceph ~]#
[root@testceph ~]# hdfs dfs -mkdir /spark
[root@testceph ~]# hdfs dfs -mkdir /spark/example
[root@testceph ~]# hdfs dfs -copyFromLocal a.sh /spark/example/


运行spark任务
[root@testceph ~]# ./spark-submit --master spark://192.168.229.133:7077 --name WordCount --class org.apache.spark.examples.JavaWordCount /data/example/pro-spark-example-2.2.0.jar /spark/example/README.md

编译spark 项目工程

JAVA_HOME=/Library/Java/JavaVirtualMachines/jdk1.8.0/Contents/Home
PATH=$JAVA_HOME/bin:$PATH:.
CLASSPATH=$JAVA_HOME/lib/tools.jar:$JAVA_HOME/lib/dt.jar:.
export JAVA_HOME
export PATH
export CLASSPATH 

cd spark
build/mvn -DskipTests clean package