设置maven JVM参数:
export MAVEN_OPTS="-Xmx2g-XX:MaxPermSize=512M -XX:ReservedCodeCacheSize=512m"
指定Hadoop,构建Spark工程。
# ApacheHadoop 2.0.5-alpha
mvn -Pyarn-alpha-Dhadoop.version=2.0.5-alpha-DskipTests clean package
#Cloudera CDH 4.2.0
mvn -Pyarn-alpha-Dhadoop.version=2.0.0-cdh4.2.0-DskipTests clean package
#Apache Hadoop 0.23.x
mvn -Pyarn-alpha-Phadoop-0.23 -Dhadoop.version=0.23.7-DskipTests clean package
#Apache Hadoop 2.2.X
mvn -Pyarn -Phadoop-2.2-Dhadoop.version=2.2.0-DskipTests clean package
#Apache Hadoop 2.3.X
mvn -Pyarn -Phadoop-2.3-Dhadoop.version=2.3.0-DskipTests clean package
#Apache Hadoop 2.4.X
mvn -Pyarn -Phadoop-2.4-Dhadoop.version=2.4.0-DskipTests clean package
#Different versions of HDFS and YARN.
mvn -Pyarn-alpha-Phadoop-2.3 -Dhadoop.version=2.3.0-Dyarn.version=0.23.7-DskipTests clean package
测试:
mvn -Dhadoop.version=... -DwildcardSuites=org.apache.spark.repl.ReplSuitetests
欢迎关注我的微信公众号