Maven方式建立Spark项目

建立maven项目

porn.xml

<?xml version="1.0" encoding="UTF-8"?>

<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0">

<modelVersion>4.0.0</modelVersion>

<groupId>Change_Phone</groupId>

<artifactId>Change_Phone</artifactId>

<version>1.0-SNAPSHOT</version>


<properties>

    <scala.version>2.11.8</scala.version>

    <spark.version>2.4.0</spark.version>

    <java.version>1.8</java.version>

</properties>


<pluginRepositories>


    <pluginRepository>

    <id>scala-tools.org</id>

    <name>Scala-Tools Maven2 Repository</name>

    <url>http://scala-tools.org/repo-releases</url>

</pluginRepository>

</pluginRepositories>


<dependencies>


    <dependency>

    <groupId>org.scala-lang</groupId>

    <artifactId>scala-library</artifactId>

    <version>${scala.version}</version>

</dependency>


    <dependency>

    <groupId>org.scala-lang</groupId>

    <artifactId>scala-compiler</artifactId>

    <version>${scala.version}</version>

</dependency>


    <dependency>

    <groupId>org.scala-lang</groupId>

    <artifactId>scala-reflect</artifactId>

    <version>${scala.version}</version>

</dependency>


    <dependency>

    <groupId>org.scala-lang</groupId>

    <artifactId>scalap</artifactId>

    <version>${scala.version}</version>

</dependency>


    <dependency>

    <groupId>org.apache.spark</groupId>

    <artifactId>spark-core_2.11</artifactId>

    <version>${spark.version}</version>

</dependency>


    <dependency>

    <groupId>org.apache.spark</groupId>

    <artifactId>spark-sql_2.11</artifactId>

    <version>${spark.version}</version>

</dependency>


    <dependency>

    <groupId>org.apache.spark</groupId>

    <artifactId>spark-hive_2.11</artifactId>

    <version>${spark.version}</version>

</dependency>


    <dependency>

    <groupId>org.apache.spark</groupId>

    <artifactId>spark-mllib_2.11</artifactId>

    <version>${spark.version}</version>

</dependency>


    <dependency>

    <groupId>org.apache.spark</groupId>

    <artifactId>spark-mllib-local_2.11</artifactId>

    <version>${spark.version}</version>

    <scope>compile</scope>

</dependency>

    <dependency>
        <groupId>mysql</groupId>
        <artifactId>mysql-connector-java</artifactId>
        <version>5.1.6</version>
    </dependency>


    <dependency>

    <groupId>org.apache.spark</groupId>

    <artifactId>spark-mllib-local_2.11</artifactId>

    <version>${spark.version}</version>

    <type>test-jar</type>

    <scope>test</scope>

</dependency>

</dependencies>


<build>


    <plugins>


    <plugin>

    <artifactId>maven-assembly-plugin</artifactId>


    <configuration>

    <classifier>dist</classifier>

    <appendAssemblyId>true</appendAssemblyId>


    <descriptorRefs>

    <descriptor>jar-with-dependencies</descriptor>

</descriptorRefs>

</configuration>


    <executions>


    <execution>

    <id>make-assembly</id>

    <phase>package</phase>


    <goals>

    <goal>single</goal>

</goals>

</execution>

</executions>

</plugin>


    <plugin>

    <artifactId>maven-compiler-plugin</artifactId>


    <configuration>

    <source>1.7</source>

    <target>1.7</target>

</configuration>

</plugin>


    <plugin>

    <groupId>net.alchim31.maven</groupId>

    <artifactId>scala-maven-plugin</artifactId>

    <version>3.2.2</version>


    <executions>


    <execution>

    <id>scala-compile-first</id>

    <phase>process-resources</phase>


    <goals>

    <goal>compile</goal>

</goals>

</execution>

</executions>


    <configuration>

    <scalaVersion>${scala.version}</scalaVersion>

    <recompileMode>incremental</recompileMode>

    <useZincServer>true</useZincServer>


    <args>

    <arg>-unchecked</arg>

    <arg>-deprecation</arg>

    <arg>-feature</arg>

</args>


    <jvmArgs>

    <jvmArg>-Xms1024m</jvmArg>

    <jvmArg>-Xmx1024m</jvmArg>

</jvmArgs>


    <javacArgs>

    <javacArg>-source</javacArg>

    <javacArg>${java.version}</javacArg>

    <javacArg>-target</javacArg>

    <javacArg>${java.version}</javacArg>

    <javacArg>-Xlint:all,-serial,-path</javacArg>

</javacArgs>

</configuration>

</plugin>


    <plugin>

    <groupId>org.antlr</groupId>

    <artifactId>antlr4-maven-plugin</artifactId>

    <version>4.3</version>


    <executions>


    <execution>

    <id>antlr</id>


    <goals>

    <goal>antlr4</goal>

</goals>

    <phase>none</phase>

</execution>

</executions>


    <configuration>

    <outputDirectory>src/test/java</outputDirectory>

    <listener>true</listener>

    <treatWarningsAsErrors>true</treatWarningsAsErrors>

</configuration>

</plugin>

</plugins>

</build>

</project>

新建文件夹 放scala程序文件

原来maven建立的java文件夹标为灰色（Umark as Sources）

新建的scala文件夹标为蓝色（Source Root）

修改Artifacts

File-projectstructure-artifacts-“+”-jar-From modules with dependecies

extract to the target jar 把所有依赖包都加入jar包中

copy to the output······ 只打包wordcount类

点击build-buildartifacts

注意modules只有一个（出问题时注意modules配置）

Global Libraries