eclipse+Maven管理Spark项目

刚开始用IDEA,用起来很不顺心，特别是快捷键的使用，批量导入依赖等，IDEA做得远不如eclipse。刚开始用sbt管理依赖，可是用得不熟，打的jar包都是肥包，编译上传等都麻烦。于是放弃IDEA，改用eclipse。

一：开发环境准备：eclipse中scala  IDE插件安装：help-eclipse marketplace 搜索scala下载

二：导入maven项目

三：初次导入时候，不能创建scala文件，这是因为还没有增加scala属性：项目右键configuration--add scala nature就可以了

附：maven项目的pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
	<modelVersion>4.0.0</modelVersion>

	<groupId>com.pxene.dmp</groupId>
	<artifactId>my-machine-learning</artifactId>
	<version>1.0-SNAPSHOT</version>

	<properties>
		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
		<hadoop.version>2.5.2</hadoop.version>
	</properties>

	<repositories>
		<repository>
			<id>mvn-repo</id>
			<url>http://maven.nlpcn.org/</url>
		</repository>
	</repositories>

	<dependencies>

		<dependency>
			<groupId>jdk.tools</groupId>
			<artifactId>jdk.tools</artifactId>
			<version>1.7</version>
			<scope>system</scope>
			<systemPath>${JAVA_HOME}/lib/tools.jar</systemPath>
		</dependency>

		<!--spark -->
		<dependency>
			<groupId>org.apache.spark</groupId>
			<artifactId>spark-core_2.10</artifactId>
			<version>1.5.2</version>
		</dependency>

		<dependency>
			<groupId>org.apache.spark</groupId>
			<artifactId>spark-sql_2.10</artifactId>
			<version>1.5.2</version>
		</dependency>

		<dependency>
			<groupId>org.apache.spark</groupId>
			<artifactId>spark-mllib_2.10</artifactId>
			<version>1.5.2</version>
		</dependency>

		<!--hbase -->
		<dependency>
			<groupId>org.apache.hbase</groupId>
			<artifactId>hbase-client</artifactId>
			<version>1.1.4</version>
		</dependency>

		<dependency>
			<groupId>org.apache.hbase</groupId>
			<artifactId>hbase-common</artifactId>
			<version>1.1.4</version>
		</dependency>

		<dependency>
			<groupId>org.apache.hbase</groupId>
			<artifactId>hbase-server</artifactId>
			<version>1.1.4</version>
		</dependency>

		<!-- ansj -->
		<dependency>
			<groupId>org.ansj</groupId>
			<artifactId>ansj_seg</artifactId>
			<version>5.0.1</version>
		</dependency>

	</dependencies>

	<build>
		<plugins>
			<plugin>
				<groupId>org.scala-tools</groupId>
				<artifactId>maven-scala-plugin</artifactId>
				<version>2.15.2</version>
				<executions>
					<execution>
						<goals>
							<goal>compile</goal>
							<goal>testCompile</goal>
						</goals>
						<configuration>
							<args>
								<arg>-dependencyfile</arg>
								<arg>${project.build.directory}/.scala_dependencies</arg>
							</args>
						</configuration>
					</execution>
				</executions>
			</plugin>

			<plugin>
				<groupId>org.apache.maven.plugins</groupId>
				<artifactId>maven-compiler-plugin</artifactId>
				<version>3.3</version>
				<configuration>
					<source>1.7</source>
					<target>1.7</target>
				</configuration>
			</plugin>

			<plugin>
				<groupId>org.apache.maven.plugins</groupId>
				<artifactId>maven-assembly-plugin</artifactId>
				<version>2.5.3</version>
				<configuration>
					<descriptors>
						<descriptor>src/main/resources/assembly/outer-lib.xml</descriptor>
						<descriptor>src/main/resources/assembly/inner-cls.xml</descriptor>
					</descriptors>
					<appendAssemblyId>false</appendAssemblyId>
					<archive>
						<manifest>
							<addClasspath>true</addClasspath>
							<classpathPrefix>lib/</classpathPrefix>
							<mainClass>pxene.NaiveBayes4Weixin</mainClass>
						</manifest>
					</archive>
				</configuration>
				<executions>
					<execution>
						<id>assembly</id>
						<phase>package</phase>
						<goals>
							<goal>single</goal>
						</goals>
					</execution>
				</executions>
			</plugin>
		</plugins>
	</build>
</project>

inner-cls.xml

<?xml version="1.0" encoding="UTF-8"?>
<assembly
	xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3"
	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
	xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 
          					  http://maven.apache.org/xsd/assembly-1.1.3.xsd">
	<id>inner-cls</id>
	<formats>
		<format>jar</format>
	</formats>
	<includeBaseDirectory>false</includeBaseDirectory>
	<fileSets>
		<fileSet>
			<directory>${project.basedir}/target/classes/</directory>
			<outputDirectory>/</outputDirectory>
			<includes>
				<include>**/*.class</include>
				<include>**/*.scala</include>
				<include>*.properties</include>
				<include>*.yaml</include>
				<include>*.xml</include>
			</includes>
		</fileSet>
	</fileSets>
	<dependencySets>
		<dependencySet>
			<outputDirectory>/</outputDirectory>
			<excludes>
				<exclude>org.apache.spark:spark-core_2.10</exclude>
				<exclude>org.apache.spark:spark-sql_2.10</exclude>
				<exclude>org.apache.spark:spark-mllib_2.10</exclude>
			</excludes>
			<unpack>true</unpack>
		</dependencySet>
	</dependencySets>
</assembly>

outer-lib.xml