写在前面
摆脱依赖IDEA手动打jar包,实现maven管理spark\scala的不同版本的依赖,排除特定依赖,包含指
定依赖,并最终生成包含Java、Scala代码的混合编译的可执行jar包程序。
使用环境
- jdk1.8
- maven3.6.1
- IDEA2017
- spark-2.4.3
- scala2.1.8
我的pom文件
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.dr.leo</groupId>
<artifactId>FastKettle</artifactId>
<version>2.0.1</version>
<description>FastKettle2.0,零售商数据清洗系统!</description>
<url>https://www.jlpyyf.com</url>
<developers>
<developer>
<id>leo</id>
<name>leo.jie</name>
<email>leo.jie@datarealglobal.com</email>
<roles>
<role>developer</role>
</roles>
</developer>
</developers>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
<spark.version>2.4.3</spark.version>
<scala.version>2.11</scala.version>
<spark.cope>compile</spark.cope>
</properties>
<distributionManagement>
<!--nexues3私服地址-->
<snapshotRepository>
<id>snapshots</id>
<url>http://192.168.x.xxx:8081/repository/maven-snapshots/</url>
<!-- nexus的仓储地址 -->
</snapshotRepository>
<repository>
<id>releases</id>
<url>http://192.168.x.xxx:8081/repository/maven-releases/</url>
</repository>
</distributionManagement>
<dependencies>
<dependency>
<!--钉钉消息预警插件(暂无发布到中央仓库)-->
<groupId>com.dr.leo</groupId>
<artifactId>dingtalk-smart-alert</artifactId>
<version>1.0.0-SNAPSHOT</version>
<classifier>repack</classifier>
</dependency>
<dependency>
<!--spark的相关依赖,直接去中央仓库搜索自己需要的版本,注意Scala和spark的版本对应,provided打包时排斥在外的依赖-->
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.version}</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
<!--<scope>${spark.cope}</scope>-->
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.spark/spark-sql -->
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.version}</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
<!--<scope>${spark.cope}</scope>-->
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.spark/spark-hive -->
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-hive_${scala.version}</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
<!--<scope>${spark.cope}</scope>-->
</dependency>
<!-- JDBC-->
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.32</version>
</dependency>
<dependency>
<groupId>com.microsoft.sqlserver</groupId>
<artifactId>mssql-jdbc</artifactId>
<version>6.1.0.jre8</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.scala-tools</groupId>
<artifactId>maven-scala-plugin</artifactId>
<version>2.15.0</version>
<executions>
<execution>
<goals>
<goal>compile</goal>
<goal>testCompile</goal>
</goals>
</execution>
</executions>
<configuration>
<scalaVersion>${scala.version}</scalaVersion>
<args>
<arg>-target:jvm-1.5</arg>
</args>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-eclipse-plugin</artifactId>
<version>2.10</version>
<configuration>
<downloadSources>true</downloadSources>
<buildcommands>
<buildcommand>ch.epfl.lamp.sdt.core.scalabuilder</buildcommand>
</buildcommands>
<additionalProjectnatures>
<projectnature>ch.epfl.lamp.sdt.core.scalanature</projectnature>
</additionalProjectnatures>
<classpathContainers>
<classpathContainer>org.eclipse.jdt.launching.JRE_CONTAINER</classpathContainer>
<classpathContainer>ch.epfl.lamp.sdt.launching.SCALA_CONTAINER</classpathContainer>
</classpathContainers>
</configuration>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<version>3.1.1</version>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
<!-- 下面两个配置,控制jar最终生成的名字 -->
<finalName>FastKettle-2.0.1-RELEASE</finalName>
<appendAssemblyId>false</appendAssemblyId>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.6.0</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<version>1.10</version>
<executions>
<execution>
<id>add-source</id>
<phase>generate-sources</phase>
<goals>
<goal>add-source</goal>
</goals>
<configuration>
<!-- 我们可以通过在这里添加多个source节点,来添加任意多个源文件夹 -->
<sources>
<source>src/main/java</source>
<source>src/main/scala</source>
</sources>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
<reporting>
<plugins>
<plugin>
<groupId>org.scala-tools</groupId>
<artifactId>maven-scala-plugin</artifactId>
<configuration>
<scalaVersion>${scala.version}</scalaVersion>
</configuration>
</plugin>
</plugins>
</reporting>
</project>
总结
以上方式,可能不是最完美的方式,但已经过实践可用,实现scala代码中调Java代码的功能模块,并实现最终的混合打包,执行。