Maven最关键的就是pom文件的配置,我们使用下面的版本:
Spring 4.3.7.RELEASE
Mybatis 3.2.8
Jackson 2.6.7
Hadoop 2.6.0
Scala 2.11.7
Spark 2.4.0
在该pom文件的配置过程中,我们先根据Spring要求确定Jackson的版本要求,然后选择Spark版本,同时确定Jackson版本,然后再选择Hadoop和Scala版本。我们需要再项目中用到Scala中的Tuple2。
在该pom文件配置过程中,最关键的是Jackson版本的选择。由于我们使用的是Spring 4.x,那么Jackson版本必须在2.6以上。而Spark也需要用到Jackson,怎么查找Spark对应的Jackson版本呢?
在Spark源码中的pom文件中,我们可以查询不同的Spark版本对应的Jackson本版
这是1.6.3版本pom文件的信息
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<akka.group>com.typesafe.akka</akka.group>
<akka.version>2.3.11</akka.version>
<java.version>1.7</java.version>
<maven.version>3.3.3</maven.version>
<sbt.project.name>spark</sbt.project.name>
<mesos.version>0.21.1</mesos.version>
<mesos.classifier>shaded-protobuf</mesos.classifier>
<slf4j.version>1.7.10</slf4j.version>
<log4j.version>1.2.17</log4j.version>
<hadoop.version>2.2.0</hadoop.version>
<protobuf.version>2.5.0</protobuf.version>
<yarn.version>${hadoop.version}</yarn.version>
<hbase.version>0.98.7-hadoop2</hbase.version>
<hbase.artifact>hbase</hbase.artifact>
<flume.version>1.6.0</flume.version>
<zookeeper.version>3.4.5</zookeeper.version>
<curator.version>2.4.0</curator.version>
<hive.group>org.spark-project.hive</hive.group>
<!-- Version used in Maven Hive dependency -->
<hive.version>1.2.1.spark</hive.version>
<!-- Version used for internal directory structure -->
<hive.version.short>1.2.1</hive.version.short>
<derby.version>10.12.1.1</derby.version>
<parquet.version>1.7.0</parquet.version>
<hive.parquet.version>1.6.0</hive.parquet.version>
<jblas.version>1.2.4</jblas.version>
<jetty.version>8.1.19.v20160209</jetty.version>
<orbit.version>3.0.0.v201112011016</orbit.version>
<chill.version>0.5.0</chill.version>
<ivy.version>2.4.0</ivy.version>
<oro.version>2.0.8</oro.version>
<codahale.metrics.version>3.1.2</codahale.metrics.version>
<avro.version>1.7.7</avro.version>
<avro.mapred.classifier>hadoop2</avro.mapred.classifier>
<jets3t.version>0.7.1</jets3t.version>
<aws.kinesis.client.version>1.4.0</aws.kinesis.client.version>
<!-- the producer is used in tests -->
<aws.kinesis.producer.version>0.10.1</aws.kinesis.producer.version>
<!-- org.apache.httpcomponents/httpclient-->
<commons.httpclient.version>4.3.2</commons.httpclient.version>
<!-- commons-httpclient/commons-httpclient-->
<httpclient.classic.version>3.1</httpclient.classic.version>
<commons.math3.version>3.4.1</commons.math3.version>
<!-- managed up from 3.2.1 for SPARK-11652 -->
<commons.collections.version>3.2.2</commons.collections.version>
<scala.version>2.10.5</scala.version>
<scala.binary.version>2.10</scala.binary.version>
<jline.version>${scala.version}</jline.version>
<jline.groupid>org.scala-lang</jline.groupid>
<codehaus.jackson.version>1.9.13</codehaus.jackson.version>
<fasterxml.jackson.version>2.4.4</fasterxml.jackson.version>
<snappy.version>1.1.2.6</snappy.version>
<netlib.java.version>1.1.2</netlib.java.version>
<calcite.version>1.2.0-incubating</calcite.version>
<commons-codec.version>1.10</commons-codec.version>
<!-- org.apache.commons/commons-lang/-->
<commons-lang2.version>2.6</commons-lang2.version>
<!-- org.apache.commons/commons-lang3/-->
<commons-lang3.version>3.3.2</commons-lang3.version>
<datanucleus-core.version>3.2.10</datanucleus-core.version>
<janino.version>2.7.8</janino.version>
<jersey.version>1.9</jersey.version>
<joda.version>2.9.3</joda.version>
<jodd.version>3.5.2</jodd.version>
<jsr305.version>1.3.9</jsr305.version>
<libthrift.version>0.9.2</libthrift.version>
<test.java.home>${java.home}</test.java.home>
<test.exclude.tags></test.exclude.tags>
<!--
Dependency scopes that can be overridden by enabling certain profiles. These profiles are
declared in the projects that build assemblies.
For other projects the scope should remain as "compile", otherwise they are not available
during compilation if the dependency is transivite (e.g. "bagel/" depending on "core/" and
needing Hadoop classes in the classpath to compile).
-->
<flume.deps.scop