idea用maven开发spark程序的pom.xml

最新推荐文章于 2022-06-10 16:21:29 发布

飞少fly

最新推荐文章于 2022-06-10 16:21:29 发布

阅读量372

点赞数

分类专栏： IDEA

本文链接：https://blog.csdn.net/qq_35400008/article/details/103462172

版权

本文介绍如何利用Maven为Spark项目构建pom.xml文件，通过Maven管理Spark和其他相关组件的依赖，详细指导查找和添加依赖的步骤。

摘要由CSDN通过智能技术生成

我们用maven构建spark项目，需要使用maven来管理各个组件jar

需要找依赖的话，地址：https://mvnrepository.com/

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
  <modelVersion>4.0.0</modelVersion>
  <groupId>com.cfqq</groupId>
  <artifactId>SparkProject</artifactId>
  <version>1.0</version>
  <inceptionYear>2008</inceptionYear>
  <properties>

    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
      <!-- 配置以下可以解决 在jdk1.8环境下打包时报错 “-source 1.5 中不支持 lambda 表达式” -->
    <maven.compiler.source>1.8</maven.compiler.source>
    <maven.compiler.target>1.8</maven.compiler.target>
    <scala.binary.version>2.11</scala.binary.version>
    <PermGen>64m</PermGen>
    <MaxPermGen>512m</MaxPermGen>

    <scala.version>2.11.8</scala.version>
    <spark.version>2.3.0</spark.version>
    <hadoop.version>2.6.0</hadoop.version>
    <scala.compat.version>2.11</scala.compat.version>
    <hbase.version>1.3.1</hbase.version>

  </properties>

  <repositories>
    <repository>
      <id>scala-tools.org</id>
      <name>Scala-Tools Maven2 Repository</name>
      <url>http://scala-tools.org/repo-releases</url>
    </repository>
  </repositories>

  <pluginRepositories>
    <pluginRepository>
      <id>scala-tools.org</id>
      <name>Scala-Tools Maven2 Repository</name>
      <url>http://scala-tools.org/repo-releases</url>
    </pluginRepository>
  </pluginRepositories>

  <dependencies>

    <!--①添加scala  -->
    <dependency>
      <groupId>org.scala-lang</groupId>
      <artifactId>scala-library</artifactId>
      <version>${scala.version}</version>
    </dependency>
    <dependency>
      <groupId>org.scala-lang</groupId>
      <artifactId>scala-reflect</artifactId>
      <version>${scala.version}</version>
    </dependency>
    <dependency>
      <groupId>org.scala-lang</groupId>
      <artifactId>scala-compiler</artifactId>
      <version>${scala.version}</version>
    </dependency>

      <!-- Spark-core -->
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-core_${scala.compat.version}</artifactId>
      <version>${spark.version}</version>
    </dependency>

    <!--③添加sparkStreaming -->
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-streaming_${scala.compat.version}</artifactId>
      <version>${spark.version}</version>
    </dependency>

      <!--③添加sparkStreaming消费kafka数据版本 -->
      <dependency>
          <groupId>org.apache.spark</groupId>
          <!--kafka0.10版本-->
          <artifactId>spark-streaming-kafka-0-10_${scala.compat.version}</artifactId>
          <!--kafka0.8版本-->
          <!--<artifactId>spark-streaming-kafka-0-8_${scala.compat.version}</artifactId>-->
          <version>${spark.version}</version>
      </dependency>

      <!-- 向kafka 生产数据需要包 -->
      <dependency&g