idea配置spark的java开发环境

1. 建立spring boot项目。Controller能够输出。搭建过程请参考我的其他博文。

2、指定spark版本：

<properties>
    <java.version>1.8</java.version>
    <spark.version>3.1.2</spark.version>
    <scala.version>2.12</scala.version>
</properties>

3、添加依赖

<dependency>
    <groupId>org.apache.spark</groupId>
    <artifactId>spark-core_${scala.version}</artifactId>
    <version>${spark.version}</version>
</dependency>
<dependency>
    <groupId>org.apache.spark</groupId>
    <artifactId>spark-streaming_${scala.version}</artifactId>
    <version>${spark.version}</version>
</dependency>
<dependency>
    <groupId>org.apache.spark</groupId>
    <artifactId>spark-sql_${scala.version}</artifactId>
    <version>${spark.version}</version>
</dependency>
<dependency>
    <groupId>org.apache.spark</groupId>
    <artifactId>spark-hive_${scala.version}</artifactId>
    <version>${spark.version}</version>
</dependency>
<dependency>
    <groupId>org.apache.spark</groupId>
    <artifactId>spark-mllib_${scala.version}</artifactId>
    <version>${spark.version}</version>
</dependency>
<dependency>
    <groupId>mysql</groupId>
    <artifactId>mysql-connector-java</artifactId>
    <version>5.1.24</version>
</dependency>
<dependency>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-common</artifactId>
    <version>3.3.1</version>
</dependency>
<dependency>
    <groupId>net.sf.json-lib</groupId>
    <artifactId>json-lib</artifactId>
    <version>2.4</version>
    <classifier>jdk15</classifier>
</dependency>

<dependency>
    <groupId>org.slf4j</groupId>
    <artifactId>slf4j-log4j12</artifactId>
    <version>1.7.21</version>
</dependency>

<dependency>
    <groupId>org.codehaus.janino</groupId>
    <artifactId>janino</artifactId>
    <version>3.0.8</version>
</dependency>

4、刷新maven，下载相关的依赖。这个配套是我搭建过程中测试过的，在spark3.1.2条件下运行成功。

5、添加以下插件：

<plugin>
    <groupId>org.scala-tools</groupId>
    <artifactId>maven-scala-plugin</artifactId>
    <version>2.12.2</version>
    <executions>
        <execution>
            <goals>
                <goal>compile</goal>
                <goal>testCompile</goal>
            </goals>
        </execution>
    </executions>
</plugin>

<plugin>
    <artifactId>maven-compiler-plugin</artifactId>
    <version>3.6.0</version>
    <configuration>
        <source>1.8</source>
        <target>1.8</target>
    </configuration>
</plugin>

<plugin>
    <groupId>org.apache.maven.plugins</groupId>
    <artifactId>maven-surefire-plugin</artifactId>
    <version>2.19</version>
    <configuration>
        <skip>true</skip>
    </configuration>
</plugin>

6、刷新maven，下载插件

7、添加TestController。文件目录结构如下图：

package com.example.demo.controller;

import com.example.demo.service.ITestService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;

import javax.servlet.http.HttpServletRequest;
import java.util.List;
import java.util.Map;
import java.util.Properties;

@RestController
@RequestMapping(value = "/mytest")
public class TestContorller {
    @Autowired
    private ITestService testService;

    @RequestMapping("/list")
    public List<Map<String ,Object>> getMessageList(HttpServletRequest req){
        List<Map<String ,Object>> list=testService.getFlowList();
        return list;
    }
}

8、添加服务接口ITestService

package com.example.demo.service;

import java.util.List;
import java.util.Map;

public interface ITestService {
    List<Map<String, Object>> getFlowList();
}

9、下载并解压hadoop2.6_Win_x64-master，这个版本与上面的pom文件中的版本是一致的

。然后配置环境变量如下：

10、配置完毕后重启系统

11、添加服务TestService

import com.example.demo.service.ITestService;
import net.sf.json.JSONObject;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SparkSession;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;

import java.util.List;
import java.util.Map;

@Service
public class TestService implements ITestService {
    private static final Logger LOGGER = LoggerFactory.getLogger(TestService.class);

    @Override
    public List<Map<String, Object>> getFlowList() {
        SparkSession spark = SparkSession
                .builder()
                .master("local")           
                .appName("stay time")
                .getOrCreate();    //指向本机，本次应用名为stay time
        Dataset<Row> results = spark.read()
                .format("jdbc")
                .option("driver", "com.mysql.jdbc.Driver")
                .option("url", "jdbc:mysql://localhost:3306/数据库名称?useSSL=false&useUnicode=true&characterEncoding=utf-8&serverTimezone=UTC")
                .option("dbtable", "testtable")   //表名
                .option("user", "root")
                .option("password", "123456")
                .load()
                ;
        results.show();  //将数据打印到控制台
        String p="vnvnvnvnvbn";
        return null;
    }
}

11.能在控制台输出行列数据，说明本环境搭建成功