前期的准备
Eclipse安装Scala IDE,为Spark开发测试准备 ,请先安装Scala IDE。
新建项目
转换为maven项目:
配置pom.xml
<dependencies>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.12</artifactId>
<version>3.0.1</version>
</dependency>
</dependencies>
<build>
<sourceDirectory>src</sourceDirectory>
<plugins>
<pluginManagement>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.0</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
<version>3.4.6</version>
<executions>
<execution>
<goals>
<goal>compile</goal>
<goal>testCompile</goal>
</goals>
</execution>
</executions>
</plugin>
</pluginManagement>
</plugins>
</build>
新建scala object
编辑代码
object WordCount {
def main(args:Array[String]){
// 1. 创建 SparkConf对象, 并设置 App名字
val conf:SparkConf = new SparkConf().setMaster("local[*]").setAppName("WordCount");
// 2. 创建SparkContext对象
val sc = new SparkContext(conf);
// 3. 使用sc创建RDD并执行操作
val res = sc.textFile("D:\\result-nnvad-2020-07-16-13-40-55.txt")
.flatMap(_.split(" "))
.map((_, 1))
.reduceByKey(_ + _).collect();
//4.循环打印
res.foreach(println);
//5.停止
sc.stop();
}
}