spark wordcount

1.创建maven项目，引入pom依赖

```xml
<dependencies>
 <dependency>
 <groupId>org.apache.spark</groupId>
 <artifactId>spark-core_2.12</artifactId>
 <version>3.0.0</version>
 </dependency>
</dependencies>
<build>
 <plugins>
 <!-- 该插件用于将 Scala 代码编译成 class 文件 -->
 <plugin>
 <groupId>net.alchim31.maven</groupId>
 <artifactId>scala-maven-plugin</artifactId>
 <version>3.2.2</version>
 <executions>
 <execution>
 <!-- 声明绑定到 maven 的 compile 阶段 -->
 <goals>
 <goal>testCompile</goal>
 </goals>
 </execution>
 </executions>
 </plugin>
 <plugin>
 <groupId>org.apache.maven.plugins</groupId>
 <artifactId>maven-assembly-plugin</artifactId>
 <version>3.1.0</version>
 <configuration>
 <descriptorRefs>
 <descriptorRef>jar-with-dependencies</descriptorRef>
 </descriptorRefs>
 </configuration>
 <executions>
 <execution>
 <id>make-assembly</id>
 <phase>package</phase>
 <goals>
 <goal>single</goal>
 </goals>
 </execution>
 </executions>
 </plugin>
 </plugins>
</build>

2.log4j.properties

log4j.rootCategory=ERROR, console
log4j.appender.console=org.apache.log4j.ConsoleAppender
log4j.appender.console.target=System.err
log4j.appender.console.layout=org.apache.log4j.PatternLayout
log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd 
HH:mm:ss} %p %c{1}: %m%n
# Set the default spark-shell log level to ERROR. When running the spark-shell,
the
# log level for this class is used to overwrite the root logger's log level, so
that
# the user can have different defaults for the shell and regular Spark apps.
log4j.logger.org.apache.spark.repl.Main=ERROR
# Settings to quiet third party logs that are too verbose
log4j.logger.org.spark_project.jetty=ERROR
log4j.logger.org.spark_project.jetty.util.component.AbstractLifeCycle=ERROR
log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=ERROR
log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=ERROR
log4j.logger.org.apache.parquet=ERROR
log4j.logger.parquet=ERROR
# SPARK-9183: Settings to avoid annoying messages when looking up nonexistent
UDFs in SparkSQL with Hive support
log4j.logger.org.apache.hadoop.hive.metastore.RetryingHMSHandler=FATAL
log4j.logger.org.apache.hadoop.hive.ql.exec.FunctionRegistry=ERROR

3.mian

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
object sparkWC {
  def main(args: Array[String]): Unit = {
    //创建spark运行配置对象
    val sparkConf=new SparkConf()
    sparkConf.setAppName("wordCount01")
    sparkConf.setMaster("local")
    //创建上下文连接对象
    val sc = new SparkContext(sparkConf)
    //读取文件数据
    val fileRDD=sc.textFile("D:\\BigData\\spark\\wordCount\\src\\main\\word\\hello")
    val value: RDD[String] = fileRDD.flatMap(_.split(" "))
    val value1: RDD[(String, Int)] = value.map((_, 1))
    val value2: RDD[(String, Int)] = value1.reduceByKey((v1: Int, v2: Int) => {
      v1 + v2
    })
    value2.foreach(println)
  }
}