大数据开发之spark基础

spark

      一、产生背景

           迭代式计算

           交互式数据挖掘

      二、安装部署

           1、下载安装包

           2、上传到linux

           3、解压

           4、配置

                 conf

                      复制cp slaves.template slaves

                            hlh002

hlh003

                      cp spark-env.sh.template spark-env.sh

                            SPARK_MASTER_HOST=hlh001

SPARK_MASTER_PORT=7077

                 启动

                      ./start-all.sh

           5、浏览器查看

                 http://hlh001:8080/

           6、配置job history server

                 cp spark-defaults.conf.template spark-defaults.conf

                      spark.master                     spark://hlh001:7077

spark.eventLog.enabled           true

spark.eventLog.dir           hdfs://namenode:9000/directory

                 修改spark-env.sh

                      export SPARK_HISTORY_OPTS="-Dspark.history.ui.port=4000 -Dspark.history.retainedApplications=3 -Dspark.history.fs.logDirectory=hdfs://hlh001:9000/directory"

                 [root@hlh001 hadoop-2.7.2]# hadoop fs -mkdir /directory

                 启动

                      [root@hlh001 sbin]# ./start-all.sh

                      启动hadoop

                      [root@hlh001 sbin]# ./start-history-server.sh

      三、执行程序

           利用蒙特•卡罗算法求PI

                 [root@hlh001 spark-2.1.1-bin-hadoop2.7]# ./bin/spark-submit --class org.apache.spark.examples.SparkPi --master spark://hlh001:7077 examples/jars/spark-examples_2.11-2.1.1.jar 100

                 [root@hlh001 conf]# cp log4j.properties.template log4j.properties

                      log4j.rootCategory=WARN, console

           spark shell Wordcount

                 [root@hlh001 hadoop-2.7.2]# ./bin/hdfs dfs -put ./README.txt /

                 [root@hlh001 bin]# ./spark-shell

                 scala> sc.textFile("hdfs://hlh001:9000/README.txt").flatMap(_.split(" ")).map((_,1)).reduceByKey(_+_).saveAsTextFile("hdfs://hlh001:9000/out")

                 查看

                      [root@hlh001 hadoop-2.7.2]# ./bin/hdfs dfs -ls /

                      [root@hlh001 hadoop-2.7.2]# ./bin/hdfs dfs -cat /out/p*