【基于Spark-core模块完成项目分析】

Spark-core模块完成项目分析

需求如下

• 年龄对合同签订的影响
• 个人贷款对合同签订的影响
• 受教育情况对合同签订的影响

数据库中的创建表

create table `age_constrate_stat`(
`age` varchar(20) default null,
`num` int(11) default null
);

create table `loan_constrate_stat`(
`loan` varchar(20) default null,
`num` int(11) default null
);

create table `edu_constrate_stat`(
`education` varchar(20) default null,
`num` int(11) default null
);

返回目录

idea开发Spark Core代码

DbUtil将处理后的数据导入MySQL

import java.sql.{Connection, DriverManager, PreparedStatement}

/**
 * @Classname DbUtil
 * @Description TODO
 * @Date 2022/10/7 19:08
 * @Created by Tiger_Li
 */
object DbUtil {
  def loadConStat2MySQL(ite: Iterator[(String, Int)]): Unit = {
    //    加载驱动
    Class.forName("com.mysql.cj.jdbc.Driver")
    //    创建连接
    var conn: Connection = null;
    //    这个对象是把数据加载到MySQL数据库中
    var ps:PreparedStatement = null;
    val sql = "insert into loan_constrate_stat(loan, num) values(?, ?);"

    try {
      conn = DriverManager.getConnection("jdbc:mysql://node1:3306/bigdata19?createDatabaseIfNotExist=true&useSSL=false", "lh", "Ypassword")
      ite.foreach(t => {
        ps = conn.prepareStatement(sql)
        ps.setString(1, t._1)
        ps.setInt(2, t._2)
        ps.executeUpdate()
      })
    } catch {
      case e:Exception => println(e.getMessage)
    } finally {
      if (ps!=null){
        ps.close();
      }
      if(conn!=null){
        conn.close()
      }
    }
  }

  def ageConStat2MySQL(ite: Iterator[(Int, Int)]): Unit = {
//    加载驱动
    Class.forName("com.mysql.cj.jdbc.Driver")
//    创建连接
    var conn: Connection = null;
//    这个对象是把数据加载到MySQL数据库中
    var ps:PreparedStatement = null;
    val sql = "insert into age_constrate_stat(age, num) values(?, ?);"

    try {
      conn = DriverManager.getConnection("jdbc:mysql://node1:3306/bigdata19?createDatabaseIfNotExist=true&useSSL=false", "lh", "Ypassword")
      ite.foreach(t => {
        ps = conn.prepareStatement(sql)
        ps.setInt(1, t._1)
        ps.setInt(2, t._2)
        ps.executeUpdate()
      })
    } catch {
      case e:Exception => println(e.getMessage)
    } finally {
      if (ps!=null){
        ps.close();
      }
      if(conn!=null){
        conn.close()
      }
    }

  }


}

返回目录

Age2ConStat年龄对营销的影响

import org.apache.spark.SparkConf
import org.apache.spark.sql.SparkSession

/**
 * @Classname Age2ConStat
 * @Description 年龄对营销的分析
 * @Date 2022/10/7 16:42
 * @Created by Tiger_Li
 */

/**
 * 年龄对营销的影响
 */
object Age2ConStat {

  def main(args:Array[String]):Unit = {
//    1. 创建SparkConf并配置master和appname
    val conf = new SparkConf().setAppName(s"${this.getClass.getSimpleName}").setMaster("spark://node1:7077")
//    2. 创建SparkSession并配置Hive支持
    val spark = SparkSession.builder().enableHiveSupport().config(conf).getOrCreate()
//    3. 读取hive数据源
    val dataSource = spark.sql("select age, poutcome from dwd.dwd_t_bank")
//    4. 转化为RDD
    import spark.implicits._
    val dataRDD = dataSource.map(row =>{
      val age = row.getAs[Int]("age")
      val poutcome = row.getAs[String]("poutcome")
      (age, poutcome)
    }).rdd
//    5. 过滤RDD
    val filterRDD = dataRDD.filter(t => t._1 > 0 && t._2 == "success")
//    6. 转化RDD
    val mapRDD = filterRDD.map(t => (t._1, 1))
//    7. 分组聚合
    val reducedRDD = mapRDD.reduceByKey(_ + _)
//    8. 写入MySQL
    reducedRDD.foreachPartition(iter => {DbUtil.ageConStat2MySQL(iter)})
//    9. 释放资源
    spark.stop()
  }

}

返回目录

LoanOnConStat贷款对营销的影响

import org.apache.spark.SparkConf
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{DataFrame, SparkSession}

/**
 * @Classname LoanOnConStat
 * @Description TODO
// * @Date 2022/10/7 21:25
 * @Created by Tiger_Li
 */
object LoanOnConStat {

  def main(args: Array[String]): Unit = {
//    1.创建SparkConf并配置master和appname
    val conf: SparkConf = new SparkConf().setAppName(s"${this.getClass.getSimpleName}").setMaster("spark://node1:7077")
//    2. 创建SparkSession并配置hive支持
    val spark: SparkSession = SparkSession.builder().enableHiveSupport().config(conf).getOrCreate()
//    3. 读取hive数据源
    val dataFrame: DataFrame = spark.sql("select loan, poutcome from dwd.dwd_t_bank where loan != '' and loan is not null;")
//    4. 转化RDD
    import spark.implicits._
    val rdddata: RDD[(String, String)] = dataFrame.map(row => {
      val loan = row.getAs[String]("loan")
      val poutcome = row.getAs[String]("poutcome")
      (loan, poutcome)
    }).rdd
//    5. 过滤RDD
    val filterRDD: RDD[(String, String)] = rdddata.filter(_._2 == "success")
//    6. 转化RDD
    val mapedrdd: RDD[(String, Int)] = filterRDD.map(t => (t._1, 1))
//    7. 分组聚合
    val reduceRDD: RDD[(String, Int)] = mapedrdd.reduceByKey(_ + _)
//    8. 写入MySQL
    reduceRDD.foreachPartition(ite => {DbUtil.loadConStat2MySQL(ite)})
  }

}

返回目录

EduOnConStat

import org.apache.spark.SparkConf
import org.apache.spark.sql.{DataFrame, SparkSession}

/**
 * @Classname EduOnConStat
 * @Description 教育对营销的分析
 * @Date 2022/10/7 16:42
 * @Created by Tiger_Li
 */

/**
 * 教育对营销的影响
 */
object EduOnConStat {

  def main(args:Array[String]):Unit = {

      if(args.length != 2){
          println("Usage: spark-submit * *.jar masterurl SaveMode[overwrite|append|ignore]")
          System.exit(1)
      }

//    1. 创建SparkConf并配置master和appname
    val MasterUrl = args(0)
    val mySaveMode = args(1)
    val conf = new SparkConf().setAppName(s"${this.getClass.getSimpleName}").setMaster(MasterUrl)
//    2. 创建SparkSession并配置Hive支持
    val spark = SparkSession.builder().enableHiveSupport().config(conf).getOrCreate()
//    3. 读取hive数据源
    val dataSource = spark.sql("select education, poutcome from dwd.dwd_t_bank where education !='' and education is not null")

    val filterdf = dataSource.filter("poutcome == 'success'")
    val resdf: DataFrame = filterdf.groupBy("education").agg("education"->"count").withColumnRenamed("count(education)", "num")


    val url="jdbc:mysql://node1:3306/bigdata19?useUnicode=true&characterEncoding=UTF-8&serverTimezone=UTC&useSSL=false"
    val table="edu_constrate_stat"
    val prop=new java.util.Properties()
    prop.put("driver","com.mysql.cj.jdbc.Driver")
    prop.put("user","lh")
    prop.put("password","Ypassword")
    resdf.write.mode(mySaveMode).jdbc(url,table,prop)

    spark.stop()
  }

}

返回目录

部署运行

vim product_market_stat.sh

#! /bin/bash
# Age2ConStatv2
/opt/soft_installed/spark-2.4.8-bin-hadoop2.7/bin/spark-submit \
--class Age2ConStatv2 \
--master yarn \
--deploy-mode cluster \
--driver-memory 512m \
--driver-cores 1 \
--executor-memory 512m \
--num-executors 1 \
--executor-cores 1 \
--driver-class-path /home/lh/softs/mysql-connector-java-8.0.30.jar \
/home/lh/hadooptest/jars/GUNbank-V2-jar-with-dependencies.jar \
spark://node1:7077 overwrite 


# EduOnConStat
opt/soft_installed/spark-2.4.8-bin-hadoop2.7/bin/spark-submit \
--name eduStatSParkcore \
--class EduOnConStat \
--master yarn \
--deploy-mode cluster \
--driver-memory 512m \
--driver-cores 1 \
--executor-memory 512m \
--num-executors 1 \
--executor-cores 1 \
--driver-class-path /home/lh/softs/mysql-connector-java-8.0.30.jar \
/home/lh/hadooptest/jars/GUNbank-V2-jar-with-dependencies.jar \
spark://node1:7077 overwrite


# loanOnConStat
opt/soft_installed/spark-2.4.8-bin-hadoop2.7/bin/spark-submit \
--name myloanOnConStat \
--class loanOnConStat \
--master yarn \
--deploy-mode cluster \
--driver-memory 512m \
--driver-cores 1 \
--executor-memory 512m \
--num-executors 1 \
--executor-cores 1 \
--driver-class-path /home/lh/softs/mysql-connector-java-8.0.30.jar \
/home/lh/hadooptest/jars/GUNbank-V2-jar-with-dependencies.jar \
spark://node1:7077 overwrite

返回目录

案例

返回目录