spark rdd mysql_Spark RDD 数据到 MySQL

数据

www.xzdream.cn 1 2 江西

www.xzdream.cn 3 4 广东

www.xzdream.cn 1 2 西藏

www.xzdream.cn 3 4 浙江

将日志文件 put 到hdfs

package com.xzdream.spark

import java.sql.DriverManager

import org.apache.spark.{SparkConf, SparkContext}

/**

* Log App

*/

object SparkContextApp {

def main(args: Array[String]): Unit = {

val sparkConf = new SparkConf();

sparkConf.setAppName("LogApp").setMaster("local[2]");

val sc = new SparkContext(sparkConf);

//file:///Users/hadoop/scala/spark_demo1/src/main/logs/2020-5-11.log

val file_path = args(0);

val lines = sc.textFile(file_path);

// lines.take(3).foreach(println)

/*

//读取每一行数据

lines.map(x => {

val splits = x.split("\t")

val length = splits.length

if(length == 4){

val domain = splits(0)

var traffic = 0L

try{

traffic = splits(1).toLong

}catch {

case e:Exception => 0L

}

(domain,traffic)

}else{

("-",0L)

}

}).reduceByKey(_+_).collect.foreach(println)

*/

//求省份访问量的top10

val res = lines.map(x=>{

val splits = x.split("\t")

val length = splits.length

if(length == 4){

val province = splits(3)

(province,1)

}else{

('-',1)

}

}).reduceByKey(_+_).sortBy(_._2,false)

// .take(10)

//保存到数据库

// Class.forName("com.mysql.jdbc.Driver")

res.foreachPartition(x=>{

var conn = DriverManager.getConnection("jdbc:mysql://127.0.0.1:3306/hive_db?user=root&password=123456&useUnicode=true&characterEncoding=UTF-8")

conn.setAutoCommit(false)

val sql = "insert into py(name) values(?)"

var pstmt = conn.prepareStatement(sql)

x.foreach{case (name,count)=>{

pstmt.setString(1,name.toString)

pstmt.addBatch()

}}

pstmt.executeBatch()

conn.commit()

conn.close()

})

sc.stop();

}

}

4.0.0

com.xzdream.spark

spark_demo1

1.0

2008

2.11.8

scala-tools.org

Scala-Tools Maven2 Repository

http://scala-tools.org/repo-releases

scala-tools.org

Scala-Tools Maven2 Repository

http://scala-tools.org/repo-releases

org.scala-lang

scala-library

${scala.version}

org.apache.spark

spark-core_2.11

2.4.0

mysql

mysql-connector-java

5.1.27

src/main/scala

src/test/scala

org.scala-tools

maven-scala-plugin

compile

testCompile

${scala.version}

-target:jvm-1.5

org.apache.maven.plugins

maven-eclipse-plugin

true

ch.epfl.lamp.sdt.core.scalabuilder

ch.epfl.lamp.sdt.core.scalanature

org.eclipse.jdt.launching.JRE_CONTAINER

ch.epfl.lamp.sdt.launching.SCALA_CONTAINER

org.scala-tools

maven-scala-plugin

${scala.version}

提交任务

./spark-submit --master yarn --class com.xzdream.spark.SparkContextApp /Users/hadoop/scala/spark_demo1/target/spark_demo1-1.0.jar hdfs:///spark/logs