数据
www.xzdream.cn 1 2 江西
www.xzdream.cn 3 4 广东
www.xzdream.cn 1 2 西藏
www.xzdream.cn 3 4 浙江
将日志文件 put 到hdfs
package com.xzdream.spark
import java.sql.DriverManager
import org.apache.spark.{SparkConf, SparkContext}
/**
* Log App
*/
object SparkContextApp {
def main(args: Array[String]): Unit = {
val sparkConf = new SparkConf();
sparkConf.setAppName("LogApp").setMaster("local[2]");
val sc = new SparkContext(sparkConf);
//file:///Users/hadoop/scala/spark_demo1/src/main/logs/2020-5-11.log
val file_path = args(0);
val lines = sc.textFile(file_path);
// lines.take(3).foreach(println)
/*
//读取每一行数据
lines.map(x => {
val splits = x.split("\t")
val length = splits.length
if(length == 4){
val domain = splits(0)
var traffic = 0L
try{
traffic = splits(1).toLong
}catch {
case e:Exception => 0L
}
(domain,traffic)
}else{
("-",0L)
}
}).reduceByKey(_+_).collect.foreach(println)
*/
//求省份访问量的top10
val res = lines.map(x=>{
val splits = x.split("\t")
val length = splits.length
if(length == 4){
val province = splits(3)
(province,1)
}else{
('-',1)
}
}).reduceByKey(_+_).sortBy(_._2,false)
// .take(10)
//保存到数据库
// Class.forName("com.mysql.jdbc.Driver")
res.foreachPartition(x=>{
var conn = DriverManager.getConnection("jdbc:mysql://127.0.0.1:3306/hive_db?user=root&password=123456&useUnicode=true&characterEncoding=UTF-8")
conn.setAutoCommit(false)
val sql = "insert into py(name) values(?)"
var pstmt = conn.prepareStatement(sql)
x.foreach{case (name,count)=>{
pstmt.setString(1,name.toString)
pstmt.addBatch()
}}
pstmt.executeBatch()
conn.commit()
conn.close()
})
sc.stop();
}
}
4.0.0
com.xzdream.spark
spark_demo1
1.0
2008
2.11.8
scala-tools.org
Scala-Tools Maven2 Repository
http://scala-tools.org/repo-releases
scala-tools.org
Scala-Tools Maven2 Repository
http://scala-tools.org/repo-releases
org.scala-lang
scala-library
${scala.version}
org.apache.spark
spark-core_2.11
2.4.0
mysql
mysql-connector-java
5.1.27
src/main/scala
src/test/scala
org.scala-tools
maven-scala-plugin
compile
testCompile
${scala.version}
-target:jvm-1.5
org.apache.maven.plugins
maven-eclipse-plugin
true
ch.epfl.lamp.sdt.core.scalabuilder
ch.epfl.lamp.sdt.core.scalanature
org.eclipse.jdt.launching.JRE_CONTAINER
ch.epfl.lamp.sdt.launching.SCALA_CONTAINER
org.scala-tools
maven-scala-plugin
${scala.version}
提交任务
./spark-submit --master yarn --class com.xzdream.spark.SparkContextApp /Users/hadoop/scala/spark_demo1/target/spark_demo1-1.0.jar hdfs:///spark/logs