netcat for windows
链接:https://pan.baidu.com/s/1crvMSL4JS_ZTCcFr9CPc9w
提取码:o2yx
因为代码是在windows 写的local模式 所以得下载一个netcat工具作为端口测试工具
package com.scala
import java.sql.DriverManager
import org.apache.spark.SparkConf
import org.apache.spark.streaming.{Seconds, StreamingContext}
object ForeachRDDApp {
def main(args: Array[String]): Unit = {
val sparkConf = new SparkConf().setAppName("ForeachRDDApp").setMaster("local[2]")
val ssc = new StreamingContext(sparkConf, Seconds(5))
val lines = ssc.socketTextStream("localhost", 9998)
val result = lines.flatMap(_.split(" ")).map((_, 1)).reduceByKey(_+_)
result.print()
result.foreachRDD { rdd =>
rdd.foreachPartition { partitionOfRecords =>
// ConnectionPool is a static, lazily initialized pool of connections
val connection = createConnection()
partitionOfRecords.foreach(record => {
var sql = "insert into wordcount(word, wordcount) values('" + record._1 + "'," + record._2 + ")"
connection.createStatement().execute(sql)
})
connection.close()
}
}
ssc.start()
ssc.awaitTermination()
}
def createConnection() = {
Class.forName("com.mysql.jdbc.Driver")
DriverManager.getConnection("jdbc:mysql://localhost:3306/test_spark","root","root")
}
}
然后再用 nc -L -p 9998 -v 才能连接这个程序 看日志说 如果不先启动监听程序 有可能会报错 我之后会在验证