第1关 QueueStream
package net.educoder;
import org.apache.spark.SparkConf;
import org.apache.spark.streaming.Durations;
import org.apache.spark.streaming.api.java.JavaDStream;
import org.apache.spark.streaming.api.java.JavaStreamingContext;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.concurrent.LinkedBlockingQueue;
public class Step1 {
private static SparkConf conf;
static {
conf = new SparkConf().setMaster("local[*]").setAppName("Step1");
conf.set("spark.streaming.stopGracefullyOnShutdown", "true");
}
public static void main(String[] args) throws InterruptedException {
JavaStreamingContext ssc = new JavaStreamingContext(conf, Durations.seconds(1));
LinkedBlockingQueue queue = QueueStream.queueStream(ssc);
JavaDStream<String> dStream = ssc.queueStream(queue);
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
JavaDStream<String> map = dStream.map(x -> {
String[] split = x.split(",");
String ip = split[0];
String time = simpleDateFormat.format(new Date(new Long(split[1])));
String startUrl = split[2].split(" ")[1];
String targetUrl = split[3];
String statusCode = split[4];
return "Ip:" + ip + ",visitTime:" + time + ",startUrl:" + startUrl + ",targetUrl:" + targetUrl + ",statusCode:" + statusCode;
});
map.foreachRDD(rdd -> {
if (rdd.isEmpty()) {
ssc.stop(false, false);
System.exit(1);
} else {
rdd.foreachPartition(partitionOfRecords -> {
JdbcTools.saveData(partitionOfRecords);
});
}
});
ssc.start();
ssc.awaitTermination();
}
}
第2关 File Streams
package com.educoder;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.VoidFunction;
import org.apache.spark.streaming.Duration;
import org.apache.spark.streaming.Durations;
import org.apache.spark.streaming.api.java.JavaDStream;
import org.apache.spark.streaming.api.java.JavaPairDStream;
import org.apache.spark.streaming.api.java.JavaStreamingContext;
import scala.Tuple2;
import java.io.Serializable;
import java.sql.*;
import java.util.Arrays;
import java.util.Iterator;
public class SparkStreaming {
public static void main(String[] args) throws Exception {
SparkConf conf=new SparkConf().setAppName("edu").setMaster("local");
JavaStreamingContext ssc = new JavaStreamingContext(conf, Durations.seconds(1));
JavaDStream<String> DStream = ssc.textFileStream("file:///root/step11_fils");
JavaPairDStream<String, Integer> wordcount = DStream.flatMap(x -> Arrays.asList(x.split(" ")).iterator())
.mapToPair(x -> new Tuple2<String, Integer>(x, 1))
.reduceByKey((x, y) -> x + y);
wordcount.foreachRDD(rdd->{
rdd.foreachPartition(new VoidFunction<Iterator<Tuple2<String, Integer>>>() {
@Override
public void call(Iterator<Tuple2<String, Integer>> r) throws Exception {
Connection connection= myconn();
while(r.hasNext()){
Tuple2<String, Integer> record = r.next();
String querySql = "SELECT t.count FROM step t WHERE t.word = '" + record._1 + "'";
ResultSet queryResultSet = connection.createStatement().executeQuery(querySql);
Boolean hasNext = queryResultSet.next();
if (!hasNext) {
String insertSql = "insert into step(word,count) values('" + record._1 + "'," + record._2 + ")";
connection.createStatement().execute(insertSql);
} else {
Integer newWordCount = queryResultSet.getInt("count") + record._2;
String updateSql = "UPDATE step SET count = " + newWordCount + " where word = '" + record._1 + "'";
connection.createStatement().execute(updateSql);
}
}
connection.close();
}
});
});
ssc.start();
Thread.sleep(15000);
ssc.stop();
ssc.awaitTermination();
}
public static Connection myconn()throws SQLException,Exception{
Class.forName("com.mysql.jdbc.Driver");
Connection conn= DriverManager.getConnection("jdbc:mysql://localhost:3306/educoder","root","123123");
return conn;
}
}
第3关 socketTextStream
package com;
import java.util.Arrays;
import java.util.List;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.Optional;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.streaming.Durations;
import org.apache.spark.streaming.api.java.JavaDStream;
import org.apache.spark.streaming.api.java.JavaPairDStream;
import org.apache.spark.streaming.api.java.JavaReceiverInputDStream;
import org.apache.spark.streaming.api.java.JavaStreamingContext;
import scala.Tuple2;
import java.sql.Connection;
import java.sql.DriverManager;
public class JSocketSpark {
public static void main(String[] args) throws InterruptedException{
SparkConf conf = new SparkConf().setAppName("socketSparkStreaming").setMaster("local[*]");
conf.set("spark.streaming.stopGracefullyOnShutdown", "true");
JavaStreamingContext ssc = new JavaStreamingContext(conf, Durations.seconds(2));
JavaReceiverInputDStream<String> lines = ssc.socketTextStream("localhost", 5566);
JavaDStream<String> rdd1 = lines.flatMap(x -> Arrays.asList(x.split(" ")).iterator());
JavaPairDStream<String, Integer> rdd2 = rdd1.mapToPair(x -> new Tuple2<>(x, 1));
ssc.checkpoint("/root/check");
JavaPairDStream<String, Integer> rdd3 = rdd2.updateStateByKey(new Function2<List<Integer>, Optional<Integer>, Optional<Integer>>() {
@Override
public Optional<Integer> call(List<Integer> values, Optional<Integer> state)
throws Exception {
Integer updatedValue = 0;
if (state.isPresent()) {
updatedValue = state.get();
}
for (Integer value : values) {
updatedValue += value;
}
return Optional.of(updatedValue);
}
});
rdd3.foreachRDD(rdd -> {
rdd.foreachPartition(x -> {
Connection myconn = myconn();
while (x.hasNext()){
Tuple2<String, Integer> record = x.next();
String sql = "insert into wordcount (word,wordcount) values('" + record._1 + "',"+record._2+") on DUPLICATE key update wordcount="+record._2;
myconn.createStatement().execute(sql);
}
myconn.close();
});
});
ssc.start();
ssc.awaitTermination();
}
public static Connection myconn()throws Exception{
Class.forName("com.mysql.jdbc.Driver");
Connection conn= DriverManager.getConnection("jdbc:mysql://localhost:3306/edu","root","123123");
return conn;
}
}
第4关 KafkaStreaming
package net.educoder;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.apache.spark.SparkConf;
import org.apache.spark.streaming.Durations;
import org.apache.spark.streaming.api.java.JavaDStream;
import org.apache.spark.streaming.api.java.JavaInputDStream;
import org.apache.spark.streaming.api.java.JavaStreamingContext;
import org.apache.spark.streaming.kafka010.ConsumerStrategies;
import org.apache.spark.streaming.kafka010.KafkaUtils;
import org.apache.spark.streaming.kafka010.LocationStrategies;
import java.text.SimpleDateFormat;
import java.util.*;
public class Step2 {
private static SparkConf conf;
static {
conf = new SparkConf().setMaster("local[*]").setAppName("Step2");
conf.set("spark.streaming.stopGracefullyOnShutdown", "true");
}
public static void main(String[] args) throws InterruptedException {
Map<String, Object> kafkaParams = new HashMap<>();
kafkaParams.put("bootstrap.servers", "127.0.0.1:9092");
kafkaParams.put("key.deserializer", StringDeserializer.class);
kafkaParams.put("value.deserializer", StringDeserializer.class);
kafkaParams.put("group.id", "sparkStreaming");
kafkaParams.put("enable.auto.commit", "false");
TopicPartition topicPartition = new TopicPartition("test", 0);
List<TopicPartition> topicPartitions = Arrays.asList(topicPartition);
HashMap<TopicPartition, Long> offsets = new HashMap<>();
offsets.put(topicPartition, 0l);
JavaStreamingContext ssc = new JavaStreamingContext(conf, Durations.seconds(1));
JavaInputDStream<ConsumerRecord<String, String>> javaInputDStream = KafkaUtils.createDirectStream(
ssc,
LocationStrategies.PreferConsistent(),
ConsumerStrategies.Assign(topicPartitions, kafkaParams, offsets));
JavaDStream<String> dStream = javaInputDStream.map(x -> x.value());
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
JavaDStream<String> map = dStream.map(x -> {
String[] split = x.split(",");
String ip = split[0];
String time = simpleDateFormat.format(new Date(new Long(split[1])));
String startUrl = split[2].split(" ")[1];
String targetUrl = split[3];
String statusCode = split[4];
return "Ip:" + ip + ",visitTime:" + time + ",startUrl:" + startUrl + ",targetUrl:" + targetUrl + ",statusCode:" + statusCode;
});
map.foreachRDD(rdd -> {
if (rdd.isEmpty()) {
ssc.stop(false, false);
System.exit(0);
} else {
rdd.foreachPartition(partitionOfRecords -> {
JdbcTools.saveData2(partitionOfRecords);
});
}
});
ssc.start();
ssc.awaitTermination();
}
}