试试这个:
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.ForeachWriter;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SparkSession;
import org.apache.spark.sql.streaming.StreamingQueryException;
import org.apache.spark.sql.streaming.Trigger;
import java.sql.*;
public class test {
public static Connection conn;
public static Statement state;
public static void main(String[] arg) throws StreamingQueryException {
Logger.getLogger("org").setLevel(Level.ERROR);
SparkSession spark = SparkSession
.builder()
.master("local[*]")
.appName("book_view")
.getOrCreate();
Dataset df = spark
.readStream()
.format("kafka")
.option("kafka.bootstrap.servers", "spark01:9092,spark02:9092,spark03:9092")
.option("group.id", "stream")
.option("auto.offset.reset", "latest")
.option("enable.auto.commit", true)
.option("subscribe", "book")
.load();
Dataset rowDataset = df.selectExpr("CAST(value AS STRING)");
Dataset value = rowDataset.groupBy("value").count().as("count");
value.writeStream().foreach(new ForeachWriter() {
@Override
public void process(Row value) {
String sql = "replace INTO `book_view` (`bookname`, `viewcount`) VALUES (?, ?)";
String bookname = value.getString(0);
int viewcount = Integer.parseInt(String.valueOf(value.getLong(1)));
try {
PreparedStatement preparedStatement = conn.prepareStatement(sql);
preparedStatement.setString(1,bookname);
preparedStatement.setInt(2,viewcount);
preparedStatement.executeUpdate();
} catch (SQLException e) {
e.printStackTrace();
}
}
@Override
public void close(Throwable errorOrNull) {
try {
conn.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
@Override
public boolean open(long partitionId, long version) {
try {
Class.forName("com.mysql.jdbc.Driver");
conn = DriverManager.getConnection("jdbc:mysql://192.168.121.134:3306/book?characterEncoding=UTF-8","root","Itcast@2020");
state = conn.createStatement();
} catch (ClassNotFoundException | SQLException e) {
e.printStackTrace();
}
return true;
}
}).outputMode("complete")
.trigger(Trigger.ProcessingTime(0))
.start()
.awaitTermination();
}
}
ion=2.2�mY!R