学期简单代码

```java
package stream.mysql

import java.sql.DriverManager

import org.apache.kafka.clients.consumer.{ConsumerConfig, ConsumerRecord}
import org.apache.spark.streaming.dstream.InputDStream
import org.apache.spark.streaming.kafka010.{ConsumerStrategies, KafkaUtils, LocationStrategies}
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.{SparkConf, SparkContext}

object InputMysqlWithKafka {

  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setAppName("ss_kafka_tomysql").setMaster("local[2]")

    val context = new SparkContext(conf)

    context.setLogLevel("WARN")

    val sc = new StreamingContext(context, Seconds(3))

    val kafkaParams = Map(
      ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG -> "master:9092,worker1:9092,worker2:9092",
      ConsumerConfig.GROUP_ID_CONFIG -> "streamingKafka",
      "key.deserializer" -> "org.apache.kafka.common.serialization.StringDeserializer",
      "value.deserializer" -> "org.apache.kafka.common.serialization.StringDeserializer"
    )

    val dStream: InputDStream[ConsumerRecord[String, String]] = KafkaUtils.createDirectStream(
      sc,
      LocationStrategies.PreferConsistent, //位置策略
      ConsumerStrategies.Subscribe(Set("test"), kafkaParams) // 消费策略：主题列表与连接kafka的参数信息
    )

    val map = dStream.map(_.value())

    val wordAndOne = map.flatMap(_.split(" "))

    // 将每个单词进行标记
    val value = wordAndOne.map(x => (x, 1))

    // 统计每个单词出现的次数
    val res = value.reduceByKey(_ + _)
    res.print()
    //    将结果输出到mysql中
    // rdd：表示统计之后的数据
    res.foreachRDD(
      rdd => {
        // 使用foreachPartitions将数据写入iteration
        rdd.foreachPartition(
          partition => {
            var url = "jdbc:mysql://banbang.ren:3306/spark_assi"
            var userName = "testuser"
            var passwd = "87654321"
            // 加载驱动
            Class.forName("com.mysql.cj.jdbc.Driver")
            val conn = DriverManager.getConnection(url, userName, passwd)
            print("连接成功")
            conn.setAutoCommit(false)
            val statement = conn.createStatement()

            // 遍历获取的数据写入数据库
            partition.foreach(record => {
              statement.addBatch("insert into Count values('" + record._1 + "','" + record._2 + "')")
            })

            // 执行并提交
            statement.executeBatch()
            conn.commit()
            conn.close()
            print("------------- 写入")

          }
        )
      }
    )
    sc.start()
    sc.awaitTermination()

    sc.stop()
  }
}

./kafka-topics.sh --bootstrap-server master:9092,worker1:9092,worker2:9092 --create --topic test --partitions 3 --replication-factor 3
生产者
./kafka-console-producer.sh --bootstrap-server master:9092,worker1:9092,worker2:9092 --topic test

import java.io.IOException;
import java.net.URI;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class 代码模板 {
    public static void main(String[] args) throws Exception {
        System.setProperty("hadoop.home.dir", "C:\\Users\\PZ_EDU\\Desktop\\资料\\hadoop-2.9.2");
        Configuration conf = new Configuration();
        //获取集群访问路径
        FileSystem fileSystem = FileSystem.get(conf);
        //判断输出路径是否已经存在，如果存在则删除
        Path outPath = new Path("C:\\Users\\PZ_EDU\\Desktop\\资料\\test");
        if (fileSystem.exists(outPath))
            fileSystem.delete(outPath, true);
        //生成job，并指定job的名称
        Job job = Job.getInstance(conf, "exam");
        //指定打成jar后的运行类
        job.setJarByClass(代码模板.class);
        //指定mapper类
        job.setMapperClass(MyMapper.class);
        //指定mapper的输出类型
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(LongWritable.class);
        //指定reducer类 
        job.setReducerClass(MyReducer.class);
        //指定reduce的输出类型 
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(LongWritable.class);
        //数据的输入目录
        FileInputFormat.addInputPath(job, new Path("C:\\Users\\PZ_EDU\\Desktop\\资料\\数据集"));
        //数据的输出目录
        FileOutputFormat.setOutputPath(job, outPath);
        System.exit(job.waitForCompletion(true) ? 0 : 1);
    }

    static class MyMapper extends Mapper<LongWritable, Text, Text, LongWritable> {
        @Override
        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            String[] line = value.toString().split(",");

//            context.write(new Text(line[0]),new LongWritable(1));
            try {
                context.write(new Text(line[0]), new LongWritable(Long.parseLong(line[8])));
//                System.out.println(line[8]);
            }catch (Exception e){
                return;
            }
        }
    }


    static class MyReducer extends Reducer<Text, LongWritable, Text, LongWritable> {
        @Override
        protected void reduce(Text key, Iterable<LongWritable> values, Context context) throws IOException, InterruptedException {
            long sum = 0,count = 0;
            long max = 0;
            long min = 500;
            for (LongWritable value: values){
                sum+= value.get();
                if (max<value.get()){
                    max = value.get();
                }
                if (min>value.get()){
                    min = value.get();
                }
                count++;
            }
//            context.write(key,new LongWritable(sum));
//            System.out.println(count);
            System.out.println(max+"-----"+min);
            context.write(key,new LongWritable(sum/count));
            context.write(key,new LongWritable(max));
            context.write(key,new LongWritable(min));
        }
    }
}

# 从flask模块中导入Flask类和render_template函数
from flask import Flask, render_template, jsonify
import pymysql

# 创建一个Flask实例
app = Flask(__name__, static_folder="static",
            template_folder="templates")


@app.route("/")
def index():
    return render_template("index.html")


@app.route("/a1")
def a1():
    conn = pymysql.connect(
        host="localhost",
        user="root",
        password="baidu123",
        database="test"
    )
    cursor = conn.cursor()
    cursor.execute("select * from tb_example")
    data = cursor.fetchall()
    exam_map = {"year":[],"num":[]}
    for row in data:
        exam_map.get("year").append(row[0])
        exam_map.get("num").append(row[1])
    return jsonify(exam_map)



if __name__ == "__main__":
    app.run(host="0.0.0.0", port=8080)

<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <title>Title</title>
    <!--引入echarts.js -->
    <script src="../static/echarts.js"></script>
    <!--引入jquery-->
    <script src="../static/jquery-1.8.3.min.js"></script>
</head>
<body>
<div id="main" style="width:1400px;height:600px;"></div>
<script type="text/javascript">
    var myChart = echarts.init(document.getElementById('main'));
    $.get("/a1").done(function (data) {
        myChart.setOption({
  xAxis: {
    data: data.id
  },
  yAxis: {},
  series: [
    {
      type: 'bar',
      data: data.sum
    }
  ]
        })
    })
</script>
</body>
</html>