Storm的Java实现（简略版）

一个简单的Storm结构通常包括Topology类、Spout类和Bolt类。

Topology类是整个数据流处理的结构，它定义了数据流的拓扑结构，包括Spout和Bolt的连接关系以及数据流的处理逻辑。

Spout类是数据流的起始点，它负责从数据源获取数据并发射到数据流中，通常用于读取外部数据源，例如数据库、消息队列等。

Bolt类是数据流的处理单元，它接收来自Spout或其他Bolt的数据并进行处理，然后将处理结果发送到下游的Bolt或者外部系统。

在一个简单的Storm结构中，Topology类会定义Spout和Bolt的连接关系和处理逻辑，以构建一个完整的数据流处理流程。Spout和Bolt类是Topology类的组成部分，它们之间通过定义的连接关系和数据流进行交互，从而实现数据的处理和传递。

总之，一个简单的Storm结构包括Topology类、Spout类和Bolt类，它们通过定义的连接关系和处理逻辑实现了数据流的处理和传递。

以下是一个对csv文件股票数据进行简单计算的storm项目

数据类型

1.创建项目

2.导入依赖

3.编写Topology类

package org.example;

import backtype.storm.Config;
import backtype.storm.LocalCluster;
import backtype.storm.generated.StormTopology;
import backtype.storm.topology.TopologyBuilder;

public class NumberTopology {
    public static void main(String[] args) {
        //创建任务的拓扑图
        TopologyBuilder topologyBuilder = new TopologyBuilder();
        //设置拓扑关系（Spout）
        topologyBuilder.setSpout("numberSpout",new NumberSpout());
        //设置拓扑关系（Bolt）
        topologyBuilder.setBolt("numberCountBolt",new NumberBolt()).shuffleGrouping("numberSpout");

        //启动Topology
        Config conf =  new Config();
        //创建一个topology
        StormTopology topology = topologyBuilder.createTopology();
        //本地模式启动集群
        LocalCluster localCluster = new LocalCluster();
        localCluster.submitTopology("numberTopology",conf,topology);
    }
}

以上是在本地模式下启动集群，用于简单测试功能

4.编写Spout类

package org.example;

import backtype.storm.spout.SpoutOutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseRichSpout;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Values;

import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentLinkedQueue;

public class NumberSpout extends BaseRichSpout {
    private SpoutOutputCollector collector;
    private ConcurrentLinkedQueue<String> lines;
    private int currentIndex;

    @Override
    public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
        this.collector = collector;
        this.lines = new ConcurrentLinkedQueue<>();

        // 启动一个新的线程来读取CSV文件
        Thread fileReaderThread = new Thread(() -> {
            try {
                // 获取CSV文件路径，可以从配置中读取或者作为参数传递
                String filePath = "C:\\Users\\ASUS\\Desktop\\办公文件\\股票交易模拟器-0.5\\dist\\股票数据1.csv";

                // 用于存储上次读取的行数，初始值为0
                int lastLineCount = 0;

                while (true) {
                    // 读取文件内容
                    List<String> newLines = readNewLines(filePath, lastLineCount);

                    if (!newLines.isEmpty()) {
                        // 将新行添加到队列中
                        lines.addAll(newLines);

                        // 更新上次读取的行数
                        lastLineCount += newLines.size();
                    }

                    // 等待一段时间后再次检查文件
                    Thread.sleep(1000);
                }
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        });

        // 启动文件读取线程
        fileReaderThread.start();

        currentIndex = 0;
    }

    private List<String> readNewLines(String filePath, int lastLineCount) {
        List<String> newLines = new ArrayList<>();

        try (BufferedReader reader = new BufferedReader(new FileReader(filePath))) {
            // 跳过已读取的行数
            for (int i = 0; i < lastLineCount; i++) {
                reader.readLine();
            }

            boolean isFirstLine = true; // 添加标志，判断是否是第一行

            String line;
            while ((line = reader.readLine()) != null) {
                if (isFirstLine) {
                    isFirstLine = false;
                    continue; // 跳过第一行
                }
                newLines.add(line);
            }
        } catch (IOException e) {
            e.printStackTrace();
        }

        return newLines;
    }

    @Override
    public void nextTuple() {
        if (!lines.isEmpty()) {
            String line = lines.poll();
            collector.emit(new Values(line));
        }
    }

    @Override
    public void declareOutputFields(OutputFieldsDeclarer declarer) {
        declarer.declare(new Fields("line"));
    }
}

5.编写Bolt类

package org.example;

import backtype.storm.task.TopologyContext;
import backtype.storm.topology.BasicOutputCollector;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseBasicBolt;
import backtype.storm.tuple.Tuple;

import java.text.DecimalFormat;
import java.util.HashMap;
import java.util.Map;

public class NumberBolt extends BaseBasicBolt {
    private static final int ONE_MINUTE = 60;
    private static final int ONE_DAY = 24 * 60 * 60;

    private Map<String, Integer> transactionCountPerMinute;
    private Map<String, Double> transactionAmountPerMinute;
    private Map<String, Integer> buyTransactionCountPerMinute;
    private Map<String, Integer> sellTransactionCountPerMinute;

    private int totalOrderCount;
    private int totalTransactionCount;
    private double totalTransactionAmount;
    private int totalBuyTransactionCount;
    private int totalSellTransactionCount;
    private long startTime;

    @Override
    public void prepare(Map stormConf, TopologyContext context) {
        transactionCountPerMinute = new HashMap<>();
        transactionAmountPerMinute = new HashMap<>();
        buyTransactionCountPerMinute = new HashMap<>();
        sellTransactionCountPerMinute = new HashMap<>();
        totalOrderCount = 0;
        totalTransactionCount = 0;
        totalTransactionAmount = 0.0;
        totalBuyTransactionCount = 0;
        totalSellTransactionCount = 0;
        startTime = System.currentTimeMillis() / 1000;
    }

    @Override
    public void execute(Tuple input, BasicOutputCollector collector) {
        String line = input.getStringByField("line");
        // 假设数据格式为: 时间戳,交易ID,股票名称,价格,数量,交易类型,所在省份,交易平台,行业
        String[] fields = line.split(",");

        String transactionType = fields[5].trim();
        double amount = Double.parseDouble(fields[3].trim()) * Integer.parseInt(fields[4].trim());
        int quantity = Integer.parseInt(fields[4].trim());

        // 更新统计数据
        updateStatistics(transactionType, amount, quantity);

        // 计算订单已处理速度
        double orderProcessingSpeed = calculateOrderProcessingSpeed();

        // 输出结果
        DecimalFormat decimalFormat = new DecimalFormat("#.00"); // 保留两位小数
        System.out.println("订单已处理速度: " + decimalFormat.format(orderProcessingSpeed) + "条/秒");
        System.out.println("近1分钟交易总金额: " + decimalFormat.format(transactionAmountPerMinute.get(getCurrentMinuteKey())));
        System.out.println("近1分钟交易总数量: " + transactionCountPerMinute.get(getCurrentMinuteKey()));
        System.out.println("当天累计交易总金额: " + decimalFormat.format(totalTransactionAmount));
        System.out.println("当天累计交易总数量: " + totalTransactionCount);
        System.out.println("近1分钟买入交易量: " + buyTransactionCountPerMinute.get(getCurrentMinuteKey()));
        System.out.println("近1分钟卖出交易量: " + sellTransactionCountPerMinute.get(getCurrentMinuteKey()));
        System.out.println("当天累计买入交易量: " + totalBuyTransactionCount);
        System.out.println("当天累计卖出交易量: " + totalSellTransactionCount);
    }

    @Override
    public void declareOutputFields(OutputFieldsDeclarer declarer) {
        // 没有输出字段，因为这个Bolt只负责输出数据
    }

    private void updateStatistics(String transactionType, double amount, int quantity) {
        DecimalFormat decimalFormat = new DecimalFormat("#.00"); // 保留两位小数

        // 更新订单处理量
        totalOrderCount++;

        // 更新近1分钟交易总金额和交易总数量
        String currentMinuteKey = getCurrentMinuteKey();
        transactionAmountPerMinute.put(currentMinuteKey, Double.parseDouble(decimalFormat.format(transactionAmountPerMinute.getOrDefault(currentMinuteKey, 0.0) + amount)));
        transactionCountPerMinute.put(currentMinuteKey, transactionCountPerMinute.getOrDefault(currentMinuteKey, 0) + quantity);

        // 更新当天累计交易总金额和交易总数量
        totalTransactionAmount = Double.parseDouble(decimalFormat.format(totalTransactionAmount + amount));
        totalTransactionCount += quantity;

        // 更新近1分钟买入交易量和卖出交易量
        if (transactionType.equalsIgnoreCase("买入")) {
            buyTransactionCountPerMinute.put(currentMinuteKey, buyTransactionCountPerMinute.getOrDefault(currentMinuteKey, 0) + quantity);
        } else if (transactionType.equalsIgnoreCase("卖出")) {
            sellTransactionCountPerMinute.put(currentMinuteKey, sellTransactionCountPerMinute.getOrDefault(currentMinuteKey, 0) + quantity);
        }

        // 更新当天累计买入交易量和卖出交易量
        if (transactionType.equalsIgnoreCase("买入")) {
            totalBuyTransactionCount += quantity;
        } else if (transactionType.equalsIgnoreCase("卖出")) {
            totalSellTransactionCount += quantity;
        }


    }

    private double calculateOrderProcessingSpeed() {
        long currentTime = System.currentTimeMillis() / 1000;
        long elapsedTime = currentTime - startTime;

        if (elapsedTime > 0) {
            return (double) totalOrderCount / elapsedTime;
        } else {
            return 0.0;
        }
    }

    private String getCurrentMinuteKey() {
        long currentTime = System.currentTimeMillis() / 1000;
        long currentMinute = (currentTime / 60) % ONE_DAY;
        return String.valueOf(currentMinute);
    }
}

6.结果展示