一个简单的Storm结构通常包括Topology类、Spout类和Bolt类。
Topology类是整个数据流处理的结构,它定义了数据流的拓扑结构,包括Spout和Bolt的连接关系以及数据流的处理逻辑。
Spout类是数据流的起始点,它负责从数据源获取数据并发射到数据流中,通常用于读取外部数据源,例如数据库、消息队列等。
Bolt类是数据流的处理单元,它接收来自Spout或其他Bolt的数据并进行处理,然后将处理结果发送到下游的Bolt或者外部系统。
在一个简单的Storm结构中,Topology类会定义Spout和Bolt的连接关系和处理逻辑,以构建一个完整的数据流处理流程。Spout和Bolt类是Topology类的组成部分,它们之间通过定义的连接关系和数据流进行交互,从而实现数据的处理和传递。
总之,一个简单的Storm结构包括Topology类、Spout类和Bolt类,它们通过定义的连接关系和处理逻辑实现了数据流的处理和传递。
以下是一个对csv文件股票数据进行简单计算的storm项目
数据类型
1.创建项目
2.导入依赖
3.编写Topology类
package org.example;
import backtype.storm.Config;
import backtype.storm.LocalCluster;
import backtype.storm.generated.StormTopology;
import backtype.storm.topology.TopologyBuilder;
public class NumberTopology {
public static void main(String[] args) {
//创建任务的拓扑图
TopologyBuilder topologyBuilder = new TopologyBuilder();
//设置拓扑关系(Spout)
topologyBuilder.setSpout("numberSpout",new NumberSpout());
//设置拓扑关系(Bolt)
topologyBuilder.setBolt("numberCountBolt",new NumberBolt()).shuffleGrouping("numberSpout");
//启动Topology
Config conf = new Config();
//创建一个topology
StormTopology topology = topologyBuilder.createTopology();
//本地模式启动集群
LocalCluster localCluster = new LocalCluster();
localCluster.submitTopology("numberTopology",conf,topology);
}
}
以上是在本地模式下启动集群,用于简单测试功能
4.编写Spout类
package org.example;
import backtype.storm.spout.SpoutOutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseRichSpout;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Values;
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentLinkedQueue;
public class NumberSpout extends BaseRichSpout {
private SpoutOutputCollector collector;
private ConcurrentLinkedQueue<String> lines;
private int currentIndex;
@Override
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
this.collector = collector;
this.lines = new ConcurrentLinkedQueue<>();
// 启动一个新的线程来读取CSV文件
Thread fileReaderThread = new Thread(() -> {
try {
// 获取CSV文件路径,可以从配置中读取或者作为参数传递
String filePath = "C:\\Users\\ASUS\\Desktop\\办公文件\\股票交易模拟器-0.5\\dist\\股票数据1.csv";
// 用于存储上次读取的行数,初始值为0
int lastLineCount = 0;
while (true) {
// 读取文件内容
List<String> newLines = readNewLines(filePath, lastLineCount);
if (!newLines.isEmpty()) {
// 将新行添加到队列中
lines.addAll(newLines);
// 更新上次读取的行数
lastLineCount += newLines.size();
}
// 等待一段时间后再次检查文件
Thread.sleep(1000);
}
} catch (InterruptedException e) {
e.printStackTrace();
}
});
// 启动文件读取线程
fileReaderThread.start();
currentIndex = 0;
}
private List<String> readNewLines(String filePath, int lastLineCount) {
List<String> newLines = new ArrayList<>();
try (BufferedReader reader = new BufferedReader(new FileReader(filePath))) {
// 跳过已读取的行数
for (int i = 0; i < lastLineCount; i++) {
reader.readLine();
}
boolean isFirstLine = true; // 添加标志,判断是否是第一行
String line;
while ((line = reader.readLine()) != null) {
if (isFirstLine) {
isFirstLine = false;
continue; // 跳过第一行
}
newLines.add(line);
}
} catch (IOException e) {
e.printStackTrace();
}
return newLines;
}
@Override
public void nextTuple() {
if (!lines.isEmpty()) {
String line = lines.poll();
collector.emit(new Values(line));
}
}
@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
declarer.declare(new Fields("line"));
}
}
5.编写Bolt类
package org.example;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.BasicOutputCollector;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseBasicBolt;
import backtype.storm.tuple.Tuple;
import java.text.DecimalFormat;
import java.util.HashMap;
import java.util.Map;
public class NumberBolt extends BaseBasicBolt {
private static final int ONE_MINUTE = 60;
private static final int ONE_DAY = 24 * 60 * 60;
private Map<String, Integer> transactionCountPerMinute;
private Map<String, Double> transactionAmountPerMinute;
private Map<String, Integer> buyTransactionCountPerMinute;
private Map<String, Integer> sellTransactionCountPerMinute;
private int totalOrderCount;
private int totalTransactionCount;
private double totalTransactionAmount;
private int totalBuyTransactionCount;
private int totalSellTransactionCount;
private long startTime;
@Override
public void prepare(Map stormConf, TopologyContext context) {
transactionCountPerMinute = new HashMap<>();
transactionAmountPerMinute = new HashMap<>();
buyTransactionCountPerMinute = new HashMap<>();
sellTransactionCountPerMinute = new HashMap<>();
totalOrderCount = 0;
totalTransactionCount = 0;
totalTransactionAmount = 0.0;
totalBuyTransactionCount = 0;
totalSellTransactionCount = 0;
startTime = System.currentTimeMillis() / 1000;
}
@Override
public void execute(Tuple input, BasicOutputCollector collector) {
String line = input.getStringByField("line");
// 假设数据格式为: 时间戳,交易ID,股票名称,价格,数量,交易类型,所在省份,交易平台,行业
String[] fields = line.split(",");
String transactionType = fields[5].trim();
double amount = Double.parseDouble(fields[3].trim()) * Integer.parseInt(fields[4].trim());
int quantity = Integer.parseInt(fields[4].trim());
// 更新统计数据
updateStatistics(transactionType, amount, quantity);
// 计算订单已处理速度
double orderProcessingSpeed = calculateOrderProcessingSpeed();
// 输出结果
DecimalFormat decimalFormat = new DecimalFormat("#.00"); // 保留两位小数
System.out.println("订单已处理速度: " + decimalFormat.format(orderProcessingSpeed) + "条/秒");
System.out.println("近1分钟交易总金额: " + decimalFormat.format(transactionAmountPerMinute.get(getCurrentMinuteKey())));
System.out.println("近1分钟交易总数量: " + transactionCountPerMinute.get(getCurrentMinuteKey()));
System.out.println("当天累计交易总金额: " + decimalFormat.format(totalTransactionAmount));
System.out.println("当天累计交易总数量: " + totalTransactionCount);
System.out.println("近1分钟买入交易量: " + buyTransactionCountPerMinute.get(getCurrentMinuteKey()));
System.out.println("近1分钟卖出交易量: " + sellTransactionCountPerMinute.get(getCurrentMinuteKey()));
System.out.println("当天累计买入交易量: " + totalBuyTransactionCount);
System.out.println("当天累计卖出交易量: " + totalSellTransactionCount);
}
@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
// 没有输出字段,因为这个Bolt只负责输出数据
}
private void updateStatistics(String transactionType, double amount, int quantity) {
DecimalFormat decimalFormat = new DecimalFormat("#.00"); // 保留两位小数
// 更新订单处理量
totalOrderCount++;
// 更新近1分钟交易总金额和交易总数量
String currentMinuteKey = getCurrentMinuteKey();
transactionAmountPerMinute.put(currentMinuteKey, Double.parseDouble(decimalFormat.format(transactionAmountPerMinute.getOrDefault(currentMinuteKey, 0.0) + amount)));
transactionCountPerMinute.put(currentMinuteKey, transactionCountPerMinute.getOrDefault(currentMinuteKey, 0) + quantity);
// 更新当天累计交易总金额和交易总数量
totalTransactionAmount = Double.parseDouble(decimalFormat.format(totalTransactionAmount + amount));
totalTransactionCount += quantity;
// 更新近1分钟买入交易量和卖出交易量
if (transactionType.equalsIgnoreCase("买入")) {
buyTransactionCountPerMinute.put(currentMinuteKey, buyTransactionCountPerMinute.getOrDefault(currentMinuteKey, 0) + quantity);
} else if (transactionType.equalsIgnoreCase("卖出")) {
sellTransactionCountPerMinute.put(currentMinuteKey, sellTransactionCountPerMinute.getOrDefault(currentMinuteKey, 0) + quantity);
}
// 更新当天累计买入交易量和卖出交易量
if (transactionType.equalsIgnoreCase("买入")) {
totalBuyTransactionCount += quantity;
} else if (transactionType.equalsIgnoreCase("卖出")) {
totalSellTransactionCount += quantity;
}
}
private double calculateOrderProcessingSpeed() {
long currentTime = System.currentTimeMillis() / 1000;
long elapsedTime = currentTime - startTime;
if (elapsedTime > 0) {
return (double) totalOrderCount / elapsedTime;
} else {
return 0.0;
}
}
private String getCurrentMinuteKey() {
long currentTime = System.currentTimeMillis() / 1000;
long currentMinute = (currentTime / 60) % ONE_DAY;
return String.valueOf(currentMinute);
}
}
6.结果展示