storm的使用

1.spout

package spout;

import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseRichSpout;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Values;
import org.apache.storm.utils.Utils;

import java.io.*;
import java.util.Map;

@SuppressWarnings("serial")
public class FileDataReaderSpout extends BaseRichSpout {
    private SpoutOutputCollector spoutOutputCollector;
    private String filename;
    private BufferedReader bufferedReader;

    @SuppressWarnings("rawtypes")
    public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
        this.spoutOutputCollector = collector;
        this.filename = "股票数据1.csv"; // 替换成你要读取的文件路径
        try {
            this.bufferedReader = new BufferedReader(new InputStreamReader(new FileInputStream(filename), "GBK"));
        } catch (IOException e) {
            throw new RuntimeException("Error reading file: " + filename, e);
        }
    }

    public void nextTuple() {
        try {
            String line = bufferedReader.readLine();
            if (line != null) {
                spoutOutputCollector.emit(new Values(line.trim().toLowerCase()));
            } else {
                Utils.sleep(100); // 如果文件读取完毕，则等待一段时间后再次检查
            }
        } catch (IOException e) {
            throw new RuntimeException("Error reading tuple from file", e);
        }
    }

    public void ack(Object id) {
        // 消息保证机制中的ack确认方法，可以为空
    }

    public void fail(Object id) {
        // 消息保证机制中的fail确认方法，可以为空
    }

    public void declareOutputFields(OutputFieldsDeclarer declarer) {
        declarer.declare(new Fields("word"));
    }

    @Override
    public void close() {
        try {
            bufferedReader.close();
        } catch (IOException e) {
            throw new RuntimeException("Error closing file: " + filename, e);
        }
    }
}

2.bolt

package bolt;

import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Values;
import org.apache.storm.topology.BasicOutputCollector;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseBasicBolt;
import org.apache.storm.tuple.Tuple;

public class OutputBolt extends BaseBasicBolt {

    @Override
    public void execute(Tuple input, BasicOutputCollector collector) {
        String stockType = input.getStringByField("stockType");
        int tradeVolume = input.getIntegerByField("totalTradeVolume");
        double tradeAmount = input.getDoubleByField("totalTradeAmount");

        System.out.println("stock Type: " + stockType + ", Trade Volume: " + tradeVolume + ", Trade Amount: " + tradeAmount);
    }

    @Override
    public void declareOutputFields(OutputFieldsDeclarer declarer) {
        // 不会发射任何数据，因为这个Bolt只是用来输出结果
    }
}

package bolt;

import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Values;
import org.apache.storm.topology.BasicOutputCollector;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseBasicBolt;
import org.apache.storm.tuple.Tuple;

import java.util.HashMap;
import java.util.Map;

public class StockTradeBolt extends BaseBasicBolt {

    private Map<String, Integer> stockTypeTotalVolumeMap;
    private Map<String, Double> stockTypeTotalAmountMap;

    @Override
    public void prepare(Map stormConf, TopologyContext context) {
        stockTypeTotalVolumeMap = new HashMap<>();
        stockTypeTotalAmountMap = new HashMap<>();
    }

    @Override
    public void execute(Tuple tuple, BasicOutputCollector collector) {
        String line = tuple.getString(0);
        String[] fields = line.split(",");

        if (fields.length >= 9 && isNumeric(fields[3]) && isNumeric(fields[4])) {
            String stockType = fields[2]; // 假设股票类型字段在数组中的索引为2
            int tradeVolume = Integer.parseInt(fields[4]);
            double price = Double.parseDouble(fields[3]);
            double tradeAmount = price * tradeVolume;

            stockTypeTotalVolumeMap.merge(stockType, tradeVolume, Integer::sum);
            stockTypeTotalAmountMap.merge(stockType, tradeAmount, Double::sum);
            collector.emit(new Values(stockType, stockTypeTotalVolumeMap.get(stockType), stockTypeTotalAmountMap.get(stockType)));
        } else {
            // 非数字字段的处理逻辑，比如记录日志或者丢弃这条数据
        }
    }

    @Override
    public void declareOutputFields(OutputFieldsDeclarer declarer) {
        declarer.declare(new Fields("stockType", "totalTradeVolume", "totalTradeAmount"));
    }

    @Override
    public void cleanup() {
        for (Map.Entry<String, Integer> entry : stockTypeTotalVolumeMap.entrySet()) {
            String stockType = entry.getKey();
            int totalVolume = entry.getValue();
            double totalAmount = stockTypeTotalAmountMap.get(stockType);
            System.out.println("Stock Type: " + stockType + ", Total Volume: " + totalVolume + ", Total Amount: " + totalAmount);
        }
    }

    private boolean isNumeric(String str) {
        try {
            Double.parseDouble(str);
            return true;
        } catch (NumberFormatException e) {
            return false;
        }
    }
}

package bolt;

import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Values;
import org.apache.storm.topology.BasicOutputCollector;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseBasicBolt;
import org.apache.storm.tuple.Tuple;

public class TimeOutputBolt extends BaseBasicBolt {

    @Override
    public void execute(Tuple input, BasicOutputCollector collector) {
        String stockType = input.getStringByField("stockType");
        int time = input.getIntegerByField("hour");
        int tradeVolume = input.getIntegerByField("hourlyTradeVolume");
        double tradeAmount = input.getDoubleByField("hourlyTradeAmount");

        System.out.println("stock Type: " + stockType + ",hour: "+time+", Trade Volume: " + tradeVolume + ", Trade Amount: " + tradeAmount);
    }

    @Override
    public void declareOutputFields(OutputFieldsDeclarer declarer) {
        // 不会发射任何数据，因为这个Bolt只是用来输出结果
    }
}

package bolt;

import org.apache.storm.task.TopologyContext;
import org.apache.storm.tuple.Fields;
import org.apache.storm.topology.BasicOutputCollector;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseBasicBolt;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;

import java.util.HashMap;
import java.util.Map;

public class TimeStockTradeBolt extends BaseBasicBolt {

    private Map<String, Map<Integer, Integer>> stockTypeHourlyVolumeMap;
    private Map<String, Map<Integer, Double>> stockTypeHourlyAmountMap;

    @Override
    public void prepare(Map stormConf, TopologyContext context) {
        stockTypeHourlyVolumeMap = new HashMap<>();
        stockTypeHourlyAmountMap = new HashMap<>();
    }

    @Override
    public void execute(Tuple tuple, BasicOutputCollector collector) {
        String line = tuple.getString(0);
        String[] fields = line.split(",");

        if (fields.length >= 9 && isNumeric(fields[3]) && isNumeric(fields[4])) {
            String time = fields[0]; // 假设时间字段在数组中的索引为0
            String stockType = fields[2]; // 假设股票类型字段在数组中的索引为2
            int tradeVolume = Integer.parseInt(fields[4]);
            double price = Double.parseDouble(fields[3]);
            double tradeAmount = price * tradeVolume;

            // 获取当前小时数
            int hour = getHourFromTime(time);

            // 更新股票类型每小时的交易量和交易总金额
            Map<Integer, Integer> hourlyVolumeMap = stockTypeHourlyVolumeMap.computeIfAbsent(stockType, k -> new HashMap<>());
            Map<Integer, Double> hourlyAmountMap = stockTypeHourlyAmountMap.computeIfAbsent(stockType, k -> new HashMap<>());

            hourlyVolumeMap.merge(hour, tradeVolume, Integer::sum);
            hourlyAmountMap.merge(hour, tradeAmount, Double::sum);
            collector.emit(new Values(stockType, hour,hourlyVolumeMap.get(hour), hourlyAmountMap.get(hour)));
        } else {
            // 非数字字段的处理逻辑，比如记录日志或者丢弃这条数据
        }
    }

    @Override
    public void declareOutputFields(OutputFieldsDeclarer declarer) {
        declarer.declare(new Fields("stockType", "hour","hourlyTradeVolume", "hourlyTradeAmount"));
    }

    @Override
    public void cleanup() {
        for (Map.Entry<String, Map<Integer, Integer>> entry : stockTypeHourlyVolumeMap.entrySet()) {
            String stockType = entry.getKey();
            Map<Integer, Integer> hourlyVolumeMap = entry.getValue();
            Map<Integer, Double> hourlyAmountMap = stockTypeHourlyAmountMap.get(stockType);

            for (Map.Entry<Integer, Integer> hourlyVolumeEntry : hourlyVolumeMap.entrySet()) {
                int hour = hourlyVolumeEntry.getKey();
                int totalVolume = hourlyVolumeEntry.getValue();
                double totalAmount = hourlyAmountMap.get(hour);
                System.out.println("Stock Type: " + stockType + ", Hour: " + hour + ", Total Volume: " + totalVolume + ", Total Amount: " + totalAmount);
            }
        }
    }

    private boolean isNumeric(String str) {
        try {
            Double.parseDouble(str);
            return true;
        } catch (NumberFormatException e) {
            return false;
        }
    }

    private int getHourFromTime(String time) {
        String[] timeParts = time.split(" ");
        String[] hourParts = timeParts[1].split(":");
        return Integer.parseInt(hourParts[0]);
    }
}

3.util

package util;

import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;

public class MapSort {

    //对map进行排序，并且进行长度截取
    @SuppressWarnings({ "unchecked", "rawtypes" })
    public static Map<String, Integer> sortByValue(Map<String, Integer> map) {

        if (map == null) {
            return null;
        }

        List list = new LinkedList(map.entrySet());

        Collections.sort(list, new Comparator() {

            public int compare(Object o1, Object o2) {
                Comparable sort1 = (Comparable) ((Map.Entry) o1).getValue();
                Comparable sort2 = (Comparable) ((Map.Entry) o2).getValue();
                return sort2.compareTo(sort1);
            }

        });

        Map result = new LinkedHashMap();

        for (Iterator it = list.iterator(); it.hasNext();) {

            Map.Entry entry = (Map.Entry) it.next();
            result.put(entry.getKey(), entry.getValue());

        }

        return result;
    }

    public static void main(String[] args) {

        Map<String, Integer> map = new HashMap<String, Integer> ();
        map.put("test", 3);
        map.put("hcy", 1);
        map.put("put", 2);

        map = sortByValue(map);

        for (String key : map.keySet()) {
            System.out.println( key + " ==> " + map.get(key));
        }
    }

}