Storm之监听文件并夹

package com.uplooking.bigdata.storm.group;

import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
import org.apache.storm.generated.StormTopology;
import org.apache.storm.shade.org.apache.commons.io.FileUtils;
import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.TopologyBuilder;
import org.apache.storm.topology.base.BaseRichBolt;
import org.apache.storm.topology.base.BaseRichSpout;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;

import java.io.File;
import java.util.Collection;
import java.util.List;
import java.util.Map;

/**
 * 使用一个FieldsGouping对tuple进行分组
 */
public class FieldsGroupingWordCountTopology {
    //做数据源，监听目录，当有新文件产生，读取其中的内容，发送到下游bolt
    static class WCSpout extends BaseRichSpout {
        private Map conf;
        private TopologyContext context;
        private SpoutOutputCollector collector;

        /**
         * 这是一个生命周期方法，一个SumNumSpout实例只运行一次，主要完成初始化的参数设置
         * @param conf      ---->storm程序以及storm集群相关的配置信息
         * @param context   ---->整个Topology上下文对象，可以通过该context获得相关topology应用属性
         * @param collector ---->主要用于收集数据，并将数据发射到下一个阶段
         */
        public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
            this.conf = conf;
            this.context = context;
            this.collector = collector;
        }

        //监听一个目录新文件的产生
        public void nextTuple() {
            /**
             * File directory ----> 要要监控的目录对象
             * String[] extensions  ---->要监控的目录下面以什么结尾(说白了就是扩展名)的文件
             *          注意，写文件扩展名的时候不能写"."
             * boolean recursive    ---->是否递归遍历
             */
            Collection<File> files = FileUtils.listFiles(new File("E:/test/storm"),
                    new String[]{"txt", "log", "csv"}, true);
            List<String> lines = null;
            try {
                for (File file : files) {
                    lines = FileUtils.readLines(file, "UTF-8");
                    for (String line : lines) {
                        System.out.println("spout读取到的内容：" + line);
                        collector.emit(new Values(line));
                    }
                    //读取完成一个文件之后，将其重命名，避免下次再读
                    FileUtils.moveFile(file, new File(file.getAbsolutePath() + "." + System.currentTimeMillis()));
                }
            }catch (Exception e) {
//                e.printStackTrace();//这里就不用输出异常信息了
            }
        }

        public void declareOutputFields(OutputFieldsDeclarer declarer) {
            declarer.declare(new Fields("line"));
        }
    }

    //读取上述spout发送过来的tuple，对tuple中的数据进行单词拆分，将拆分之后的单词发送给下游bolt
    static class SplitBolt extends BaseRichBolt {
        private Map conf;
        private TopologyContext context;
        private OutputCollector collector;

        public void prepare(Map conf, TopologyContext context, OutputCollector collector) {
            this.conf = conf;
            this.context = context;
            this.collector = collector;
        }

        public void execute(Tuple tuple) {
            String line = tuple.getStringByField("line");
            String[] splits = line.split(" ");
            for (String word : splits) {
                collector.emit(new Values(word, 1));
            }
        }

        public void declareOutputFields(OutputFieldsDeclarer declarer) {
            declarer.declare(new Fields("word", "times"));
        }
    }

    //接收上游bolt发送过来的单词，对单词进行统计
    static class WordCountBolt extends BaseRichBolt {
        private Map conf;
        private TopologyContext context;
        private OutputCollector collector;

        public void prepare(Map conf, TopologyContext context, OutputCollector collector) {
            this.conf = conf;
            this.context = context;
            this.collector = collector;
        }

        int sum = 0;
        public void execute(Tuple tuple) {
            String word = tuple.getStringByField("word");
            int times = tuple.getIntegerByField("times");
            System.out.println("=================" + Thread.currentThread().getId() + "=================");
            System.out.println("thread id: " + Thread.currentThread().getId() + "-----：" + word);
            sum += times;
            System.out.println("截止到目前为止出现的单词个数：" + sum);
            System.out.println("======================================================");
        }

        public void declareOutputFields(OutputFieldsDeclarer declarer) {

        }
    }

    public static void main(String[] args) {

        TopologyBuilder builder = new TopologyBuilder();
        //设置spout和bolt
        builder.setSpout("wcSpout_id", new WCSpout());
        builder.setBolt("splitBolt_id", new SplitBolt()).shuffleGrouping("wcSpout_id");
        builder.setBolt("wcBolt_id", new WordCountBolt(), 3)//
                //说明在石宏fieldsGrouping的时候，第二个参数就是制定按照那个一字段来进行分组
                .fieldsGrouping("splitBolt_id", new Fields("word"));

        StormTopology stormTopology = builder.createTopology();
        LocalCluster lCluster = new LocalCluster();
        String topologyName = FieldsGroupingWordCountTopology.class.getSimpleName();
        Config config = new Config();

        lCluster.submitTopology(topologyName, config, stormTopology);
    }
}