Storm的消息队列接入以及多种方式落地实例代码实现

最新推荐文章于 2019-04-30 13:41:49 发布

chengqiuming

最新推荐文章于 2019-04-30 13:41:49 发布

阅读量766

点赞数

分类专栏： storm 文章标签： Storm

本文链接：https://blog.csdn.net/chengqiuming/article/details/79052692

版权

storm 专栏收录该内容

37 篇文章 0 订阅

订阅专栏

一 pom.xml

<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
     xsi:schemaLocation="http://maven.apache.org/POM/4.0.0http://maven.apache.org/xsd/maven-4.0.0.xsd";>
     <modelVersion>4.0.0</modelVersion>
     <groupId>com.cakin</groupId>
     <artifactId>stormexapmle</artifactId>
     <version>0.0.1-SNAPSHOT</version>
     <packaging>jar</packaging>
     <name>stormexapmle</name>
     <url>http://maven.apache.org</url>
     <properties>
          <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     </properties>
     <dependencies>
           <dependency>
                <groupId>org.apache.storm</groupId>
                <artifactId>storm-core</artifactId>
                <version>0.9.2-incubating</version>
                <exclusions>
                     <exclusion>
                          <artifactId>log4j-over-slf4j</artifactId>
                           <groupId>org.slf4j</groupId>
                     </exclusion>
                     <!--与 slf4j-log4j12 冲突，所以需要进行exclude移除 -->
                </exclusions>
           </dependency>
           <!--metaq依赖 -->
           <dependency>
                <groupId>com.taobao.metamorphosis</groupId>
                <artifactId>metamorphosis-commons</artifactId>
                <version>1.4.6.2</version>
           </dependency>
           <dependency>
                <groupId>com.taobao.metamorphosis</groupId>
                <artifactId>metamorphosis-client</artifactId>
                <version>1.4.6.2</version>
           </dependency>
           <!--mysql依赖 -->
           <dependency>
                <groupId>mysql</groupId>
                <artifactId>mysql-connector-java</artifactId>
                <version>5.1.22</version>
           </dependency>
           <dependency>
                <groupId>junit</groupId>
                <artifactId>junit</artifactId>
                <version>3.8.1</version>
                <scope>test</scope>
           </dependency>
     </dependencies>
</project>

二数据生产类

1 DataProDucer

package sourcedata;
import com.taobao.metamorphosis.Message;
import com.taobao.metamorphosis.client.MessageSessionFactory;
import com.taobao.metamorphosis.client.MetaClientConfig;
import com.taobao.metamorphosis.client.MetaMessageSessionFactory;
import com.taobao.metamorphosis.client.producer.MessageProducer;
import com.taobao.metamorphosis.client.producer.SendResult;
import com.taobao.metamorphosis.utils.ZkUtils;
import java.util.Random;
//该的目的是构造一个随机的domain数据集
public class DataProDucer {
    private static String topic = "test";
    private static String zkRoot = "/meta";
    private static String zkConnect = "192.168.0.110:2181";
    private static  MetaClientConfig metaClientConfig;
    private transient static MessageSessionFactory sessionFactory;
    private transient static MessageProducer messageProducer;
    private transient static SendResult sendResult;
    public static void main(String[] args) throws Exception {
        Random random = new Random();
        //构造一个随机记录
        String[] net0 = {"baidu", "hitwh", "google", "jikexueyuan", "hadoop",
                "storm", "blogchong", "cooje", "mite8", "mygod"};
        String[] net1 = {"com", "net", "cn", "edu", "tv", "org", "us", "jp","club" ,"pub",
                "rec", "info"};
        String[] times = {"2000", "2001", "2002", "2005", "2007", "2010",
                "2011", "2012", "2013", "1998", "2014", "2015"};
        String[] value = {"1326", "1446", "1401", "1202", "1871", "2000",
                "122", "23000", "400", "240", "8888", "100000", "34123"};
        String[] validity = {"3", "5", "20", "100", "32", "12", "50", "1",
                "23", "45", "200"};
        String[] seller = {"Huang", "Lina", "James", "Gale", "Kathryn",
                "Acong", "Green", "Facke", "Nina", "Litao", "Pony", "Blogchong", "Mite"};
        ZkUtils.ZKConfig zkconf = new ZkUtils.ZKConfig();
        zkconf.zkConnect = zkConnect;
        zkconf.zkRoot = zkRoot;
        MetaClientConfig metaconf = new MetaClientConfig();
        metaconf.setZkConfig(zkconf);
        metaClientConfig = metaconf;
        sessionFactory = new MetaMessageSessionFactory(metaClientConfig);
        messageProducer = sessionFactory.createProducer();
        messageProducer.publish(topic);
        while (true) {
            // 构造域名
            String net = "www."; + net0[random.nextInt(net0.length)] + "."
                    + net1[random.nextInt(net1.length)];
            String records = net + "\t" + value[random.nextInt(value.length)] + "\t"
                    + times[random.nextInt(times.length)] + "\t"
                    + validity[random.nextInt(validity.length)] + "\t"
                    + seller[random.nextInt(seller.length)];
            sendResult = messageProducer
                    .sendMessage(new Message(topic, (records).getBytes()));
            if (sendResult.isSuccess()) {
                System.out.println("消息：[" + records + "] 发送成功！");
            } else {
                System.err.println("消息：[" + records + "] 发送失败！");
            }
            Thread.sleep(100);
        }
    }
}

2 GetSource

package sourcedata;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.UnsupportedEncodingException;
import java.util.Random;
/**
 * @Des 构造一个随机的域名交易信息源，写入文件中
 */
//该的目的是构造一个随机的domain数据集
public class GetSource {
    public static void main(String[] args) {
        Random random = new Random();
        int note_num = 10000;
        //构造一个随机记录
        String[] net0 = {"baidu", "hitwh", "google", "jikexueyuan", "hadoop",
                "storm", "blogchong", "cooje", "mite8", "mygod"};
        String[] net1 = {"com", "net", "cn", "edu", "tv", "org", "us", "jp","club" ,"pub",
                "rec", "info"};
        String[] times = {"2000", "2001", "2002", "2005", "2007", "2010",
                "2011", "2012", "2013", "1998", "2014", "2015"};
        String[] value = {"1326", "1446", "1401", "1202", "1871", "2000",
                "122", "23000", "400", "240", "8888", "100000", "34123"};
        String[] validity = {"3", "5", "20", "100", "32", "12", "50", "1",
                "23", "45", "200"};
        String[] seller = {"Huang", "Lina", "James", "Gale", "Kathryn",
                "Acong", "Green", "Facke", "Nina", "Litao", "Pony", "Blogchong", "Mite"};
        // 写入中文字符时解决中文乱码问题
        FileOutputStream fos = null;
        try {
            fos = new FileOutputStream(new File("domain.log"));
        } catch (FileNotFoundException e1) {
            e1.printStackTrace();
        }
        OutputStreamWriter osw = null;
        try {
            osw = new OutputStreamWriter(fos, "UTF-8");
        } catch (UnsupportedEncodingException e) {
            e.printStackTrace();
        }
        BufferedWriter bw = new BufferedWriter(osw);
        for (int i = 0; i < note_num; i++) {
            // 构造域名
            String net = "www."; + net0[random.nextInt(net0.length)] + "."
                    + net1[random.nextInt(net1.length)];
            String records = net + "\t" + value[random.nextInt(value.length)] + "\t"
                    + times[random.nextInt(times.length)] + "\t"
                    + validity[random.nextInt(validity.length)] + "\t"
                    + seller[random.nextInt(seller.length)];
            try {
                bw.write(records);
                bw.newLine();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        // 注意关闭的先后顺序，先打开的后关闭，后打开的先关闭
        try {
            bw.close();
            osw.close();
            fos.close();
            System.out.println("write ok !");
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}

三 Spout类

1 MetaSpout类

package spout;

import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executor;
import java.util.concurrent.TimeUnit;
import com.taobao.gecko.core.util.LinkedTransferQueue;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import backtype.storm.spout.Scheme;
import backtype.storm.spout.SpoutOutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.IRichSpout;
import backtype.storm.topology.OutputFieldsDeclarer;
import util.ConfCheck;
import util.MacroDef;
import util.SpoutMetaq.MetaMessageWrapper;
import util.SpoutMetaq.StringScheme;
import xml.SpoutXml;
import com.taobao.metamorphosis.client.MetaClientConfig;
import com.taobao.metamorphosis.exception.MetaClientException;
import com.taobao.metamorphosis.utils.ZkUtils.ZKConfig;
import com.taobao.metamorphosis.Message;
import com.taobao.metamorphosis.client.MessageSessionFactory;
import com.taobao.metamorphosis.client.MetaMessageSessionFactory;
import com.taobao.metamorphosis.client.consumer.ConsumerConfig;
import com.taobao.metamorphosis.client.consumer.MessageConsumer;
import com.taobao.metamorphosis.client.consumer.MessageListener;
/**
 * @Des Spout数据源，从metaq中消费数据
 */
public class MetaSpout implements IRichSpout {
    public static final String FETCH_MAX_SIZE = "meta.fetch.max_size";
    public static final String TOPIC = "meta.topic";
    public static final int DEFAULT_MAX_SIZE = 128 * 1024;
    private transient MessageConsumer messageConsumer;
    private transient MessageSessionFactory sessionFactory;
    private MetaClientConfig metaClientConfig;
    private ConsumerConfig consumerConfig;
    static final Log log = LogFactory.getLog(MetaSpout.class);
    public static final long WAIT_FOR_NEXT_MESSAGE = 1L;
    private transient ConcurrentHashMap<Long, MetaMessageWrapper> id2wrapperMap;
    private transient SpoutOutputCollector collector;
    private transient LinkedTransferQueue<MetaMessageWrapper> messageQueue;
    private long spout_debug = 5;
    private long register = 0;
    private long reg_tmp = 0;
    private boolean spout_flag = true;
    String topic = "storm-test";
    private final Scheme scheme = new StringScheme();
    private boolean flag_par = true;
    private String spoutXml = "MetaSpout.xml";
    //是否加载配置标志位
    private static boolean flag_load = false;
    @SuppressWarnings("rawtypes")
    private Map conf = null;
    public MetaSpout(String SpoutXml) {
        super();
        if (SpoutXml == null) {
            this.flag_par = false;
        } else {
            this.spoutXml = SpoutXml;
        }
    }
    @SuppressWarnings("rawtypes")
    public void open(final Map conf, final TopologyContext context,
                     final SpoutOutputCollector collector) {
        System.out.println("MetaSpout--   Start!");
        this.collector = collector;
        this.conf = conf;
        this.spout_debug = MacroDef.SPOUT_DEBUG;
        this.spout_flag = MacroDef.SPOUT_FLAG;
        this.reg_tmp = this.spout_debug;
        if (!this.flag_par) {
            System.out
                    .println("MetaSpout-- Erre: can't get the path of Spout.xml!");
        } else {
            // 调用检测线程
            new ConfCheck(this.spoutXml, MacroDef.HEART_BEAT,
                    MacroDef.Thread_type_metaqspout).start();
        }
    }
    private void setUpMeta(final String topic, final Integer maxSize)
            throws MetaClientException {
        this.sessionFactory = new MetaMessageSessionFactory(
                this.metaClientConfig);
        this.messageConsumer = this.sessionFactory
                .createConsumer(this.consumerConfig);
        this.messageConsumer.subscribe(topic, maxSize, new MessageListener() {
            public void recieveMessages(final Message message) {
                final MetaMessageWrapper wrapper = new MetaMessageWrapper(
                        message);
                MetaSpout.this.id2wrapperMap.put(message.getId(), wrapper);
                MetaSpout.this.messageQueue.offer(wrapper);
                try {
                    wrapper.latch.await();
                } catch (final InterruptedException e) {
                    Thread.currentThread().interrupt();
                }
                // 获取数据失败
                if (!wrapper.success) {
                    throw new RuntimeException("MetaSpout  --     Obtain data fail!");
                }
            }
            public Executor getExecutor() {
                return null;
            }
        }).completeSubscribe();
    }
    public void close() {
        try {
            this.messageConsumer.shutdown();
        } catch (final MetaClientException e) {
            log.error("Shutdown consumer failed", e);
        }
        try {
            this.sessionFactory.shutdown();
        } catch (final MetaClientException e) {
            log.error("Shutdown session factory failed", e);
        }
    }
    //更改标志位
    public static void isload() {
        flag_load = false;
    }
    // 加载参数操作
    public void Loading() {
        // MetaSpout参数
        new SpoutXml(this.spoutXml).read();
        // 读取接收Topic
        String MetaRevTopic = SpoutXml.MetaRevTopic;
        // 读取MetaQ所在zk地址
        String MetaZkConnect = SpoutXml.MetaZkConnect;
        // 读取MetaQ所在的zk配置
        String MetaZkRoot = SpoutXml.MetaZkRoot;
        // 读取metaqconsumer配置“strom-test01”
        String MetaConsumerGroup = SpoutXml.MetaConsumerConf;
        // 获取连接zk配置(metaq)
        ZKConfig zkconf = new ZKConfig();
        // zkconf.zkConnect // ="192.168.5.240:2181";//
        zkconf.zkConnect = MetaZkConnect;
        // zkconf.zkRoot = "/meta";
        zkconf.zkRoot = MetaZkRoot;
        MetaClientConfig metaconf = new MetaClientConfig();
        metaconf.setZkConfig(zkconf);
        this.metaClientConfig = metaconf;
        this.consumerConfig = new ConsumerConfig(MetaConsumerGroup);
        this.topic = MetaRevTopic;
        if (this.topic == null) {
            throw new IllegalArgumentException(TOPIC + " is null");
        }
        Integer maxSize = (Integer) conf.get(FETCH_MAX_SIZE);
        if (maxSize == null) {
            log.warn("Using default FETCH_MAX_SIZE");
            maxSize = DEFAULT_MAX_SIZE;
        }
        this.id2wrapperMap = new ConcurrentHashMap<Long, MetaMessageWrapper>();
        this.messageQueue = new LinkedTransferQueue<MetaMessageWrapper>();
        try {
            this.setUpMeta(this.topic, maxSize);
        } catch (final MetaClientException e) {
            log.error("Setup meta consumer failed", e);
        }
        flag_load = true;
    }
    // 数据发布操作
    public void nextTuple() {
        if (!this.flag_par) {
            // 配置文件中参数为空(无法从配置文件中获取正确参数)
            System.out
                    .println("MetaSpout-- Erre: can't get the path of Spout.xml!");
        } else {
            // 检测配置文件是否更改
            if (!flag_load) {
                // 检测配置文件是否更改
                Loading();
                if (register != 0) {
                    System.out.println("MetaSpout-- Conf Change: "
                            + this.spoutXml);
                } else {
                    System.out.println("MetaSpout-- Conf Loaded: "
                            + this.spoutXml);
                }
            }
            if (this.messageConsumer != null) {
                try {
                    final MetaMessageWrapper wrapper = this.messageQueue.poll(
                            WAIT_FOR_NEXT_MESSAGE, TimeUnit.MILLISECONDS);
                    if (wrapper == null) {
                        return;
                    }
                    final Message message = wrapper.message;
                    // 统计数据量，并且输出打印
                    this.register++;
                    if (this.register >= this.reg_tmp) {
                        if (this.spout_flag) {
                            System.out
                                    .println("MetaSpout    --     Send Tuple Count: "
                                            + this.register);
                        }
                        this.reg_tmp = this.register + this.spout_debug;
                    }
                    // 数据发布操作
                    this.collector.emit(
                            this.scheme.deserialize(message.getData()),
                            message.getId());
                    Thread.sleep(100);
                } catch (final InterruptedException e) {
                }
            }
        }
    }
    public void ack(final Object msgId) {
        if (msgId instanceof Long) {
            final long id = (Long) msgId;
            final MetaMessageWrapper wrapper = this.id2wrapperMap.remove(id);
            if (wrapper == null) {
                System.out.println("MetaSpout--ack");
                log.warn(String.format("don't know how to ack(%s: %s)", msgId
                        .getClass().getName(), msgId));
                return;
            }
            wrapper.success = true;
            wrapper.latch.countDown();
        } else {
            System.out.println("MetaSpout--ack");
            log.warn(String.format("don't know how to ack(%s: %s)", msgId
                    .getClass().getName(), msgId));
        }
    }
    public void fail(final Object msgId) {
        if (msgId instanceof Long) {
            final long id = (Long) msgId;
            final MetaMessageWrapper wrapper = this.id2wrapperMap.remove(id);
            if (wrapper == null) {
                System.out.println("MetaSpout--fail");
                log.warn(String.format("don't know how to reject(%s: %s)",
                        msgId.getClass().getName(), msgId));
                return;
            }
            wrapper.success = false;
            wrapper.latch.countDown();
        } else {
            System.out.println("MetaSpout--fail");
            log.warn(String.format("don't know how to reject(%s: %s)", msgId
                    .getClass().getName(), msgId));
        }
    }
    public void declareOutputFields(final OutputFieldsDeclarer declarer) {
        declarer.declare(this.scheme.getOutputFields());
    }
    public boolean isDistributed() {
        return true;
    }
    public void activate() {
    }
    public void deactivate() {
    }
    public Map<String, Object> getComponentConfiguration() {
        return null;
    }
}

2 ReadLogSpout类

package spout;
import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.util.Map;
import backtype.storm.spout.SpoutOutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.IRichSpout;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Values;
import util.MacroDef;
/**
 * @Des Spout数据源，从log文件中直接读取数据
 */
public class ReadLogSpout implements IRichSpout {
     private SpoutOutputCollector collector;
     FileInputStream fis;
     InputStreamReader isr;
     BufferedReader br;
     @SuppressWarnings("rawtypes")
     public void open(Map conf, TopologyContext context,
                SpoutOutputCollector collector) {
           this.collector = collector;
           
           String file = "domain.log";
           try {
                this.fis = new FileInputStream(file);
                this.isr = new InputStreamReader(fis, MacroDef.ENCODING);
                this.br = new BufferedReader(isr);
           } catch (Exception e) {
                e.printStackTrace();
           }
     }
     public void close() {
     }
     public void activate() {
     }
     public void deactivate() {
     }
     public void nextTuple() {
           String str = "";
           try {
                while ((str = this.br.readLine()) != null) {
                     this.collector.emit(new Values(str));
                     Thread.sleep(100);
                }
           } catch (Exception e) {
                e.printStackTrace();
           }
     }
     public void ack(Object msgId) {
     }
     public void fail(Object msgId) {
     }
     public void declareOutputFields(OutputFieldsDeclarer declarer) {
           declarer.declare(new Fields("str"));
     }
     public Map<String, Object> getComponentConfiguration() {
           return null;
     }
}

四数据过滤bolt

package bolt;
import backtype.storm.task.OutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.IRichBolt;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Tuple;
import backtype.storm.tuple.Values;
import util.ConfCheck;
import util.MacroDef;
import xml.FilterXml;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
 * @Des 过滤Bolt，进行数据的正则过滤，范围过滤以及普通字符串过滤
 */
@SuppressWarnings("serial")
public class FilterBolt implements IRichBolt {
     private OutputCollector collector;
    //是否加载配置标志位
     private static boolean flag_load = false;
     private long register = 0;
    //默认参数~~
     String monitorXml = "Filter.xml";
    // 参数判空标志
     private boolean flag_par = true;
    // 匹配条件间的逻辑关系
     String MatchLogic = "AND";
    // !--匹配类型列表
     String MatchType = "regular::range::routine0";
    // !--匹配字段列表-
     String MatchField = "1::2::5";
    // !--字段值列表-
     String FieldValue = ".*baidu.*::1000,2000::ina";
     public FilterBolt(String MonitorXML) {
           if (MonitorXML == null) {
                flag_par = false;
           } else {
                this.monitorXml = MonitorXML;
           }
     }
     @SuppressWarnings("rawtypes")
     public void prepare(Map stormConf, TopologyContext context,
                OutputCollector collector) {
           System.out.println("FilterBolt  --   Start!");
           this.collector = collector;
                     
           if (!this.flag_par) {
                System.out
                           .println("MetaSpout-- Erre: can't get the path of Spout.xml!");
           } else {
            // 调用检测线程
                new ConfCheck(this.monitorXml, MacroDef.HEART_BEAT,
                          MacroDef.Thread_type_filterbolt).start();
           }
     }
     public void execute(Tuple input) {
           String str = input.getString(0);
           if (!this.flag_par) {
                System.out
                           .println("FilterBolt-- Erre: can't get the path of Filter.xml!");
           } else {
            // 检测配置文件是否更改
                if (!flag_load ) {
                // 配置文件发生更改则进行加载参数操作
                     Loading();
                     if (register != 0) {
                           System.out.println("FilterBolt-- Conf Change: "
                                     + this.monitorXml);
                     } else {
                           System.out.println("FilterBolt-- Conf Loaded: "
                                     + this.monitorXml);
                     }
                }
                boolean moni = Monitor(str, this.MatchLogic, this.MatchType,
                           this.MatchField, this.FieldValue);
                if (moni) {
                     this.collector.emit(new Values(str));
                }
            this.collector.ack(input);
           }
     }
    //更改标志位
     public static void isload() {
           flag_load = false;
     }
    // 加载参数操作
     public void Loading() {
        // 从conf中获取参数
           System.out.println("FilterXml:     " + this.monitorXml);
           new FilterXml(this.monitorXml).read();
           this.MatchLogic = FilterXml.MatchLogic;
           this.MatchType = FilterXml.MatchType;
           this.MatchField = FilterXml.MatchField;
           this.FieldValue = FilterXml.FieldValue;
           flag_load = true;
     }
     private boolean Monitor(String str, String logic, String type,
                String field, String value) {
           String[] types = type.split(MacroDef.FLAG_COLON);
           String[] fields = field.split(MacroDef.FLAG_COLON);
           String[] values = value.split(MacroDef.FLAG_COLON);
           int flag_init = types.length;
           int flag = 0;
           if (logic.equals(MacroDef.RULE_AND)) {
                for (int i = 0; i < flag_init; i++) {
                     if (types[i].equals(MacroDef.RLUE_REGULAR)) {
                           boolean regu = regular(str, fields[i], values[i]);
                           if (regu) {
                                flag++;
                           }
                     } else if (types[i].equals(MacroDef.RULE_RANGE)) {
                           boolean ran = range(str, fields[i], values[i]);
                           if (ran) {
                                flag++;
                           }
                     } else if (types[i].equals(MacroDef.RULE_ROUTINE0)) {
                           boolean rou0 = routine0(str, fields[i], values[i]);
                           if (rou0) {
                                flag++;
                           }
                     } else if (types[i].equals(MacroDef.RULE_ROUTINE1)) {
                           boolean rou1 = routine1(str, fields[i], values[i]);
                           if (rou1) {
                                flag++;
                           }
                     }
                }
                if (flag == flag_init) {
                     return true;
                } else {
                     return false;
                }
                
           } else if (logic.equals(MacroDef.RULE_OR)) {
                
                for (int i = 0; i < flag_init; i++) {
                     if (types[i].equals(MacroDef.RLUE_REGULAR)) {
                           boolean regu = regular(str, fields[i], values[i]);
                           if (regu) {
                                flag++;
                           }
                     } else if (types[i].equals(MacroDef.RULE_RANGE)) {
                           boolean ran = range(str, fields[i], values[i]);
                           if (ran) {
                                flag++;
                           }
                     } else if (types[i].equals(MacroDef.RULE_ROUTINE0)) {
                           boolean rou0 = routine0(str, fields[i], values[i]);
                           if (rou0) {
                                flag++;
                           }
                     } else if (types[i].equals(MacroDef.RULE_ROUTINE1)) {
                           boolean rou1 = routine1(str, fields[i], values[i]);
                           if (rou1) {
                                flag++;
                           }
                     }
                }
                if (flag != 0) {
                     return true;
                } else {
                     return false;
                }
           }
           return false;
     }
    // 正则匹配判断
     private boolean regular(String str, String field, String value) {
           String[] strs = str.split(MacroDef.FLAG_TABS);
           Pattern p = Pattern.compile(value);
           Matcher m = p.matcher(strs[Integer.parseInt(field) - 1]);
           boolean result = m.matches();
           if (result) {
                return true;
           } else {
                return false;
           }
     }
    // 范围匹配
     private boolean range(String str, String field, String value) {
           String[] strs = str.split(MacroDef.FLAG_TABS);
           String[] values = value.split(MacroDef.FLAG_COMMA);
           int strss = Integer.parseInt(strs[Integer.parseInt(field) - 1]);
           if (values.length == 1) {
                if (strss > Integer.parseInt(values[0])) {
                     return true;
                } else {
                     return false;
                }
           } else if (values.length == 2 && values[0].length() == 0) {
                if (strss < Integer.parseInt(values[1])) {
                     return true;
                } else {
                     return false;
                }
           } else if (values.length == 2 && values[0].length() != 0) {
                if (strss > Integer.parseInt(values[0])
                           && strss < Integer.parseInt(values[1])) {
                     return true;
                } else {
                     return false;
                }
           } else {
                return false;
           }
     }
    // 常规模糊匹配
     private boolean routine0(String str, String field, String value) {
           String[] strs = str.split(MacroDef.FLAG_TABS);
           String strss = strs[Integer.parseInt(field) - 1];
           if (strss.contains(value)) {
                return true;
           } else {
                return false;
           }
     }
    // 常规完全匹配
     private boolean routine1(String str, String field, String value) {
           String[] strs = str.split(MacroDef.FLAG_TABS);
           String strss = strs[Integer.parseInt(field) - 1];
           if (strss.equals(value)) {
                return true;
           } else {
                return false;
           }
     }
     public void cleanup() {
     }
     public void declareOutputFields(OutputFieldsDeclarer declarer) {
           declarer.declare(new Fields("str"));
     }
     public Map<String, Object> getComponentConfiguration() {
           return null;
     }
}

五数据落地

1 MetaBolt

package bolt;
import java.util.Map;
import com.taobao.metamorphosis.client.MetaClientConfig;
import com.taobao.metamorphosis.exception.MetaClientException;
import com.taobao.metamorphosis.utils.ZkUtils.ZKConfig;
import com.taobao.metamorphosis.Message;
import com.taobao.metamorphosis.client.MessageSessionFactory;
import com.taobao.metamorphosis.client.MetaMessageSessionFactory;
import com.taobao.metamorphosis.client.producer.MessageProducer;
import com.taobao.metamorphosis.client.producer.SendResult;
import backtype.storm.task.OutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.IRichBolt;
import backtype.storm.tuple.Tuple;
import util.ConfCheck;
import util.MacroDef;
import xml.MetaXml;
/**
 * @Des Metaq回写Bolt，数据生产
 */
@SuppressWarnings("serial")
public class MetaBolt implements IRichBolt {
    private MetaClientConfig metaClientConfig;
    // 是否加载配置标志位
    private static boolean flag_load = false;
    private transient MessageSessionFactory sessionFactory;
    private transient MessageProducer messageProducer;
    public static String Topic;
    private transient SendResult sendResult;
    String metaXml = "MetaBolt.xml";
    private boolean flag_par = true;
    private long meta_debug = 5;
    private long register = 0;
    private long reg_tmp = 0;
    public MetaBolt(String MetaXml) {
        super();
        if (MetaXml == null) {
            flag_par = false;
        } else {
            this.metaXml = MetaXml;
        }
    }
    @SuppressWarnings({"rawtypes"})
    public void prepare(final Map conf, TopologyContext context,
                        final OutputCollector collector) {
        System.out.println("MetaBolt --   Start!");
        this.reg_tmp = MacroDef.meta_debug;
        if (!this.flag_par) {
            System.out
                    .println("MetaSpout-- Erre: can't get the path of Spout.xml!");
        } else {
            // 调用检测线程
            new ConfCheck(this.metaXml, MacroDef.HEART_BEAT,
                    MacroDef.Thread_type_metaqbolt).start();
        }
    }
    @SuppressWarnings({"static-access", "unused"})
    public void execute(Tuple input) {
        String str = input.getString(0);
        if (!this.flag_par) {
            System.out
                    .println("MeataBolt-- Erre: can't get the path of MetaBolt.xml!");
        } else {
            try {
                // 检测配置文件是否更改
                if (!flag_load ) {
                    // 配置文件发生更改则进行加载参数操作
                    Loading();
                    if (register != 0) {
                        System.out.println("MetaBolt-- Conf Change: "
                                + this.metaXml);
                    } else {
                        System.out.println("MetaBolt-- Conf Loaded: "
                                + this.metaXml);
                    }
                }
                if (str != null) {
                    // 写入metaq的数据必须加上换行
                    str = str + MacroDef.FLAG_ROW;
                    this.sendResult = this.messageProducer
                            .sendMessage(new Message(this.Topic, str.getBytes()));
                    // 往metaq写数据
                    if (!sendResult.isSuccess()) {
                        System.err
                                .println("MetaBolt-- Send message failed,error message:"
                                        + sendResult.getErrorMessage());
                        System.err.println("MetaBolt-- Error Tuple: " + str);
                    } else {
                        System.out
                                .println("MetaBolt   --   Send Message success: "+ str);
                        this.register++;
                        if (this.register >= this.reg_tmp) {
                            if (!MacroDef.meta_flag ) {
                                System.out
                                        .println("MetaBolt --     Send Tuple Count: "
                                                + this.register);
                            }
                            this.reg_tmp = this.register + this.meta_debug;
                        }// 数量统计
                    }
                }
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }
    // 更改标志位
    public static void isload() {
        flag_load = false;
    }
    // 加载参数操作
    public void Loading() {
        new MetaXml(this.metaXml).read();
        // MetaSpout参数
        // 读取接收Topic
        this.Topic = MetaXml.MetaTopic;
        // 获取连接zk配置(metaq)
        ZKConfig zkconf = new ZKConfig();
        // zkconf.zkConnect
        zkconf.zkConnect = MetaXml.MetaZkConnect;
        // zkconf.zkRoot = "/meta";
        zkconf.zkRoot = MetaXml.MetaZkRoot;
        MetaClientConfig metaconf = new MetaClientConfig();
        metaconf.setZkConfig(zkconf);
        this.metaClientConfig = metaconf;
        if (this.Topic == null) {
            throw new IllegalArgumentException(this.Topic + ":" + " is null");
        }
        try {
            this.sessionFactory = new MetaMessageSessionFactory(
                    this.metaClientConfig);
            this.messageProducer = this.sessionFactory.createProducer();
            this.messageProducer.publish(this.Topic);
        } catch (final MetaClientException e) {
            e.printStackTrace();
        }
        flag_load = true;
    }
    public void cleanup() {
    }
    public void declareOutputFields(OutputFieldsDeclarer declarer) {
    }
    public Map<String, Object> getComponentConfiguration() {
        return null;
    }
}

2 MysqlBolt类

package bolt;
import java.util.Map;
import backtype.storm.task.OutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.IRichBolt;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.tuple.Tuple;
import util.ConfCheck;
import util.MacroDef;
import util.MysqlOpt;
import xml.MysqlXml;
/**
 * @Des 数据落地Mysql接口
 */
@SuppressWarnings("serial")
public class MysqlBolt implements IRichBolt {
     @SuppressWarnings("unused")
     private OutputCollector collector;
    //是否加载配置标志位
     private static boolean flag_load = false;
     
     private long register = 0;
     String mysqlXml = "Mysql.xml";
     MysqlOpt mysql = new MysqlOpt();
     private boolean flag_par = true;
     private boolean flag_xml = true;
     String from = "monitor"; // 表名
    // 构造函数
     public MysqlBolt(String MysqlXML) {
           if (MysqlXML == null) {
                flag_par = false;
           } else {
                this.mysqlXml = MysqlXML;
           }
     }
     public static void main(String[] args) {
     }
     @SuppressWarnings("rawtypes")
     public void prepare(Map stormConf, TopologyContext context,
                OutputCollector collector) {
           System.out.println("MysqlBolt   --   Start!");
           this.collector = collector;
           
           if (!this.flag_par) {
                System.out
                           .println("MetaSpout-- Erre: can't get the path of Spout.xml!");
           } else {
            // 调用检测线程
                new ConfCheck(this.mysqlXml, MacroDef.HEART_BEAT,
                          MacroDef.Thread_type_mysqlbolt).start();
           }
     }
    // 更改标志位
     public static void isload() {
           flag_load = false;
     }
    // 参数初始化
     public void Loading() {
           new MysqlXml(this.mysqlXml).read();
        // mysql地址及端口
           String host_port = MysqlXml.Host_port;
        // 数据库名
           String database = MysqlXml.Database;
        // 用户名
           String username = MysqlXml.Username;
        // 密码
           String password = MysqlXml.Password;
        // 表名
           this.from = MysqlXml.From;
           if (!this.mysql.connSQL(host_port, database, username, password)) {
                System.out
                           .println("MysqlBolt--Config errer, Please check Mysql-conf: "
                                     + this.mysqlXml);
                flag_xml = false;
           } else {
                System.out.println("MysqlBolt-- test connect mysql success: " + this.mysqlXml);
           }
        flag_load = true;
     }
     public void execute(Tuple input) {
           String str = input.getString(0);
           if (!this.flag_par) {
                System.out
                           .println("MysqlBolt-- Erre: can't get the path of Mysql.xml!");
           } else {
            // 检测配置文件是否更改
                if (!flag_load ) {
                // 配置文件发生更改则进行加载参数操作
                     Loading();
                     if (register != 0) {
                           System.out.println("MysqlBolt-- Conf Change: "
                                     + this.mysqlXml);
                     } else {
                           System.out.println("MysqlBolt-- Conf Loaded: "
                                     + this.mysqlXml);
                     }
                }
                if (this.flag_xml) {
                     String sql = send_str(str);
                     if (!this.mysql.insertSQL(sql)) {
                           System.out
                                     .println("MysqlBolt-- Erre: can't insert tuple into database!");
                           System.out.println("MysqlBolt-- Error Tuple: " + str);
                           System.out.println("SQL: " + sql);
                     }
                }
           }
     }
     public String send_str(String str) {
           String send_tmp = null;
           String field[] = str.split(MacroDef.FLAG_TABS);
           for (int i = 0; i < field.length; i++) {
                if (i == 0) {
                     send_tmp = "'" + field[0] + "', '";
                } else if (i == (field.length - 1)) {
                     send_tmp = send_tmp + field[i] + "'";
                } else {
                     send_tmp = send_tmp + field[i] + "', '";
                }
           }
           String send = "insert into " + this.from
                     + "(domain, value, time, validity, seller) values (" + send_tmp
                     + ");";
           return send;
     }
     public void cleanup() {
     }
     public void declareOutputFields(OutputFieldsDeclarer declarer) {
     }
     public Map<String, Object> getComponentConfiguration() {
           return null;
     }
}

3 PrintBolt

package bolt;

import backtype.storm.topology.BasicOutputCollector;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseBasicBolt;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Tuple;

/**
* @author blogchong
* @version 2015年06月07日 上午14:31:25
* @Blog www.blogchong.com
* @米特吧大数据论坛　www.mite8.com
* @email blogchong@163.com
* @QQ_G 191321336
* @Weixin: blogchong
* @Des 数据打印Bolt
*/

@SuppressWarnings("serial")
public class PrintBolt extends BaseBasicBolt {

    public static void main(String[] args) {

    }

    public void execute(Tuple input, BasicOutputCollector collector) {
        try {
            String mesg = input.getString(0);
            if (mesg != null)
                // 打印数据
                System.out.println("Tuple: " + mesg);
        } catch (Exception e) {
            e.printStackTrace();
        }

    }

    public void declareOutputFields(OutputFieldsDeclarer declarer) {
        declarer.declare(new Fields("mesg"));

    }

}

六拓扑构建

import backtype.storm.Config;
import backtype.storm.LocalCluster;
import backtype.storm.StormSubmitter;
import backtype.storm.generated.AlreadyAliveException;
import backtype.storm.generated.InvalidTopologyException;
import backtype.storm.topology.TopologyBuilder;
import bolt.FilterBolt;
import bolt.MetaBolt;
import bolt.MysqlBolt;
import bolt.PrintBolt;
import spout.MetaSpout;
/**
 * @Des  数据源Spout，从metaq中消费数据/从文本中读取数据
 */
public class Topology {
    // 实例化TopologyBuilder类。
     private static TopologyBuilder builder = new TopologyBuilder();
     public static void main(String[] args) throws InterruptedException,
                AlreadyAliveException, InvalidTopologyException {
           Config config = new Config();
        //数据源-->读取log文件/从消息队列metaq中消费
//         builder.setSpout("spout", new ReadLogSpout(), 1);
        builder.setSpout("spout", new MetaSpout("MetaSpout.xml"), 1);
        // 创建filter过滤节点
           builder.setBolt("filter", new FilterBolt("FilterBolt.xml"), 1)
                     .shuffleGrouping("spout");
        // 创建mysql数据存储节点
           builder.setBolt("mysql", new MysqlBolt("MysqlBolt.xml"), 1)
                     .shuffleGrouping("filter");
        //创建metaq回写节点
        builder.setBolt("meta", new MetaBolt("MetaBolt.xml"), 1)
                .shuffleGrouping("filter");
        //创建print消息打印节点
           builder.setBolt("print", new PrintBolt(), 1).shuffleGrouping("filter");
           config.setDebug(false);
           if (args != null && args.length > 0) {
                config.setNumWorkers(1);
                StormSubmitter.submitTopology(args[0], config,
                           builder.createTopology());
           } else {
            // 这里是本地模式下运行的启动代码。
                config.setMaxTaskParallelism(1);
                LocalCluster cluster = new LocalCluster();
                cluster.submitTopology("dataopttopology", config, builder.createTopology());
           }
     }
}

七其他类

参考https://github.com/blogchong/storm-example

八参考

http://www.jikexueyuan.com/course/1724.html

https://github.com/blogchong/storm-example

chengqiuming

关注

0
点赞
踩
0

收藏

觉得还不错? 一键收藏
0
评论
Storm的消息队列接入以及多种方式落地实例代码实现

一 pom.xml<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0http://maven.apache.org/
复制链接

扫一扫