关闭

【storm-kafka】storm和kafka结合处理流式数据

标签: hadoopstorm流式计算分布式
5034人阅读 评论(3) 收藏 举报
分类:

首先简单描述下storm

Storm是一个免费开源、分布式、高容错的实时计算系统。Storm令持续不断的流计算变得容易,弥补了Hadoop批处理所不能满足的实时要求。Storm经常用于在实时分析、在线机器学习、持续计算、分布式远程调用和ETL等领域。Storm的部署管理非常简单,而且,在同类的流式计算工具,Storm的性能也是非常出众的。

关于kafka

Kafka是一种高吞吐量的分布式发布订阅消息系统。

最近项目中有个需求是需要从kafka中将订阅到的流式数据实时持久化到elasticsearch和accumulo中。这里主要记录的是关于kafka和storm的整合。关于zookeeper,kafka,storm,elasticsearch等的安装可以网上找。
首先就是配置maven

<dependencies>
    <dependency>
        <groupId>org.apache.storm</groupId>
        <artifactId>storm-core</artifactId>
        <version>1.0.2</version>
        <!-- 由于storm环境中有该jar,所以不用pack到最终的task.jar中 -->
        <scope>provided</scope>
    </dependency>
    <dependency>
        <groupId>org.apache.storm</groupId>
        <artifactId>storm-kafka</artifactId>
        <version>1.0.2</version>
    </dependency>
    <dependency>
        <groupId>org.apache.commons</groupId>
        <artifactId>commons-lang3</artifactId>
        <version>3.3.2</version>
    </dependency>
    <!-- kafka目前已经有2.10了,但是我用了,任务执行报错,目前只能用kafka_2.9.2,我kafka服务端也是用最新的2.10版本 -->
    <dependency>
        <groupId>org.apache.kafka</groupId>
        <artifactId>kafka_2.9.2</artifactId>
        <version>0.8.2.2</version>
        <!-- 排除以下jar,由于storm服务端有log4j,避免冲突报错-->
        <exclusions>
            <exclusion>
                <groupId>org.apache.zookeeper</groupId>
                <artifactId>zookeeper</artifactId>
            </exclusion>
            <exclusion>
                <groupId>log4j</groupId>
                <artifactId>log4j<artifactId>
            </exclusion>
            <exclusion>
                <groupId>org.slf4j</groupId>
                <artifactId>slf4j-log4j12</artifactId>
            </exclusion>
        </exclusions>
    </dependency>
    <dependency>
        <groupId>org.elasticsearch</groupId>
        <artifactId>elasticsearch</artifactId>
        <version>1.4.4</version>
        <exclusions>
            <exclusion>
                <groupId>log4j</groupId>
                <artifactId>log4j<artifactId>
            </exclusion>
            <exclusion>
                <groupId>org.slf4j</groupId>
                <artifactId>slf4j-log4j12</artifactId>
            </exclusion>
        </exclusions>
    </dependency>
</dependencies>

<build>
    <plugins>
        <plugin>
            <artifactId>maven-compiler-plugin</artifactId>
            <configuration>
                <encoding>utf-8</encoding>
                <source>1.8</source>
                <target>1.8</target>
            </configuration>
        </plugin>
        <plugin>
            <artifactId>maven-assembly-plugin</artifactId>
            <configuration>
                <archive>
                    <manifest><manifest>
                </archive>
                <descriptorRefs>
                    <descriptorRef>jar-with-dependencies</descriptorRef>
                </descriptorRefs>
                <encoding>utf-8</encoding>
            </configuration>
        </plugin>
    </plugins>
</build>

由于storm-kafka已经实现了spout,我们直接用就可以。
Bolt代码

public class FilterBolt extends BaseRichBolt{
    private OutputCollector collector;

    /**
    * 初始化工作
    * /
    public void prepare(Map map, TopologyContext context, OutputCollector collector){
        this.collector = collector;
    }

    /**
    * 执行逻辑,目的是过滤无用的字符串
    */
    public void execute(Tuple input){
        String str = input.getString(0);
        if(StringUtils.isNotBlank(str)){
            String [] lines = str.split("\n");
            for(String line : lines){
                if(StringUtils.isBlank(line) || line.charAt(0) == '#'){
                    continue;
                }
                //发射到下一个bolt
                collector.emit(new Values(line));
            }
            //汇报给storm,该tuple执行成功
            collector.ack(input);
        }else{
            //执行失败
            collector.fail(input);
        }
    }

    /**
    * 申明传入到一个Bolt的字段名称
    */
    public void declareOutputFields(OutputFieldsDeclarer declarer){
        declarer.declare(new Fields("line"));
    }
}

下面是转换解析string,生成json,并将json保存到elasticsearch中。

public class TransferBolt extends BaseRichBolt{

    private Logger LOG = LoggerFactory.getLogger(TransferBolt.class);

    private OutputCollector collector;

    public void prepare(Map map, TopologyContext context, OutputCollector collector){
        this.collector = collector;
    }

    public void execute(Tuple input){
        String line = input.getString(0);
        JSONObject json = JSONObject.toJson(line);
        BulkRequest bulkRequest = new BulkRequest();
        IndexRequest indexRequest = new IndexRequest("test","element",json.getString("id")).source(json.getJSONObject("source").toString());
        bulkRequest.add(indexRequest);
        BulkResponse response = client.bulk(bulkRequest).actionGet();
        client.admin().indices().prepareRefresh("test").execute().actionGet();
    }

}

topology

public class KafkaTopology{

    public static void main(String[] args) throws Exception{
        String zks = PropertiesUtils.getString(KafkaProperties.ZK_HOSTS);
        String topic = PropertiesUtils.getString(KafkaProperties.TOPIC);
        String zkRoot = PropertiesUtils.getString(KafkaProperties.ZK_ROOT);
        String id = PropertiesUtils.getString(KafkaProperties.STORM_ID);
        BrokerHosts brokerHosts = new ZkHosts(zks);
        SpoutConfig spoutConfig = new SpoutConfig(brokerHosts,topic,zkRoot,id);
        spoutConfig.scheme = new SchemeAsMultiScheme(new StringScheme());
        spoutConfig.zkServers = Arrays.asList(PropertiesUtil.getString(KafkaProperties.ZK_SERVERS).split(","));
        spoutConfig.zkPort = PropertiesUtil.getInt(KafkaProperties.ZK_PORT);
        TopologyBuilder builder = new TopologyBuilder();
        builder.setSpout("kafka-reader",new KafkaSpout(spoutConfig),1);
        builder.setBolt("filter-bolt",new FilterBolt(),1).shuffleGrouping("kafka-reader");
        builder.setBolt("input-line",new TransferBolt(),1).shuffleGrouping("reader-input");
        Config config = new Config();
        String name = KafkaTopology.class.getSimpleName();
        config.setNumWorkers(PropertiesUtil.getInt(KafkaProperties.NUM_WORKERS));
        StormSubmitter.submitTopologyWithProgressBar(name,config,builder.createTopology());
    }
}
0
0

查看评论
* 以上用户言论只代表其个人观点,不代表CSDN网站的观点或立场