flume+kafka+sparkstreaming+hbase

最新推荐文章于 2022-02-21 14:31:46 发布
最新推荐文章于 2022-02-21 14:31:46 发布
阅读量303 收藏 1
点赞数
分类专栏： 大数据实时项目文档 文章标签： flume kafka hbase spark
版权声明：本文为博主原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。
本文链接：https://blog.csdn.net/m0_53683186/article/details/120808180
版权

文章目录

爬虫代码

Monitor

package ln;

import java.io.File;

public class Monitor extends Thread{
    @Override
    public void run() {
        super.run();
        File file = new File("Data.txt");
        while(true){
            System.out.println("文件大小:"+file.length()+"bytes");
            //1M等于1048576byte
            //file.length返回的是byte
            if(file.length()>=100000000){
                boolean delete = file.delete();
                if (delete==true)
                    System.out.println("删除成功");
            }
            try {
                sleep(2000);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
    }
}

Catcher

package ln;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.WebClient;
import org.apache.commons.io.FileUtils;

import java.io.File;
import java.io.IOException;

public class Catcher {
    public static void main(String[] args) throws IOException, InterruptedException {
        //股票列表url
        String StockListURL="http://43.push2.eastmoney.com/api/qt/clist/get?&pn=1&pz=10000&po=0&np=1&fltt=2&invt=2&fid=f12&fs=m:0+t:6,m:0+t:80,m:1+t:2,m:1+t:23&fields=f12,f13";
        WebClient webClient = new WebClient(BrowserVersion.CHROME);
        Monitor monitor = new Monitor();
        monitor.start();
        //json数据
        String contentAsString = webClient.getPage(StockListURL).getWebResponse().getContentAsString();
        Object[] objects = JSONObject.parseObject(contentAsString).getJSONObject("data").getJSONArray("diff").toArray();
        for (Object object : objects) {
            String F12 = JSONObject.parseObject(object.toString()).getString("f12");
            String F13 = JSONObject.parseObject(object.toString()).getString("f13");
            String DataURL = "http://push2his.eastmoney.com/api/qt/stock/kline/get?fields1=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13&fields2=f51,f52,f53,f54,f55,f56,f57,f58,f59,f60,f61&beg=0&end=20500101&rtntype=6&secid="+F13+"."+F12+"&klt=101&fqt=1";
            //原始数据
            String contentAsString1 = webClient.getPage(DataURL).getWebResponse().getContentAsString();
            Thread.sleep(5000);
            FileUtils.writeStringToFile(new File("Data.txt"), contentAsString1 + "\n", "UTF-8", true);
        }
    }
}

pom.xml

<?xml version="1.0" encoding="UTF-8"?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>

  <groupId>org.example</groupId>
  <artifactId>Spider</artifactId>
  <version>1.0-SNAPSHOT</version>

  <name>Spider</name>
  <!-- FIXME change it to the project's website -->
  <url>http://www.example.com</url>

  <properties>
    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    <maven.compiler.source>1.7</maven.compiler.source>
    <maven.compiler.target>1.7</maven.compiler.target>
  </properties>

  <dependencies>
    <dependency>
      <groupId>junit</groupId>
      <artifactId>junit</artifactId>
      <version>4.11</version>
      <scope>test</scope>
    </dependency>
    <!--爬虫组件-->
    <dependency>
      <groupId>net.sourceforge.htmlunit</groupId>
      <artifactId>htmlunit</artifactId>
      <version>2.49.1</version>
    </dependency>
    <!--json解析组件-->
    <dependency>
      <groupId>com.alibaba</groupId>
      <artifactId>fastjson</artifactId>
      <version>1.2.76</version>
    </dependency>
    <!--文件输出组件-->
    <dependency>
      <groupId>commons-io</groupId>
      <artifactId>commons-io</artifactId>
      <version>2.8.0</version>
    </dependency>
  </dependencies>

  <build>
    <pluginManagement><!-- lock down plugins versions to avoid using Maven defaults (may be moved to parent pom) -->
      <plugins>
        <!-- clean lifecycle, see https://maven.apache.org/ref/current/maven-core/lifecycles.html#clean_Lifecycle -->
        <plugin>
          <artifactId>maven-clean-plugin</artifactId>
          <version>3.1.0</version>
        </plugin>
        <!-- default lifecycle, jar packaging: see https://maven.apache.org/ref/current/maven-core/default-bindings.html#Plugin_bindings_for_jar_packaging -->
        <plugin>
          <artifactId>maven-resources-plugin</artifactId>
          <version>3.0.2</version>
        </plugin>
        <plugin>
          <artifactId>maven-compiler-plugin</artifactId>
          <version>3.8.0</version>
        </plugin>
        <plugin>
          <artifactId>maven-surefire-plugin</artifactId>
          <version>2.22.1</version>
        </plugin>
        <plugin>
          <artifactId>maven-jar-plugin</artifactId>
          <version>3.0.2</version>
        </plugin>
        <plugin>
          <artifactId>maven-install-plugin</artifactId>
          <version>2.5.2</version>
        </plugin>
        <plugin>
          <artifactId>maven-deploy-plugin</artifactId>
          <version>2.8.2</version>
        </plugin>
        <!-- site lifecycle, see https://maven.apache.org/ref/current/maven-core/lifecycles.html#site_Lifecycle -->
        <plugin>
          <artifactId>maven-site-plugin</artifactId>
          <version>3.7.1</version>
        </plugin>
        <plugin>
          <artifactId>maven-project-info-reports-plugin</artifactId>
          <version>3.0.0</version>
        </plugin>
      </plugins>
    </pluginManagement>
  </build>
</project>

启动爬虫

nohup java -jar xxxx.jar > /opt/spiderlog.file  2>&1 &

flume配置文件

a1.sources = r1
a1.sinks = k1
a1.channels = c1
a1.sources.r1.type = exec
a1.sources.r1.command = tail -F /home/server/Data.txt
a1.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink
a1.sinks.k1.kafka.topic = mytopic
a1.sinks.k1.kafka.bootstrap.servers = server1:9092,server2:9092,server3:9092
a1.sinks.k1.kafka.flumeBatchSize = 20
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1

启动flume命令

cd /opt/flume
nohup bin/flume-ng agent --conf conf --conf-file conf/example.conf --name a1 -Dflume.root.logger=INFO,console > /opt/flumelog.file  2>&1 &

kafka相关命令

bin/kafka-server-start.sh config/server.properties &
bin/kafka-topics.sh --create --topic mytopic --bootstrap-server localhost:9092
bin/kafka-topics.sh --describe --topic mytopic --bootstrap-server localhost:9092
bin/kafka-console-producer.sh --topic mytopic --bootstrap-server localhost:9092
bin/kafka-console-consumer.sh --topic mytopic --from-beginning --bootstrap-server localhost:9092
bin/kafka-topics.sh --zookeeper server1:2181 --list

Hive建立HBase关联表

CREATE TABLE StockInfo(
stockkey string,
stockmarket string,
stockcode string,
stockname string,
stockdate string,
stockopen string,
stockend string,
highest string,
lowest string,
ts string,
tn string,
zf string,
zdf string,
zde string,
ch string)
STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
WITH SERDEPROPERTIES ("hbase.columns.mapping" = 
":key,
info:stockmarket,
info:stockcode,
info:stockname,
info:stockdate,
info:stockopen,
info:stockend,
info:highest,
info:lowest,
info:ts,
info:tn,
info:zf,
info:zdf,
info:zde,
info:ch
")
TBLPROPERTIES ("hbase.table.name" = "StockInfo");

Spark Streaming

SparkStreamTest

import com.alibaba.fastjson.JSON.parseObject
import com.alibaba.fastjson.{JSON, JSONArray, JSONObject}
import org.apache.hadoop.hbase.{HBaseConfiguration, TableName}
import org.apache.hadoop.hbase.client.{ConnectionFactory, Put}
import org.apache.hadoop.hbase.io.ImmutableBytesWritable
import org.apache.hadoop.hbase.mapred.TableOutputFormat
import org.apache.hadoop.hbase.util.Bytes
import org.apache.hadoop.mapred.JobConf
import org.apache.kafka.clients.consumer.ConsumerRecord
import org.apache.spark.streaming._
import org.apache.spark.SparkConf
import org.apache.kafka.common.serialization.StringDeserializer
import org.apache.log4j.{Level, Logger}
import org.apache.spark.rdd.RDD
import org.apache.spark.streaming.dstream.{DStream, InputDStream}
import org.apache.spark.streaming.kafka010.{KafkaUtils, LocationStrategies, LocationStrategy}
import org.apache.spark.streaming.kafka010.ConsumerStrategies.Subscribe

import scala.collection.mutable.ArrayBuffer

object SparkStreamTest {
  Logger.getLogger("org").setLevel(Level.ERROR);
  def main(args: Array[String]): Unit = {
    println("spark启动中")
    val conf = new SparkConf()
      .setMaster("local[*]")
      .setAppName("StreamingKafkaTest")
    val ssc = new StreamingContext(conf,Seconds(1))
    ssc.checkpoint("hdfs://server1:9000/spark-checkpoint")
    val kafkaTopic = Array("mytopic")
    val kafkaParams = Map[String,Object](
      "bootstrap.servers" -> "server1:9092,server2:9092,server3:9092",
      "key.deserializer"->classOf[StringDeserializer],
      "value.deserializer"->classOf[StringDeserializer],
      "group.id"->"1",
      "enable.auto.commit"->(false : java.lang.Boolean)
    )


    val inputStream:InputDStream[ConsumerRecord[String,String]]= KafkaUtils.createDirectStream[String,String](ssc, LocationStrategies.PreferConsistent, Subscribe[String,String](kafkaTopic,kafkaParams))
    val dataDStream = inputStream.map(record =>(record.key,record.value)).map(_._2)
    dataDStream.foreachRDD{y=>y.foreach{x=>
      val data = parseObject(x).getJSONObject("data")
      val market = data.getString("market")
      val code = data.getString("code")
      val name = data.getString("name")
      val day_klines = data.getJSONArray("klines")
      val hbaseConf = HBaseConfiguration.create()
      hbaseConf.set("hbase.zookeeper.quorum","10.206.0.6")
      hbaseConf.set("hbase.property.clientPort","2181")
      val conn = ConnectionFactory.createConnection(hbaseConf)
      val tableName = TableName.valueOf("StockInfo")
      val table = conn.getTable(tableName)
      day_klines.forEach{data=>{
        val s = data.toString.split(",")
        val date = s(0)
        val open = s(1)
        val end = s(2)
        val highest = s(3)
        val lowest  = s(4)
        val ts  = s(5)
        val tn	= s(6)
        val zf	= s(7)
        val zdf	= s(8)
        val zde	= s(9)
        val ch= s(10)

        val putin = new Put(Bytes.toBytes(market+"."+code+","+date))
        putin.addColumn(Bytes.toBytes("info"), Bytes.toBytes("stockname"), Bytes.toBytes(name))
        putin.addColumn(Bytes.toBytes("info"), Bytes.toBytes("stockmarket"), Bytes.toBytes(market))
        putin.addColumn(Bytes.toBytes("info"), Bytes.toBytes("stockcode"), Bytes.toBytes(code))
        putin.addColumn(Bytes.toBytes("info"), Bytes.toBytes("stockdate"), Bytes.toBytes(date))
        putin.addColumn(Bytes.toBytes("info"), Bytes.toBytes("stockopen"), Bytes.toBytes(open))
        putin.addColumn(Bytes.toBytes("info"), Bytes.toBytes("stockend"), Bytes.toBytes(end))
        putin.addColumn(Bytes.toBytes("info"), Bytes.toBytes("highest"), Bytes.toBytes(highest))
        putin.addColumn(Bytes.toBytes("info"), Bytes.toBytes("lowest"), Bytes.toBytes(lowest))
        putin.addColumn(Bytes.toBytes("info"), Bytes.toBytes("ts"), Bytes.toBytes(ts))
        putin.addColumn(Bytes.toBytes("info"), Bytes.toBytes("tn"), Bytes.toBytes(tn))
        putin.addColumn(Bytes.toBytes("info"), Bytes.toBytes("zf"), Bytes.toBytes(zf))
        putin.addColumn(Bytes.toBytes("info"), Bytes.toBytes("zdf"), Bytes.toBytes(zdf))
        putin.addColumn(Bytes.toBytes("info"), Bytes.toBytes("zde"), Bytes.toBytes(zde))
        putin.addColumn(Bytes.toBytes("info"), Bytes.toBytes("ch"), Bytes.toBytes(ch))

        table.put(putin)
      }}
    }}
    ssc.start()
    ssc.awaitTermination()
  }
}

pom.xml

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
  <modelVersion>4.0.0</modelVersion>
  <groupId>org.example</groupId>
  <artifactId>SparkProgram</artifactId>
  <version>1.0-SNAPSHOT</version>
  <inceptionYear>2008</inceptionYear>
  <properties>
    <scala.version>2.12.10</scala.version>
  </properties>

  <repositories>
    <repository>
      <id>scala-tools.org</id>
      <name>Scala-Tools Maven2 Repository</name>
      <url>http://scala-tools.org/repo-releases</url>
    </repository>
  </repositories>

  <pluginRepositories>
    <pluginRepository>
      <id>scala-tools.org</id>
      <name>Scala-Tools Maven2 Repository</name>
      <url>http://scala-tools.org/repo-releases</url>
    </pluginRepository>
  </pluginRepositories>

  <dependencies>
    <dependency>
      <groupId>org.scala-tools</groupId>
      <artifactId>maven-scala-plugin</artifactId>
      <version>2.12</version>
    </dependency>
    <dependency>
      <groupId>org.apache.maven.plugins</groupId>
      <artifactId>maven-eclipse-plugin</artifactId>
      <version>2.5.1</version>
    </dependency>
    <dependency>
      <groupId>org.scala-lang</groupId>
      <artifactId>scala-library</artifactId>
      <version>${scala.version}</version>
    </dependency>
    <dependency>
      <groupId>junit</groupId>
      <artifactId>junit</artifactId>
      <version>4.4</version>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>org.specs</groupId>
      <artifactId>specs</artifactId>
      <version>1.2.5</version>
      <scope>test</scope>
    </dependency>
    <!--...............................................................................................................-->
    <!--以上是基础依赖-->
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-core_2.12</artifactId>
      <version>3.1.2</version>
    </dependency>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-streaming_2.12</artifactId>
      <version>3.1.2</version>
    </dependency>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-streaming-kafka-0-10_2.12</artifactId>
      <version>3.1.2</version>
    </dependency>
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-mapreduce-client-core</artifactId>
      <version>2.7.2</version>
    </dependency>
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-common</artifactId>
      <version>2.7.2</version>
    </dependency>
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-mapreduce-client-common</artifactId>
      <version>2.7.2</version>
    </dependency>
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
      <version>2.7.2</version>
    </dependency>
    <dependency>
      <groupId>org.apache.hbase</groupId>
      <artifactId>hbase-client</artifactId>
      <version>1.4.13</version>
    </dependency>
    <dependency>
      <groupId>org.apache.hbase</groupId>
      <artifactId>hbase-server</artifactId>
      <version>1.4.13</version>
    </dependency>
    <dependency>
      <groupId>com.alibaba</groupId>
      <artifactId>fastjson</artifactId>
      <version>1.2.76</version>
    </dependency>

  </dependencies>

  <build>
    <sourceDirectory>src/main/scala</sourceDirectory>
    <testSourceDirectory>src/test/scala</testSourceDirectory>
    <plugins>
      <plugin>
        <groupId>org.scala-tools</groupId>
        <artifactId>maven-scala-plugin</artifactId>
        <executions>
          <execution>
            <goals>
              <goal>compile</goal>
              <goal>testCompile</goal>
            </goals>
          </execution>
        </executions>
        <configuration>
          <scalaVersion>${scala.version}</scalaVersion>
          <args>
            <arg>-target:jvm-1.5</arg>
          </args>
        </configuration>
      </plugin>
      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-eclipse-plugin</artifactId>
        <configuration>
          <downloadSources>true</downloadSources>
          <buildcommands>
            <buildcommand>ch.epfl.lamp.sdt.core.scalabuilder</buildcommand>
          </buildcommands>
          <additionalProjectnatures>
            <projectnature>ch.epfl.lamp.sdt.core.scalanature</projectnature>
          </additionalProjectnatures>
          <classpathContainers>
            <classpathContainer>org.eclipse.jdt.launching.JRE_CONTAINER</classpathContainer>
            <classpathContainer>ch.epfl.lamp.sdt.launching.SCALA_CONTAINER</classpathContainer>
          </classpathContainers>
        </configuration>
      </plugin>
    </plugins>
  </build>
  <reporting>
    <plugins>
      <plugin>
        <groupId>org.scala-tools</groupId>
        <artifactId>maven-scala-plugin</artifactId>
        <configuration>
          <scalaVersion>${scala.version}</scalaVersion>
        </configuration>
      </plugin>
    </plugins>
  </reporting>
</project>

启动命令

nohup bin/spark-submit /home/server/SparkProgram.jar --class SparkStreamTest > /opt/sparkstreamlog  2>&1 &
李南想做条咸鱼
关注
  • 0
    点赞
  • 踩
  • 1
    收藏
    觉得还不错? 一键收藏
  • 打赏
    打赏
  • 0
    评论
专栏目录
flume=》kafka=》hbase
Keanu的博客
09-07 189
flume=》kafka=》hbase ps:无说明流程（未完成） 环境准备： 1.启动hadoop：start-all.sh 2.启动zookeeper：zkServer.sh start 3.启动kafka：/opt/soft/kafka211/bin/kafka-server-start.sh /opt/soft/kafka211/config/server.properties 4.启动hbase：start-hbase.sh 5.上传数据到 /opt/sanyu/f2k2hbase_data/ 1
Flume+Kafka+Storm+Hbase实现日志抓取和实施网站流量统计
07-02
搭建Hadoop集群，并使用flume+kafka+storm+hbase实现日志抓取分析，使用一个主节点master、两个slave节点
大数据篇：flume+kafka+spark stream+hbase做日志收集
热门推荐
yycc
06-30 1万+
flume+kafka+spark stream+hbase做日志收集 前言 flume+kafka+spark stream 是目前比较常用的一套大数据消息日志收集管理框架，至于最后是入到Hive或者者Hbase需看不同业务场景，下面以HBase为场景简述下整个配置与搭建流程以及这些框架如此搭配的优点。 flume+kafka+spark stream+hbase做日志收集...
Flume+Kafka+HBase实例
01-11
结合电信客服项目，将数据存储模块独立成项目，欢迎下载
flume+kafka+storm+hbase
AlbertLiangzt的博客
06-11 267
1.启动服务 1.1 flume kafka storm 同flume+kafka+storm 1.2 hbase服务 hbase-0.98.6-hadoop2/bin/目录下 ./start-hbase.sh 2.启动任务 python /usr/local/src/apache-storm-0.9.3/bin/storm jar \ /usr/local/src/learn/albert/24_storm_extend/extend.jar \ stormHbase.StormKafka \
新闻网大数据实时分析可视化系统项目——9、Flume+HBase+Kafka集成与开发
weixin_30364325的博客
05-10 335
1.下载Flume源码并导入Idea开发工具 1）将apache-flume-1.7.0-src.tar.gz源码下载到本地解压 2）通过idea导入flume源码 打开idea开发工具，选择File——》Open 然后找到flume源码解压文件，选中flume-ng-hbase-sink，点击ok加载相应模块的源码。 2.官方flume与hbase集成的参数介绍 3.下载...
【十八Spark Streaming实战】日志分析之Flume+Kafka+Spark Streaming+Hbase
Sid小杰的博客
07-23 1934
【十七Spark Streaming实战】日志分析之用python生成日志 对接Python日志产生器输出的日志到Flume 1.编写flume agent的配置文件 在node1上进入flume目录 cd /app/flume/flume/conf 创建flume的agent配置文件 vi test-streaming-project.conf #streaming-projec...
Hadoop+Spark+Hive+HBase+Oozie+Kafka+Flume+Flink+ES+Redash等详细安装部署
06-22
1、内容概要：Hadoop+Spark+Hive+HBase+Oozie+Kafka+Flume+Flink+Elasticsearch+Redash等大数据集群及组件搭建指南（详细搭建步骤+实践过程问题总结）。 2、适合人群：大数据运维、大数据相关技术及组件初学者。 3、...
flume+kafka+sparkstreaming
02-23
通过flume监控文件，让kafka消费flume数据，再将sparkstreaming连接kafka作为消费者进行数据处理，文档整理实现
SparkStreaming：Spark Streaming + Flume + Kafka + HBase + Hadoop + Zookeeper实现实时日志分析统计； SpringBoot + Echarts实现数据可视化展示
02-05
生成的数据主要是模拟某学习网站学习视频课程的访问量（其中*以“ / class”开头的表示实战课程，然后通过流水线Flume + Kafka + SparkStreaming进行实时日志的收集，HBase来存储数据）*注意事项（使用的软件工具及...
基于 Flume+ Kafka+ Spark Streaming 实现实时监控输出日志的报警系统的 Spark Streaming 程序代码
04-01
基于 Flume+ Kafka+ Spark Streaming 实现实时监控输出日志的报警系统的 Spark Streaming 程序代码。 基于 Flume+Kafka+Spark Streaming 实现实时监控输出日志的报警系统的 Spark Streaming 程序代码，博客链接： https://blog.csdn.net/linge1995/article/details/81326146 Spark Stream 实时监控
流计算项目实战【KAFKA+FLUME+SPARKSTRAMING+HBASE】
weixin_42991982的博客
01-23 221
1、 使用python 造数据 #!/usr/bin/env python # coding: utf-8 # In[1]: import random import time # In[2]: url_paths=[ "/class/112.html", "/class/132.html", "/class/146.html", "/class/177.html", "/class/212.html", "/class/342.html",
Flume与HBASE、Kafka集成
ThreeAspects的博客
10-25 739
Flume与HBASE、Kafka集成相关配置Flume与HBASE集成Flume与Kafka集成   这里首先设置两台Flume采集应用服务日志，将数据Push到第三台Flume进行日志合并、预处理。然后通过两个Channel分别将数据发送到HBASE和Kafka中。关于Flume基础可以参照Flume解析。   这里配置三台节点机器，其中agent2、agent3节点配置flume，用于从应用...
Linux下面Maven、Kafka、HBASE、flume、Hive安装
lbship的博客
09-29 372
一、安装Maven 1.下载并解压wget http://archive.apache.org/dist/maven/maven-3/3.5.2/binaries/apache-maven-3.5.2-bin.tar.gz  tar -xvzf apache-maven-3.5.2-bin.tar.gz 2.添加环境变量 vim ~/.bashrc export MAVEN_HOME=/...
flume - kafka - spark
lazyman001的博客
03-17 209
转载：开源中国 https://my.oschina.net/sunmin/blog/692994 1.架构 第一步，Flume和Kakfa对接，Flume抓取日志，写到Kafka中 第二部，Spark Streaming读取Kafka中的数据，进行实时分析 本文首先使用Kakfa自带的消息处理（脚本）来获取消息，走通Flume和Kafka的对接 2.安装flume,kafka flume ins...
基于Flume+Kafka+HBase+Mapreduce的电信客服项目（下）
u013337425的博客
04-15 628
文章目录1、数据消费阶段2、数据分析阶段 1、数据消费阶段 2、数据分析阶段
102.Spark2Streaming在Kafka和HBase下的读写
大勇若怯任卷舒
02-21 609
102.1 演示环境介绍 CM版本：5.14.3 CDH版本：5.14.2 Apache Kafka版本：0.10.2 SPARK版本：2.2.0 Redhat版本：7.3 已启用Kerberos,用root用户进行操作 102.2 操作演示 1.准备环境 导出keytab文件 [root@cdh01 ~]# kadmin.local Authenticating as principal hbase/admin@FAYSON.COM with password. kadmin.local:
[大数据] 搜索日志数据采集系统 flume+hbase+kafka架构 （数据搜狗实验室）
eye2cro
05-17 2926
1 收集规划 说明： D1 日志所在服务器1 —bigdata02.com D2 日志所在服务器2 —bigdata03.com A flume2 — bigdata02.com 日志收集 C flume3 — bigdata03.com 日志收集 B flume1 — bigdata01.com 日志整合 E kafka —bigdata01.com,bigd...
flume+hbase+kafka集成部署
梦想天空
09-10 1076
目录 一、架构图 二、flume的安装和配置 1. 下载安装 2. 配置 2.1 日志收集节点配置 2.2 日志聚集节点配置 2.3 Flume的二次开发 三、模拟程序开发 四、服务启动测试 1. 服务启动 1.1 zookeeper 1.2 HDFS 1.3 Hbase 1.4 kafka 2. 测试 Flume是一个高可用的，高可靠的，分布式的海量日志采集、聚合...
Flume+Kafka+Hbase集成配置
最新发布
05-14
Flume、Kafka和HBase都是大数据领域常用的组件，它们可以很好地协同工作来实现数据的实时采集、传输和存储。下面是它们的集成配置。 1. 安装Flume Flume是Apache基金会下的分布式、可靠、高可用的海量日志采集、...

“相关推荐”对你有帮助么？

  • 非常没帮助
  • 没帮助
  • 一般
  • 有帮助
  • 非常有帮助
提交
评论
添加红包

请填写红包祝福语或标题

个

红包个数最小为10个

元

红包金额最低5元

当前余额3.43元 前往充值 >
需支付：10.00元
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包

打赏作者

李南想做条咸鱼

你的鼓励将是我创作的最大动力

¥1 ¥2 ¥4 ¥6 ¥10 ¥20
扫码支付：¥1
获取中
扫码支付

您的余额不足，请更换扫码支付或充值

打赏作者

实付元
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明：

1.余额是钱包充值的虚拟货币，按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载，可以购买VIP、付费专栏及课程。

余额充值