基于Hbase的Spark Sql示例 一

最新推荐文章于 2024-08-22 10:09:18 发布
最新推荐文章于 2024-08-22 10:09:18 发布
阅读量1k 收藏
点赞数
分类专栏： spark
版权声明：本文为博主原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。
本文链接：https://blog.csdn.net/u012036736/article/details/78350274
版权 
spark sql读取hbase数据简单实例：
package enn.cn.dataimport


/**
  * Created by Administrator 
  */

import java.io.Serializable
import java.util.logging.Logger

import enn.cn.util.{CommonContent, KerberosUtil}
import org.apache.hadoop.hbase.HBaseConfiguration
import org.apache.hadoop.hbase.mapreduce.TableInputFormat
import org.apache.spark.{SparkConf, SparkContext}

object MySparkSql extends Serializable {

  case class Score(name: String,metric: String, value: String, timestamp: String, deviceTypeId: String, deviceNo: String, deviceId: String, gatewayId: String)

  val logger = Logger.getLogger(MySparkSql.getClass.getName)

  def main(args: Array[String]) {

    /*val jars: Array[String] = Array("D:\\workspace\\mysparksql_2.10-1.0.jar")
    System.setProperty("hadoop.home.dir", "E:\\Program Files\\hadoop-2.7.0") //win7环境下运行须加*/
    KerberosUtil.loginUserFromKeytab(CommonContent.KERBEROS_NAME, CommonContent.KERBEROS_KEYTAB)
    val sconf = new SparkConf()
      .setMaster("local")
      //      .setMaster("spark://h230:7077")//在集群测试下设置,h230是我的hostname，须自行修改
      .setAppName("MySparkSql") //win7环境下设置
      .set("spark.executor.memory", "1g")
      .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
    //      .setJars(jars)//在集群测试下，设置应用所依赖的jar包
    val sc = new SparkContext(sconf)

    val conf = HBaseConfiguration.create()
   /* conf.set("hbase.zookeeper.property.clientPort", "2181")
    conf.set("hbase.zookeeper.quorum", "h230")
    conf.set("hbase.master", "h230:60000")*/
    conf.addResource("hbase-site.xml")//替代以上三条配置信息
    conf.set(TableInputFormat.INPUT_TABLE, "test:iot_electric_sale")

    //    Scan操作
    val hBaseRDD = sc.newAPIHadoopRDD(conf, classOf[TableInputFormat],
      classOf[org.apache.hadoop.hbase.io.ImmutableBytesWritable],
      classOf[org.apache.hadoop.hbase.client.Result])


    val score = hBaseRDD.map(m => m._2.listCells()).map(c =>
      Score(new String(c.get(0).getRow()),
        new String(c.get(0).getValue),
        new String(c.get(1).getValue),
        new String(c.get(2).getValue),
        new String(c.get(3).getValue),
        new String(c.get(4).getValue),
        new String(c.get(5).getValue),
        new String(c.get(6).getValue))
    )
    score.foreach(println)
    val sqlContext = new org.apache.spark.sql.SQLContext(sc)
    val scoreSchema = sqlContext.createDataFrame(score)
    scoreSchema.registerTempTable("test:iot_electric_sale")
    var result = sqlContext.sql("SELECT * FROM test:iot_electric_sale")
    result.collect().foreach(println)

  }
}
package enn.cn.dataimport;

/**
 * Created by Administrator 
 */
public class Score {
    public String name;

    public String metric;

    public String value;

    public String timestamp;

    public String deviceTypeId;

    public String deviceNo;

    public String deviceId;

    public String gatewayId;
    public Score(String name, String metric, String value, String timestamp, String deviceTypeId, String deviceNo, String deviceId, String gatewayId){
        this.name=name;
        this.metric=metric;
        this.value=value;
        this.timestamp=timestamp;
        this.deviceTypeId=deviceTypeId;
        this.deviceNo=deviceNo;
        this.deviceId=deviceId;
        this.gatewayId=gatewayId;
    }
    public String getMetric() {
        return metric;
    }

    public void setMetric(String metric) {
        this.metric = metric;
    }

    public String getValue() {
        return value;
    }

    public void setValue(String value) {
        this.value = value;
    }

    public String getTimestamp() {
        return timestamp;
    }

    public void setTimestamp(String timestamp) {
        this.timestamp = timestamp;
    }

    public String getDeviceTypeId() {
        return deviceTypeId;
    }

    public void setDeviceTypeId(String deviceTypeId) {
        this.deviceTypeId = deviceTypeId;
    }

    public String getDeviceNo() {
        return deviceNo;
    }

    public void setDeviceNo(String deviceNo) {
        this.deviceNo = deviceNo;
    }

    public String getDeviceId() {
        return deviceId;
    }

    public void setDeviceId(String deviceId) {
        this.deviceId = deviceId;
    }

    public String getGatewayId() {
        return gatewayId;
    }

    public void setGatewayId(String gatewayId) {
        this.gatewayId = gatewayId;
    }
}

依赖包，摘出hbase， spark相关的包即可：
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
       xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>

   <groupId>enn.cn</groupId>
   <artifactId>dataimport</artifactId>
   <version>0.0.1-SNAPSHOT</version>
   <packaging>jar</packaging>

   <name>dataimport</name>
   <url>http://maven.apache.org</url>

   <properties>
      <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
      <elasticsearch.version>2.1.2</elasticsearch.version>
   </properties>

   <dependencies>
      <!-- https://mvnrepository.com/artifact/com.metamx/scala-util_2.10 -->
      <dependency>
         <groupId>com.metamx</groupId>
         <artifactId>scala-util_2.10</artifactId>
         <version>1.11.6</version>
      </dependency>

      <dependency>
         <groupId>org.apache.spark</groupId>
         <artifactId>spark-core_2.10</artifactId>
         <version>1.6.0-cdh5.10.0</version>
         <exclusions>
            <exclusion>
               <artifactId>jackson-annotations</artifactId>
               <groupId>com.fasterxml.jackson.core</groupId>
            </exclusion>
            <exclusion>
               <artifactId>jackson-databind</artifactId>
               <groupId>com.fasterxml.jackson.core</groupId>
            </exclusion>
         </exclusions>
      </dependency>
      <dependency>
         <groupId>org.apache.spark</groupId>
         <artifactId>spark-streaming-kafka_2.10</artifactId>
         <version>1.6.0-cdh5.10.0</version>
      </dependency>
      <dependency>
         <groupId>org.apache.spark</groupId>
         <artifactId>spark-streaming_2.10</artifactId>
         <version>1.6.0-cdh5.10.0</version>
      </dependency>
      <dependency>
         <groupId>unknown</groupId>
         <artifactId>PspaceClientSDK</artifactId>
         <version>1.4</version>
         <scope>system</scope>
         <systemPath>${basedir}/libs/PspaceClientSDK-1.4.jar</systemPath>
      </dependency>
      <dependency>
         <groupId>com.likong</groupId>
         <artifactId>pspace1.0</artifactId>
         <version>1.0.0</version>
      </dependency>
      <dependency>
         <groupId>org.apache.kafka</groupId>
         <artifactId>kafka_2.10</artifactId>
         <version>0.10.0.0</version>
      </dependency>

      <dependency>
         <groupId>org.apache.hbase</groupId>
         <artifactId>hbase-checkstyle</artifactId>
         <version>1.2.0-cdh5.10.0</version>
      </dependency>
      <dependency>
         <groupId>org.apache.hbase</groupId>
         <artifactId>hbase-client</artifactId>
         <version>1.2.0-cdh5.10.0</version>
      </dependency>
      <dependency>
         <groupId>org.apache.hbase</groupId>
         <artifactId>hbase-common</artifactId>
         <version>1.2.0-cdh5.10.0</version>
      </dependency>
      <dependency>
         <groupId>org.apache.hbase</groupId>
         <artifactId>hbase-hadoop2-compat</artifactId>
         <version>1.2.0-cdh5.10.0</version>
      </dependency>
      <dependency>
         <groupId>org.apache.hbase</groupId>
         <artifactId>hbase-protocol</artifactId>
         <version>1.2.0-cdh5.10.0</version>
      </dependency>
      <dependency>
         <groupId>org.apache.hadoop</groupId>
         <artifactId>hadoop-hdfs</artifactId>
         <version>2.6.0-cdh5.10.0</version>
         <exclusions>
            <exclusion>
               <groupId>tomcat</groupId>
               <artifactId>jasper-runtime</artifactId>
            </exclusion>
            <exclusion>
               <groupId>tomcat</groupId>
               <artifactId>jasper-compiler</artifactId>
            </exclusion>
         </exclusions>
      </dependency>

      <dependency>
         <groupId>commons-dbcp</groupId>
         <artifactId>commons-dbcp</artifactId>
         <version>1.4</version>
      </dependency>
      <dependency>
         <groupId>junit</groupId>
         <artifactId>junit</artifactId>
         <version>3.8.1</version>
         <scope>test</scope>
      </dependency>
      <!-- https://mvnrepository.com/artifact/javax.servlet/javax.servlet-api -->
      <dependency>
         <groupId>javax.servlet</groupId>
         <artifactId>javax.servlet-api</artifactId>
         <version>3.1.0</version>
      </dependency>

      <dependency>
         <groupId>org.springframework.data</groupId>
         <artifactId>spring-data-jpa</artifactId>
         <version>1.5.0.RELEASE</version>
      </dependency>
      <dependency>
         <groupId>org.springframework</groupId>
         <artifactId>spring-tx</artifactId>
         <version>3.2.3.RELEASE</version>
      </dependency>
      <dependency>
         <groupId>org.hibernate</groupId>
         <artifactId>hibernate-entitymanager</artifactId>
         <version>4.2.20.Final</version>
      </dependency>
      <dependency>
         <groupId>c3p0</groupId>
         <artifactId>c3p0</artifactId>
         <version>0.9.1.2</version>
      </dependency>
      <dependency>
         <groupId>org.hibernate</groupId>
         <artifactId>hibernate-c3p0</artifactId>
         <version>3.5.6-Final</version>
      </dependency>

      <dependency>
         <groupId>org.springframework</groupId>
         <artifactId>spring-webmvc</artifactId>
         <version>3.2.3.RELEASE</version>
      </dependency>
      <dependency>
         <groupId>org.springframework</groupId>
         <artifactId>spring-webmvc-portlet</artifactId>
         <version>3.2.3.RELEASE</version>
      </dependency>
      <dependency>
         <groupId>org.springframework</groupId>
         <artifactId>spring-expression</artifactId>
         <version>3.2.3.RELEASE</version>
      </dependency>
      <dependency>
         <groupId>org.springframework</groupId>
         <artifactId>spring-aspects</artifactId>
         <version>3.2.3.RELEASE</version>
      </dependency>
      <dependency>
         <groupId>org.springframework</groupId>
         <artifactId>spring-oxm</artifactId>
         <version>3.2.3.RELEASE</version>
      </dependency>
      <dependency>
         <groupId>org.apache.commons</groupId>
         <artifactId>commons-lang3</artifactId>
         <version>3.1</version>
      </dependency>
      <dependency>
         <groupId>org.apache.shiro</groupId>
         <artifactId>shiro-all</artifactId>
         <version>1.2.4</version>
      </dependency>
      <dependency>
         <groupId>commons-codec</groupId>
         <artifactId>commons-codec</artifactId>
         <version>1.10</version>
      </dependency>
      <dependency>
         <groupId>mysql</groupId>
         <artifactId>mysql-connector-java</artifactId>
         <version>5.1.25</version>
      </dependency>
      <dependency>
         <groupId>com.google.guava</groupId>
         <artifactId>guava-base</artifactId>
         <version>r03</version>
      </dependency>
      <dependency>
         <groupId>com.google.guava</groupId>
         <artifactId>guava-collections</artifactId>
         <version>r03</version>
      </dependency>
      <dependency>
         <groupId>org.hibernate</groupId>
         <artifactId>hibernate-ehcache</artifactId>
         <version>4.2.20.Final</version>
      </dependency>
      <!-- http://mvnrepository.com/artifact/net.sf.ehcache/ehcache-core -->
      <dependency>
         <groupId>net.sf.ehcache</groupId>
         <artifactId>ehcache-core</artifactId>
         <version>2.6.11</version>
      </dependency>

      <dependency>
         <groupId>com.fasterxml.jackson.core</groupId>
         <artifactId>jackson-annotations</artifactId>
         <version>2.7.5</version>
      </dependency>

      <dependency>
         <groupId>cglib</groupId>
         <artifactId>cglib</artifactId>
         <version>3.2.2</version>
      </dependency>
      <dependency>
         <groupId>org.codehaus.jackson</groupId>
         <artifactId>jackson-mapper-asl</artifactId>
         <version>1.8.6</version>
      </dependency>
      <dependency>
         <groupId>commons-fileupload</groupId>
         <artifactId>commons-fileupload</artifactId>
         <version>1.3</version>
      </dependency>
      <dependency>
         <groupId>org.slf4j</groupId>
         <artifactId>slf4j-log4j12</artifactId>
         <version>1.7.6</version>
      </dependency>
      <dependency>
         <groupId>javax.servlet</groupId>
         <artifactId>jstl</artifactId>
         <version>1.2</version>
      </dependency>
      <dependency>
         <groupId>com.alibaba</groupId>
         <artifactId>fastjson</artifactId>
         <version>1.2.10</version>
      </dependency>
      <dependency>
         <groupId>javax.mail</groupId>
         <artifactId>mail</artifactId>
         <version>1.4.7</version>
      </dependency>
      <dependency>
         <groupId>org.opensymphony.quartz</groupId>
         <artifactId>quartz-all</artifactId>
         <version>1.6.1</version>
      </dependency>
      <dependency>
         <groupId>commons-collections</groupId>
         <artifactId>commons-collections</artifactId>
         <version>3.2.1</version>
      </dependency>
      <dependency>
         <groupId>com.belerweb</groupId>
         <artifactId>pinyin4j</artifactId>
         <version>2.5.0</version>
      </dependency>
      <dependency>
         <groupId>org.apache.commons</groupId>
         <artifactId>commons-pool2</artifactId>
         <version>2.2</version>
      </dependency>
      <dependency>
         <groupId>commons-pool</groupId>
         <artifactId>commons-pool</artifactId>
         <version>1.6</version>
      </dependency>
      <dependency>
         <groupId>org.ow2.asm</groupId>
         <artifactId>asm-util</artifactId>
         <version>4.0</version>
      </dependency>
      <dependency>
         <groupId>commons-net</groupId>
         <artifactId>commons-net</artifactId>
         <version>3.3</version>
      </dependency>
      <dependency>
         <groupId>commons-lang</groupId>
         <artifactId>commons-lang</artifactId>
         <version>2.4</version>
      </dependency>
      <!-- http://mvnrepository.com/artifact/commons-beanutils/commons-beanutils -->
      <dependency>
         <groupId>commons-beanutils</groupId>
         <artifactId>commons-beanutils</artifactId>
         <version>1.9.2</version>
      </dependency>
      <!-- http://mvnrepository.com/artifact/commons-beanutils/commons-beanutils-core -->
      <dependency>
         <groupId>commons-beanutils</groupId>
         <artifactId>commons-beanutils-core</artifactId>
         <version>1.8.3</version>
      </dependency>
      <!-- http://mvnrepository.com/artifact/commons-beanutils/commons-beanutils-bean-collections -->
      <dependency>
         <groupId>commons-beanutils</groupId>
         <artifactId>commons-beanutils-bean-collections</artifactId>
         <version>1.8.3</version>
      </dependency>
      <!-- http://mvnrepository.com/artifact/redis.clients/jedis -->
      <dependency>
         <groupId>redis.clients</groupId>
         <artifactId>jedis</artifactId>
         <version>2.7.0</version>
      </dependency>
      <!-- http://mvnrepository.com/artifact/net.heartsavior/jedis <dependency>
         <groupId>net.heartsavior</groupId> <artifactId>jedis</artifactId> <version>2.2.1.1</version>
         </dependency> -->
      <!-- shiro 分布式session及cache管理 -->
      <!-- https://mvnrepository.com/artifact/org.bouncycastle/bcprov-jdk16 -->
      <dependency>
         <groupId>org.bouncycastle</groupId>
         <artifactId>bcprov-jdk16</artifactId>
         <version>1.45</version>
      </dependency>

      <dependency>
         <groupId>org.crazycake</groupId>
         <artifactId>shiro-redis</artifactId>
         <version>2.4.2.1-RELEASE</version>
      </dependency>
      <!-- https://mvnrepository.com/artifact/net.sf.json-lib/json-lib-ext-spring -->
      <dependency>
         <groupId>net.sf.json-lib</groupId>
         <artifactId>json-lib-ext-spring</artifactId>
         <version>1.0.2</version>
      </dependency>
      <!-- https://mvnrepository.com/artifact/commons-httpclient/commons-httpclient -->
      <dependency>
         <groupId>commons-httpclient</groupId>
         <artifactId>commons-httpclient</artifactId>
         <version>3.0</version>
      </dependency>
      <!-- https://mvnrepository.com/artifact/com.thoughtworks.xstream/xstream -->
      <dependency>
         <groupId>com.thoughtworks.xstream</groupId>
         <artifactId>xstream</artifactId>
         <version>1.4.8</version>
      </dependency>
      <dependency>
         <groupId>jaxen</groupId>
         <artifactId>jaxen</artifactId>
         <version>1.1.6</version>
      </dependency>
      <dependency>
         <groupId>dom4j</groupId>
         <artifactId>dom4j</artifactId>
         <version>1.6.1</version>
      </dependency>
      <dependency>
         <groupId>org.apache.httpcomponents</groupId>
         <artifactId>httpcore</artifactId>
         <version>4.4.6</version>
      </dependency>
      <dependency>
         <groupId>commons-codec</groupId>
         <artifactId>commons-codec</artifactId>
         <version>1.9</version>
      </dependency>
      <dependency>
         <groupId>org.apache.httpcomponents</groupId>
         <artifactId>httpclient</artifactId>
         <version>4.4</version>
      </dependency>
      <!-- https://mvnrepository.com/artifact/org.apache.zookeeper/zookeeper -->
      <dependency>
         <groupId>org.apache.zookeeper</groupId>
         <artifactId>zookeeper</artifactId>
         <version>3.4.10</version>
         <type>pom</type>
      </dependency>
      <!-- https://mvnrepository.com/artifact/org.apache.hive/hive-jdbc -->
      <dependency>
         <groupId>org.apache.hive</groupId>
         <artifactId>hive-jdbc</artifactId>
         <version>1.1.0-cdh5.10.0</version>
      </dependency>
      <dependency>
         <groupId>org.apache.spark</groupId>
         <artifactId>spark-sql_2.10</artifactId>
         <version>1.6.0-cdh5.10.0</version>
      </dependency>
      <dependency>
         <groupId>org.apache.hbase</groupId>
         <artifactId>hbase-server</artifactId>
         <version>1.2.0-cdh5.10.0</version>
      </dependency>
   </dependencies>
   <build>
      <plugins>
         <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-jar-plugin</artifactId>
            <version>2.6</version>
            <configuration>
               <archive>
                  <manifest>
                     <addClasspath>true</addClasspath>
                     <classpathPrefix>lib/</classpathPrefix>
                     <!--<mainClass>enn.cn.pspace.EnnPspaceConn</mainClass>-->
                  </manifest>
               </archive>
            </configuration>
         </plugin>
         <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-dependency-plugin</artifactId>
            <version>2.10</version>
            <executions>
               <execution>
                  <id>copy-dependencies</id>
                  <phase>package</phase>
                  <goals>
                     <goal>copy-dependencies</goal>
                  </goals>
                  <configuration>
                     <outputDirectory>${project.build.directory}/lib</outputDirectory>
                  </configuration>
               </execution>
            </executions>
         </plugin>
         <plugin>
            <groupId>org.scala-tools</groupId>
            <artifactId>maven-scala-plugin</artifactId>
            <executions>
               <execution>
                  <goals>
                     <goal>compile</goal>
                     <goal>testCompile</goal>
                  </goals>
               </execution>
            </executions>
            <configuration>
               <scalaVersion>2.10.6</scalaVersion>
            </configuration>
         </plugin>
         <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-shade-plugin</artifactId>
            <executions>
               <execution>
                  <phase>package</phase>
                  <goals>
                     <goal>shade</goal>
                  </goals>
                  <configuration>
                     <relocations>
                        <relocation>
                           <pattern>org.apache.http</pattern>
                           <shadedPattern>shade.org.apache.http</shadedPattern>
                        </relocation>
                     </relocations>
                     <artifactSet>
                        <includes>
                           <include>*:*</include>
                        </includes>
                     </artifactSet>
                     <filters>
                        <filter>
                           <artifact>*:*</artifact>
                           <excludes>
                              <exclude>META-INF/*.SF</exclude>
                              <exclude>META-INF/*.DSA</exclude>
                              <exclude>META-INF/*.RSA</exclude>
                           </excludes>
                        </filter>
                     </filters>
                      <transformers>
                            <transformer
                                implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
                              <!--  <mainClass>enn.cn.reports.IotToHiveMain</mainClass>-->
                            </transformer>
                            <transformer
                                implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
                                <resource>META-INF/spring.handlers</resource>
                            </transformer>
                            <transformer
                                implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
                                <resource>META-INF/spring.schemas</resource>
                            </transformer>
                            <transformer
                                    implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
                                <resource>META-INF/services/org.apache.hadoop.fs.FileSystem</resource>
                            </transformer>
                            </transformers>
                  </configuration>
               </execution>
            </executions>
         </plugin>
      </plugins>
   </build>
</project>


                

        

        

    

  


    

      

        

            

              
                
                  挖矿的小强
                
              
            

            

                关注
              
            

        

        

          
      

      

            

                专栏目录
          









                



	

		

			

				
					
Spark读取HBase数据库

				
			

			

				

					CSDN 精品推荐
				

				

					08-14
					
					832
					
				

			

		

		

			
				
Apache Spark 是一个快速、通用的大数据处理引擎，支持在分布式环境中进行大规模数据处理和分析。它提供了丰富的数据处理功能，并且可以与多种数据存储系统集成，包括 HBase。

			
		

	



                

              
                



	

		

			

				
					
Spark RDD编程 读写HBase数据

				
			

			

				

					晓晓的天空
				

				

					03-13
					
					1303
					
				

			

		

		

			
				
Spark RDD编程 读写HBase数据

			
		

	



                


  
              

                


	

		

			

				
					
spark读取hbase数据，并使用spark sql保存到mysql

				
			

			

				

					03-22
				

			

		

		

			
				
使用spark读取hbase中的数据，并插入到mysql中

			
		

	





	

		

			

				
					
Spark(四): Spark-sql 读hbase

				
			

			

				

					SunWuKong_Hadoop的博客
				

				

					12-30
					
					945
					
				

			

		

		

			
				
SparkSQL是指整合了Hive的spark-sql cli, 本质上就是通过Hive访问HBase表，具体就是通过hive-hbase-handler, 具体配置参见：Hive(五):hive与hbase整合 

目录：

SparkSql 访问 hbase配置测试验证

SparkSql 访问 hbase配置:


 拷贝HBase的相关jar包到Spark节点上的$SPA

			
		

	



		

			
		



	

		

			

				
					
Spark SQL on HBase 使用教程

					
最新发布

				
			

			

				

					gitblog_00652的博客
				

				

					08-22
					
					446
					
				

			

		

		

			
				
Spark SQL on HBase 使用教程
 Spark-SQL-on-HBaseNative, optimized access to HBase Data through Spark SQL/Dataframe Interfaces项目地址:https://gitcode.com/gh_mirrors/sp/Spark-SQL-on-HBase 项目介绍
Spark SQL on HBas...

			
		

	





	

		

			

				
					
sparkSQL 查询hbase

				
			

			

				

					weixin_40532625的博客
				

				

					05-31
					
					890
					
				

			

		

		

			
				
1.在spark/jars目录下建立./hbase目录，然后拷贝以下jar包导该目录


cd  /usr/local/spark/jars
mkdir  hbase
cd  hbase
cp  /usr/local/hbase/lib/hbase*.jar  ./
cp  /usr/local/hbase/lib/guava-12.0.1.jar  ./
cp  /usr/local/hbas...

			
		

	





	

		

			

				
					
sparksql查询hbase中的数据

				
			

			

				

					u013164612的专栏
				

				

					05-29
					
					3965
					
				

			

		

		

			
				
1.demo环境说明    demo是根据spark2.2+hadoop2.7+hbase1.3（亲测版本兼容，其他不同版本可能会出现不兼容的情况）2.代码案例package com.yzc.test;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
i...

			
		

	





	

		

			

				
					
Spark  SQL与HBase集成

				
			

			

				

					qq_52137951的博客
				

				

					05-04
					
					1054
					
				

			

		

		

			
				
文章目录一、Spark  SQL与HBase集成1.把hbase，hive，mysql相关jar包拷贝到spark的jars目录下2.启动hbae各节点3.查看hive中的表4.启动spark-shell获取表

一、Spark  SQL与HBase集成
Spark SQL与HBase集成，其核心就是Spark Sql通过hive外部表来获取HBase的表数据。
1.把hbase，hive，mysql相关jar包拷贝到spark的jars目录下

我使用的是spark2.2.0，根据这些包版本都是基于spa

			
		

	





	

		

			

				
					
flink+hbase+spark_linux.rar

				
			

			

				

					07-24
				

			

		

		

			
				
标题中的"flink+hbase+spark_linux.rar"表明这是一个关于大数据处理框架Flink、分布式数据库HBase以及大数据处理引擎Spark在Linux操作系统环境下的综合应用。这个压缩包可能包含相关的配置文件、示例代码、教程文档...

			
		

	





	

		

			

				
					
Spark-SQL-HBase:利用spark sql在HBase上搭建的sql查询， 支持标准sql查询操作，后续有空闲时间会增加支持插入，删除，建表相关的ddl 语法（rowkey生成策略 部分尚未找到较好的解决方案，hbase查询 table也有待修改[目前暂定为TableMapper]）

				
			

			

				

					05-13
				

			

		

		

			
				
SparkSQLOnHBase
利用spark sql在HBase上搭建的sql查询， 支持标准sql查询操作，后续有空闲时间会增加支持插入，删除，建表相关的ddl 语法（rowkey生成策略 部分尚未找到较好的解决方案，hbase查询 table也有待修改[目前暂定为TableMapper]）。
目前列名部分暂定为cf_qualifier 形式。。。。  例如 cf:1列的查询为 select cf_1 from ***
此版本只支持spark2.0及以上
注: relation目前暂定为CatalogRelation，后续将使用自定义的relation更好的关联hbase操作。

			
		

	





	

		

			

				
					
基于Kafka+SparkStreaming+HBase某信贷实时数据采集存储

				
			

			

				

					02-11
				

			

		

		

			
				
项目以宜信贷风控系统实时数据采集系统为背景，主要描述了技术架构、核心技术难点及代码实现全过程，涉及技术包括但不限于：Kafka\zookeeper,Spark，SparkStreaming，HBase，实时访问技术，爬虫技术等

			
		

	





	

		

			

				
					
Astro —— 华为开源的 SparkSQL on HBase

				
			

			

				

					大数据技术杂谈
				

				

					10-19
					
					2258
					
				

			

		

		

			
				
华为2015年7月20日在O'Reilly Open Source Convention (OSCON) 上宣布Spark SQL on HBase package正式开源。Spark SQL on HBase package 项目又名 Astro，端到端整合了Spark，Spark SQL和HBase的能力，有助于推动帮助Spark进入NoSQL的广泛客户群，并提供强大的在线查询和分析以及在垂直

			
		

	





	

		

			

				
					
Spark:Spark-sql 读hbase

				
			

			

				

					tugangkai的专栏
				

				

					08-03
					
					1962
					
				

			

		

		

			
				
&nbsp;&nbsp;&nbsp;&nbsp; SparkSQL是指整合了Hive的spark-sql cli, 本质上就是通过Hive访问HBase表，具体就是通过hive-hbase-handler, 具体配置参见：Hive(五):hive与hbase整合&nbsp;
目录：



SparkSql 访问 hbase配置
测试验证



SparkSql 访问 hbase配置:




&n...

			
		

	





	

		

			

				
					
sparksql读取hbase

				
			

			

				

					weisongming的博客
				

				

					08-08
					
					1692
					
				

			

		

		

			
				

test("hbase") {
  val sqlContext = SparkSession.builder().getOrCreate().sqlContext
  //拉取hbase数据
  val hBaseConfiguration = HBaseConfiguration.create()
  hBaseConfiguration.set(TableInputFormat.INPUT...

			
		

	





	

		

			

				
					
spark sql——6. spark sql操作hbase

				
			

			

				

					ant的博客
				

				

					05-30
					
					3922
					
				

			

		

		

			
				
目标：

在hbase建一张表，使用spark sql操作它



参考：

https://blog.csdn.net/eyeofeagle/article/details/84571756

https://blog.csdn.net/eyeofeagle/article/details/89943913



hbase数据准备：

进入hbase shell：

[root@hadoo...

			
		

	





	

		

			

				
					
使用SparkSQL读取Hbase表

				
			

			

				

					qazwsx1225a的博客
				

				

					12-05
					
					6775
					
				

			

		

		

			
				
HBase-Spark Connector(在HBase-Spark 模块中）利用了在Spark-1.2.0中引入的DataSource API(SPARK-3247)，在简单的HBase KV存储和复杂的关系型SQL查询之间架起了桥梁，使得用户可以在HBase上使用Spark执行复杂的数据分析工作。HBase Dataframe是一个标准的Spark Dataframe，能够与任何其他的数据源进

			
		

	





	

		

			

				
					
Spark sql之集成Hbase-connectors

				
			

			

				

					南风知我意
				

				

					08-23
					
					758
					
				

			

		

		

			
				
1.spark sql使用hbase-connector整合Hbase，实现df写hbase和读取hbase数据为df
2.hbase-connector源码编译

			
		

	





	

		

			

				
					
spark  sql  hive 写入hbase

				
			

			

				

					05-13
				

			

		

		

			
				
将 Spark SQL 或 Hive 的数据写入 HBase 可以通过两种方式实现：使用 HBase API 或使用 HBase Connector。  1. 使用 HBase API：通过在 Spark 或 Hive 中编写 Java 代码，使用 HBase API 将数据写入 HBase。这种方法...

			
		

	



              




          




        



    





    



      

      

        
      

      





	

		评论	
	

	
	




  

    

      添加红包
      
    

    

      

        
        

          
          
        

        
请填写红包祝福语或标题

      

      

        
        

          
          个
        

        
红包个数最小为10个

      

      

        
        

          
          元
        

        
红包金额最低5元

      

      

        
        

          当前余额3.43元
          前往充值 >
        

      

      

        

          需支付：10.00元
        

        
        
      

    

  





  

    
    
  

  

    
    
  








  

    

      

        

          

            
            
成就一亿技术人!

          

          

        

        

          

          

            领取后你会自动成为博主和红包主的粉丝
            规则
          

        

      

      

        

          

            
              
            
            

              
hope_wisdom
 发出的红包
            

          

        

        

          

          

          

        

      

    

    

  



      
      

      
实付元

      
使用余额支付

      

        

          

            
            点击重新获取
          

        

        
扫码支付

      

      

        
          
            
          
          
        
      

      

        
        钱包余额
          0
          

            
            

              

                
抵扣说明：

                
 1.余额是钱包充值的虚拟货币，按照1:1的比例进行支付金额的抵扣。
 2.余额无法直接购买下载，可以购买VIP、付费专栏及课程。

              

            

          

      

      余额充值