scala spark hbase 操作案例

最新推荐文章于 2020-12-28 22:50:24 发布

VIP文章小草君

最新推荐文章于 2020-12-28 22:50:24 发布

阅读量6.8k

点赞数 1

分类专栏： scala java

本文链接：https://blog.csdn.net/ldds_520/article/details/51860232

版权

</pre><pre code_snippet_id="1751778" snippet_file_name="blog_20160708_1_3020430" name="code" class="plain">s

1、软件版本Hadoop 2.6.4、 HBase 1.0.3、spark 1.6.0 、 scala-2.10.5

程序jar包版本：

hbase-server.1.0.3、 spark-core_2.10.1.1.0、

2、maven配置文件

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
  <modelVersion>4.0.0</modelVersion>
  <groupId>spark-hbase2</groupId>
  <artifactId>spark-hbase2</artifactId>
  <version>0.0.1-SNAPSHOT</version>
  <name>${project.artifactId}</name>
  <description>My wonderfull scala app</description>
  <inceptionYear>2015</inceptionYear>
  <licenses>
    <license>
      <name>My License</name>
      <url>http://....</url>
      <distribution>repo</distribution>
    </license>
  </licenses> 

	<repositories>
	
		<repository>
			<id>maven-apache-repo2</id>
			<name>Maven Plugin Repository 2</name>
			<url>http://repo2.maven.org/maven2</url>
			<releases>
				<enabled>true</enabled>
			</releases>
			<snapshots>
				<enabled>false</enabled>
			</snapshots>
		</repository>
      
		 <repository>
			<id>maven-centeral</id>
			<name>Maven Plugin Repository 2</name>
			<url>http://central.maven.org/maven2</url>
			<releases>
				<enabled>true</enabled>
			</releases> 
			<snapshots>
				<enabled>false</enabled>
			</snapshots> 
		</repository> 
		<repository>
			<id>maven-com-sun</id>
			<name>Maven2 repository of SUN</name>
			<url>http://download.java.net/maven/2</url>
			<releases>
				<enabled>true</enabled>
			</releases>
			<snapshots>
				<enabled>false</enabled>
			</snapshots>
		</repository>
		<repository>
			<id>logicaldoc.sourceforge.net</id>
			<name>logicaldoc.sourceforge.net</name>
			<url>http://logicaldoc.sourceforge.net/maven/</url>
			<releases>
				<enabled>true</enabled>
			</releases>
			<snapshots>
				<enabled>false</enabled>
			</snapshots>
		</repository>
		<repository>
			<id>server.oosnmp.net</id>
			<name>server.oosnmp.net</name>
			<url>https://server.oosnmp.net/dist/release</url>
			<releases>
				<enabled>true</enabled>
			</releases>
			<snapshots>
				<enabled>false</enabled>
			</snapshots>
		</repository>
		<repository>
			<id>codehaus.org</id>
			<name>codehaus.org</name>
			<url>http://repository.codehaus.org</url>
			<releases>
				<enabled>true</enabled>
			</releases>
			<snapshots>
				<enabled>false</enabled>
			</snapshots>
		</repository>
		<repository>
			<id>sonatype.org</id>
			<name>sonatype.org</name>
			<url>https://oss.sonatype.org/content/groups/jetty</url>
			<releases>
				<enabled>true</enabled>
			</releases>
			<snapshots>
				<enabled>false</enabled>
			</snapshots>
		</repository>
	</repositories>

  <properties>
    <maven.compiler.source>1.6</maven.compiler.source>
    <maven.compiler.target>1.6</maven.compiler.target>
    <encoding>UTF-8</encoding>
    <scala.version>2.11.5</scala.version>
    <scala.compat.version>2.11</scala.compat.version>
  </properties>

  <dependencies>
  
    <dependency>
      <groupId>org.scala-lang</groupId>
      <artifactId>scala-library</artifactId>
      <version>${scala.version}</version>
    </dependency>

    <!-- Test -->
    <dependency>
      <groupId>junit</groupId>
      <artifactId>junit</artifactId>
      <version>4.11</version>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>org.specs2</groupId>
      <artifactId>specs2-core_${scala.compat.version}</artifactId>
      <version>2.4.16</version>
      <scope>test</scope>
    </dependency> 
    <dependency>
      <groupId>org.scalatest</groupId>
      <artifactId>scalatest_${scala.compat.version}</artifactId>
      <version>2.2.4</version>
      <scope>test</scope>
    </dependency>
    
<!--      <dependency>
		<groupId>org.apache.hbase</groupId>
		<artifactId>hbase-client</artifactId>  
		<version>1.0.3</version>
	</dependency>  
    
	<dependency>
		<groupId>org.apache.hbase</groupId>
		<artifactId>hbase-server</artifactId>
		<version>1.0.3</version>
	</dependency>   -->
   <dependency>
		<groupId>org.apache.hbase</groupId>
		<artifactId>hbase-server</artifactId>
		<version>1.0.3</version>
	</dependency> 
	     <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-hdfs -->
<!-- 	<dependency>
	    <groupId>org.apache.hadoop</groupId>
	    <artifactId>hadoop-client</artifactId>
	    <version>2.6.0</version>
	</dependency> 
  -->  
    <dependency>
			<groupId>org.apache.spark</groupId>
			<artifactId>spark-core_2.10</artifactId>
			<version>1.1.0</version>
	</dependency>  
    
    <dependency>
   		 <groupId>jdk.tools</groupId>
   		 <artifactId>jdk.tools</artifactId>
   		 <version>1.6</version>
   		 <scope>system</scope>
   		 <systemPath>${JAVA_HOME}/lib/tools.jar</systemPath>
	</dependency>
    
  </dependencies>

 <build>
    <sourceDirectory>src/main/scala</sourceDirectory>
    <testSourceDirectory>src/test/scala</testSourceDirectory>
    <plugins>
       <plugin>
        <groupId>org.scala-tools</groupId>
        <artifactId>maven-scala-plugin</artifactId>
        <version>2.15.2</version&