<dependency> <groupId>org.apache.hbase</groupId> <artifactId>hbase-client</artifactId> <version>2.3.0</version> </dependency> <dependency> <groupId>org.apache.hbase</groupId> <artifactId>hbase-server</artifactId> <version>2.3.0</version> </dependency> <!-- https://mvnrepository.com/artifact/org.apache.hbase/hbase-mapreduce --> <dependency> <groupId>org.apache.hbase</groupId> <artifactId>hbase-mapreduce</artifactId> <version>2.3.0</version> </dependency>
1.首先搭建hbase集群
habse是基于zookper搭建的
先上传jdk的rpm文件
jdk-8u211-linux-x64.rpm peiyajie@10.202.234.56:/home/peiyajie
rpm -ivh jdk-8u211-linux-x64.rpm
sudo rpm -ivh jdk-8u211-linux-x64.rpm
然后完成安装、
解压zookper的安装文件
进入conf
cp zoo_sample.cfg zoo.cfg
在zoo.cfg增加:
server.2=10.202.234.233:2888:3888
server.3=10.202.234.56:2888:3888
server.1=10.202.234.244:2888:3888
修改
dataDir=/home/peiyajie/zookeeper-3.4.14/data
新建/home/peiyajie/zookeeper-3.4.14/data下面myid里面的值和集群的id对应
然后解压hhase的压缩文件
修改hbase-site.xml并且新增
<property>
<name>hbase.zookeeper.quorum</name>
<value>10.202.234.244,10.202.234.233,10.202.234.56</value>
<description>The directory shared by RegionServers.
</description>
</property>
<property>
<name>hbase.rootdir</name>
<value>hdfs://10.202.234.244:9000/hbase</value>
<description>The directory shared by RegionServers.
</description>
</property>
<property>
<name>hbase.cluster.distributed</name>
<value>true</value>
<description>The mode the cluster will be in. Possible values are
false: standalone and pseudo-distributed setups with managed Zookeeper
true: fully-distributed with unmanaged Zookeeper Quorum (see hbase-env.sh)
</description>
</property>
<property>
<name>hbase.master.info.port</name>
<value>60010</value>
</property>
在hbase-env.sh里面新增
export JAVA_HOME=/usr/java/jdk1.8.0_211-amd64
export HBASE_MANAGES_ZK=false
上代码:
package com.qihoo.hadoop.util; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Mapper; import java.io.IOException; public class WCHbaseMapper extends Mapper<LongWritable, Text,Text, NullWritable> { @Override protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { context.write(value,NullWritable.get()); } }
package com.qihoo.hadoop.util; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.mapreduce.TableReducer; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.Text; import java.io.IOException; /** * */ public class WCHbaseReducer extends TableReducer<Text, NullWritable, ImmutableBytesWritable> { @Override protected void reduce(Text key, Iterable<NullWritable> values, Context context) throws IOException, InterruptedException { String[] split = key.toString().split(" "); Put put = new Put(Bytes.toBytes(split[0])); put.addColumn("info".getBytes(),"name".getBytes(),split[1].getBytes()); put.addColumn("info".getBytes(),"age".getBytes(),Bytes.toBytes(Integer.parseInt(split[2]))); context.write(new ImmutableBytesWritable(Bytes.toBytes(split[0])),put); } }
package com.qihoo.hadoop.util; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import java.io.IOException; public class HbaseUtil { public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException { Configuration conf = new Configuration(); conf.set("hbase.zookeeper.quorum","10.202.234.244,10.202.234.233,10.202.234.56"); conf.set("hbase.zookeeper.property.clientPort", "2181"); Job job=Job.getInstance(conf, "Runner"); // Path path = new Path("hdfs://10.202.234.244:9000/wordcount/inputpeiyajie/3.txt"); FileInputFormat.addInputPath(job, path); job.setJarByClass(HbaseUtil.class) ; job.setMapOutputKeyClass(ImmutableBytesWritable.class) ; job.setMapOutputValueClass(Put.class) ; job.setMapperClass(WCHbaseMapper.class) ; // job.setReducerClass(WCHbaseReducer.class) ; // org.apache.hadoop.mapreduce.lib.output.FileOutputFormat.setOutputPath(job, new Path("D:/testFile/mr1/" + new Date().getTime())) ; TableMapReduceUtil.initTableReducerJob("member", WCHbaseReducer.class, job) ; job.waitForCompletion(true); } }