从hadoop2.2，HBase0.96 mapreduce操作

最新推荐文章于 2016-08-29 21:17:12 发布

hxpjava1

最新推荐文章于 2016-08-29 21:17:12 发布

阅读量2.8k

点赞数

分类专栏： hadoop

本文链接：https://blog.csdn.net/hxpjava1/article/details/20047077

版权

hadoop 专栏收录该内容

73 篇文章 0 订阅

订阅专栏

发布一个k8s部署视频：https://edu.csdn.net/course/detail/26967

课程内容：各种k8s部署方式。包括minikube部署，kubeadm部署，kubeasz部署，rancher部署，k3s部署。包括开发测试环境部署k8s，和生产环境部署k8s。

腾讯课堂连接地址https://ke.qq.com/course/478827?taid=4373109931462251&tuin=ba64518

第二个视频发布 https://edu.csdn.net/course/detail/27109

腾讯课堂连接地址https://ke.qq.com/course/484107?tuin=ba64518

介绍主要的k8s资源的使用配置和命令。包括configmap，pod，service，replicaset，namespace，deployment，daemonset，ingress，pv，pvc，sc，role，rolebinding，clusterrole，clusterrolebinding，secret，serviceaccount，statefulset，job，cronjob，podDisruptionbudget，podSecurityPolicy,networkPolicy,resourceQuota，limitrange，endpoint，event，conponentstatus，node，apiservice，controllerRevision等。

第三个视频发布：https://edu.csdn.net/course/detail/27574

详细介绍helm命令，学习helm chart语法，编写helm chart。深入分析各项目源码，学习编写helm插件
————————————————------------------------------------------------------------------------------------------------------------------

从hadoop取出文件写入hbase表中

package example2;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.hbase.mapreduce.TableOutputFormat;
import org.apache.hadoop.hbase.mapreduce.TableReducer;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.util.GenericOptionsParser;

//Administrator
public class ImportFromFileExample {
	
	public static class ImportMapper extends Mapper<LongWritable, Text, ImmutableBytesWritable,Text >{
		
		
		@Override
		protected void map(LongWritable key, Text value, Context context)
				throws IOException, InterruptedException {
			context.write(new ImmutableBytesWritable(Bytes.toBytes(key.get())), value);
		}

		
	}
	
	public static class Reducer1 extends TableReducer<ImmutableBytesWritable, Text, ImmutableBytesWritable> {
		private byte[] family=null;
		private byte[]qualifier=null;
		

		@Override
		protected void setup(Context context) throws IOException,
				InterruptedException {
			String column=context.getConfiguration().get("conf.column");
			byte[][]colkey=KeyValue.parseColumn(Bytes.toBytes(column));
			family=colkey[0];
			if(colkey.length>1){
				qualifier=colkey[1];
			}
		}
		
        public void reduce(ImmutableBytesWritable key, Iterable<Text> values, Context context)
                throws IOException, InterruptedException {
        	String valueCon=null;
        	for(Text text:values){
        		valueCon+=text.toString();
        	}
            Put put = new Put(key.get());
            put.add(family, qualifier, Bytes.toBytes(valueCon));
            context.write(key, put);
        }
    }

	/**
	 * @param args
	 */
	public static void main(String[] args)throws Exception {
		Configuration conf=HBaseConfiguration.create();
		String []argArray=new GenericOptionsParser(conf, args).getRemainingArgs();
		if(argArray.length!=1){
			System.exit(1);
		}
		conf.set("conf.column", "family1:text");
		Job job=new Job(conf,"import from hdfs to hbase");
		job.setJarByClass(ImportFromFileExample.class);
		job.setMapperClass(ImportMapper.class);
		job.setOutputFormatClass(TableOutputFormat.class);
		job.getConfiguration().set(TableOutputFormat.OUTPUT_TABLE, "testtable");
		job.setMapOutputKeyClass(ImmutableBytesWritable.class);
		job.setMapOutputValueClass(Text.class);
		job.setOutputKeyClass(ImmutableBytesWritable.class);
		job.setOutputValueClass(Put.class);
		TableMapReduceUtil.initTableReducerJob("testtable", Reducer1.class, job);
		FileInputFormat.addInputPaths(job, argArray[0]);
		System.exit(job.waitForCompletion(true) ? 0 : 1);

	}

}

2从hbase加载数据到hdfs

package example2;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.hbase.mapreduce.TableMapper;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;

//Administrator
public class FromHBaseToHDFSExample {
	
	public static class HBaseMapper extends TableMapper<IntWritable, Text>{

		@Override
		protected void map(ImmutableBytesWritable key, Result value,
				Context context) throws IOException, InterruptedException {
			 for(KeyValue kv:value.raw()){
				 context.write(null, new Text(Bytes.toString(kv.getValue())));
			 }
		}
		
	}

	/**
	 * @param args
	 */
	public static void main(String[] args)throws Exception {
		Configuration conf=HBaseConfiguration.create();
		String []argArray=new GenericOptionsParser(conf, args).getRemainingArgs();
		if(argArray.length!=1){
			System.exit(1);
		}
		Job job=new Job(conf,"import hbase to hdfs");
		job.setJarByClass(FromHBaseToHDFSExample.class);
		TableMapReduceUtil.initTableMapperJob("testtable", new Scan(), HBaseMapper.class, IntWritable.class, Text.class, job);
		FileOutputFormat.setOutputPath(job, new Path(argArray[0]));
		job.setNumReduceTasks(0);
		System.exit(job.waitForCompletion(true) ? 0 : 1);
		

	}

}

3hbase 到hbase

package example2;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.hbase.mapreduce.TableMapper;
import org.apache.hadoop.hbase.mapreduce.TableReducer;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;

//Administrator
public class FromHBaseToHBaseExample {
	
	public static class HBaseMapper extends TableMapper<ImmutableBytesWritable, Text>{

		@Override
		protected void map(ImmutableBytesWritable key, Result value,
				Context context) throws IOException, InterruptedException {
			for(Cell kv:value.rawCells()){
				Text out=new Text(Bytes.toString(kv.getFamilyArray())+"|"+Bytes.toString(kv.getQualifierArray())+"|"+Bytes.toString(kv.getValueArray()));
				context.write(new ImmutableBytesWritable(kv.getRowArray()), out);
			}
		}
	}
	
	public static class HBaseReducer extends TableReducer<ImmutableBytesWritable, Text, ImmutableBytesWritable>{

		@Override
		protected void reduce(ImmutableBytesWritable key, Iterable<Text> values,
				Context context)
				throws IOException, InterruptedException {
			for(Text value:values){
				String text=value.toString();
				String [] textArray=text.split("|");
				Put put=new Put(key.get());
				System.out.println("---------------------------------------"+Bytes.toBytes(textArray[0]));
				put.add(Bytes.toBytes("family1"), Bytes.toBytes(textArray[1]), Bytes.toBytes(textArray[2]));
				context.write(null, put);
			}
			
		}
	}

	/**
	 * @param args
	 */
	public static void main(String[] args)throws Exception {
		Configuration conf=HBaseConfiguration.create();
		//String [] argArray=new GenericOptionsParser(conf, args).getRemainingArgs();
		conf.set("from.table", "testtable");
		conf.set("to.table", "hbase");
		//conf.set("family", "family1");
		Job job=new Job(conf,"hbase to hbase");
		job.setJarByClass(FromHBaseToHBaseExample.class);
		TableMapReduceUtil.initTableMapperJob(conf.get("from.table"), new Scan(), HBaseMapper.class,ImmutableBytesWritable.class, Text.class, job);
		TableMapReduceUtil.initTableReducerJob(conf.get("to.table"), HBaseReducer.class, job);
		System.exit(job.waitForCompletion(true)?0:1);
		

	}

}