import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.hbase.mapreduce.TableMapper;
import org.apache.hadoop.hbase.mapreduce.TableReducer;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import java.io.IOException;
public class HBaseToHdfs {
public static void main(String[] args)throws Exception {
Configuration conf = HBaseConfiguration.create();
conf.set("hbase.zookeeper.quorum","master:2181,slave1:2181,slave2:2181");//当然这些都可以作为参数传入,这里只是实验,所以写死在了代码里,实际成产过程中肯定要用参数的方式
conf.set("hbase.rootdir","hdfs://master:9000/hbase");
// conf.set(TableOutputFormat.OUTPUT_TABLE,"im");
Job job = Job.getInstance(conf,HBaseToHdfs.class.getSimpleName());
String outPath = "C:\\Users\\Administrator\\Desktop\\member.csv";
Path path = new Path(outPath);
FileSystem fs = FileSystem.get(conf);
if (fs.exists(path)) {
System.out.println("path 存在 删除!!"+path);
fs.delete(path, true);
}
FileOutputFormat.setOutputPath(job, new Path(outPath));
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
// job.setReducerClass(HdfsToHBaseReducer.class);
Scan scan = new Scan();
String cf[]={"address","address","id","info","info","info"};
String qualifier[]={"city","country","","age","birthday","industry"};
for (int i = 0; i < cf.length; i++) {
scan.addColumn(Bytes.toBytes(cf[i]),Bytes.toBytes(qualifier[i]));
}
TableMapReduceUtil.initTableMapperJob("member", scan,
HBaseToHdfs.HdfsToHBaseMapper.class, Text.class,
Text.class, job);
System.out.println("ok................");
job.waitForCompletion(true);
}
// <ImmutableBytesWritable, Result ,Text,Text>
public static class HdfsToHBaseMapper extends TableMapper<Text, Text> {
private String Id;
private String columnFamily;
private String qualifier;
private Long timestamp;
private Object val;
private Cell[] kv;
private static Text keyOut = new Text();
private static Text valOut = new Text();
private static int count = 0;
@Override
protected void map(ImmutableBytesWritable key, Result value, Context context)throws IOException,InterruptedException {
String fam = "";
String va = "";
Id = new String(key.get());
kv = value.rawCells();
for (int i = 0; i < kv.length; i++) {
timestamp = kv[i].getTimestamp();
val = new String(kv[i].getValue());
qualifier = new String(kv[i].getQualifier());
columnFamily = new String(kv[i].getFamily());
fam += columnFamily + ":" + qualifier +",";
va +=val+"-"+timestamp+",";
}
keyOut.set(Id);
if(count <= 0) {
valOut.set(fam);
context.write(new Text(), valOut);
}
count++;
valOut.set(va);
context.write(keyOut,valOut);
System.out.println("map.............");
}
}
public static class HdfsToHBaseReducer extends TableReducer<Text,Text,Text> {
@Override
protected void reduce(Text k2, Iterable<Text> v2s, Context context) throws IOException, InterruptedException {
}
}
}
hbase 导出 文件
最新推荐文章于 2024-02-03 16:47:52 发布