Hbase中存储图片
话不多说.直接先上代码Demo:
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.util.ReflectionUtils;
import java.net.URI;
/**
* Created by xuyi on 2019/6/17
*/
public class SequenceFileTest {
//HDFS路径
//picHbase为hdfs保存图片的地址
static String inpath = "/xuyi/picHbase";
//out为hdfs图片文件转为Sequence后的文件名
static String outpath = "/xuyi/out";
static SequenceFile.Writer writer = null;
static HTable htable = null;
public static void main(String[] args) throws Exception {
//inpath = args[0];
//outpath = args[1];
//String zklist = args[2];
//HBase入库
Configuration hbaseConf = HBaseConfiguration.create();
hbaseConf.set("hbase.zookeeper.property.clientPort", "2181");
//ZK列表
hbaseConf.setStrings("hbase.zookeeper.quorum", "masternode2,masternode1,masternode3");
//指定表名
htable = new HTable(hbaseConf, "pic2Hbase");
//设置读取本地磁盘文件
Configuration conf = new Configuration();
conf.addResource(new Path("/Users/xuyi123/Desktop/pic2Hbase/src/main/resources/core-site.xml"));
conf.addResource(new Path("/Users/xuyi123/Desktop/pic2Hbase/src/main/resources/hdfs-site.xml"));
URI uri = new URI(inpath);
FileSystem fileSystem = FileSystem.get(uri, conf, "hdfs");
//实例化writer对象
writer = SequenceFile.createWriter(fileSystem, conf, new Path(outpath), Text.class, BytesWritable.class);
//递归遍历文件夹,并将文件下的文件写入sequenceFile文件
listFileAndWriteToSequenceFile(fileSystem, inpath);
//关闭流
org.apache.hadoop.io.IOUtils.closeStream(writer);
//读取所有文件
URI seqURI = new URI(outpath);
FileSystem fileSystemSeq = FileSystem.get(seqURI, conf);
SequenceFile.Reader reader = new SequenceFile.Reader(fileSystemSeq, new Path(outpath), conf);
Text key = new Text();
BytesWritable val = new BytesWritable();
key = (Text) ReflectionUtils.newInstance(reader.getKeyClass(), conf);
val = (BytesWritable) ReflectionUtils.newInstance(reader.getValueClass(), conf);
while (reader.next(key, val)) {
String temp = key.toString();
//截取文件名作为rowkey
temp = temp.substring(temp.lastIndexOf("/") + 1);
// temp = temp.substring(temp.indexOf("Image")+6, temp.indexOf("."));
// String[] tmp = temp.split("/");
//rowKey 设计
String rowKey = temp;
// String rowKey = Integer.valueOf(tmp[0])-1+"_"+Integer.valueOf(tmp[1])/2+"_"+Integer.valueOf(tmp[2])/2;
//为了测试起见,这里只将文件名作为rowkey
System.out.println(rowKey);
//指定ROWKEY的值
Put put = new Put(Bytes.toBytes(rowKey));
//指定列簇名称、列修饰符、列值 temp.getBytes()
put.addColumn("cf".getBytes(), "content".getBytes(), val.getBytes());
htable.put(put);
}
htable.close();
org.apache.hadoop.io.IOUtils.closeStream(reader);
}
/****
* 递归文件;并将文件写成SequenceFile文件
* @param fileSystem
* @param path
* @throws Exception
*/
public static void listFileAndWriteToSequenceFile(FileSystem fileSystem, String path) throws Exception {
final FileStatus[] listStatuses = fileSystem.listStatus(new Path(path));
for (FileStatus fileStatus : listStatuses) {
if (fileStatus.isFile()) {
Text fileText = new Text(fileStatus.getPath().toString());
System.out.println(fileText.toString());
//返回一个SequenceFile.Writer实例 需要数据流和path对象 将数据写入了path对象
FSDataInputStream in = fileSystem.open(new Path(fileText.toString()));
byte[] buffer = IOUtils.toByteArray(in);
in.read(buffer);
BytesWritable value = new BytesWritable(buffer);
//写成SequenceFile文件
writer.append(fileText, value);
}
if (fileStatus.isDirectory()) {
listFileAndWriteToSequenceFile(fileSystem, fileStatus.getPath().toString());
}
}
}
}
注意:
1.首先hdfs上有图片文件,比如代码中路径:/xuyi/picHbase
[root@masternode3 xuyi]# hdfs dfs -ls /xuyi/picHbase
Found 2 items
-rw-r--r-- 3 root supergroup 189498 2019-06-17 13:58 /xuyi/picHbase/WeChatc5fac7c8d052b3654b8686bdaf700d47.png
-rw-r--r-- 3 root supergroup 285593 2019-06-17 13:59 /xuyi/picHbase/WeChatde27ea7ad5cb65a2f9ce128ac17a08db.png
2.查看hdfs图片文件被写成sequence文件
[root@masternode3 xuyi]# hdfs dfs -ls /xuyi/
Found 5 items
-rw-r--r-- 3 hdfs supergroup 1039843 2019-06-17 16:28 /xuyi/out
3.去hue查看图片是否被写入到Hbase
Hbase中读取图片
同样先上代码Demo:
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.*;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
/**
* Created by xuyi on 2019/6/17
*/
public class Pic2Local {
private static Configuration conf = HBaseConfiguration.create();
static {
conf.set("hbase.zookeeper.quorum", "masternode1:2181,masternode2:2181,masternode3:2181");
}
public static void scanTable(String tableName) throws IOException {
String parentPath = "/Users/xuyi123/Desktop/worklog/pic/";
Connection conn = ConnectionFactory.createConnection(conf);
HTable table = (HTable) conn.getTable(org.apache.hadoop.hbase.TableName.valueOf(tableName));
Scan scan = new Scan();
ResultScanner results = table.getScanner(scan);
for (Result result : results) {//遍历所有的Rowkey(行)
for (Cell cell : result.rawCells()) {//遍历某一行中的所有cell
String rk = new String(CellUtil.cloneRow(cell));//行键
String cf = new String(CellUtil.cloneFamily(cell));//列族
String cq = new String(CellUtil.cloneQualifier(cell));//列名
byte[] imageByte = CellUtil.cloneValue(cell);//图片内容
String geoid = rk.substring(0, rk.indexOf("."));//rowkey后缀前名称
String imageName = rk.substring(0, rk.indexOf("."));//rowkey后缀前名称
//将要在本地生成的文件地址
String dirPath = parentPath + geoid;
File dirFile = new File(dirPath);
//content为存储图片的列名
if (cq.equals("content")) {
//以rowkey后缀名前的内容作为本地存储图片的地址
dirFile.mkdir();
System.out.println("you image " + imageName);
String imgPath = dirPath + "/" + imageName + ".png";//我这里存hbase的时候是png格式
File imgFile = new File(imgPath);
FileOutputStream foStream = new FileOutputStream(imgFile);
foStream.write(imageByte);
foStream.close();
}
}
}
results.close();
table.close();
conn.close();
}
public static void main(String[] args) throws Exception {
System.out.println("start");
scanTable("default:pic2Hbase");//冒号前是hbase表的namespace,后边是表名
}
}
注意:
1.parentPath是本地将要存储的地址,我这里是MAC,Windows版本的路径格式需要注意
运行代码后: