笨鸟的平凡之路-hbase中图片的存储与读写

Hbase中存储图片

话不多说.直接先上代码Demo:

import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.util.ReflectionUtils;

import java.net.URI;

/**
 * Created by xuyi on 2019/6/17
 */
public class SequenceFileTest {
    //HDFS路径
    //picHbase为hdfs保存图片的地址
    static String inpath = "/xuyi/picHbase";
    //out为hdfs图片文件转为Sequence后的文件名
    static String outpath = "/xuyi/out";
    
    static SequenceFile.Writer writer = null;
    static HTable htable = null;

    public static void main(String[] args) throws Exception {

        //inpath = args[0];
        //outpath = args[1];
        //String zklist = args[2];

        //HBase入库
        Configuration hbaseConf = HBaseConfiguration.create();
        hbaseConf.set("hbase.zookeeper.property.clientPort", "2181");
        //ZK列表
        hbaseConf.setStrings("hbase.zookeeper.quorum", "masternode2,masternode1,masternode3");
        //指定表名
        htable = new HTable(hbaseConf, "pic2Hbase");

        //设置读取本地磁盘文件
        Configuration conf = new Configuration();
        conf.addResource(new Path("/Users/xuyi123/Desktop/pic2Hbase/src/main/resources/core-site.xml"));
        conf.addResource(new Path("/Users/xuyi123/Desktop/pic2Hbase/src/main/resources/hdfs-site.xml"));

        URI uri = new URI(inpath);
        FileSystem fileSystem = FileSystem.get(uri, conf, "hdfs");
        //实例化writer对象
        writer = SequenceFile.createWriter(fileSystem, conf, new Path(outpath), Text.class, BytesWritable.class);

        //递归遍历文件夹，并将文件下的文件写入sequenceFile文件
        listFileAndWriteToSequenceFile(fileSystem, inpath);


        //关闭流
        org.apache.hadoop.io.IOUtils.closeStream(writer);


        //读取所有文件
        URI seqURI = new URI(outpath);
        FileSystem fileSystemSeq = FileSystem.get(seqURI, conf);
        SequenceFile.Reader reader = new SequenceFile.Reader(fileSystemSeq, new Path(outpath), conf);

        Text key = new Text();
        BytesWritable val = new BytesWritable();

        key = (Text) ReflectionUtils.newInstance(reader.getKeyClass(), conf);
        val = (BytesWritable) ReflectionUtils.newInstance(reader.getValueClass(), conf);
        
        while (reader.next(key, val)) {
            String temp = key.toString();
      
            //截取文件名作为rowkey
            temp = temp.substring(temp.lastIndexOf("/") + 1);
//            temp = temp.substring(temp.indexOf("Image")+6, temp.indexOf("."));
//            String[] tmp = temp.split("/");
         
            //rowKey 设计
            String rowKey = temp;
//            String rowKey = Integer.valueOf(tmp[0])-1+"_"+Integer.valueOf(tmp[1])/2+"_"+Integer.valueOf(tmp[2])/2;
            //为了测试起见,这里只将文件名作为rowkey
            System.out.println(rowKey);

            //指定ROWKEY的值
            Put put = new Put(Bytes.toBytes(rowKey));
            //指定列簇名称、列修饰符、列值 temp.getBytes()
            put.addColumn("cf".getBytes(), "content".getBytes(), val.getBytes());
            htable.put(put);

        }
        htable.close();
        org.apache.hadoop.io.IOUtils.closeStream(reader);

    }


    /****
     * 递归文件;并将文件写成SequenceFile文件
     * @param fileSystem
     * @param path
     * @throws Exception
     */
    public static void listFileAndWriteToSequenceFile(FileSystem fileSystem, String path) throws Exception {
        final FileStatus[] listStatuses = fileSystem.listStatus(new Path(path));
        for (FileStatus fileStatus : listStatuses) {

            if (fileStatus.isFile()) {
                Text fileText = new Text(fileStatus.getPath().toString());
                System.out.println(fileText.toString());

                //返回一个SequenceFile.Writer实例 需要数据流和path对象 将数据写入了path对象
                FSDataInputStream in = fileSystem.open(new Path(fileText.toString()));
                byte[] buffer = IOUtils.toByteArray(in);
                in.read(buffer);
                BytesWritable value = new BytesWritable(buffer);

                //写成SequenceFile文件
                writer.append(fileText, value);

            }

            if (fileStatus.isDirectory()) {
                listFileAndWriteToSequenceFile(fileSystem, fileStatus.getPath().toString());
            }
        }
    }
}

注意:
1.首先hdfs上有图片文件,比如代码中路径:/xuyi/picHbase

[root@masternode3 xuyi]# hdfs dfs -ls /xuyi/picHbase
Found 2 items
-rw-r--r--   3 root supergroup     189498 2019-06-17 13:58 /xuyi/picHbase/WeChatc5fac7c8d052b3654b8686bdaf700d47.png
-rw-r--r--   3 root supergroup     285593 2019-06-17 13:59 /xuyi/picHbase/WeChatde27ea7ad5cb65a2f9ce128ac17a08db.png

2.查看hdfs图片文件被写成sequence文件

[root@masternode3 xuyi]# hdfs dfs -ls /xuyi/
Found 5 items
-rw-r--r--   3 hdfs supergroup    1039843 2019-06-17 16:28 /xuyi/out

3.去hue查看图片是否被写入到Hbase

---

Hbase中读取图片

同样先上代码Demo:

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.*;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;

/**
 * Created by xuyi on 2019/6/17
 */
public class Pic2Local {
    private static Configuration conf = HBaseConfiguration.create();

    static {
        conf.set("hbase.zookeeper.quorum", "masternode1:2181,masternode2:2181,masternode3:2181");
    }

    public static void scanTable(String tableName) throws IOException {
        String parentPath = "/Users/xuyi123/Desktop/worklog/pic/";
        Connection conn = ConnectionFactory.createConnection(conf);
        HTable table = (HTable) conn.getTable(org.apache.hadoop.hbase.TableName.valueOf(tableName));

        Scan scan = new Scan();
        ResultScanner results = table.getScanner(scan);
        for (Result result : results) {//遍历所有的Rowkey(行)
            for (Cell cell : result.rawCells()) {//遍历某一行中的所有cell

                String rk = new String(CellUtil.cloneRow(cell));//行键
                String cf = new String(CellUtil.cloneFamily(cell));//列族
                String cq = new String(CellUtil.cloneQualifier(cell));//列名

                byte[] imageByte = CellUtil.cloneValue(cell);//图片内容
                
                
                String geoid = rk.substring(0, rk.indexOf("."));//rowkey后缀前名称
                String imageName = rk.substring(0, rk.indexOf("."));//rowkey后缀前名称

				//将要在本地生成的文件地址
                String dirPath = parentPath + geoid;
                File dirFile = new File(dirPath);

				//content为存储图片的列名
                if (cq.equals("content")) {
                //以rowkey后缀名前的内容作为本地存储图片的地址
                    dirFile.mkdir();
                    System.out.println("you image " + imageName);
                    String imgPath = dirPath + "/" + imageName + ".png";//我这里存hbase的时候是png格式
                    File imgFile = new File(imgPath);
                    FileOutputStream foStream = new FileOutputStream(imgFile);
                    foStream.write(imageByte);
                    foStream.close();
                }
            }
        }
        results.close();
        table.close();
        conn.close();
    }

    public static void main(String[] args) throws Exception {
        System.out.println("start");
        scanTable("default:pic2Hbase");//冒号前是hbase表的namespace，后边是表名
    }
}

注意:
1.parentPath是本地将要存储的地址,我这里是MAC,Windows版本的路径格式需要注意
运行代码后: