Hadoop 上传小文件 合成sequencefile 记录

    在Hadoop中，支持对二进制文件的处理，而sequencefile的方法，便是其中的重点。

    以下为实践中实现的经验：（本次记录上传文件。）

package test;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Text;

public class SequenceFileWriteDemo {
	private static String[] Data={
		"one,two show one",
		"three,four show two",
		"five,six show three",
		"seven,eight show four",
		"nine,ten show five"
	};
	public void writeText() throws IOException{
		String uri="hdfs://192.168.50.28:8020/user/root/jyl/testImageSequenceFile";
		Configuration con=new Configuration();
		FileSystem fs=FileSystem.get(URI.create(uri), con);
		Path path=new Path("uri");
		IntWritable key=new IntWritable();
		Text value=new Text();
		SequenceFile.Writer writer=null;
		writer=SequenceFile.createWriter(fs, con, path, key.getClass(), value.getClass());
		for(int i=0;i<100;i++){
			key.set(100-i);
			value.set(Data[i%Data.length]);
			System.out.printf("[%s]\t%s\t%s\n",writer.getLength(),key,value);
			writer.append(key, value);
		}
		IOUtils.closeStream(writer);
	}
	
	public void writeImage() throws IOException{
		String uri="hdfs://192.168.50.28:8020/user/root/jyl/testByteImageSequenceFile";
		Configuration con=new Configuration();
		FileSystem fs=FileSystem.get(URI.create(uri), con);
		Path path=new Path(uri);
		BytesWritable key=new BytesWritable();
		
		BytesWritable value=new BytesWritable();
		
		File file1=new File("/mnt/disk1/yl/images/zhouzhou.jpg");
		File file2=new File("/mnt/disk1/yl/images/gouhuo.jpg");
		
		InputStream in1=new FileInputStream(file1);
		InputStream in2=new FileInputStream(file2);
		
		byte[] byte1=new byte[(int) file1.length()];
		byte[] byte2=new byte[(int) file2.length()];
		
		
		in1.read(byte1);
		in2.read(byte2);
		
		SequenceFile.Writer writer=null;
		writer=SequenceFile.createWriter(fs, con, path, BytesWritable.class,value.getClass());
		
		byte[] b1=new byte[1];
		b1[0]=1;
		BytesWritable bw1 = new BytesWritable(b1);
		
		byte[] b2=new byte[1];
		b2[0]=2;
		BytesWritable bw2 = new BytesWritable(b2);
		
		
		value.set(byte1, 0, byte1.length);
		writer.append(bw1,value);
		value.set(byte2, 0, byte2.length);
		writer.append(bw2,value);
		
		IOUtils.closeStream(writer);
	}
	
	public static void main(String[] args) throws IOException {
		SequenceFileWriteDemo demo=new SequenceFileWriteDemo();
		demo.writeImage();
//		demo.writeText();
	}
}