HDFS之SequenceFile的读写操作

本文代码参考hadoop权威指南里第四章hadoop的i/o操作

SequenceFile写操作：

import java.io.IOException;
import java.net.URI;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Text;
//sequenceFile追加操作byte
public class SequenceFileWriteDemo {
	private static final String[] data = {
		"kcbhducdi","jhcui","lashdcuohco"
	};
	public static void main(String args[]){
		String uri = args[0];
		Configuration conf = new Configuration();
		FileSystem fs = null;;
		try {
			fs = FileSystem.get(URI.create(uri), conf);
		} catch (IOException e1) {
			// TODO Auto-generated catch block
			e1.printStackTrace();
		}
		Path path = new Path(uri);
		
		IntWritable key = new IntWritable();
		Text value = new Text();
		SequenceFile.Writer write = null;
		try {
			write = SequenceFile.createWriter(fs, conf, path,key.getClass(),value.getClass());
		for(int i=0;i< 100;i++){
			key.set(100-i);
			value.set(data[i%data.length]);
			System.out.printf("[%d]\t%s\t%s\n",write.getLength(),key,value);
			write.append(key, value);
		} 
		}catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}finally{IOUtils.closeStream(write);}
		}
	}

输出：

[128]	100	kcbhducdi
[158]	99	jhcui
[184]	98	lashdcuohco
[216]	97	kcbhducdi
[246]	96	jhcui
[272]	95	lashdcuohco
[304]	94	kcbhducdi
[334]	93	jhcui
[360]	92	lashdcuohco
[392]	91	kcbhducdi
[422]	90	jhcui
[448]	89	lashdcuohco
[480]	88	kcbhducdi
[510]	87	jhcui
[536]	86	lashdcuohco
[568]	85	kcbhducdi
[598]	84	jhcui
[624]	83	lashdcuohco
[656]	82	kcbhducdi
[686]	81	jhcui
[712]	80	lashdcuohco
[744]	79	kcbhducdi
[774]	78	jhcui
[800]	77	lashdcuohco
[832]	76	kcbhducdi
[862]	75	jhcui
[888]	74	lashdcuohco
[920]	73	kcbhducdi
[950]	72	jhcui
[976]	71	lashdcuohco
[1008]	70	kcbhducdi
[1038]	69	jhcui
[1064]	68	lashdcuohco
[1096]	67	kcbhducdi
[1126]	66	jhcui
[1152]	65	lashdcuohco
[1184]	64	kcbhducdi
[1214]	63	jhcui
[1240]	62	lashdcuohco
[1272]	61	kcbhducdi
[1302]	60	jhcui
[1328]	59	lashdcuohco
[1360]	58	kcbhducdi
[1390]	57	jhcui
[1416]	56	lashdcuohco
[1448]	55	kcbhducdi
[1478]	54	jhcui
[1504]	53	lashdcuohco
[1536]	52	kcbhducdi
[1566]	51	jhcui
[1592]	50	lashdcuohco
[1624]	49	kcbhducdi
[1654]	48	jhcui
[1680]	47	lashdcuohco
[1712]	46	kcbhducdi
[1742]	45	jhcui
[1768]	44	lashdcuohco
[1800]	43	kcbhducdi
[1830]	42	jhcui
[1856]	41	lashdcuohco
[1888]	40	kcbhducdi
[1918]	39	jhcui
[1944]	38	lashdcuohco
[1976]	37	kcbhducdi
[2006]	36	jhcui
[2052]	35	lashdcuohco
[2084]	34	kcbhducdi
[2114]	33	jhcui
[2140]	32	lashdcuohco
[2172]	31	kcbhducdi
[2202]	30	jhcui
[2228]	29	lashdcuohco
[2260]	28	kcbhducdi
[2290]	27	jhcui
[2316]	26	lashdcuohco
[2348]	25	kcbhducdi
[2378]	24	jhcui
[2404]	23	lashdcuohco
[2436]	22	kcbhducdi
[2466]	21	jhcui
[2492]	20	lashdcuohco
[2524]	19	kcbhducdi
[2554]	18	jhcui
[2580]	17	lashdcuohco
[2612]	16	kcbhducdi
[2642]	15	jhcui
[2668]	14	lashdcuohco
[2700]	13	kcbhducdi
[2730]	12	jhcui
[2756]	11	lashdcuohco
[2788]	10	kcbhducdi
[2818]	9	jhcui
[2844]	8	lashdcuohco
[2876]	7	kcbhducdi
[2906]	6	jhcui
[2932]	5	lashdcuohco
[2964]	4	kcbhducdi
[2994]	3	jhcui
[3020]	2	lashdcuohco
[3052]	1	kcbhducdi

SequenceFile读操作：

import java.io.IOException;
import java.net.URI;

import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.util.ReflectionUtils;

//sequenceFile读取操作
public class SequenceFileReadDemo {
	public static void main(String args[]) throws IOException{
		String uri = args[0];
		Configuration conf = new Configuration();
		FileSystem fs = FileSystem.get(URI.create(uri),conf);
		Path path = new Path(uri);
		SequenceFile.Reader reader = null;
		try {
			reader = new SequenceFile.Reader(fs, path, conf);
		Writable key = (Writable)ReflectionUtils.newInstance(reader.getKeyClass(), conf);
		Writable value = (Writable)ReflectionUtils.newInstance(reader.getValueClass(), conf);
		long position = reader.getPosition();
		while(reader.next(key,value)){
			String syncSeen = reader.syncSeen() ? "*" : "" ;//set the reader mark
			System.out.printf("[%s%s]\t%s\t%s\n",position,syncSeen,key,value);
			position = reader.getPosition();
		}
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}finally{
			reader.close();
			//IOUtils.closeStream(reader);;
		}	
	}
}

输出：

[128]	100	kcbhducdi
[158]	99	jhcui
[184]	98	lashdcuohco
[216]	97	kcbhducdi
[246]	96	jhcui
[272]	95	lashdcuohco
[304]	94	kcbhducdi
[334]	93	jhcui
[360]	92	lashdcuohco
[392]	91	kcbhducdi
[422]	90	jhcui
[448]	89	lashdcuohco
[480]	88	kcbhducdi
[510]	87	jhcui
[536]	86	lashdcuohco
[568]	85	kcbhducdi
[598]	84	jhcui
[624]	83	lashdcuohco
[656]	82	kcbhducdi
[686]	81	jhcui
[712]	80	lashdcuohco
[744]	79	kcbhducdi
[774]	78	jhcui
[800]	77	lashdcuohco
[832]	76	kcbhducdi
[862]	75	jhcui
[888]	74	lashdcuohco
[920]	73	kcbhducdi
[950]	72	jhcui
[976]	71	lashdcuohco
[1008]	70	kcbhducdi
[1038]	69	jhcui
[1064]	68	lashdcuohco
[1096]	67	kcbhducdi
[1126]	66	jhcui
[1152]	65	lashdcuohco
[1184]	64	kcbhducdi
[1214]	63	jhcui
[1240]	62	lashdcuohco
[1272]	61	kcbhducdi
[1302]	60	jhcui
[1328]	59	lashdcuohco
[1360]	58	kcbhducdi
[1390]	57	jhcui
[1416]	56	lashdcuohco
[1448]	55	kcbhducdi
[1478]	54	jhcui
[1504]	53	lashdcuohco
[1536]	52	kcbhducdi
[1566]	51	jhcui
[1592]	50	lashdcuohco
[1624]	49	kcbhducdi
[1654]	48	jhcui
[1680]	47	lashdcuohco
[1712]	46	kcbhducdi
[1742]	45	jhcui
[1768]	44	lashdcuohco
[1800]	43	kcbhducdi
[1830]	42	jhcui
[1856]	41	lashdcuohco
[1888]	40	kcbhducdi
[1918]	39	jhcui
[1944]	38	lashdcuohco
[1976]	37	kcbhducdi
[2006*]	36	jhcui
[2052]	35	lashdcuohco
[2084]	34	kcbhducdi
[2114]	33	jhcui
[2140]	32	lashdcuohco
[2172]	31	kcbhducdi
[2202]	30	jhcui
[2228]	29	lashdcuohco
[2260]	28	kcbhducdi
[2290]	27	jhcui
[2316]	26	lashdcuohco
[2348]	25	kcbhducdi
[2378]	24	jhcui
[2404]	23	lashdcuohco
[2436]	22	kcbhducdi
[2466]	21	jhcui
[2492]	20	lashdcuohco
[2524]	19	kcbhducdi
[2554]	18	jhcui
[2580]	17	lashdcuohco
[2612]	16	kcbhducdi
[2642]	15	jhcui
[2668]	14	lashdcuohco
[2700]	13	kcbhducdi
[2730]	12	jhcui
[2756]	11	lashdcuohco
[2788]	10	kcbhducdi
[2818]	9	jhcui
[2844]	8	lashdcuohco
[2876]	7	kcbhducdi
[2906]	6	jhcui
[2932]	5	lashdcuohco
[2964]	4	kcbhducdi
[2994]	3	jhcui
[3020]	2	lashdcuohco
[3052]	1	kcbhducdi