序列化

序列化基础：

1、串行化：将结构化对象转换成字节流，可用于进程间通信和永久存储

2、在Text中，输入和输出的文本编码方式必须为UTF-8！

3、hadoop序列文件是由序列化的"k-v"对组成，而k和v即hadoop的writable格式

SequenceFile：序列文件，用于包装文本文件（日志文件）

1、SeqFile的特性：

（1）扁平化文件，包括二进制的k-v

（2）可读可写可排序

（3）有三种压缩方式：不压缩，记录压缩（只压value），块压缩（将多组k-v聚集成一个"block"进行压缩，默认1M）

2、SeqFile格式：

（1）SEQ三字节的头 + 数字（如6）作为版本号

（2）key的完整类名

（3）value的完整类名

（4）布尔值，指定SeqFile是否采用压缩

（5）布尔值，指定SeqFile是否采用块压缩

（6）压缩编解码器

（7）metadata：元数据信息

（8）sync：同步点

3、写SeqFile文件

public void testWriteSeq() throws IOException {
	Configuration conf = new Configuration();
	conf.set("fs.defaultFS", "file:///");
	FileSystem fs = FileSystem.get(conf);
	Path p = new Path("f:/seqfile/1-block.seq");
	//通过反射获取压缩编解码器
	BZip2Codec codec = ReflectionUtils.newInstance(BZip2Codec.class, conf);
	SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, p, IntWritable.class, Text.class,SequenceFile.CompressionType.BLOCK, codec);
	for (int i = 1; i < 1000; i++) {
		IntWritable key = new IntWritable(i);
		Text value = new Text("ma" + i);
		writer.append(key, value);
	}
	writer.close();
}

4、读SeqFile文件

public void testReadSeq() throws IOException {
	Configuration conf = new Configuration();
	conf.set("fs.defaultFS", "file:///");
	FileSystem fs = FileSystem.get(conf);
	Path p = new Path("f:/seqfile/1.seq");
	SequenceFile.Reader reader = new SequenceFile.Reader(fs, p, conf);
	//初始化两个writable对象
	IntWritable key = new IntWritable();
	Text value = new Text();
	while (reader.next(key, value)) {
		long pos = reader.getPosition();//得到指针的位置
		System.out.println("key:" + key.get() + ",value:" + value.toString() + ",pos:" + pos);
	}
	reader.close();
}

5、在"cmd"显示序列化文件的内容：hdfs dfs -text file:///f:/seqfile/1-block.seq

6、手动指定指针位置，读取SeqFile文件内容，若指针不在文件头，则报错

public void testSeek() throws IOException {
    Configuration conf = new Configuration();
    conf.set("fs.defaultFS", "file:///");
    FileSystem fs = FileSystem.get(conf);
    Path p = new Path("f:/seqfile/1.seq");
    SequenceFile.Reader reader = new SequenceFile.Reader(fs, p, conf);
    //初始化两个writable对象
    IntWritable key = new IntWritable();
    Text value = new Text();
    //手动指定指针位置
    reader.seek(176);
    reader.next(key, value);
    System.out.println("key:" + key.get() + ",val:" + value.toString());
}

7、使用同步点来读写SeqFile文件

//写
public void testWriteSeq2() throws IOException {
	Configuration conf = new Configuration();
	conf.set("fs.defaultFS", "file:///");
	FileSystem fs = FileSystem.get(conf);
	Path p = new Path("f:/seqfile/2.seq");
	SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, p, IntWritable.class, Text.class);
	for (int i = 1; i < 1000; i++) {
		writer.sync(); //在记录边界添加同步点
		IntWritable key = new IntWritable(i);
		Text value = new Text("ma" + i);
		writer.append(key, value);
	}
	writer.close();
}
//读
public void testReadSeq2() throws IOException {
	Configuration conf = new Configuration();
	conf.set("fs.defaultFS", "file:///");
	FileSystem fs = FileSystem.get(conf);
	Path p = new Path("f:/seqfile/2.seq");
	SequenceFile.Reader reader = new SequenceFile.Reader(fs, p, conf);
	//初始化两个writable对象
	IntWritable key = new IntWritable();
	Text value = new Text();
	reader.sync(100);
	reader.next(key, value);
	System.out.println("key:" + key.get() + ",val:" + value.toString());
}

8、MapFile文件的读写

//写文件
public void testWrite() throws Exception {
	Configuration conf = new Configuration();
	conf.set("fs.defaultFS", "file:///");
	FileSystem fs = FileSystem.get(conf);
	String p = "f:/seqfile/test.map";
	MapFile.Writer writer = new MapFile.Writer(conf, fs, p, IntWritable.class, Text.class);
	for (int i = 1; i < 100; i++) {
		IntWritable key = new IntWritable(i);
		Text value = new Text("ma" + i);
		writer.append(key, value);
	}
	writer.close();
}
//读文件
public void testRead() throws Exception {
	Configuration conf = new Configuration();
	conf.set("fs.defaultFS", "file:///");
	FileSystem fs = FileSystem.get(conf);
	String p = "f:/seqfile/test.map";
	MapFile.Reader reader = new MapFile.Reader(fs, p, conf);
	IntWritable key = new IntWritable();
	Text value = new Text();
	while(reader.next(key,value)){
		System.out.println(key + "---" + value);
	}
}