SequenceFileInputFormat用法

SequenceFileInputFormat只能处理SequenceFile类型的文件。

代码：

[java] view plain copy print ?

1. package inputformat;  
2.   
3. import java.io.IOException;  
4.   
5. import org.apache.hadoop.conf.Configuration;  
6. import org.apache.hadoop.fs.Path;  
7. import org.apache.hadoop.io.LongWritable;  
8. import org.apache.hadoop.io.Text;  
9. import org.apache.hadoop.mapreduce.Job;  
10. import org.apache.hadoop.mapreduce.Mapper;  
11. import org.apache.hadoop.mapreduce.Reducer;  
12. import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;  
13. import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;  
14. import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;  
15. import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;  
16. import org.apache.hadoop.mapreduce.lib.partition.HashPartitioner;  
17. //用之前SequenceFile类型的文件作为处理数据，用那个for循环生成的数据，那个数据指定的类型是<LongWritable,Text>  
18. //SequenceFileInputFormat只能处理SequenceFile类型的数据  
19. public class SequenceFileInputFormatTest {  
20.     public static class MyMapper extends  
21.             Mapper<LongWritable, Text, Text, LongWritable> {  
22.         final Text k2 = new Text();  
23.         final LongWritable v2 = new LongWritable();  
24.   
25.         protected void map(LongWritable key, Text value,  
26.                 Mapper<LongWritable, Text, Text, LongWritable>.Context context)  
27.                 throws InterruptedException, IOException {  
28.             final String line = value.toString();  
29.             final String[] splited = line.split("\\s");  
30.             for (String word : splited) {  
31.                 k2.set(word);  
32.                 v2.set(1);  
33.                 context.write(k2, v2);  
34.             }  
35.         }  
36.     }  
37.   
38.     public static class MyReducer extends  
39.             Reducer<Text, LongWritable, Text, LongWritable> {  
40.         LongWritable v3 = new LongWritable();  
41.   
42.         protected void reduce(Text k2, Iterable<LongWritable> v2s,  
43.                 Reducer<Text, LongWritable, Text, LongWritable>.Context context)  
44.                 throws IOException, InterruptedException {  
45.             long count = 0L;  
46.             for (LongWritable v2 : v2s) {  
47.                 count += v2.get();  
48.             }  
49.             v3.set(count);  
50.             context.write(k2, v3);  
51.         }  
52.     }  
53.   
54.     public static void main(String[] args) throws Exception {  
55.         final Configuration conf = new Configuration();  
56.         final Job job = Job.getInstance(conf, SequenceFileInputFormatTest.class.getSimpleName());  
57.         // 1.1  
58.         FileInputFormat.setInputPaths(job,  
59.                 "hdfs://192.168.1.10:9000/sf1");  
60.           
61.         //这里改了一下，把TextInputFormat改成了SequenceFileInputFormat  
62.         job.setInputFormatClass(SequenceFileInputFormat.class);  
63.           
64.           
65.         // 1.2  
66.         job.setMapperClass(MyMapper.class);  
67.         job.setMapOutputKeyClass(Text.class);  
68.         job.setMapOutputValueClass(LongWritable.class);  
69.         // 1.3 默认只有一个分区  
70.         job.setPartitionerClass(HashPartitioner.class);  
71.         job.setNumReduceTasks(1);  
72.         // 1.4省略不写  
73.         // 1.5省略不写  
74.   
75.         // 2.2  
76.         job.setReducerClass(MyReducer.class);  
77.         job.setOutputKeyClass(Text.class);  
78.         job.setOutputValueClass(LongWritable.class);  
79.         // 2.3  
80.         FileOutputFormat.setOutputPath(job, new Path(  
81.                 "hdfs://192.168.1.10:9000/out1"));  
82.         job.setOutputFormatClass(TextOutputFormat.class);  
83.         // 执行打成jar包的程序时，必须调用下面的方法  
84.         job.setJarByClass(SequenceFileInputFormatTest.class);  
85.         job.waitForCompletion(true);  
86.     }  
87. }  

生成SequenceFile类型的文件，供上述SequenceFileInputFormat使用，作为输入数据：

[java] view plain copy print ?

1. package sequenceFile;  
2.   
3. import java.net.URI;  
4.   
5. import org.apache.hadoop.conf.Configuration;  
6. import org.apache.hadoop.fs.FileSystem;  
7. import org.apache.hadoop.fs.Path;  
8. import org.apache.hadoop.io.LongWritable;  
9. import org.apache.hadoop.io.SequenceFile;  
10. import org.apache.hadoop.io.Text;  
11. import org.apache.zookeeper.common.IOUtils;  
12.   
13. //for循环读写操作演示  
14. public class Forduxie {  
15.     public static void main(String args[]) throws Exception {  
16.         final Path path = new Path("/sf1");  
17.         Configuration conf = new Configuration();  
18.         final FileSystem fs = FileSystem.get(new URI("hdfs://192.168.1.10:9000/"), conf);  
19.           
20.         @SuppressWarnings("deprecation")  
21.         final SequenceFile.Writer writer = new SequenceFile.Writer(fs, conf,path, LongWritable.class,Text.class);  
22.         for (int i = 0; i < 10; i++) {  
23.             writer.append(new LongWritable(i), new Text(i+"=_="));  
24.         }  
25.         IOUtils.closeStream(writer);  
26.   
27.         @SuppressWarnings({ "deprecation" })  
28.         final SequenceFile.Reader reader = new SequenceFile.Reader(fs, path,conf);  
29.         LongWritable key = new LongWritable();  
30.         Text val = new Text();  
31.         while (reader.next(key, val)) {  
32.             System.out.println(key.get() + "\t" + val.toString());  
33.         }IOUtils.closeStream(reader);  
34.     }  
35. }  

如果创建的是Maven项目，需要在pom包里添加：

[java] view plain copy print ?

1. <span style="white-space:pre">        </span><dependency>  
2.             <groupId>commons-io</groupId>  
3.             <artifactId>commons-io</artifactId>  
4.             <version>2.4</version>  
5.         </dependency>