mapreduce读取sequencefile文件中的数据

[java]  view plain copy

1. sequencefile中的数据是以key，value对存储的。  

[java]  view plain copy

1. 通过mapreduce模式，可以读取sequencefile中的数据。  

[java]  view plain copy

1. public class MapReduceReadFile {  
2.       
3.     private static SequenceFile.Reader reader = null;  
4.     private static Configuration conf = new Configuration();  
5.   
6.     public static class ReadFileMapper extends  
7.             Mapper<LongWritable, Text, LongWritable, Text> {  
8.   
9.         /* (non-Javadoc) 
10.          * @see org.apache.hadoop.mapreduce.Mapper#map(KEYIN, VALUEIN, org.apache.hadoop.mapreduce.Mapper.Context) 
11.          */  
12.         @Override  
13.         public void map(LongWritable key, Text value, Context context) {  
14.             key = (LongWritable) ReflectionUtils.newInstance(  
15.                     reader.getKeyClass(), conf);  
16.             value = (Text) ReflectionUtils.newInstance(  
17.                     reader.getValueClass(), conf);  
18.             try {  
19.                 while (reader.next(key, value)) {  
20.                     System.out.printf("%s\t%s\n", key, value);  
21.                     context.write(key, value);  
22.                 }  
23.             } catch (IOException e1) {  
24.                 e1.printStackTrace();  
25.             } catch (InterruptedException e) {  
26.                 e.printStackTrace();  
27.             }  
28.         }  
29.   
30.     }  
31.     /** 
32.      * @param args 
33.      * @throws IOException 
34.      * @throws InterruptedException 
35.      * @throws ClassNotFoundException 
36.      */  
37.     public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {  
38.           
39.         Job job = new Job(conf,"read seq file");  
40.         job.setJarByClass(MapReduceReadFile.class);  
41.         job.setMapperClass(ReadFileMapper.class);  
42.         job.setMapOutputValueClass(Text.class);  
43.         Path path = new Path("logfile2");  
44.         FileSystem fs = FileSystem.get(conf);  
45.         reader = new SequenceFile.Reader(fs, path, conf);  
46.         FileInputFormat.addInputPath(job, path);  
47.         FileOutputFormat.setOutputPath(job, new Path("result"));  
48.         System.exit(job.waitForCompletion(true)?0:1);  
49.     }  
50.