MapReduce输出格式压缩

 1 public class MaxTemprature {
 2 
 3     /**  
 4      * 程序入口
 5      * @author: wangyanfu
 6      * @version: Jul 10, 2013 11:00:26 AM  
 7      */
 8 
 9     public static void main(String[] args) {
10         
11         try {
12             //########################
13             //此处直接操作在hadoop集群中
14             //########################
15             Path inputPath = new Path("hdfs://192.168.1.15:9000/hadoopcase/maxtemprature/input/data.txt");
16             Path outPath = new Path("hdfs://192.168.1.15:9000/hadoopcase/maxtemprature/output");
17             
18             JobConf jobConf = new JobConf(MaxTempratureMapper.class);
19             
20             jobConf.setJobName("Calculate Max Temprature");
21             
22             FileInputFormat.setInputPaths(jobConf, inputPath);
23             FileOutputFormat.setOutputPath(jobConf, outPath);
24             
25             //#################################
26             //设置输出属性,输出时压缩
27             //#################################
28             jobConf.setBoolean("mapred.output.compress", true);
29             //#################################
30             //指定输出压缩编码为GZIP
31             //#################################
32             jobConf.setClass("mapred.output.compression.codec", GzipCodec.class, CompressionCodec.class);
33             
34             
35             //#################################
36             //设置Map后结果压缩,压缩编码为GZIP
37             //#################################
38             jobConf.setCompressMapOutput(true);
39             jobConf.setMapOutputCompressorClass(GzipCodec.class);
40             
41             jobConf.setMapperClass(MaxTempratureMapper.class);
42             jobConf.setCombinerClass(MaxTempratureReducer.class);
43             jobConf.setReducerClass(MaxTempratureReducer.class);
44             
45             //**********************************************
46             //TextInputFormat格式默认行号为key,每一行的字符为value
47             //**********************************************
48             jobConf.setInputFormat(TextInputFormat.class);
49             
50             //**********************************************
51             //设置Maper后的输出格式,不设置则出现参数类型不匹配错误
52             //**********************************************
53             jobConf.setOutputKeyClass(Text.class);
54             jobConf.setOutputValueClass(IntWritable.class);
55             
56             JobClient.runJob(jobConf);
57         } catch (IOException e) {
58             e.printStackTrace();
59         }
60         
61     }
62 
63 }

 

转载于:https://www.cnblogs.com/geopanda/p/3183597.html