MapReduce自定义输出目录和输出文件名

MapReduce自定义输出目录和输出文件名

 

添加临时的输出目录（非Reduce或Mapper输出文件目录）

        String outputPath = "/tmp/test/output/";
        hdfsApi.loadFileSystem();
        if (hdfsApi.exist(outputPath)) {
            hdfsApi.delete(outputPath);
        }
        //相当于job.getConfiguration().set(FileOutputFormat.OUTDIR, tablePath);
        FileOutputFormat.setOutputPath(job, new Path(outputPath));

添加自定义输出目录

Job job = Job.getInstance(conf);

MultipleOutputs.addNamedOutput(job, "mult", FileOutputFromate.class, NullWritable.class, Test.class);
//去掉 临时输出目录会生成part-r-00000或者part-m-00000的空文件
LazyOutputFormat.setOutputFormatClass(job, OrcOutputFormat.class);

自定义输出目录和文件名

在Map或Reduce实现类中添加如下代码

private final NullWritable nw = NullWritable.get();
private MultipleOutputs<NullWritable, OrcStruct> mos;

  /**  
     * output written to /自定义输出目录/自定义输出文件名-r-00000 (etc)
     *
     * @param outputPath 自定义输出目录
     * @param filename 自定义输出文件名称
     * @return 返回输出文件和文件名的全路径
     */
 private String generateFileName(String outputPath, String filename) {
        // example for k = Smith|John
        return outputPath + "/" + filename;
}

 @Override
protected void setup(Context context) throws IOException, InterruptedException {
    mos = new MultipleOutputs<>(context);
｝

 @Override
    protected void reduce(Text key, Iterable<Text> values, Context output) throws IOException, InterruptedException {
   for (Text value : values) {
     mos.write(nw, value, generateFileName(key.toString(), value.toString.splilt("\t")[0]));  
   ｝

 @Override
    protected void cleanup(Context context) throws IOException, InterruptedException {
        if(mos != null)
        mos.close();
    }
   
｝