java获取hadoop目录结构_Hadoop Map-Reduce程序读取OrcStruct类型的文件

表结构:

map-reduce程序(只要map阶段,没有reduce):

Driver:

import java.time.LocalDate;

import java.time.format.DateTimeFormatter;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.conf.Configured;

import org.apache.hadoop.fs.FileSystem;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Job;

import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

import org.apache.hadoop.mapreduce.lib.input.SequenceFileAsBinaryInputFormat;

import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;

import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import org.apache.hadoop.util.Tool;

import org.apache.hadoop.util.ToolRunner;

import org.apache.orc.mapreduce.OrcInputFormat;

public class EvBikeColorDriver extends Configured implements Tool {

@Override

public int run(String[] args) throws Exception {

LocalDate runDate = null;

DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd");

// 参数传入判断

if (args == null || args.length == 0) {

System.err.println("参数为空，请至少输入时间参数！！！！！！");

System.exit(1);

} else if (args.length == 1) {

runDate = LocalDate.parse(args[0], formatter);

} else if (args.length == 2) {

System.err.println("参数个数不能多于1个 ！！！！");

System.exit(1);

}

// user/hive/dataware/dw/dw_bike_alert_status_clean/pt=xxx

String inputPath = "/xxxx/xxxxxxxin/pt=" + formatter.format(runDate).replace("-", "");

String outputPath = "/xxxx/xxxxxxout" + "/pt=" + formatter.format(runDate).replace("-", "");

System.out.println("输入路径: "+inputPath);

System.out.println("输出路径: "+outputPath);

Configuration conf = getConf();

conf.set("orc.mapred.output.schema","struct");

FileSystem fs = FileSystem.get(conf);

conf.set("inputPath", inputPath);

conf.set("outputPath", outputPath);

Job job = Job.getInstance(conf, "OrcInputFormat reader");

job.setJarByClass(MyDriver.class);

// 设置Map

job.setMapperClass(MyMapper.class);

job.setMapOutputKeyClass(Text.class);

job.setMapOutputValueClass(Text.class);

FileInputFormat.addInputPaths(job, inputPath);

//设置Orc格式

job.setInputFormatClass(OrcInputFormat.class);

FileOutputFormat.setOutputPath(job, new Path(outputPath));

// mapreduce输出的临时目录删除

if (fs.exists(new Path(outputPath))) {

fs.delete(new Path(outputPath), true);

}

return job.waitForCompletion(true) ? 0 : 1;

}

public static void main(String[] args) throws Exception {

Configuration configuration = new Configuration();

int status = ToolRunner.run(configuration, new MyDriver(), args);

System.exit(status);

}

}

Mapper:

import java.io.IOException;

import org.apache.hadoop.io.NullWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Mapper;

import org.apache.orc.mapred.OrcStruct;

public class MyMapper extends Mapper {

@Override

protected void map(NullWritable key, OrcStruct vlaue, Context context) throws IOException, InterruptedException {

//vlaue.getFieldValue(0),vlaue.getFieldValue(1),vlaue.getFieldValue(2)代表列index

if(vlaue.getFieldValue(0) != null && vlaue.getFieldValue(1) != null) {

context.write((Text)vlaue.getFieldValue(1), (Text)vlaue.getFieldValue(2));

}

}

}

转载至链接:https://my.oschina.net/u/2338224/blog/1830266