表结构:
map-reduce程序(只要map阶段,没有reduce):
Driver:
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileAsBinaryInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.apache.orc.mapreduce.OrcInputFormat;
public class EvBikeColorDriver extends Configured implements Tool {
@Override
public int run(String[] args) throws Exception {
LocalDate runDate = null;
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd");
// 参数传入判断
if (args == null || args.length == 0) {
System.err.println("参数为空,请至少输入时间参数!!!!!!");
System.exit(1);
} else if (args.length == 1) {
runDate = LocalDate.parse(args[0], formatter);
} else if (args.length == 2) {
System.err.println("参数个数不能多于1个 !!!!");
System.exit(1);
}
// user/hive/dataware/dw/dw_bike_alert_status_clean/pt=xxx
String inputPath = "/xxxx/xxxxxxxin/pt=" + formatter.format(runDate).replace("-", "");
String outputPath = "/xxxx/xxxxxxout" + "/pt=" + formatter.format(runDate).replace("-", "");
System.out.println("输入路径: "+inputPath);
System.out.println("输出路径: "+outputPath);
Configuration conf = getConf();
conf.set("orc.mapred.output.schema","struct");
FileSystem fs = FileSystem.get(conf);
conf.set("inputPath", inputPath);
conf.set("outputPath", outputPath);
Job job = Job.getInstance(conf, "OrcInputFormat reader");
job.setJarByClass(MyDriver.class);
// 设置Map
job.setMapperClass(MyMapper.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(Text.class);
FileInputFormat.addInputPaths(job, inputPath);
//设置Orc格式
job.setInputFormatClass(OrcInputFormat.class);
FileOutputFormat.setOutputPath(job, new Path(outputPath));
// mapreduce输出的临时目录删除
if (fs.exists(new Path(outputPath))) {
fs.delete(new Path(outputPath), true);
}
return job.waitForCompletion(true) ? 0 : 1;
}
public static void main(String[] args) throws Exception {
Configuration configuration = new Configuration();
int status = ToolRunner.run(configuration, new MyDriver(), args);
System.exit(status);
}
}
Mapper:
import java.io.IOException;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.orc.mapred.OrcStruct;
public class MyMapper extends Mapper {
@Override
protected void map(NullWritable key, OrcStruct vlaue, Context context) throws IOException, InterruptedException {
//vlaue.getFieldValue(0),vlaue.getFieldValue(1),vlaue.getFieldValue(2)代表列index
if(vlaue.getFieldValue(0) != null && vlaue.getFieldValue(1) != null) {
context.write((Text)vlaue.getFieldValue(1), (Text)vlaue.getFieldValue(2));
}
}
}
转载至链接:https://my.oschina.net/u/2338224/blog/1830266