MainTest.java
package MyinputFormat;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
/**
* @author carssun 2018年3月19日下午6:41:39
*自定义outputformat
*/
public class MainTest extends Configured implements Tool {
public static void main(String[] args) throws Exception {
int exitCode = ToolRunner.run(new MainTest(), args);
System.exit(exitCode);
}
private static Text filename;
public static class MyInMapper extends Mapper<NullWritable, Text, Text, Text>{
@Override
protected void setup(Mapper<NullWritable, Text, Text, Text>.Context context)
throws IOException, InterruptedException {
InputSplit split=context.getInputSplit();
Path filePath=((FileSplit)split).getPath();
filename=new Text(filePath.toString());//记得试一下换成getNames
}
@Override
protected void map(NullWritable key, Text value, Mapper<NullWritable, Text, Text, Text>.Context context)
throws IOException, InterruptedException {
context.write(filename, value);
System.err.println("读了一次");
}
}
//reduce原样输出即可
public static class MyInReducer extends Reducer<Text, Text, Text, NullWritable>{
@Override
protected void reduce(Text key, Iterable<Text> value, Context context)
throws IOException, InterruptedException {
context.write(value.iterator().next(),NullWritable.get());//不用循环 一个key只对应一个value
}
}
@Override
public int run(String[] args) throws Exception {
Configuration conf = new Configuration();
Job job = Job.getInstance(conf);
job.setJarByClass(MainTest.class);
job.setMapperClass(MyInMapper.class);
job.setReducerClass(MyInReducer.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(Text.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(NullWritable.class);
Path inputPath=new Path("D:/result/small");
Path outputPath=new Path("D:/result/small/output");
FileInputFormat.setInputPaths(job, inputPath);
FileOutputFormat.setOutputPath(job,outputPath);
job.setNumReduceTasks(0);
job.setInputFormatClass(WholeInputFormat.class);
FileSystem fs=FileSystem.get(conf);
if(fs.exists(outputPath)){
fs.delete(outputPath,true);
}
int status = job.waitForCompletion(true) ? 0 : 1;
return status;
}
}
MyReaderRecorder
package MyinputFormat;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
/**
* @author lhx 2018年3月19日下午8:37:16
*
*/
public class MyReaderRecorder extends RecordReader<NullWritable, Text>{
//四个成员变量
private Configuration conf=null;//配置
private FileSplit split=null;//文件切片
private boolean process=false;//记录进度
private Text value=new Text();
@Override
public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException {
conf=context.getConfiguration();
this.split=(FileSplit)split;
}
// nextKeyValue()方法是RecordReader最重要的方法,也就是RecordReader读取文件的读取逻辑所在地
// 所以我们要自定义RecordReader,就需要重写nextKeyValue()的实现
@Override
public boolean nextKeyValue() throws IOException, InterruptedException {
if(!process){
//创建缓冲区
byte[] contents=new byte[(int)split.getLength()];//文件的确可能超过int最大值
//这里的强转也是迫不得已 如果超过int最大值 就可以不考虑整个文件一起切
FileSystem fs=FileSystem.get(conf);
Path filePath=split.getPath();//通过filesplit切片获取文件路劲
FSDataInputStream inputStream=fs.open(filePath);//hdfs文件读流
//读取文件
IOUtils.readFully(inputStream, contents, 0, contents.length);
//把文件内容都读取到缓缓冲区后 再把内容写到value
value.set(contents,0,contents.length);
IOUtils.closeStream(inputStream);//关闭输入流
process=true;//进度完成
return true;
}
return false;
}
@Override
public NullWritable getCurrentKey() throws IOException, InterruptedException {
return NullWritable.get();
}
@Override
public Text getCurrentValue() throws IOException, InterruptedException {
// TODO Auto-generated method stub
return value;
}
@Override
public float getProgress() throws IOException, InterruptedException {
// TODO Auto-generated method stub
return process?0f:1f;//进度 要不就是100%要不就是0%
}
@Override
public void close() throws IOException {
// do nothing
}
}
WholeInputFormat
package MyinputFormat;
import java.io.IOException;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
public class WholeInputFormat extends FileInputFormat<NullWritable, Text>{
@Override
protected boolean isSplitable(JobContext context, Path filename) {//设置成不可再切分
return false;
}
@Override
public RecordReader<NullWritable, Text> createRecordReader(InputSplit split, TaskAttemptContext context)
throws IOException, InterruptedException {
MyReaderRecorder reader=new MyReaderRecorder();
reader.initialize(split, context);
return reader;
}
}