还是dataguru第五周的视频练习,感谢老师的分享
import java.io.IOException;
import java.io.InterruptedIOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapreduce.Mapper.Context;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
public class Test2 extends Configured implements Tool {
enum Counter {
LINESKIP, // error line
}
public static class Map extends
Mapper<LongWritable, Text, Text, Text> {
public void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
String line = value.toString();
try {
String[] lineSplit = line.split(" ");
String anum= lineSplit[0];
String bnum = lineSplit[1];
context.write(new Text(anum) ,new Text(bnum));
} catch (ArrayIndexOutOfBoundsException e) {
context.getCounter(Counter.LINESKIP).increment(1);//error counter add 1
return;
}
}
}// end map
public static class Reduce extends Reducer<Text,Text,Text,Text>
{
public void reduce(Text key,Iterable<Text> values,Context context)throws IOException,InterruptedException{
String valueString;
String out="";
for(Text value:values)
{
valueString=value.toString();
out+=valueString+"|";
}//end for
context.write(key, new Text(out));
}//end reduce
}//end Reduce
public int run(String[] args) throws Exception {
Configuration conf = getConf();
Job job = new Job(conf, "Test2");
job.setJarByClass(Test2.class);
FileInputFormat.addInputPath(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
job.setMapperClass(Map.class);
job.setReducerClass(Reducer.class);
job.setOutputFormatClass(TextOutputFormat.class);
job.setOutputKeyClass(NullWritable.class);
job.setOutputValueClass(Text.class);
job.waitForCompletion(true);
return job.isSuccessful() ? 0 : 1;
}// end run
public static void main(String[] args) throws Exception {
int res = ToolRunner.run(new Configuration(), new Test2(), args);
System.exit(res);
}// end main
}//end Test2
1.上面的代码会出现标题的错误,要改掉标记的地方
job.setOutputKeyClass(Text.class);
import java.io.IOException; import java.io.InterruptedIOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.*; import org.apache.hadoop.mapreduce.Mapper.Context; import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; public class Test2 extends Configured implements Tool { enum Counter { LINESKIP, // error line } public static class Map extends Mapper<LongWritable, Text, Text, Text> { public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { String line = value.toString(); try { String[] lineSplit = line.split(" "); String anum= lineSplit[0]; String bnum = lineSplit[1]; context.write(new Text(anum) ,new Text(bnum)); } catch (ArrayIndexOutOfBoundsException e) { context.getCounter(Counter.LINESKIP).increment(1);//error counter add 1 return; } } }// end map
import java.io.InterruptedIOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.*; import org.apache.hadoop.mapreduce.Mapper.Context; import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; public class Test2 extends Configured implements Tool { enum Counter { LINESKIP, // error line } public static class Map extends Mapper<LongWritable, Text, Text, Text> { public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { String line = value.toString(); try { String[] lineSplit = line.split(" "); String anum= lineSplit[0]; String bnum = lineSplit[1]; context.write(new Text(anum) ,new Text(bnum)); } catch (ArrayIndexOutOfBoundsException e) { context.getCounter(Counter.LINESKIP).increment(1);//error counter add 1 return; } } }// end map public static class Reduce extends Reducer<Text,Text,Text,Text> { public void reduce(Text key,Iterable<Text> values,Context context)throws IOException,InterruptedException{ String valueString; String out=""; for(Text value:values) { valueString=value.toString(); out+=valueString+"|"; }//end for context.write(key, new Text(out)); }//end reduce }//end Reduce public int run(String[] args) throws Exception { Configuration conf = getConf(); Job job = new Job(conf, "Test2"); job.setJarByClass(Test2.class); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); job.setMapperClass(Map.class); job.setReducerClass(Reduce.class); job.setOutputFormatClass(TextOutputFormat.class); job.setOutputKeyClass(NullWritable.class); job.setOutputValueClass(Text.class); job.waitForCompletion(true); return job.isSuccessful() ? 0 : 1; }// end run public static void main(String[] args) throws Exception { int res = ToolRunner.run(new Configuration(), new Test2(), args); System.exit(res); }// end main }//end Test2
job.setMapperClass(Map.class);
job.setReducerClass(Reducer.class);
job.setOutputFormatClass(TextOutputFormat.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
job.waitForCompletion(true);
return job.isSuccessful() ? 0 : 1}// end runpublic static void main(String[] args) throws Exception {int res = ToolRunner.run(new Configuration(), new Test2(), args);System.exit(res);}// end main}//end Test2 但结果依然不正确,只是输入的倒序后来把红色改为
job.setReducerClass(Reduce.class);
所以要Reducer的类名必须一致,否则可能使用系统默认的Reducer,就是倒序输出