老的API:
public class MaxTemperature {
public static class MaxTemperatureMapper extends MapReduceBase implements Mapper<LongWritable, Text, Text, IntWritable> {
private static final int MISSING = 9999;
@Override
public void map(LongWritable key, Text value, OutputCollector<Text, IntWritable> output, Reporter reporter)
throws IOException {
String line = value.toString();
String year = line.substring(15, 19);
int airTemperature;
if(line.charAt(87) == '+') {
airTemperature = Integer.parseInt(line.substring(88,92));
} else {
airTemperature = Integer.parseInt(line.substring(87, 92));
}
String quality = line.substring(92, 93);
if(airTemperature != MISSING && quality.matches("[01459]")) {
output.collect(new Text(year), new IntWritable(airTemperature));
}
}
}
public static class MaxTemperatureReducer extends MapReduceBase implements Reducer<Text,IntWritable, Text, IntWritable>{
@Override
public void reduce(Text key, Iterator<IntWritable> values, OutputCollector<Text, IntWritable> output, Reporter reporter)
throws IOException {
int maxvalue = Integer.MIN_VALUE;
while (values.hasNext()) {
maxvalue = Math.max(maxvalue, values.next().get());
}
output.collect(key, new IntWritable(maxvalue));
}
}
public static void main(String[] args) throws IOException {
if (args.length != 2) {
System.err.println("Usage: MaxTemperature <input path> <output path>");
}
JobConf conf = new JobConf(MaxTemperature.class);
conf.setJobName("Max temperature");
FileInputFormat.addInputPath(conf, new Path(args[0]));
FileOutputFormat.setOutputPath(conf, new Path(args[1]));
conf.setMapperClass(MaxTemperatureMapper.class);
conf.setReducerClass(MaxTemperatureReducer.class);
conf.setOutputKeyClass(Text.class);
conf.setOutputValueClass(IntWritable.class);
JobClient.runJob(conf);
}
}
新的api:
public class NewMaxTemperature {
public static class NewMaxTemperatureMapper extends Mapper<LongWritable, Text, Text, IntWritable> {
private static final int MISSING = 9999;
public void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, IntWritable>.Context context)
throws IOException, InterruptedException {
String line = value.toString();
String year = line.substring(15, 19);
int airTemperature;
if (line.charAt(87) == '+') {
airTemperature = Integer.parseInt(line.substring(88, 92));
} else {
airTemperature = Integer.parseInt(line.substring(87, 92));
}
String quality = line.substring(92, 93);
if (airTemperature != MISSING && quality.matches("[01459]")) {
context.write(new Text(year), new IntWritable(airTemperature));
}
}
}
public static class NewMaxTemperatureReducer extends Reducer<Text, IntWritable, Text, IntWritable> {
@Override
public void reduce(Text key, Iterable<IntWritable> values,
Reducer<Text, IntWritable, Text, IntWritable>.Context context) throws IOException, InterruptedException {
int maxvalue = Integer.MIN_VALUE;
while (values.iterator().hasNext()) {
maxvalue = Math.max(maxvalue, values.iterator().next().get());
}
context.write(key, new IntWritable(maxvalue));
}
}
public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
if (args.length != 2) {
System.err.println("Usage: MaxTemperature <input path> <output path>");
}
Configuration conf = new Configuration();
String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
HashMap<String, String> params = ParamsUtil.getJobParameters(otherArgs);
Job job = Job.getInstance(new Cluster(conf), conf);
job.setJarByClass(NewMaxTemperature.class);
job.setMapperClass(NewMaxTemperatureMapper.class);
job.setReducerClass(NewMaxTemperatureReducer.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
String reduceTasksNum = params.get("r");
if (reduceTasksNum == null) {
System.out.println("No reduce tasks num");
System.exit(-1);
}
String input = params.get("input");
if (input == null) {
System.out.println("No input");
System.exit(-1);
}
String output = params.get("output");
if (output == null) {
System.out.println("No output");
System.exit(-1);
}
String name = params.get("name");
if (name == null || "".equals(name)) {
name = "MaxTemperature Test";
}
job.setNumReduceTasks(Integer.valueOf(reduceTasksNum));
FileInputFormat.setInputPaths(job, input);
FileOutputFormat.setOutputPath(job, new Path(output));
job.setJobName(name);
job.waitForCompletion(true);
}
}
在运行新的程序时,刚开始内部类没有用静态修饰,运行时报错:
WARN org.apache.hadoop.mapred.Child: Exception running child : java.lang.RuntimeException: java.lang.NoSuchMethodException: com.renren.dp.mapred.NewMaxTemperature$NewMaxTemperatureMapper.<init>() at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:123) at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:612) at org.apache.hadoop.mapred.MapTask.run(MapTask.java:328) at org.apache.hadoop.mapred.Child$4.run(Child.java:217) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:396) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:742) at org.apache.hadoop.mapred.Child.main(Child.java:211) Caused by: java.lang.NoSuchMethodException: com.renren.dp.mapred.NewMaxTemperature$NewMaxTemperatureMapper.<init>() at java.lang.Class.getConstructor0(Class.java:2706) at java.lang.Class.getDeclaredConstructor(Class.java:1985) at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:117) ... 7 more 原因是:hadoop在调用map和reduce类时采用的反射调用,内部类不是静态的,没有获取到内部类的实例,因为必须要获取外部类的实例,new NewMaxTemperature().new NewMaxTemperatureMapper();如果是静态的内部类的话:直接new NewMaxTemperatureMapper() 得出实例。 在新api中封装了参数解析类: public class ParamsUtil { public static HashMap<String, String> getJobParameters(String[] args) { Options options = new Options(); for (String arg : args) { if (arg.startsWith("-")) { Option option = OptionBuilder.hasArg(true).create( arg.substring(1)); option.setArgName(arg.substring(1)); options.addOption(option); } } PosixParser posixerParser = new PosixParser(); CommandLine cmd = null; try { cmd = posixerParser.parse(options, args); HashMap<String, String> params = new HashMap<String, String>(); for (Option op : cmd.getOptions()) { params.put(op.getArgName(), op.getValue()); System.out.println("Params:" + op.getArgName() + "=" + op.getValue()); } return params; } catch (org.apache.commons.cli.ParseException e) { e.printStackTrace(); System.exit(0); } return null; }