搞Hadoop开发,尤其是编写MR程序:总是难免遇到许多奇奇怪怪的问题,第一次编写一个排序求和的MR程序时就是如此。
一;遇到的第一个问题:16/01/10 15:40:19 WARN mapred.LocalJobRunner: job_local2074055527_0001
java.lang.Exception: java.lang.RuntimeException: java.lang.NoSuchMethodException: Hadoop.Hu.Maprduce.sort.SumStep
SumMapper.()atorg.apache.hadoop.mapred.LocalJobRunner
Job.run(LocalJobRunner.java:403)
Caused by: java.lang.RuntimeException: java.lang.NoSuchMethodException: Hadoop.Hu.Maprduce.sort.SumStep$SumMapper.()
at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:131)
at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:721)
at org.apache.hadoop.mapred.MapTask.run(MapTask.java:339)
出现这种情况的原因是:1自己的编写的序列化类中,如果有有参构造函数,必须写无参构造函数,但事实上我是用的void set()方法。并不需要写无参构造函数
2,后现是自己太大意了,在Mapper和Reduceer方法中没有加static,修改后继续运行
二,又出来错误,显示数组越界,奇异值以为是自己参数弄错了,导致数组越界,后来就远程debug,检查来检查去都感觉不是代码错误,后来在群里求救,然后冷静下来是自己的数据有错,所以要随时注意光标的位置!最后把自己的怠代码拿出来!
Java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
public class SumStep {
public class SumMapper extends Mapper<LongWritable, Text, Text, InfoBean>{
private InfoBean v = new InfoBean();
private Text k = new Text();
@Override
protected void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
String line = value.toString();
String[] fileds = line.split("\t");
String account = fileds[0];
double in = Double.parseDouble(fileds[1]);
double out = Double.parseDouble(fileds[2]);
k.set(account);
v.set(account, in, out);
context.write(k, v);
}
}
public class SumReducer extends Reducer<Text, InfoBean, Text, InfoBean>{
private InfoBean v = new InfoBean();
@Override
protected void reduce(Text key, Iterable<InfoBean> v2s, Context context)
throws IOException, InterruptedException {
double in_sum = 0;
double out_sum = 0;
for( InfoBean bean : v2s){
in_sum += bean.getIncome();
out_sum += bean.getExpenses();
}
v.set("",in_sum,out_sum);
context.write(key, v);
}
}
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
Job job = Job.getInstance(conf);
job.setJarByClass(SumStep.class);
job.setMapperClass(SumMapper.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(InfoBean.class);
FileInputFormat.setInputPaths(job, new Path(args[0]));
job.setReducerClass(SumReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(InfoBean.class);
FileOutputFormat.setOutputPath(job,new Path(args[1]));
job.waitForCompletion(true);
}
}
写代码片`
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.io.WritableComparable;
public class InfoBean implements WritableComparable {
private String account;
private double income;
private double expenses;
private double surplus;
public void set(String account,double income,double expenses ){
this.account = account;
this.income = income;
this.expenses = expenses;
this.surplus = income - expenses;
}
@Override
public String toString() {
return this.income+ "\t" +this.expenses+ "\t" +this.surplus;
}
@Override
public void write(DataOutput out) throws IOException {
out.writeUTF(account);
out.writeDouble(income);
out.writeDouble(expenses);
out.writeDouble(surplus);
}
@Override
public void readFields(DataInput in) throws IOException {
this.account =in.readUTF();
this.income = in.readDouble();
this.expenses = in.readDouble();
this.surplus = in.readDouble();
}
@Override
public int compareTo(InfoBean o) {
if(this.income == o.getIncome()){
return this.expenses > o.getExpenses() ? 1 : -1;
}
else{
return this.income > o.getIncome() ? -1 : 1;
}
}
public String getAccount() {
return account;
}
public void setAccount(String account) {
this.account = account;
}
public double getIncome() {
return income;
}
public void setIncome(double income) {
this.income = income;
}
public double getExpenses() {
return expenses;
}
public void setExpenses(double expenses) {
this.expenses = expenses;
}
public double getSurplus() {
return surplus;
}
public void setSurplus(double surplus) {
this.surplus = surplus;
}
}