mapreduce清洗mysql_MapReduce实验——数据清洗

packagemapreduce;importjava.io.IOException;importorg.apache.hadoop.fs.Path;importorg.apache.hadoop.io.IntWritable;importorg.apache.hadoop.io.Text;importorg.apache.hadoop.mapreduce.Job;importorg.apache.hadoop.mapreduce.Mapper;importorg.apache.hadoop.mapreduce.Reducer;importorg.apache.hadoop.mapreduce.lib.input.FileInputFormat;importorg.apache.hadoop.mapreduce.lib.output.FileOutputFormat;public classCleanResult {public static void main(String[] args) throwsIOException,ClassNotFoundException,InterruptedException{

Job job=Job.getInstance();

job.setJobName("CleanResult");

job.setJarByClass(WordCount.class);

job.setMapperClass(doMapper.class);

job.setReducerClass(doReducer.class);

job.setOutputKeyClass(Text.class);

job.setOutputValueClass(IntWritable.class);

Path in= new Path("hdfs://192.168.12.128:9000/mapreduce/in/result.txt");

Path out= new Path("hdfs://192.168.12.128:9000/mapreduce/out/clean");

FileInputFormat.addInputPath(job,in);

FileOutputFormat.setOutputPath(job,out);

System.exit(job.waitForCompletion(true) ? 0:1);

}public static class doMapper extends Mapper{public static Text word = newText();public static final IntWritable id = newIntWritable();

@Overrideprotected void map(Object key,Text value,Context context) throwsIOException,InterruptedException{

String[] data= value.toString().split(",");

String[] time= data[1].split(":");

data[1] = "2016-11-10 " + time[1] + ":" + time[2] + ":" + time[3].split(" ")[0];

String traffic= "";for(int i=0;i= 48 && data[3].charAt(i)<=57) {

traffic+= data[3].charAt(i);

}

}

data[3] =traffic;

String newValue= data[0];for(int i=1;i<5;i++)

newValue+= "\t" +data[i];

id.set(Integer.parseInt(data[5]));

word.set(newValue);

context.write(word, id);

}

}public static class doReducer extends Reducer{private IntWritable result = newIntWritable();

@Overrideprotected void reduce(Text key, Iterable values, Context context) throwsIOException, InterruptedException {

System.out.println("key:" +key);for(IntWritable value:values){

result.set(value.get());

}

System.out.println("value:" +result);

context.write(key, result);

}

}

}