mapreduce清洗mysql_MapReduce---数据清洗

package mapreduce;

import java.io.IOException;

import java.util.ArrayList;

import java.util.List;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.FileSystem;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.IntWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Job;

import org.apache.hadoop.mapreduce.Mapper;

import org.apache.hadoop.mapreduce.Reducer;

import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;

import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;public classdataclean{public static List ips=new ArrayList();public static List times=new ArrayList();public static List traffic=new ArrayList();public static List wen=new ArrayList();public static List shi=new ArrayList();public static class Map extends Mapper{private static Text Name =newText();private static Text num=newText();public voidmap(Object key,Text value,Context context) throws IOException, InterruptedException{

String line=value.toString();

String arr[]=line.split(",");

Name.set(arr[0]);

num.set(arr[0]);

context.write(Name,num);

}

}public static class Reduce extends Reducer< Text, Text,Text, Text>{private static Text result= newText();int i=0;public void reduce(Text key,Iterablevalues,Context context) throws IOException, InterruptedException{for(Text val:values){

context.write(key, val);

ips.add(val.toString());

}

}

}public static intrun()throws IOException, ClassNotFoundException, InterruptedException

{

Configuration conf=newConfiguration();

conf.set("fs.defaultFS", "hdfs://localhost:9000");

FileSystem fs=FileSystem.get(conf);

Job job=new Job(conf,"OneSort");

job.setJarByClass(dataclean.class);

job.setMapperClass(Map.class);

job.setReducerClass(Reduce.class);

job.setOutputKeyClass(Text.class);

job.setOutputValueClass(Text.class);

job.setInputFormatClass(TextInputFormat.class);

job.setOutputFormatClass(TextOutputFormat.class);

Pathin=new Path("hdfs://localhost:9000/test2/in/result.txt");

Pathout=new Path("hdfs://localhost:9000/test2/out/ip/1");

FileInputFormat.addInputPath(job,in);

fs.delete(out,true);

FileOutputFormat.setOutputPath(job,out);return(job.waitForCompletion(true) ? 0 : 1);

}public static voidmain(String[] args) throws IOException, ClassNotFoundException, InterruptedException{

run();

}

}

}