mapreducde中的超类

 

 

package com.sxt.hadoop.mr;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.db.DBInputFormat;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class MyJOB {
	
	
	public static void main(String[] args) throws Exception {
		
		
		Configuration conf = new Configuration(true);
		
		Job  job =  Job.getInstance(conf);
		
		  // Create a new Job
//	     Job job = Job.getInstance();
		
	     job.setJarByClass(MyJOB.class);
	    
	     
	     // Specify various job-specific parameters     
	     job.setJobName("ooxx");
	     
//	     job.setInputPath(new Path("in"));
//	     job.setOutputPath(new Path("out"));
	     
	     Path fileIn= new Path("/user/root/test.txt");
		FileInputFormat.addInputPath(job, fileIn);
	     
		Path fileout = new Path("/data/wc/output01");
		if(fileout.getFileSystem(conf).exists(fileout)){
			fileout.getFileSystem(conf).delete(fileout,true);
			
		}
		FileOutputFormat.setOutputPath(job, fileout );
	     
		
		
		FileInputFormat.setMaxInputSplitSize(job, 64);
	     
	     job.setMapperClass(MyMapper.class);
	     job.setMapOutputKeyClass(Text.class);
	     job.setMapOutputValueClass(IntWritable.class);
	     
	     job.setInputFormatClass(cls);
	     job.setReducerClass(MyReducer.class);

	     
	     job.setNumReduceTasks(0);
	     
	     
	     
	     
	     job.setCombinerClass(ooxx.class);
	     
	     
	     
	     
	     
	     
	     
	     
	     // Submit the job, then poll for progress until the job is complete
	     job.waitForCompletion(true);
	 
		
		
		
		
		
		
		
		
	}

}

package com.sxt.hadoop.mr;

import java.io.IOException;
import java.util.StringTokenizer;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

public class MyMapper  extends  Mapper<Object, Text, Text, IntWritable> {
	
	  private final static IntWritable one = new IntWritable(1);
	   private Text word = new Text();
	   
	   public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
		   
		   
		   
	     StringTokenizer itr = new StringTokenizer(value.toString());
	     
	     
	     while (itr.hasMoreTokens()) {
	       word.set(itr.nextToken());
	       
	       context.write(word, one);
	       
	     }
	   }
	 

	

}

package com.sxt.hadoop.mr;

import java.io.IOException;
import java.util.Iterator;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

public class MyReducer extends  Reducer<Text, IntWritable, Text, IntWritable>{
	
	
	
	
//	bj.hd.sxt
//	bj.hd.qh
//	bj.cp.bd
	
//	sh.hq.jd
//	sh.hq.sxt
//	sh.pd.oo

	
	
	
	private IntWritable result = new IntWritable();
	 
	   public void reduce(Text key, Iterable<IntWritable> values,
	                      Context context) throws IOException, InterruptedException {
		   
	     int sum = 0;
	     //sxt  1
	     //sxt  1
	     //sxt  1
	     //sxt  1
	     //sxt  1
	     //sxt  1
	     for (IntWritable val : values) {
	       sum += val.get();
	     }
	     
	     
	     Iterator<IntWritable> it = values.iterator();
	     
	     
//	 	bj.hd.sxt    1E
//	 	bj.hd.qh     10
//	 	bj.cp.bd     20
	     
	     
	     while(it.hasNext()){
	    	 IntWritable n = it.next();
	    	 System.out.println(n);
	    	 //1E
	    	 //10
	    	 //20
	    	 System.out.println(key);
	    	 //bj.hd.sxt
	    	 //bj.hd.qh
	    	 //bj.cp.bd
	    	 
	     }
	     
	     
	     result.set(sum);
	     context.write(key, result);
	   }

	
	
	

}