mapreduce页面访问总次数的全局倒序排序-编程 （ten day）

先梳理逻辑：

数据量大的话，在treemap里面会放不下，

map产生的很多key-value数据，会发给reduce去聚合，key相同的会作为一组聚合(框架内部有排序机制，按key排序)，

1、先写一个mapreduce ,输出访问页面和访问它的总次数，产生一个结果文件，不过这个里面的顺序是按照页面来排序的

2、再写一个mapReduce，用map去读上一个mapreduce产生的文件，把（页面，总次数）整体当作key，value为null，

此时key为对象，控制compareTo如何排序即可（mapreduce内部排序机制，只要告诉key上的对象怎么比大小）

WritableComparable：要排序实现Comparable，要输出实现Writable

public class PageCount implements WritableComparable<PageCount>{
	
	private String page;
	private int count;
	
	public void set(String page, int count) {
		this.page = page;
		this.count = count;
	}
	
	public String getPage() {
		return page;
	}
	public void setPage(String page) {
		this.page = page;
	}
	public int getCount() {
		return count;
	}
	public void setCount(int count) {
		this.count = count;
	}

	@Override
	public int compareTo(PageCount o) {
		
		return o.getCount()-this.count==0?this.page.compareTo(o.getPage()):o.getCount()-this.count;
	}

	@Override
	public void write(DataOutput out) throws IOException {
		out.writeUTF(this.page);
		out.writeInt(this.count);
		
	}

	@Override
	public void readFields(DataInput in) throws IOException {
		this.page= in.readUTF();
		this.count = in.readInt();
		
	}
	
	
	@Override
	public String toString() {
		return this.page + "," + this.count;
	}
}

 第一个mapreduce

public class PageCountStep1 {
	
	public static class PageCountStep1Mapper extends Mapper<LongWritable, Text, Text, IntWritable>{
		
		@Override
		protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, IntWritable>.Context context)
				throws IOException, InterruptedException {
			String line = value.toString();
			String[] split = line.split(" ");
			context.write(new Text(split[1]), new IntWritable(1));
		}
		
	}
	
	public static class PageCountStep1Reducer extends Reducer<Text, IntWritable, Text, IntWritable>{
		@Override
		protected void reduce(Text key, Iterable<IntWritable> values,
				Reducer<Text, IntWritable, Text, IntWritable>.Context context) throws IOException, InterruptedException {

			int count = 0;
			for (IntWritable v : values) {
				count += v.get();
			}
			
			context.write(key, new IntWritable(count));
			
		}
	}
	
	public static void main(String[] args) throws Exception {
		


		/**
		 * 通过加载classpath下的*-site.xml文件解析参数
		 */
		Configuration conf = new Configuration();
		
		Job job = Job.getInstance(conf);

		job.setJarByClass(PageCountStep1.class);

		job.setMapperClass(PageCountStep1Mapper.class);
		job.setReducerClass(PageCountStep1Reducer.class);

		job.setMapOutputKeyClass(Text.class);
		job.setMapOutputValueClass(IntWritable.class);
		
		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(IntWritable.class);

		FileInputFormat.setInputPaths(job, new Path("F:\\mrdata\\url\\input"));
		FileOutputFormat.setOutputPath(job, new Path("F:\\mrdata\\url\\countout"));

		job.setNumReduceTasks(3);
		
		job.waitForCompletion(true);
		
	}
}

第二个mapreduce

public class PageCountStep2 {
	
	public static class PageCountStep2Mapper extends Mapper<LongWritable, Text, PageCount, NullWritable>{
		
		@Override
		protected void map(LongWritable key, Text value,
				Mapper<LongWritable, Text, PageCount, NullWritable>.Context context)
				throws IOException, InterruptedException {
			
			String[] split = value.toString().split("\t");
			
			PageCount pageCount = new PageCount();
			pageCount.set(split[0], Integer.parseInt(split[1]));
			
			context.write(pageCount, NullWritable.get());
		}
		
	}
	
	public static class PageCountStep2Reducer extends Reducer<PageCount, NullWritable, PageCount, NullWritable>{
		
		
		@Override
		protected void reduce(PageCount key, Iterable<NullWritable> values,
				Reducer<PageCount, NullWritable, PageCount, NullWritable>.Context context)
				throws IOException, InterruptedException {
			context.write(key, NullWritable.get());
		}
		
	}
	
public static void main(String[] args) throws Exception {

	Configuration conf = new Configuration();
		
		Job job = Job.getInstance(conf);

		job.setJarByClass(PageCountStep2.class);

		job.setMapperClass(PageCountStep2Mapper.class);
		job.setReducerClass(PageCountStep2Reducer.class);

		job.setMapOutputKeyClass(PageCount.class);
		job.setMapOutputValueClass(NullWritable.class);
		
		job.setOutputKeyClass(PageCount.class);
		job.setOutputValueClass(NullWritable.class);

		FileInputFormat.setInputPaths(job, new Path("F:\\mrdata\\url\\countout"));
		FileOutputFormat.setOutputPath(job, new Path("F:\\mrdata\\url\\sortout"));

		job.setNumReduceTasks(1);
		
		job.waitForCompletion(true);
		
	}
}

mapreduce排序实例