求最大值和最大k个值

最新推荐文章于 2022-03-27 14:01:03 发布

小江_xiaojiang

最新推荐文章于 2022-03-27 14:01:03 发布

阅读量735

点赞数

分类专栏： MapReduce

本文链接：https://blog.csdn.net/jiangsanfeng1111/article/details/45765977

版权

MapReduce 专栏收录该内容

14 篇文章 1 订阅

订阅专栏

来源吴超7天视频

1.求最大K个值

import java.io.IOException;
import java.net.URI;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class TopKnum {

	static final String INPUT_PATH = "hdfs://hadoop0:9000/input";
	static final String OUT_PATH = "hdfs://hadoop0:9000/out";
	static final int K=100;
	
	
	public static void main(String[] args) throws Exception {
		Configuration conf = new Configuration();
		final FileSystem fileSystem = FileSystem.get(new URI(INPUT_PATH), conf);
		final Path outPath = new Path(OUT_PATH);
		if(fileSystem.exists(outPath)){
			fileSystem.delete(outPath, true);
		}
		
		final Job job = new Job(conf , TopKnum.class.getSimpleName());
		FileInputFormat.setInputPaths(job, INPUT_PATH);
		job.setMapperClass(MyMapper.class);
		job.setMapOutputKeyClass(LongWritable.class);
		job.setMapOutputValueClass(LongWritable.class);
		job.setReducerClass(MyReducer.class);
		job.setOutputKeyClass(LongWritable.class);
		job.setOutputValueClass(NullWritable.class);
		FileOutputFormat.setOutputPath(job, outPath);
		job.waitForCompletion(true);
	}
	
	static class MyMapper extends Mapper<LongWritable, Text, LongWritable, LongWritable>{
		private Long[] top = new Long[K];
		{
			for(int j=0;j<100;j++)
				top[j]=0L;
		}
		@Override
		protected void map(LongWritable k1, Text v1,
				org.apache.hadoop.mapreduce.Mapper<LongWritable, Text, LongWritable, LongWritable>.Context context)
				throws IOException, InterruptedException {
			// TODO Auto-generated method stub
			Long temp=Long.parseLong(v1.toString());
			if(temp>top[0]){
				top[0]=temp;
				int i=0;
				for(;i<99&&temp>top[i+1];i++){
					top[i]=top[i+1];
				}
				top[i]=temp;
			}
			
		}
		
		@Override
		protected void cleanup(
				org.apache.hadoop.mapreduce.Mapper<LongWritable, Text, LongWritable, LongWritable>.Context context)
				throws IOException, InterruptedException {
			// TODO Auto-generated method stub
			for(int i=0;i<100;i++){
				context.write(new LongWritable(top[i]),new LongWritable(top[i]));
			}
		}
	}
	
	static class MyReducer extends Reducer<LongWritable, LongWritable, LongWritable, NullWritable>{
		private Long[] top = new Long[K];
		{
			for(int j=0;j<100;j++)
				top[j]=0L;
		}
		@Override
		protected void reduce(LongWritable k2, Iterable<LongWritable> v2s,
				org.apache.hadoop.mapreduce.Reducer<LongWritable, LongWritable, LongWritable, NullWritable>.Context arg2)
				throws IOException, InterruptedException {
			// TODO Auto-generated method stub
			for (LongWritable v2 : v2s) {
				Long temp=v2.get();
				if(temp>top[0])
				{
					int i=0;
					for(;i<99&&temp>top[i+1];i++){
						top[i]=top[i+1];
					}
					top[i]=temp;
				}
			}
		}
		
		@Override
		protected void cleanup(
				org.apache.hadoop.mapreduce.Reducer<LongWritable, LongWritable, LongWritable, NullWritable>.Context context)
				throws IOException, InterruptedException {
			// TODO Auto-generated method stub
			for(int i=0;i<100;i++){
				context.write(new LongWritable(top[i]), NullWritable.get());
			}
		}
	}
}

2.求最大值

import java.net.URI;

import mapreduce.WordCountApp;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
/**
 * 作业：求最大的100个值
 */
public class TopKApp {
	static final String INPUT_PATH = "hdfs://hadoop0:9000/input";
	static final String OUT_PATH = "hdfs://hadoop0:9000/out";
	
	public static void main(String[] args) throws Exception {
		Configuration conf = new Configuration();
		final FileSystem fileSystem = FileSystem.get(new URI(INPUT_PATH), conf);
		final Path outPath = new Path(OUT_PATH);
		if(fileSystem.exists(outPath)){
			fileSystem.delete(outPath, true);
		}
		
		final Job job = new Job(conf , TopKApp.class.getSimpleName());
		FileInputFormat.setInputPaths(job, INPUT_PATH);
		job.setMapperClass(MyMapper.class);
		job.setReducerClass(MyReducer.class);
		job.setOutputKeyClass(LongWritable.class);
		job.setOutputValueClass(NullWritable.class);
		FileOutputFormat.setOutputPath(job, outPath);
		job.waitForCompletion(true);
	}
	static class MyMapper extends Mapper<LongWritable, Text, LongWritable, NullWritable>{
		long max = Long.MIN_VALUE;
		protected void map(LongWritable k1, Text v1, Context context) throws java.io.IOException ,InterruptedException {
			final long temp = Long.parseLong(v1.toString());
			if(temp>max){
				max = temp;
			}
		};
		
		protected void cleanup(org.apache.hadoop.mapreduce.Mapper<LongWritable,Text,LongWritable, NullWritable>.Context context) throws java.io.IOException ,InterruptedException {
			context.write(new LongWritable(max), NullWritable.get());
		};
	}
	
	static class MyReducer extends Reducer<LongWritable, NullWritable, LongWritable, NullWritable>{
		long max = Long.MIN_VALUE;
		protected void reduce(LongWritable k2, java.lang.Iterable<NullWritable> arg1, org.apache.hadoop.mapreduce.Reducer<LongWritable,NullWritable,LongWritable,NullWritable>.Context arg2) throws java.io.IOException ,InterruptedException {
			final long temp = k2.get();
			if(temp>max){
				max = temp;
			}
		};
		
		protected void cleanup(org.apache.hadoop.mapreduce.Reducer<LongWritable,NullWritable,LongWritable,NullWritable>.Context context) throws java.io.IOException ,InterruptedException {
			context.write(new LongWritable(max), NullWritable.get());
		};
	}		
}