经典面试题-出现最多的十个数

问题：有1/10/100 亿个 数/字符串(URL)，找到其中出现次数最多的10个 数/字符串(URL)

解决思路：

    1.通过hash取模的方式分成电脑可以运行的数据大小；

    2.在每一块中排序，然后使用一个最小堆维护出现次数最多的几个；

    3.取所有的块中出现次数最多的十个 数/字符串。

先运用hadoop中的mapreduce来解决一波--

开始生成了10亿个数的文件 (大概6G多)，然后放到集群中把电脑跑爆了(垃圾电脑开了四个虚拟机)

然后就重新生成了1亿个数的数据(653M），才得以运行成功。

生成数据代码：

#include<stdio.h>
#include<stdlib.h>
#include<math.h> 
int main(){
	freopen("data.txt","w",stdout);
	int x;
	for (int i=0;i<100000000;i++)
	{
		x = rand()%1000000000;
		printf("%d\n",x);
	}
	return 0;
}

生成的数据：

上传数据到hdfs：

map代码：

package com.rm_int;

import java.io.IOException;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

public class myMapper extends Mapper<LongWritable, Text, IntWritable, IntWritable>{
	
	@Override
	protected void map(LongWritable key, Text value,
			Mapper<LongWritable, Text, IntWritable, IntWritable>.Context context)
					throws IOException, InterruptedException {

		Integer x = Integer.parseInt(value.toString());
		context.write(new IntWritable(x.hashCode()%20)  , new IntWritable(x));
	}
	
}

reduce代码：

package com.rm_int;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.PriorityQueue;
import java.util.Queue;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;


public class myReduce extends Reducer<IntWritable, IntWritable,IntWritable, Text>{
	
	@Override
	protected void reduce(IntWritable key, Iterable<IntWritable> value,
			Reducer<IntWritable, IntWritable, IntWritable, Text>.Context context)
					throws IOException, InterruptedException {
		
		class Node{
			int value;//值
			int num;//个数
		}
		
		List list = new ArrayList<Integer>();
		for (IntWritable x:value){
			list.add(x.get());
		}
		
		Queue que = new PriorityQueue<Node>(10,new Comparator<Node>() {

			@Override
			public int compare(Node o1, Node o2) {
				return o1.num - o2.num;
			}
		});
		
		list.sort(new Comparator<Integer>() {
			@Override
			public int compare(Integer x, Integer y) {
				return x-y;
			}
		});
		
		Node node = new Node();
		Node nodeTo;
		for (int i=0;i<list.size();i++){
			if (i==0){
				node.value=(int) list.get(i);
				node.num=1;
			}else {
				if ((int)list.get(i) == (int)list.get(i-1)){
					node.num++;
				}else{
					if (que.size()==10){
						nodeTo = (Node) que.peek();
						if (nodeTo.num<node.num){
							que.remove();
							que.add(node);
						}
					}else{
						que.add(node);
					}
					node = new Node();
					node.value = (int) list.get(i);
					node.num=1;
				}
			}
			if (i==list.size()-1){
				if (que.size()==10){
					nodeTo = (Node) que.peek();
					if (nodeTo.num<node.num){
						que.remove();
						que.add(node);
					}
				}else{
					que.add(node);
				}
			}
		}
		
		while (!que.isEmpty()){
			node = (Node) que.poll();
			context.write(key, new Text(node.value+" "+node.num));
		}
		
	}
}

mapreduce结果：

将结果从hdfs中下载下来：

合并取最多次数的代码：

import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStreamWriter;
import java.util.Comparator;
import java.util.PriorityQueue;
import java.util.Queue;
import java.util.Scanner;



public class Main {

	public static void main(String[] args) throws IOException {
		
		class Node{
			int value;//值
			int num;//个数
		}
		
		InputStream is = new FileInputStream("F://output.txt");
		OutputStreamWriter osw = new OutputStreamWriter(new FileOutputStream("F://answer.txt"));
		
		Scanner cin = new Scanner(is);
		
		Queue que = new PriorityQueue<Node>(10,new Comparator<Node>() {

			@Override
			public int compare(Node o1, Node o2) {
				return o1.num - o2.num;
			}
		});
		
		Node node;
		Node nodeTo;
		
		while (cin.hasNext()){
			String str = cin.nextLine();
			String[] x = str.split(" |\t");
			node = new Node();
			node.value = Integer.parseInt(x[1]);
			node.num = Integer.parseInt(x[2]);
			if (que.size()==10){
				nodeTo = (Node) que.peek();
				if (nodeTo.num<node.num){
					que.remove();
					que.add(node);
				}
			}else{
				que.add(node);
			}
		}
		
		Node ans[] = new Node[10];
		int index = 10;
		
		while (!que.isEmpty()){
			node = (Node) que.poll();
			ans[--index] = node;
		}
		for (int i=0;i<10;i++)
			osw.write(ans[i].value+"\t"+ans[i].num+"\r\n");
		osw.close();
		is.close();
	}
}

最后结果：