hadoop-java：计算平均值分布式程序编写

<pre name="code" class="java">类一：

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

public class MapperReduce {
<span style="white-space:pre">	</span>public MapperReduce(){//文件初始化，如果存在则删除
<span style="white-space:pre">		</span>File file=new File("F:","map_result.txt");
<span style="white-space:pre">		</span>if(file.exists()){
<span style="white-space:pre">			</span>file.delete();
<span style="white-space:pre">		</span>}
<span style="white-space:pre">		</span>File file1=new File("F:","output.txt");
<span style="white-space:pre">		</span>if(file1.exists()){
<span style="white-space:pre">			</span>file1.delete();
<span style="white-space:pre">		</span>}
<span style="white-space:pre">	</span>}
<span style="white-space:pre">	</span>
<span style="white-space:pre">	</span>public Map<String,List<String>> shuff() throws IOException{//shuffle过程，将mapper处理结果相同key的记录合并
<span style="white-space:pre">		</span>BufferedReader br=new BufferedReader(new FileReader("F:\\map_result.txt"));
        String line="";
        Map<String, List<String>> map=new HashMap<String, List<String>>();
        while((line=br.readLine())!=null){
        <span style="white-space:pre">	</span>String[] content=line.split(" ",2);
        <span style="white-space:pre">	</span>if(map.containsKey(content[0])){
        <span style="white-space:pre">		</span>List<String> list=new ArrayList<String>();
        <span style="white-space:pre">		</span>list=map.get(content[0]);
        <span style="white-space:pre">		</span>if(content.length==2){
        <span style="white-space:pre">		</span>    list.add(content[1]);
        <span style="white-space:pre">		</span>}else{
        <span style="white-space:pre">			</span>list.add(" ");
        <span style="white-space:pre">		</span>}
        <span style="white-space:pre">		</span>map.remove(content[0]);
        <span style="white-space:pre">		</span>map.put(content[0],list);
        <span style="white-space:pre">	</span>}else{
        <span style="white-space:pre">			</span>List<String> list=new ArrayList<String>();
            <span style="white-space:pre">		</span>if(content.length==2){
        <span style="white-space:pre">		</span>        list.add(content[1]);
            <span style="white-space:pre">		</span>}else{
            <span style="white-space:pre">			</span>list.add(" ");
            <span style="white-space:pre">		</span>}
        <span style="white-space:pre">		</span>    map.put(content[0],list);
        <span style="white-space:pre">	</span>}
        }
        map.remove("");//把key为空格的行删除
<span style="white-space:pre">		</span>return map;
<span style="white-space:pre">	</span>}
<span style="white-space:pre">	</span>
<span style="white-space:pre">	</span>public void IterMapper() throws IOException{
<span style="white-space:pre">		</span>//指定input数据文本路径，循环地按行输入，每一行中分key和value，其中区分标志为第一个空格,按行遍历mapper
<span style="white-space:pre">		</span>BufferedReader br=new BufferedReader(new FileReader("F:\\input.txt"));
        String line="";
        int count=1;
        while((line=br.readLine())!=null){
        <span style="white-space:pre">	</span>if(!line.trim().equals("")){
        <span style="white-space:pre">		</span>Mapper(String.valueOf(count),line);
        <span style="white-space:pre">	</span>} 
        <span style="white-space:pre">	</span>count +=1;
        }
<span style="white-space:pre">	</span>}
<span style="white-space:pre">	</span>
<span style="white-space:pre">	</span>public void IterReducer() throws IOException{
<span style="white-space:pre">		</span>//shuffle过程的生成的数据是一个map数据类型记录，按每条记录遍历reducer
<span style="white-space:pre">		</span>Map<String,List<String>> map=shuff();
<span style="white-space:pre">		</span>Iterator<String> Iter=map.keySet().iterator();
    <span style="white-space:pre">	</span>while(Iter.hasNext()){
    <span style="white-space:pre">		</span>String tmp=Iter.next();
    <span style="white-space:pre">		</span>Iterator<String> IterList=map.get(tmp).iterator();
    <span style="white-space:pre">		</span>Reducer(tmp,IterList);
    <span style="white-space:pre">	</span>}
<span style="white-space:pre">		</span>
<span style="white-space:pre">	</span>}
<span style="white-space:pre">	</span>
<span style="white-space:pre">	</span>public void WriteMaper(String new_key,String new_value) throws IOException{
<span style="white-space:pre">		</span>//指定mapper输出的文本路径，然后写入一行  new_key 与 new_value 用空格空开
<span style="white-space:pre">		</span>FileWriter fw=new FileWriter("F:\\map_result.txt",true); 
<span style="white-space:pre">		</span>String line=new_key+" "+new_value+"\r\n";
<span style="white-space:pre">		</span>fw.write(line);
<span style="white-space:pre">		</span>fw.close();
<span style="white-space:pre">	</span>}
<span style="white-space:pre">	</span>
<span style="white-space:pre">	</span>public void WriteReducer(String new_key,String new_value) throws IOException{
<span style="white-space:pre">		</span>//指定reducer输出的文件路径，然后写入一行  new_key 与 new_value 用空格空开
<span style="white-space:pre">		</span>FileWriter fw=new FileWriter("F:\\output.txt",true); 
<span style="white-space:pre">		</span>String line=new_key+" "+new_value+"\r\n";
<span style="white-space:pre">		</span>fw.write(line);
<span style="white-space:pre">		</span>fw.close();
<span style="white-space:pre">	</span>}
<span style="white-space:pre">	</span>
//map函数开始
<span style="white-space:pre">	</span>public void Mapper(String key,String value) throws IOException{


<span style="white-space:pre">	</span>}
//map函数结束
<span style="white-space:pre">	</span>
//reduce函数开始<span style="white-space:pre">	</span>
<span style="white-space:pre">	</span>public void Reducer(String key,Iterator<String> value) throws IOException{


<span style="white-space:pre">	</span>}
//reduce函数结束<span style="white-space:pre">		</span>
}

类二：

import java.io.IOException;
import java.util.Iterator;
import java.util.StringTokenizer;


public class MapReduce extends MapperReduce{

	public MapReduce(){
		super();
	}
	
	//map函数开始
		public void Mapper(String key,String value) throws IOException{
			//注意：当input文本传过来的数据中行为 （      str），传给Mapper函数参数则key有可能为空，即(key="",value="  str")
			String[] count=value.split(" ", 2);
			String new_key=count[0];
			String new_value=count[1].trim();
			WriteMaper(new_key,new_value);
		}
	//map函数结束
		
	//reduce函数开始	
		public void Reducer(String key,Iterator<String> value) throws IOException{
			String new_key = key;
			int count=0;
			int sum=0;
			while(value.hasNext()){
				String tmp=value.next();
				sum +=Integer.parseInt(tmp.trim());
				count +=1;
			}
			float avg=sum/count;
			String new_value=String.valueOf(avg);
			WriteReducer(new_key,new_value);
		}
	//reduce函数结束
	public static void main(String[] args) throws IOException {
		MapReduce a=new MapReduce();
		a.IterMapper();
		a.IterReducer();
	}
}

程序输入：

张三  99
李四 90
王五  90
赵六 60

张三  79
李四 75
王五   80
赵六  90


张三   89
李四  75
王五   70
赵六  90

程序输出：

李四 80.0
张三 89.0
王五 80.0
赵六 80.0