大矩阵乘法

原理参考：http://blog.csdn.net/xyilu/article/details/9066973

 

我的代码实现

 

****************Starter********************************
 
import java.io.IOException;
import java.util.regex.Pattern;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class Starter
{
 public static final Pattern DELIMITER =Pattern.compile("[\t,]");
 
 public static void main(String[] args) throwsException
 {
  Configuration conf = newConfiguration();
  
  // global variable, set therows and cols of the final matrix
  conf.setInt("rows", 3);
  conf.setInt("cols", 2);
  
  Job job = newJob(conf,"MatrixMultiply");
  job.setInputFormatClass(TextInputFormat.class);
  
       //  specify mapper & reducer
  job.setMapperClass(MatrixMultiplyMapper.class);
  job.setReducerClass(MatrixMultiplyReducer.class);
  
  // specify output types ofmapper and reducer
  job.setOutputKeyClass(Text.class);
  job.setOutputValueClass(Text.class);
  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(Text.class);
  
  // specify input and outputDIRECTORIES
  Path inPath = newPath("hdfs://192.168.1.201:9000/user/stevie/week9/data");
  Path outPath = newPath("hdfs://192.168.1.201:9000/user/stevie/week9/result");
  FileInputFormat.addInputPath(job,inPath);
       FileOutputFormat.setOutputPath(job,outPath);    //  output path
       
       
  // delete outputdirectory
  try{
   FileSystemhdfs = outPath.getFileSystem(conf);
   if(hdfs.exists(outPath))
    hdfs.delete(outPath);
   hdfs.close();
  } catch (Exception e){
   e.printStackTrace();
   return;
  }
  
  //  run thejob
  System.exit(job.waitForCompletion(true)? 0 : 1);
 }
}

*************************************

Starter是整个MR的启动器

 

  conf.setInt("rows",3);
  conf.setInt("cols", 2);
上面这两行用于指定结果集的维度

 

***************MatrixMultiplyMapper**********************
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
 
public class MatrixMultiplyMapper extends Mapper 
{
 private String fileName;
 private int rows;
 private int cols;
 
 protected void setup(Context context)
 {
  FileSplit fs = (FileSplit)context.getInputSplit();
  fileName =fs.getPath().getName();
  Configuration conf =context.getConfiguration();
  rows = conf.getInt("rows",1);
  cols = conf.getInt("cols",1);
 }
 
 public void map(Object key, Text value, Contextcontext) throws IOException, InterruptedException
 {  
  String[] values =Starter.DELIMITER.split(value.toString());
  String x = values[0];
  String y = values[1];
  String content =values[2];
  
  if(fileName.startsWith("data1"))
  {
   for(int i =1;i <= cols;i++)
   {
    StringoutputKey = x + "," + i;
    StringoutputValue = "A," + y + "," + content;
    context.write(newText(outputKey), new Text(outputValue));
   }
  }
  else
  {
   for(int i =1; i <= rows;i++)
   {
    StringoutputKey = i + "," + y;
    StringoutputValue = "B," + x + "," + content;
    context.write(newText(outputKey), new Text(outputValue));
   }
  }
  
  
 }
}

*************************************

 

******************MatrixMultiplyReducer*******************
import java.io.IOException;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

public class MatrixMultiplyReducer extends Reducer
{
 public void reduce(Text key, Iterable values,Context context) throws IOException, InterruptedException
 {
  int times =context.getConfiguration().getInt("rows", 1);
  
  int[] AGroup = newint[times];
  int[] BGroup = newint[times];
  
  for(Text value: values)
  {
   String[]array = value.toString().split(",");
   String matrix= array[0];
   int position= Integer.parseInt(array[1]);
   int content =Integer.parseInt(array[2]);
   
   if(matrix.startsWith("A"))
   {
    AGroup[position- 1]  = content;
   }
   else
   {
    BGroup[position- 1]  = content;
   }
  }
  
  int result = 0;
  for(int i=0;i
  {
   result +=AGroup[i] * BGroup[i];
  }
  
  String outputValue = "," +result;
  
  context.write(key, newText(outputValue));
 }
}
*************************************