package com.matrix;
import java.io.BufferedReader;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.filecache.DistributedCache;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;
import java.util.*;
/**************************************
*
* @author Lincolnfather
* @date 20131107
* hadoop 矩阵乘法
* 在http://hadoop.nchc.org.tw/
* 测试通过
*
*/
public class MatrixMR {
public static class MatrixMapper extends Mapper<Object,Text,MatrixPair,IntWritable>{
private List<List<Integer>> cache = new ArrayList< List<Integer>>();
private int i = 0;
public void setup(Context context){
BufferedReader br = null;
try {
Path[] cacheFiles = DistributedCache.getLocalCacheFiles(context.getConfiguration());
System.out.println(cacheFiles== null);
br =
new BufferedReader(new InputStreamReader(new FileInputStream(cacheFiles[0].toString())));
String line = null;
while((line = br.readLine())!= null){
String[] strs = line.split("#");
List<Integer> is = new ArrayList<Integer>();
for(String str :strs){
is.add(Integer.parseInt(str));
}
cache.add(is);
}
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}finally{
try {
if(br != null) br.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
public void map(Object key,Text value,Context context){
String[] vals = value.toString().split("#");
if(i<cache.get(0).size()){
for(int j = 0;j<cache.size();j++){
for(int k = 0;k<vals.length;k++){
try {
MatrixPair mp = new MatrixPair();
mp.setRowindex(j);
mp.setColumnindex(k);
context.write(mp,
new IntWritable(cache.get(j).get(i)*Integer.parseInt(vals[k])));
} catch (NumberFormatException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
}
i++;
}
}
public static class MatrixReducer extends Reducer<MatrixPair,IntWritable,Text,Text>{
private StringBuffer sb = new StringBuffer();
public void reduce(MatrixPair key,Iterable<IntWritable> value,Context context){
int sum = 0;
for(IntWritable i:value){
sum += i.get();
}
try {
sb.append(sum);
if(key.getColumnindex() == 4){
context.write(new Text(sb.toString()), new Text(""));
sb = new StringBuffer();
}else
sb.append("#");
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
public static class MatrixPair implements WritableComparable<MatrixPair> {
Integer rowindex;
Integer columnindex;
@Override
public void readFields(DataInput read) throws IOException {
// TODO Auto-generated method stub
rowindex = read.readInt();
columnindex = read.readInt();
}
public void set(){
setRowindex(rowindex);
setColumnindex(columnindex);
}
@Override
public void write(DataOutput write) throws IOException {
// TODO Auto-generated method stub
write.writeInt(rowindex);
write.writeInt(columnindex);
}
@Override
public int compareTo(MatrixPair o) {
return rowindex - o.getRowindex() != 0?
rowindex - o.getRowindex():
columnindex - o.getColumnindex() != 0?
columnindex - o.getColumnindex():0;
}
public Integer getRowindex() {
return rowindex;
}
public void setRowindex(Integer rowindex) {
this.rowindex = rowindex;
}
public Integer getColumnindex() {
return columnindex;
}
public void setColumnindex(Integer columnindex) {
this.columnindex = columnindex;
}
}
public static void main(String args[]) throws IOException, ClassNotFoundException, InterruptedException{
Configuration conf = new Configuration();
String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
if (otherArgs.length != 2) {
System.err.println("matrix multi <in> <out>");
System.exit(2);
}
DistributedCache.addCacheFile(new Path("hdfs://hadoop.nchc.org.tw/user/h3969/matrix1.txt").toUri(), conf);
Job job = new Job(conf, "my matrix multiply");
job.setJarByClass(MatrixMR.class);
job.setMapperClass(MatrixMapper.class);
// job.setCombinerClass(FirstReducer.class);
job.setReducerClass(MatrixReducer.class);
job.setMapOutputKeyClass(MatrixPair.class);
job.setMapOutputValueClass(IntWritable.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
import java.io.BufferedReader;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.filecache.DistributedCache;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;
import java.util.*;
/**************************************
*
* @author Lincolnfather
* @date 20131107
* hadoop 矩阵乘法
* 在http://hadoop.nchc.org.tw/
* 测试通过
*
*/
public class MatrixMR {
public static class MatrixMapper extends Mapper<Object,Text,MatrixPair,IntWritable>{
private List<List<Integer>> cache = new ArrayList< List<Integer>>();
private int i = 0;
public void setup(Context context){
BufferedReader br = null;
try {
Path[] cacheFiles = DistributedCache.getLocalCacheFiles(context.getConfiguration());
System.out.println(cacheFiles== null);
br =
new BufferedReader(new InputStreamReader(new FileInputStream(cacheFiles[0].toString())));
String line = null;
while((line = br.readLine())!= null){
String[] strs = line.split("#");
List<Integer> is = new ArrayList<Integer>();
for(String str :strs){
is.add(Integer.parseInt(str));
}
cache.add(is);
}
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}finally{
try {
if(br != null) br.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
public void map(Object key,Text value,Context context){
String[] vals = value.toString().split("#");
if(i<cache.get(0).size()){
for(int j = 0;j<cache.size();j++){
for(int k = 0;k<vals.length;k++){
try {
MatrixPair mp = new MatrixPair();
mp.setRowindex(j);
mp.setColumnindex(k);
context.write(mp,
new IntWritable(cache.get(j).get(i)*Integer.parseInt(vals[k])));
} catch (NumberFormatException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
}
i++;
}
}
public static class MatrixReducer extends Reducer<MatrixPair,IntWritable,Text,Text>{
private StringBuffer sb = new StringBuffer();
public void reduce(MatrixPair key,Iterable<IntWritable> value,Context context){
int sum = 0;
for(IntWritable i:value){
sum += i.get();
}
try {
sb.append(sum);
if(key.getColumnindex() == 4){
context.write(new Text(sb.toString()), new Text(""));
sb = new StringBuffer();
}else
sb.append("#");
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
public static class MatrixPair implements WritableComparable<MatrixPair> {
Integer rowindex;
Integer columnindex;
@Override
public void readFields(DataInput read) throws IOException {
// TODO Auto-generated method stub
rowindex = read.readInt();
columnindex = read.readInt();
}
public void set(){
setRowindex(rowindex);
setColumnindex(columnindex);
}
@Override
public void write(DataOutput write) throws IOException {
// TODO Auto-generated method stub
write.writeInt(rowindex);
write.writeInt(columnindex);
}
@Override
public int compareTo(MatrixPair o) {
return rowindex - o.getRowindex() != 0?
rowindex - o.getRowindex():
columnindex - o.getColumnindex() != 0?
columnindex - o.getColumnindex():0;
}
public Integer getRowindex() {
return rowindex;
}
public void setRowindex(Integer rowindex) {
this.rowindex = rowindex;
}
public Integer getColumnindex() {
return columnindex;
}
public void setColumnindex(Integer columnindex) {
this.columnindex = columnindex;
}
}
public static void main(String args[]) throws IOException, ClassNotFoundException, InterruptedException{
Configuration conf = new Configuration();
String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
if (otherArgs.length != 2) {
System.err.println("matrix multi <in> <out>");
System.exit(2);
}
DistributedCache.addCacheFile(new Path("hdfs://hadoop.nchc.org.tw/user/h3969/matrix1.txt").toUri(), conf);
Job job = new Job(conf, "my matrix multiply");
job.setJarByClass(MatrixMR.class);
job.setMapperClass(MatrixMapper.class);
// job.setCombinerClass(FirstReducer.class);
job.setReducerClass(MatrixReducer.class);
job.setMapOutputKeyClass(MatrixPair.class);
job.setMapOutputValueClass(IntWritable.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}
数据:matrix1.txt
3#4#1
2#4#8
1#2#7
5#6#5
数据matrix2.txt:
5#8#7#4#9
2#6#5#4#2
9#1#3#4#7
本 算法适合数亿行以下的一个矩阵和另一个不限规模的矩阵想成,如果想实现两个不限规模的矩阵相乘则需要在数据加入DistributeCache之前切分