mahout源码KMeansDriver分析之五KmeansDriver(完结篇)

好吧，我上篇博客的仿造CIReducer的代码的确是有误的，错在哪？map输出的中心点是两个不错，但是其key是不一样的，所以要送到不同的reduce函数中（注意这里是函数，而非reducer，如果是同一个key则要送到同一个reduce中），所以不是一个for循环就搞定的，应该要key的个数个for循环才行。那就是说reducer中的reduce函数一次只能接受一个中心点了（针对一直使用的数据来说），好吧但是在reduce函数中只是有一个赋值语句呀，并没有更改这个中心点的center属性呀所以只会进入 if(first==null)中而已（如果有多个map的话可能会进入else中）。那么是在哪里更改center属性值的呢？往下看，

List<Cluster> models = new ArrayList<Cluster>();
    models.add(first.getValue());
    classifier = new ClusterClassifier(models, policy);
    classifier.close();

看到close-->AbstractClusteringPolicy.close()-->AbstractCluster.computeParameters():

public void computeParameters() {
    if (getS0() == 0) {
      return;
    }
    setNumObservations((long) getS0());
    setTotalObservations(getTotalObservations() + getNumObservations());
    setCenter(getS1().divide(getS0()));
    // compute the component stds
    if (getS0() > 1) {
      setRadius(getS2().times(getS0()).minus(getS1().times(getS1())).assign(new SquareRootFunction()).divide(getS0()));
    }
    setS0(0);
    setS1(center.like());
    setS2(center.like());
  }

好吧，这里有改center属性的方法了。

怎么知道以上猜测全部正确呢？

编写下面的类进行测试（这里主要是mahout的源码，只是里面加入了相关信息的显示而已），或者直接下载 kmeans.jarjar包也行。

Driver：

package mahout.fansy.kmeans1;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
//import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
import org.apache.mahout.clustering.Cluster;
//import org.apache.mahout.clustering.classify.ClusterClassifier;
import org.apache.mahout.clustering.iterator.ClusterIterator;
import org.apache.mahout.clustering.iterator.ClusterWritable;
//import org.apache.mahout.clustering.iterator.ClusteringPolicy;

public class CIDriver {

	/**
	 * @param args
	 * @throws InterruptedException 
	 * @throws IOException 
	 * @throws ClassNotFoundException 
	 */
	public static void main(String[] args) throws ClassNotFoundException, IOException, InterruptedException {
		if(args.length!=3){
			System.err.println("Usage:<inPath> <priorPath> <outPath>");
			System.exit(-1);
		}
		Path inPath=new Path(args[0]);
		Path priorPath=new Path(args[1]);
		Path outPath=new Path(args[2]);
		runMR(inPath,priorPath,outPath);
	}
	
	
	
	
	public static void runMR(Path inPath, Path priorPath, Path outPath)
		      throws IOException, InterruptedException, ClassNotFoundException {
		//    ClusteringPolicy policy = ClusterClassifier.readPolicy(priorPath);
		    Path clustersOut = null;
		    Configuration conf=new Configuration();
		    conf.set("mapred.job.tracker", "hadoop:9001");
		      conf.set(ClusterIterator.PRIOR_PATH_KEY, priorPath.toString());
		      
		      String jobName = "Cluster Iterator running iteration " + 1 + " over priorPath: " + priorPath;
		      System.out.println(jobName);
		      Job job = new Job(conf, jobName);
		      job.setMapOutputKeyClass(IntWritable.class);
		      job.setMapOutputValueClass(ClusterWritable.class);
		      job.setOutputKeyClass(IntWritable.class);
		      job.setOutputValueClass(ClusterWritable.class);
		      
		      job.setInputFormatClass(SequenceFileInputFormat.class);
		      job.setOutputFormatClass(SequenceFileOutputFormat.class);
		      job.setMapperClass(CIMapper.class);
		      job.setReducerClass(CIReducer.class);
		      
		      FileInputFormat.addInputPath(job, inPath);
		      clustersOut = new Path(outPath, Cluster.CLUSTERS_DIR + 1);
		      priorPath = clustersOut;
		      FileOutputFormat.setOutputPath(job, clustersOut);
		      
		      job.setJarByClass(ClusterIterator.class);
		      if (!job.waitForCompletion(true)) {
		        throw new InterruptedException("Cluster Iteration " + 1 + " failed processing " + priorPath);
		      }
		      /*
		      ClusterClassifier.writePolicy(policy, clustersOut);
		      FileSystem fs = FileSystem.get(outPath.toUri(), conf);
		      iteration++;
		      if (isConverged(clustersOut, conf, fs)) {
		        break;
		      }
		        */
		
		    /*
		    Path finalClustersIn = new Path(outPath, Cluster.CLUSTERS_DIR + (iteration - 1) + Cluster.FINAL_ITERATION_SUFFIX);
		    FileSystem.get(clustersOut.toUri(), conf).rename(clustersOut, finalClustersIn);
		  */
		  }

}

Mapper：

package mahout.fansy.kmeans1;

import java.io.IOException;
import java.util.Iterator;
import java.util.List;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.mahout.clustering.AbstractCluster;
import org.apache.mahout.clustering.Cluster;
import org.apache.mahout.clustering.GetAbstractClusterS_Fan;
import org.apache.mahout.clustering.classify.ClusterClassifier;
import org.apache.mahout.clustering.iterator.ClusterIterator;
import org.apache.mahout.clustering.iterator.ClusterWritable;
import org.apache.mahout.clustering.iterator.ClusteringPolicy;
import org.apache.mahout.clustering.iterator.DistanceMeasureCluster;
import org.apache.mahout.math.Vector;
import org.apache.mahout.math.Vector.Element;
import org.apache.mahout.math.VectorWritable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class CIMapper extends Mapper<WritableComparable<?>,VectorWritable,IntWritable,ClusterWritable> {
  
  private ClusterClassifier classifier;
  
  private ClusteringPolicy policy;
  private static final Logger log=LoggerFactory.getLogger(CIMapper.class);
  
  @Override
  protected void setup(Context context) throws IOException, InterruptedException {
    Configuration conf = context.getConfiguration();
    String priorClustersPath = conf.get(ClusterIterator.PRIOR_PATH_KEY);
    classifier = new ClusterClassifier();
    classifier.readFromSeqFiles(conf, new Path(priorClustersPath));
    policy = classifier.getPolicy();
    policy.update(classifier);
    
    // 查看setup函数的具体操作
    log.info("setup : classifier.getModels().get(0).toString():"+classifier.getModels().get(0).toString());
    log.info("setup : classifier.getModels().get(1).toString():"+classifier.getModels().get(1).toString());
    super.setup(context);
  }
  
  @Override
  protected void map(WritableComparable<?> key, VectorWritable value, Context context) throws IOException,
      InterruptedException {
    Vector probabilities = classifier.classify(value.get());
    Vector selections = policy.select(probabilities);
    for (Iterator<Element> it = selections.iterateNonZero(); it.hasNext();) {
      Element el = it.next();
      classifier.train(el.index(), value.get(), el.get());
    }
    /**
     * 在对应的包中编写一个类把原来protected的方法变为public
     */
    GetAbstractClusterS_Fan gs0= new GetAbstractClusterS_Fan( (AbstractCluster)(DistanceMeasureCluster) classifier.getModels().get(0));
    GetAbstractClusterS_Fan gs1= new GetAbstractClusterS_Fan( (AbstractCluster)(DistanceMeasureCluster) classifier.getModels().get(1));
    
    log.info("map : classifier.getModels().get(0).toString():"+classifier.getModels().get(0).toString()+",s0:"+gs0.getS0()+",s1:"+gs0.getS1().asFormatString());
    log.info("map : classifier.getModels().get(1).toString():"+classifier.getModels().get(1).toString()+","+gs1.getS0()+",s1:"+gs1.getS1().asFormatString());
  }
  
  @Override
  protected void cleanup(Context context) throws IOException, InterruptedException {
    List<Cluster> clusters = classifier.getModels();
    ClusterWritable cw = new ClusterWritable();
    for (int index = 0; index < clusters.size(); index++) {
      cw.setValue(clusters.get(index));
      log.info("cleanup : index:"+index+", clusterwritable : "+cw.getValue().toString());
      context.write(new IntWritable(index), cw);
    }
    super.cleanup(context);
  }
  
}

Reducer：

package mahout.fansy.kmeans1;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.mahout.clustering.Cluster;
import org.apache.mahout.clustering.classify.ClusterClassifier;
import org.apache.mahout.clustering.iterator.ClusterIterator;
import org.apache.mahout.clustering.iterator.ClusterWritable;
import org.apache.mahout.clustering.iterator.ClusteringPolicy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class CIReducer extends Reducer<IntWritable,ClusterWritable,IntWritable,ClusterWritable> {
  
  private ClusterClassifier classifier;
  private ClusteringPolicy policy;
  private static final Logger log=LoggerFactory.getLogger(CIReducer.class);
  
  @Override
  protected void reduce(IntWritable key, Iterable<ClusterWritable> values, Context context) throws IOException,
      InterruptedException {
    Iterator<ClusterWritable> iter = values.iterator();
    ClusterWritable first = null;
    log.info("**********************before while *****************");
    while (iter.hasNext()) {
      ClusterWritable cw = iter.next();
      if (first == null) {
        first = cw;
        log.info("reduce : first ==null:first:"+first.getValue().toString());
      } else {
        first.getValue().observe(cw.getValue());
        log.info("reduce : first !=null:first:"+first.getValue().toString());
      }
    }
    log.info("**********************after while *****************");
    List<Cluster> models = new ArrayList<Cluster>();
    models.add(first.getValue());
    classifier = new ClusterClassifier(models, policy);
    classifier.close();
    log.info("last ------------------reduce : first !=null:first:"+first.getValue().toString());
    context.write(key, first);
  }
  
  @Override
  protected void setup(Context context) throws IOException, InterruptedException {
    Configuration conf = context.getConfiguration();
    String priorClustersPath = conf.get(ClusterIterator.PRIOR_PATH_KEY);
    classifier = new ClusterClassifier();
    classifier.readFromSeqFiles(conf, new Path(priorClustersPath));
    policy = classifier.getPolicy();
    policy.update(classifier);
    log.info("r setup : classifier.getModels().get(0).toString():"+classifier.getModels().get(0).toString());
    log.info("r setup : classifier.getModels().get(1).toString():"+classifier.getModels().get(1).toString());
    super.setup(context);
  }
  
}

因为在Mapper中要访问KCluster中的S0和S1属性，所以在org.apache.mahout.clustering包中编写了一个类，用于把AbstractCluster类中的getS0和getS1()方法放出来。如下：

package org.apache.mahout.clustering;

import org.apache.mahout.math.Vector;

public class GetAbstractClusterS_Fan {
	private AbstractCluster aCluster;
	
	public GetAbstractClusterS_Fan(){
		
	}
	
	public GetAbstractClusterS_Fan(AbstractCluster aCluster_){
		this.aCluster=aCluster_;
	}
	/**
	 * get s0 value
	 * @return
	 */
	public double getS0(){
		return this.aCluster.getS0();
	}
	
	/**
	 * get s1 vector value
	 * @return
	 */
	public  Vector getS1(){
		return this.aCluster.getS1();
	}
	/**
	 * get s2 vector value
	 * @return
	 */
	public Vector getS2(){
		return this.aCluster.getS2();
	}
	
}

然后运行就可以在web页面查看log信息了，如下：mapper：

如上红色框中可以看到map输出的中心点的center属性没有改变，但是它的s0和s1属性都是改变的；下面是reducer：

红色框中即为reduce的输出了，可以看出新的中心点的center值为s1除以s0；和上面分析说最后close方法运行设置center值的方法是一样的。

分享，快乐，成长

转载请注明出处：http://blog.csdn.net/fansy1990