JavaDay17

学习来源：日撸 Java 三百行（61-70天，决策树与集成学习）_闵帆的博客——CSDN博客

集成学习之AdaBoosting

集成学习之AdaBoosting回顾：第一部分（带权数据集）

1.树桩分类器：

        单层决策树，也称决策树桩。是一种简单的决策树，通过给定的阈值进行分类。决策树桩根据对单个属性值的判断进行分类，适合作为集成学习中的弱分类器。

2.抽象分类器：

        在此基础上可以实现树桩分类器，也可以支持其它的基础分类器。

代码如下：

package JavaDay17;

import java.util.Random;

import JavaDay16.WeightedInstances;
import weka.core.Instance;

/**
 * @author Ke-Xiong Wang
 *
 * The super class of any simple classifier.
 */

public abstract class SimpleClassifier {

    /**
     * The index of the current attribute.
     */
    int selectedAttribute;

    /**
     * Weighted data.
     */
    WeightedInstances weightedInstances;

    /**
     * The accuracy on the training set.
     */
    double trainingAccuracy;

    /**
     * The number of classes. For binary classification it is 2.
     */
    int numClasses;

    /**
     * The number of instances.
     */
    int numInstances;

    /**
     * The number of conditional attributes.
     */
    int numConditions;

    /**
     * For random number generation.
     */
    Random random = new Random();

    /**
     ******************
     * The first constructor.
     *
     * @param paraWeightedInstances
     *            The given instances.
     ******************
     */
    public SimpleClassifier(WeightedInstances paraWeightedInstances) {
        weightedInstances = paraWeightedInstances;

        numConditions = weightedInstances.numAttributes() - 1;
        numInstances = weightedInstances.numInstances();
        numClasses = weightedInstances.classAttribute().numValues();
    }// Of the first constructor

    /**
     ******************
     * Train the classifier.
     ******************
     */
    public abstract void train();

    /**
     ******************
     * Classify an instance.
     *
     * @param paraInstance
     *            The given instance.
     * @return Predicted label.
     ******************
     */
    public abstract int classify(Instance paraInstance);

    /**
     ******************
     * Which instances in the training set are correctly classified.
     *
     * @return The correctness array.
     ******************
     */
    public boolean[] computeCorrectnessArray() {
        boolean[] resultCorrectnessArray = new boolean[weightedInstances.numInstances()];
        for (int i = 0; i < resultCorrectnessArray.length; i++) {
            Instance tempInstance = weightedInstances.instance(i);
            if ((int) (tempInstance.classValue()) == classify(tempInstance)) {
                resultCorrectnessArray[i] = true;
            } // Of if

            // System.out.print("\t" + classify(tempInstance));
        } // Of for i
        // System.out.println();
        return resultCorrectnessArray;
    }// Of computeCorrectnessArray

    /**
     ******************
     * Compute the accuracy on the training set.
     *
     * @return The training accuracy.
     ******************
     */
    public double computeTrainingAccuracy() {
        double tempCorrect = 0;
        boolean[] tempCorrectnessArray = computeCorrectnessArray();
        for (int i = 0; i < tempCorrectnessArray.length; i++) {
            if (tempCorrectnessArray[i]) {
                tempCorrect++;
            } // Of if
        } // Of for i

        double resultAccuracy = tempCorrect / tempCorrectnessArray.length;

        return resultAccuracy;
    }// Of computeTrainingAccuracy

    /**
     ******************
     * Compute the weighted error on the training set. It is at least 1e-6 to
     * avoid NaN.
     *
     * @return The weighted error.
     ******************
     */
    public double computeWeightedError() {
        double resultError = 0;
        boolean[] tempCorrectnessArray = computeCorrectnessArray();
        for (int i = 0; i < tempCorrectnessArray.length; i++) {
            if (!tempCorrectnessArray[i]) {
                resultError += weightedInstances.getWeight(i);
            } // Of if
        } // Of for i

        if (resultError < 1e-6) {
            resultError = 1e-6;
        } // Of if

        return resultError;
    }// Of computeWeightedError
} // Of class SimpleClassifier

3.树桩分类器：

代码如下：

package JavaDay17;

import JavaDay16.WeightedInstances;
import weka.core.Instance;
import java.io.FileReader;
import java.util.*;

/**
 * @author Ke-Xiong Wang
 *
 * The stump classifier.
 */
public class StumpClassifier extends SimpleClassifier {

    /**
     * The best cut for the current attribute on weightedInstances.
     */
    double bestCut;

    /**
     * The class label for attribute value less than bestCut.
     */
    int leftLeafLabel;

    /**
     * The class label for attribute value no less than bestCut.
     */
    int rightLeafLabel;

    /**
     ******************
     * The only constructor.
     *
     * @param paraWeightedInstances
     *            The given instances.
     ******************
     */
    public StumpClassifier(WeightedInstances paraWeightedInstances) {
        super(paraWeightedInstances);
    }// Of the only constructor

    /**
     ******************
     * Train the classifier.
     ******************
     */
    public void train() {
        // Step 1. Randomly choose an attribute.
        selectedAttribute = random.nextInt(numConditions);

        // Step 2. Find all attribute values and sort.
        double[] tempValuesArray = new double[numInstances];
        for (int i = 0; i < tempValuesArray.length; i++) {
            tempValuesArray[i] = weightedInstances.instance(i).value(selectedAttribute);
        } // Of for i
        Arrays.sort(tempValuesArray);

        // Step 3. Initialize, classify all instances as the same with the
        // original cut.
        int tempNumLabels = numClasses;
        double[] tempLabelCountArray = new double[tempNumLabels];
        int tempCurrentLabel;

        // Step 3.1 Scan all labels to obtain their counts.
        for (int i = 0; i < numInstances; i++) {
            // The label of the ith instance
            tempCurrentLabel = (int) weightedInstances.instance(i).classValue();
            tempLabelCountArray[tempCurrentLabel] += weightedInstances.getWeight(i);
        } // Of for i

        // Step 3.2 Find the label with the maximal count.
        double tempMaxCorrect = 0;
        int tempBestLabel = -1;
        for (int i = 0; i < tempLabelCountArray.length; i++) {
            if (tempMaxCorrect < tempLabelCountArray[i]) {
                tempMaxCorrect = tempLabelCountArray[i];
                tempBestLabel = i;
            } // Of if
        } // Of for i

        // Step 3.3 The cut is a little bit smaller than the minimal value.
        bestCut = tempValuesArray[0] - 0.1;
        leftLeafLabel = tempBestLabel;
        rightLeafLabel = tempBestLabel;

        // Step 4. Check candidate cuts one by one.
        // Step 4.1 To handle multi-class data, left and right.
        double tempCut;
        double[][] tempLabelCountMatrix = new double[2][tempNumLabels];

        for (int i = 0; i < tempValuesArray.length - 1; i++) {
            // Step 4.1 Some attribute values are identical, ignore them.
            if (tempValuesArray[i] == tempValuesArray[i + 1]) {
                continue;
            } // Of if
            tempCut = (tempValuesArray[i] + tempValuesArray[i + 1]) / 2;

            // Step 4.2 Scan all labels to obtain their counts wrt. the cut.
            // Initialize again since it is used many times.
            for (int j = 0; j < 2; j++) {
                for (int k = 0; k < tempNumLabels; k++) {
                    tempLabelCountMatrix[j][k] = 0;
                } // Of for k
            } // Of for j

            for (int j = 0; j < numInstances; j++) {
                // The label of the jth instance
                tempCurrentLabel = (int) weightedInstances.instance(j).classValue();
                if (weightedInstances.instance(j).value(selectedAttribute) < tempCut) {
                    tempLabelCountMatrix[0][tempCurrentLabel] += weightedInstances.getWeight(j);
                } else {
                    tempLabelCountMatrix[1][tempCurrentLabel] += weightedInstances.getWeight(j);
                } // Of if
            } // Of for i

            // Step 4.3 Left leaf.
            double tempLeftMaxCorrect = 0;
            int tempLeftBestLabel = 0;
            for (int j = 0; j < tempLabelCountMatrix[0].length; j++) {
                if (tempLeftMaxCorrect < tempLabelCountMatrix[0][j]) {
                    tempLeftMaxCorrect = tempLabelCountMatrix[0][j];
                    tempLeftBestLabel = j;
                } // Of if
            } // Of for i

            // Step 4.4 Right leaf.
            double tempRightMaxCorrect = 0;
            int tempRightBestLabel = 0;
            for (int j = 0; j < tempLabelCountMatrix[1].length; j++) {
                if (tempRightMaxCorrect < tempLabelCountMatrix[1][j]) {
                    tempRightMaxCorrect = tempLabelCountMatrix[1][j];
                    tempRightBestLabel = j;
                } // Of if
            } // Of for i

            // Step 4.5 Compare with the current best.
            if (tempMaxCorrect < tempLeftMaxCorrect + tempRightMaxCorrect) {
                tempMaxCorrect = tempLeftMaxCorrect + tempRightMaxCorrect;
                bestCut = tempCut;
                leftLeafLabel = tempLeftBestLabel;
                rightLeafLabel = tempRightBestLabel;
            } // Of if
        } // Of for i

        System.out.println("Attribute = " + selectedAttribute + ", cut = " + bestCut + ", leftLeafLabel = "
                + leftLeafLabel + ", rightLeafLabel = " + rightLeafLabel);
    }// Of train

    /**
     ******************
     * Classify an instance.
     *
     * @param paraInstance
     *            The given instance.
     * @return Predicted label.
     ******************
     */
    public int classify(Instance paraInstance) {
        int resultLabel = -1;
        if (paraInstance.value(selectedAttribute) < bestCut) {
            resultLabel = leftLeafLabel;
        } else {
            resultLabel = rightLeafLabel;
        } // Of if
        return resultLabel;
    }// Of classify

    /**
     ******************
     * For display.
     ******************
     */
    public String toString() {
        String resultString = "I am a stump classifier.\r\n" + "I choose attribute #" + selectedAttribute
                + " with cut value " + bestCut + ".\r\n" + "The left and right leaf labels are " + leftLeafLabel
                + " and " + rightLeafLabel + ", respectively.\r\n" + "My weighted error is: " + computeWeightedError()
                + ".\r\n" + "My weighted accuracy is : " + computeTrainingAccuracy() + ".";

        return resultString;
    }// Of toString

    /**
     ******************
     * For unit test.
     *
     * @param args
     *            Not provided.
     ******************
     */
    public static void main(String args[]) {
        WeightedInstances tempWeightedInstances = null;
        String tempFilename = "D:/iris.arff";
        try {
            FileReader tempFileReader = new FileReader(tempFilename);
            tempWeightedInstances = new WeightedInstances(tempFileReader);
            tempFileReader.close();
        } catch (Exception ee) {
            System.out.println("Cannot read the file: " + tempFilename + "\r\n" + ee);
            System.exit(0);
        } // Of try

        StumpClassifier tempClassifier = new StumpClassifier(tempWeightedInstances);
        tempClassifier.train();
        System.out.println(tempClassifier);

        System.out.println(Arrays.toString(tempClassifier.computeCorrectnessArray()));
    }// Of main
}// Of class StumpClassifier

 运行结果：

 4.集成器：

        将所有的弱分类器结合起来，通过加权投票的方式确定分类结果。

代码如下：

package JavaDay17;

import java.io.FileReader;

import JavaDay16.WeightedInstances;
import weka.core.Instance;
import weka.core.Instances;

/**
 * @author Ke-Xing Wang
 *
 * The booster which ensembles base classifiers.
 */
public class Booster {

    /**
     * Classifiers.
     */
    SimpleClassifier[] classifiers;

    /**
     * Number of classifiers.
     */
    int numClassifiers;

    /**
     * Weather or not stop after the training error is 0.
     */
    boolean stopAfterConverge = false;

    /**
     * The weights of classifiers.
     */
    double[] classifierWeights;

    /**
     * The training data.
     */
    Instances trainingData;

    /**
     * The testing data.
     */
    Instances testingData;

    /**
     ******************
     * The first constructor. The testing set is the same as the training set.
     *
     * @param paraTrainingFilename
     *            The data filename.
     ******************
     */
    public Booster(String paraTrainingFilename) {
        // Step 1. Read training set.
        try {
            FileReader tempFileReader = new FileReader(paraTrainingFilename);
            trainingData = new Instances(tempFileReader);
            tempFileReader.close();
        } catch (Exception ee) {
            System.out.println("Cannot read the file: " + paraTrainingFilename + "\r\n" + ee);
            System.exit(0);
        } // Of try

        // Step 2. Set the last attribute as the class index.
        trainingData.setClassIndex(trainingData.numAttributes() - 1);

        // Step 3. The testing data is the same as the training data.
        testingData = trainingData;

        stopAfterConverge = true;

        //System.out.println("****************Data**********\r\n" + trainingData);
    }// Of the first constructor

    /**
     ******************
     * Set the number of base classifier, and allocate space for them.
     *
     * @param paraNumBaseClassifiers
     *            The number of base classifier.
     ******************
     */
    public void setNumBaseClassifiers(int paraNumBaseClassifiers) {
        numClassifiers = paraNumBaseClassifiers;

        // Step 1. Allocate space (only reference) for classifiers
        classifiers = new SimpleClassifier[numClassifiers];

        // Step 2. Initialize classifier weights.
        classifierWeights = new double[numClassifiers];
    }// Of setNumBaseClassifiers

    /**
     ******************
     * Train the booster.
     *
     * @see JavaDay17.StumpClassifier#train()
     ******************
     */
    public void train() {
        // Step 1. Initialize.
        WeightedInstances tempWeightedInstances = null;
        double tempError;
        numClassifiers = 0;

        // Step 2. Build other classifiers.
        for (int i = 0; i < classifiers.length; i++) {
            // Step 2.1 Key code: Construct or adjust the weightedInstances
            if (i == 0) {
                tempWeightedInstances = new WeightedInstances(trainingData);
            } else {
                // Adjust the weights of the data.
                tempWeightedInstances.adjustWeights(classifiers[i - 1].computeCorrectnessArray(),
                        classifierWeights[i - 1]);
            } // Of if

            // Step 2.2 Train the next classifier.
            classifiers[i] = new StumpClassifier(tempWeightedInstances);
            classifiers[i].train();

            tempError = classifiers[i].computeWeightedError();

            // Key code: Set the classifier weight.
            classifierWeights[i] = 0.5 * Math.log(1 / tempError - 1);
            if (classifierWeights[i] < 1e-6) {
                classifierWeights[i] = 0;
            } // Of if

            System.out.println("Classifier #" + i + " , weighted error = " + tempError + ", weight = "
                    + classifierWeights[i] + "\r\n");

            numClassifiers++;

            // The accuracy is enough.
            if (stopAfterConverge) {
                double tempTrainingAccuracy = computeTrainingAccuray();
                System.out.println("The accuracy of the booster is: " + tempTrainingAccuracy + "\r\n");
                if (tempTrainingAccuracy > 0.999999) {
                    System.out.println("Stop at the round: " + i + " due to converge.\r\n");
                    break;
                } // Of if
            } // Of if
        } // Of for i
    }// Of train

    /**
     ******************
     * Classify an instance.
     *
     * @param paraInstance
     *            The given instance.
     * @return The predicted label.
     ******************
     */
    public int classify(Instance paraInstance) {
        double[] tempLabelsCountArray = new double[trainingData.classAttribute().numValues()];
        for (int i = 0; i < numClassifiers; i++) {
            int tempLabel = classifiers[i].classify(paraInstance);
            tempLabelsCountArray[tempLabel] += classifierWeights[i];
        } // Of for i

        int resultLabel = -1;
        double tempMax = -1;
        for (int i = 0; i < tempLabelsCountArray.length; i++) {
            if (tempMax < tempLabelsCountArray[i]) {
                tempMax = tempLabelsCountArray[i];
                resultLabel = i;
            } // Of if
        } // Of for

        return resultLabel;
    }// Of classify

    /**
     ******************
     * Test the booster on the training data.
     *
     * @return The classification accuracy.
     ******************
     */
    public double test() {
        System.out.println("Testing on " + testingData.numInstances() + " instances.\r\n");

        return test(testingData);
    }// Of test

    /**
     ******************
     * Test the booster.
     *
     * @param paraInstances
     *            The testing set.
     * @return The classification accuracy.
     ******************
     */
    public double test(Instances paraInstances) {
        double tempCorrect = 0;
        paraInstances.setClassIndex(paraInstances.numAttributes() - 1);

        for (int i = 0; i < paraInstances.numInstances(); i++) {
            Instance tempInstance = paraInstances.instance(i);
            if (classify(tempInstance) == (int) tempInstance.classValue()) {
                tempCorrect++;
            } // Of if
        } // Of for i

        double resultAccuracy = tempCorrect / paraInstances.numInstances();
        System.out.println("The accuracy is: " + resultAccuracy);

        return resultAccuracy;
    } // Of test

    /**
     ******************
     * Compute the training accuracy of the booster. It is not weighted.
     *
     * @return The training accuracy.
     ******************
     */
    public double computeTrainingAccuray() {
        double tempCorrect = 0;

        for (int i = 0; i < trainingData.numInstances(); i++) {
            if (classify(trainingData.instance(i)) == (int) trainingData.instance(i).classValue()) {
                tempCorrect++;
            } // Of if
        } // Of for i

        double tempAccuracy = tempCorrect / trainingData.numInstances();

        return tempAccuracy;
    }// Of computeTrainingAccuray

    /**
     ******************
     * For integration test.
     *
     * @param args
     *            Not provided.
     ******************
     */
    public static void main(String args[]) {
        System.out.println("Starting AdaBoosting...");
        Booster tempBooster = new Booster("D:/iris.arff");
        // Booster tempBooster = new Booster("src/data/smalliris.arff");

        tempBooster.setNumBaseClassifiers(100);
        tempBooster.train();

        System.out.println("The training accuracy is: " + tempBooster.computeTrainingAccuray());
        tempBooster.test();
    }// Of main
}// Of class Booster

运行结果：