Flink实现K-Means算法案例实战

package com.cnic.algorithm.flink.kmeans001;


import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.api.common.functions.RichMapFunction;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.functions.FunctionAnnotation;
import org.apache.flink.api.java.operators.IterativeDataSet;
import org.apache.flink.api.java.operators.MapOperator;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.api.java.utils.ParameterTool;
import org.apache.flink.configuration.Configuration;

import java.io.Serializable;
import java.util.Collection;
import java.util.LinkedList;
import java.util.List;

/***
 * 提供使用K-Means示例程序的默认数据集，如果没有为程序提供参数，则使用默认数据集
 */
public class KMeansData {
    public static class Point implements Serializable {
        // x坐标，y坐标
        public double x, y;

        public Point() {}

        public Point(double x, double y) {
            this.x = x;
            this.y = y;
        }
        // 点坐标的加法器
        public Point add(Point other) {
            x += other.x;
            y += other.y;
            return this;
        }

        // 点坐标的除法器
        public Point div(long val) {
            x /= val;
            y /= val;
            return this;
        }
        // 计算点之间的欧式距离
        public double euclideanDistance(Point other) {
            return Math.sqrt((x - other.x) * (x - other.x) + (y - other.y) * (y - other.y));
        }

        public void clear() {
            x = y = 0.0;
        }

        @Override
        public String toString() {
            return x + " " + y;
        }
    }

    /**
     * 质心类, 基于点坐标和id.
     */
    public static class Centroid extends Point {

        public int id;

        public Centroid() {}

        public Centroid(int id, double x, double y) {
            super(x, y);
            this.id = id;
        }

        public Centroid(int id, Point p) {
            super(p.x, p.y);
            this.id = id;
        }

        @Override
        public String toString() {
            return id + " " + super.toString();
        }
    }

    /***
     * 簇中心（质心）数据
     */
    public static final Object[][] CENTROIDS = new Object[][]{
            new Object[]{1, -31.85, -44.77},
            new Object[]{2, 35.16, 17.46},
            new Object[]{3, -5.16, 21.93},
            new Object[]{4, -24.06, 6.81}
    };

    /***
     * 输入点的数据集
     *
     */
    public static final Object[][] POINTS = new Object[][]{
            new Object[]{-14.22, -48.01},
            new Object[]{-22.78, 37.10},
            new Object[]{56.18, -42.99},
            new Object[]{35.04, 50.29},
            new Object[]{-9.53, -46.26},
            new Object[]{-34.35, 48.25},
            new Object[]{55.82, -57.49},
            new Object[]{21.03, 54.64},
            new Object[]{-13.63, -42.26},
            new Object[]{-36.57, 32.63},
            new Object[]{50.65, -52.40},
            new Object[]{24.48, 34.04},
            new Object[]{-2.69, -36.02},
            new Object[]{-38.80, 36.58},
            new Object[]{24.00, -53.74},
            new Object[]{32.41, 24.96},
            new Object[]{-4.32, -56.92},
            new Object[]{-22.68, 29.42},
            new Object[]{59.02, -39.56},
            new Object[]{24.47, 45.07},
            new Object[]{5.23, -41.20},
            new Object[]{-23.00, 38.15},
            new Object[]{44.55, -51.50},
            new Object[]{14.62, 59.06},
            new Object[]{7.41, -56.05},
            new Object[]{-26.63, 28.97},
            new Object[]{47.37, -44.72},
            new Object[]{29.07, 51.06},
            new Object[]{0.59, -31.89},
            new Object[]{-39.09, 20.78},
            new Object[]{42.97, -48.98},
            new Object[]{34.36, 49.08},
            new Object[]{-21.91, -49.01},
            new Object[]{-46.68, 46.04},
            new Object[]{48.52, -43.67},
            new Object[]{30.05, 49.25},
            new Object[]{4.03, -43.56},
            new Object[]{-37.85, 41.72},
            new Object[]{38.24, -48.32},
            new Object[]{20.83, 57.85}
    };

    /***
     * 得到默认的质心数据
     */
    public static DataSet<Centroid> getDefaultCentroidDataSet(ParameterTool params, ExecutionEnvironment env) {
        List<Centroid> centroidList = new LinkedList<Centroid>();
        // 遍历质心数据
        for (Object[] centroid : CENTROIDS) {
            // 将质心数据集添加到centroidList中
            centroidList.add(
                    new Centroid((Integer) centroid[0], (Double) centroid[1], (Double) centroid[2]));
        }
        return env.fromCollection(centroidList);
    }

    // 得到默认的点数据
    public static DataSet<Point> getDefaultPointDataSet(ParameterTool params, ExecutionEnvironment env){
        List<Point> pointList = new LinkedList<Point>();
        // 遍历点数据
        for (Object[] point : POINTS) {
            pointList.add(new Point((Double) point[0], (Double) point[1]));
        }
        return env.fromCollection(pointList);
    }


    // 确定数据点最近的集群中心
    @FunctionAnnotation.ForwardedFields("* -> 1")
    public static final class SelectNearestCenter extends RichMapFunction<Point,Tuple2<Integer,Point>>{

        private Collection<Centroid> centroids;
        // 将广播变量中的质心数据集读到集合中

        @Override
        public void open(Configuration parameters) throws Exception {
            this.centroids = getRuntimeContext().getBroadcastVariable("centroids");
        }

        public Tuple2<Integer, Point> map(Point p) throws Exception {
            double minDistance = Double.MAX_VALUE;
            int closestCentroidId = -1;
            // 遍历所有的簇中心
            for (Centroid centroid : centroids) {
                // 计算点和簇中心的欧式距离
                double distance = p.euclideanDistance(centroid);
                // 找到距离点最近的簇中心
                if (distance < minDistance) {
                    minDistance = distance;
                    closestCentroidId = centroid.id;
                }
            }
            //  输出一条新的记录，由簇中心id和point组成
            return new Tuple2<Integer, Point>(closestCentroidId,p);
        }
    }

    /**
     * 对Tuple2<Integer, Point> 进行计数
     *
     */

    /** 对 Tuple2<Integer, Point>进行计数 */
    @FunctionAnnotation.ForwardedFields("f0;f1")
    public static final class CountAppender implements MapFunction<Tuple2<Integer, Point>, Tuple3<Integer, Point, Long>> {

        @Override
        public Tuple3<Integer, Point, Long> map(Tuple2<Integer, Point> t) {
            // 对簇内点进行计数
            return new Tuple3<Integer, Point, Long>(t.f0, t.f1, 1L);
        }
    }

    /***
     *  对簇内点计数以及对簇内点的坐标进行累加
     */
    /** 对簇内点计数以及对簇内点的坐标进行累加 */
    @FunctionAnnotation.ForwardedFields("0")
    public static final class CentroidAccumulator implements ReduceFunction<Tuple3<Integer, Point, Long>> {

        @Override
        public Tuple3<Integer, Point, Long> reduce(Tuple3<Integer, Point, Long> val1, Tuple3<Integer, Point, Long> val2) {
            // 这一步逻辑很关键，对簇内点坐标累计，然后对簇内元素个数计数。
            return new Tuple3<Integer, Point, Long>(val1.f0, val1.f1.add(val2.f1), val1.f2 + val2.f2);
        }
    }

    /** 从簇内点的个数和这些点的坐标和计算出新的簇中心*/
    @FunctionAnnotation.ForwardedFields("0->id")
    public static final class CentroidAverager implements MapFunction<Tuple3<Integer, Point, Long>, Centroid> {

        @Override
        public Centroid map(Tuple3<Integer, Point, Long> value) {
            // 坐标和/簇内点个数作为新的簇中心
            return new Centroid(value.f0, value.f1.div(value.f2));
        }
    }


    public static void main(String[] args) throws Exception {
        // 1.解析命令行参数
        final ParameterTool params = ParameterTool.fromArgs(args);
        // 2.构建执行环境
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        // 3.使参数在web界面中可用
        env.getConfig().setGlobalJobParameters(params);
        // 4.得到输入数据：从提供的路径读取点和质心，或返回默认数据
        DataSet<Point> points = getDefaultPointDataSet(params, env);
        DataSet<Centroid> centroids = getDefaultCentroidDataSet(params, env);
        // 5.为K-Means算法设置批量迭代次数
        IterativeDataSet<Centroid> loop = centroids.iterate(params.getInt("iteration", 10));
        // 6.K-Means算法计算过程
        DataSet<Centroid> newCentroids = points
                // 6.1. 计算每个点距离最近的质心
                .map(new SelectNearestCenter()).withBroadcastSet(loop, "centroids")
                // 6.2. 每个簇内的所有点坐标求和
                .map(new CountAppender())
                .groupBy(0).reduce(new CentroidAccumulator())
                // 6.3. 根据点计数和坐标和计算新的质心
                .map(new CentroidAverager());
        // 7. 将新的质心数据反馈到下一个迭代中
        DataSet<Centroid> finalCentroids = loop.closeWith(newCentroids);
        // 8. 将点归宿给最终的簇
        // 8. 将点归宿给最终的簇
        DataSet<Tuple2<Integer, Point>> clusteredPoints = points
                .map(new SelectNearestCenter()).withBroadcastSet(finalCentroids, "centroids");

        // 9. 指定输出结果路径和执行
        if (params.has("output")) {
            clusteredPoints.writeAsCsv(params.get("output"), "\n", "data/output");
            env.execute("KMeans Example");
        } else {
            System.out.println("Printing result to stdout. Use --output to specify output path.");
            clusteredPoints.print();
        }

    }


}