Kmean聚类编写

spark Example GOGOGO!

kmean算法的聚类算是好理解的

依旧画图好理解

 

from __future__ import print_function

import sys

import numpy as np

from pyspark import SparkContext



def closestPoint(p, centers):
    bestIndex = 0
    closest = float("+inf")
    for i in range(len(centers)):
        tempDist = np.sum((p-centers[i]) ** 2)
        # 找出p最靠近那个点
        if tempDist < closest:
            closest = tempDist
            bestIndex = i
        return bestIndex



if __name__ == '__main__':
    if len(sys.argv) != 4:
        print("Usage: kmeans <file> <k> <convergeDist>", file=sys.stderr)
        exit(-1)
    
        print(
              """WARN: This is a naive implementation of KMeans Clustering and is given as an example! 
              Please refer to examples/src/main/python/mllib/kmeans.py for an example on how to use MLlib's 
              KMeans implementation.""", file=sys.stderr)
        
        sc = SparkContext(appName="PythonKMean")
        lines = sc.textFile(sys.argv[1])
        # 分割数据
        data = lines.map(lambda line: np.array([float(x) for x in line.split(' ')])).cache()
        K = int(sys.argv[2])
        # 平均距离
        convergeDist = float(sys.argv[3])
        # 该函数是将数据加载到内存中的,所以使用时必须注意,数据量不能太大
        # False 表示num不大于原本数据
        # 初始化 K 个默认 的点(最后不断的迭代,至最小距离)
        kPoints = data.takeSample(False, K, 1)
        tempDist = 1.0
        
        while tempDist > convergeDist:
            closest = data.map(
                               lambda p: (closestPoint(p, kPoints), (p, 1)))
            # 把每一个K的集合的距离相加 (sum,num)
            pointStats = closest.reduceByKey(
                                             lambda p1_c1,p2_c2:(p1_c1[0] + p2_c2[0],p1_c1[1] + p2_c2[1]))
            newPoints = pointStats.map(
            # st[0] => K
            # st[1][0]距离和
            # st[1][0] / st[1][1] 平均距离
            lambda st: (st[0], st[1][0] / st[1][1])).collect()
            
            tempDist = sum(np.sum((kPoints[iK] - p) ** 2) for (iK, p) in newPoints)
            
            # 保存聚类点
            for (iK,p) in newPoints:
                kPoints[iK] = p
        print("Final centers:" + str(kPoints))
        sc.stop()