Spark常用机器学习算法(scala+java)

最新推荐文章于 2024-04-23 00:39:17 发布
最新推荐文章于 2024-04-23 00:39:17 发布
阅读量327 收藏 1
点赞数
文章标签： 大数据 java 人工智能
原文链接：http://www.cnblogs.com/kingshine007/p/8082679.html
版权

kmeans

Scala程序

import org.apache.spark.{SparkConf, SparkContext}

import org.apache.spark.mllib.clustering.{KMeans, KMeansModel}

import org.apache.spark.mllib.linalg.Vectors

/**

  * Created by hui on 2017/11/21.

  * K-means算法

  */

object kmeans {

  def main(args:Array[String]): Unit ={

    val conf= new SparkConf().setAppName("kmeans").setMaster("local")

    val sc = new SparkContext(conf)

    val data=sc.textFile("data/mllib/kmeans_data.txt")

    val parsedData=data.map(s=>Vectors.dense(s.split(' ').map(_.toDouble))).cache()

    val numClusters=2

    val numIterations=20

    val clusters=KMeans.train(parsedData,numClusters,numIterations)

    val WSSSE=clusters.computeCost(parsedData)

    println("Within Set Sum of Squared Errors = " + WSSSE)

    clusters.save(sc,"my_kmeans")

    val sameModel=KMeansModel.load(sc,"my_kmeans")

  }

}

Java程序

import org.apache.spark.SparkConf;

import org.apache.spark.api.java.JavaSparkContext;

import org.apache.spark.api.java.JavaRDD;

import org.apache.spark.api.java.function.Function;

import org.apache.spark.mllib.clustering.KMeans;

import org.apache.spark.mllib.clustering.KMeansModel;

import org.apache.spark.mllib.linalg.Vector;

import org.apache.spark.mllib.linalg.Vectors;

// $example off$

public class JavaKMeansExample {

  public static void main(String[] args) {

 

    SparkConf conf = new SparkConf().setAppName("JavaKMeansExample").setMaster("local");

    JavaSparkContext jsc = new JavaSparkContext(conf);

    // $example on$

    // Load and parse data

    String path = "data/mllib/kmeans_data.txt";

    JavaRDD<String> data = jsc.textFile(path);

    JavaRDD<Vector> parsedData = data.map(

      new Function<String, Vector>() {

        public Vector call(String s) {

          String[] sarray = s.split(" ");

          double[] values = new double[sarray.length];

          for (int i = 0; i < sarray.length; i++) {

            values[i] = Double.parseDouble(sarray[i]);

          }

          return Vectors.dense(values);

        }

      }

    );

    parsedData.cache();

    // Cluster the data into two classes using KMeans

    int numClusters = 2;

    int numIterations = 20;

    KMeansModel clusters = KMeans.train(parsedData.rdd(), numClusters, numIterations);

    System.out.println("Cluster centers:");

    for (Vector center: clusters.clusterCenters()) {

      System.out.println(" " + center);

    }

    double cost = clusters.computeCost(parsedData.rdd());

    System.out.println("Cost: " + cost);

    // Evaluate clustering by computing Within Set Sum of Squared Errors

    double WSSSE = clusters.computeCost(parsedData.rdd());

    System.out.println("Within Set Sum of Squared Errors = " + WSSSE);

    // Save and load model

    clusters.save(jsc.sc(), "target/org/apache/spark/JavaKMeansExample/KMeansModel");

    KMeansModel sameModel = KMeansModel.load(jsc.sc(),

      "target/org/apache/spark/JavaKMeansExample/KMeansModel");

    // $example off$

    jsc.stop();

  }

}

运行结果

decisiontree

Scala程序

import org.apache.spark.mllib.tree.DecisionTree

import org.apache.spark.mllib.tree.model.DecisionTreeModel

import org.apache.spark.mllib.util.MLUtils

import org.apache.spark.SparkConf

import org.apache.spark.SparkContext

/**

  * Created by hui on 2017/11/21.

  * 使用树深为5的决策树进行分类

  */

object decisiontree {

  def main(args:Array[String]): Unit = {

    val conf = new SparkConf().setAppName("decisiontree").setMaster("local")

    val sc = new SparkContext(conf)

    val data = MLUtils.loadLibSVMFile(sc, "E:\\ideaProjects\\TestBook\\data\\mllib\\sample_libsvm_data.txt")

    val splits = data.randomSplit(Array(0.7, 0.3))

    val (trainingData, testData) = (splits(0), splits(1))

    val numClass = 2

    val categoricalFeaturesInfo = Map[Int, Int]()

    val impurity = "gini"

    val maxDepth = 5

    val maxBins = 32

    val model = DecisionTree.trainClassifier(trainingData, numClass, categoricalFeaturesInfo, impurity, maxDepth, maxBins)

    val labelAndPreds = testData.map { point =>

      val predicition = model.predict(point.features)

      (point.label, predicition)

    }

    val testErr = labelAndPreds.filter(r => r._1 != r._2).count.toDouble / testData.count()

    println("Test Error=" + testErr)

    println("Learn classification tree model:\n" + model.toDebugString)

    model.save(sc, "my_decisiontree")

    val sameModel = DecisionTreeModel.load(sc, "my_decisiontree")

  }

}

Java程序

import java.util.HashMap;

import java.util.Map;

import scala.Tuple2;

import org.apache.spark.SparkConf;

import org.apache.spark.api.java.JavaPairRDD;

import org.apache.spark.api.java.JavaRDD;

import org.apache.spark.api.java.JavaSparkContext;

import org.apache.spark.api.java.function.Function;

import org.apache.spark.api.java.function.PairFunction;

import org.apache.spark.mllib.regression.LabeledPoint;

import org.apache.spark.mllib.tree.DecisionTree;

import org.apache.spark.mllib.tree.model.DecisionTreeModel;

import org.apache.spark.mllib.util.MLUtils;

// $example off$

class JavaDecisionTreeClassificationExample {

  public static void main(String[] args) {

    // $example on$

    SparkConf sparkConf = new SparkConf().setAppName("JavaDecisionTreeClassificationExample").setMaster("local");

    JavaSparkContext jsc = new JavaSparkContext(sparkConf);

    // Load and parse the data file.

    String datapath = "data/mllib/sample_libsvm_data.txt";

    JavaRDD<LabeledPoint> data = MLUtils.loadLibSVMFile(jsc.sc(), datapath).toJavaRDD();

    // Split the data into training and test sets (30% held out for testing)

    JavaRDD<LabeledPoint>[] splits = data.randomSplit(new double[]{0.7, 0.3});

    JavaRDD<LabeledPoint> trainingData = splits[0];

    JavaRDD<LabeledPoint> testData = splits[1];

    // Set parameters.

    //  Empty categoricalFeaturesInfo indicates all features are continuous.

    Integer numClasses = 2;

    Map<Integer, Integer> categoricalFeaturesInfo = new HashMap<>();

    String impurity = "gini";

    Integer maxDepth = 5;

    Integer maxBins = 32;

    // Train a DecisionTree model for classification.

    final DecisionTreeModel model = DecisionTree.trainClassifier(trainingData, numClasses,

      categoricalFeaturesInfo, impurity, maxDepth, maxBins);

    // Evaluate model on test instances and compute test error

    JavaPairRDD<Double, Double> predictionAndLabel =

      testData.mapToPair(new PairFunction<LabeledPoint, Double, Double>() {

        @Override

        public Tuple2<Double, Double> call(LabeledPoint p) {

          return new Tuple2<>(model.predict(p.features()), p.label());

        }

      });

    Double testErr =

      1.0 * predictionAndLabel.filter(new Function<Tuple2<Double, Double>, Boolean>() {

        @Override

        public Boolean call(Tuple2<Double, Double> pl) {

          return !pl._1().equals(pl._2());

        }

      }).count() / testData.count();

    System.out.println("Test Error: " + testErr);

    System.out.println("Learned classification tree model:\n" + model.toDebugString());

    // Save and load model

    model.save(jsc.sc(), "target/tmp/myDecisionTreeClassificationModel");

    DecisionTreeModel sameModel = DecisionTreeModel

      .load(jsc.sc(), "target/tmp/myDecisionTreeClassificationModel");

    // $example off$

  }

}

运行结果

randforest_classifier

Scala程序

import org.apache.spark.SparkConf

import org.apache.spark.SparkContext

import org.apache.spark.mllib.util.MLUtils

import org.apache.spark.mllib.tree.RandomForest

import org.apache.spark.mllib.tree.model.RandomForestModel

/**

  * Created by hui on 2017/11/21.

  * 使用随机森林进行分类

  */

object randforest_classifier {

  def main(args:Array[String]): Unit = {

    val conf = new SparkConf().setAppName("randforest_classifier").setMaster("local")

    val sc = new SparkContext(conf)

    val data = MLUtils.loadLibSVMFile(sc, "data/mllib/sample_libsvm_data.txt")

    val splits = data.randomSplit(Array(0.7, 0.3))

    val (trainingData, testData) = (splits(0), splits(1))

    val numClass = 2

    val categoricalFeaturesInfo = Map[Int, Int]()

    val numTrees = 3

    val featureSubsetStrategy = "auto"

    val impurity = "gini"

    val maxDepth = 4

    val maxBins = 32

    val model = RandomForest.trainClassifier(trainingData, numClass, categoricalFeaturesInfo, numTrees, featureSubsetStrategy, impurity, maxDepth, maxBins)

    val labelAndPreds = testData.map { point =>

      val prediction = model.predict(point.features)

      (point.label, prediction)

    }

    val testErr = labelAndPreds.filter(r => r._1 != r._2).count.toDouble / testData.count()

    println("Test Error=" + testErr)

    println("Learned classification forest model:\n" + model.toDebugString)

    model.save(sc, "myModelPath")

    val sameModel = RandomForestModel.load(sc, "myModelPath")

  }

}

Java程序

import java.util.HashMap;

import scala.Tuple2;

import org.apache.spark.SparkConf;

import org.apache.spark.api.java.JavaPairRDD;

import org.apache.spark.api.java.JavaRDD;

import org.apache.spark.api.java.JavaSparkContext;

import org.apache.spark.api.java.function.Function;

import org.apache.spark.api.java.function.PairFunction;

import org.apache.spark.mllib.regression.LabeledPoint;

import org.apache.spark.mllib.tree.RandomForest;

import org.apache.spark.mllib.tree.model.RandomForestModel;

import org.apache.spark.mllib.util.MLUtils;

// $example off$

public class JavaRandomForestClassificationExample {

  public static void main(String[] args) {

    // $example on$

    SparkConf sparkConf = new SparkConf().setAppName("JavaRandomForestClassificationExample").setMaster("local");

    JavaSparkContext jsc = new JavaSparkContext(sparkConf);

    // Load and parse the data file.

    String datapath = "data/mllib/sample_libsvm_data.txt";

    JavaRDD<LabeledPoint> data = MLUtils.loadLibSVMFile(jsc.sc(), datapath).toJavaRDD();

    // Split the data into training and test sets (30% held out for testing)

    JavaRDD<LabeledPoint>[] splits = data.randomSplit(new double[]{0.7, 0.3});

    JavaRDD<LabeledPoint> trainingData = splits[0];

    JavaRDD<LabeledPoint> testData = splits[1];

    // Train a RandomForest model.

    // Empty categoricalFeaturesInfo indicates all features are continuous.

    Integer numClasses = 2;

    HashMap<Integer, Integer> categoricalFeaturesInfo = new HashMap<>();

    Integer numTrees = 3; // Use more in practice.

    String featureSubsetStrategy = "auto"; // Let the algorithm choose.

    String impurity = "gini";

    Integer maxDepth = 5;

    Integer maxBins = 32;

    Integer seed = 12345;

    final RandomForestModel model = RandomForest.trainClassifier(trainingData, numClasses,

      categoricalFeaturesInfo, numTrees, featureSubsetStrategy, impurity, maxDepth, maxBins,

      seed);

    // Evaluate model on test instances and compute test error

    JavaPairRDD<Double, Double> predictionAndLabel =

      testData.mapToPair(new PairFunction<LabeledPoint, Double, Double>() {

        @Override

        public Tuple2<Double, Double> call(LabeledPoint p) {

          return new Tuple2<>(model.predict(p.features()), p.label());

        }

      });

    Double testErr =

      1.0 * predictionAndLabel.filter(new Function<Tuple2<Double, Double>, Boolean>() {

        @Override

        public Boolean call(Tuple2<Double, Double> pl) {

          return !pl._1().equals(pl._2());

        }

      }).count() / testData.count();

    System.out.println("Test Error: " + testErr);

    System.out.println("Learned classification forest model:\n" + model.toDebugString());

    // Save and load model

    model.save(jsc.sc(), "target/tmp/myRandomForestClassificationModel");

    RandomForestModel sameModel = RandomForestModel.load(jsc.sc(),

      "target/tmp/myRandomForestClassificationModel");

    // $example off$

    jsc.stop();

  }

}

运行结果

randforest_regressor

Scala程序

import org.apache.spark.{SparkConf, SparkContext}

import org.apache.spark.mllib.util.MLUtils

import org.apache.spark.SparkConf

import org.apache.spark.SparkContext

import org.apache.spark.mllib.util.MLUtils

import org.apache.spark.mllib.tree.RandomForest

import org.apache.spark.mllib.tree.model.RandomForestModel

/**

  * Created by hui on 2017/11/21.

  * 使用随机森林进行回归

  */

object randforest_regressor {

  def main(args:Array[String]): Unit = {

    val conf = new SparkConf().setAppName("randforest_regressor").setMaster("local")

    val sc = new SparkContext(conf)

    val data = MLUtils.loadLibSVMFile(sc, "data/mllib/sample_libsvm_data.txt")

    val splits = data.randomSplit(Array(0.7, 0.3))

    val (trainingData, testData) = (splits(0), splits(1))

    val numClass = 2

    val categoricalFeaturesInfo = Map[Int, Int]()

    val numTrees = 3

    val featureSubsetStrategy = "auto"

    val impurity = "variance"

    val maxDepth = 4

    val maxBins = 32

    val model = RandomForest.trainRegressor(trainingData, categoricalFeaturesInfo, numTrees, featureSubsetStrategy, impurity, maxDepth, maxBins)

    val labelAndPredictions = testData.map { point =>

      val prediction = model.predict(point.features)

      (point.label, prediction)

    }

    val testMSE = labelAndPredictions.map { case (v, p) => math.pow((v - p), 2) }.mean()

    println("Test Mean Squared Error=" + testMSE)

    println("Learned regression forest model:\n" + model.toDebugString)

    model.save(sc, "myModelPath")

    val sameModel = RandomForestModel.load(sc, "myModelPath")

  }

}

Java程序

import java.util.HashMap;

import java.util.Map;

import scala.Tuple2;

import org.apache.spark.api.java.function.Function2;

import org.apache.spark.api.java.JavaPairRDD;

import org.apache.spark.api.java.JavaRDD;

import org.apache.spark.api.java.JavaSparkContext;

import org.apache.spark.api.java.function.Function;

import org.apache.spark.api.java.function.PairFunction;

import org.apache.spark.mllib.regression.LabeledPoint;

import org.apache.spark.mllib.tree.RandomForest;

import org.apache.spark.mllib.tree.model.RandomForestModel;

import org.apache.spark.mllib.util.MLUtils;

import org.apache.spark.SparkConf;

// $example off$

public class JavaRandomForestRegressionExample {

  public static void main(String[] args) {

    // $example on$

    SparkConf sparkConf = new SparkConf().setAppName("JavaRandomForestRegressionExample").setMaster("local");

    JavaSparkContext jsc = new JavaSparkContext(sparkConf);

    // Load and parse the data file.

    String datapath = "data/mllib/sample_libsvm_data.txt";

    JavaRDD<LabeledPoint> data = MLUtils.loadLibSVMFile(jsc.sc(), datapath).toJavaRDD();

    // Split the data into training and test sets (30% held out for testing)

    JavaRDD<LabeledPoint>[] splits = data.randomSplit(new double[]{0.7, 0.3});

    JavaRDD<LabeledPoint> trainingData = splits[0];

    JavaRDD<LabeledPoint> testData = splits[1];

    // Set parameters.

    // Empty categoricalFeaturesInfo indicates all features are continuous.

    Map<Integer, Integer> categoricalFeaturesInfo = new HashMap<>();

    Integer numTrees = 3; // Use more in practice.

    String featureSubsetStrategy = "auto"; // Let the algorithm choose.

    String impurity = "variance";

    Integer maxDepth = 4;

    Integer maxBins = 32;

    Integer seed = 12345;

    // Train a RandomForest model.

    final RandomForestModel model = RandomForest.trainRegressor(trainingData,

      categoricalFeaturesInfo, numTrees, featureSubsetStrategy, impurity, maxDepth, maxBins, seed);

    // Evaluate model on test instances and compute test error

    JavaPairRDD<Double, Double> predictionAndLabel =

      testData.mapToPair(new PairFunction<LabeledPoint, Double, Double>() {

        @Override

        public Tuple2<Double, Double> call(LabeledPoint p) {

          return new Tuple2<>(model.predict(p.features()), p.label());

        }

      });

    Double testMSE =

      predictionAndLabel.map(new Function<Tuple2<Double, Double>, Double>() {

        @Override

        public Double call(Tuple2<Double, Double> pl) {

          Double diff = pl._1() - pl._2();

          return diff * diff;

        }

      }).reduce(new Function2<Double, Double, Double>() {

        @Override

        public Double call(Double a, Double b) {

          return a + b;

        }

      }) / testData.count();

    System.out.println("Test Mean Squared Error: " + testMSE);

    System.out.println("Learned regression forest model:\n" + model.toDebugString());

    // Save and load model

    model.save(jsc.sc(), "target/tmp/myRandomForestRegressionModel");

    RandomForestModel sameModel = RandomForestModel.load(jsc.sc(),

      "target/tmp/myRandomForestRegressionModel");

    // $example off$

    jsc.stop();

  }

}

运行结果

svm

Scala程序

import org.apache.spark.{SparkConf, SparkContext}

import org.apache.spark.mllib.classification.{SVMModel, SVMWithSGD}

import org.apache.spark.mllib.evaluation.BinaryClassificationMetrics

import org.apache.spark.mllib.util.MLUtils

/**

  * Created by hui on 2017/11/21.

  * 支持向量机分类

  */

object svm {

  def main(args:Array[String]): Unit = {

    val conf = new SparkConf().setAppName("svm").setMaster("local")

    val sc = new SparkContext(conf)

    val data=MLUtils.loadLibSVMFile(sc,"data/mllib/sample_libsvm_data.txt")

    val splits=data.randomSplit(Array(0.6,0.4),seed=11L)

    val training=splits(0).cache()

    val test=splits(1)

    val numIterations=100

    val model=SVMWithSGD.train(training,numIterations)

    model.clearThreshold()

    val scoreAndLabels=test.map{point=>

      val score=model.predict(point.features)

      (score,point.label)

    }

    val metrics=new BinaryClassificationMetrics(scoreAndLabels)

    val auROC=metrics.areaUnderROC()

    println("Area under ROC="+ auROC)

    model.save(sc,"my_svm")

    val sameModel=SVMModel.load(sc,"my_svm")

  }

}

Java程序

import org.apache.spark.SparkConf;

import org.apache.spark.SparkContext;

// $example on$

import scala.Tuple2;

import org.apache.spark.api.java.JavaRDD;

import org.apache.spark.api.java.function.Function;

import org.apache.spark.mllib.classification.SVMModel;

import org.apache.spark.mllib.classification.SVMWithSGD;

import org.apache.spark.mllib.evaluation.BinaryClassificationMetrics;

import org.apache.spark.mllib.regression.LabeledPoint;

import org.apache.spark.mllib.util.MLUtils;

// $example off$

/**

 * Example for SVMWithSGD.

 */

public class JavaSVMWithSGDExample {

  public static void main(String[] args) {

    SparkConf conf = new SparkConf().setAppName("JavaSVMWithSGDExample").setMaster("local");

    SparkContext sc = new SparkContext(conf);

    // $example on$

    String path = "data/mllib/sample_libsvm_data.txt";

    JavaRDD<LabeledPoint> data = MLUtils.loadLibSVMFile(sc, path).toJavaRDD();

    // Split initial RDD into two... [60% training data, 40% testing data].

    JavaRDD<LabeledPoint> training = data.sample(false, 0.6, 11L);

    training.cache();

    JavaRDD<LabeledPoint> test = data.subtract(training);

    // Run training algorithm to build the model.

    int numIterations = 100;

    final SVMModel model = SVMWithSGD.train(training.rdd(), numIterations);

    // Clear the default threshold.

    model.clearThreshold();

    // Compute raw scores on the test set.

    JavaRDD<Tuple2<Object, Object>> scoreAndLabels = test.map(

      new Function<LabeledPoint, Tuple2<Object, Object>>() {

        public Tuple2<Object, Object> call(LabeledPoint p) {

          Double score = model.predict(p.features());

          return new Tuple2<Object, Object>(score, p.label());

        }

      }

    );

    // Get evaluation metrics.

    BinaryClassificationMetrics metrics =

      new BinaryClassificationMetrics(JavaRDD.toRDD(scoreAndLabels));

    double auROC = metrics.areaUnderROC();

    System.out.println("Area under ROC = " + auROC);

    // Save and load model

    model.save(sc, "target/tmp/javaSVMWithSGDModel");

    SVMModel sameModel = SVMModel.load(sc, "target/tmp/javaSVMWithSGDModel");

    // $example off$

    sc.stop();

  }

}

运行结果

转载于:https://www.cnblogs.com/kingshine007/p/8082679.html

weixin_34269583
关注
  • 0
    点赞
  • 踩
  • 1
    收藏
    觉得还不错? 一键收藏
  • 0
    评论
spark机器学习算法研究
从地图看世界的专栏
11-18 2803
spark提供了一个机器学习库，spark ml包，可以在spark中直接引入使用 import org.apache.spark.ml.clustering.{KMeans,KMeansModel} import org.apache.spark.ml.linalg.Vectors 1. 常用聚类算法 k-means||算法 GMM（高斯混合模型） PIC（快速迭代聚类） LDA（隐式狄利克雷分布) 二分k-means算法 流式k-means算法 2. 常用分类算法 决策树 朴素贝叶斯 3. 常
基于spark的机器学习算法实现
06-29
本项目主要是对一些常用的算法的实现，以及基于spark的机器学习算法实现。 目前以实现的功能有： * 数据预处理 * 基于spark的LR逻辑回归分类 * 基于spark的DT决策树分类
scala dataFrame 一列向量（部分为稀疏向量）转化为dense vector（rg.apache.spark.ml.linalg.SparseVector cannot be cast ）
yangnianjinxin的博客
06-14 311
scala在数据输入模型前，用VectorAssembler函数将特征转化为向量，但是扔进模型fit时报错： org.apache.spark.ml.linalg.SparseVector cannot be cast to org.apache.spark.ml.linalg.DenseVector。）， 那么如何将dataFrame的一列VectorAssembler转化完成的feature列全部转化为稠密向量呢？尝试了很多方法，最后成功的做法很简单，用UDF转换，代码如下。
Spark机器学习--矩阵的定义——scala版本
北京小辉
11-06 583
目录 一、本地向量 二、含类标签的点 三、稀疏数据Sparse data 四、本地矩阵 五、分布式矩阵 5.1 面向行的分布式矩阵(RowMatrix) 5.2行索引矩阵(IndexedRowMatrix) 5.3三元组矩阵(CoordinateMatrix) 一、本地向量        本地向量的基类是 Vector,我们提供了两个实现 DenseVec...
【Spark】Spark的机器学习算法库——Spark MLilb
毕业于东北大学软件工程专业，现就职于字节跳动
01-03 2539
文章目录1 导入1.1 基本概念1.2 spark.mlib和spark.ml2 机器学习工作流(ML Pipelines)2.1 基本概念2.2 工作流的构建构建SparkSession对象引入要包含的包构建训练数据集定义Pipeline中的各个工作流阶段PipelineStage创建一个Pipeline构建测试数据预测 1 导入 1.1 基本概念 MLlib是Spark的机器学习（Machine Learning）库，旨在简化机器学习的工程实践工作，并方便扩展到更大规模。MLlib由一些通用的学习算法和
Spark机器学习算法
weixin_34163553的博客
03-29 1144
2019独角兽企业重金招聘Python工程师标准>>> ...
spark之常见的机器学习算法
霜叶的博客
10-05 2967
1、分类、回归、聚类 分类回归是一种重要的机器学习和数据挖掘技术。分类的目的是根据数据集的特点构造一个分类函数或分类模型(也常常称作分类器)，该模型能把未知类别的样本映射到给定类别中的一种技术。 即： 向量X=[x1,x2...xn]但标签C=[c1,c2...,ck]的映射F(W,X)=C 聚类是一种无监督学习的方法，将无标签数据聚类到不同的簇中 spark.ml支持的分类与回归算法 2、常见算法实例介绍 2.1、逻辑回归： 2.1.1、逻辑斯蒂分布 2.1.2、二项LR：
大数据笔记--Spark机器学习（第二篇）
我睡觉的时候不困
04-13 3290
一、梯度下降法 二、推荐系统模型 三、ALS算法与显示矩阵分解
[Spark机器学习]基于Spark 2.0 机器学习之推荐系统实现
热门推荐
图解AI
01-18 1万+
1 什么是ALS  ALS是交替最小二乘（alternating least squares）的简称。在机器学习中，ALS特指使用交替最小二乘求解的一个协同推荐算法。它通过观察到的所有用户给商品的打分，来推断每个用户的喜好并向用户推荐适合的商品。举个例子，我们看下面一个8*8的用户打分矩阵。   这个矩阵的每一行代表一个用户（u1,u2,…,u8）、每一列代表一个商品（v1,v2,…,v8）、用户的
Spark MLlib 机器学习算法（一）
sunqingok的博客
12-25 424
一 协同过滤算法 协同过滤（Collaborative filtering）算法是一种基于群体用户或者物品的典型推荐算法，主要有两种：一种是通过考察具有相同爱好的用户对相同物品的评分标准进行计算。一种是考察具有相同特质的物品从而推荐给选择了某件物品的用户。 协同过滤算法关键是计算相似度，主要有以下几种方法： 1、 基于欧几里得距离计算 公式： 主要从不同目标的绝对差异性考虑 2、 基于余弦角度计算 公式： 主要从方向趋势上考虑 3、 交替最小二乘法（ALS） Val ratings = data.map
Scala实现逻辑回归分类，Titanic
u013240812的专栏
10-31 1373
1.读取文件 import scala.io.Source object myfirst { //titanic,LR def main(args: Array[String]) { val data= Source.fromFile("D:\\IDEA\\_01\\train.csv") data.foreach(print) } } import or
spark机器学习算法实现
03-21
基于spark实现了logisticregression以及softmax算法，仅供参考 基于spark实现了logisticregression以及softmax算法，仅供参考
基于Scala实现的Spark算法实例+源代码+文档说明
03-30
2、本项目适合计算机相关专业(如计科、人工智能、通信工程、自动化、电子信息等)的在校学生、老师或者企业员工下载学习，也适合小白学习进阶，当然也可作为毕设项目、课程设计、作业、项目初期立项演示等。...
SparkML算法详解(关于DataFrame的API操作)--数据挖掘(Scala与Java版)
06-28
一.Machine Learning 二.完整代码示例 三.数据挖掘 四.机器学习过程中的一些问题 五.各种机器学习算法的总结 (共119页,代码部分以Spark的DateFrame为主)
scala-spark线性代数
weixin_42365868的博客
11-14 90
DenseVector比较大小（其实也不能叫比大小吧，而应该是根据条件筛选符合的元素） DenseVector a(1,2,3) 与 DenseVector b(1,4,1)比较大小： println(a :== b) # output: BitVector(0) println(a :> b) # output: BitVector(2) println(a :< b) # output: BitVector(1) println(a :<= b) # output: BitVector
spark 协方差计算
zhangxinyu0827的博客
12-27 1044
       val data = Array(       Vectors.dense(4.0, 2.0, 3.0),       Vectors.dense(5.0, 6.0, 1.0),       Vectors.dense(2.0, 4.0, 7.0),       Vectors.dense(3.0, 6.0, 5.0)     )               rdd是可以直接算协方...
scala-MLlib官方文档---spark.ml package--ML Pipelines+Collaborative Filtering+Frequent Pattern Mining
pt798633929的博客
01-20 1198
三、ML Pipeline 1、Main concepts in Pipelines DataFrame 2)Pipeline components (1)Transformers (2)Estimators (3)Properties of pipeline components 3)pipeline (1)How it works (2)Details 4)Parameters 5)ML p...
spark mllib和spark ml机器学习基础知识
caocry的博客
09-23 1275
spark机械学系基础知识,同时可以通过pyspark库用在python上，语法一样
ELK日志系统的搭建
最新发布
adminstate的博客
04-23 818
ELK日志系统搭建
spark的机器学习库
06-02
MLlib包括常见的机器学习算法和工具，如分类、回归、聚类、协同过滤、降维等。以下是MLlib的主要特点： 1. 分布式机器学习：MLlib支持Spark的分布式计算框架，可以处理大规模的数据集。 2. 丰富的机器学习算法：...

“相关推荐”对你有帮助么？

  • 非常没帮助
  • 没帮助
  • 一般
  • 有帮助
  • 非常有帮助
提交
评论
添加红包

请填写红包祝福语或标题

个

红包个数最小为10个

元

红包金额最低5元

当前余额3.43元 前往充值 >
需支付：10.00元
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付元
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明：

1.余额是钱包充值的虚拟货币，按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载，可以购买VIP、付费专栏及课程。

余额充值