1. build a sh(sparkrun.sh) scripts as follows;
#!/usr/bin/env bash
export SPARK_MEM=3g
CP=$CLASSPATH:/opt/spark-0.9.0-incubating-bin-hadoop2/conf:/opt/spark-0.9.0-incubating-bin-hadoop2/assembly/target/scala-2.10/spark-assembly_2.10-0.9.0-incubating-hadoop2.2.0.jar:./*
exec java -cp "$CP" "$@"
you can run as: ./sparkrun.sh [MainClass]
2. build a spark demo, package it to a jar file, & upload it to ./*, can use java or scala.
the spark application depend jar file as follows:
spark-mllib_2.10-0.9.0-incubating.jar
spark-streaming_2.10-0.9.0-incubating.jar
spark-graphx_2.10-0.9.0-incubating.jar
spark-assembly_2.10-0.9.0-incubating-hadoop2.2.0.jar
example 1 use java: a machine learning method Kmeans using spark.
import java.io.Serializable;
import java.util.List;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.broadcast.Broadcast;
import scala.Tuple2;
public class MyKMeans2 {
/**
* @param args
*/
public static void main(String[] args) {
int k = 3;
String[] arr = new String[2];
arr[0] = "spark://masterIP:7077";
arr[1] = "hdfs://namenodeIP:9000/test/kmeans_data.txt";
JavaSparkContext ctx = new JavaSparkContext(arr[0], "MyKeans",
System.getenv("SPARK_HOME"),
JavaSparkContext.jarOfClass(MyKMeans2.class));
JavaRDD<String> data = ctx.textFile(arr[1]);
int iter = 20;
double threshold = 0.0001;
List<String> ls = data.takeSample(false, k, 1);
System.out.println("LS:=" + ls.size());
Point[] curCenterPoint;
curCenterPoint = new Point[k];
for (int i = 0; i < k; i++) {
curCenterPoint[i] = new Point();
String str[] = ls.get(i).split(" ");
curCenterPoint[i].setX1(Double.parseDouble(str[0]));
curCenterPoint[i].setX2(Double.parseDouble(str[1]));
curCenterPoint[i].setX3(Double.parseDouble(str[2]));
}
JavaRDD<Point> data1=null;
JavaPairRDD<Integer, String> data2;
Broadcast<Point[]> bp = ctx.broadcast(curCenterPoint);
for (int i = 0; i < iter; i++) {
data1 = data.map(new ComputerClass(curCenterPoint, k));
data2 = data1.map(new PairFunction<Point, Integer, String>() {
@Override
public Tuple2<Integer, String> call(Point p) throws Exception {
int k = p.getType();
String v = "" + p.getX1() + " " + p.getX2() + " "
+ p.getX3();
Tuple2<Integer, String> t = new Tuple2<Integer, String>(k,
v);
return t;
}
});
JavaPairRDD<Integer, String> data3 = data2
.reduceByKey(new Function2<String, String, String>() {
@Override
public String call(String arg0, String arg1)
throws Exception {
String strs1[] = arg0.split(" ");
String strs2[] = arg1.split(" ");
double x1 = (Double.parseDouble(strs1[0]) + Double
.parseDouble(strs2[0])) / 2;
double x2 = (Double.parseDouble(strs1[1]) + Double
.parseDouble(strs2[1])) / 2;
double x3 = (Double.parseDouble(strs1[2]) + Double
.parseDouble(strs2[2])) / 2;
return x1 + " " + x2 + " " + x3;
}
});
Point[] NewCenter = new Point[k];
List<Tuple2<Integer, String>> ls3 = data3.collect();
for (int j = 0; j < k; j++) {
NewCenter[j] = new Point();
Tuple2<Integer, String> t = ls3.get(j);
NewCenter[j].setType(t._1);
String strs[] = t._2.split(" ");
NewCenter[j].setX1(Double.parseDouble(strs[0]));
NewCenter[j].setX2(Double.parseDouble(strs[1]));
NewCenter[j].setX3(Double.parseDouble(strs[2]));
}
double ms = computerCenterDistance(curCenterPoint, NewCenter, k);
if (ms < threshold) {
break;
}
data = data1.map(new Function<Point, String>() {
@Override
public String call(Point p) throws Exception {
return p.getX1() + " " + p.getX2() + " " + p.getX3();
}
});
System.out.println("ite:=" + i);
curCenterPoint = NewCenter;
}
System.out.println("classfy point====================");
List<Point> ls1 = data1.collect();
for (Point p : ls1) {
System.out.println(p.getType() + " " + p.getX1() + " "
+ p.getX2() + " " + p.getX3());
}
System.out.println("center size:="+curCenterPoint.length);
for (int i = 0; i < k; i++) {
Integer type = curCenterPoint[i].getType();
String x1 = curCenterPoint[i].getX1() + "";
String x2 = curCenterPoint[i].getX2() + "";
String x3 = curCenterPoint[i].getX3() + "";
System.out.println(type + " " + x1 + " " + x2 + " " + x3);
}
}
static double computerPointDistance(Point p1, Point p2) {
double ret = 0;
double x1 = p1.getX1();
double x2 = p1.getX2();
double x3 = p1.getX3();
double y1 = p2.getX1();
double y2 = p2.getX2();
double y3 = p2.getX3();
ret = (Math.pow((x1 - y1), 2) + Math.pow((x2 - y2), 2) + Math.pow(
(x3 - y3), 2));
return ret;
}
static double computerCenterDistance(Point[] oldCenter, Point[] newCenter,
int k) {
double ret = 0;
for (int i = 0; i < k; i++) {
double ox1 = oldCenter[i].getX1();
double ox2 = oldCenter[i].getX2();
double ox3 = oldCenter[i].getX3();
double nx1 = newCenter[i].getX1();
double nx2 = newCenter[i].getX2();
double nx3 = newCenter[i].getX3();
ret += (Math.pow((nx1 - ox1), 2) + Math.pow((nx2 - ox2), 2) + Math
.pow((nx3 - ox3), 2));
}
return ret;
}
static class ComputerClass extends Function<String, Point> {
private Point[] centerPoints;
private int k;
public ComputerClass(Point[] points, int k) {
this.centerPoints = points;
this.k = k;
}
@Override
public Point call(String line) throws Exception {
Point p = new Point();
String[] strs = line.split(" ");
p.setX1(Double.parseDouble(strs[0]));
p.setX2(Double.parseDouble(strs[1]));
p.setX3(Double.parseDouble(strs[2]));
int type1 = 0;
double D = 99999;
for (int j = 0; j < k; j++) {
double temp1 = computerPointDistance(p, centerPoints[j]);
if (temp1 < D) {
D = temp1;
type1 = j;
}
}
p.setType(type1);
return p;
}
}
static class Point implements Serializable {
private static final long serialVersionUID = 1L;
private int type;
private double x1;
private double x2;
private double x3;
public int getType() {
return type;
}
public void setType(int type) {
this.type = type;
}
public double getX1() {
return x1;
}
public void setX1(double x1) {
this.x1 = x1;
}
public double getX2() {
return x2;
}
public void setX2(double x2) {
this.x2 = x2;
}
public double getX3() {
return x3;
}
public void setX3(double x3) {
this.x3 = x3;
}
}
}
the testing data as follows:
0.0 0.0 0.0
0.1 0.1 0.1
0.2 0.2 0.2
9.0 9.0 9.0
9.1 9.1 9.1
9.2 9.2 9.2
2 2 2
2.1 2.1 2.1
2.3 2.4 2.6
example 2: spark code use scala
import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import org.apache.spark.serializer.KryoRegistrator
import com.esotericsoftware.kryo.Kryo
import org.apache.spark.rdd._
import org.apache.spark.SparkContext._
import org.apache.spark._
import org.apache.spark.api.java._
class MyRegistrator extends KryoRegistrator {
override def registerClasses(kryo: Kryo) {
}
}
object SortTest {
def main(args: Array[String]): Unit = {
val arr = new Array[String](2)
arr(0) = "spark://masterIP:7077"
arr(1) = "hdfs://nameNodeIP:9000/test/" + args(0)
val conf = new SparkConf().setMaster(arr(0)).setAppName("sorttest")
conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
conf.set("spark.kryo.registrator", "demo.MyRegistrator")
conf.set("spark.storage.memoryFraction", "0.5")
conf.setSparkHome(System.getenv("SPARK_HOME"))
conf.setJars(SparkContext.jarOfClass(this.getClass()))
val sc = new SparkContext(conf);
val dataset1:RDD[(Long, String)] = sc.textFile(arr(1), 1000).map(line => (line.split(',')(0).toLong, line))
val datasets=dataset1.sortByKey(false, 1000);
val arrdata2:Array[(Long, String)] =datasets.take(100);
for(i<-0 to arrdata2.length-1)
{
println(i+" "+arrdata2(i)._1,arrdata2(i)._2)
}
println(arrdata2.length)
}
}