参考地址：
1、http://spark.apache.org/docs/latest/quick-start.html
2、https://github.com/apache/spark/tree/v2.2.0
快速开始
与Spark Shell的交互式分析
基本
./bin/pyspark # 启动交互模式 >>> textFile = spark.read.text("README.md") # 读取文件 返回DataFrame >>> textFile.count() # Number of rows in this DataFrame 行数 >>> textFile.first() # First row in this DataFrame 第一行 >>> linesWithSpark = textFile.filter(textFile.value.contains("Spark")) # 按条件过滤 >>> textFile.filter(textFile.value.contains("Spark")).count() # How many lines contain "Spark"?
更多关于数据集操作
>>> from pyspark.sql.functions import * >>> textFile.select(size(split(textFile.value, "\s+")).name("numWords")).agg(max(col("numWords"))).collect() # [Row(max(numWords)=15)] >>> wordCounts = textFile.select(explode(split(textFile.value, "\s+")).as("word")).groupBy("word").count() >>> wordCounts.collect() [Row(word=u'online', count=1), Row(word=u'graphs', count=1), ...]
高速缓存
>>> linesWithSpark.cache() >>> linesWithSpark.count() 15 >>> linesWithSpark.count() 15
自包含应用程序
"""SimpleApp.py""" from pyspark.sql import SparkSession logFile = "README.md" # Should be some file on your system spark = SparkSession.builder \ .master("local") \ .appName("Word Count") \ .config("spark.some.config.option", "some-value") \ .getOrCreate()logData = spark.read.text(logFile).cache()# spark = SparkSession.builder.appName("First App").master(master).getOrCreate()
numAs = logData.filter(logData.value.contains('a')).count()
numBs = logData.filter(logData.value.contains('b')).count()
print("Lines with a: %i, lines with b: %i" % (numAs, numBs))
spark.stop()
-------------------------------------------------------
# Use spark-submit to run your application
$ YOUR_SPARK_HOME/bin/spark-submit \
--master local[4] \
SimpleApp.py
...Lines with a: 46, Lines with b: 23
spark自带的例子
更多例子参考路径：spark/examples/src/main/python/
pi.py
from __future__ import print_function import sys from random import random from operator import add from pyspark.sql import SparkSession if __name__ == "__main__": """ Usage: pi [partitions] """ spark = SparkSession\ .builder\ .appName("PythonPi")\ .getOrCreate() partitions = int(sys.argv[1]) if len(sys.argv) > 1 else 2 n = 100000 * partitions def f(_): x = random() * 2 - 1 y = random() * 2 - 1 return 1 if x ** 2 + y ** 2 < 1 else 0 count = spark.sparkContext.parallelize(range(1, n + 1), partitions).map(f).reduce(add) print("Pi is roughly %f" % (4.0 * count / n)) spark.stop()
als.py
""" This is an example implementation of ALS for learning how to use Spark. Please refer to pyspark.ml.recommendation.ALS for more conventional use. This example requires numpy (http://www.numpy.org/) """ from __future__ import print_function import sys import numpy as np from numpy.random import rand from numpy import matrix from pyspark.sql import SparkSession LAMBDA = 0.01 # regularization np.random.seed(42) def rmse(R, ms, us): diff = R - ms * us.T return np.sqrt(np.sum(np.power(diff, 2)) / (M * U)) def update(i, mat, ratings): uu = mat.shape[0] ff = mat.shape[1] XtX = mat.T * mat Xty = mat.T * ratings[i, :].T for j in range(ff): XtX[j, j] += LAMBDA * uu return np.linalg.solve(XtX, Xty) if __name__ == "__main__": """ Usage: als [M] [U] [F] [iterations] [partitions]" """ print("""WARN: This is a naive implementation of ALS and is given as an example. Please use pyspark.ml.recommendation.ALS for more conventional use.""", file=sys.stderr) spark = SparkSession\ .builder\ .appName("PythonALS")\ .getOrCreate() sc = spark.sparkContext M = int(sys.argv[1]) if len(sys.argv) > 1 else 100 U = int(sys.argv[2]) if len(sys.argv) > 2 else 500 F = int(sys.argv[3]) if len(sys.argv) > 3 else 10 ITERATIONS = int(sys.argv[4]) if len(sys.argv) > 4 else 5 partitions = int(sys.argv[5]) if len(sys.argv) > 5 else 2 print("Running ALS with M=%d, U=%d, F=%d, iters=%d, partitions=%d\n" % (M, U, F, ITERATIONS, partitions)) R = matrix(rand(M, F)) * matrix(rand(U, F).T) ms = matrix(rand(M, F)) us = matrix(rand(U, F)) Rb = sc.broadcast(R) msb = sc.broadcast(ms) usb = sc.broadcast(us) for i in range(ITERATIONS): ms = sc.parallelize(range(M), partitions) \ .map(lambda x: update(x, usb.value, Rb.value)) \ .collect() # collect() returns a list, so array ends up being # a 3-d array, we take the first 2 dims for the matrix ms = matrix(np.array(ms)[:, :, 0]) msb = sc.broadcast(ms) us = sc.parallelize(range(U), partitions) \ .map(lambda x: update(x, msb.value, Rb.value.T)) \ .collect() us = matrix(np.array(us)[:, :, 0]) usb = sc.broadcast(us) error = rmse(R, ms, us) print("Iteration %d:" % i) print("\nRMSE: %5.4f\n" % error) spark.stop()
kmeans.py
""" The K-means algorithm written from scratch against PySpark. In practice, one may prefer to use the KMeans algorithm in ML, as shown in examples/src/main/python/ml/kmeans_example.py. This example requires NumPy (http://www.numpy.org/). """ from __future__ import print_function import sys import numpy as np from pyspark.sql import SparkSession def parseVector(line): return np.array([float(x) for x in line.split(' ')]) def closestPoint(p, centers): bestIndex = 0 closest = float("+inf") for i in range(len(centers)): tempDist = np.sum((p - centers[i]) ** 2) if tempDist < closest: closest = tempDist bestIndex = i return bestIndex if __name__ == "__main__": if len(sys.argv) != 4: print("Usage: kmeans <file> <k> <convergeDist>", file=sys.stderr) exit(-1) print("""WARN: This is a naive implementation of KMeans Clustering and is given as an example! Please refer to examples/src/main/python/ml/kmeans_example.py for an example on how to use ML's KMeans implementation.""", file=sys.stderr) spark = SparkSession\ .builder\ .appName("PythonKMeans")\ .getOrCreate() lines = spark.read.text(sys.argv[1]).rdd.map(lambda r: r[0]) data = lines.map(parseVector).cache() K = int(sys.argv[2]) convergeDist = float(sys.argv[3]) kPoints = data.takeSample(False, K, 1) tempDist = 1.0 while tempDist > convergeDist: closest = data.map( lambda p: (closestPoint(p, kPoints), (p, 1))) pointStats = closest.reduceByKey( lambda p1_c1, p2_c2: (p1_c1[0] + p2_c2[0], p1_c1[1] + p2_c2[1])) newPoints = pointStats.map( lambda st: (st[0], st[1][0] / st[1][1])).collect() tempDist = sum(np.sum((kPoints[iK] - p) ** 2) for (iK, p) in newPoints) for (iK, p) in newPoints: kPoints[iK] = p print("Final centers: " + str(kPoints)) spark.stop()
pagerank.py
""" This is an example implementation of PageRank. For more conventional use, Please refer to PageRank implementation provided by graphx Example Usage: bin/spark-submit examples/src/main/python/pagerank.py data/mllib/pagerank_data.txt 10 """ from __future__ import print_function import re import sys from operator import add from pyspark.sql import SparkSession def computeContribs(urls, rank): """Calculates URL contributions to the rank of other URLs.""" num_urls = len(urls) for url in urls: yield (url, rank / num_urls) def parseNeighbors(urls): """Parses a urls pair string into urls pair.""" parts = re.split(r'\s+', urls) return parts[0], parts[1] if __name__ == "__main__": if len(sys.argv) != 3: print("Usage: pagerank <file> <iterations>", file=sys.stderr) exit(-1) print("WARN: This is a naive implementation of PageRank and is given as an example!\n" + "Please refer to PageRank implementation provided by graphx", file=sys.stderr) # Initialize the spark context. spark = SparkSession\ .builder\ .appName("PythonPageRank")\ .getOrCreate() # Loads in input file. It should be in format of: # URL neighbor URL # URL neighbor URL # URL neighbor URL # ... lines = spark.read.text(sys.argv[1]).rdd.map(lambda r: r[0]) # Loads all URLs from input file and initialize their neighbors. links = lines.map(lambda urls: parseNeighbors(urls)).distinct().groupByKey().cache() # Loads all URLs with other URL(s) link to from input file and initialize ranks of them to one. ranks = links.map(lambda url_neighbors: (url_neighbors[0], 1.0)) # Calculates and updates URL ranks continuously using PageRank algorithm. for iteration in range(int(sys.argv[2])): # Calculates URL contributions to the rank of other URLs. contribs = links.join(ranks).flatMap( lambda url_urls_rank: computeContribs(url_urls_rank[1][0], url_urls_rank[1][1])) # Re-calculates URL ranks based on neighbor contributions. ranks = contribs.reduceByKey(add).mapValues(lambda rank: rank * 0.85 + 0.15) # Collects all URL ranks and dump them to console. for (link, rank) in ranks.collect(): print("%s has rank: %s." % (link, rank)) spark.stop()
logistic_regression.py
""" A logistic regression implementation that uses NumPy (http://www.numpy.org) to act on batches of input data using efficient matrix operations. In practice, one may prefer to use the LogisticRegression algorithm in ML, as shown in examples/src/main/python/ml/logistic_regression_with_elastic_net.py. """ from __future__ import print_function import sys import numpy as np from pyspark.sql import SparkSession D = 10 # Number of dimensions # Read a batch of points from the input file into a NumPy matrix object. We operate on batches to # make further computations faster. # The data file contains lines of the form <label> <x1> <x2> ... <xD>. We load each block of these # into a NumPy array of size numLines * (D + 1) and pull out column 0 vs the others in gradient(). def readPointBatch(iterator): strs = list(iterator) matrix = np.zeros((len(strs), D + 1)) for i, s in enumerate(strs): matrix[i] = np.fromstring(s.replace(',', ' '), dtype=np.float32, sep=' ') return [matrix] if __name__ == "__main__": if len(sys.argv) != 3: print("Usage: logistic_regression <file> <iterations>", file=sys.stderr) exit(-1) print("""WARN: This is a naive implementation of Logistic Regression and is given as an example! Please refer to examples/src/main/python/ml/logistic_regression_with_elastic_net.py to see how ML's implementation is used.""", file=sys.stderr) spark = SparkSession\ .builder\ .appName("PythonLR")\ .getOrCreate() points = spark.read.text(sys.argv[1]).rdd.map(lambda r: r[0])\ .mapPartitions(readPointBatch).cache() iterations = int(sys.argv[2]) # Initialize w to a random value w = 2 * np.random.ranf(size=D) - 1 print("Initial w: " + str(w)) # Compute logistic regression gradient for a matrix of data points def gradient(matrix, w): Y = matrix[:, 0] # point labels (first column of input file) X = matrix[:, 1:] # point coordinates # For each point (x, y), compute gradient function, then sum these up return ((1.0 / (1.0 + np.exp(-Y * X.dot(w))) - 1.0) * Y * X.T).sum(1) def add(x, y): x += y return x for i in range(iterations): print("On iteration %i" % (i + 1)) w -= points.map(lambda m: gradient(m, w)).reduce(add) print("Final w: " + str(w)) spark.stop()
sort.py
from __future__ import print_function import sys from pyspark.sql import SparkSession if __name__ == "__main__": if len(sys.argv) != 2: print("Usage: sort <file>", file=sys.stderr) exit(-1) spark = SparkSession\ .builder\ .appName("PythonSort")\ .getOrCreate() lines = spark.read.text(sys.argv[1]).rdd.map(lambda r: r[0]) sortedCount = lines.flatMap(lambda x: x.split(' ')) \ .map(lambda x: (int(x), 1)) \ .sortByKey() # This is just a demo on how to bring all the sorted data back to a single node. # In reality, we wouldn't want to collect all the data to the driver node. output = sortedCount.collect() for (num, unitcount) in output: print(num) spark.stop()
transitive_closure.py
from __future__ import print_function import sys from random import Random from pyspark.sql import SparkSession numEdges = 200 numVertices = 100 rand = Random(42) def generateGraph(): edges = set() while len(edges) < numEdges: src = rand.randrange(0, numVertices) dst = rand.randrange(0, numVertices) if src != dst: edges.add((src, dst)) return edges if __name__ == "__main__": """ Usage: transitive_closure [partitions] """ spark = SparkSession\ .builder\ .appName("PythonTransitiveClosure")\ .getOrCreate() partitions = int(sys.argv[1]) if len(sys.argv) > 1 else 2 tc = spark.sparkContext.parallelize(generateGraph(), partitions).cache() # Linear transitive closure: each round grows paths by one edge, # by joining the graph's edges with the already-discovered paths. # e.g. join the path (y, z) from the TC with the edge (x, y) from # the graph to obtain the path (x, z). # Because join() joins on keys, the edges are stored in reversed order. edges = tc.map(lambda x_y: (x_y[1], x_y[0])) oldCount = 0 nextCount = tc.count() while True: oldCount = nextCount # Perform the join, obtaining an RDD of (y, (z, x)) pairs, # then project the result to obtain the new (x, z) paths. new_edges = tc.join(edges).map(lambda __a_b: (__a_b[1][1], __a_b[1][0])) tc = tc.union(new_edges).distinct().cache() nextCount = tc.count() if nextCount == oldCount: break print("TC has %i edges" % tc.count()) spark.stop()
wordcount.py
from __future__ import print_function import sys from operator import add from pyspark.sql import SparkSession if __name__ == "__main__": if len(sys.argv) != 2: print("Usage: wordcount <file>", file=sys.stderr) exit(-1) spark = SparkSession\ .builder\ .appName("PythonWordCount")\ .getOrCreate() lines = spark.read.text(sys.argv[1]).rdd.map(lambda r: r[0]) counts = lines.flatMap(lambda x: x.split(' ')) \ .map(lambda x: (x, 1)) \ .reduceByKey(add) output = counts.collect() for (word, count) in output: print("%s: %i" % (word, count)) spark.stop()
avro_inputformat.py
from __future__ import print_function import sys from functools import reduce from pyspark.sql import SparkSession """ Read data file users.avro in local Spark distro: $ cd $SPARK_HOME $ ./bin/spark-submit --driver-class-path /path/to/example/jar \ > ./examples/src/main/python/avro_inputformat.py \ > examples/src/main/resources/users.avro {u'favorite_color': None, u'name': u'Alyssa', u'favorite_numbers': [3, 9, 15, 20]} {u'favorite_color': u'red', u'name': u'Ben', u'favorite_numbers': []} To read name and favorite_color fields only, specify the following reader schema: $ cat examples/src/main/resources/user.avsc {"namespace": "example.avro", "type": "record", "name": "User", "fields": [ {"name": "name", "type": "string"}, {"name": "favorite_color", "type": ["string", "null"]} ] } $ ./bin/spark-submit --driver-class-path /path/to/example/jar \ > ./examples/src/main/python/avro_inputformat.py \ > examples/src/main/resources/users.avro examples/src/main/resources/user.avsc {u'favorite_color': None, u'name': u'Alyssa'} {u'favorite_color': u'red', u'name': u'Ben'} """ if __name__ == "__main__": if len(sys.argv) != 2 and len(sys.argv) != 3: print(""" Usage: avro_inputformat <data_file> [reader_schema_file] Run with example jar: ./bin/spark-submit --driver-class-path /path/to/example/jar \ /path/to/examples/avro_inputformat.py <data_file> [reader_schema_file] Assumes you have Avro data stored in <data_file>. Reader schema can be optionally specified in [reader_schema_file]. """, file=sys.stderr) exit(-1) path = sys.argv[1] spark = SparkSession\ .builder\ .appName("AvroKeyInputFormat")\ .getOrCreate() sc = spark.sparkContext conf = None if len(sys.argv) == 3: schema_rdd = sc.textFile(sys.argv[2], 1).collect() conf = {"avro.schema.input.key": reduce(lambda x, y: x + y, schema_rdd)} avro_rdd = sc.newAPIHadoopFile( path, "org.apache.avro.mapreduce.AvroKeyInputFormat", "org.apache.avro.mapred.AvroKey", "org.apache.hadoop.io.NullWritable", keyConverter="org.apache.spark.examples.pythonconverters.AvroWrapperToJavaConverter", conf=conf) output = avro_rdd.map(lambda x: x[0]).collect() for k in output: print(k) spark.stop()
parquet_inputformat.py
import sys from pyspark.sql import SparkSession """ Read data file users.parquet in local Spark distro: $ cd $SPARK_HOME $ export AVRO_PARQUET_JARS=/path/to/parquet-avro-1.5.0.jar $ ./bin/spark-submit --driver-class-path /path/to/example/jar \\ --jars $AVRO_PARQUET_JARS \\ ./examples/src/main/python/parquet_inputformat.py \\ examples/src/main/resources/users.parquet <...lots of log output...> {u'favorite_color': None, u'name': u'Alyssa', u'favorite_numbers': [3, 9, 15, 20]} {u'favorite_color': u'red', u'name': u'Ben', u'favorite_numbers': []} <...more log output...> """ if __name__ == "__main__": if len(sys.argv) != 2: print(""" Usage: parquet_inputformat.py <data_file> Run with example jar: ./bin/spark-submit --driver-class-path /path/to/example/jar \\ /path/to/examples/parquet_inputformat.py <data_file> Assumes you have Parquet data stored in <data_file>. """, file=sys.stderr) exit(-1) path = sys.argv[1] spark = SparkSession\ .builder\ .appName("ParquetInputFormat")\ .getOrCreate() sc = spark.sparkContext parquet_rdd = sc.newAPIHadoopFile( path, 'org.apache.parquet.avro.AvroParquetInputFormat', 'java.lang.Void', 'org.apache.avro.generic.IndexedRecord', valueConverter='org.apache.spark.examples.pythonconverters.IndexedRecordToJavaConverter') output = parquet_rdd.map(lambda x: x[1]).collect() for k in output: print(k) spark.stop()
status_api_demo.py
from __future__ import print_function import time import threading import Queue from pyspark import SparkConf, SparkContext def delayed(seconds): def f(x): time.sleep(seconds) return x return f def call_in_background(f, *args): result = Queue.Queue(1) t = threading.Thread(target=lambda: result.put(f(*args))) t.daemon = True t.start() return result def main(): conf = SparkConf().set("spark.ui.showConsoleProgress", "false") sc = SparkContext(appName="PythonStatusAPIDemo", conf=conf) def run(): rdd = sc.parallelize(range(10), 10).map(delayed(2)) reduced = rdd.map(lambda x: (x, 1)).reduceByKey(lambda x, y: x + y) return reduced.map(delayed(2)).collect() result = call_in_background(run) status = sc.statusTracker() while result.empty(): ids = status.getJobIdsForGroup() for id in ids: job = status.getJobInfo(id) print("Job", id, "status: ", job.status) for sid in job.stageIds: info = status.getStageInfo(sid) if info: print("Stage %d: %d tasks total (%d active, %d complete)" % (sid, info.numTasks, info.numActiveTasks, info.numCompletedTasks)) time.sleep(1) print("Job results are:", result.get()) sc.stop() if __name__ == "__main__": main()
6064
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
