二十一 Spark Kafka操作集合

spark-env.sh
export SPARK_DIST_CLASSPATH=$(/opt/hadoop/bin/hadoop classpath)
export SPARK_MASTER_IP=hmaster

slaves
hslave1
hslave2

http://www.powerxing.com/spark-quick-start-guide/

./spark-submit   --master spark://hmaster:7077 --name JavaWorkCount --class org.apache.spark.examples.JavaWordCount  /data/spark-workspace/spark-examples_2.10-1.6.2.jar  /spark/doc/1.txt


./spark-submit   --master spark://hmaster:7077 --name JavaALS --class org.apache.spark.examples.mllib.JavaALS   /data/spark-workspace/spark-examples_2.10-1.6.2.jar


./spark-submit --master spark://hmaster:7077 --name JavaKafkaWordCount --jars $LIBJARS --class org.apache.spark.examples.streaming.JavaKafkaWordCount /data/example/pro-spark-example-1.6.2.jar

./spark-submit  --name JavaKafkaWordCount --jars $LIBJARS --class org.apache.spark.examples.streaming.JavaKafkaWordCount /data/example/pro-spark-example-1.6.2.jar

./spark-submit --master spark://hmaster:7077 --name JavaKafkaWordCount --jars $LIBJARS --class org.apache.spark.examples.streaming.JavaKafkaWordCount /data/spark-workspace/pro-spark-example-1.6.2.jar

./spark-submit --master spark://hmaster:7077 --name KafkaAls--jars $LIBJARS --class org.apache.spark.dx.KafkaAls /data/spark-workspace/pro-spark-example-1.6.2.jar


Kafka操作
bin/kafka-server-start.sh config/server.properties &

bin/kafka-topics.sh --create --zookeeper localhost:2181/kafka --replication－factor 3 --partitions 1 --topic test
bin/kafka-topics.sh --describe --zookeeper localhost:2181 /kafka --topic test

bin/kafka-console-producer.sh --broker-list 192.168.10.155:9092 --topic test

bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic test --from-beginning