spark-7-spark streaming向kafka生产数据

如何正确使用pyspark将数据发送到kafka经纪人?
【Pyspark】Spark导入zip文件/上传zip文件
从kafka主题中接收数据，对该数据进行一些转换，然后将转换后的数据放在另一个kafka主题中。

1 参数写在代码里

#encoding=utf8
from pyspark import SparkConf, SparkContext
from operator import add
import sys
from pyspark.streaming import StreamingContext
from pyspark.streaming.kafka import KafkaUtils
import json
from kafka import SimpleProducer, KafkaClient
from kafka import KafkaProducer

producer = KafkaProducer(bootstrap_servers='192.168.0.91:9092')

def handler(message):
    #message类型pyspark.streaming.kafka.KafkaRDD
    records = message.collect()
    #records类型list
    for record in records:
        #record类型tuple
        value = record[0]
        #value类型unicode
        producer.send('yourtest', value.encode("utf8"))
        producer.flush()

def main():
    sc = SparkContext(appName="PythonStreamingDirectKafkaWordCount")
    ssc = StreamingContext(sc, 10)
    brokers='192.168.0.91:9092'
    topic = 'mytest'
    #brokers, topic = sys.argv[1:]
    kvs = KafkaUtils.createDirectStream(ssc, [topic], {"metadata.broker.list": brokers})
    kvs.foreachRDD(handler)

    ssc.start()
    ssc.awaitTermination()
if __name__ == "__main__":

   main()

spark-submit --master spark://192.168.0.91:7077 xxx.py

2 从控制台获取参数

#encoding=utf8
from pyspark import SparkConf, SparkContext
from operator import add
import sys
from pyspark.streaming import StreamingContext
from pyspark.streaming.kafka import KafkaUtils
import json
from kafka import SimpleProducer, KafkaClient
from kafka import KafkaProducer

producer = KafkaProducer(bootstrap_servers='192.168.0.91:9092')

def handler(message):
    #message类型pyspark.streaming.kafka.KafkaRDD
    records = message.collect()
    #records类型list
    for record in records:
        #record类型tuple
        value = record[1]
        #value类型unicode
        value1 = value.encode("utf8")
        #value1类型str
        producer.send('yourtest', value1)
        producer.flush()

def main():
    sc = SparkContext(appName="PythonStreamingDirectKafkaWordCount")
    ssc = StreamingContext(sc, 10)
    #brokers='192.168.0.91:9092'
    #topic = 'mytest'
    brokers, topic = sys.argv[1:]
    kvs = KafkaUtils.createDirectStream(ssc, [topic], {"metadata.broker.list": brokers})
    kvs.foreachRDD(handler)

    ssc.start()
    ssc.awaitTermination()
if __name__ == "__main__":

   main()

提交代码

spark-submit --master spark://192.168.0.91:7077 xxx.py 192.168.0.91:9092 mytest

3 py-files提交额外代码

3.1 主程序内部执行qq.py

print("my name is lucy")

3.2 提交代码

#encoding=utf8
from pyspark import SparkConf, SparkContext
from operator import add
import sys
from pyspark.streaming import StreamingContext
from pyspark.streaming.kafka import KafkaUtils
import json
from kafka import SimpleProducer, KafkaClient
from kafka import KafkaProducer
import os
producer = KafkaProducer(bootstrap_servers='192.168.0.91:9092')

def handler(message):
    #message类型pyspark.streaming.kafka.KafkaRDD
    records = message.collect()
    #records类型list
    for record in records:
        #record类型tuple
        value = record[1]
        #value类型unicode
        value1= value.encode("utf8")
        re = os.popen("python qq.py").read()
        print("2222222222",re)
        producer.send('yourtest', re)
        producer.flush()

def main():
    sc = SparkContext(appName="PythonStreamingDirectKafkaWordCount")
    ssc = StreamingContext(sc, 10)
    #brokers='192.168.0.91:9092'
    #topic = 'mytest'
    brokers, topic = sys.argv[1:]
    print("111111",topic)
    kvs = KafkaUtils.createDirectStream(ssc, [topic], {"metadata.broker.list": brokers})
    kvs.foreachRDD(handler)

    ssc.start()
    ssc.awaitTermination()
if __name__ == "__main__":

   main()

提交代码

spark-submit --py-files qq.py --master spark://192.168.0.91:7077 dd.py 192.168.0.91:9092 mytest

4 py-files提交zip压缩文件

压缩文件yatest.zip，内含qq.py文件。
通过命令上传。
然后需要注意在程序中解压。

from kafka import KafkaProducer
import os
import zipfile
producer = KafkaProducer(bootstrap_servers='192.168.0.91:9092')

def handler(message):
    #message类型pyspark.streaming.kafka.KafkaRDD
    records = message.collect()
    #records类型list
    for record in records:
        #record类型tuple
        value = record[1]
        #value类型unicode
        value1= value.encode("utf8")
        re = os.popen("python ./yatest/qq.py").read()
        print("2222222222",re)
        producer.send('yourtest', re)
        producer.flush()

def main():
    sc = SparkContext(appName="PythonStreamingDirectKafkaWordCount")
    #zip文件原始路径
    #file_path = "./yatest.zip"
    # 添加文件到spark的空间,位于根目录下
    #sc.addFile(file_path)
    root = './'
    file_name = 'yatest.zip'
    zip_file_path = os.path.join(root, file_name)
    with zipfile.ZipFile(zip_file_path) as zf:
        zf.extractall(root)

    ssc = StreamingContext(sc, 10)
    #brokers='192.168.0.91:9092'
    #topic = 'mytest'
    brokers, topic = sys.argv[1:]
    print("111111",topic)
    kvs = KafkaUtils.createDirectStream(ssc, [topic], {"metadata.broker.list": brokers})
    kvs.foreachRDD(handler)

    ssc.start()
    ssc.awaitTermination()
if __name__ == "__main__":

   main()

提交代码

spark-submit --py-files yatest.zip --master spark://192.168.0.91:7077 dd.py 192.168.0.91:9092 mytest

5 该进使用os.chdir

test.zip中放了很多的py文件。

#encoding=utf8
from pyspark import SparkConf, SparkContext
from operator import add
import sys
from pyspark.streaming import StreamingContext
from pyspark.streaming.kafka import KafkaUtils
import json
from kafka import SimpleProducer, KafkaClient
from kafka import KafkaProducer
import os
import zipfile
producer = KafkaProducer(bootstrap_servers='192.168.0.91:9092')

def handler(message):
    #message类型pyspark.streaming.kafka.KafkaRDD
    records = message.collect()
    #records类型list
    for record in records:
        #record类型tuple
        value = record[1]
        #value类型unicode
        value1= value.encode("utf8")
        os.chdir("./test")
        msg_dict = os.popen("python3 __init__.py").read()
        os.chdir("../")
        print("2222222222",msg_dict)
        msg = json.dumps(msg_dict, ensure_ascii=False).encode('utf-8')
        producer.send('yourtest', msg)
        producer.flush()

def main():
    sc = SparkContext(appName="PythonStreamingDirectKafkaWordCount")
    #zip文件原始路径
    #file_path = "./yatest.zip"
    # 添加文件到spark的空间,位于根目录下
    #sc.addFile(file_path)
    root = './'
    file_name = 'test.zip'
    zip_file_path = os.path.join(root, file_name)
    with zipfile.ZipFile(zip_file_path) as zf:
        zf.extractall(root)

    ssc = StreamingContext(sc, 10)
    #brokers='192.168.0.91:9092'
    #topic = 'mytest'
    brokers, topic = sys.argv[1:]
    print("111111",topic)
    kvs = KafkaUtils.createDirectStream(ssc, [topic], {"metadata.broker.list": brokers})
    kvs.foreachRDD(handler)

    ssc.start()
    ssc.awaitTermination()
if __name__ == "__main__":

   main()

提交代码

spark-submit --py-files test.zip --master spark://192.168.0.91:7077 dd.py 192.168.0.91:9092 mytest