pyspark streaming统计kafka埋点日志

需求背景：恶意注册账号疯狂发im消息，请求加商家微信。建立黑名单，凡是一天内访问200次详情页的加黑名单。

#启动zk

nohup /usr/local/kafka-2.2.1/bin/zookeeper-server-start.sh /usr/local/kafka-2.2.1/config/zookeeper.properties &

#启动kafka
nohup /usr/local/kafka-2.2.1/bin/kafka-server-start.sh /usr/local/kafka-2.2.1/config/server.properties &

#模拟生产者
while read line; do echo $line; sleep 1; done  < /root/demo/app-pv.log | ./kafka-console-producer.sh --broker-list 10.8.63.7:9092 --topic test

#pv埋点示例
 {"appkey":"c-app-pv","domain":"analytics.corp.doumi.com","hostIP":"10.16.72.21","hostname":"g1-img-web-03.dns.doumi.com","http_referer":"-","http_user_agent":"doumi-php-1.0","http_x_forwarded_for":"10.16.72.76","logpath":"/app_log.php","ng_request_id":"2f9a6c11dc81eb9800ccc904ac38ebff","quotaCount":190534,