python使用spark-sql读取数据并可视化_python中使用pyspark 读取和整理日志数据并将数据写入到es中去...

importdatetimefrom pyspark importSparkContextfrom elasticsearch importElasticsearch

sc=SparkContext.getOrCreate()

log_data= sc.textFile("/Desktop/data_doc/data_Log/utm/GX04-UTM1000D-1")"""一条日志的格式如下

Mar 1 00:00:08 172.21.208.21 date=2019-03-01 time=00:00:08 devname=GX04-UTM1000D-1 devid=FGT1KD3914800909

logid=0001000014 type=traffic subtype=local level=notice vd=root srcip=195.142.115.111 srcport=54045 srcintf="port12"

dstip=114.242.119.194 dstport=80 dstintf="root" sessionid=1013402601 status=deny policyid=0 dstcountry="China"

srccountry="Turkey" trandisp=noop service=FortiGuard proto=6 app="Web Management" duration=0 sentbyte=0

rcvdbyte=0 sentpkt=0"""es=Elasticsearch()#打印加载的用户信息第一条

filed