https://stackoverflow.com/questions/25407550/how-do-i-log-from-my-python-spark-script
log_util.py
#!/usr/bin/python
# coding=utf-8
import socket
def main():
print("hello ....")
tags = dict()
tags["flight"] = "MU2916"
tags["key"] = "0001"
# log_info(logger=None, appname="wordcount", map=map, message="test message")
get_text( appname="wordcount", tags=tags, message="test message")
get_text( appname="wordcount", message="test message")
# 获取本机IP
def get_host_ip():
"""
查询本机ip地址
:return:
"""
try:
s=socket.socket(socket.AF_INET,socket.SOCK_DGRAM)
s.connect(('8.8.8.8', 80))
ip=s.getsockname()[0]
finally:
s.close()
return ip
def get_text(appname='', tags=dict(),message=''):
result ="@@"+ get_host_ip()+"@@"+appname+"@@"+str(tags)+"@@"+message+"@@"
print result
return result
if __name__ == "__main__":
main()
sc, sqlContext = s.startContext(appName)
log4jLogger = sc._jvm.org.apache.log4j
log4jLogger.LogManager.getLogger("com.umetrip.redis").setLevel(log4jLogger.Level.ERROR )
log4jLogger.LogManager.getLogger("org.apache.spark").setLevel(log4jLogger.Level.WARN )
log4jLogger.LogManager.getLogger("akka").setLevel(log4jLogger.Level.ERROR )
global log
log = log4jLogger.LogManager.getLogger("umetrip_pyspark")
info=log_util.get_text( appname=appName, message="test message")
log.warn(info)