2.6 spark实战案例：实时日志分析
- 2.6.1 交互流程图
- 2.6.2 客户端监听器（java）
@SuppressWarnings("static-access") private void handleSocket() { lock.lock(); Writer writer = null; RandomAccessFile raf = null; try { File file = new File(filepath); raf = new RandomAccessFile(file, "r"); raf.seek(pointer); writer = new OutputStreamWriter(socket.getOutputStream(), "UTF-8"); String line = null; while ((line = raf.readLine()) != null) { if (Strings.isBlank(line)) { continue; } line = new String(line.getBytes("ISO-8859-1"), "UTF-8"); writer.write(line.concat("\n")); writer.flush(); logger.info("线程：{}----起始位置：{}----读取文件\n{} :",Thread.currentThread().getName(), pointer, line); pointer = raf.getFilePointer(); } Thread.currentThread().sleep(2000); } catch (Exception e) { logger.error(e.getMessage()); e.printStackTrace(); } finally { lock.unlock(); fclose(writer, raf); } }
- 2.6.3 sparkStream实时数据接收（python）
conf = SparkConf() conf.setAppName("HIS实时日志分析") conf.setMaster('yarn') # spark standalone conf.set('spark.executor.instances', 8) # cluster on yarn conf.set('spark.executor.memory', '1g') conf.set('spark.executor.cores', '1') # conf.set('spark.cores.max', '2') # conf.set('spark.logConf', True) conf.set('spark.streaming.blockInterval', 1000*4) # restart receiver interval sc = SparkContext(conf = conf) sc.setLogLevel('ERROR') sc.setCheckpointDir('hdfs://hadoop01:9000/hadoop/upload/checkpoint/') ssc = StreamingContext(sc, 30) # time interval at which splits streaming data into block lines = ssc.socketTextStream(str(ip), int(port)) # lines.pprint() lines.foreachRDD(requestLog) lines.foreachRDD(errorLog) ssc.start() ssc.awaitTermination()
- 2.6.4 sparklSQL、RDD结算、结构化搜索、结构存储mongoDB（python）
def getSparkSessionInstance(sparkConf): ''' :@desc 多个RDD全局共享sparksession .config("spark.mongodb.input.uri", "mongodb://127.0.0.1/test.coll") \ .config("spark.mongodb.output.uri", "mongodb://adxkj:123456@192.168.0.252:27017/") \ :param sparkConf: :return: ''' if ('sparkSessionSingletonInstance' not in globals()): globals()['sparkSessionSingletonInstance'] = SparkSession \ .builder \ .config(conf=sparkConf) \ .getOrCreate() return globals()['sparkSessionSingletonInstance'] def timeFomate(x): ''' :@desc 处理时间 :param x: :return: ''' if not isinstance(x, list): return None # filter microsenconds x.insert(0, ' '.join(x[0:2])) x.pop(1) x.pop(1) # filter '[]' rx = re.compile('([\[\]\',])') # text = rx.sub(r'\\\1', text) x = [rx.sub(r'', x[i]) for i in range(len(x))] # string to time x[0] = x[0][: x[0].find('.')] x[0] = ''.join(x[0]) x[0] = datetime.strptime(x[0], '%Y-%m-%d %H:%M:%S') return x def sqlMysql(sqlResult, table, url="jdbc:mysql://192.168.0.252:3306/hisLog", user='root', password=""): ''' :@desc sql结果保存 :param sqlResult: :param table: :param url: :param user: :param password: :return: ''' try: sqlResult.write \ .mode('append') \ .format("jdbc") \ .option("url", url) \ .option("dbtable", table) \ .option("user", user) \ .option("password", password) \ .save() except: excType, excValue, excTraceback = sys.exc_info() traceback.print_exception(excType, excValue, excTraceback, limit=3) # print(excValue) # traceback.print_tb(excTraceback) def sqlMongodb(sqlResult, table): ''' :@desc sql结果保存 :param sqlResult: :param table: :param url: :param user: :param password: :return: ''' try: sqlResult.\ write.\ format("com.mongodb.spark.sql.DefaultSource"). \ options(uri="mongodb://adxkj:123456@192.168.0.252:27017/hislog", database="hislog", collection=table, user="adxkj", password="123456").\ mode("append").\ save() except: excType, excValue, excTraceback = sys.exc_info() traceback.print_exception(excType, excValue, excTraceback, limit=3) # print(excValue) # traceback.print_tb(excTraceback) def decodeStr(x) : ''' :@desc base64解码 :param x: :return: ''' try: if x[9].strip() != '' : x[9] = base64.b64decode(x[9].encode("utf-8")).decode("utf-8") # x[9] = x[9][:5000] #mysql if x[11].strip() != '': x[11] = base64.b64decode(x[11].encode("utf-8")).decode("utf-8") # x[11] = x[11][:5000] #mysql if len(x) > 12 and x[12].strip() != '': x[12] = base64.b64decode(x[12].encode("utf-8")).decode("utf-8") except Exception as e: print("不能解码：", x, e) return x def analyMod(x) : ''' :@desc 通过uri匹配模块 :param x: :return: ''' if x[6].strip() == ' ': return None hasMatch = False for k, v in URI_MODULES.items() : if x[6].strip().startswith('/' + k) : hasMatch = True x.append(v) if not hasMatch: x.append('公共模块') return x def requestLog(time, rdd): ''' :@desc 请求日志分析 :param time: :param rdd: :return: ''' logging.info("+++++handle request log：length：%d，获取内容：++++++++++" % (rdd.count())) if rdd.isEmpty(): return None logging.info("++++++++++++++++++++++处理requestLog+++++++++++++++++++++++++++++++") reqrdd = rdd.map(lambda x: x.split(' ')).\ filter(lambda x: len(x) > 12 and x[4].find('http-nio-') > 0 and x[2].strip() == 'INFO').\ filter(lambda x: x[8].strip().upper().startswith('POST') or x[8].strip().upper().startswith('GET')).\ map(timeFomate).\ map(decodeStr).\ map(analyMod) reqrdd.cache() reqrdd.checkpoint() # checkpoint先cache避免计算两次，以前的rdd销毁 sqlRdd = reqrdd.map(lambda x: Row(time=x[0], level=x[1], clz=x[2], thread=x[3], user=x[4], depart=x[5], uri=x[6], method=x[7], ip=x[8], request=x[9], oplen=x[10], respone=x[11], mod=x[12])) # rdd持久化，降低内存消耗, cache onliy for StorageLevel.MEMORY_ONLY # reqrdd.persist(storageLevel=StorageLevel.MEMORY_AND_DISK_SER) if reqrdd.isEmpty(): return None spark = getSparkSessionInstance(rdd.context.getConf()) df = spark.createDataFrame(sqlRdd) df.createOrReplaceTempView(REQUEST_TABLE) # 结构化后再分析 sqlresult = spark.sql("SELECT * FROM " + REQUEST_TABLE) sqlresult.show() # 保存 sqlMongodb(sqlresult, REQUEST_TABLE) def errorLog(time, rdd): ''' :@desc 错误日志分析 :param time: :param rdd: :return: ''' logging.info("+++++handle error log：length：%d，获取内容：++++++++++" % (rdd.count())) if rdd.isEmpty(): return None logging.info("++++++++++++++++++++++处理errorLog+++++++++++++++++++++++++++++++") errorrdd = rdd.map(lambda x: x.split(' ')). \ filter(lambda x: len(x) > 13 and x[2].strip().upper().startswith('ERROR')). \ map(timeFomate). \ map(decodeStr). \ map(analyMod). \ map(lambda x: Row(time=x[0], level=x[1], clz=x[2], thread=x[3], user=x[4], depart=x[5], uri=x[6], method=x[7], ip=x[8], request=x[9], oplen=x[10], respone=x[11], stack=x[12], mod=x[13])) # rdd持久化，降低内存消耗 errorrdd.persist(storageLevel=StorageLevel.MEMORY_AND_DISK_SER) if errorrdd.isEmpty(): return None spark = getSparkSessionInstance(rdd.context.getConf()) df = spark.createDataFrame(errorrdd) df.createOrReplaceTempView(ERROR_TABLE) # 结构化后再分析 sqlresult = spark.sql("SELECT * FROM " + ERROR_TABLE) sqlresult.show() # 保存 sqlMongodb(sqlresult, ERROR_TABLE)
备注：需要完整代码请联系作者@狼
641
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
