基于nestjs与log4js的日志模块集成

日志模块集成

1. 版本需求

新平台项目需集成日志系统, 用于记录项目中各类信息.

2. 需求分析

日志模块选型:
	社区主流日志实现有 log4js, Winston, pino.
	综合网络教程及star数, 选择 log4js 作为日志系统

预期实现功能:
	日志格式化输出至log文件, 信息易读
	日志模块参数化配置, 易维护
	请求与响应信息记录, 异常信息的全局捕获, 便于排错
	自定义集成typeorm 日志模块, 统一输出 

3. 需求实现

0. 安装npm包

npm install log4js
npm install stacktrace-js

1. log4js 配置

官方文档

// 项目根目录新建config文件用于保存配置文件, 新建log4jsConfig.ts配置文件

import * as path from 'path';
const baseLogPath = path.resolve(__dirname, '../../logs');


const log4jsConfig = {
  appenders: {
    console: { type: 'console' },	// 控制打印至控制台
    // 统计日志
    access: {
      type: 'dateFile',	// 写入文件格式，并按照日期分类
      filename: `${baseLogPath}/access/access.log`,	// 日志文件名，会命名为：access.2021-04-01.log
      alwaysIncludePattern: true,	// 为true, 则每个文件都会按pattern命名，否则最新的文件不会按照pattern命名
      pattern: 'yyyy-MM-dd',	// 日期格式
      // maxLogSize: 10485760,  // 日志大小
      daysToKeep: 30,	// 文件保存日期30天
      numBackups: 3,	//  配置日志文件最多存在个数
      compress: true,   // 配置日志文件是否压缩
      category: 'http',	// category 类型
      keepFileExt: true,	// 是否保留文件后缀
    },
    // 一些app的 应用日志
    app: {
      type: 'dateFile',
      filename: `${baseLogPath}/app-out/app.log`,
      alwaysIncludePattern: true,
      layout: {
        type: 'pattern',
        pattern: "[%d{yyyy-MM-dd hh:mm:ss SSS}] [%p] -h: %h -pid: %z  msg: \'%m\' "
      },	// 自定义的输出格式, 可参考 https://blog.csdn.net/hello_word2/article/details/79295344
      pattern: 'yyyy-MM-dd',
      daysToKeep: 30,
      numBackups: 3,
      keepFileExt: true,
    },
    // 异常日志 
    errorFile: {
      type: 'dateFile',
      filename: `${baseLogPath}/error/error.log`,
      alwaysIncludePattern: true,
      layout: {
        type: 'pattern',
        pattern: "[%d{yyyy-MM-dd hh:mm:ss SSS}] [%p] -h: %h -pid: %z  msg: \'%m\' "
      },
      pattern: 'yyyy-MM-dd',
      daysToKeep: 30,
      numBackups: 3,
      keepFileExt: true,
    },
    errors: {
      type: 'logLevelFilter',
      level: 'ERROR',
      appender: 'errorFile',
    },
  },
  
  categories: {
    default: { appenders: ['console', 'access', 'app', 'errors'], level: 'DEBUG' },
    mysql: { appenders: ['access', 'errors'], level: 'info' },
    http: { appenders: ['access'], level: 'DEBUG' },
  },
};

export default log4jsConfig;

注意点, 配置类中有两个主要参数

- appenders
	作用是配置输出源, 用于定义输出日志的各种格式, 后续我们真正输出日志的对象就是log4js的下属的输出源.
	
- categories
	category 类型, 可以设置一个 Logger 实例的类型，按照另外一个维度来区分日志.
	在通过 getLogger 获取 Logger 实例时, 可指定获取具体的 Logger 实例

2. 实例化日志类

// 目前把实例化类配置放在 src\utils\log4js.ts 中

import * as Path from 'path';
import * as Log4js from 'log4js';
import * as Util from 'util';
import * as Moment from 'moment'; // 处理时间的工具
import * as StackTrace from 'stacktrace-js';
import Chalk from 'chalk';
import log4jsConfig from 'config/log4jsConfig';
import { QueryRunner } from 'typeorm';


// 定义日志级别
export enum LoggerLevel {
  ALL = 'ALL',
  MARK = 'MARK',
  TRACE = 'TRACE',
  DEBUG = 'DEBUG',
  INFO = 'INFO',
  WARN = 'WARN',
  ERROR = 'ERROR',
  FATAL = 'FATAL',
  OFF = 'OFF',
}

// 内容跟踪类
export class ContextTrace {
  constructor(
    public readonly context: string,
    public readonly path?: string,
    public readonly lineNumber?: number,
    public readonly columnNumber?: number,
  ) { }
}

// 添加用户自定义的格式化布局函数。 可参考: https://log4js-node.github.io/log4js-node/layouts.html
Log4js.addLayout('json', (logConfig: any) => {
  return (logEvent: Log4js.LoggingEvent): string => {
    let moduleName: string = '';
    let position: string = '';

    // 日志组装
    const messageList: string[] = [];
    logEvent.data.forEach((value: any) => {
      if (value instanceof ContextTrace) {
        moduleName = value.context;
        // 显示触发日志的坐标（行，列）
        if (value.lineNumber && value.columnNumber) {
          position = `${value.lineNumber}, ${value.columnNumber}`;
        }
        return;
      }

      if (typeof value !== 'string') {
        value = Util.inspect(value, false, 3, true);
      }

      messageList.push(value);
    });

    // 日志组成部分
    const messageOutput: string = messageList.join(' ');
    const positionOutput: string = position ? ` [${position}]` : '';
    const typeOutput: string = `[${logConfig.type}] ${logEvent.pid.toString()}   - `;
    const dateOutput: string = `${Moment(logEvent.startTime).format('YYYY-MM-DD HH:mm:ss')}`;
    const moduleOutput: string = moduleName ? `[${moduleName}] ` : '[LoggerService] ';
    let levelOutput: string = `[${logEvent.level}] ${messageOutput}`;

    // 根据日志级别，用不同颜色区分
    switch (logEvent.level.toString()) {
      case LoggerLevel.DEBUG:
        levelOutput = Chalk.green(levelOutput);
        break;
      case LoggerLevel.INFO:
        levelOutput = Chalk.cyan(levelOutput);
        break;
      case LoggerLevel.WARN:
        levelOutput = Chalk.yellow(levelOutput);
        break;
      case LoggerLevel.ERROR:
        levelOutput = Chalk.red(levelOutput);
        break;
      case LoggerLevel.FATAL:
        levelOutput = Chalk.hex('#DD4C35')(levelOutput);
        break;
      default:
        levelOutput = Chalk.grey(levelOutput);
        break;
    }

    return `${Chalk.green(typeOutput)}${dateOutput}  ${Chalk.yellow(moduleOutput)}${levelOutput}${positionOutput}`;
  };
});

// 注入配置
Log4js.configure(log4jsConfig);

// 实例化
const logger = Log4js.getLogger("default");
const mysqlLogger = Log4js.getLogger('mysql');	// 添加了typeorm 日志实例
logger.level = LoggerLevel.TRACE;

// 定义log类方法
export class Logger {
  static trace(...args) {
    logger.trace(Logger.getStackTrace(), ...args);
  }

  static debug(...args) {
    logger.debug(Logger.getStackTrace(), ...args);
  }

  static log(...args) {
    logger.info(Logger.getStackTrace(), ...args);
  }

  static info(...args) {
    logger.info(Logger.getStackTrace(), ...args);
  }

  static warn(...args) {
    logger.warn(Logger.getStackTrace(), ...args);
  }

  static warning(...args) {
    logger.warn(Logger.getStackTrace(), ...args);
  }

  static error(...args) {
    logger.error(Logger.getStackTrace(), ...args);
  }

  static fatal(...args) {
    logger.fatal(Logger.getStackTrace(), ...args);
  }

  static access(...args) {
    const loggerCustom = Log4js.getLogger('http');
    loggerCustom.info(Logger.getStackTrace(), ...args);
  }

  // 日志追踪，可以追溯到哪个文件、第几行第几列
  // StackTrace 可参考 https://www.npmjs.com/package/stacktrace-js
  static getStackTrace(deep: number = 2): string {
    const stackList: StackTrace.StackFrame[] = StackTrace.getSync();
    const stackInfo: StackTrace.StackFrame = stackList[deep];
    const lineNumber: number = stackInfo.lineNumber;
    const columnNumber: number = stackInfo.columnNumber;
    const fileName: string = stackInfo.fileName;
    const basename: string = Path.basename(fileName);
    return `${basename}(line: ${lineNumber}, column: ${columnNumber}): \n`;
  }
}


// 自定义typeorm 日志器, 可参考 https://blog.csdn.net/huzzzz/article/details/103191803/
export class DbLogger implements Logger {
  logQuery(query: string, parameters?: any[], queryRunner?: QueryRunner) {
    mysqlLogger.info(query);
  }

  logQueryError(error: string, query: string, parameters?: any[], queryRunner?: QueryRunner) {
    mysqlLogger.error(query, error);
  }

  logQuerySlow(time: number, query: string, parameters?: any[], queryRunner?: QueryRunner) {
    mysqlLogger.info(query, time);
  }

  logSchemaBuild(message: string, queryRunner?: QueryRunner) {
    mysqlLogger.info(message);
  }

  logMigration(message: string, queryRunner?: QueryRunner) {
    mysqlLogger.info(message);
  }
  log(level: 'log' | 'info' | 'warn', message: any, queryRunner?: QueryRunner) {
    switch (level) {
      case 'info': {
        mysqlLogger.info(message);
        break;
      }
      case 'warn': {
        mysqlLogger.warn(message);
      }
    }
  }
}

注意点

主要是实例化 log4js 的过程，整合了日志的组成部分（包含了时间、类型，调用文件以及调用的坐标），可以参考文档自行更改参数配置.

3. 利用中间件记录请求信息

注意点

参考文档:
	https://docs.nestjs.cn/7/middlewares
	https://blog.csdn.net/qq_42852301/article/details/103295541

记得在main中注册: 
    app.use(new HttpRequestMiddleware().use)

src\middleware\request.middleware.ts

/**
 * 自定义请求信息日志记录中间件
 */
import { NextFunction, Request, Response } from 'express'
import { HttpLogger } from '../logger/logger'
import { requestLoggerData } from '../logger/appenders/log4js-kafka-appender/kafka.appender.interface'
import { Injectable, NestMiddleware } from '@nestjs/common'

@Injectable()
export class HttpRequestMiddleware implements NestMiddleware {
  use(req: Request, res: Response, next: NextFunction) {
    next()

    // 组装日志信息
    const logFormat: requestLoggerData = {
      httpType: 'Request',
      ip: req.headers?.remoteip ? String(req.headers.remoteip) : req.ip.split(':').pop(),
      reqUrl: `${req.headers.host}${req.originalUrl}`,
      reqMethod: req.method,
      httpCode: res.statusCode,
      params: req.params,
      query: req.query,
      body: req.body
    }

    // 根据状态码，进行日志类型区分
    if (res.statusCode >= 400) {
      HttpLogger.error(JSON.stringify(logFormat))
    } else {
      HttpLogger.access(JSON.stringify(logFormat))
    }
  }
}

4. 利用拦截器记录出参信息

注意点

入参使用中间件达成获取记录, 出参需要通过 nestjs 的拦截器获取, 所有参数都会经过这里进行处理

功能:
	- 在函数执行之前/之后绑定额外的逻辑
    - 转换从函数返回的结果
    - 转换从函数抛出的异常
    - 根据所选条件完全重写函数 (例如, 缓存目的) 
		- 后续可参考 https://www.jianshu.com/p/e7b0f3eb3aed?tdsourcetag=s_pcqq_aiomsg 实现缓存

参考文档:
	https://docs.nestjs.cn/7/interceptors	// 官方文档
	https://www.wenjiangs.com/doc/8usoxl0vg
	https://tuture.co/2020/05/12/@uXOOfFmhS/

intercept 接受两个参数，当前的上下文和传递函数，这里还使用了 pipe（管道）

记得在main中注册: 
	app.useGlobalInterceptors(new HttpResponseInterceptor());

src\middleware\response.interceptor.ts

/**
 * 自定义响应日志记录拦截器
 */
import { CallHandler, ExecutionContext, Injectable, NestInterceptor } from '@nestjs/common'
import { Observable } from 'rxjs'
import { map } from 'rxjs/operators'
import { HttpLogger } from '../logger/logger'
import { requestLoggerData } from '../logger/appenders/log4js-kafka-appender/kafka.appender.interface'

@Injectable()
export class HttpResponseInterceptor implements NestInterceptor {
  intercept(context: ExecutionContext, next: CallHandler): Observable<any> {
    const req = context.getArgByIndex(1).req

    return next.handle().pipe(
      map(data => {
        const logFormat: requestLoggerData = {
          httpType: 'Response',
          ip: req.headers?.remoteip ? String(req.headers.remoteip) : req.ip.split(':').pop(),
          reqUrl: `${req.headers.host}${req.originalUrl}`,
          reqMethod: req.method,
          params: req.params,
          query: req.query,
          body: req.body
          // data: data
        }
        HttpLogger.access(JSON.stringify(logFormat))
        return data
      })
    )
  }
}

5. 利用过滤器实现全局异常的捕捉

注意点

参考文档:
	https://docs.nestjs.cn/7/exceptionfilters

异常过滤, nestjs官方提供的已经很多, 很完整了
	所有这些都可以在 @nestjs/common包中找到：

可以使用命令创建
	nest g filter http-exception filter	// http相关的异常
    nest g filter any-exception filter	// 项目中所有的异常

记得在main中注册: 
	  app.useGlobalFilters(new AllExceptionsFilter)

src\middleware\any-exception.filter.ts

/**
 * any-exception filter.
 * @file 全局异常捕获
 * 全局范围的异常捕获, 统一处理, 并输出error日志
 * 原则上error信息要全部记录, 可以有选择的提取信息进行前置
 */

import { ExceptionFilter, Catch, ArgumentsHost, HttpException, HttpStatus } from '@nestjs/common'
import { HttpLogger } from '../logger/logger'
import * as TEXT from '../constants/text.constant'
import * as CODE from '../constants/code.constant'
import { errorResponseLoggerData } from './error.interface'

@Catch()
export class AllExceptionsFilter implements ExceptionFilter {
  catch(exception: any, host: ArgumentsHost) {
    const request = host.switchToHttp().getRequest()
    const response = host.switchToHttp().getResponse()
    const status =
      exception instanceof HttpException ? exception.getStatus() : CODE.EHttpStatus.INTERNAL_SERVER_ERROR_CODE

    // 自定义的异常信息结构, 响应用
    const error_info = exception.response ? exception.response : exception
    const error_data = exception.response?.data ? exception.response.data : {}
    const error_msg = exception.response
      ? exception.response.message
        ? exception.response.message
        : exception.response.errorMsg
      : TEXT.INTERNAL_SERVER_ERROR_TEXT
    const error_code = exception.response?.errorCode ? exception.response.errorCode : CODE.StatusCode.ErrorCode

    // 自定义异常结构体, 日志用
    const data: errorResponseLoggerData = {
      timestamp: new Date().toISOString(),
      ip: request.ip,
      reqUrl: request.originalUrl,
      reqMethod: request.method,
      httpCode: status,
      params: request.params,
      query: request.query,
      body: request.body,
      statusCode: error_code,
      errorMsg: error_msg,
      errorData: error_data,
      errorInfo: error_info
    }

    // 404 异常响应
    if (status === HttpStatus.NOT_FOUND) {
      data.errorMsg = `资源不存在! 接口 ${request.method} -> ${request.url} 无效!`
    }
    HttpLogger.error(data)

    // 程序内异常捕获返回
    response.status(status).json({
      data: data.errorData,
      msg: data.errorMsg,
      code: data.statusCode
    })
  }
}

6. 自定义typeorm 日志模块

注意点

参考文档:
	官方文档: https://typeorm.biunav.com/zh/logging.html#%E6%9B%B4%E6%94%B9%E9%BB%98%E8%AE%A4%E8%AE%B0%E5%BD%95%E5%99%A8
	https://blog.csdn.net/huzzzz/article/details/103191803/

直接在配置项更改即可 
	typeorm 的配置文件已修改为根据env启动对应文件
	

config/prod.env.ts

import { ConnectionOptions } from 'typeorm';
import { DbLogger } from 'src/utils/log4js';


export const db: ConnectionOptions = {
  "type": "mysql",
  "host": "",
  "port": 3306,
  "username": "",
  "password": "",
  "database": "vmp_dev",
  "entities": ["dist/**/*.entity{.ts,.js}"],
  "synchronize": true,
  "logging": true,
  "maxQueryExecutionTime": 1000,
  "logger": new DbLogger(),	// 配置项添加自定义的log类
  "timezone": '+08:00',
};

7. 根据env获取对应的配置信息

注意点

配置orm自定日志时, 扩展了配置文件的读取方式, 记录一下根据环境变量的配置文件读取

基本思路:
	使用 process.env 设置 NODE_ENV
	项目启动时, 根据线上与测试的不同, 设置不同的环境变量. 
	项目启动后, 会根据 NODE_ENV 进行读取不同的配置

参考文章:
	https://blog.csdn.net/xiaolinlife/article/details/107032533
	

项目目录下新建 config文件夹

config\index.ts

import * as dev from './dev.env';
import * as prop from './prod.env';


const envconfigs = {
  development: dev,
  production: prop,
};

const environment = envconfigs[process.env.NODE_ENV || 'development']

export { environment };

config\prod.env.ts

import { ConnectionOptions } from 'typeorm';
import { DbLogger } from 'src/utils/log4js';


export const db: ConnectionOptions = {
  "type": "mysql",
  "host": "",
  "port": 3306,
  "username": "",
  "password": "",
  "database": "vmp_dev",
  "entities": ["dist/**/*.entity{.ts,.js}"],
  "synchronize": true,
  "logging": true,
  "maxQueryExecutionTime": 1000,
  "logger": new DbLogger(),
  "timezone": '+08:00',
};

config\dev.env.ts

import { ConnectionOptions } from 'typeorm';
import { DbLogger } from 'src/utils/log4js';


export const db: ConnectionOptions = {
  "type": "mysql",
  "host": "127.0.0.1",
  "port": 3306,
  "username": "root",
  "password": "",
  "database": "vmp_dev",
  "entities": ["dist/**/*.entity{.ts,.js}"],
  "synchronize": true,
  "maxQueryExecutionTime": 1000,
  "logger": new DbLogger(),
  "timezone": '+08:00',
};

package.json

设置 scripts 启动命令
{
  ...
  "scripts": {
    "start": "cross-env NODE_ENV=development nest start | pino-colada ",
    "start:dev": "webpack --config webpack.config.js",
    "start:prod": "cross-env NODE_ENV=production node dist/main",
  },
  ...
}