用syslog服务器收集java日志

初来公司时，java日志收集使用的nfs方式，将nfs server上的一块大容量磁盘挂载到所有java服务器的$catalina_home/logs目录，解决服务器本地空间不足问题；

这个方案的弊端是，如果nfs server想停机做个维护，几乎不可行；同时，多节点的日志在nfs服务器上是分开的，如果某个应用有8个节点，那查日志要查8个文件。

用syslog收集java日志，能完美解决这个问题。

安装syslog-ng

有兴趣建议看看官方文档：https://syslog-ng.com/documents/html/syslog-ng-ose-latest-guides/en/syslog-ng-ose-guide-admin/html/index.html

以下是Centos7.4上的安装过程
yum install zlib-devel libffi-devel 
yum install pcre pcre-devel glib2-devel json-c json-c-devel openssl-devel

安装以下4个依赖，方式都一样./configure && make && makeinstall
flex-2.6.4
bison-3.0.4
autoconf-archive-2017.09.28
pcre-8.41

安装syslog-ng
cd syslog-ng-3.13.2
./configure --prefix=/syslog-ng --with-jsonc=system
make 
make install

java输出日志主要通过log4j或logback，咱公司这两种都用到了，因此提供两种方式的syslog服务器。

syslog-ng for logback配置

syslog-ng.conf 

#############################################################################
# Default syslog-ng.conf file which collects all local logs into a
# single file called /var/log/messages.
#

@version: 3.7
@include "scl.conf"

options {
    keep-hostname(no);
    chain-hostnames(no);
    use-dns(no);
    flush-lines(0);
    time-reap(1);
    log-msg-size(4096);
    create-dirs(yes);  
    stats-freq(3600);
    owner (root);
    group(root);
    perm(0666);
    dir-perm(0777);
};

source s_local {
        system();
        internal();
};

destination d_local {
        file("/var/log/messages");
};

log {
    source(s_local);
    destination(d_local);
};
#logback只支持udp方式输出udp，因此这里通过udp接受日志
source s_app {
    udp(
        ip(0.0.0.0) 
        port(515)
        log_iw_size(50000)
    );
};

filter f_app {
    facility(local2) and (level(error) or level(info) or level(debug));
};
# syslog通过解析日志格式化输出，通过“|”分割日志；同时“[]”中作为一个整体,不做分割；
# 日志分割后，各个字段名称通过columns（）来命名；
# template，指定从syslog协议中取的日志来源
parser p_msg {
    csv-parser(
        columns("APPNAME","FILENAME","MYDATE","LOGGER","INVOKENO","CONTENT")
        flags(escape-double-char,strip-whitespace,greedy)
        delimiters("|")
        quote-pairs('[]')
        template("${MSGHDR}${MESSAGE}")
    );
};
# 日志输出到/syslog/${APPNAME}下，以日期做后缀
# 日志格式通过template指定
destination d_appfile {
    file(
        "/syslog/${APPNAME}/${FILENAME}-${LEVEL}.log.$YEAR$MONTH$DAY" 
        template("${MYDATE}|$HOST|${LOGGER}|${INVOKENO}|${CONTENT}\n")
        log-fifo-size(500000)
    );
};

# 日志从收集到解析、输出的处理流
log {
    source(s_app);
    filter(f_app);
    parser(p_msg);
    destination(d_appfile);

};

# java程序logback.xml的配置
# 每个应用的日志可以按2个维度区分：级别、来源
# 级别（debug\info\warning\error）
# 日志来源（通过appender区分不同的java程序打印的日志，如用户登录访问日志，组件mybaits的日志等，可选）
# suffixPattern指定输出的格式，根据syslog-ng的指定，以“|”分割，myapp用于创建目录，myfile1用于文件的前缀

<?xml version="1.0" encoding="UTF-8"?>
<configuration scan="true" scanPeriod="60 seconds" debug="false">
    
    <appender name="append1" class="ch.qos.logback.classic.net.SyslogAppender">
        <syslogHost>syslogserverip</syslogHost>
        <port>515</port>
        <facility>local2</facility>
        <throwableExcluded>true</throwableExcluded>
        <suffixPattern>[myapp]|myfile1|%date|%logger{50}|%X{invokeNo}|%X{userIP}|%X{userName}|%msg%n%xException</suffixPattern>
    </appender>
	 
    <appender name="append2" class="ch.qos.logback.classic.net.SyslogAppender">
        <syslogHost>syslogserver</syslogHost>
        <port>515</port>
        <facility>local2</facility>
        <throwableExcluded>true</throwableExcluded>
        <suffixPattern>[myapp]|myfile2|%date|%logger{50}|%X{invokeNo}|%X{userIP}|%X{userName}|%msg%n%xException</suffixPattern>
    </appender>
	 
    <appender name="other" class="ch.qos.logback.classic.net.SyslogAppender">
        <syslogHost>syslogserver</syslogHost>
        <port>515</port>
        <facility>local2</facility>
        <throwableExcluded>true</throwableExcluded>
        <suffixPattern>[myapp]|myfile3|%date|%logger{50}|%X{invokeNo}|%X{userIP}|%X{userName}|%msg%n%xException</suffixPattern>
    </appender>
	 
	 
    <logger name="package1" level="INFO" additivity="false">
	<appender-ref ref="append1" />
    </logger>
	 
    <logger name="package2" level="INFO" additivity="false">
	<appender-ref ref="append2" />
    </logger>
 
    <root level="info">
        <appender-ref ref="other" />
    </root>
 
</configuration>

syslog-ng for log4j配置

syslog-ng.conf 
#############################################################################
# Default syslog-ng.conf file which collects all local logs into a
# single file called /var/log/messages.
#
@version: 3.7
@include "scl.conf"

options {
    keep-hostname(no);
    chain-hostnames(no);
    use-dns(no);
    flush-lines(0);
    time-reap(1);
    log-msg-size(4096);
    create-dirs(yes);  
    stats-freq(3600);
    owner (root);
    group(root);
    perm(0666);
    dir-perm(0777);
};

source s_local {
        system();
        internal();
};
# log4j同时支持tcp\udp方式输出日志，但压力测试过程发现log4j的udp输出到日志服务器日志会串，即A应用的日志会写到B应用的文件中，用tcp没这样的问题
source s_app {
    tcp(
        ip(0.0.0.0) 
        port(515)
        max-connections(500)
        log_iw_size(50000)
    );
};

filter f_app {
    facility(local2) and (level(error) or level(info) or level(debug));
};
# 因为log4j输出的格式和logback不同，因此syslog-ng解析方式也不同
parser p_msg {
    csv-parser(
        columns("MSG.TIME", "MSG.HOSTNAME", "MSG.APPNAME","MSG.PID","MSG.MESSAGEID","MSG.APPINFO","MSG.CONTENT")
        flags(escape-double-char,strip-whitespace,greedy)
        delimiters(" ")
        quote-pairs('[]')
    );
};


rewrite r_nl {
    subst("#NewLine#","\n",value("MSG.CONTENT") flags(global));
    subst("mdc@18060 ","",value("MSG.APPINFO") flags(global));
    subst("syslogk[1-9]=\"","|",value("MSG.APPINFO") flags(global));
    subst("\" ","",value("MSG.APPINFO") flags(global));
    subst("\"","|",value("MSG.APPINFO") flags(global));
};

destination d_local {
        file("/var/log/messages");
};

destination d_appfile {
    file(
        "/syslog/${MSG.APPNAME}/${MSG.MESSAGEID}-$LEVEL.log.$YEAR$MONTH$DAY" 
        template("$(substr $(replace-delimiter 'T' ' ' ${MSG.TIME}) 0 23)|$HOST${MSG.APPINFO}${MSG.CONTENT} \n")
        log-fifo-size(500000)
    );
};

log {
    source(s_local);
    destination(d_local);
};

log {
    source(s_app);
    filter(f_app);
    parser(p_msg);
    rewrite(r_nl);
    destination(d_appfile);
    flags(flow-control);
};


# log4j2.xml
# <LoggerFields>中syslogk1是固定字段，对应类名称；syslogk2- syslogk9都是自定义字段，可用于记录登录的用户名，用户的客户端IP等个性化的信息；
# 不需要用的syskogkn，建议删除，否则日志文件中会多余的||，影响日志查看。
# 自定义内容的添加，在java程序中执行：MDC.put("syslogk2", "1111");

<?xml version="1.0" encoding="UTF-8"?>
<Configuration status="off">
	<Appenders>
		<Syslog name="appender1" format="RFC5424" host="syslogserverip"
			port="515" protocol="TCP" appName="myappname" messageId="myfilename1"
			facility="LOCAL2" newLine="true" mdcId="mdc" includeMDC="true" 
			newLineEscape="#NewLine#" exceptionPattern="%ex"
			immediateFlush="false" connectTimeoutMillis="3000" reconnectionDelayMillis="3000">
			<LoggerFields>
				<KeyValuePair key="syslogk1" value="%l" />
				<KeyValuePair key="syslogk2" value="%X{syslogk2}" /> 
				<KeyValuePair key="syslogk3" value="%X{syslogk3}" />
				<KeyValuePair key="syslogk4" value="%X{syslogk4}" />
				<KeyValuePair key="syslogk5" value="%X{syslogk5}" />
			</LoggerFields>
		</Syslog>
		<Syslog name="appender2" format="RFC5424" host="syslogserverip"
			port="515" protocol="TCP" appName="myappname" messageId="myfilename2"
			facility="LOCAL2" newLine="true" mdcId="mdc" includeMDC="true" 
			newLineEscape="#NewLine#" exceptionPattern="%ex"
			immediateFlush="false" connectTimeoutMillis="3000" reconnectionDelayMillis="3000">
			<LoggerFields>
				<KeyValuePair key="syslogk1" value="%l" />
				<KeyValuePair key="syslogk2" value="%X{syslogk2}" />
				<KeyValuePair key="syslogk3" value="%X{syslogk3}" />
				<KeyValuePair key="syslogk4" value="%X{syslogk4}" />
				<KeyValuePair key="syslogk5" value="%X{syslogk5}" />
			</LoggerFields>
		</Syslog>
	</Appenders>
	<Loggers>
	    <Logger name="com.liebao.yunwei" level="debug" additivity="false">
	        <AppenderRef ref="appender1" />
	    </Logger>
            <Root  level="debug">
	        <AppenderRef ref="appender2" />
	    </Root>
	</Loggers>
</Configuration>