syslog-ng构建集中型日志服务器

RHEL5 下、Centos 5.5 x86_64下均测试通过。

在生产环境中，存在一台日志服务器，专门用来记录其他服务器的日志信息是个很好的主意，不过用红帽自带的syslog，配置虽然简单，但是日志却没有办法分离，默认都堆在/var/log/message文件里面，用来超不爽，下面来介绍下用syslog-ng来构建日志服务器，这个还支持将日志导入数据库和通过网页来发布日志，听起来功能相当的强大，接下来要好好的研究下咯……

环境介绍
日志服务器IP：192.168.90.20；客户端IP：192.168.90.10
系统：RHEL5.4
实现目标：将客户端的日志自动保存在服务器端的相应目录，并根据日期，IP地址和日志类型进行分开保存
备注：由于在虚拟机环境下操作，服务器于客户端时间未同步，所以会存在记录日志时间不一致的现象；

[root@server2 ~]#mkdir -p /usr/local/src/tarbag/
[root@server2 ~]# cd /usr/local/src/tarbag/
[root@server2 tarbag]# wget http://www.balabit.com/downloads/files/eventlog/0.2/eventlog_0.2.9.tar.gz
[root@server2 tarbag]# tar -zxvf eventlog_0.2.9.tar.gz -C ../software/
[root@server2 tarbag]# cd ../software/eventlog-0.2.9/
[root@server2 eventlog-0.2.9]# ./configure  --prefix=/usr/local/eventlog && make && make install
[root@server2 eventlog-0.2.9]# ls /usr/local/eventlog/
include   lib

[root@server2 syslog-ng-3.0.5]# cd -
/usr/local/src/tarbag
[root@server2 tarbag]# wgethttp://www.balabit.com/downloads/files/libol/0.3/libol-0.3.9.tar.gz
[root@server2 tarbag]# tar -zxvf libol-0.3.9.tar.gz -C ../software/
[root@server2 tarbag]# cd ../software/libol-0.3.9/
[root@server2 libol-0.3.9]# ./configure --prefix=/usr/local/libol &&make && make install
[root@server2 libol-0.3.9]# ls /usr/local/libol/
bin   include   lib

[root@server2 tarbag]# wgethttp://www.balabit.com/downloads/files/syslog-ng/sources/3.0.5/source/syslog-ng_3.0.5.tar.gz
[root@server2 tarbag]# tar -zxvf syslog-ng_3.0.5.tar.gz -C ../software/
[root@server2 tarbag]# cd ../software/syslog-ng-3.0.5/
[root@server2 syslog-ng-3.0.5]#   exportPKG_CONFIG_PATH=/usr/local/eventlog/lib/pkgconfig
[root@server2 syslog-ng-3.0.5]# ./configure --prefix=/usr/local/syslog-ng--with-libol=/usr/local/libol && make && make install
configure: error: Cannot find eventlog version >= 0.2: is pkg-config inpath? (若出现这个错误，基本上是由于前面的PKG_CONFIG_PATH变量没指定好)
[root@server2 syslog-ng-3.0.5]# ls /usr/local/syslog-ng/
bin   libexec   sbin   share
[root@server2 syslog-ng-3.0.5]# mkdir /usr/local/syslog-ng/etc
[root@server2 syslog-ng-3.0.5]# mkdir /usr/local/syslog-ng/var
[root@server2 syslog-ng-3.0.5]# cp contrib/syslog-ng.conf.RedHat  /usr/local/syslog-ng/etc/
[root@server2 syslog-ng-3.0.5]# cp contrib/init.d.RedHat /etc/init.d/syslog-ng

[root@server2 syslog-ng-3.0.5]# cd /usr/local/syslog-ng/etc/
[root@server2 etc]# mv syslog-ng.conf.RedHat syslog-ng.conf

[root@server2 etc]# cat syslog-ng.conf                 #复制以下配置

[root@server2 etc]

@version:3.0
options {
long_hostnames(off);
log_msg_size(8192);
flush_lines(1);
log_fifo_size(20480);
time_reopen(10);
use_dns(yes);
dns_cache(yes);
use_fqdn(yes);
keep_hostname(yes);
chain_hostnames(no);
perm(0644);
stats_freq(43200);
};
source s_internal { internal(); };
destination d_syslognglog { file("/var/log/syslog-ng.log"); };
log { source(s_internal); destination(d_syslognglog); };

source s_local {
        unix-dgram("/dev/log");
         file("/proc/kmsg"program_override("kernel:"));
};

filter f_messages { level(info..emerg); };   #定义7种日志类型
filter f_secure { facility(authpriv); };
filter f_mail { facility(mail); };
filter f_cron { facility(cron); };
filter f_emerg { level(emerg); };
filter f_spooler { level(crit..emerg) and facility(uucp, news); };
filter f_local7 { facility(local7); };
destination d_messages { file("/var/log/messages"); };  #定义7种类型日志在客户端的位置
destination d_secure { file("/var/log/secure"); };
destination d_maillog { file("/var/log/maillog"); };
destination d_cron { file("/var/log/cron"); };
destination d_console { usertty("root"); };
destination d_spooler { file("/var/log/spooler"); };
destination d_bootlog { file("/var/log/dmesg"); };
log { source(s_local); filter(f_emerg); destination(d_console); };
log { source(s_local); filter(f_secure); destination(d_secure); flags(final);};
log { source(s_local); filter(f_mail); destination(d_maillog); flags(final); };
log { source(s_local); filter(f_cron); destination(d_cron); flags(final); };
log { source(s_local); filter(f_spooler); destination(d_spooler); };
log { source(s_local); filter(f_local7); destination(d_bootlog); };
log { source(s_local); filter(f_messages); destination(d_messages); };

# Remote logging  #定义监听的端口
source s_remote {
         tcp(ip(0.0.0.0)port(514));
         udp(ip(0.0.0.0) port(514));
};
#定义客户端日志在服务器上保存的格式，位置和权限等
destination r_console{file("/var/log/syslog-ng/$YEAR$MONTH$DAY/$HOST/console"owner("root") group("root") perm(0640) dir_perm(0750)create_dirs(yes));};
destination r_secure{file("/var/log/syslog-ng/$YEAR$MONTH$DAY/$HOST/secure" owner("root")group("root") perm(0640) dir_perm(0750) create_dirs(yes));};
destination r_cron{file("/var/log/syslog-ng/$YEAR$MONTH$DAY/$HOST/cron"owner("root") group("root") perm(0640) dir_perm(0750)create_dirs(yes));};
destination r_spooler {file("/var/log/syslog-ng/$YEAR$MONTH$DAY/$HOST/spooler"owner("root") group("root") perm(0640) dir_perm(0750)create_dirs(yes));};
destination r_bootlog{file("/var/log/syslog-ng/$YEAR$MONTH$DAY/$HOST/bootlog"owner("root") group("root") perm(0640) dir_perm(0750)create_dirs(yes));};
destination r_messages{file("/var/log/syslog-ng/$YEAR$MONTH$DAY/$HOST/messages"owner("root") group("root") perm(0640) dir_perm(0750)create_dirs(yes));};
log { source(s_remote); filter(f_emerg); destination(r_console); };
log { source(s_remote); filter(f_secure); destination(r_secure); flags(final);};
log { source(s_remote); filter(f_cron); destination(r_cron); flags(final); };
log { source(s_remote); filter(f_spooler); destination(r_spooler); };
log { source(s_remote); filter(f_local7); destination(r_bootlog); };
log { source(s_remote); filter(f_messages); destination(r_messages); };

[root@server2 etc]# chmod +x /etc/init.d/syslog-ng
[root@server2 etc]# chkconfig --add syslog-ng
service syslog-ng does not support chkconfig(若出现该错误，请修改该脚本前四行如下)
[root@server2 etc]# head -4 /etc/init.d/syslog-ng //加services不是在usr下的etc
#!/bin/bash
#chkconifg: --add syslog-ng
#chkconfig: 2345 12 88
#Description: syslog-ng

该脚本还需要修改下面的三个位置
[root@server2 etc]# grep ‘PATH‘ /etc/init.d/syslog-ng
PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/syslog-ng/bin:/usr/local/syslog-ng/sbin
[root@server2 etc]# grep 'INIT' /etc/init.d/syslog-ng |head -2
INIT_PROG="/usr/local/syslog-ng/sbin/syslog-ng"    # Full path to daemon
INIT_OPTS="-f /usr/local/syslog-ng/etc/syslog-ng.conf"                      # options passed to daemon


[root@server2 etc]# service syslog-ng start // 注意cd /usr/local/syslog-ng/etc/
Starting syslog-ng:/usr/local/syslog-ng/sbin/syslog-ng: error while loading shared libraries:libevtlog.so.0: cannot open shared object file: No such file or directory
Starting Kernel Logger: 出现此错误是因为共享库链接没做好
[root@server2 etc]# ln -s /usr/local/eventlog/lib/* /lib/

出现下面的问题是因为主配置文件中缺少：@version:3.0这行
Starting syslog-ng: Configuration file has no version number, assumingsyslog-ng 2.1 format. Please add @version: maj.min to the beginning of thefile;

[root@server2 ~]# service syslog-ng start
Starting KernelLogger:                                    [   OK   ]
[root@server2 etc]# cat /var/log/syslog-ng.log
Oct 15 11:29:47 localhost.localdomain syslog-ng[28022]: syslog-ng starting up; version='3.0.5'
[root@server2 etc]# netstat -tpln|grep 514
tcp        0      0 0.0.0.0:514                 0.0.0.0:*                   LISTEN      28407/syslog-ng     

[root@server2 etc]# ps aux|grep syslog
root      2385  0.0  0.1   5908   608 ?        Ss   02:26   0:00 syslogd -m 0
root     28406  0.0  0.1  49176   596 ?        S    11:47   0:00 supervising syslog-ng                                                         
root     28407  0.0  0.4  53468  2100 ?        Ss   11:47   0:00 /usr/local/syslog-ng/sbin/syslog-ng -f /usr/local/syslog-ng/etc/syslog-ng.conf
root     28417  0.0  0.1  61156   720 pts/3    R+   11:48   0:00 grep syslog

开启成功

客户端配置：
[root@client ~]#tail -1 /etc/syslog.conf
*.*                                                     @192.168.90.20
[root@client ~]# service syslog restart
[root@client ~]# logger -i just one test
[root@client ~]# tail -1 /var/log/messages
Oct 15 11:38:45 localhost kernel: klogd 1.4.1, log source = /proc/kmsg started.
[root@server2 ~]# cat /var/log/syslog-ng/20121015/192.168.90.10/messages
Oct 15 11:38:50 192.168.12.138 root[24984]: just one test

参考网站：http://blog.sina.com.cn/s/blog_4a071ed80100cssu.html

前面配置好了syslog-ng,下面简要的概述下如何将系统日志存入mysql
1：将mysql的头文件和库文件链接到/usr/local下
[root@server2 ~]# ln -s /usr/local/mysql/lib/mysql /usr/local/lib/mysql
[root@server2 ~]# ln -s /usr/local/mysql/include/mysql/ /usr/local/include
[root@server2 ~]# cd /usr/local/src/software/sqlsyslogd
2：下载sqlsyslogd源码包，由于是整个目录下载，所以会下载index.html打头的索引文件
[root@server2 software]# wget -d -r -np http://www.frasunek.com/sources/security/sqlsyslogd/
[root@server2 software]# cd www.frasunek.com/sources/security/sqlsyslogd/
[root@server2 sqlsyslogd]# rm -rf index.html*
[root@server2 sqlsyslogd]# cd contrib/
[root@server2 contrib]# rm -rf index.html*
[root@server2 contrib]# cd
[root@server2 ~]# mv/usr/local/src/software/www.frasunek.com/sources/security/sqlsyslogd/ /usr/
local/src/software/
3:make,复制sqlsyslogd二进制程序到/usr/local/sbin目录下
[root@server2 ~]# cd /usr/local/src/software/sqlsyslogd/
[root@server2 sqlsyslogd]# make
cc -O6 -Wall -pipe -I/usr/local/include-DCONF=\"/usr/local/etc/sqlsyslogd.conf\" -L/usr/local/lib/mysql-lmysqlclient sqlsyslogd.c   -o sqlsyslogd
[root@server2 sqlsyslogd]# cp sqlsyslogd /usr/local/sbin/
4：执行下sqlsyslogd程序，出现下面的命令选项则说明安装成功
[root@server2 sqlsyslogd]# sqlsyslogd
usage: sqlsyslogd [-h hostname] <-u username> [-p] <-t table>[database]

5：修改/etc/ld.so.conf文件，并使其生效，这个文件维护着编译的动态链接库位置
[root@server2sqlsyslogd]# cat /etc/ld.so.conf
include ld.so.conf.d/*.conf
/usr/local/lib/mysql
[root@server2 sqlsyslogd]# ldconfig

6：在数据库中创建相应的库和表
[root@server2sqlsyslogd]# mysql
Welcome to the MySQL monitor. Commands end with ; or \g.
Your MySQL connection id is 158
Server version: 5.1.36-log Source distribution

Type 'help;' or '\h' for help. Type'\c' to clear the current input statement.

mysql> create database syslog;
Query OK, 1 row affected (0.00 sec)

mysql> use syslog
Database changed
mysql> create table logs (Id int(10) NOT NULL auto_increment,Timestampvarchar(16),Host varchar(50),Prog varchar(50),Mesg text,PRIMARY KEY (id));
Query OK, 0 rows affected (0.01 sec)

mysql> exit
Bye

7：该文件定义了连接数据库的密码
[root@server2sqlsyslogd]# cat /usr/local/etc/sqlsyslogd.conf
123456

8：在syslog-ng主配置文件中添加下列配置
[root@server2sqlsyslogd]# vi /usr/local/syslog-ng/etc/syslog-ng.conf

destination sqlsyslogd{
program("/usr/local/sbin/sqlsyslogd -u root -t logs syslog -p");
};

log {
        source(s_remote);
        destination(sqlsyslogd);
};

9:重启syslog-ng服务
[root@server2sqlsyslogd]# service syslog-ng restart
Stopping KernelLogger:                                   [ OK ]
Starting KernelLogger:                                   [ OK ]

10:客户端发消息测试
[root@server2 sqlsyslogd]#tail -1 /var/log/syslog-ng/20100226/192.168.90.1/messages
Feb 26 14:25:47 192.168.90.1 root[6058]: just for fun