rSyslog + logstash 安装部署 [多端口多目录]

syslog 部署

1. 修改syslog 配置文件

vi /etc/rsyslog.conf
添加：

$ModLoad imfile #装载imfile模块
$InputFileName /data/src/wb.log #syslog追加日志的文件
$InputFileFacility local1 #日志类型
$InputFileTag wbtag:  # 日志类型标签
$InputFileSeverity info #日志等级
$InputFileStateFile sssologs.log_state #定义记录偏移量数据文件名
$InputFilePollInterval 1 #检查日志文件间隔（秒）
$InputFilePersistStateInterval 1 #回写偏移量数据到文件间隔时间（秒）
$InputRunFileMonitor #激活读取，可以设置多组日志读取，每组结束时设置本参数。以示生效。

\# 接收日志的远程地址
local1.*                             @@IP:port

2. 启动服务

systemctl start rsyslog【.service】

3. 测试

echo >> /data/src/wb.log
查看 logstash 日志，数据已加载过来

logstash 部署

1. 文件解压

1.	解压：tar -xvf logstash-7.9.3.tar.gz -C /opt
2.	文件重命名：
	cd /usr/local/
	mv logstash-7.9.3 logstash

2. 配置文件修改

cd /usr/local/logstash/config
vi liufangyun.conf ，添加如下内容

input {
	# 配置多个即可
	syslog { 
		host => "0.0.0.0" 
		add_field => { 
			"[@metadata][company][name]" => "syslog-10514" 
		} 
		port => 10514 
		codec => plain { 
			charset => "UTF-8" 
		} 
	 } 

	syslog { 
		host => "0.0.0.0" 
		add_field => { 
			"[@metadata][company][name]" => "syslog-10515" 
		} 
		port => 10515 
		codec => plain { 
			charset => "UTF-8" 
		} 
	 } 
}


output {

	file{
		path => "/opt/boxdata/data/syslog/%{[@metadata][company][name]}/%{+YYYYMMdd}%{+HH}.log"
	}

}

3. 服务启动 [后台]

/usr/local/logstash/bin/logstash --path.settings /usr/local/logstash/config -f /usr/local/logstash/config/liufangyun.conf &

备注:logstash部分配置使用python 脚本生成

path = "D:/company/com-notes/project/smallbox2021.1.12/logstash_conf/input.txt"
with open(path,'a+') as f:
    for i in range(10514,10547):
        # input
        '''
        str1 = r"input { syslog { host => '0.0.0.0' type => 'syslog-"+ str(i) + r"' port => " + str(i) + r" codec => plain {charset => 'GB2312'}}}"
        print(str1)
        f.write(str1 + '\n')
        '''

        # filter

        str2 = rf"if[type] == \"syslog-{i}\" " \
                r"{" + '\n'\
                r"	mutate {" + '\n'\
                rf"		add_tag => [\"syslog-{i}\"]" + '\n'\
                r"	}" + '\n'\
                r"	ruby { " + '\n'\
                r"		code => \"event.set('timestamp', event.get('@timestamp').time.localtime + 8*60*60)\" "  + '\n'\
                r"	}" + '\n' \
                r"	ruby {" + '\n'\
                r"		code => \"event.set('@timestamp',event.get('timestamp'))\"" + '\n'\
                r"	}" + '\n'\
                r"	mutate {" + '\n'\
                r"		remove_field => [\"timestamp\"]" + '\n'\
                r"	}" + '\n'                                                                                                                                                                                                                                                                                                                                                          r"}"
    
        # print(str2.replace('\\\"','"') + "\n")
        # f.write(str2.replace('\\\"','"') + '\n\n')


        # output

        str3 =rf"if \"syslog-{i}\" in [tags] "\
              r"{" + '\n'\
                r"	file{" + '\n'\
                rf"		path => \"/opt/boxdata/data/syslog/syslog-{i}/syslog_{i}"\
                r"_%{+YYYYMMdd}%{+HH}.log\"" + '\n'\
                r"	}" + '\n'\
                r"}"

        # print(str3.replace('\\\"','"') + '\n\n')
        f.write(str3.replace('\\\"','"') + '\n\n')

    f.close()

备注：logstash 监听接收32个端口详细配置文件网址

https://blog.csdn.net/a123147abc/article/details/112621210