先对logstash进行安装测试和配置
[root@localhost software]# java -version
openjdk version "1.8.0_65"
OpenJDK Runtime Environment (build 1.8.0_65-b17)
OpenJDK 64-Bit Server VM (build 25.65-b01, mixed mode)
[root@localhost software]#
[root@localhost software]# pwd
/software
[root@localhost software]#
[root@localhost software]# wget https://download.elastic.co/logstash/logstash/packages/centos/logstash-all-plugins-2.4.0.noarch.rpm
--2016-09-20 16:17:18-- https://download.elastic.co/logstash/logstash/packages/centos/logstash-all-plugins-2.4.0.noarch.rpm
正在解析主机 download.elastic.co (download.elastic.co)... 23.23.240.27, 23.21.83.64, 107.22.244.203, ...
正在连接 download.elastic.co (download.elastic.co)|23.23.240.27|:443... 已连接。
已发出 HTTP 请求，正在等待回应... 200 OK
长度：140945933 (134M) [application/octet-stream]
正在保存至: “logstash-all-plugins-2.4.0.noarch.rpm”
100%[===============================================================>] 140,945,933 262KB/s 用时 8m 53s
2016-09-20 16:26:21 (258 KB/s) - 已保存 “logstash-all-plugins-2.4.0.noarch.rpm” [140945933/140945933])
[root@localhost software]# ls
logstash-all-plugins-2.4.0.noarch.rpm
[root@localhost software]# chmod +x logstash-all-plugins-2.4.0.noarch.rpm
[root@localhost software]# rpm -ivh logstash-all-plugins-2.4.0.noarch.rpm
准备中... ################################# [100%]
正在升级/安装...
1:logstash-all-plugins-1:2.4.0-1 ################################# [100%]
[root@localhost software]#
[root@localhost software]# cd ~
[root@localhost ~]# service logstash start
logstash started.
[root@localhost ~]#
[root@localhost ~]# whereis logstash
logstash: /etc/logstash /opt/logstash/bin/logstash /opt/logstash/bin/logstash.bat
[root@localhost ~]# ls /etc/logstash/
conf.d
[root@localhost ~]# ls /etc/logstash/conf.d/
[root@localhost opt]# ll /opt/logstash/bin
总用量 44
-rwxrwxr-x. 1 logstash logstash 1854 8月 31 04:36 logstash
-rw-rw-r--. 1 logstash logstash 689 8月 31 04:36 logstash.bat
-rwxrwxr-x. 1 logstash logstash 5330 8月 31 04:36 logstash.lib.sh
-rwxrwxr-x. 1 logstash logstash 439 8月 31 04:36 logstash-plugin
-rw-rw-r--. 1 logstash logstash 251 8月 31 04:36 logstash-plugin.bat
-rwxrwxr-x. 1 logstash logstash 199 8月 31 04:36 plugin
-rw-rw-r--. 1 logstash logstash 203 8月 31 04:36 plugin.bat
-rwxrwxr-x. 1 logstash logstash 322 8月 31 04:36 rspec
-rw-rw-r--. 1 logstash logstash 245 8月 31 04:36 rspec.bat
-rw-rw-r--. 1 logstash logstash 2947 8月 31 04:36 setup.bat
[root@localhost opt]# ll /opt/logstash
总用量 164
drwxr-xr-x. 2 logstash logstash 4096 9月 20 16:36 bin
-rw-rw-r--. 1 logstash logstash 102879 8月 31 04:36 CHANGELOG.md
-rw-rw-r--. 1 logstash logstash 2249 8月 31 04:36 CONTRIBUTORS
-rw-rw-r--. 1 logstash logstash 7799 8月 31 04:36 Gemfile
-rw-rw-r--. 1 logstash logstash 34057 8月 31 04:36 Gemfile.jruby-1.9.lock
drwxr-xr-x. 4 logstash logstash 42 9月 20 16:36 lib
-rw-rw-r--. 1 logstash logstash 589 8月 31 04:36 LICENSE
-rw-rw-r--. 1 logstash logstash 149 8月 31 04:36 NOTICE.TXT
drwxr-xr-x. 4 logstash logstash 31 9月 20 16:36 vendor
[root@localhost opt]#
[root@localhost opt]# cd /etc/logstash/conf.d/
[root@localhost conf.d]#
[root@localhost conf.d]# ls
first.conf
[root@localhost conf.d]# cat first.conf
input {
stdin { }
}
output {
stdout {
codec => rubydebug
}
}
[root@localhost conf.d]#
[root@localhost conf.d]# chmod +x ./*
[root@localhost conf.d]# /opt/logstash/bin/logstash -f first.conf
Hello
Settings: Default pipeline workers: 1
Pipeline main started
{
"message" => "Hello",
"@version" => "1",
"@timestamp" => "2016-09-20T08:48:37.097Z",
"host" => "localhost.localdomain"
}
[root@localhost software]# cd /opt
[root@localhost opt]# cd logstash/
[root@localhost logstash]# ls
bin CHANGELOG.md CONTRIBUTORS Gemfile Gemfile.jruby-1.9.lock lib LICENSE NOTICE.TXT vendor
[root@localhost logstash]# bin/plugin install logstash-output-webhdfs
The use of bin/plugin is deprecated and will be removed in a feature release. Please use bin/logstash-plugin.
Validating logstash-output-webhdfs
Installing logstash-output-webhdfs
Installation successful
[root@localhost logstash]# bin/plugin install logstash-filter-elasticsearch
The use of bin/plugin is deprecated and will be removed in a feature release. Please use bin/logstash-plugin.
Validating logstash-filter-elasticsearch
Installing logstash-filter-elasticsearch
Installation successful
[root@localhost logstash]#
[root@localhost logstash]# bin/plugin install logstash-input-beats
The use of bin/plugin is deprecated and will be removed in a feature release. Please use bin/logstash-plugin.
Validating logstash-input-beats
Installing logstash-input-beats
Installation successful
[root@localhost logstash]# bin/plugin install logstash-input-kafka
The use of bin/plugin is deprecated and will be removed in a feature release. Please use bin/logstash-plugin.
Validating logstash-input-kafka
Installing logstash-input-kafka
Installation successful
[root@localhost logstash]# bin/plugin install logstash-output-lumberjack
The use of bin/plugin is deprecated and will be removed in a feature release. Please use bin/logstash-plugin.
Validating logstash-output-lumberjack
Installing logstash-output-lumberjack
Installation successful
[root@localhost logstash]# bin/plugin install logstash-output-kafka
The use of bin/plugin is deprecated and will be removed in a feature release. Please use bin/logstash-plugin.
Validating logstash-output-kafka
Installing logstash-output-kafka
Installation successful
[root@localhost logstash]# bin/plugin install logstash-input-file
The use of bin/plugin is deprecated and will be removed in a feature release. Please use bin/logstash-plugin.
Validating logstash-input-file
Installing logstash-input-file
Installation successful
[root@localhost logstash]# bin/plugin install logstash-input-syslog
The use of bin/plugin is deprecated and will be removed in a feature release. Please use bin/logstash-plugin.
Validating logstash-input-syslog
Installing logstash-input-syslog
Installation successful
[root@localhost logstash]# bin/plugin install logstash-output-file
The use of bin/plugin is deprecated and will be removed in a feature release. Please use bin/logstash-plugin.
Validating logstash-output-file
Installing logstash-output-file
Installation successful
[root@localhost logstash]# bin/plugin install logstash-input-tcp
The use of bin/plugin is deprecated and will be removed in a feature release. Please use bin/logstash-plugin.
Validating logstash-input-tcp
Installing logstash-input-tcp
Installation successful
[root@localhost logstash]# cd /software/
[root@localhost software]#
[root@localhost software]# wget https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/rpm/elasticsearch/2.4.0/elasticsearch-2.4.0.rpm
--2016-09-20 18:10:09-- https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/rpm/elasticsearch/2.4.0/elasticsearch-2.4.0.rpm
正在解析主机 download.elastic.co (download.elastic.co)... 23.23.240.27, 107.22.226.93, 107.22.240.120, ...
正在连接 download.elastic.co (download.elastic.co)|23.23.240.27|:443... 已连接。
已发出 HTTP 请求，正在等待回应... 200 OK
长度：27256532 (26M) [application/x-rpm]
正在保存至: “elasticsearch-2.4.0.rpm”
100%[===============================================================>] 27,256,532 160KB/s 用时 2m 51s
2016-09-20 18:13:09 (156 KB/s) - 已保存 “elasticsearch-2.4.0.rpm” [27256532/27256532])
[root@localhost software]# chmod +x elasticsearch-2.4.0.rpm
[root@localhost software]# rpm -ivh elasticsearch-2.4.0.rpm
警告：elasticsearch-2.4.0.rpm: 头V4 RSA/SHA1 Signature, 密钥 ID d88e42b4: NOKEY
准备中... ################################# [100%]
Creating elasticsearch group... OK
Creating elasticsearch user... OK
正在升级/安装...
1:elasticsearch-2.4.0-1 ################################# [100%]
### NOT starting on installation, please execute the following statements to configure elasticsearch service to start automatically using systemd
sudo systemctl daemon-reload
sudo systemctl enable elasticsearch.service
### You can start elasticsearch service by executing
sudo systemctl start elasticsearch.service
[root@localhost software]# sudo systemctl enable elasticsearch.service
Created symlink from /etc/systemd/system/multi-user.target.wants/elasticsearch.service to /usr/lib/systemd/system/elasticsearch.service.
[root@localhost software]# service elasticsearch start
Starting elasticsearch (via systemctl): [ 确定 ]
[root@localhost software]#
[root@localhost software]# cd /etc/logstash/conf.d
[root@localhost conf.d]# cat first.conf
input {
file {
path => ["/var/log/*log","/var/log/message"]
type => "system"
start_position => "beginning"
}
}
output {
stdout {
codec => rubydebug
}
}
[root@localhost conf.d]# /opt/logstash/bin/logstash -f first.conf
..........日志输入......
————————————————————我是分割线——————————————————————————————————
收集本地日志
[root@localhost bin]# cd /etc/logstash/conf.d/
[root@localhost conf.d]# ls
collect_email_svr_log.conf collect_lenovobook_log.conf collect_local_log.conf
[root@localhost conf.d]# cat collect_local_log.conf
input {
file {
path => ["/var/log/*log","/var/log/message"]
type => "system"
start_position => "beginning"
}
}
output {
file {
path => "/coll_log/localhost/%{+yyyy}/%{+MM}/%{+dd}/%{host}.log"
message_format => "%{message}"
#gzip => true
}
}
[root@localhost conf.d]#
————————————————————我是分割线——————————————————————————————————
[root@localhost conf.d]# cd /opt/logstash/
[root@localhost logstash]# ls
bin CHANGELOG.md CONTRIBUTORS Gemfile Gemfile.jruby-1.9.lock lib LICENSE NOTICE.TXT vendor
[root@localhost logstash]# bin/plugin list
The use of bin/plugin is deprecated and will be removed in a feature release. Please use bin/logstash-plugin.
logstash-codec-avro
logstash-codec-cef
logstash-codec-cloudfront
logstash-codec-collectd
logstash-codec-compress_spooler
logstash-codec-csv
logstash-codec-dots
logstash-codec-edn
logstash-codec-edn_lines
logstash-codec-es_bulk
logstash-codec-fluent
logstash-codec-graphite
logstash-codec-gzip_lines
logstash-codec-json
logstash-codec-json_lines
logstash-codec-line
logstash-codec-msgpack
logstash-codec-multiline
logstash-codec-netflow
logstash-codec-oldlogstashjson
logstash-codec-plain
logstash-codec-protobuf
logstash-codec-rubydebug
logstash-codec-s3plain
logstash-codec-sflow
logstash-filter-aggregate
logstash-filter-alter
logstash-filter-anonymize
logstash-filter-checksum
logstash-filter-cidr
logstash-filter-cipher
logstash-filter-clone
logstash-filter-collate
logstash-filter-csv
logstash-filter-date
logstash-filter-de_dot
logstash-filter-dns
logstash-filter-drop
logstash-filter-elapsed
logstash-filter-elasticsearch
logstash-filter-environment
logstash-filter-extractnumbers
logstash-filter-fingerprint
logstash-filter-geoip
logstash-filter-grok
logstash-filter-i18n
logstash-filter-json
logstash-filter-json_encode
logstash-filter-kv
logstash-filter-metaevent
logstash-filter-metricize
logstash-filter-metrics
logstash-filter-multiline
logstash-filter-mutate
logstash-filter-oui
logstash-filter-prune
logstash-filter-punct
logstash-filter-range
logstash-filter-ruby
logstash-filter-sleep
logstash-filter-split
logstash-filter-syslog_pri
logstash-filter-throttle
logstash-filter-tld
logstash-filter-translate
logstash-filter-unique
logstash-filter-urldecode
logstash-filter-useragent
logstash-filter-uuid
logstash-filter-xml
logstash-filter-zeromq
logstash-input-beats
logstash-input-cloudwatch
logstash-input-couchdb_changes
logstash-input-elasticsearch
logstash-input-eventlog
logstash-input-exec
logstash-input-file
logstash-input-fluentd
logstash-input-ganglia
logstash-input-gelf
logstash-input-gemfire
logstash-input-generator
logstash-input-github
logstash-input-graphite
logstash-input-heartbeat
logstash-input-http
logstash-input-http_poller
logstash-input-imap
logstash-input-irc
logstash-input-jdbc
logstash-input-jmx
logstash-input-kafka
logstash-input-kinesis
logstash-input-log4j
logstash-input-lumberjack
logstash-input-meetup
logstash-input-pipe
logstash-input-puppet_facter
logstash-input-rabbitmq
logstash-input-redis
logstash-input-relp
logstash-input-rss
logstash-input-s3
logstash-input-salesforce
logstash-input-snmptrap
logstash-input-sqlite
logstash-input-sqs
logstash-input-stdin
logstash-input-stomp
logstash-input-syslog
logstash-input-tcp
logstash-input-twitter
logstash-input-udp
logstash-input-unix
logstash-input-varnishlog
logstash-input-websocket
logstash-input-wmi
logstash-input-xmpp
logstash-input-zenoss
logstash-input-zeromq
logstash-output-boundary
logstash-output-circonus
logstash-output-cloudwatch
logstash-output-csv
logstash-output-datadog
logstash-output-datadog_metrics
logstash-output-elasticsearch
logstash-output-elasticsearch-ec2
logstash-output-elasticsearch_java
logstash-output-email
logstash-output-exec
logstash-output-file
logstash-output-ganglia
logstash-output-gelf
logstash-output-graphite
logstash-output-graphtastic
logstash-output-hipchat
logstash-output-http
logstash-output-influxdb
logstash-output-irc
logstash-output-juggernaut
logstash-output-kafka
logstash-output-librato
logstash-output-loggly
logstash-output-lumberjack
logstash-output-metriccatcher
logstash-output-monasca_log_api
logstash-output-mongodb
logstash-output-nagios
logstash-output-nagios_nsca
logstash-output-null
logstash-output-opentsdb
logstash-output-pagerduty
logstash-output-pipe
logstash-output-rabbitmq
logstash-output-rados
logstash-output-redis
logstash-output-redmine
logstash-output-riemann
logstash-output-s3
logstash-output-sns
logstash-output-solr_http
logstash-output-sqs
logstash-output-statsd
logstash-output-stdout
logstash-output-stomp
logstash-output-syslog
logstash-output-tcp
logstash-output-udp
logstash-output-webhdfs
logstash-output-websocket
logstash-output-xmpp
logstash-output-zabbix
logstash-output-zeromq
logstash-output-zookeeper
logstash-patterns-core
[root@localhost logstash]#
————————————————————————————————我是分割线——————————————————————————————
收取远程linux主机日志记录
[root@localhost ~]# cd /etc/logstash/
[root@localhost logstash]# ls
conf.d
[root@localhost logstash]# cd conf.d/
[root@localhost conf.d]# ls
collect_email_svr_log.conf collect_lenovobook_log.conf collect_local_log.conf
[root@localhost conf.d]# vi collect_lenovobook_log.conf
[root@localhost conf.d]# vi collect_lenovobook_log.conf
[root@localhost conf.d]# cat collect_lenovobook_log.conf
input {
beats {
port => 5044
}
}
output {
file {
path => "/coll_log/collect_lenovobook_log/%{+yyyy}/%{+MM}/%{+dd}/%{host}.log"
message_format => "%{message}"
#gzip => true
}
}
[root@localhost conf.d]#
——————————————————————————————————我是分割线——————————————————————————
http://blog.csdn.net/fenglailea/article/details/52469671
[root@localhost bin]# ./filebeat -e -c /etc/filebeat/filebeat.yml -d "Publish"
[root@localhost filebeat]# cat filebeat.yml
################### Filebeat Configuration Example #########################
############################# Filebeat ######################################
filebeat:
# List of prospectors to fetch data.
prospectors:
# Each - is a prospector. Below are the prospector specific configurations
-
# Paths that should be crawled and fetched. Glob based paths.
# To fetch all ".log" files from a specific level of subdirectories
# /var/log/*/*.log can be used.
# For each file found under this path, a harvester is started.
# Make sure not file is defined twice as this can lead to unexpected behaviour.
paths:
- /var/log/*.log
#- c:\programdata\elasticsearch\logs\*
# Configure the file encoding for reading files with international characters
# following the W3C recommendation for HTML5 (http://www.w3.org/TR/encoding).
# Some sample encodings:
# plain, utf-8, utf-16be-bom, utf-16be, utf-16le, big5, gb18030, gbk,
# hz-gb-2312, euc-kr, euc-jp, iso-2022-jp, shift-jis, ...
#encoding: plain
# Type of the files. Based on this the way the file is read is decided.
# The different types cannot be mixed in one prospector
#
# Possible options are:
# * log: Reads every line of the log file (default)
# * stdin: Reads the standard in
input_type: log
# Exclude lines. A list of regular expressions to match. It drops the lines that are
# matching any regular expression from the list. The include_lines is called before
# exclude_lines. By default, no lines are dropped.
# exclude_lines: ["^DBG"]
# Include lines. A list of regular expressions to match. It exports the lines that are
# matching any regular expression from the list. The include_lines is called before
# exclude_lines. By default, all the lines are exported.
# include_lines: ["^ERR", "^WARN"]
# Exclude files. A list of regular expressions to match. Filebeat drops the files that
# are matching any regular expression from the list. By default, no files are dropped.
# exclude_files: [".gz$"]
# Optional additional fields. These field can be freely picked
# to add additional information to the crawled log files for filtering
#fields:
# level: debug
# review: 1
# Set to true to store the additional fields as top level fields instead
# of under the "fields" sub-dictionary. In case of name conflicts with the
# fields added by Filebeat itself, the custom fields overwrite the default
# fields.
#fields_under_root: false
# Ignore files which were modified more then the defined timespan in the past.
# In case all files on your system must be read you can set this value very large.
# Time strings like 2h (2 hours), 5m (5 minutes) can be used.
#ignore_older: 0
# Close older closes the file handler for which were not modified
# for longer then close_older
# Time strings like 2h (2 hours), 5m (5 minutes) can be used.
#close_older: 1h
# Type to be published in the 'type' field. For Elasticsearch output,
# the type defines the document type these entries should be stored
# in. Default: log
#document_type: log
# Scan frequency in seconds.
# How often these files should be checked for changes. In case it is set
# to 0s, it is done as often as possible. Default: 10s
#scan_frequency: 10s
# Defines the buffer size every harvester uses when fetching the file
#harvester_buffer_size: 16384
# Maximum number of bytes a single log event can have
# All bytes after max_bytes are discarded and not sent. The default is 10MB.
# This is especially useful for multiline log messages which can get large.
#max_bytes: 10485760
# Mutiline can be used for log messages spanning multiple lines. This is common
# for Java Stack Traces or C-Line Continuation
#multiline:
# The regexp Pattern that has to be matched. The example pattern matches all lines starting with [
#pattern: ^\[
# Defines if the pattern set under pattern should be negated or not. Default is false.
#negate: false
# Match can be set to "after" or "before". It is used to define if lines should be append to a pattern
# that was (not) matched before or after or as long as a pattern is not matched based on negate.
# Note: After is the equivalent to previous and before is the equivalent to to next in Logstash
#match: after
# The maximum number of lines that are combined to one event.
# In case there are more the max_lines the additional lines are discarded.
# Default is 500
#max_lines: 500
# After the defined timeout, an multiline event is sent even if no new pattern was found to start a new event
# Default is 5s.
#timeout: 5s
# Setting tail_files to true means filebeat starts readding new files at the end
# instead of the beginning. If this is used in combination with log rotation
# this can mean that the first entries of a new file are skipped.
#tail_files: false
# Backoff values define how agressively filebeat crawls new files for updates
# The default values can be used in most cases. Backoff defines how long it is waited
# to check a file again after EOF is reached. Default is 1s which means the file
# is checked every second if new lines were added. This leads to a near real time crawling.
# Every time a new line appears, backoff is reset to the initial value.
#backoff: 1s
# Max backoff defines what the maximum backoff time is. After having backed off multiple times
# from checking the files, the waiting time will never exceed max_backoff idenependent of the
# backoff factor. Having it set to 10s means in the worst case a new line can be added to a log
# file after having backed off multiple times, it takes a maximum of 10s to read the new line
#max_backoff: 10s
# The backoff factor defines how fast the algorithm backs off. The bigger the backoff factor,
# the faster the max_backoff value is reached. If this value is set to 1, no backoff will happen.
# The backoff value will be multiplied each time with the backoff_factor until max_backoff is reached
#backoff_factor: 2
# This option closes a file, as soon as the file name changes.
# This config option is recommended on windows only. Filebeat keeps the files it's reading open. This can cause
# issues when the file is removed, as the file will not be fully removed until also Filebeat closes
# the reading. Filebeat closes the file handler after ignore_older. During this time no new file with the
# same name can be created. Turning this feature on the other hand can lead to loss of data
# on rotate files. It can happen that after file rotation the beginning of the new
# file is skipped, as the reading starts at the end. We recommend to leave this option on false
# but lower the ignore_older value to release files faster.
#force_close_files: false
# Additional prospector
#-
# Configuration to use stdin input
#input_type: stdin
# General filebeat configuration options
#
# Event count spool threshold - forces network flush if exceeded
#spool_size: 2048
# Enable async publisher pipeline in filebeat (Experimental!)
#publish_async: false
# Defines how often the spooler is flushed. After idle_timeout the spooler is
# Flush even though spool_size is not reached.
#idle_timeout: 5s
# Name of the registry file. Per default it is put in the current working
# directory. In case the working directory is changed after when running
# filebeat again, indexing starts from the beginning again.
registry_file: /var/lib/filebeat/registry
# Full Path to directory with additional prospector configuration files. Each file must end with .yml
# These config files must have the full filebeat config part inside, but only
# the prospector part is processed. All global options like spool_size are ignored.
# The config_dir MUST point to a different directory then where the main filebeat config file is in.
#config_dir:
###############################################################################
############################# Libbeat Config ##################################
# Base config file used by all other beats for using libbeat features
############################# Output ##########################################
# Configure what outputs to use when sending the data collected by the beat.
# Multiple outputs may be used.
output:
### Elasticsearch as output
# elasticsearch:
# Array of hosts to connect to.
# Scheme and port can be left out and will be set to the default (http and 9200)
# In case you specify and additional path, the scheme is required: http://localhost:9200/path
# IPv6 addresses should always be defined as: https://[2001:db8::1]:9200
# hosts: ["172.19.122.99:9200"]
# Optional protocol and basic auth credentials.
#protocol: "https"
#username: "admin"
#password: "s3cr3t"
# Number of workers per Elasticsearch host.
#worker: 1
# Optional index name. The default is "filebeat" and generates
# [filebeat-]YYYY.MM.DD keys.
#index: "filebeat"
# A template is used to set the mapping in Elasticsearch
# By default template loading is disabled and no template is loaded.
# These settings can be adjusted to load your own template or overwrite existing ones
#template:
# Template name. By default the template name is filebeat.
#name: "filebeat"
# Path to template file
#path: "filebeat.template.json"
# Overwrite existing template
#overwrite: false
# Optional HTTP Path
#path: "/elasticsearch"
# Proxy server url
#proxy_url: http://proxy:3128
# The number of times a particular Elasticsearch index operation is attempted. If
# the indexing operation doesn't succeed after this many retries, the events are
# dropped. The default is 3.
#max_retries: 3
# The maximum number of events to bulk in a single Elasticsearch bulk API index request.
# The default is 50.
#bulk_max_size: 50
# Configure http request timeout before failing an request to Elasticsearch.
#timeout: 90
# The number of seconds to wait for new events between two bulk API index requests.
# If `bulk_max_size` is reached before this interval expires, addition bulk index
# requests are made.
#flush_interval: 1
# Boolean that sets if the topology is kept in Elasticsearch. The default is
# false. This option makes sense only for Packetbeat.
#save_topology: false
# The time to live in seconds for the topology information that is stored in
# Elasticsearch. The default is 15 seconds.
#topology_expire: 15
# tls configuration. By default is off.
#tls:
# List of root certificates for HTTPS server verifications
#certificate_authorities: ["/etc/pki/root/ca.pem"]
# Certificate for TLS client authentication
#certificate: "/etc/pki/client/cert.pem"
# Client Certificate Key
#certificate_key: "/etc/pki/client/cert.key"
# Controls whether the client verifies server certificates and host name.
# If insecure is set to true, all server host names and certificates will be
# accepted. In this mode TLS based connections are susceptible to
# man-in-the-middle attacks. Use only for testing.
#insecure: true
# Configure cipher suites to be used for TLS connections
#cipher_suites: []
# Configure curve types for ECDHE based cipher suites
#curve_types: []
# Configure minimum TLS version allowed for connection to logstash
#min_version: 1.0
# Configure maximum TLS version allowed for connection to logstash
#max_version: 1.2
output:
logstash:
enabled: true
hosts: ["172.19.122.99:5044"]
### Logstash as output
#logstash:
# The Logstash hosts
# hosts: ["172.19.122.99:5044"]
# Number of workers per Logstash host.
#worker: 1
# The maximum number of events to bulk into a single batch window. The
# default is 2048.
#bulk_max_size: 2048
# Set gzip compression level.
#compression_level: 3
# Optional load balance the events between the Logstash hosts
#loadbalance: true
# Optional index name. The default index name depends on the each beat.
# For Packetbeat, the default is set to packetbeat, for Topbeat
# top topbeat and for Filebeat to filebeat.
#index: filebeat
# Optional TLS. By default is off.
#tls:
# List of root certificates for HTTPS server verifications
#certificate_authorities: ["/etc/pki/root/ca.pem"]
# Certificate for TLS client authentication
#certificate: "/etc/pki/client/cert.pem"
# Client Certificate Key
#certificate_key: "/etc/pki/client/cert.key"
# Controls whether the client verifies server certificates and host name.
# If insecure is set to true, all server host names and certificates will be
# accepted. In this mode TLS based connections are susceptible to
# man-in-the-middle attacks. Use only for testing.
#insecure: true
# Configure cipher suites to be used for TLS connections
#cipher_suites: []
# Configure curve types for ECDHE based cipher suites
#curve_types: []
### File as output
#file:
# Path to the directory where to save the generated files. The option is mandatory.
#path: "/tmp/filebeat"
# Name of the generated files. The default is `filebeat` and it generates files: `filebeat`, `filebeat.1`, `filebeat.2`, etc.
#filename: filebeat
# Maximum size in kilobytes of each file. When this size is reached, the files are
# rotated. The default value is 10 MB.
#rotate_every_kb: 10000
# Maximum number of files under path. When this number of files is reached, the
# oldest file is deleted and the rest are shifted from last to first. The default
# is 7 files.
#number_of_files: 7
### Console output
# console:
# Pretty print json event
#pretty: false
############################# Shipper #########################################
shipper:
# The name of the shipper that publishes the network data. It can be used to group
# all the transactions sent by a single shipper in the web interface.
# If this options is not defined, the hostname is used.
#name:
# The tags of the shipper are included in their own field with each
# transaction published. Tags make it easy to group servers by different
# logical properties.
#tags: ["service-X", "web-tier"]
# Uncomment the following if you want to ignore transactions created
# by the server on which the shipper is installed. This option is useful
# to remove duplicates if shippers are installed on multiple servers.
#ignore_outgoing: true
# How often (in seconds) shippers are publishing their IPs to the topology map.
# The default is 10 seconds.
#refresh_topology_freq: 10
# Expiration time (in seconds) of the IPs published by a shipper to the topology map.
# All the IPs will be deleted afterwards. Note, that the value must be higher than
# refresh_topology_freq. The default is 15 seconds.
#topology_expire: 15
# Internal queue size for single events in processing pipeline
#queue_size: 1000
# Configure local GeoIP database support.
# If no paths are not configured geoip is disabled.
#geoip:
#paths:
# - "/usr/share/GeoIP/GeoLiteCity.dat"
# - "/usr/local/var/GeoIP/GeoLiteCity.dat"
############################# Logging #########################################
# There are three options for the log ouput: syslog, file, stderr.
# Under Windos systems, the log files are per default sent to the file output,
# under all other system per default to syslog.
logging:
# Send all logging output to syslog. On Windows default is false, otherwise
# default is true.
#to_syslog: true
# Write all logging output to files. Beats automatically rotate files if rotateeverybytes
# limit is reached.
#to_files: false
# To enable logging to files, to_files option has to be set to true
files:
# The directory where the log files will written to.
#path: /var/log/mybeat
# The name of the files where the logs are written to.
#name: mybeat
# Configure log file size limit. If limit is reached, log file will be
# automatically rotated
rotateeverybytes: 10485760 # = 10MB
# Number of rotated log files to keep. Oldest files will be deleted first.
#keepfiles: 7
# Enable debug output for selected components. To enable all selectors use ["*"]
# Other available selectors are beat, publish, service
# Multiple selectors can be chained.
#selectors: [ ]
# Sets log level. The default log level is error.
# Available log levels are: critical, error, warning, info, debug
#level: error
[root@localhost filebeat]#
——————————————————————————————我还是分割线————————————————————————————————————————————————
windows主机日志收集
下载nxlog软件，安装步骤省略，在系统服务中启动nxlog服务
http://nxlog.org/system/files/products/files/1/nxlog-ce-2.9.1716.msi
windows端配置文件修改如下，linux日志服务器地址为 172.19.122.99
## This is a sample configuration file. See the nxlog reference manual about the
## configuration options. It should be installed locally and is also available
## online at http://nxlog.org/docs/
## Please set the ROOT to the folder your nxlog was installed into,
## otherwise it will not start.
#define ROOT C:\Program Files\nxlog
define ROOT C:\Program Files (x86)\nxlog
Moduledir %ROOT%\modules
CacheDir %ROOT%\data
Pidfile %ROOT%\data\nxlog.pid
SpoolDir %ROOT%\data
LogFile %ROOT%\data\nxlog.log
<Extension _syslog>
Module xm_syslog
</Extension>
<Input in>
Module im_msvistalog
# For windows 2003 and earlier use the following:
# Module im_mseventlog
</Input>
<Output out>
Module om_tcp
Host 172.19.122.99
Port 514
Exec to_syslog_snare();
</Output>
<Route 1>
Path in => out
</Route>
logstash服务器端配置
[root@localhost conf.d]# cat win.conf
input {
tcp {
port => 514 }
}
output {
file {
path => "/coll_log/collect_Mywindows10_log/%{+yyyy}/%{+MM}/%{+dd}/%{host}.log"
message_format => "%{message}"
#gzip => true
}
}
917
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
