fluentd安装

Ruby安装

---

fluentd安装之前

ulimit -n
65535

如果是1024修改/etc/security/limits.conf并重启

root soft nofile 65536
root hard nofile 65536
* soft nofile 65536
* hard nofile 65536

修改/etc/sysctl.conf

net.core.somaxconn = 1024
net.core.netdev_max_backlog = 5000
net.core.rmem_max = 16777216
net.core.wmem_max = 16777216
net.ipv4.tcp_wmem = 4096 12582912 16777216
net.ipv4.tcp_rmem = 4096 12582912 16777216
net.ipv4.tcp_max_syn_backlog = 8096
net.ipv4.tcp_slow_start_after_idle = 0
net.ipv4.tcp_tw_reuse = 1
net.ipv4.ip_local_port_range = 10240 65535

使其生效并重启

sysctl -p

Red Hat / Centos 安装

# td-agent 4
$ curl -L https://toolbelt.treasuredata.com/sh/install-redhat-td-agent4.sh | sh

# td-agent 3
$ curl -L https://toolbelt.treasuredata.com/sh/install-redhat-td-agent3.sh | sh

记得修改读取目标文件以及pos文件的权限为td-agent

chown -R td-agent:td-agent /var/log/containers

生成密钥

gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB

修改host

vi /etc/hosts
199.232.28.133 raw.githubusercontent.com

安装RVM

curl -sSL https://get.rvm.io | bash -s stable
source /etc/profile.d/rvm.sh
rvm list known
rvm install 2.4.1
#查看版本
ruby -v
gem -v

ruby安装

yum install ruby

ruby仓库

gem sources --add https://gems.ruby-china.com/ --remove https://rubygems.org/

查看

gem sources -l
#更新缓存
gem sources -u

ruby安装

# 安装
gem install fluentd --no-doc
fluentd --setup ./fluent
# 启动
fluentd -c ./fluent/fluent.conf -vv &
# 测试
echo '{"json":"message"}' | fluent-cat debug.test
# 停止
pkill -f fluentd

安装

curl -L https://toolbelt.treasuredata.com/sh/install-redhat-td-agent3.sh | sh

启动

systemctl start td-agent

验证

curl -X POST -d 'json={"json":"message"}' http://localhost:8888/debug.test

查看日志

tail /var/log/td-agent/td-agent.log

安装插件

gem install fluent-plugin-elasticsearch

文件追加记录

echo "测试" >> /etc/td-agent/huitest.log

配置文件

####
## Output descriptions:
##

# Treasure Data (http://www.treasure-data.com/) provides cloud based data
# analytics platform, which easily stores and processes data from td-agent.
# FREE plan is also provided.
# @see http://docs.fluentd.org/articles/http-to-td
#
# This section matches events whose tag is td.DATABASE.TABLE
<match td.*.*>
  @type tdlog
  apikey YOUR_API_KEY

  auto_create_table
  buffer_type file
  buffer_path /var/log/td-agent/buffer/td

  <secondary>
    @type file
    path /var/log/td-agent/failed_records
  </secondary>
</match>

## match tag=debug.** and dump to console
<match debug.**>
  @type stdout
</match>

<source>
  @type tail
  path /etc/td-agent/huitest.log
#  pos_file /etc/td-agent/huitest.log.pos
  tag hui
#  refresh_interval 10
  <parse>
    @type none
  </parse>
</source>

<match **>
  @type elasticsearch
  host	172.28.6.119
  port	9200
#  user  elastic
#  password  123456
#  time_slice_format %Y%m%d
  index_name  ${tag}.%Y%m%d
  logstash_format    true
  logstash_dateformat %Y%m%d
  include_tag_key true
  type_name testlog
#  buffer_chunk_limit 2M
#  buffer_queue_limit 32
#  max_retry_wait 30
#  disable_retry_limit
#  num_threads 8
  logstash_prefix    hui
  flush_interval 1s
  <buffer tag, time>
    timekey 1h
    timekey_wait 1s
    flush_interval 1s
  </buffer>
</match>
#<match hui>
#  @type file
#  path /ect/td-agent/generaltest.log
#  time_slice_format %Y%m%d
#  time_slice_wait 10m
#  comperss gzip
#</match>

####
## Source descriptions:
##

## built-in TCP input
## @see http://docs.fluentd.org/articles/in_forward
<source>
  @type forward
</source>

## built-in UNIX socket input
#<source>
#  @type unix
#</source>

# HTTP input
# POST http://localhost:8888/<tag>?json=<json>
# POST http://localhost:8888/td.myapp.login?json={"user"%3A"me"}
# @see http://docs.fluentd.org/articles/in_http
<source>
  @type http
  bind 127.0.0.1
  port 8888
#  tag debug.log
</source>

## live debugging agent
<source>
  @type debug_agent
  bind 127.0.0.1
  port 24230
</source>

####
## Examples:
##

## File input
## read apache logs continuously and tags td.apache.access
#<source>
#  @type tail
#  format apache
#  path /var/log/httpd-access.log
#  tag td.apache.access
#</source>

## File output
## match tag=local.** and write to file
#<match local.**>
#  @type file
#  path /var/log/td-agent/access
#</match>

## Forwarding
## match tag=system.** and forward to another td-agent server
#<match system.**>
#  @type forward
#  host 192.168.0.11
#  # secondary host is optional
#  <secondary>
#    host 192.168.0.12
#  </secondary>
#</match>

## Multiple output
## match tag=td.*.* and output to Treasure Data AND file
#<match td.*.*>
#  @type copy
#  <store>
#    @type tdlog
#    apikey API_KEY
#    auto_create_table
#    buffer_type file
#    buffer_path /var/log/td-agent/buffer/td
#  </store>
#  <store>
#    @type file
#    path /var/log/td-agent/td-%Y-%m-%d/%H.log
#  </store>
#</match>

可用配置

https://archive.apache.org/dist/kafka/2.3.0/kafka_2.12-2.3.0.tgz