elasticsearch & elastiflow & logstash & Kibana 搭建开源 netflow 流量监控系统

elasticsearch & elastiflow & logstash & Kibana 搭建开源 netflow 流量监控系统

################################################################

@官方安装文档

 

环境

#####

[ Ubuntu ]

# lsb_release -a
No LSB modules are available.
Distributor ID: Ubuntu
Description:    Ubuntu 18.04.3 LTS
Release:        18.04
Codename:       bionic

[ Java ]

# java -version
openjdk version "11.0.4" 2019-07-16
OpenJDK Runtime Environment (build 11.0.4+11-post-Ubuntu-1ubuntu218.04.3)
OpenJDK 64-Bit Server VM (build 11.0.4+11-post-Ubuntu-1ubuntu218.04.3, mixed mode, sharing)

[ nodejs ]

/usr/local/nodejs/bin/node -v      #kibana 使用nodejs 提供web服务 
v10.16.3

npm 的一些模块可能没装，产生一些问题。

 

#############################

 

Elasticsearch

################

@下载elasticsearch

tar -zxvf elasticsearch-7.3.2-linux-x86_64.tar.gz             #解压
mv elasticsearch-7.3.2/  /usr/local/elsticsearch              

groupadd  elasticsearch                                       #添加组
useradd  elasticsearch -s /sbin/nologin -M -g elastic         #添加用户，不可登入

chown  elasticsearch:elasticsearch -R /usr/local/elasticsearch#更改所属
chmod 744 -R /usr/local/elasticsearch                         #更改权限

vim /usr/local/elasticsearch/config/jvm.options
# Xms represents the initial size of total heap space
# Xmx represents the maximum size of total heap space

-Xms1g    #jvm初始内存
-Xmx4g    #jvm最大内存    建议主机的内存大于4G,不然会有点卡

vim /usr/local/elasticsearch/config/elasticsearch.yml
确认以下配置：
network.host: 127.0.0.1    #运行时的主机IP
http.port: 9200            #运行时的主机端口

indices.query.bool.max_clause_count: 8192    #官方建议
search.max_buckets: 100000                   #官方建议

 

/usr/local/elasticsearch/bin/elasticsearch    #手动运行elasticsearch
curl http://127.0.0.1:9200                    #测试是否正常
{
  "name" : "szqsm",
  "cluster_name" : "elasticsearch",
  "cluster_uuid" : "aaJilAe_TImsaf0sZH2_UA",
  "version" : {
    "number" : "7.3.2",
    "build_flavor" : "default",
    "build_type" : "tar",
    "build_hash" : "1c1faf1",
    "build_date" : "2019-09-06T14:40:30.409026Z",
    "build_snapshot" : false,
    "lucene_version" : "8.1.0",
    "minimum_wire_compatibility_version" : "6.8.0",
    "minimum_index_compatibility_version" : "6.0.0-beta1"
  },
  "tagline" : "You Know, for Search"
}

vim /etc/systemd/system/elasticsearch.service    #设定开机启动

[Unit]
Description=Elasticsearch

[Service]
Type=simple
WorkingDirectory=/usr/local/elasticsearch        #程序运行目录
User=elasticsearch
Group=elasticsearch
ExecStart=/usr/local/elasticsearch/bin/elasticsearch -p elasticsearch.pid  #开始运行命令
Restart=always

[Install]
WantedBy=multi-user.target

Logstash

###########

@下载logstash

@下载elastiflow（logstash下的netflow模块）

@官方文档

#没有安装java环境先执行以先安装
#java jdk && jre
#apt-get install openjdk-11-jdk openjdk-11-jre

tar -zxvf logstash-7.3.2.tar.gz
mv logstash-7.3.2/  /usr/local/logstash

groupadd  logstash                                         #添加组
useradd   logstash -s /sbin/nologin -M -g logstash         #添加用户，不可登入

chown  logstash:logstash -R /usr/local/logstash            #更改所属
chmod 744 -R /usr/local/logstash                           #更改权限

#安装及更新logstash的各个工具模块，确保是最新版本

/usr/local/logstash/bin/logstash-plugin install logstash-codec-sflow
/usr/local/logstash/bin/logstash-plugin update logstash-codec-netflow
/usr/local/logstash/bin/logstash-plugin update logstash-input-udp
/usr/local/logstash/bin/logstash-plugin update logstash-input-tcp
/usr/local/logstash/bin/logstash-plugin update logstash-filter-dns
/usr/local/logstash/bin/logstash-plugin update logstash-filter-geoip
/usr/local/logstash/bin/logstash-plugin update logstash-filter-translate

cp -R /usr/local/logstash/config/  /etc/logstash    #把配置目录放到/etc 下

tar -axvf logstash-7.3.2.tar.gz
mv elastiflow-3.5.1/logstash/elastiflow/ /etc/logstash/logstash  #elastiflow模块配置文件
cp -R elastiflow-3.5.1/logstash.service.d/ /etc/systemd/system/  #elastiflow启动参数文件

 

vim /etc/logstash/pipelines.yml    #关联模块elastiflow配置到logstash
- pipeline.id: elastiflow
  path.config: "/etc/logstash/elastiflow/conf.d/*.conf"
                                                         

/usr/local/logstash/bin/logstash "--path.settings" "/etc/logstash"  #检查手动运行是否正常
netstat -tunlp  | grep java   #查看此时服务端口有哪些
tcp6       0      0 :::4739                 :::*                    LISTEN      1285/java           
tcp6       0      0 127.0.0.1:9200          :::*                    LISTEN      1238/java           
tcp6       0      0 127.0.0.1:9300          :::*                    LISTEN      1238/java           
tcp6       0      0 127.0.0.1:9600          :::*                    LISTEN      1285/java           
udp        0      0 0.0.0.0:2055            0.0.0.0:*                           1285/java           
udp        0      0 0.0.0.0:6343            0.0.0.0:*                           1285/java           
udp        0      0 0.0.0.0:4739            0.0.0.0:*                           1285/java 

@注意
2055 端口是默认的 netflow v5 v9
4739 端口是默认的 ipfix v10
其实ipfix_10 就是netflow v10
端口修改
vim /etc/systemd/system/logstash.service.d/elastiflow.conf


ps -aux | grep logstash        #查看进程
kill -9 xxx                    #手动终止

./usr/local/logstash/bin/system-install     #自动安装启动服务
vim /etc/systemd/system/logstash.service    #更据情况修改启动参数

[Unit]
Description=logstash
After=elasticsearch.service                  
#由于logstash启动会自动搜索9200端口，所以需要在elasticsearch启动后运行

[Service]
Type=simple
User=logstash
Group=logstash
# Load env vars from /etc/default/ and /etc/sysconfig/ if they exist.
# Prefixing the path with '-' makes it try to load, but if the file doesn't
# exist, it continues onward.
EnvironmentFile=-/etc/default/logstash
EnvironmentFile=-/etc/sysconfig/logstash
ExecStart=/usr/local/logstash/bin/logstash "--path.settings" "/etc/logstash"
Restart=always
WorkingDirectory=/
Nice=19
LimitNOFILE=16384

[Install]
WantedBy=multi-user.target

systemctl sart logstash      #开启服务
systemctl status logstash    #服务状态
systemctl enable logstash    #加入自启动

cat /usr/local/logstash/logs/logstash-plain.log    #查看运行日志

Kibana

#########

@下载kibana

tar -zxvf kibana-7.3.1-linux-x86_64.tar.gz
mv kibana-7.3.1/  /usr/local/kibana

groupadd  kibana                                       #添加组
useradd   kibana -s /sbin/nologin -M -g kibana         #添加用户，不可登入

chown  kibana:kibana -R /usr/local/kibana              #更改所属
chmod 744 -R /usr/local/kibana                         #更改权限

vim /usr/local/kibana/config/kibana.yml    #更改及确认配置

server.name: "SZQSM-Kibana"
server.host: "192.168.91.35"    #对外web服务的IP(服务器IP)
server.port: 5601               #web服务端口(默认)
elasticsearch.hosts: ["http://127.0.0.1:9200"]    #elasticsearch URL
i18n.locale: "zh-CN"            #中文显示

vim /etc/systemd/system/kibana.service     #添加到管理服务

[Unit]
Description=kibana
After=logstash.service                     #kibana 最后启用，获取logstash提供的数据

[Service]
Type=simple
WorkingDirectory=/usr/local/kibana
User=kibana
Group=kibana
ExecStart=/usr/local/kibana/bin/kibana
Restart=always

[Install]
WantedBy=multi-user.target


systemctl start/status/enable kibana

#Kibana 上传Elastiflow 模板文件
../elastiflow-3.5.1/kibana/elastiflow.kibana.7.3.x.ndjson  最新版本为ndjson文件 

Kibana 上传Elastiflow 模板文件

 

效果图：

##############