说明:logstash通过执行logstash.conf文件操作,其中logstash.conf文件包含(需配置信息):导入数据文件位置,上次加载位置,转化格式。
- 配置logstash.conf
例如:需要导入电影相关数据
input {
file {
path => "YOUR_FULL_PATH_OF_movies.csv"//数据路径
start_position => "beginning"
sincedb_path => "/dev/123" //历史位置存放文件,没有就新建
}
}
filter {
csv {
separator => ","
columns => ["id","content","genre"]
}
mutate {
split => { "genre" => "|" }
remove_field => ["path", "host","@timestamp","message"]
}
mutate {
split => ["content", "("]
add_field => { "title" => "%{[content][0]}"}
add_field => { "year" => "%{[content][1]}"}
}
mutate {
convert => {
"year" => "integer"
}
strip => ["title"]
remove_field => ["path", "host","@timestamp","message","content"]
}
}
output {
elasticsearch {
hosts => "http://localhost:9200"
index => "movies"
document_id => "%{id}"
}
stdout {}
}
- 在logstash的bin目录下执行conf文件
命令如下:
logstash -f YOUR_FULL_PATH_OF_logstash.conf
结果如下: