Logstash从Oracle全量、增量、定时抽取数据至ES

下载Logstash

[https://www.elastic.co/cn/downloads/logstash]

Oracle->ES全量

1. 解压缩下载的Logstash包,如下图所示
   
2. 在/bin路径下创建 jdbc_orcl_all.config文件

input {
   
    stdin {
   }
    jdbc {
   
	  #ES中type
	  type => "pc_bkrwxxsjx"
	  #数据库链接信息
      jdbc_connection_string =>  "jdbc:oracle:thin:username/password@//10.4.124.170:1521/orcl"
	  #用户名
      jdbc_user => "username"
	  #密码
      jdbc_password => "password"
	  #ojdbc6.jar包所在运行logstash程序服务器的路径
      jdbc_driver_library => "F:\05_Kettle\oracle_jdbc_jar\ojdbc6.jar"
	  #驱动类型
      jdbc_driver_class => "Java::oracle.jdbc.driver.OracleDriver"
	  #源表字段名导入ES后是否忽略大小写
	  lowercase_column_names=> false
	  #分页
      jdbc_paging_enabled => "true"
	  #每页数据量
      jdbc_page_size => "50000"
	  #默认时区
      jdbc_default_timezone => "UTC"
	  #执行sql语句或者指定.sql文件路径。statement_filepath => "D:\logstash\oracle_sql\PC_AJWJWPJBXXSJX.sql"
	  statement => "SELECT * FROM PC_BKRWXXSJX"
	  #定时设置，corn表达式
      schedule => "* * * * *"  
    }
}

filter {
   
   json {
   
        source => "message"
        remove_field => ["message"]
    }
}
output {
   
	if[type] == "pc_bkrwxxsjx"{
   
		elasticsearch {
   
			#ES ip:port
			hosts => "ip:port"
			#ES索引名
			index => "pc_bkrwxxsjx"
			document_type => "pc_bkrwxxsjx"
			document_id => "%{
   ID}"
		}
	}
    stdout {
   
        codec =