logstash大量数据同步

https://blog.csdn.net/weixin_43934561/article/details/103904791

truncate base_apt_del  清空表

vm.option改xmx xms参数 内存大小

数据同步logstash真实数据到测试环境   同步到es或者mysql
    分批次同步
./logstash -f mysql.conf    input filter output

utf-8->ani编码  格式  安装logstash-output-jdbc！！！！

 

input {
        jdbc {

                jdbc_connection_string => "jdbc:mysql://10.151.18.134:3306/ag_security_db?characterEncoding=utf8&useSSL=false&useTimezone=true&serverTimezone=GMT%2B8"
                jdbc_user => "root"
                jdbc_password => "pgaqdb_1qazZAQ!"
                jdbc_driver_library => "/opt/logstash-7.4.2/logstash-7.4.2/mysql/mysql-connector-java-5.1.48.jar"
                jdbc_driver_class => "com.mysql.jdbc.Driver"
                jdbc_paging_enabled => "true"
                jdbc_page_size => "50000"

                statement => "select * from base_apt where id > :sql_last_value limit 100000"
                # 是否记录上次执行结果，true表示会将上次执行结果的tracking_column字段的值保存到last_run_metadata_path指定的文件中；
                record_last_run => true
                # 需要记录查询结果某字段的值时，此字段为true，否则默认tracking_column为timestamp的值；
                use_column_value => true
                # 需要记录的字段，用于增量同步，需是数据库字段
                tracking_column => "id"
                # Value can be any of: numeric,timestamp，Default value is "numeric"
                tracking_column_type => numeric
                # record_last_run上次数据存放位置；
                last_run_metadata_path => "/opt/logstash-7.4.2/logstash-7.4.2/mysql/last_id.txt"
                # 是否清除last_run_metadata_path的记录，需要增量同步时此字段必须为false；
                clean_run => false
                schedule => "* * * * *"

    }
}

output {
        stdout {
                codec=>rubydebug{}
    }

        jdbc {
                driver_jar_path => "/opt/logstash-7.4.2/mysql/mysql-connector-java-5.1.48.jar"
                connection_string => "jdbc:mysql://10.151.31.126:3306/ag_security_db?user=root&password=pgaqdb_1qazZAQ!&characterEncoding=utf8&useSSL=false&useTimezone=true&serverTimezone=GMT%2B8"
                driver_class => "com.mysql.jdbc.Driver"
                statement => [ "INSERT INTO base_apt (hearder,s_ip,s_ip_num,s_port,create_time,happen_time,d_ip,d_ip_num,dport,rule_name,message,attack_grade,access_id,file_name,pay_load,file_md5,host,reply_code,direction,s_mac,d_mac,reply_len,request_header,post_body,reply_content,local_ip,confirm_flag,attack_stage,attack_file_type,app_type,attack_type,msg,unit_id,area_id,device_id,s_area_id,s_unit_id,d_area_id,d_unit_id ) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", "hearder","s_ip","s_ip_num","s_port","create_time","happen_time","d_ip","d_ip_num","dport","rule_name","message","attack_grade","access_id","file_name","pay_load","file_md5","host","reply_code","direction","s_mac","d_mac","reply_len","request_header","post_body","reply_content","local_ip","confirm_flag","attack_stage","attack_file_type","app_type","attack_type","msg","unit_id","area_id","device_id","s_area_id","s_unit_id","d_area_id","d_unit_id" ]
        }
}