DataX学习笔记-Writer插件开发

人生偌只如初见

已于 2023-01-30 18:44:08 修改

阅读量7.3k

点赞数 1

分类专栏： DataX 文章标签： DataX ElasticSearch Writer

于 2016-05-25 22:54:46 首次发布

本文链接：https://blog.csdn.net/fighting_one_piece/article/details/51502741

版权

本文详细介绍了如何开发DataX的ElasticSearch Writer插件，包括检出源码、创建插件目录、配置plugins.xml、修改build.xml、编译打包、配置运行及使用Java API的方式进行操作。

摘要由CSDN通过智能技术生成

本文主要是基于将数据写入ElasticSearch开发的Writer插件

1、检出DataX源码（svn checkout http://code.taobao.org/svn/datax/trunk）

2、在com.taobao.datax.plugins.writer包下面创建一个eswriter包，新建ESWriter.java，ParamKey.java

package com.taobao.datax.plugins.writer.eswriter;

import org.apache.log4j.Logger;

import com.taobao.datax.common.exception.DataExchangeException;
import com.taobao.datax.common.exception.ExceptionTracker;
import com.taobao.datax.common.plugin.Line;
import com.taobao.datax.common.plugin.LineReceiver;
import com.taobao.datax.common.plugin.PluginStatus;
import com.taobao.datax.common.plugin.Writer;
import com.taobao.datax.plugins.writer.eswriter.ParamKey;

public class ESWriter extends Writer {

    private String singleCurl = "";

    private String nullString = "";

    private String columnNameString = "";

    private String columnNameSplit = "";

    private String[] columnNames = null;

    private Logger logger = Logger.getLogger(ESWriter.class.getCanonicalName());

    @Override
    public int init() {
        this.singleCurl = param.getValue(ParamKey.singleCurl, 
                "curl -XPOST 'http://192.168.0.108:9200/user/student/{id}' -d '{data}'");
        this.nullString = param.getValue(ParamKey.nullChar, this.nullString);
        this.columnNameString = param.getValue(ParamKey.columnNames, "userid,name,phone");
        this.columnNameSplit = param.getValue(ParamKey.columnNameSplit, ",");
        this.columnNames = columnNameString.split(columnNameSplit);
        return PluginStatus.SUCCESS.value();
    }

    @Override
    public int connect() {
        return PluginStatus.SUCCESS.value();
    }

    private String makeCurl(Line line) {
        if (line == null || line.getFieldNum() == 0) {
            return this.singleCurl + "\n";
        }
        String item = null;
        int num = line.getFieldNum();
        this.singleCurl = this.singleCurl.replace("{id}", line.getField(0));
        StringBuilder sb = new StringBuilder().append("{");
        for (int i = 1; i < num; i++) {
            item = line.getField(i);
            sb.append("\"").append(columnNames[i-1]).append("\":\"");
            if (null == item) {
                sb.append(nullString);
            } else {
                sb.append(item);
            }
            sb.append("\",");
        }
        sb.deleteCharAt(sb.length() - 1);
        sb.append("}");
        return new StringBuilder(this.singleCurl).toString().replace("{data}", sb.toString());
    }

    @Override
    public int startWrite(LineReceiver receiver) {
        Line line;
        try {
            while ((line = receiver.getFromReader()) != null) {
                String curlCommand = makeCurl(line);