kettle7.0写入elasticsearch5.11插件
kettle7.0中自带es版本支持为2.3,但是5.x版本获取客户端连接的api已改,因此需要改源码重新编译。
下载kettle源码后,只提取es部分
建立普通java工程,目录层级与上面一致,导入依赖jar包。
jar包参照安装好的pentaho-kettle-7.0.0.0-R\plugins\elasticsearch-bulk-insert\lib下的从安装好的es5.11中提取对应版本的jar包。
更改 src\org\pentaho\di\trans\steps\elasticsearchbulk\ElasticSearchBulk.java
//import org.elasticsearch.node.NodeBuilder;
import org.elasticsearch.transport.client.PreBuiltTransportClient;
private void initClient() throws UnknownHostException {
Settings.Builder settingsBuilder = Settings.builder();
settingsBuilder.put( Settings.Builder.EMPTY_SETTINGS ); // keep default classloader
settingsBuilder.put( meta.getSettingsMap() );
Settings settings = settingsBuilder.build();
client = new PreBuiltTransportClient(settings);
// //TransportClient.Builder tClientBuilder = TransportClient.builder().settings( settingsBuilder );
// TransportClient client = new PreBuiltTransportClient(settings);
//
if ( !meta.servers.isEmpty() ) {
node = null;
//TransportClient tClient = tClientBuilder.build();
for ( ElasticSearchBulkMeta.Server s : meta.servers ) {
((TransportClient) client).addTransportAddress(s.getAddr());
}
//client = tClient;
}
// } else {
// NodeBuilder nodeBuilder = NodeBuilder.nodeBuilder();
// nodeBuilder.settings( settingsBuilder );
// node = nodeBuilder.client( true ).node(); // this node will not hold data
// client = node.client();
// node.start();
// }
// for ( ElasticSearchBulkMeta.Server s : meta.servers ) {
// logError(s.getAddr().toString());
// }
//
// logError("-----");
// Settings settings = Settings.builder().put("cluster.name", "estest").build();;// 集群名
// client = new PreBuiltTransportClient(settings).addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName("10.0.0.20"), 12300));
}
private void disposeClient() {
if ( client != null ) {
client.close();
}
// if ( node != null ) {
// node.close();
// }
}
更改 src\org\pentaho\di\ui\trans\steps\elasticsearchbulk\ElasticSearchBulkDialog.java
private void test( TestType testType ) {
// Save off the thread's context class loader to restore after the test
ClassLoader originalClassloader = Thread.currentThread().getContextClassLoader();
// Now ensure that the thread's context class loader is the plugin's classloader
Thread.currentThread().setContextClassLoader( this.getClass().getClassLoader() );
TransportClient client = null;
Node node = null;
try {
ElasticSearchBulkMeta tempMeta = new ElasticSearchBulkMeta();
toModel( tempMeta );
Settings.Builder settingsBuilder = Settings.builder();
settingsBuilder.put( Settings.Builder.EMPTY_SETTINGS ); // keep default classloader
settingsBuilder.put( tempMeta.getSettingsMap() );
Settings settings = settingsBuilder.build();
client = new PreBuiltTransportClient(settings);
if ( !tempMeta.getServers().isEmpty() ) {
node = null;
//TransportClient tClient = tClientBuilder.build();
for ( ElasticSearchBulkMeta.Server s : tempMeta.getServers() ) {
((TransportClient) client).addTransportAddress(s.getAddr());
}
//client = tClient;
}
TransportClient.Builder tClientBuilder = TransportClient.builder().settings( settingsBuilder );
// Settings settings = settingsBuilder.build();
// client = new PreBuiltTransportClient(settings);
// if ( !tempMeta.getServers().isEmpty() ) {
// node = null;
TransportClient tClient = tClientBuilder.build();
// for ( ElasticSearchBulkMeta.Server s : tempMeta.getServers() ) {
// client.addTransportAddress( s.getAddr() );
// }
client = tClient;
// }
// else {
// NodeBuilder nodeBuilder = NodeBuilder.nodeBuilder();
// nodeBuilder.settings( settingsBuilder );
// node = nodeBuilder.client( true ).node();
// client = node.client();
// node.start();
// }
// logError(settingsBuilder.toString());
// logError("-----");
// Settings settings = Settings.builder().put("cluster.name", "estest").build();;// 集群名
// client = new PreBuiltTransportClient(settings).addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName("10.0.0.20"), 12300));
AdminClient admin = client.admin();
switch ( testType ) {
case INDEX:
if ( StringUtils.isBlank( tempMeta.getIndex() ) ) {
showError( BaseMessages.getString( PKG, "ElasticSearchBulk.Error.NoIndex" ) );
break;
}
// First check to see if the index exists
IndicesExistsRequestBuilder indicesExistBld = admin.indices().prepareExists( tempMeta.getIndex() );
IndicesExistsResponse indicesExistResponse = indicesExistBld.execute().get();
if ( !indicesExistResponse.isExists() ) {
showError( BaseMessages.getString( PKG, "ElasticSearchBulkDialog.Error.NoIndex" ) );
return;
}
RecoveryRequestBuilder indicesBld = admin.indices().prepareRecoveries( tempMeta.getIndex() );
ListenableActionFuture<RecoveryResponse> lafInd = indicesBld.execute();
String shards = "" + lafInd.get().getSuccessfulShards() + "/" + lafInd.get().getTotalShards();
showMessage( BaseMessages.getString( PKG, "ElasticSearchBulkDialog.TestIndex.TestOK", shards ) );
break;
case CLUSTER:
ClusterStateRequestBuilder clusterBld = admin.cluster().prepareState();
ListenableActionFuture<ClusterStateResponse> lafClu = clusterBld.execute();
ClusterStateResponse cluResp = lafClu.actionGet();
String name = cluResp.getClusterName().value();
ClusterState cluState = cluResp.getState();
int numNodes = cluState.getNodes().getSize();
showMessage( BaseMessages.getString( PKG, "ElasticSearchBulkDialog.TestCluster.TestOK", name, numNodes ) );
break;
default:
break;
}
} catch ( NoNodeAvailableException e ) {
showError( BaseMessages.getString( PKG, "ElasticSearchBulkDialog.Error.NoNodesFound" ) );
} catch ( MasterNotDiscoveredException e ) {
showError( BaseMessages.getString( PKG, "ElasticSearchBulkDialog.Error.NoNodesFound" ) );
} catch ( Exception e ) {
showError( e.getLocalizedMessage() );
} finally {
if ( client != null ) {
client.close();
}
// if ( node != null ) {
// node.close();
// }
}
// Restore the original classloader
Thread.currentThread().setContextClassLoader( originalClassloader );
}
导出为elasticsearch-bulk-insert-plugin-7.0.0.0-25.jar,将jar及lib替换原来的即可。
插件lib替换注意,在原2.3插件的基础上,复制es6安装目录下lib及modules下所有jar至插件lib下,然后删除相同名称版本号较小的jar。
源码及打包好的插件见网盘:链接: https://pan.baidu.com/s/1F5-6mGiV_LgFf912-hc6rw 密码: cun9