环境:
kudu:kudu-1.9.0+cdh6.2.0-967373.el6.x86_64.rpm
问题:
使用kudu的java api执行插入操作的时候,一次性插入10000条,报错:
org.apache.kudu.client.NonRecoverableException: MANUAL_FLUSH is enabled but the buffer is too big
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduSession.apply(KuduSession.java:94)
at com.cloudera.utils.KuduUtils.upsert(KuduUtils.java:102)
at com.cloudera.OperateKuduTable.operate(OperateKuduTable.java:18)
at com.cloudera.KuduExample.main(KuduExample.java:24)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduSession.apply(AsyncKuduSession.java:590)
at org.apache.kudu.client.KuduSession.apply(KuduSession.java:80)
... 3 more
翻开源码
public Deferred<OperationResponse> apply(final Operation operation) throws KuduException {
Preconditions.checkNotNull(operation, "Can not apply a null operation");
// Freeze the row so that the client can not concurrently modify it while it is in flight.
operation.getRow().freeze();
// If immediate flush mode, send the operation directly.
if (flushMode == FlushMode.AUTO_FLUSH_SYNC) {
if (timeoutMs != 0) {
operation.setTimeoutMillis(timeoutMs);
}
operation.setExternalConsistencyMode(this.consistencyMode);
operation.setIgnoreAllDuplicateRows(ignoreAllDuplicateRows);
// Add a callback to update the propagated timestamp returned from the server.
Callback<Deferred<OperationResponse>, OperationResponse> cb =
new Callback<Deferred<OperationResponse>, OperationResponse>() {
@Override
public Deferred<OperationResponse> call(OperationResponse resp) throws Exception {
client.updateLastPropagatedTimestamp(resp.getWriteTimestampRaw());
return Deferred.fromResult(resp);
}
};
return client.sendRpcToTablet(operation)
.addCallbackDeferring(cb)
.addErrback(new SingleOperationErrCallback(operation));
}
// Kick off a location lookup.
Deferred<LocatedTablet> tablet = client.getTabletLocation(operation.getTable(),
operation.partitionKey(),
timeoutMs);
// Holds a buffer that should be flushed outside the synchronized block, if necessary.
Buffer fullBuffer = null;
try {
synchronized (monitor) {
Deferred<Void> notification = flushNotification.get();
if (activeBuffer == null) {
// If the active buffer is null then we recently flushed. Check if there
// is an inactive buffer available to replace as the active.
if (inactiveBufferAvailable()) {
refreshActiveBuffer();
} else {
Status statusServiceUnavailable =
Status.ServiceUnavailable("All buffers are currently flushing");
// This can happen if the user writes into a buffer, flushes it, writes
// into the second, flushes it, and immediately tries to write again.
throw new PleaseThrottleException(statusServiceUnavailable,
null, operation, notification);
}
}
if (flushMode == FlushMode.MANUAL_FLUSH) {
if (activeBuffer.getOperations().size() < mutationBufferSpace) {
activeBuffer.getOperations().add(new BufferedOperation(tablet, operation));
} else {
Status statusIllegalState =
Status.IllegalState("MANUAL_FLUSH is enabled but the buffer is too big");
throw new NonRecoverableException(statusIllegalState);
}
} else {
assert flushMode == FlushMode.AUTO_FLUSH_BACKGROUND;
int activeBufferSize = activeBuffer.getOperations().size();
if (activeBufferSize >= mutationBufferSpace) {
// Save the active buffer into fullBuffer so that it gets flushed when we leave this
// synchronized block.
fullBuffer = activeBuffer;
activeBuffer = null;
activeBufferSize = 0;
if (inactiveBufferAvailable()) {
refreshActiveBuffer();
} else {
Status statusServiceUnavailable =
Status.ServiceUnavailable("All buffers are currently flushing");
throw new PleaseThrottleException(statusServiceUnavailable,
null, operation, notification);
}
}
if (mutationBufferLowWatermark < mutationBufferSpace && // low watermark is enabled
activeBufferSize >= mutationBufferLowWatermark && // buffer is over low water mark
!inactiveBufferAvailable()) { // no inactive buffers
// Check if we are over the low water mark.
int randomWatermark = activeBufferSize + 1 +
randomizer.nextInt(mutationBufferSpace -
mutationBufferLowWatermark);
if (randomWatermark > mutationBufferSpace) {
Status statusServiceUnavailable =
Status.ServiceUnavailable("The previous buffer hasn't been flushed and the " +
"current buffer is over the low watermark, please retry later");
throw new PleaseThrottleException(statusServiceUnavailable,
null, operation, notification);
}
}
activeBuffer.getOperations().add(new BufferedOperation(tablet, operation));
if (activeBufferSize + 1 >= mutationBufferSpace && inactiveBufferAvailable()) {
// If the operation filled the buffer, then flush it.
Preconditions.checkState(fullBuffer == null);
fullBuffer = activeBuffer;
activeBuffer = null;
activeBufferSize = 0;
} else if (activeBufferSize == 0) {
// If this is the first operation in the buffer, start a background flush timer.
client.newTimeout(activeBuffer.getFlusherTask(), interval);
}
}
}
} finally {
// Flush the buffer outside of the synchronized block, if required.
if (fullBuffer != null) {
doFlush(fullBuffer);
}
}
return operation.getDeferred();
}
问题解决,设置kuduSession的大小 即可:
kuduSession.setMutationBufferSpace(20000);