elasticsearch translog 去除问题
背景:有许多elasticsearch 的业务都是对速度的追求,数据丢失不常见或者可以忍受。比如日志服务;
对此一个优化点是去掉 translog 写入保证数据流程;
参考博客:
https://lequ7.com/guan-yu-shu-ju-ku-di-di-elasticsearch-qian-wan-ji-tps-xie-ru-xing-neng-fan-bei-ji-shu-pou-xi.html
优化translog,反对动静开启索引不写translog,不写translog的话,咱们能够不再触发translog的锁问题,也能够缓解了IO压力。然而这可能带来数据失落,所以目前咱们做成动静开关,能够在须要追数据的时候长期开启。后续咱们也在思考跟flink团队联合,通过flink checkpoint保证数据可靠性,就能够不依赖写入translog。从生产环境咱们验证的状况看,在写入压力较大的索引上开启不写translog,能有10-30%不等的性能晋升。
具体实现
通过查看 elasticsearch的 写入流程可以发现针对translog 的触发时机只有两处,一个数据写入一个是数据删除。
所以只改这两处就可以了。
diff --git a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
index cb5052c..cdf348b 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
@@ -486,8 +486,9 @@ public class InternalEngine extends Engine {
}
if (indexResult.hasFailure() == false &&
index.origin() != Operation.Origin.LOCAL_TRANSLOG_RECOVERY) {
- Translog.Location location =
- translog.add(new Translog.Index(index, indexResult));
+// Translog.Location location =
+// translog.add(new Translog.Index(index, indexResult));
+ Translog.Location location = translog.getLocation();
indexResult.setTranslogLocation(location);
}
indexResult.setTook(System.nanoTime() - index.startTime());
@@ -741,8 +742,9 @@ public class InternalEngine extends Engine {
}
if (!deleteResult.hasFailure() &&
delete.origin() != Operation.Origin.LOCAL_TRANSLOG_RECOVERY) {
- Translog.Location location =
- translog.add(new Translog.Delete(delete, deleteResult));
+// Translog.Location location =
+// translog.add(new Translog.Delete(delete, deleteResult));
+ Translog.Location location = translog.getLocation();
deleteResult.setTranslogLocation(location);
}
deleteResult.setTook(System.nanoTime() - delete.startTime());
diff --git a/core/src/main/java/org/elasticsearch/index/translog/Translog.java b/core/src/main/java/org/elasticsearch/index/translog/Translog.java
index 71d2e52..ec2672c 100644
--- a/core/src/main/java/org/elasticsearch/index/translog/Translog.java
+++ b/core/src/main/java/org/elasticsearch/index/translog/Translog.java
@@ -428,6 +428,19 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
}
}
+ /**
+ * 替换上面的函数
+ * Adds a delete / index operations to the transaction log.
+ *
+ * @see org.elasticsearch.index.translog.Translog.Operation
+ * @see Index
+ * @see org.elasticsearch.index.translog.Translog.Delete
+ */
+ public Location getLocation() throws IOException {
+ Location location = current.add(null);
+ return location;
+ }
+
/**
* The a {@linkplain Location} that will sort after the {@linkplain Location} returned by the last write but before any locations which
* can be returned by the next write.
diff --git a/core/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java b/core/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java
index 241729c..07de07a 100644
--- a/core/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java
+++ b/core/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java
@@ -131,22 +131,44 @@ public class TranslogWriter extends BaseTranslogReader implements Closeable {
/**
* add the given bytes to the translog and return the location they were written at
*/
+// public synchronized Translog.Location add(BytesReference data) throws IOException {
+// ensureOpen();
+// final long offset = totalOffset;
+// try {
+// data.writeTo(outputStream);
+// } catch (Exception ex) {
+// try {
+// closeWithTragicEvent(ex);
+// } catch (Exception inner) {
+// ex.addSuppressed(inner);
+// }
+// throw ex;
+// }
+// totalOffset += data.length();
+// operationCounter++;
+// return new Translog.Location(generation, offset, data.length());
+// }
+
+ /**
+ * 改造上面的代码
+ * add the given bytes to the translog and return the location they were written at
+ */
public synchronized Translog.Location add(BytesReference data) throws IOException {
- ensureOpen();
+// ensureOpen();
final long offset = totalOffset;
- try {
- data.writeTo(outputStream);
- } catch (Exception ex) {
- try {
- closeWithTragicEvent(ex);
- } catch (Exception inner) {
- ex.addSuppressed(inner);
- }
- throw ex;
- }
- totalOffset += data.length();
+// try {
+// data.writeTo(outputStream);
+// } catch (Exception ex) {
+// try {
+// closeWithTragicEvent(ex);
+// } catch (Exception inner) {
+// ex.addSuppressed(inner);
+// }
+// throw ex;
+// }
+ totalOffset += 0;
operationCounter++;
- return new Translog.Location(generation, offset, data.length());
+ return new Translog.Location(generation, offset, 0);
}
/**
后记
后期优化, 看着博客的意思是有一个 全局配置开关改变elasticsearch的行为,或者是通过 一些index的配置,改变 index是否保留translog 的作用,这些需要额外开发支持。这里要做的东西还是很多。此博客紧起到抛砖引玉的作用。