2021SC@SDUSC
/** 将段文件按原样复制到 IndexWriter 的目录中。 */
private SegmentCommitInfo copySegmentAsIs(SegmentCommitInfo info, String segName, IOContext context) throws IOException {
// 与以前相同的 SI,但我们更改了目录和名称
SegmentInfo newInfo = new SegmentInfo(directoryOrig, info.info.getVersion(), info.info.getMinVersion(), segName, info.info.maxDoc(),
info.info.getUseCompoundFile(), info.info.getCodec(),
info.info.getDiagnostics(), info.info.getId(), info.info.getAttributes(), info.info.getIndexSort());
SegmentCommitInfo newInfoPerCommit = new SegmentCommitInfo(newInfo, info.getDelCount(), info.getSoftDelCount(), info.getDelGen(),
info.getFieldInfosGen(), info.getDocValuesGen(), info.getId());
newInfo.setFiles(info.info.files());
newInfoPerCommit.setFieldInfosFiles(info.getFieldInfosFiles());
newInfoPerCommit.setDocValuesUpdatesFiles(info.getDocValuesUpdatesFiles());
boolean success = false;
Set<String> copiedFiles = new HashSet<>();
try {
// Copy the segment's files
for (String file: info.files()) {
final String newFileName = newInfo.namedForThisSegment(file);
directory.copyFrom(info.info.dir, file, newFileName, context);
copiedFiles.add(newFileName);
}
success = true;
} finally {
if (!success) {
// Safe: these files must exist
deleteNewFiles(copiedFiles);
}
}
assert copiedFiles.equals(newInfoPerCommit.files()): "copiedFiles=" + copiedFiles + " vs " + newInfoPerCommit.files();
return newInfoPerCommit;
}
/**
* 用于扩展类以在待添加和挂起后执行操作的钩子
* 已删除的文档已刷新到目录但在更改之前
* 已提交(写入新的 Segments_N 文件)。
*/
protected void doAfterFlush() throws IOException {}
/**
* 用于扩展类以在挂起添加和之前执行操作的钩子
* 删除的文件被刷新到目录中。
*/
protected void doBeforeFlush() throws IOException {}
/** <p>专家:准备提交。 这使
* 两阶段提交的第一阶段。 这个方法全
* 自此作者以来提交更改所需的步骤
* 已打开:刷新待添加和删除的文档,
* 同步索引文件,写入大部分下一段_N
* 文件。 调用此后,您必须调用 {@link
* #commit()} 完成提交,或 {@link
* #rollback()} 恢复提交并撤消所有更改
* 自作者打开后完成。</p>
*
* <p>你也可以直接调用{@link #commit()}
* 没有prepareCommit 首先在这种情况下该方法
* 将在内部调用 prepareCommit。
*
* @return <a href="#sequence_number">序列号</a>
* 提交中的最后一个操作。 所有序列号 <= 这个值
* 将反映在提交中,所有其他人都不会。
*/
@Override
public final long prepareCommit() throws IOException {
ensureOpen();
pendingSeqNo = prepareCommitInternal();
// 我们必须在 commitLock 之外执行此操作,否则我们可能会死锁:
if (maybeMerge.getAndSet(false)) {
maybeMerge(config.getMergePolicy(), MergeTrigger.FULL_FLUSH, UNBOUNDED_MAX_MERGE_SEGMENTS);
}
return pendingSeqNo;
}
/**
* <p>专家:如果可用或最大的活动线程缓冲区,则刷新每个线程缓冲区的下一个挂起写入器
* 调用线程中每个线程缓冲区的非挂起写入器。
* 这可用于在索引线程之外将文档刷新到磁盘。 与 {@link #flush()} 相反
* 这不会将所有当前活动的索引缓冲区标记为刷新待处理。
*
* 注意:此方法是尽力而为,可能不会将任何段刷新到磁盘。 如果发生全冲
* 同时多个段可能已被刷新。
* 此 API 的用户可以通过 {@link #ramBytesUsed()} 访问 IndexWriters 当前的内存消耗
* </p>
* @return <code>true</code> iff this method flushed at least on segment to disk.
* @lucene.experimental
*/
public final boolean flushNextBuffer() throws IOException {
try {
if (docWriter.flushOneDWPT()) {
processEvents(true);
return true; // we wrote a segment
}
return false;
} catch (VirtualMachineError tragedy) {
tragicEvent(tragedy, "flushNextBuffer");
throw tragedy;
} finally {
maybeCloseOnTragicEvent();
}
}
private long prepareCommitInternal() throws IOException {
startCommitTime = System.nanoTime();
synchronized(commitLock) {
ensureOpen(false);
if (infoStream.isEnabled("IW")) {
infoStream.message("IW", "prepareCommit: flush");
infoStream.message("IW", " index before flush " + segString());
}
if (tragedy.get() != null) {
throw new IllegalStateException("this writer hit an unrecoverable error; cannot commit", tragedy.get());
}
if (pendingCommit != null) {
throw new IllegalStateException("prepareCommit was already called with no corresponding call to commit");
}
doBeforeFlush();
testPoint("startDoFlush");
SegmentInfos toCommit = null;
boolean anyChanges = false;
long seqNo;
MergePolicy.MergeSpecification pointInTimeMerges = null;
AtomicBoolean stopAddingMergedSegments = new AtomicBoolean(false);
final long maxCommitMergeWaitMillis = config.getMaxFullFlushMergeWaitMillis();
// 这是从 doFlush 复制的,除了它被修改为
// 克隆 & incRef 内部刷新的 SegmentInfos
// 同步块:
try {
synchronized (fullFlushLock) {
boolean flushSuccess = false;
boolean success = false;
try {
seqNo = docWriter.flushAllThreads();
if (seqNo < 0) {
anyChanges = true;
seqNo = -seqNo;
}
if (anyChanges == false) {
// 防止双增量,因为 docWriter#doFlush 增加了flushcount
// 如果我们刷新了任何东西。
flushCount.incrementAndGet();
}
publishFlushedSegments(true);
// cannot pass triggerMerges=true here else it can lead to deadlock:
processEvents(false);
flushSuccess = true;
applyAllDeletesAndUpdates();
synchronized(this) {
writeReaderPool(true);
if (changeCount.get() != lastCommitChangeCount) {
// 有变化要提交,所以我们将在 startCommit 中写入一个新的segments_N。
// 提交的行为本身就是一个 NRT 可见的变化(一个 NRT 阅读器是
// 刚刚打开之前应该在重新打开时看到它)所以我们增加changeCount
// 和段版本,以便未来的 NRT 重新打开将看到更改:
changeCount.incrementAndGet();
segmentInfos.changed();
}
if (commitUserData != null) {
Map<String,String> userData = new HashMap<>();
for(Map.Entry<String,String> ent : commitUserData) {
userData.put(ent.getKey(), ent.getValue());
}
segmentInfos.setUserData(userData, false);
}
toCommit = segmentInfos.clone();
pendingCommitChangeCount = changeCount.get();
// 这在以下情况下很重要,例如,虽然
// 我们正在尝试同步所有引用的文件的
// 合并完成,否则会
// 删除了我们现在正在同步的文件。
deleter.incRef(toCommit.files(false));
if (anyChanges && maxCommitMergeWaitMillis > 0) {
// 我们可以安全地调用 preparePointInTimeMerge 因为上面的 writeReaderPool(true) 写了所有
// 必要的文件到磁盘并检查点它们。
pointInTimeMerges = preparePointInTimeMerge(toCommit, stopAddingMergedSegments::get, MergeTrigger.COMMIT, sci->{});
}
}
success = true;
} finally {
if (!success) {
if (infoStream.isEnabled("IW")) {
infoStream.message("IW", "hit exception during prepareCommit");
}
}
assert Thread.holdsLock(fullFlushLock);
// Done: finish the full flush!
docWriter.finishFullFlush(flushSuccess);
doAfterFlush();
}
}
} catch (VirtualMachineError tragedy) {
tragicEvent(tragedy, "prepareCommit");
throw tragedy;
} finally {
maybeCloseOnTragicEvent();
}
if (pointInTimeMerges != null) {
if (infoStream.isEnabled("IW")) {
infoStream.message("IW", "now run merges during commit: " + pointInTimeMerges.segString(directory));
}
mergeScheduler.merge(mergeSource, MergeTrigger.COMMIT);
pointInTimeMerges.await(maxCommitMergeWaitMillis, TimeUnit.MILLISECONDS);
if (infoStream.isEnabled("IW")) {
infoStream.message("IW", "done waiting for merges during commit");
}
synchronized (this) {
// we need to call this under lock since mergeFinished above is also called under the IW lock
stopAddingMergedSegments.set(true);
}
}
// 在处理任何 pointInTimeMerges 之后执行此操作,因为如果有任何合并,文件将发生更改完成
filesToCommit = toCommit.files(false);
try {
if (anyChanges) {
maybeMerge.set(true);
}
startCommit(toCommit);
if (pendingCommit == null) {
return -1;
} else {
return seqNo;
}
} catch (Throwable t) {
synchronized (this) {
if (filesToCommit != null) {
try {
deleter.decRef(filesToCommit);
} catch (Throwable t1) {
t.addSuppressed(t1);
} finally {
filesToCommit = null;
}
}
}
throw t;
}
}
}
/**
* 此优化允许 commit/getReader 等待小段上的合并以
* 减少提交点/NRT 阅读器中微小段的最终数量。 我们将 {@code OneMerge} 包装到
* 合并完成后更新 {@code mergingSegmentInfos}。 我们替换源段
* 在 SIS 中,我们将使用新合并的段提交/打开阅读器,但忽略所有删除和更新
* 在合并时对合并段中的文档进行处理。 所做的更新不属于
* 时间点提交点/NRT READER,因此不应包括在内。 查看 {@code onMergeComplete} 中的克隆调用
* 以下。 我们还确保在持有 {@code IndexWriter} 的锁的同时拉出合并阅读器。 否则
* 我们可以看到不属于该段的并发删除/更新。
*/
private MergePolicy.MergeSpecification preparePointInTimeMerge(SegmentInfos mergingSegmentInfos, BooleanSupplier stopCollectingMergeResults,
MergeTrigger trigger,
IOUtils.IOConsumer<SegmentCommitInfo> mergeFinished) throws IOException {
assert Thread.holdsLock(this);
assert trigger == MergeTrigger.GET_READER || trigger == MergeTrigger.COMMIT : "illegal trigger: " + trigger;
MergePolicy.MergeSpecification pointInTimeMerges = updatePendingMerges(new OneMergeWrappingMergePolicy(config.getMergePolicy(), toWrap ->
new MergePolicy.OneMerge(toWrap.segments) {
SegmentCommitInfo origInfo;
final AtomicBoolean onlyOnce = new AtomicBoolean(false);
@Override
public void mergeFinished(boolean committed, boolean segmentDropped) throws IOException {
assert Thread.holdsLock(IndexWriter.this);
// 如果允许的最大挂钟,includeInCommit 将被设置(上面,由我们的调用者)为 false
// 时间 (IWC.getMaxCommitMergeWaitMillis()) 已经过去,这意味着我们没有设置超时
// 并且不会将我们的合并提交到要提交的 SegmentInfos
if (segmentDropped == false
&& committed
&& stopCollectingMergeResults.getAsBoolean() == false) {
// make sure onMergeComplete really was called:
assert origInfo != null;
if (infoStream.isEnabled("IW")) {
infoStream.message("IW", "now apply merge during commit: " + toWrap.segString());
}
if (trigger == MergeTrigger.COMMIT) {
// 如果我们在此处的 getReader 调用中执行此操作,则此操作已过时,因为我们已经包含这些文件
deleter.incRef(origInfo.files());
}
Set<String> mergedSegmentNames = new HashSet<>();
for (SegmentCommitInfo sci : segments) {
mergedSegmentNames.add(sci.info.name);
}
List<SegmentCommitInfo> toCommitMergedAwaySegments = new ArrayList<>();
for (SegmentCommitInfo sci : mergingSegmentInfos) {
if (mergedSegmentNames.contains(sci.info.name)) {
toCommitMergedAwaySegments.add(sci);
if (trigger == MergeTrigger.COMMIT) {
// 如果我们在此处的 getReader 调用中执行此操作,则此操作已过时,因为我们已经拥有一个具有
// incRef'd 这些文件,并在关闭时将它们 decRef
deleter.decRef(sci.files());
}
}
}
// Construct a OneMerge that applies to toCommit
MergePolicy.OneMerge applicableMerge = new MergePolicy.OneMerge(toCommitMergedAwaySegments);
applicableMerge.info = origInfo;
long segmentCounter = Long.parseLong(origInfo.info.name.substring(1), Character.MAX_RADIX);
mergingSegmentInfos.counter = Math.max(mergingSegmentInfos.counter, segmentCounter + 1);
mergingSegmentInfos.applyMergeChanges(applicableMerge, false);
} else {
if (infoStream.isEnabled("IW")) {
infoStream.message("IW", "skip apply merge during commit: " + toWrap.segString());
}
}
toWrap.mergeFinished(committed, segmentDropped);
super.mergeFinished(committed, segmentDropped);
}
@Override
void onMergeComplete() throws IOException {
assert Thread.holdsLock(IndexWriter.this);
if (stopCollectingMergeResults.getAsBoolean() == false
&& isAborted() == false
&& info.info.maxDoc() > 0/* never do this if the segment if dropped / empty */) {
mergeFinished.accept(info);
// clone the target info to make sure we have the original info without the updated del and update gens
origInfo = info.clone();
}
toWrap.onMergeComplete();
super.onMergeComplete();
}
@Override
void initMergeReaders(IOUtils.IOFunction<SegmentCommitInfo, MergePolicy.MergeReader> readerFactory) throws IOException {
if (onlyOnce.compareAndSet(false, true)) {
// 我们只在下面这样做一次,以将读者作为提交点的时间点读者
super.initMergeReaders(readerFactory);
}
}
@Override
public CodecReader wrapForMerge(CodecReader reader) throws IOException {
return toWrap.wrapForMerge(reader); // must delegate
}
}
), trigger, UNBOUNDED_MAX_MERGE_SEGMENTS);
if (pointInTimeMerges != null) {
boolean closeReaders = true;
try {
for (MergePolicy.OneMerge merge : pointInTimeMerges.merges) {
IOContext context = new IOContext(merge.getStoreMergeInfo());
merge.initMergeReaders(
sci -> {
final ReadersAndUpdates rld = getPooledInstance(sci, true);
// 调用 setIsMerging 很重要,因为它会导致 RaU 记录所有 DV 更新
// 在一个单独的地图中,以便在完成后应用于合并的段
rld.setIsMerging();
return rld.getReaderForMerge(context);
});
}
closeReaders = false;
} finally {
if (closeReaders) {
IOUtils.applyToAll(pointInTimeMerges.merges, merge -> {
// 合并被破坏了,我们需要在它之后进行清理 - 很好,我们仍然有 IW 锁来执行此操作
boolean removed = pendingMerges.remove(merge);
assert removed: "merge should be pending but isn't: " + merge.segString();
try {
abortOneMerge(merge);
} finally {
mergeFinish(merge);
}
});
}
}
}
return pointInTimeMerges;
}
/**
* 确保读取器池中的所有更改都写入磁盘。
* @param writeDeletes if <code>true</code> 如果删除也应该写入磁盘。
*/
private void writeReaderPool(boolean writeDeletes) throws IOException {
assert Thread.holdsLock(this);
if (writeDeletes) {
if (readerPool.commit(segmentInfos)) {
checkpointNoSIS();
}
} else { // only write the docValues
if (readerPool.writeAllDocValuesUpdates()) {
checkpoint();
}
}
// 现在尽最大努力检查一个段是否被完全删除
List<SegmentCommitInfo> toDrop = new ArrayList<>(); // don't modify segmentInfos in-place
for (SegmentCommitInfo info : segmentInfos) {
ReadersAndUpdates readersAndUpdates = readerPool.get(info, false);
if (readersAndUpdates != null) {
if (isFullyDeleted(readersAndUpdates)) {
toDrop.add(info);
}
}
}
for (SegmentCommitInfo info : toDrop) {
dropDeletedSegment(info);
}
if (toDrop.isEmpty() == false) {
checkpoint();
}
}