优化索引只能提高搜索速度,而不是索引速度。
索引优化过程中会消耗大量的CPU和I/O资源。
Optimize() :将索引压缩至一个段
Optimize(intmaxNumSegments) :部分优化:将索引最多压缩至maxNumSegments个段
Optimize(booleandoWait) : doWait=false的时候合并工作在后台运行,只适合用于后台线程调用合并程序,如默认的ConcurrentMergeScheduler
2.9版本以前的
参数名 | 参数作用 |
mergeFactor | 当大小相当的段数达到这个数的时候开始合并 |
minMergeSize | 所有大小小于此值的段,都按这个值算 |
maxMergeSize | 当段大小超过这个值,不再参与合并 |
maxMergeDocs | 一个段包含的文档数大于此值,不再参与合并 |
合并段由MergeScheduler执行,策略有MergePolicy决定。
比如mergeFactor=3,开始来的段大小为10M,当凑够3个10M的时候,0.cfs, 1.cfs, 2.cfs则合并成一个新的段3.cfs,大小为30M,然后再来4.cfs, 5.cfs, 6.cfs,合并成7.cfs,大小为30M,然后再来8.cfs,9.cfs, a.cfs合并成b.cfs, 大小为30M,这时候又凑够了3个30M的,合并成90M的c.cfs,然后又来d.cfs, e.cfs, f.cfs合并成10.cfs,大小为30M,然后11.cfs大小为10M,这时候硬盘上的段为:c.cfs(90M)10.cfs(30M),11.cfs(10M)
3.5版本以后
TieredMergePolicy根据每一层允许的段数合并大小相似的段。这个策略和2.9版本的LogByteSizeMergepolicy策略区别在于它可以合并不相邻的段,并且区分最多允许一次合并的段数和一层最多容许的段数。
对于一次普通的合并,首先计算一个“budget”。如果index超过了这个budget,排列所有段按照字节大小从大到小的顺序。找出最小cost的段。段的cost结合skew,总段大小以及pct删除回收计算。越小的Skew,越小的段总大小和更多的删除空间回收的段是cost最低的,也是最适合合并的。
Score= skew(最大的段在maxMergeAtOnce个段里面的比重)*pow(totAfterMergeByte,0.05)*
pow((double)(totAfterMergeByte/totBeforeMergeByte),reclaimDeletesWeight);
下面是个人书写翻译的段合并部分代码注释:
@Override
public MergeSpecification findMerges(SegmentInfos infos) throws IOException {
final Collection<SegmentInfo> merging = writer.get().getMergingSegments();//获取合并中的段
final Collection<SegmentInfo> toBeMerged = new HashSet<SegmentInfo>(); //获取将要合并的段
final List<SegmentInfo> infosSorted = new ArrayList<SegmentInfo>(infos.asList());
Collections.sort(infosSorted, segmentByteSizeDescending); //按字节大小降序排列段
long totIndexBytes = 0; //初始化总索引大小
long minSegmentBytes = Long.MAX_VALUE; //初始化最小值
//打印索引信息
for(SegmentInfo info : infosSorted) {
final long segBytes = size(info);
if (verbose()) {
String extra = merging.contains(info) ? " [merging]" : "";
if (segBytes >= maxMergedSegmentBytes/2.0) { //5*1024*1024*1024L (5GB) /2
extra += " [skip: too large]";
} else if (segBytes < floorSegmentBytes) { //2*1024*1024L (2MB)
extra += " [floored]";
}
}
minSegmentBytes = Math.min(segBytes, minSegmentBytes); //获取最小值
totIndexBytes += segBytes; //求总的索引字节数
}
//如果有过大的段,将它从Sorted队列里面剥离出来
int tooBigCount = 0;
while (tooBigCount < infosSorted.size() && size(infosSorted.get(tooBigCount)) >= maxMergedSegmentBytes/2.0) {
totIndexBytes -= size(infosSorted.get(tooBigCount));
tooBigCount++; //过大段的个数计数
}
minSegmentBytes = floorSize(minSegmentBytes); //和floorSegementBytes(2MB)比,取较大者
// 计算索引内部最大允许合并的段数
long levelSize = minSegmentBytes;
long bytesLeft = totIndexBytes;
double allowedSegCount = 0;
while(true) {
final double segCountLevel = bytesLeft / (double) levelSize; //计算最大需要的段数
if (segCountLevel < segsPerTier) { //segesPerTier 10
allowedSegCount += Math.ceil(segCountLevel);
break; //退出循环条件
}
allowedSegCount += segsPerTier; //如果比10大,最多的段数为10
bytesLeft -= segsPerTier * levelSize; //总大小重新计算
levelSize *= maxMergeAtOnce; //maxMergeAtOnce 一次循环合并10段
}
int allowedSegCountInt = (int) allowedSegCount;
MergeSpecification spec = null;
// Cycle to possibly select more than one merge:
while(true) {
long mergingBytes = 0;
// 获取所有可用的合并段
final List<SegmentInfo> eligible = new ArrayList<SegmentInfo>();
for(int idx = tooBigCount; idx<infosSorted.size(); idx++) {
final SegmentInfo info = infosSorted.get(idx);
if (merging.contains(info)) {
mergingBytes += info.sizeInBytes(true);
} else if (!toBeMerged.contains(info)) {
eligible.add(info);
}
}
//判断是否超过最大正在合并段的大小 5gb
final boolean maxMergeIsRunning = mergingBytes >= maxMergedSegmentBytes;
if (eligible.size() == 0) {
return spec; //没有可用段
}
if (eligible.size() >= allowedSegCountInt) { //大小超过最大容许范围,寻找最佳合并方案
MergeScore bestScore = null;
List<SegmentInfo> best = null;
boolean bestTooLarge = false;
long bestMergeBytes = 0;
// 大于一次合并段数,则进入条件
for(int startIdx = 0;startIdx <= eligible.size()-maxMergeAtOnce; startIdx++) {
long totAfterMergeBytes = 0;
final List<SegmentInfo> candidate = new ArrayList<SegmentInfo>();
boolean hitTooLarge = false;
//没有超过最大合并大小
for(int idx = startIdx;idx<eligible.size() && candidate.size() < maxMergeAtOnce;idx++) {
final SegmentInfo info = eligible.get(idx);
final long segBytes = size(info);
if (totAfterMergeBytes + segBytes > maxMergedSegmentBytes) { //检查是否过大
hitTooLarge = true;
continue;
}
candidate.add(info);
totAfterMergeBytes += segBytes;
}
//score最低的为最佳的合并方案
final MergeScore score = score(candidate, hitTooLarge, mergingBytes);
if ((bestScore == null || score.getScore() < bestScore.getScore()) && (!hitTooLarge || !maxMergeIsRunning)) {
best = candidate;
bestScore = score;
bestTooLarge = hitTooLarge;
bestMergeBytes = totAfterMergeBytes;
}
}
if (best != null) {
if (spec == null) {
spec = new MergeSpecification();
}
final OneMerge merge = new OneMerge(best);
spec.add(merge);
for(SegmentInfo info : merge.segments) {
toBeMerged.add(info);
}
} else {
return spec;
}
} else {
return spec;
}
}
}
/** 为段评分,可以重写该类*/
protected MergeScore score(List<SegmentInfo> candidate, boolean hitTooLarge, long mergingBytes) throws IOException {
long totBeforeMergeBytes = 0;
long totAfterMergeBytes = 0;
long totAfterMergeBytesFloored = 0;
for(SegmentInfo info : candidate) {
final long segBytes = size(info);
totAfterMergeBytes += segBytes;
totAfterMergeBytesFloored += floorSize(segBytes);
totBeforeMergeBytes += info.sizeInBytes(true);
}
//计算skew 1.0/numSegsBeingMerged (good) to 1.0
final double skew;
if (hitTooLarge) {
//这个skew没有意义
skew = 1.0/maxMergeAtOnce;
} else {
skew = ((double) floorSize(size(candidate.get(0))))/totAfterMergeBytesFloored;
}
// smaller mergeScore is better
double mergeScore = skew;
//指数缩放
mergeScore *= Math.pow(totAfterMergeBytes, 0.05);
//回收删除:
final double nonDelRatio = ((double) totAfterMergeBytes)/totBeforeMergeBytes;
mergeScore *= Math.pow(nonDelRatio, reclaimDeletesWeight);
final double finalMergeScore = mergeScore;
return new MergeScore() {
@Override
public double getScore() {
return finalMergeScore;
}
@Override
public String getExplanation() {
return "skew=" + String.format("%.3f", skew) + " nonDelRatio=" + String.format("%.3f", nonDelRatio);
}
};
}