之前我们说道,VTM帧内预测在对RMD
和MPM
构建的候选列表进行full RDO的时候,会调用xRecurIntraCodingLumaQT()
进行信号完整预测、变换和量化,选择变换核,设置cbf
,计算完整率失真代价。
xRecurIntraCodingLumaQT()
要点如下:
- 确定tu是否需要继续划分
– 如果划分(isp),递归调用自身 - 确定变换模式候选集
- 遍历变换模式,调用
xIntraCodingTUBlock()
计算残差并进行变换量化,反变换,反量化,计算重建信号
void IntraSearch::xRecurIntraCodingLumaQT( CodingStructure &cs, Partitioner &partitioner, const double bestCostSoFar, const int subTuIdx, const PartSplit ispType )
{
int subTuCounter = subTuIdx; // tu idx (if isp, otherwise equal to -1)
const UnitArea &currArea = partitioner.currArea();
const CodingUnit &cu = *cs.getCU( currArea.lumaPos(), partitioner.chType );
bool earlySkipISP = false; // if early skip isp
uint32_t currDepth = partitioner.currTrDepth;
const PPS &pps = *cs.pps;
const bool keepResi = pps.getPpsRangeExtension().getCrossComponentPredictionEnabledFlag() || KEEP_PRED_AND_RESI_SIGNALS; // 默认为0, for 4:2:0
bool bCheckFull = true; // pu不可继续划分
bool bCheckSplit = false; // pu继续划分
bCheckFull = !partitioner.canSplit( TU_MAX_TR_SPLIT, cs ); // always false here
bCheckSplit = partitioner.canSplit( TU_MAX_TR_SPLIT, cs ); // always true here
#if JVET_M0102_INTRA_SUBPARTITIONS
if( cu.ispMode )
{
bCheckSplit = partitioner.canSplit( ispType, cs ); // if isp, update bchecksplit
bCheckFull = !bCheckSplit;
}
#endif
uint32_t numSig = 0;
double dSingleCost = MAX_DOUBLE; // 最优rdcost
Distortion uiSingleDistLuma = 0; // 最优distortion
uint64_t singleFracBits = 0; // 最优bits
int bestModeId[MAX_NUM_COMPONENT] = { 0, 0, 0 }; // 记录最优tu模式
const TempCtx ctxStart ( m_CtxCache, m_CABACEstimator->getCtx() ); // 获取上下文信息
TempCtx ctxBest ( m_CtxCache );
CodingStructure *csSplit = nullptr;
CodingStructure *csFull = nullptr;
if( bCheckSplit ) // 根据pu划分类型,更新csSplit 或 csFull
{
csSplit = &cs;
}
else if( bCheckFull )
{
csFull = &cs;
}
if( bCheckFull )
{
csFull->cost = 0.0;
// add tu and initial
TransformUnit &tu = csFull->addTU( CS::getArea( *csFull, currArea, partitioner.chType ), partitioner.chType );
tu.depth = currDepth;
const bool tsAllowed = TU::isTSAllowed ( tu, COMPONENT_Y ); // if transform skip allowed 是否进行TS模式
const bool mtsAllowed = TU::isMTSAllowed( tu, COMPONENT_Y ); // 是否进行mts模式选择 is mts alled -> not isp and not sub-block transform(sbt only for inter blocks)
uint8_t nNumTransformCands = 1 + ( tsAllowed ? 1 : 0 ) + ( mtsAllowed ? 4 : 0 ); // DCT + TS + 4 MTS = 6 tests,总变换模式候选
std::vector<TrMode> trModes; // typedef std::pair<int, bool> TrMode; 存放变换模式候选
trModes.push_back( TrMode( 0, true ) ); // DCT2 默认添加
if( tsAllowed )
{
trModes.push_back( TrMode( 1, true ) );
}
if( mtsAllowed )
{
for( int i = 2; i < 6; i++ )
{
trModes.push_back( TrMode( i, true) );
}
}
CHECK( !tu.Y().valid(), "Invalid TU" );
CodingStructure &saveCS = *m_pSaveCS[0];
TransformUnit *tmpTU = nullptr;
Distortion singleDistTmpLuma = 0;
uint64_t singleTmpFracBits = 0;
double singleCostTmp = 0;
int firstCheckId = 0; // 第一次标志位
#if JVET_M0464_UNI_MTS
int lastCheckId = trModes[nNumTransformCands-1].first; // 最后一个tu模式
bool isNotOnlyOneMode = nNumTransformCands != 1; // 是否只能进行DCT II
#else
//we add the EMT candidates to the loop. TransformSkip will still be the last one to be checked (when modeId == lastCheckId) as long as checkTransformSkip is true
int lastCheckId = numTransformIndexCands - ( firstCheckId + 1 ) + ( int ) checkTransformSkip;
bool isNotOnlyOneMode = lastCheckId != firstCheckId && !checkInitTrDepthTransformSkipWinner;
#endif
if( isNotOnlyOneMode ) // 如果不止dctII一个变换核,保存临时数据
{
saveCS.pcv = cs.pcv;
saveCS.picture = cs.picture;
saveCS.area.repositionTo(cs.area);
saveCS.clearTUs();
tmpTU = &saveCS.addTU(currArea, partitioner.chType);
}
#if JVET_M0464_UNI_MTS
bool cbfDCT2 = true; //
#else
bool cbfBestMode = false;
#endif
for( int modeId = firstCheckId; modeId < nNumTransformCands; modeId++ ) // 遍历所有tu模式
{
// 如果dct2的cbf==0或者TS为亮度最优tu模式,则终止变换核选择
if( !cbfDCT2 || ( m_pcEncCfg->getUseTransformSkipFast() && bestModeId[COMPONENT_Y] == 1 ) )
{
break;
}
if( !trModes[modeId].second ) // 如果当前tu模式关闭,跳到下一个模式
{
continue;
}
tu.mtsIdx = trModes[modeId].first; // 更新当前tu变换核
// 不是dct2,且不止一个模式时,更新上下文信息
if ((modeId != firstCheckId) && isNotOnlyOneMode)
{
m_CABACEstimator->getCtx() = ctxStart;
}
int default0Save1Load2 = 0;
singleDistTmpLuma = 0;
#if JVET_M0464_UNI_MTS
if( modeId == firstCheckId && nNumTransformCands > 1 ) // 第一次循环(DCT2),且不止一个tu模式时,更新default0Save1Load2 为 1
#else
if (modeId == firstCheckId && modeId != lastCheckId && !checkInitTrDepthTransformSkipWinner )
#endif
{
default0Save1Load2 = 1;
}
else if (modeId != firstCheckId)// 不是(DCT2)更新default0Save1Load2 为 2
{
default0Save1Load2 = 2;
}
#if JVET_M0102_INTRA_SUBPARTITIONS
if( cu.ispMode ) // isp预测模式,default0Save1Load2置0
{
default0Save1Load2 = 0;
}
#endif
if( nNumTransformCands > 1 ) // 如果不止一个tu模式,在第一次信号预测,并对残差进行变换量化后,记录最后一个有效的tu模式索引
{
// 对预测信号计算残差,并进行变换量化,计算distortion
// dct2时,更新tu模式候选的有效标志?
xIntraCodingTUBlock( tu, COMPONENT_Y, false, singleDistTmpLuma, default0Save1Load2, &numSig, modeId == 0 ? &trModes : nullptr, true );
if( modeId == 0 ) // 记录
{
for( int i = 0; i < nNumTransformCands; i++ )
{
if( trModes[i].second )
{
lastCheckId = trModes[i].first;
}
}
}
}
else
{
// 只有DCT2,进行预测变换量化
xIntraCodingTUBlock( tu, COMPONENT_Y, false, singleDistTmpLuma, default0Save1Load2, &numSig );
}
//----- determine rate and r-d cost ----- 计算率失真代价
#if JVET_M0464_UNI_MTS
if( ( trModes[modeId].first != 0 && !TU::getCbfAtDepth( tu, COMPONENT_Y, currDepth ) ) )
#else
//the condition (transformIndex != DCT2_EMT) seems to be irrelevant, since DCT2_EMT=7 and the highest value of transformIndex is 4
if( ( modeId == lastCheckId && checkTransformSkip && !TU::getCbfAtDepth( tu, COMPONENT_Y, currDepth ) ) )
#endif
{
//In order not to code TS flag when cbf is zero, the case for TS with cbf being zero is forbidden.
// 为了保证在cbf不为0时,不对TS标志进行编码,当cbf为0时,禁止TS,将代价设为最大
singleCostTmp = MAX_DOUBLE;
}
else
{
#if JVET_M0102_INTRA_SUBPARTITIONS
if( cu.ispMode && m_pcRdCost->calcRdCost( csFull->fracBits, csFull->dist + singleDistTmpLuma ) > bestCostSoFar )
{ // 如果是ISP模式,且进行到当前这个tu时,RDCost已经比之前最优的RDCost大,终止ISP模式
earlySkipISP = true;
}
else
{ // 计算bits(模式头信息,变换量化后的系数,cbf)
singleTmpFracBits = xGetIntraFracBitsQT( *csFull, partitioner, true, false, subTuCounter, ispType );
}
#else
singleTmpFracBits = xGetIntraFracBitsQT( *csFull, partitioner, true, false );
#endif
singleCostTmp = m_pcRdCost->calcRdCost( singleTmpFracBits, singleDistTmpLuma ); // 依据bits和distortion 计算RDCost
}
if (singleCostTmp < dSingleCost) // 当前模式RD代价更小,更新最优率失真信息
{
dSingleCost = singleCostTmp;
uiSingleDistLuma = singleDistTmpLuma;
singleFracBits = singleTmpFracBits;
#if JVET_M0464_UNI_MTS
bestModeId[COMPONENT_Y] = trModes[modeId].first; // 最优变换模式
if( trModes[modeId].first == 0 ) // 记录DCT2的cbf,用于提前终止
{
cbfDCT2 = TU::getCbfAtDepth( tu, COMPONENT_Y, currDepth );
}
#else
bestModeId[COMPONENT_Y] = modeId;
cbfBestMode = TU::getCbfAtDepth( tu, COMPONENT_Y, currDepth );
#endif
if( bestModeId[COMPONENT_Y] != lastCheckId ) // 如果当前tu模式不是最后一个模式,记录最优的预测信号和重建信号,
{
#if KEEP_PRED_AND_RESI_SIGNALS || JVET_M0427_INLOOP_RESHAPER
saveCS.getPredBuf( tu.Y() ).copyFrom( csFull->getPredBuf( tu.Y() ) );
#endif
saveCS.getRecoBuf( tu.Y() ).copyFrom( csFull->getRecoBuf( tu.Y() ) );
if( keepResi )
{
saveCS.getResiBuf ( tu.Y() ).copyFrom( csFull->getResiBuf ( tu.Y() ) );
saveCS.getOrgResiBuf( tu.Y() ).copyFrom( csFull->getOrgResiBuf( tu.Y() ) );
}
tmpTU->copyComponentFrom( tu, COMPONENT_Y );
ctxBest = m_CABACEstimator->getCtx(); // 更新上下文
}
}
}
if( bestModeId[COMPONENT_Y] != lastCheckId ) // 如果最优tu模式不是最后测试的模式,记录最后一次预测变换重建信息
{
#if KEEP_PRED_AND_RESI_SIGNALS || JVET_M0427_INLOOP_RESHAPER
csFull->getPredBuf( tu.Y() ).copyFrom( saveCS.getPredBuf( tu.Y() ) );
#endif
csFull->getRecoBuf( tu.Y() ).copyFrom( saveCS.getRecoBuf( tu.Y() ) );
if( keepResi )
{
csFull->getResiBuf ( tu.Y() ).copyFrom( saveCS.getResiBuf ( tu.Y() ) );
csFull->getOrgResiBuf( tu.Y() ).copyFrom( saveCS.getOrgResiBuf( tu.Y() ) );
}
tu.copyComponentFrom( *tmpTU, COMPONENT_Y );
if( !bCheckSplit )
{
m_CABACEstimator->getCtx() = ctxBest;
}
}
else if( bCheckSplit )
{
ctxBest = m_CABACEstimator->getCtx();
}
// 更新率失真代价信息
csFull->cost += dSingleCost;
csFull->dist += uiSingleDistLuma;
csFull->fracBits += singleFracBits;
}
if( bCheckSplit ) // 如果tu需要划分(isp),计算各个子tu的rdcost
{
//----- store full entropy coding status, load original entropy coding status -----
if( bCheckFull )
{
m_CABACEstimator->getCtx() = ctxStart;
}
//----- code splitted block -----
csSplit->cost = 0;
bool uiSplitCbfLuma = false;
bool splitIsSelected = true; // 最优的是否是划分
if( partitioner.canSplit( TU_MAX_TR_SPLIT, cs ) )
{
partitioner.splitCurrArea( TU_MAX_TR_SPLIT, cs );
}
#if JVET_M0102_INTRA_SUBPARTITIONS
if( cu.ispMode )
{
partitioner.splitCurrArea( ispType, *csSplit ); // 区域划分
}
#endif
do
{
#if JVET_M0102_INTRA_SUBPARTITIONS
xRecurIntraCodingLumaQT( *csSplit, partitioner, bestCostSoFar, subTuCounter, ispType ); // 递归调用,计算子区域rdcost,cbf,残差变换量化
subTuCounter += subTuCounter != -1 ? 1 : 0;
#else
xRecurIntraCodingLumaQT( *csSplit, partitioner );
#endif
#if JVET_M0102_INTRA_SUBPARTITIONS
if( !cu.ispMode )
{
csSplit->setDecomp( partitioner.currArea().Y() );
}
else if( CU::isISPFirst( cu, partitioner.currArea().Y(), COMPONENT_Y ) )
{
csSplit->setDecomp( cu.Y() ); // 更新重建
}
#else
csSplit->setDecomp( partitioner.currArea().Y() );
#endif
#if JVET_M0102_INTRA_SUBPARTITIONS
uiSplitCbfLuma |= TU::getCbfAtDepth( *csSplit->getTU( partitioner.currArea().lumaPos(), partitioner.chType, subTuCounter - 1 ), COMPONENT_Y, partitioner.currTrDepth );
if( cu.ispMode )
{
//exit condition if the accumulated cost is already larger than the best cost so far (no impact in RD performance) 终止,如果cost已经大于最优cost
if( csSplit->cost > bestCostSoFar )
{
earlySkipISP = true;
splitIsSelected = false;
break;
}
else
{
//more restrictive exit condition
bool tuIsDividedInRows = CU::divideTuInRows( cu );
int nSubPartitions = tuIsDividedInRows ? cu.lheight() >> g_aucLog2[cu.firstTU->lheight()] : cu.lwidth() >> g_aucLog2[cu.firstTU->lwidth()];
double threshold = nSubPartitions == 2 ? 0.95 : subTuCounter == 1 ? 0.83 : 0.91;
if( subTuCounter < nSubPartitions && csSplit->cost > bestCostSoFar*threshold )
{
earlySkipISP = true;
splitIsSelected = false;
break;
}
}
}
#else
uiSplitCbfLuma |= TU::getCbfAtDepth( *csSplit->getTU( partitioner.currArea().lumaPos(), partitioner.chType ), COMPONENT_Y, partitioner.currTrDepth );
#endif
} while( partitioner.nextPart( *csSplit ) );
partitioner.exitCurrSplit();
if( splitIsSelected )
{ // 如果选择split,遍历tus,设置cbf
for( auto &ptu : csSplit->tus )
{
if( currArea.Y().contains( ptu->Y() ) )
{
TU::setCbfAtDepth( *ptu, COMPONENT_Y, currDepth, uiSplitCbfLuma ? 1 : 0 );
}
}
//----- restore context states -----
m_CABACEstimator->getCtx() = ctxStart;
//----- determine rate and r-d cost -----
#if JVET_M0102_INTRA_SUBPARTITIONS
csSplit->fracBits = xGetIntraFracBitsQT( *csSplit, partitioner, true, false, cu.ispMode ? 0 : -1, ispType );
#else
csSplit->fracBits = xGetIntraFracBitsQT(*csSplit, partitioner, true, false);
#endif
//--- update cost ---
csSplit->cost = m_pcRdCost->calcRdCost(csSplit->fracBits, csSplit->dist);
}
}
if( csFull || csSplit ) // 更新picture信号信息,计算cs rdcost
{
{
// otherwise this would've happened in useSubStructure
cs.picture->getRecoBuf( currArea.Y() ).copyFrom( cs.getRecoBuf( currArea.Y() ) );
#if JVET_M0427_INLOOP_RESHAPER
cs.picture->getPredBuf( currArea.Y() ).copyFrom( cs.getPredBuf( currArea.Y() ) );
#endif
}
#if JVET_M0102_INTRA_SUBPARTITIONS
if( cu.ispMode && earlySkipISP )
{
cs.cost = MAX_DOUBLE;
}
else
{
cs.cost = m_pcRdCost->calcRdCost( cs.fracBits, cs.dist );
}
#else
cs.cost = m_pcRdCost->calcRdCost( cs.fracBits, cs.dist );
#endif
}
}