------VTM9.0更新----
xRecurIntraCodingLumaQT函数主要是使用上层函数选中的预测模式,遍历上层函数传来的变换模式,通过调用xIntraCodingTUBlock计算相应的失真,从而计算RD Cost,从而选出最佳变换。主要流程如下:
- 确定当前分区是否可以继续划分,如果需要划分则递归调用自身。
- 确定候选模式变换集
- 遍历候选模式遍历集,调用xIntraCodingTUBlock函数进行预测计算残差、进行变换、量化、重建,并计算失真。
- 调用xGetIntraFracBitsQT函数进行编码计算比特数和调用calcRdCost函数计算RD Cost
确定候选模式变换集分为两种情况:
(1)SPS层开启LFNST、当前块允许TransformSkip、使用DCT-2变换且lfnstIdx==0
此时,需要检查DCT-2变换和TransformSkip,从二者之间选出最佳值
(2)SPS层关闭LFNST
如果此时允许TransformSkip和MTS,则待测候选变换模式为DCT-2、TS以及MTS的四种组合
注意:
TrModes变量包含两个值,第一个是int值,表示变换模式,第二个值是bool值,表示该模式是否可用
如果TrModes变量的size大于1时,则会通过xIntraCodingTUBlock函数调用第一种TransformNxN函数,该函数比较TrModes中的几种变换后的绝对残差系数和,来修改TrModes中的bool值,然后在xRecurIntraCodingLumaQT函数中决定是否需要跳过该模式。
具体代码注释如下:
bool IntraSearch::xRecurIntraCodingLumaQT( CodingStructure &cs, Partitioner &partitioner, const double bestCostSoFar, const int subTuIdx, const PartSplit ispType, const bool ispIsCurrentWinner, bool mtsCheckRangeFlag, int mtsFirstCheckId, int mtsLastCheckId, bool moreProbMTSIdxFirst )
{
int subTuCounter = subTuIdx;
const UnitArea &currArea = partitioner.currArea();
const CodingUnit &cu = *cs.getCU( currArea.lumaPos(), partitioner.chType );
bool earlySkipISP = false;
uint32_t currDepth = partitioner.currTrDepth;
const SPS &sps = *cs.sps;
//检查是否需要继续划分,即判断当前TU宽度和高度是否大于最大支持变换尺寸
bool bCheckFull = true;
bool bCheckSplit = false;
bCheckFull = !partitioner.canSplit( TU_MAX_TR_SPLIT, cs );
bCheckSplit = partitioner.canSplit( TU_MAX_TR_SPLIT, cs );
#if JVET_R0110_MIXED_LOSSLESS
const Slice &slice = *cs.slice;
#endif
if( cu.ispMode )//当前CU是ISP模式 是否需要该判断???
{
bCheckSplit = partitioner.canSplit( ispType, cs );
bCheckFull = !bCheckSplit;
}
uint32_t numSig = 0;
double dSingleCost = MAX_DOUBLE;//失真
Distortion uiSingleDistLuma = 0;
uint64_t singleFracBits = 0;//比特数
bool checkTransformSkip = sps.getTransformSkipEnabledFlag();//检查变换跳过标志
int bestModeId[ MAX_NUM_COMPONENT ] = { 0, 0, 0 };
//cu.mtsflag在xCheckRDCostIntra函数里控制,如果为0表示一次变换为DCT-2,否则为DST7与DCT8的四种组合
uint8_t nNumTransformCands = cu.mtsFlag ? 4 : 1;
uint8_t numTransformIndexCands = nNumTransformCands;
//初始化上下文
const TempCtx ctxStart ( m_CtxCache, m_CABACEstimator->getCtx() );
TempCtx ctxBest ( m_CtxCache );
CodingStructure *csSplit = nullptr;
CodingStructure *csFull = nullptr;
CUCtx cuCtx;
cuCtx.isDQPCoded = true;
cuCtx.isChromaQpAdjCoded = true;
if( bCheckSplit )
{
csSplit = &cs;
}
else if( bCheckFull )
{
csFull = &cs;
}
bool validReturnFull = false;
if( bCheckFull )//不需要进一步划分
{
csFull->cost = 0.0;
TransformUnit &tu = csFull->addTU( CS::getArea( *csFull, currArea, partitioner.chType ), partitioner.chType );
tu.depth = currDepth;
const bool tsAllowed = TU::isTSAllowed( tu, COMPONENT_Y );//当前TU是否允许TransformSkip
const bool mtsAllowed = CU::isMTSAllowed( cu, COMPONENT_Y );//当前CU是否允许MTS
//trModes的第一个值是变换模式,第二个值表示是否使用该模式
std::vector<TrMode> trModes;
if( sps.getUseLFNST() )//SPS层开启LFNST
{
//判断是否需要检查TransformSkip
checkTransformSkip &= tsAllowed;
checkTransformSkip &= !cu.mtsFlag;
checkTransformSkip &= !cu.lfnstIdx;
if( !cu.mtsFlag && checkTransformSkip )
{
trModes.push_back( TrMode( 0, true ) ); //DCT2
trModes.push_back( TrMode( 1, true ) ); //TS
}
}
else //SPS层关闭LFNST
{
//此时如果需要检查TS和MTS,则总共需要检查6种变换模式
nNumTransformCands = 1 + ( tsAllowed ? 1 : 0 ) + ( mtsAllowed ? 4 : 0 ); // DCT + TS + 4 MTS = 6 tests
#if JVET_R0110_MIXED_LOSSLESS
if (m_pcEncCfg->getCostMode() == COST_LOSSLESS_CODING && slice.isLossless())//无损编码
#else
if (m_pcEncCfg->getCostMode() == COST_LOSSLESS_CODING)
#endif
{
nNumTransformCands = 1;
CHECK(!tsAllowed && !cu.bdpcmMode, "transform skip should be enabled for LS");
if (cu.bdpcmMode)
{
trModes.push_back(TrMode(0, true));
}
else
{
trModes.push_back(TrMode(1, true));
}
}
else //将6种变换模式保存在trModes中
{
trModes.push_back( TrMode( 0, true ) ); //DCT2
if( tsAllowed )
{
trModes.push_back( TrMode( 1, true ) );
}
if( mtsAllowed )
{
for( int i = 2; i < 6; i++ )
{
trModes.push_back( TrMode( i, true ) );
}
}
}
}
CHECK( !tu.Y().valid(), "Invalid TU" );
CodingStructure &saveCS = *m_pSaveCS[0];
TransformUnit *tmpTU = nullptr;
Distortion singleDistTmpLuma = 0;//失真
uint64_t singleTmpFracBits = 0;//比特数
double singleCostTmp = 0;//RD Cost
//mtsFirstCheckId和mtsLastCheckId参数仅在mtsCheckRangeFlag为true时才会用到
//如果SPS层开启LFNST且当前正在检查MTS且需要检查后三种MTS组合
int firstCheckId = ( sps.getUseLFNST() && mtsCheckRangeFlag && cu.mtsFlag ) ? mtsFirstCheckId : 0;
//we add the MTS candidates to the loop. TransformSkip will still be the last one to be checked (when modeId == lastCheckId) as long as checkTransformSkip is true
//我们将MTS候选添加到循环中。只要checkTransformSkip为true,TransformSkip仍然是最后一个要检查的(当modeId == lastCheckId时)
int lastCheckId = sps.getUseLFNST() ? ( ( mtsCheckRangeFlag && cu.mtsFlag ) ? ( mtsLastCheckId + ( int ) checkTransformSkip ) : ( numTransformIndexCands - ( firstCheckId + 1 ) + ( int ) checkTransformSkip ) ) :
trModes[ nNumTransformCands - 1 ].first;
//不只检查一种模式
bool isNotOnlyOneMode = sps.getUseLFNST() ? lastCheckId != firstCheckId : nNumTransformCands != 1;
if( isNotOnlyOneMode )
{
saveCS.pcv = cs.pcv;
saveCS.picture = cs.picture;
saveCS.area.repositionTo(cs.area);
saveCS.clearTUs();
tmpTU = &saveCS.addTU(currArea, partitioner.chType);
}
bool cbfBestMode = false;//最佳的CBF值
bool cbfBestModeValid = false;
bool cbfDCT2 = true;
double bestDCT2cost = MAX_DOUBLE;//DCT-2变换的最佳Cost
double threshold = m_pcEncCfg->getUseFastISP() && !cu.ispMode && ispIsCurrentWinner && nNumTransformCands > 1 ? 1 + 1.4 / sqrt( cu.lwidth() * cu.lheight() ) : 1;
//遍历所有可能的变换模式
for( int modeId = firstCheckId; modeId <= ( sps.getUseLFNST() ? lastCheckId : ( nNumTransformCands - 1 ) ); modeId++ )
{
uint8_t transformIndex = modeId;
if( sps.getUseLFNST() )//SPS层开启LFNST
{
//如果当前检查的模式不是最后一种模式或者当前检查的是最后一种模式且该模式不是变换跳过模式
if( ( transformIndex < lastCheckId ) || ( ( transformIndex == lastCheckId ) && !checkTransformSkip ) ) //we avoid this if the mode is transformSkip
{
// Skip checking other transform candidates if zero CBF is encountered and it is the best transform so far
// 如果遇到零CBF,则跳过检查其他候选变换,这是迄今为止最好的转换
if( m_pcEncCfg->getUseFastLFNST() && transformIndex && !cbfBestMode && cbfBestModeValid )
{
continue;
}
}
}
else//SPS层关闭LFNST
{
#if JVET_R0110_MIXED_LOSSLESS
if (!(m_pcEncCfg->getCostMode() == COST_LOSSLESS_CODING && slice.isLossless()))//无损编码
#else
if( !( m_pcEncCfg->getCostMode() == COST_LOSSLESS_CODING) )
#endif
{
if( !cbfDCT2 || ( m_pcEncCfg->getUseTransformSkipFast() && bestModeId[ COMPONENT_Y ] == MTS_SKIP))
{
break;
}
if( !trModes[ modeId ].second )
{
continue;
}
//we compare the DCT-II cost against the best ISP cost so far (except for TS)
if (m_pcEncCfg->getUseFastISP() && !cu.ispMode && ispIsCurrentWinner && trModes[modeId].first != MTS_DCT2_DCT2 && (trModes[modeId].first != MTS_SKIP || !tsAllowed) && bestDCT2cost > bestCostSoFar * threshold)
{
continue;
}
}
tu.mtsIdx[COMPONENT_Y] = trModes[modeId].first;
}
if ((modeId != firstCheckId) && isNotOnlyOneMode)
{
m_CABACEstimator->getCtx() = ctxStart;
}
int default0Save1Load2 = 0;
singleDistTmpLuma = 0;
//如果当前检查的是第一种变换模式
//且当SPS层开启LFNST时,检查的不是最后一种模式
//且当SPS层关闭LFNST时,待检测模式大于1
//default0Save1Load2该参数为1时在xIntraCodingTUBlock函数中进行预测时将预测信号保存
//default0Save1Load2该参数为2时在xIntraCodingTUBlock函数中之间加载预测信号
if( modeId == firstCheckId && ( sps.getUseLFNST() ? ( modeId != lastCheckId ) : ( nNumTransformCands > 1 ) ) )
{
default0Save1Load2 = 1;
}
else if (modeId != firstCheckId)//当前检查的不是第一个模式
{
if( sps.getUseLFNST() && !cbfBestModeValid )//SPS层开启LFNST且最佳CBF不可用时
{
default0Save1Load2 = 1;
}
else
{
default0Save1Load2 = 2;
}
}
if( cu.ispMode )//当前CU是ISP模式
{
default0Save1Load2 = 0;
}
if( sps.getUseLFNST() )//SPS层开启LFNST
{
if( cu.mtsFlag )//当前正在检查MTS
{
if( moreProbMTSIdxFirst )
{
const ChannelType chType = toChannelType( COMPONENT_Y );
const CompArea& area = tu.blocks[ COMPONENT_Y ];
const PredictionUnit& pu = *cs.getPU( area.pos(), chType );
uint32_t uiIntraMode = pu.intraDir[ chType ];
if( transformIndex == 1 )
{
tu.mtsIdx[COMPONENT_Y] = (uiIntraMode < 34) ? MTS_DST7_DCT8 : MTS_DCT8_DST7;
}
else if( transformIndex == 2 )
{
tu.mtsIdx[COMPONENT_Y] = (uiIntraMode < 34) ? MTS_DCT8_DST7 : MTS_DST7_DCT8;
}
else
{
tu.mtsIdx[COMPONENT_Y] = MTS_DST7_DST7 + transformIndex;
}
}
else
{
tu.mtsIdx[COMPONENT_Y] = MTS_DST7_DST7 + transformIndex;
}
}
else
{
tu.mtsIdx[COMPONENT_Y] = transformIndex;
}
//如果当前检查的不是MTS(即DCT-2)且需要检查TransformSkip
//xIntraCodingTUBlock进行预测、变换之后再进行反变换后获得重建值,然后计算相应的失真
if( !cu.mtsFlag && checkTransformSkip )
{
xIntraCodingTUBlock( tu, COMPONENT_Y, singleDistTmpLuma, default0Save1Load2, &numSig, modeId == 0 ? &trModes : nullptr, true );
if( modeId == 0 )
{
for( int i = 0; i < 2; i++ )
{
//根据DCT-2变换和TS变换后的绝对残差系数和决定是否需要跳过TS变换
if( trModes[ i ].second )//根据是否使用该变换模式修改最后一个测试模式
{
lastCheckId = trModes[ i ].first;
}
}
}
}
else
{
xIntraCodingTUBlock( tu, COMPONENT_Y, singleDistTmpLuma, default0Save1Load2, &numSig );
}
}//End if( sps.getUseLFNST() )
else
{
if( nNumTransformCands > 1 )
{
xIntraCodingTUBlock( tu, COMPONENT_Y, singleDistTmpLuma, default0Save1Load2, &numSig, modeId == 0 ? &trModes : nullptr, true );
if( modeId == 0 )
{
for( int i = 0; i < nNumTransformCands; i++ )
{
if( trModes[ i ].second )
{
lastCheckId = trModes[ i ].first;
}
}
}
}
else
{
xIntraCodingTUBlock( tu, COMPONENT_Y, singleDistTmpLuma, default0Save1Load2, &numSig );
}
}
cuCtx.mtsLastScanPos = false;
//----- determine rate and r-d cost -----
//----- 确定码率和RD Cost -----
//SPS层开启LFNST时判断当前模式是否是最后一个测试模式且模式号不等于0且需要检查TransformSkip
//SPS层关闭LFNST时,判断当前模式不是DCT-2变换
if( ( sps.getUseLFNST() ? ( modeId == lastCheckId && modeId != 0 && checkTransformSkip ) : ( trModes[ modeId ].first != 0 ) ) && !TU::getCbfAtDepth( tu, COMPONENT_Y, currDepth ) )
{
//In order not to code TS flag when cbf is zero, the case for TS with cbf being zero is forbidden.
//为了在cbf为零时不编码TS标志,禁止cbf为零的TS情况。
#if JVET_R0110_MIXED_LOSSLESS
if (m_pcEncCfg->getCostMode() != COST_LOSSLESS_CODING || !slice.isLossless())//无损编码
#else
if (m_pcEncCfg->getCostMode() != COST_LOSSLESS_CODING)
#endif
singleCostTmp = MAX_DOUBLE;
else
{
singleTmpFracBits = xGetIntraFracBitsQT(*csFull, partitioner, true, false, subTuCounter, ispType, &cuCtx);
singleCostTmp = m_pcRdCost->calcRdCost(singleTmpFracBits, singleDistTmpLuma);
}
}
else
{
if( cu.ispMode && m_pcRdCost->calcRdCost( csFull->fracBits, csFull->dist + singleDistTmpLuma ) > bestCostSoFar )
{
earlySkipISP = true;
}
else
{ //xGetIntraFracBitsQT函数对当前变换模式及残差进行编码,获取所需编码比特数
singleTmpFracBits = xGetIntraFracBitsQT( *csFull, partitioner, true, false, subTuCounter, ispType, &cuCtx );
}
if (tu.mtsIdx[COMPONENT_Y] > MTS_SKIP)//如果当前变换模式是MTS的四种组合
{
if (!cuCtx.mtsLastScanPos)//如果没有编码MTS的最后非零系数位置
{
singleCostTmp = MAX_DOUBLE;
}
else
{
singleCostTmp = m_pcRdCost->calcRdCost(singleTmpFracBits, singleDistTmpLuma);//计算RD Cost
}
}
else
singleCostTmp = m_pcRdCost->calcRdCost( singleTmpFracBits, singleDistTmpLuma );//计算RD Cost
}
//如果不是ISP模式且待测模式大于1且当前检查的模式是第一个检查模式
if ( !cu.ispMode && nNumTransformCands > 1 && modeId == firstCheckId )
{
bestDCT2cost = singleCostTmp;//保存DCT-2变换的RD Cost
}
if (singleCostTmp < dSingleCost)//当前Cost小于最佳Cost
{
dSingleCost = singleCostTmp;
uiSingleDistLuma = singleDistTmpLuma;
singleFracBits = singleTmpFracBits;
if( sps.getUseLFNST() ) //SPS层开启LFNST
{
//设置最佳模式和最佳CBF
bestModeId[ COMPONENT_Y ] = modeId;
cbfBestMode = TU::getCbfAtDepth( tu, COMPONENT_Y, currDepth );
cbfBestModeValid = true;
validReturnFull = true;
}
else //SPS层关闭LFNST
{
bestModeId[ COMPONENT_Y ] = trModes[ modeId ].first;
if( trModes[ modeId ].first == 0 )
{
cbfDCT2 = TU::getCbfAtDepth( tu, COMPONENT_Y, currDepth );
}
}
if( bestModeId[COMPONENT_Y] != lastCheckId )//最佳模式不是最后检查的模式
{
//将当前的预测和重建信号保存下来
saveCS.getPredBuf( tu.Y() ).copyFrom( csFull->getPredBuf( tu.Y() ) );
saveCS.getRecoBuf( tu.Y() ).copyFrom( csFull->getRecoBuf( tu.Y() ) );
if( KEEP_PRED_AND_RESI_SIGNALS )
{
saveCS.getResiBuf ( tu.Y() ).copyFrom( csFull->getResiBuf ( tu.Y() ) );
saveCS.getOrgResiBuf( tu.Y() ).copyFrom( csFull->getOrgResiBuf( tu.Y() ) );
}
tmpTU->copyComponentFrom( tu, COMPONENT_Y );
ctxBest = m_CABACEstimator->getCtx();
}
}
}//End for(mode)
//如果SPS层开启LFNST且没有有效的返回值
if( sps.getUseLFNST() && !validReturnFull )
{
csFull->cost = MAX_DOUBLE;
if( bCheckSplit )
{
ctxBest = m_CABACEstimator->getCtx();
}
}
else
{
//最佳的模式不是最后一个检查的模式(说明提前截止)
if( bestModeId[COMPONENT_Y] != lastCheckId )
{
csFull->getPredBuf( tu.Y() ).copyFrom( saveCS.getPredBuf( tu.Y() ) );
csFull->getRecoBuf( tu.Y() ).copyFrom( saveCS.getRecoBuf( tu.Y() ) );
if( KEEP_PRED_AND_RESI_SIGNALS )
{
csFull->getResiBuf ( tu.Y() ).copyFrom( saveCS.getResiBuf ( tu.Y() ) );
csFull->getOrgResiBuf( tu.Y() ).copyFrom( saveCS.getOrgResiBuf( tu.Y() ) );
}
tu.copyComponentFrom( *tmpTU, COMPONENT_Y );
if( !bCheckSplit )
{
m_CABACEstimator->getCtx() = ctxBest;
}
}
else if( bCheckSplit )
{
ctxBest = m_CABACEstimator->getCtx();
}
csFull->cost += dSingleCost;
csFull->dist += uiSingleDistLuma;
csFull->fracBits += singleFracBits;
}
}
bool validReturnSplit = false;
if( bCheckSplit )
{
//----- store full entropy coding status, load original entropy coding status -----
if( bCheckFull )
{
m_CABACEstimator->getCtx() = ctxStart;
}
//----- code splitted block -----
csSplit->cost = 0;
bool uiSplitCbfLuma = false;
bool splitIsSelected = true;
if( partitioner.canSplit( TU_MAX_TR_SPLIT, cs ) )
{
partitioner.splitCurrArea( TU_MAX_TR_SPLIT, cs );
}
if( cu.ispMode )
{
partitioner.splitCurrArea( ispType, *csSplit );
}
do
{
bool tmpValidReturnSplit = xRecurIntraCodingLumaQT( *csSplit, partitioner, bestCostSoFar, subTuCounter, ispType, false, mtsCheckRangeFlag, mtsFirstCheckId, mtsLastCheckId );
subTuCounter += subTuCounter != -1 ? 1 : 0;
if( sps.getUseLFNST() && !tmpValidReturnSplit )
{
splitIsSelected = false;
break;
}
if( !cu.ispMode )
{
csSplit->setDecomp( partitioner.currArea().Y() );
}
else if( CU::isISPFirst( cu, partitioner.currArea().Y(), COMPONENT_Y ) )
{
csSplit->setDecomp( cu.Y() );
}
uiSplitCbfLuma |= TU::getCbfAtDepth( *csSplit->getTU( partitioner.currArea().lumaPos(), partitioner.chType, subTuCounter - 1 ), COMPONENT_Y, partitioner.currTrDepth );
if( cu.ispMode )
{
//exit condition if the accumulated cost is already larger than the best cost so far (no impact in RD performance)
//如果累计成本已经大于目前为止的最佳成本(对研发绩效没有影响),则退出条件
if( csSplit->cost > bestCostSoFar )
{
earlySkipISP = true;
splitIsSelected = false;
break;
}
else
{
//more restrictive exit condition
//更严格的退出条件
bool tuIsDividedInRows = CU::divideTuInRows( cu );
int nSubPartitions = tuIsDividedInRows ? cu.lheight() >> floorLog2(cu.firstTU->lheight()) : cu.lwidth() >> floorLog2(cu.firstTU->lwidth());
double threshold = nSubPartitions == 2 ? 0.95 : subTuCounter == 1 ? 0.83 : 0.91;
if( subTuCounter < nSubPartitions && csSplit->cost > bestCostSoFar*threshold )
{
earlySkipISP = true;
splitIsSelected = false;
break;
}
}
}
} while( partitioner.nextPart( *csSplit ) );
partitioner.exitCurrSplit();
if( splitIsSelected )
{
for( auto &ptu : csSplit->tus )
{
if( currArea.Y().contains( ptu->Y() ) )
{
TU::setCbfAtDepth( *ptu, COMPONENT_Y, currDepth, uiSplitCbfLuma ? 1 : 0 );
}
}
//----- restore context states -----
m_CABACEstimator->getCtx() = ctxStart;
cuCtx.violatesLfnstConstrained[CHANNEL_TYPE_LUMA] = false;
cuCtx.violatesLfnstConstrained[CHANNEL_TYPE_CHROMA] = false;
cuCtx.lfnstLastScanPos = false;
cuCtx.violatesMtsCoeffConstraint = false;
cuCtx.mtsLastScanPos = false;
//----- determine rate and r-d cost -----
//更新比特数
csSplit->fracBits = xGetIntraFracBitsQT( *csSplit, partitioner, true, false, cu.ispMode ? 0 : -1, ispType, &cuCtx );
//--- update cost ---
//更新RD Cost
csSplit->cost = m_pcRdCost->calcRdCost(csSplit->fracBits, csSplit->dist);
validReturnSplit = true;
}
}
bool retVal = false;
if( csFull || csSplit )
{
if( !sps.getUseLFNST() || validReturnFull || validReturnSplit )//如果存在合理的返回值
{
{
// otherwise this would've happened in useSubStructure
cs.picture->getRecoBuf( currArea.Y() ).copyFrom( cs.getRecoBuf( currArea.Y() ) );
cs.picture->getPredBuf( currArea.Y() ).copyFrom( cs.getPredBuf( currArea.Y() ) );
}
if( cu.ispMode && earlySkipISP ) //ISP模式
{
cs.cost = MAX_DOUBLE;
}
else
{
cs.cost = m_pcRdCost->calcRdCost( cs.fracBits, cs.dist );
retVal = true;
}
}
}
return retVal;
}
VTM9.0中的该函数涉及了许多ISP模式的东西,我很怀疑这些代码是否有用???