1 VTM4.0版本
初始化后分为两种情况：1.可以继续划分，2.不可继续划分
1.可以继续划分时，根据cost的大小进行TU的划分（递归调用多次划分），设置cbf，获取上下文模型并保存代价
2.不可继续划分时，设置变换的模式（6种，决定是否开启及开启几个），对可以进行变换的模式进行后续残差计算、变换量化和反变换重建等操作，具体函数为xIntraCodingTUBlock（DCT2与其余5种不同），更新率失真信息。
可点击查看详情：H.266/VVC代码学习9：xIntraCodingTUBlock函数
最终获取缓存，确定代价。
具体代码和部分注释如下：
#if JVET_M0102_INTRA_SUBPARTITIONS
void IntraSearch::xRecurIntraCodingLumaQT( CodingStructure &cs, Partitioner &partitioner, const double bestCostSoFar, const int subTuIdx, const PartSplit ispType )
{
/*************************** 初始化 **********************/
int subTuCounter = subTuIdx;
const UnitArea &currArea = partitioner.currArea();
const CodingUnit &cu = *cs.getCU( currArea.lumaPos(), partitioner.chType );
bool earlySkipISP = false;
#else
void IntraSearch::xRecurIntraCodingLumaQT( CodingStructure &cs, Partitioner &partitioner )
{
const UnitArea &currArea = partitioner.currArea();
#if !JVET_M0464_UNI_MTS
const CodingUnit &cu = *cs.getCU(currArea.lumaPos(), partitioner.chType);
#endif
#endif
uint32_t currDepth = partitioner.currTrDepth;
const PPS &pps = *cs.pps;
const bool keepResi = pps.getPpsRangeExtension().getCrossComponentPredictionEnabledFlag() || KEEP_PRED_AND_RESI_SIGNALS;
bool bCheckFull = true; // 不能继续划分：当前PU可进行变换和量化
bool bCheckSplit = false; // 是否进行分割：当前PU应继续划分
bCheckFull = !partitioner.canSplit( TU_MAX_TR_SPLIT, cs );
bCheckSplit = partitioner.canSplit( TU_MAX_TR_SPLIT, cs );
#if JVET_M0102_INTRA_SUBPARTITIONS
if( cu.ispMode )
{
bCheckSplit = partitioner.canSplit( ispType, cs );
bCheckFull = !bCheckSplit;
}
#endif
uint32_t numSig = 0;
#if JVET_M0464_UNI_MTS
double dSingleCost = MAX_DOUBLE;
Distortion uiSingleDistLuma = 0;
uint64_t singleFracBits = 0;
int bestModeId[MAX_NUM_COMPONENT] = { 0, 0, 0 };
#else
bool checkInitTrDepth = false, checkInitTrDepthTransformSkipWinner = false;
double dSingleCost = MAX_DOUBLE;
Distortion uiSingleDistLuma = 0;
uint64_t singleFracBits = 0;
bool checkTransformSkip = pps.getUseTransformSkip();
int bestModeId[MAX_NUM_COMPONENT] = {0, 0, 0};
uint8_t nNumTransformCands = cu.emtFlag ? 4 : 1; //4 is the number of transforms of emt
bool isAllIntra = m_pcEncCfg->getIntraPeriod() == 1;
uint8_t numTransformIndexCands = nNumTransformCands;
#endif
//处理上下文
const TempCtx ctxStart ( m_CtxCache, m_CABACEstimator->getCtx() );
TempCtx ctxBest ( m_CtxCache );
/************************** 创建进行操作的cs ********************/
CodingStructure *csSplit = nullptr;
CodingStructure *csFull = nullptr;
if( bCheckSplit )
{
csSplit = &cs;
}
else if( bCheckFull )
{
csFull = &cs;
}
/*************************** 如果PU划分完成的操作 ************************/
if( bCheckFull )
{
csFull->cost = 0.0;
TransformUnit &tu = csFull->addTU( CS::getArea( *csFull, currArea, partitioner.chType ), partitioner.chType );
tu.depth = currDepth;
#if JVET_M0464_UNI_MTS
const bool tsAllowed = TU::isTSAllowed ( tu, COMPONENT_Y );//TS：变换跳过的模式，能否进行
const bool mtsAllowed = TU::isMTSAllowed( tu, COMPONENT_Y );//MTS：选变换核的模式，能否进行
uint8_t nNumTransformCands = 1 + ( tsAllowed ? 1 : 0 ) + ( mtsAllowed ? 4 : 0 ); // DCT + TS + 4 MTS = 6 tests
std::vector<TrMode> trModes;
trModes.push_back( TrMode( 0, true ) ); //0：DCT2直接加入
if( tsAllowed )//1：如果可以进行TS，则加入
{
trModes.push_back( TrMode( 1, true ) );
}
if( mtsAllowed )//2~5：如果可以进行MTS，则依次加入
{
for( int i = 2; i < 6; i++ )
{
trModes.push_back( TrMode( i, true) );
}
}
CHECK( !tu.Y().valid(), "Invalid TU" );
#else
checkTransformSkip &= TU::hasTransformSkipFlag( *tu.cs, tu.Y() );
checkTransformSkip &= !cu.transQuantBypass;
checkTransformSkip &= !cu.emtFlag;
#if JVET_M0102_INTRA_SUBPARTITIONS
checkTransformSkip &= !cu.ispMode;
#endif
CHECK( !tu.Y().valid(), "Invalid TU" );
//this prevents transformSkip from being checked because we already know it's not the best mode
checkTransformSkip = ( checkInitTrDepth && !checkInitTrDepthTransformSkipWinner ) ? false : checkTransformSkip;
CHECK( checkInitTrDepthTransformSkipWinner && !checkTransformSkip, "Transform Skip must be enabled if it was the winner in the previous call of xRecurIntraCodingLumaQT!" );
#endif
CodingStructure &saveCS = *m_pSaveCS[0];
TransformUnit *tmpTU = nullptr;
Distortion singleDistTmpLuma = 0;
uint64_t singleTmpFracBits = 0;
double singleCostTmp = 0;
int firstCheckId = 0;//第一个模式的标号
#if JVET_M0464_UNI_MTS
int lastCheckId = trModes[nNumTransformCands-1].first;//最后一个模式的标号
bool isNotOnlyOneMode = nNumTransformCands != 1;//变换模式不止一种
#else
//we add the EMT candidates to the loop. TransformSkip will still be the last one to be checked (when modeId == lastCheckId) as long as checkTransformSkip is true
int lastCheckId = numTransformIndexCands - ( firstCheckId + 1 ) + ( int ) checkTransformSkip;
bool isNotOnlyOneMode = lastCheckId != firstCheckId && !checkInitTrDepthTransformSkipWinner;
#endif
if( isNotOnlyOneMode )//如除了DCT2还有其他模式的变换时，创建临时cs以保留数据
{
saveCS.pcv = cs.pcv;
saveCS.picture = cs.picture;
saveCS.area.repositionTo(cs.area);
saveCS.clearTUs();
tmpTU = &saveCS.addTU(currArea, partitioner.chType);
}
#if JVET_M0464_UNI_MTS
bool cbfDCT2 = true;
#else
bool cbfBestMode = false;
#endif
#if JVET_M0464_UNI_MTS
for( int modeId = firstCheckId; modeId < nNumTransformCands; modeId++ )//遍历变换模式
{
if( !cbfDCT2 || ( m_pcEncCfg->getUseTransformSkipFast() && bestModeId[COMPONENT_Y] == 1 ) )
{
break;
}
if( !trModes[modeId].second )//如果变换模式不开启，则跳过
{
continue;
}
tu.mtsIdx = trModes[modeId].first;//mtsIdx，变换方式的序号
#else
for( int modeId = firstCheckId; modeId <= lastCheckId; modeId++ )
{
if( checkInitTrDepthTransformSkipWinner )
{
//If this is a full RQT call and the winner of the first call (checkFirst=true) was transformSkip, then we skip the first iteration of the loop, since transform skip always comes at the end
if( modeId == firstCheckId )
{
continue;
}
}
uint8_t transformIndex = modeId;
if( ( transformIndex < lastCheckId ) || ( ( transformIndex == lastCheckId ) && !checkTransformSkip ) ) //we avoid this if the mode is transformSkip
{
// Skip checking other transform candidates if zero CBF is encountered and it is the best transform so far
if( m_pcEncCfg->getFastIntraEMT() && isAllIntra && transformIndex && !cbfBestMode )
{
continue;
}
}
#endif
//如果不是第一种模式，并且不只有一种模式，则获取上下文
if ((modeId != firstCheckId) && isNotOnlyOneMode)
{
m_CABACEstimator->getCtx() = ctxStart;
}
int default0Save1Load2 = 0;//模式重复使用标志
singleDistTmpLuma = 0;
//如果是第一种模式，且有多种可用模式
#if JVET_M0464_UNI_MTS
if( modeId == firstCheckId && nNumTransformCands > 1 )
#else
if (modeId == firstCheckId && modeId != lastCheckId && !checkInitTrDepthTransformSkipWinner )
#endif
{
default0Save1Load2 = 1;
}
//如果不是第一种模式（这时当然有多种可用模式）
else if (modeId != firstCheckId)
{
default0Save1Load2 = 2;
}
#if JVET_M0102_INTRA_SUBPARTITIONS
if( cu.ispMode )
{
default0Save1Load2 = 0;
}
#endif
/******** 进行残差变换重建等操作 ********/
#if JVET_M0464_UNI_MTS
if( nNumTransformCands > 1 )//如果不止一种模式
{
xIntraCodingTUBlock( tu, COMPONENT_Y, false, singleDistTmpLuma, default0Save1Load2, &numSig, modeId == 0 ? &trModes : nullptr, true );
if( modeId == 0 )//把可用模式的标号记下，并用lastCheckId存可用模式的数量
{
for( int i = 0; i < nNumTransformCands; i++ )
{
if( trModes[i].second )
{
lastCheckId = trModes[i].first;
}
}
}
}
else//如果只有一种模式，即如果只做DCT变换，直接简单的变换即可
{
xIntraCodingTUBlock( tu, COMPONENT_Y, false, singleDistTmpLuma, default0Save1Load2, &numSig );
}
#else
if (cu.emtFlag)
{
tu.emtIdx = transformIndex;
}
if( !checkTransformSkip )
{
tu.transformSkip[COMPONENT_Y] = false;
}
else
{
tu.transformSkip[COMPONENT_Y] = modeId == lastCheckId;
}
xIntraCodingTUBlock( tu, COMPONENT_Y, false, singleDistTmpLuma, default0Save1Load2, &numSig );
#endif
//----- determine rate and r-d cost 存储率失真信息 -----
#if JVET_M0464_UNI_MTS
if( ( trModes[modeId].first != 0 && !TU::getCbfAtDepth( tu, COMPONENT_Y, currDepth ) ) )
#else
//the condition (transformIndex != DCT2_EMT) seems to be irrelevant, since DCT2_EMT=7 and the highest value of transformIndex is 4
if( ( modeId == lastCheckId && checkTransformSkip && !TU::getCbfAtDepth( tu, COMPONENT_Y, currDepth ) ) )
#endif
{//为了在cbf为零时不对TS标志进行编码，禁止cbf为零的TS的情况。
//In order not to code TS flag when cbf is zero, the case for TS with cbf being zero is forbidden.
singleCostTmp = MAX_DOUBLE;
}
else//获取亮度比特值和代价信息
{
#if JVET_M0102_INTRA_SUBPARTITIONS
if( cu.ispMode && m_pcRdCost->calcRdCost( csFull->fracBits, csFull->dist + singleDistTmpLuma ) > bestCostSoFar )
{
earlySkipISP = true;
}
else
{
singleTmpFracBits = xGetIntraFracBitsQT( *csFull, partitioner, true, false, subTuCounter, ispType );
}
#else
singleTmpFracBits = xGetIntraFracBitsQT( *csFull, partitioner, true, false );
#endif
singleCostTmp = m_pcRdCost->calcRdCost( singleTmpFracBits, singleDistTmpLuma );
}
/********** 更新最优率失真信息 ***********/
if (singleCostTmp < dSingleCost)
{
dSingleCost = singleCostTmp;
uiSingleDistLuma = singleDistTmpLuma;
singleFracBits = singleTmpFracBits;
#if JVET_M0464_UNI_MTS
bestModeId[COMPONENT_Y] = trModes[modeId].first;//最佳变换模式
if( trModes[modeId].first == 0 )//如果是DCT2最优，则获取cbf
{
cbfDCT2 = TU::getCbfAtDepth( tu, COMPONENT_Y, currDepth );
}
#else
bestModeId[COMPONENT_Y] = modeId;
cbfBestMode = TU::getCbfAtDepth( tu, COMPONENT_Y, currDepth );
#endif
if( bestModeId[COMPONENT_Y] != lastCheckId )//如果还没循环结束，临时保存一下各种信息
{
#if KEEP_PRED_AND_RESI_SIGNALS || JVET_M0427_INLOOP_RESHAPER
saveCS.getPredBuf( tu.Y() ).copyFrom( csFull->getPredBuf( tu.Y() ) );
#endif
saveCS.getRecoBuf( tu.Y() ).copyFrom( csFull->getRecoBuf( tu.Y() ) );
if( keepResi )
{
saveCS.getResiBuf ( tu.Y() ).copyFrom( csFull->getResiBuf ( tu.Y() ) );
saveCS.getOrgResiBuf( tu.Y() ).copyFrom( csFull->getOrgResiBuf( tu.Y() ) );
}
tmpTU->copyComponentFrom( tu, COMPONENT_Y );
ctxBest = m_CABACEstimator->getCtx();
}
}
}
if( bestModeId[COMPONENT_Y] != lastCheckId )//如果还没循环结束，最终保存一下各种信息
{
#if KEEP_PRED_AND_RESI_SIGNALS || JVET_M0427_INLOOP_RESHAPER
csFull->getPredBuf( tu.Y() ).copyFrom( saveCS.getPredBuf( tu.Y() ) );
#endif
csFull->getRecoBuf( tu.Y() ).copyFrom( saveCS.getRecoBuf( tu.Y() ) );
if( keepResi )
{
csFull->getResiBuf ( tu.Y() ).copyFrom( saveCS.getResiBuf ( tu.Y() ) );
csFull->getOrgResiBuf( tu.Y() ).copyFrom( saveCS.getOrgResiBuf( tu.Y() ) );
}
tu.copyComponentFrom( *tmpTU, COMPONENT_Y );
if( !bCheckSplit )
{
m_CABACEstimator->getCtx() = ctxBest;
}
}
else if( bCheckSplit )
{
ctxBest = m_CABACEstimator->getCtx();
}
/******** 更新率失真信息 *****/
csFull->cost += dSingleCost;
csFull->dist += uiSingleDistLuma;
csFull->fracBits += singleFracBits;
#if LUMA_AND_CHROMA_BITS
if (csFull->chType == 0) csFull->lumaBits += singleFracBits;
if (csFull->chType == 1) csFull->chromaBits += singleFracBits;
#endif
}
/*************************** 如果PU划分未完的操作 ************************/
if( bCheckSplit )
{
//----- store full entropy coding status, load original entropy coding status 存储全熵编码状态，加载原始熵编码状态 -----
if( bCheckFull )
{
m_CABACEstimator->getCtx() = ctxStart;
}
//----- code splitted block 分割编码 -----
csSplit->cost = 0;
bool uiSplitCbfLuma = false;
bool splitIsSelected = true;
if( partitioner.canSplit( TU_MAX_TR_SPLIT, cs ) )
{
partitioner.splitCurrArea( TU_MAX_TR_SPLIT, cs );
}
#if JVET_M0102_INTRA_SUBPARTITIONS
if( cu.ispMode )
{
partitioner.splitCurrArea( ispType, *csSplit );
}
#endif
do
{
#if JVET_M0102_INTRA_SUBPARTITIONS
xRecurIntraCodingLumaQT( *csSplit, partitioner, bestCostSoFar, subTuCounter, ispType );//递归调用，继续划分
subTuCounter += subTuCounter != -1 ? 1 : 0;//划分了几次
#else
xRecurIntraCodingLumaQT( *csSplit, partitioner );
#endif
#if JVET_M0102_INTRA_SUBPARTITIONS
if( !cu.ispMode )
{
csSplit->setDecomp( partitioner.currArea().Y() );
}
else if( CU::isISPFirst( cu, partitioner.currArea().Y(), COMPONENT_Y ) )
{
csSplit->setDecomp( cu.Y() );
}
#else
csSplit->setDecomp( partitioner.currArea().Y() );
#endif
#if JVET_M0102_INTRA_SUBPARTITIONS
uiSplitCbfLuma |= TU::getCbfAtDepth( *csSplit->getTU( partitioner.currArea().lumaPos(), partitioner.chType, subTuCounter - 1 ), COMPONENT_Y, partitioner.currTrDepth );
if( cu.ispMode )//splitIsSelected是默认为true的，表示要划分到什么成都。以下是令splitIsSelected = false的原因
{ //如果累计成本已经大于目前的最佳成本，则跳出循环条件（对RD性能没有影响）
//exit condition if the accumulated cost is already larger than the best cost so far (no impact in RD performance)
if( csSplit->cost > bestCostSoFar )//如果大于ISP临时的代价
{
earlySkipISP = true;
splitIsSelected = false;
break;
}
else
{
//more restrictive exit condition 更严格的跳出循环条件
bool tuIsDividedInRows = CU::divideTuInRows( cu );
int nSubPartitions = tuIsDividedInRows ? cu.lheight() >> g_aucLog2[cu.firstTU->lheight()] : cu.lwidth() >> g_aucLog2[cu.firstTU->lwidth()];//短边
double threshold = nSubPartitions == 2 ? 0.95 : subTuCounter == 1 ? 0.83 : 0.91;//阈值
if( subTuCounter < nSubPartitions && csSplit->cost > bestCostSoFar*threshold )//如果大于ISP临时的代价*阈值
{
earlySkipISP = true;
splitIsSelected = false;
break;
}
}
}
#else
uiSplitCbfLuma |= TU::getCbfAtDepth( *csSplit->getTU( partitioner.currArea().lumaPos(), partitioner.chType ), COMPONENT_Y, partitioner.currTrDepth );
#endif
} while( partitioner.nextPart( *csSplit ) );//划分完成
partitioner.exitCurrSplit();
if( splitIsSelected )//splitIsSelected = true时（划分到底了）
{
for( auto &ptu : csSplit->tus )//将每个tu设置cbf
{
if( currArea.Y().contains( ptu->Y() ) )
{
TU::setCbfAtDepth( *ptu, COMPONENT_Y, currDepth, uiSplitCbfLuma ? 1 : 0 );
}
}
//----- restore context states 恢复上下文 -----
m_CABACEstimator->getCtx() = ctxStart;
//----- determine rate and r-d cost 确定比特率和RDcost -----
#if JVET_M0102_INTRA_SUBPARTITIONS
csSplit->fracBits = xGetIntraFracBitsQT( *csSplit, partitioner, true, false, cu.ispMode ? 0 : -1, ispType );
#if LUMA_AND_CHROMA_BITS
if (csSplit->chType == 0) csSplit->fracBits = xGetIntraFracBitsQT(*csSplit, partitioner, true, false, cu.ispMode ? 0 : -1, ispType);
if (csSplit->chType == 1) csSplit->fracBits = xGetIntraFracBitsQT(*csSplit, partitioner, true, false, cu.ispMode ? 0 : -1, ispType);
#endif
#else
csSplit->fracBits = xGetIntraFracBitsQT(*csSplit, partitioner, true, false);
#endif
//--- update cost 更新代价 ---
csSplit->cost = m_pcRdCost->calcRdCost(csSplit->fracBits, csSplit->dist);
}
}
if( csFull || csSplit )//获取缓存，确定代价
{
{
// otherwise this would've happened in useSubStructure 否则这将发生在useSubStructure中
cs.picture->getRecoBuf( currArea.Y() ).copyFrom( cs.getRecoBuf( currArea.Y() ) );
#if JVET_M0427_INLOOP_RESHAPER
cs.picture->getPredBuf( currArea.Y() ).copyFrom( cs.getPredBuf( currArea.Y() ) );
#endif
}
#if JVET_M0102_INTRA_SUBPARTITIONS
if( cu.ispMode && earlySkipISP )
{
cs.cost = MAX_DOUBLE;
}
else
{
cs.cost = m_pcRdCost->calcRdCost( cs.fracBits, cs.dist );
}
#else
cs.cost = m_pcRdCost->calcRdCost( cs.fracBits, cs.dist );
#endif
}
}
2 VTM7.0版本(20191128更新)
和4.0版本基本相同，具体步骤在代码中。
bool IntraSearch::xRecurIntraCodingLumaQT( CodingStructure &cs, Partitioner &partitioner, const double bestCostSoFar, const int subTuIdx, const PartSplit ispType, const bool ispIsCurrentWinner, bool mtsCheckRangeFlag, int mtsFirstCheckId, int mtsLastCheckId, bool moreProbMTSIdxFirst )
{
/*********************************************************** 1.初始化 ********************************************************/
int subTuCounter = subTuIdx;
const UnitArea &currArea = partitioner.currArea();
const CodingUnit &cu = *cs.getCU( currArea.lumaPos(), partitioner.chType );
bool earlySkipISP = false;//属于ISP模式时提前跳出
uint32_t currDepth = partitioner.currTrDepth;
const SPS &sps = *cs.sps;
const PPS &pps = *cs.pps;
const bool keepResi = pps.getPpsRangeExtension().getCrossComponentPredictionEnabledFlag() || KEEP_PRED_AND_RESI_SIGNALS;
bool bCheckFull = true;
bool bCheckSplit = false;
bCheckFull = !partitioner.canSplit( TU_MAX_TR_SPLIT, cs );//如果不可以继续划分为TU
bCheckSplit = partitioner.canSplit( TU_MAX_TR_SPLIT, cs );//如果可继续划分为TU
if( cu.ispMode )
{
bCheckSplit = partitioner.canSplit( ispType, cs );//如果不可以继续划分为TU
bCheckFull = !bCheckSplit;//如果可继续划分为TU
}
uint32_t numSig = 0;
double dSingleCost = MAX_DOUBLE;
Distortion uiSingleDistLuma = 0;
uint64_t singleFracBits = 0;
bool checkTransformSkip = sps.getTransformSkipEnabledFlag();
int bestModeId[ MAX_NUM_COMPONENT ] = { 0, 0, 0 };
uint8_t nNumTransformCands = cu.mtsFlag ? 4 : 1;
uint8_t numTransformIndexCands = nNumTransformCands;
const TempCtx ctxStart ( m_CtxCache, m_CABACEstimator->getCtx() );
TempCtx ctxBest ( m_CtxCache );
CodingStructure *csSplit = nullptr;
CodingStructure *csFull = nullptr;
#if JVET_P1026_ISP_LFNST_COMBINATION
CUCtx cuCtx;
cuCtx.isDQPCoded = true;
cuCtx.isChromaQpAdjCoded = true;
#endif
if( bCheckSplit )
{
csSplit = &cs;
}
else if( bCheckFull )
{
csFull = &cs;
}
/*********************************************************** 2.不继续划分为TU的变换 ******************************************************/
bool validReturnFull = false;
if( bCheckFull )
{
/***************** 2.1 初始化要遍历的变换模式列表 ******************/
csFull->cost = 0.0;//初始化代价
TransformUnit &tu = csFull->addTU( CS::getArea( *csFull, currArea, partitioner.chType ), partitioner.chType );
tu.depth = currDepth;
const bool tsAllowed = TU::isTSAllowed( tu, COMPONENT_Y );//是否允许TS
#if JVET_P1026_MTS_SIGNALLING
const bool mtsAllowed = CU::isMTSAllowed( cu, COMPONENT_Y );//是否允许MTS
#else
const bool mtsAllowed = TU::isMTSAllowed( tu, COMPONENT_Y );
#endif
std::vector<TrMode> trModes;//创建要经过的变换模式
/* 如果是二次变换，只测试DCT2和TS */
if( sps.getUseLFNST() )
{
checkTransformSkip &= tsAllowed;
checkTransformSkip &= !cu.mtsFlag;
checkTransformSkip &= !cu.lfnstIdx;
if( !cu.mtsFlag && checkTransformSkip )
{
trModes.push_back( TrMode( 0, true ) ); //DCT2
trModes.push_back( TrMode( 1, true ) ); //TS
}
}
/* 如果不是二次变换，测试DCT2和TS和4种MTS */
else
{
nNumTransformCands = 1 + ( tsAllowed ? 1 : 0 ) + ( mtsAllowed ? 4 : 0 ); // DCT + TS + 4 MTS = 6 tests
trModes.push_back( TrMode( 0, true ) ); //DCT2
if( tsAllowed )
{
trModes.push_back( TrMode( 1, true ) );//TS
}
if( mtsAllowed )
{
for( int i = 2; i < 6; i++ )
{
trModes.push_back( TrMode( i, true ) );//MTS
}
}
}
CHECK( !tu.Y().valid(), "Invalid TU" );
/************************ 2.2 进行各种模式的变换并比较 ************************/
/******************* 2.2.1 初始化 ********************/
CodingStructure &saveCS = *m_pSaveCS[0];
TransformUnit *tmpTU = nullptr;
Distortion singleDistTmpLuma = 0;//初始化当前部分的失真D
uint64_t singleTmpFracBits = 0;//初始化当前部分的比特数R
double singleCostTmp = 0;//初始化临时代价
int firstCheckId = ( sps.getUseLFNST() && mtsCheckRangeFlag && cu.mtsFlag ) ? mtsFirstCheckId : 0;//第一个变换模式序号
//we add the MTS candidates to the loop. TransformSkip will still be the last one to be checked (when modeId == lastCheckId) as long as checkTransformSkip is true
int lastCheckId = sps.getUseLFNST() ? ( ( mtsCheckRangeFlag && cu.mtsFlag ) ? ( mtsLastCheckId + ( int ) checkTransformSkip ) :
( numTransformIndexCands - ( firstCheckId + 1 ) + ( int ) checkTransformSkip ) ) :
trModes[ nNumTransformCands - 1 ].first; //最后一个变换模式序号
bool isNotOnlyOneMode = sps.getUseLFNST() ? lastCheckId != firstCheckId : nNumTransformCands != 1;
if( isNotOnlyOneMode )//如果只有一种模式，直接存入这一模式
{
saveCS.pcv = cs.pcv;
saveCS.picture = cs.picture;
saveCS.area.repositionTo(cs.area);
saveCS.clearTUs();
tmpTU = &saveCS.addTU(currArea, partitioner.chType);
}
bool cbfBestMode = false;
bool cbfBestModeValid = false;
bool cbfDCT2 = true;
double bestDCT2cost = MAX_DOUBLE;
double threshold = m_pcEncCfg->getUseFastISP() && !cu.ispMode && ispIsCurrentWinner && nNumTransformCands > 1 ? 1 + 1.4 / sqrt( cu.lwidth() * cu.lheight() ) : 1;//设一个阈值，可能是快速算法？
/************ 2.2.2 遍历变换模式进行变换操作 ***********/
for( int modeId = firstCheckId; modeId <= ( sps.getUseLFNST() ? lastCheckId : ( nNumTransformCands - 1 ) ); modeId++ )
{
uint8_t transformIndex = modeId;
/******** 2.2.2.1 提前跳出本次循环条件 ********/
/** 2.2.2.1.1 如果可以进行二次变换 **/
if( sps.getUseLFNST() )//如果可以使用二次变换但不是DCT2，需要跳过
{
if( ( transformIndex < lastCheckId ) || ( ( transformIndex == lastCheckId ) && !checkTransformSkip ) ) //we avoid this if the mode is transformSkip
{//如果遇到零CBF，则跳过检查其他变换候选对象，这是迄今为止最好的变换
// Skip checking other transform candidates if zero CBF is encountered and it is the best transform so far
if( m_pcEncCfg->getUseFastLFNST() && transformIndex && !cbfBestMode && cbfBestModeValid )
{
continue;//如果进行二次变换但不是DCT2-DCT2，并且不是最佳模式
}
}
}
/* 2.2.2.1.2 如果不进行二次变换 */
else//TS跳过，当前模式不可用则跳过等
{
#if JVET_AHG14_LOSSLESS
if( !( m_pcEncCfg->getCostMode() == COST_LOSSLESS_CODING ) )//如果有残差才看
{
#endif
#if JVET_P0058_CHROMA_TS
if( !cbfDCT2 || ( m_pcEncCfg->getUseTransformSkipFast() && bestModeId[ COMPONENT_Y ] == MTS_SKIP))
#else
if( !cbfDCT2 || ( m_pcEncCfg->getUseTransformSkipFast() && bestModeId[ COMPONENT_Y ] == 1 ) )
#endif
{
break;//如果最佳模式是skip，跳出循环
}
if( !trModes[ modeId ].second )
{
continue;//如果当前模式不进行，跳出本次循环
}
//we compare the DCT-II cost against the best ISP cost so far (except for TS)
#if JVET_P0058_CHROMA_TS
if (m_pcEncCfg->getUseFastISP() && !cu.ispMode && ispIsCurrentWinner && trModes[modeId].first != MTS_DCT2_DCT2 && (trModes[modeId].first != MTS_SKIP || !tsAllowed) && bestDCT2cost > bestCostSoFar * threshold)
#else
if( m_pcEncCfg->getUseFastISP() && !cu.ispMode && ispIsCurrentWinner && trModes[ modeId ].first != 0 && ( trModes[ modeId ].first != 1 || !tsAllowed ) && bestDCT2cost > bestCostSoFar * threshold )
#endif
{
continue;//如果满足一定条件，跳出本次循环
}
#if JVET_AHG14_LOSSLESS
}
#endif
#if JVET_P0058_CHROMA_TS
tu.mtsIdx[COMPONENT_Y] = trModes[modeId].first;
#else
tu.mtsIdx = trModes[ modeId ].first;
#endif
}
/***** 2.2.2.2 确定变换方向，进行变换过程 *****/
if ((modeId != firstCheckId) && isNotOnlyOneMode)
{
m_CABACEstimator->getCtx() = ctxStart;
}
int default0Save1Load2 = 0;
singleDistTmpLuma = 0;
if( modeId == firstCheckId && ( sps.getUseLFNST() ? ( modeId != lastCheckId ) : ( nNumTransformCands > 1 ) ) )
{
default0Save1Load2 = 1;//只有一个模式，则为1
}
else if (modeId != firstCheckId)
{
if( sps.getUseLFNST() && !cbfBestModeValid )
{
default0Save1Load2 = 1;//如果是二次变换，也相当于只有一个模式
}
else
{
default0Save1Load2 = 2;//如果有多个模式，则为2
}
}
if( cu.ispMode )//如果是ISP模式，default0Save1Load2为0
{
default0Save1Load2 = 0; //如果继续划分，则为0
}
/** 2.2.2.2.1 如果可以进行二次变换 **/
if( sps.getUseLFNST() )
{
/** 如果是多核变换，进行多核变换与帧内模式对应 **/
if( cu.mtsFlag )//多核变换的两个变换方向对应起来
{
if( moreProbMTSIdxFirst )
{
const ChannelType chType = toChannelType( COMPONENT_Y );
const CompArea& area = tu.blocks[ COMPONENT_Y ];
const PredictionUnit& pu = *cs.getPU( area.pos(), chType );
uint32_t uiIntraMode = pu.intraDir[ chType ];
if( transformIndex == 1 )//多核变换的两个方向
{
#if JVET_P0058_CHROMA_TS
tu.mtsIdx[COMPONENT_Y] = (uiIntraMode < 34) ? MTS_DST7_DCT8 : MTS_DCT8_DST7;
#else
tu.mtsIdx = ( uiIntraMode < 34 ) ? MTS_DST7_DCT8 : MTS_DCT8_DST7;
#endif
}
else if( transformIndex == 2 )//多核变换的两个方向
{
#if JVET_P0058_CHROMA_TS
tu.mtsIdx[COMPONENT_Y] = (uiIntraMode < 34) ? MTS_DCT8_DST7 : MTS_DST7_DCT8;
#else
tu.mtsIdx = ( uiIntraMode < 34 ) ? MTS_DCT8_DST7 : MTS_DST7_DCT8;
#endif
}
else//多核变换的两个方向
{
#if JVET_P0058_CHROMA_TS
tu.mtsIdx[COMPONENT_Y] = MTS_DST7_DST7 + transformIndex;
#else
tu.mtsIdx = MTS_DST7_DST7 + transformIndex;
#endif
}
}
else//多核变换的两个方向
{
#if JVET_P0058_CHROMA_TS
tu.mtsIdx[COMPONENT_Y] = MTS_DST7_DST7 + transformIndex;
#else
tu.mtsIdx = MTS_DST7_DST7 + transformIndex;
#endif
}
}
/** 如果不是多核变换 **/
else
{
#if JVET_P0058_CHROMA_TS
tu.mtsIdx[COMPONENT_Y] = transformIndex;
#else
tu.mtsIdx = transformIndex;
#endif
}
/** 如果是TS变换模式 **/
if( !cu.mtsFlag && checkTransformSkip )
{
xIntraCodingTUBlock( tu, COMPONENT_Y, false, singleDistTmpLuma, default0Save1Load2, &numSig, modeId == 0 ? &trModes : nullptr, true );//进行多核变换
if( modeId == 0 )
{
for( int i = 0; i < 2; i++ )//遍历DCT2和TS
{
if( trModes[ i ].second )//如果当前模式可用
{
lastCheckId = trModes[ i ].first;//最后一个检查的模式设置为当前模式
}
}
}
}
/** 如果不是TS变换模式 **/
else
{
xIntraCodingTUBlock( tu, COMPONENT_Y, false, singleDistTmpLuma, default0Save1Load2, &numSig );//进行普通变换
}
}
/* 2.2.2.2.2 如果不可以进行二次变换 */
else
{
/******************* 如果是TS或多核变换 ***************/
if( nNumTransformCands > 1 )
{
xIntraCodingTUBlock( tu, COMPONENT_Y, false, singleDistTmpLuma, default0Save1Load2, &numSig, modeId == 0 ? &trModes : nullptr, true );//进行多核变换
if( modeId == 0 )
{
for( int i = 0; i < nNumTransformCands; i++ )
{
if( trModes[ i ].second )
{
lastCheckId = trModes[ i ].first;
}
}
}
}
/******************* 如果是正常变换 ***************/
else
{
xIntraCodingTUBlock( tu, COMPONENT_Y, false, singleDistTmpLuma, default0Save1Load2, &numSig );//进行普通变换
}
}
/******* 2.2.2.3 通过RD决定最佳模式 *********/
//----- determine rate and r-d cost -----
/** 2.2.2.3.1 如果进行二次变换 **/
if( ( sps.getUseLFNST() ? ( modeId == lastCheckId && modeId != 0 && checkTransformSkip ) : ( trModes[ modeId ].first != 0 ) ) && !TU::getCbfAtDepth( tu, COMPONENT_Y, currDepth ) )
{
//In order not to code TS flag when cbf is zero, the case for TS with cbf being zero is forbidden.
singleCostTmp = MAX_DOUBLE;
}
/* 2.2.2.2.1 如果不进行二次变换 */
else
{
if( cu.ispMode && m_pcRdCost->calcRdCost( csFull->fracBits, csFull->dist + singleDistTmpLuma ) > bestCostSoFar )
{
earlySkipISP = true;
}
else
{
#if JVET_P1026_ISP_LFNST_COMBINATION
singleTmpFracBits = xGetIntraFracBitsQT( *csFull, partitioner, true, false, subTuCounter, ispType, &cuCtx );
#else
singleTmpFracBits = xGetIntraFracBitsQT( *csFull, partitioner, true, false, subTuCounter, ispType );
#endif
}
singleCostTmp = m_pcRdCost->calcRdCost( singleTmpFracBits, singleDistTmpLuma );
}
if ( !cu.ispMode && nNumTransformCands > 1 && modeId == firstCheckId )
{
bestDCT2cost = singleCostTmp;
}
/********* 2.2.2.4 比较最终决定代价 **********/
if (singleCostTmp < dSingleCost)
{
dSingleCost = singleCostTmp;
uiSingleDistLuma = singleDistTmpLuma;
singleFracBits = singleTmpFracBits;
if( sps.getUseLFNST() )
{
bestModeId[ COMPONENT_Y ] = modeId;
cbfBestMode = TU::getCbfAtDepth( tu, COMPONENT_Y, currDepth );
cbfBestModeValid = true;
validReturnFull = true;
}
else
{
bestModeId[ COMPONENT_Y ] = trModes[ modeId ].first;
if( trModes[ modeId ].first == 0 )
{
cbfDCT2 = TU::getCbfAtDepth( tu, COMPONENT_Y, currDepth );
}
}
if( bestModeId[COMPONENT_Y] != lastCheckId )
{
saveCS.getPredBuf( tu.Y() ).copyFrom( csFull->getPredBuf( tu.Y() ) );
saveCS.getRecoBuf( tu.Y() ).copyFrom( csFull->getRecoBuf( tu.Y() ) );
if( keepResi )
{
saveCS.getResiBuf ( tu.Y() ).copyFrom( csFull->getResiBuf ( tu.Y() ) );
saveCS.getOrgResiBuf( tu.Y() ).copyFrom( csFull->getOrgResiBuf( tu.Y() ) );
}
tmpTU->copyComponentFrom( tu, COMPONENT_Y );
ctxBest = m_CABACEstimator->getCtx();
}
}
}
/*********************** 2.3 返回代价 ************************/
if( sps.getUseLFNST() && !validReturnFull )//如果是二次变换，但没有正确的返回，则不这样进行
{
csFull->cost = MAX_DOUBLE;
if( bCheckSplit )
{
ctxBest = m_CABACEstimator->getCtx();
}
}
else//其他情况是正常情况，获取代价
{
if( bestModeId[COMPONENT_Y] != lastCheckId )
{
csFull->getPredBuf( tu.Y() ).copyFrom( saveCS.getPredBuf( tu.Y() ) );
csFull->getRecoBuf( tu.Y() ).copyFrom( saveCS.getRecoBuf( tu.Y() ) );
if( keepResi )
{
csFull->getResiBuf ( tu.Y() ).copyFrom( saveCS.getResiBuf ( tu.Y() ) );
csFull->getOrgResiBuf( tu.Y() ).copyFrom( saveCS.getOrgResiBuf( tu.Y() ) );
}
tu.copyComponentFrom( *tmpTU, COMPONENT_Y );
if( !bCheckSplit )
{
m_CABACEstimator->getCtx() = ctxBest;
}
}
else if( bCheckSplit )
{
ctxBest = m_CABACEstimator->getCtx();
}
csFull->cost += dSingleCost;//代价RD
csFull->dist += uiSingleDistLuma;//失真D
csFull->fracBits += singleFracBits;//比特数R
}
}
/************************************************************ 3.继续划分为TU的变换 *******************************************************/
bool validReturnSplit = false;
if( bCheckSplit )//如果继续划分为TU，则递归调用xRecurIntraCodingLumaQT，对每个子块继续变换
{
/*************** 3.1 初始化上下文和划分 ***************/
//----- store full entropy coding status, load original entropy coding status -----
if( bCheckFull )
{
m_CABACEstimator->getCtx() = ctxStart;//存储完整的熵编码状态，加载原始熵编码状态（上下文信息）
}
//----- code splitted block -----
csSplit->cost = 0;
bool uiSplitCbfLuma = false;
bool splitIsSelected = true;
if( partitioner.canSplit( TU_MAX_TR_SPLIT, cs ) )//如果TU可以划分，则继续划分
{
partitioner.splitCurrArea( TU_MAX_TR_SPLIT, cs );
}
if( cu.ispMode )//如果是ISP，则继续划分
{
partitioner.splitCurrArea( ispType, *csSplit );
}
/*************** 3.2 递归进行TU级的运算 ***************/
do//递归调用xRecurIntraCodingLumaQT
{
bool tmpValidReturnSplit = xRecurIntraCodingLumaQT( *csSplit, partitioner, bestCostSoFar, subTuCounter, ispType, false, mtsCheckRangeFlag, mtsFirstCheckId, mtsLastCheckId );
subTuCounter += subTuCounter != -1 ? 1 : 0;
if( sps.getUseLFNST() && !tmpValidReturnSplit )
{
splitIsSelected = false;
break;
}
if( !cu.ispMode )
{
csSplit->setDecomp( partitioner.currArea().Y() );
}
else if( CU::isISPFirst( cu, partitioner.currArea().Y(), COMPONENT_Y ) )
{
csSplit->setDecomp( cu.Y() );
}
uiSplitCbfLuma |= TU::getCbfAtDepth( *csSplit->getTU( partitioner.currArea().lumaPos(), partitioner.chType, subTuCounter - 1 ), COMPONENT_Y, partitioner.currTrDepth );
if( cu.ispMode )//如果是ISP
{
//exit condition if the accumulated cost is already larger than the best cost so far (no impact in RD performance)
if( csSplit->cost > bestCostSoFar )//并不十分严格的跳出条件：如果ISP到了当前块，代价已经超出了之前的最佳
{
earlySkipISP = true;//就提前跳出MIP
splitIsSelected = false;
break;
}
else//快速算法，比较严格的跳出条件：如果ISP到了当前块，代价达到了一定阈值
{
//more restrictive exit condition
bool tuIsDividedInRows = CU::divideTuInRows( cu );
int nSubPartitions = tuIsDividedInRows ? cu.lheight() >> floorLog2(cu.firstTU->lheight()) : cu.lwidth() >> floorLog2(cu.firstTU->lwidth());
double threshold = nSubPartitions == 2 ? 0.95 : subTuCounter == 1 ? 0.83 : 0.91;
if( subTuCounter < nSubPartitions && csSplit->cost > bestCostSoFar*threshold )
{
earlySkipISP = true;
splitIsSelected = false;
break;
}
}
}
} while( partitioner.nextPart( *csSplit ) );
partitioner.exitCurrSplit();
/******** 3.3 若ISP没有提前跳出，算代价设标志 *******/
if( splitIsSelected )//如果一直没有提前跳出
{
for( auto &ptu : csSplit->tus )//遍历每一个TU
{
if( currArea.Y().contains( ptu->Y() ) )
{
TU::setCbfAtDepth( *ptu, COMPONENT_Y, currDepth, uiSplitCbfLuma ? 1 : 0 );//设置CBF
}
}
//----- restore context states -----
m_CABACEstimator->getCtx() = ctxStart;
#if JVET_P1026_ISP_LFNST_COMBINATION
cuCtx.violatesLfnstConstrained[CHANNEL_TYPE_LUMA] = false;
cuCtx.violatesLfnstConstrained[CHANNEL_TYPE_CHROMA] = false;
cuCtx.lfnstLastScanPos = false;
#if JVET_P1026_MTS_SIGNALLING
cuCtx.violatesMtsCoeffConstraint = false;
#endif
#endif
//----- determine rate and r-d cost -----
#if JVET_P1026_ISP_LFNST_COMBINATION
csSplit->fracBits = xGetIntraFracBitsQT( *csSplit, partitioner, true, false, cu.ispMode ? 0 : -1, ispType, &cuCtx );//计算R
#else
csSplit->fracBits = xGetIntraFracBitsQT( *csSplit, partitioner, true, false, cu.ispMode ? 0 : -1, ispType );
#endif
//--- update cost ---
csSplit->cost = m_pcRdCost->calcRdCost(csSplit->fracBits, csSplit->dist);//更新RD
validReturnSplit = true;//TU划分成功
}
}
/******************************************************** 4.处理返回值是true还是false *****************************************************/
bool retVal = false;
if( csFull || csSplit )//本部分总结：返回true的条件：1.不使用二次变换或使用二次变换了但有合理返回，2.没有提前跳出isp
{
if( !sps.getUseLFNST() || validReturnFull || validReturnSplit )
{
{
// otherwise this would've happened in useSubStructure
cs.picture->getRecoBuf( currArea.Y() ).copyFrom( cs.getRecoBuf( currArea.Y() ) );
cs.picture->getPredBuf( currArea.Y() ).copyFrom( cs.getPredBuf( currArea.Y() ) );
}
if( cu.ispMode && earlySkipISP )//这种情况依然返回false：isp模式提前跳出（用了ISP发现不好）
{
cs.cost = MAX_DOUBLE;
}
else
{
cs.cost = m_pcRdCost->calcRdCost( cs.fracBits, cs.dist );
retVal = true;
}
}
}
return retVal;
}
304
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
