对于帧内预测来说，主要包括角度预测和划分。
在xCompressCU函数中体现为
xCheckRDCostIntra( tempCS, bestCS, partitioner, currTestMode );以及
xCheckModeSplit( tempCS, bestCS, partitioner, currTestMode
, tempMotCandLUTs
, bestMotCandLUTs
, partitioner.currArea()
);。
xCheckModeSplit函数中主要是根据划分模式进行划分，再递归回xCompressCU，与HEVC中递归差不多。
这里主要讲帧内亮度角度预测过程，VTM版本为4.0.1。
xCheckRDCostIntra函数进来之后，会进行相应参数的初始化，之后调用亮度角度预测
m_pcIntraSearch->estIntraPredLumaQT( cu, partitioner, bestCostSoFar );和色度角度预测
m_pcIntraSearch->estIntraPredChromaQT( cu, ( !useIntraSubPartitions || ( CS::isDualITree( *cu.cs ) && !isLuma( CHANNEL_TYPE_CHROMA ) ) ) ? partitioner : subTuPartitioner, maxCostAllowedForChroma );。
帧内亮度预测主要过程为：
1、参考像素的获取与滤波；具体可见---https://blog.csdn.net/pengyouyou/article/details/89027301
2、在原HEVC的35种模式中，挑选出numModesForFullRD种哈达玛代价最小的模式-------RMD第一步；
其中planner模式讲解见---https://blog.csdn.net/pengyouyou/article/details/89041932
3、在RMD第一步得到的角度模式中，对其相邻的新增角度模式计算哈达玛代价值，同样保留numModesForFullRD种代价最小的模式-------RMD第二步；
4、添加MPM模式；
5、对候选模式进行RDO过程，选出代价最小的模式。
虽然大致过程为这5个步骤，但是在MRL和ISP等技术的引入使得具体过程更加复杂。
具体程序注释如下：
#if JVET_M0102_INTRA_SUBPARTITIONS
void IntraSearch::estIntraPredLumaQT( CodingUnit &cu, Partitioner &partitioner, const double bestCostSoFar )//亮度角度预测
#else
void IntraSearch::estIntraPredLumaQT( CodingUnit &cu, Partitioner &partitioner )
#endif
{
CodingStructure &cs = *cu.cs;
const SPS &sps = *cs.sps;
const uint32_t uiWidthBit = g_aucLog2[partitioner.currArea().lwidth() ];//
const uint32_t uiHeightBit = g_aucLog2[partitioner.currArea().lheight()];
// Lambda calculation at equivalent Qp of 4 is recommended because at that Qp, the quantization divisor is 1.
const double sqrtLambdaForFirstPass = m_pcRdCost->getMotionLambda(cu.transQuantBypass) / double(1 << SCALE_BITS);
//===== loop over partitions =====
const TempCtx ctxStart ( m_CtxCache, m_CABACEstimator->getCtx() );
const TempCtx ctxStartIntraMode(m_CtxCache, SubCtx(Ctx::IntraLumaMpmFlag, m_CABACEstimator->getCtx()));
const TempCtx ctxStartMHIntraMode ( m_CtxCache, SubCtx( Ctx::MHIntraPredMode, m_CABACEstimator->getCtx() ) );
const TempCtx ctxStartMrlIdx ( m_CtxCache, SubCtx( Ctx::MultiRefLineIdx, m_CABACEstimator->getCtx() ) );
CHECK( !cu.firstPU, "CU has no PUs" );
const bool keepResi = cs.pps->getPpsRangeExtension().getCrossComponentPredictionEnabledFlag() || KEEP_PRED_AND_RESI_SIGNALS;
uint32_t extraModes = 0; // add two extra modes, which would be used after uiMode <= DC_IDX is removed for cu.nsstIdx == 3
#if !JVET_M0464_UNI_MTS
const int width = partitioner.currArea().lwidth();
const int height = partitioner.currArea().lheight();
// Marking EMT usage for faster EMT
// 0: EMT is either not applicable for current CU (cuWidth > EMT_INTRA_MAX_CU or cuHeight > EMT_INTRA_MAX_CU), not active in the config file or the fast decision algorithm is not used in this case
// 1: EMT fast algorithm can be applied for the current CU, and the DCT2 is being checked
// 2: EMT is being checked for current CU. Stored results of DCT2 can be utilized for speedup
uint8_t emtUsageFlag = 0;
const int maxSizeEMT = EMT_INTRA_MAX_CU_WITH_QTBT;
if( width <= maxSizeEMT && height <= maxSizeEMT && sps.getUseIntraEMT() )
{
emtUsageFlag = cu.emtFlag == 1 ? 2 : 1;
}
bool isAllIntra = m_pcEncCfg->getIntraPeriod() == 1;
if( width * height < 64 && !isAllIntra )
{
emtUsageFlag = 0; //this forces the recalculation of the candidates list. Why is this necessary? (to be checked)
}
#endif
#if JVET_M0102_INTRA_SUBPARTITIONS
#if JVET_M0464_UNI_MTS
const int width = partitioner.currArea().lwidth();
const int height = partitioner.currArea().lheight();
int nOptionsForISP = NUM_INTRA_SUBPARTITIONS_MODES;//ISP划分模式：HOR_INTRA_SUBPARTITIONS = 1, VER_INTRA_SUBPARTITIONS = 2,
//默认的NUM_INTRA_SUBPARTITIONS_MODES=3
#else
int nOptionsForISP = cu.emtFlag == 0 ? NUM_INTRA_SUBPARTITIONS_MODES : 1;
#endif
double bestCurrentCost = bestCostSoFar;//当前最佳代价
int ispOptions[NUM_INTRA_SUBPARTITIONS_MODES] = { 0 };//ISP模式初始为non-ISP
if( nOptionsForISP > 1 ) {
auto splitsThatCanBeUsedForISP = CU::canUseISPSplit( width, height, cu.cs->sps->getMaxTrSize() );//得到可能进行的ISP模式
if( splitsThatCanBeUsedForISP == CAN_USE_VER_AND_HORL_SPLITS ) {//水平和垂直ISP均可以
const CodingUnit* cuLeft = cu.ispMode != NOT_INTRA_SUBPARTITIONS ? cs.getCU( cs.area.blocks[partitioner.chType].pos().offset( -1, 0 ), partitioner.chType ) : nullptr;
const CodingUnit* cuAbove = cu.ispMode != NOT_INTRA_SUBPARTITIONS ? cs.getCU( cs.area.blocks[partitioner.chType].pos().offset( 0, -1 ), partitioner.chType ) : nullptr;
bool ispHorIsFirstTest = CU::firstTestISPHorSplit( width, height, COMPONENT_Y, cuLeft, cuAbove );
if( ispHorIsFirstTest )//先水平ISP，默认nOptionsForISP=3
{
ispOptions[1] = HOR_INTRA_SUBPARTITIONS;
ispOptions[2] = VER_INTRA_SUBPARTITIONS;
}
else//先垂直ISP，默认nOptionsForISP=3
{
ispOptions[1] = VER_INTRA_SUBPARTITIONS;
ispOptions[2] = HOR_INTRA_SUBPARTITIONS;
}
}
else if( splitsThatCanBeUsedForISP == HOR_INTRA_SUBPARTITIONS )
{
nOptionsForISP = 2;
ispOptions[1] = HOR_INTRA_SUBPARTITIONS;//水平ISP
}
else if( splitsThatCanBeUsedForISP == VER_INTRA_SUBPARTITIONS )
{
nOptionsForISP = 2;
ispOptions[1] = VER_INTRA_SUBPARTITIONS;//垂直ISP
}
else
{
nOptionsForISP = 1;//non-ISP
}
}
if( nOptionsForISP > 1 )//ISP模式清空列表
{
//variables for the full RD list without MRL modes
m_rdModeListWithoutMrl .clear();
m_rdModeListWithoutMrlHor .clear();
m_rdModeListWithoutMrlVer .clear();
//variables with data from regular intra used to skip ISP splits
m_intraModeDiagRatio .clear();
m_intraModeHorVerRatio .clear();
m_intraModeTestedNormalIntra.clear();
}
#endif
static_vector<uint32_t, FAST_UDI_MAX_RDMODE_NUM> uiHadModeList;//哈达玛模式列表
static_vector<double, FAST_UDI_MAX_RDMODE_NUM> CandCostList;//代价候选列表
static_vector<double, FAST_UDI_MAX_RDMODE_NUM> CandHadList;//绝对变换残差和列表
static_vector<int, FAST_UDI_MAX_RDMODE_NUM> extendRefList;//扩展参考列表MRL索引或ISP（MRL模式下ISP索引最终被取消）
static_vector<int, FAST_UDI_MAX_RDMODE_NUM>* nullList = NULL;
auto &pu = *cu.firstPU;//得到PU
#if !JVET_M0464_UNI_MTS
int puIndex = 0;
#endif
{
//列表清空
CandHadList.clear();
CandCostList.clear();
uiHadModeList.clear();
extendRefList.clear();
CHECK(pu.cu != &cu, "PU is not contained in the CU");
//===== determine set of modes to be tested (using prediction signal only) =====
int numModesAvailable = NUM_LUMA_MODE; // 可使用的模式数量
static_vector< uint32_t, FAST_UDI_MAX_RDMODE_NUM > uiRdModeList;//率失真模式列表
int numModesForFullRD = 3;//采用全RDO的模式数
numModesForFullRD = g_aucIntraModeNumFast_UseMPM_2D[uiWidthBit - MIN_CU_LOG2][uiHeightBit - MIN_CU_LOG2];//映射表
#if INTRA_FULL_SEARCH
numModesForFullRD = numModesAvailable;//全搜索
#endif
#if !JVET_M0464_UNI_MTS
if( emtUsageFlag != 2 )
#endif
{
// this should always be true
CHECK( !pu.Y().valid(), "PU is not valid" );
#if ENABLE_JVET_L0283_MRL//多行参考
bool isFirstLineOfCtu = (((pu.block(COMPONENT_Y).y)&((pu.cs->sps)->getMaxCUWidth() - 1)) == 0);//pu是否在图像第一行
int numOfPassesExtendRef = (isFirstLineOfCtu ? 1 : MRL_NUM_REF_LINES);//MRL最大值
#endif
pu.multiRefIdx = 0;//non-MRL索引
//===== init pattern for luma prediction =====
initIntraPatternChType( cu, pu.Y(), IntraPrediction::useFilteredIntraRefSamples( COMPONENT_Y, pu, false, pu ) );
if( numModesForFullRD != numModesAvailable )//不进行全搜索
{
CHECK( numModesForFullRD >= numModesAvailable, "Too many modes for full RD search" );
const CompArea &area = pu.Y();
PelBuf piOrg = cs.getOrgBuf(area);//原始像素
PelBuf piPred = cs.getPredBuf(area);//预测像素
DistParam distParam;//失真参数
const bool bUseHadamard = cu.transQuantBypass == 0;//是否采用哈达玛变换
#if JVET_M0427_INLOOP_RESHAPER
if (cu.slice->getReshapeInfo().getUseSliceReshaper() && m_pcReshape->getCTUFlag())
{
CompArea tmpArea(COMPONENT_Y, area.chromaFormat, Position(0, 0), area.size());
PelBuf tmpOrg = m_tmpStorageLCU.getBuf(tmpArea);
tmpOrg.copyFrom(piOrg);
tmpOrg.rspSignal(m_pcReshape->getFwdLUT());
m_pcRdCost->setDistParam(distParam, tmpOrg, piPred, sps.getBitDepth(CHANNEL_TYPE_LUMA), COMPONENT_Y, bUseHadamard);//设置失真参数
}
else
#endif
m_pcRdCost->setDistParam(distParam, piOrg, piPred, sps.getBitDepth(CHANNEL_TYPE_LUMA), COMPONENT_Y, bUseHadamard);
distParam.applyWeight = false;//不使用权重预测
bool bSatdChecked[NUM_INTRA_MODE];//模式校验标志
memset( bSatdChecked, 0, sizeof( bSatdChecked ) );//未校验
{//模式校验，RMD第一步--在传统的35模式中进行
for( int modeIdx = 0; modeIdx < numModesAvailable; modeIdx++ )
{
uint32_t uiMode = modeIdx;
Distortion uiSad = 0;
// Skip checking extended Angular modes in the first round of SATD
if( uiMode > DC_IDX && ( uiMode & 1 ) )//跳过新增的角度模式，RMD第一步
{
continue;
}
bSatdChecked[uiMode] = true;//校验确认
pu.intraDir[0] = modeIdx;//存储亮度角度预测模式
if( useDPCMForFirstPassIntraEstimation( pu, uiMode ) )//使用DPCM
{
encPredIntraDPCM( COMPONENT_Y, piOrg, piPred, uiMode );
}
else
{
predIntraAng( COMPONENT_Y, piPred, pu, IntraPrediction::useFilteredIntraRefSamples( COMPONENT_Y, pu, true, pu ) );//进行角度预测
}
// use Hadamard transform here
uiSad += distParam.distFunc(distParam);//获得绝对变换残差和
// NB xFracModeBitsIntra will not affect the mode for chroma that may have already been pre-estimated.
m_CABACEstimator->getCtx() = SubCtx(Ctx::IntraLumaMpmFlag, ctxStartIntraMode);
m_CABACEstimator->getCtx() = SubCtx( Ctx::MHIntraPredMode, ctxStartMHIntraMode );
m_CABACEstimator->getCtx() = SubCtx( Ctx::MultiRefLineIdx, ctxStartMrlIdx );
uint64_t fracModeBits = xFracModeBitsIntra(pu, uiMode, CHANNEL_TYPE_LUMA);//获得编码比特数
double cost = ( double ) uiSad + ( double ) fracModeBits * sqrtLambdaForFirstPass;//计算哈达玛代价值
DTRACE( g_trace_ctx, D_INTRA_COST, "IntraHAD: %u, %llu, %f (%d)\n", uiSad, fracModeBits, cost, uiMode );
updateCandList( uiMode, cost, uiRdModeList, CandCostList
, extendRefList, 0
, numModesForFullRD + extraModes );//更新率失真模式、代价列表、扩展参考列表
updateCandList(uiMode, (double) uiSad, uiHadModeList, CandHadList
, *nullList, -1
, 3 + extraModes);//更新哈达玛模式、绝对变换残差和（哈达玛变换，不考虑比特率）列表
}
} // NSSTFlag
// forget the extra modes
uiRdModeList.resize( numModesForFullRD );//仅保留全RDO数量的模式
CandCostList.resize(numModesForFullRD);
extendRefList.resize(numModesForFullRD);
static_vector<unsigned, FAST_UDI_MAX_RDMODE_NUM> parentCandList(FAST_UDI_MAX_RDMODE_NUM);
std::copy_n(uiRdModeList.begin(), numModesForFullRD, parentCandList.begin());//将RMD第一步的模式候选保存
//RMD第二步--细化角度搜索 Second round of SATD for extended Angular modes//
for (int modeIdx = 0; modeIdx < numModesForFullRD; modeIdx++) {
unsigned parentMode = parentCandList[modeIdx];//得到第一步中的模式
if (parentMode > (DC_IDX + 1) && parentMode < (NUM_LUMA_MODE - 1))//有效的角度模式
{
for (int subModeIdx = -1; subModeIdx <= 1; subModeIdx += 2)//对第一步的角度进行细化
{
unsigned mode = parentMode + subModeIdx;//角度偏移
if (!bSatdChecked[mode])//确认此角度没有被校验过
{
pu.intraDir[0] = mode;//保存模式
if (useDPCMForFirstPassIntraEstimation(pu, mode)) {
encPredIntraDPCM(COMPONENT_Y, piOrg, piPred, mode);
}
else {
predIntraAng(COMPONENT_Y, piPred, pu,
IntraPrediction::useFilteredIntraRefSamples(COMPONENT_Y, pu, true, pu));//角度预测
}
// use Hadamard transform here
Distortion sad = distParam.distFunc(distParam);//绝对变换残差和
// NB xFracModeBitsIntra will not affect the mode for chroma that may have already been pre-estimated.
m_CABACEstimator->getCtx() = SubCtx(Ctx::IntraLumaMpmFlag, ctxStartIntraMode);
m_CABACEstimator->getCtx() = SubCtx( Ctx::MHIntraPredMode, ctxStartMHIntraMode );
m_CABACEstimator->getCtx() = SubCtx( Ctx::MultiRefLineIdx, ctxStartMrlIdx );
uint64_t fracModeBits = xFracModeBitsIntra(pu, mode, CHANNEL_TYPE_LUMA);//比特数
double cost = (double) sad + (double) fracModeBits * sqrtLambdaForFirstPass;//哈达玛代价
updateCandList(mode, cost, uiRdModeList, CandCostList
, extendRefList, 0
, numModesForFullRD);//
updateCandList(mode, (double)sad, uiHadModeList, CandHadList
, *nullList, -1
, 3);
bSatdChecked[mode] = true;//确认校验
}
}
}
}
#if JVET_M0102_INTRA_SUBPARTITIONS
if( nOptionsForISP > 1 ) {//如果能进行ISP
//we save the list with no mrl modes to keep only the Hadamard selected modes (no mpms)
m_rdModeListWithoutMrl.resize( numModesForFullRD );
std::copy_n( uiRdModeList.begin(), numModesForFullRD, m_rdModeListWithoutMrl.begin() );//保存non-MRL的哈达玛代价模式
}
#endif
#if ENABLE_JVET_L0283_MRL//MRL-MPM
pu.multiRefIdx = 1;//MRL索引
const int numMPMs = NUM_MOST_PROBABLE_MODES;//MPM模式数
unsigned multiRefMPM [numMPMs];//MPM模式
PU::getIntraMPMs(pu, multiRefMPM);//获得MPM模式
for (int mRefNum = 1; mRefNum < numOfPassesExtendRef; mRefNum++)//遍历MRL，扩展索引1和3
{
int multiRefIdx = MULTI_REF_LINE_IDX[mRefNum];//转换MRL索引
pu.multiRefIdx = multiRefIdx;
{//对参考像素滤波并获取
initIntraPatternChType(cu, pu.Y(), IntraPrediction::useFilteredIntraRefSamples(COMPONENT_Y, pu, false, pu));
}
for (int x = 0; x < numMPMs; x++)//遍历MRL-MPM
{
uint32_t mode = multiRefMPM[x];//得到MPM模式
{
pu.intraDir[0] = mode;//存储模式
if (useDPCMForFirstPassIntraEstimation(pu, mode)) {
encPredIntraDPCM(COMPONENT_Y, piOrg, piPred, mode);
}
else{
predIntraAng(COMPONENT_Y, piPred, pu, IntraPrediction::useFilteredIntraRefSamples(COMPONENT_Y, pu, true, pu));//角度预测
}
// use Hadamard transform here
Distortion sad = distParam.distFunc(distParam);
// NB xFracModeBitsIntra will not affect the mode for chroma that may have already been pre-estimated.
m_CABACEstimator->getCtx() = SubCtx(Ctx::IntraLumaMpmFlag, ctxStartIntraMode);
m_CABACEstimator->getCtx() = SubCtx( Ctx::MHIntraPredMode, ctxStartMHIntraMode );
m_CABACEstimator->getCtx() = SubCtx( Ctx::MultiRefLineIdx, ctxStartMrlIdx );
uint64_t fracModeBits = xFracModeBitsIntra(pu, mode, CHANNEL_TYPE_LUMA);
double cost = (double)sad + (double)fracModeBits * sqrtLambdaForFirstPass;
updateCandList(mode, cost, uiRdModeList, CandCostList, extendRefList, multiRefIdx, numModesForFullRD);
}
}
}
#endif
CandCostList.resize(numModesForFullRD);//保留代价最小的numModesForFullRD种模式
extendRefList.resize(numModesForFullRD);
if( m_pcEncCfg->getFastUDIUseMPMEnabled() )
{//non-MRL MPM//正常MPM模式
const int numMPMs = NUM_MOST_PROBABLE_MODES;
unsigned uiPreds[numMPMs];
pu.multiRefIdx = 0;//non-MRL
const int numCand = PU::getIntraMPMs( pu, uiPreds );
for( int j = 0; j < numCand; j++ )//遍历MPM
{
bool mostProbableModeIncluded = false;//是否已经包括MPM模式
int mostProbableMode = uiPreds[j];//MPM模式
for( int i = 0; i < numModesForFullRD; i++ )//MPM校验
{
mostProbableModeIncluded |= (mostProbableMode == uiRdModeList[i] && extendRefList[i] == 0);//non-MRL MPM中是否包括了当前MPM
}
if( !mostProbableModeIncluded )//MPM没有被包括
{
extendRefList.push_back(0);//non-MRL标记
numModesForFullRD++;//增加全RDO模式数
uiRdModeList.push_back( mostProbableMode );//保存MPM到列表
}
}
#if JVET_M0102_INTRA_SUBPARTITIONS
//ISP-MPM
if( nOptionsForISP > 1 )//ISP模式
{
//we add the ISP MPMs to the list without mrl modes
//添加ISP的MPM模式到non-MRL列表
m_rdModeListWithoutMrlHor = m_rdModeListWithoutMrl;
m_rdModeListWithoutMrlVer = m_rdModeListWithoutMrl;
static_vector<uint32_t, FAST_UDI_MAX_RDMODE_NUM>* listPointer;
for( int k = 1; k < nOptionsForISP; k++ ) {//遍历ISP模式
cu.ispMode = ispOptions[k];//得到ISP模式
listPointer = &( cu.ispMode == HOR_INTRA_SUBPARTITIONS ? m_rdModeListWithoutMrlHor : m_rdModeListWithoutMrlVer );//水平或垂直
const int numCandISP = PU::getIntraMPMs( pu, uiPreds );//MPM
for( int j = 0; j < numCandISP; j++ )
{
bool mostProbableModeIncluded = false;
int mostProbableMode = uiPreds[j];
for( int i = 0; i < listPointer->size(); i++ ){//校验
mostProbableModeIncluded |= ( mostProbableMode == listPointer->at( i ) );
}
if( !mostProbableModeIncluded ) {
listPointer->push_back( mostProbableMode );
}
}
}
cu.ispMode = NOT_INTRA_SUBPARTITIONS;
}
#endif
}
}
else{//进行角度全搜索
for( int i = 0; i < numModesForFullRD; i++ ) {
uiRdModeList.push_back( i );
}
}
#if !JVET_M0464_UNI_MTS
if( emtUsageFlag == 1 ){
// Store the modes to be checked with RD
m_savedNumRdModes[puIndex] = numModesForFullRD;
std::copy_n( uiRdModeList.begin(), numModesForFullRD, m_savedRdModeList[puIndex] );
std::copy_n(extendRefList.begin(), numModesForFullRD, m_savedExtendRefList[puIndex]);
}
#endif
}
#if !JVET_M0464_UNI_MTS
else //emtUsage = 2 (here we potentially reduce the number of modes that will be full-RD checked)
{
if( isAllIntra && m_pcEncCfg->getFastIntraEMT() )
{
double thresholdSkipMode = 1.0 + 1.4 / sqrt( ( double ) ( width*height ) );
numModesForFullRD = 0;
// Skip checking the modes with much larger R-D cost than the best mode
for( int i = 0; i < m_savedNumRdModes[puIndex]; i++ )
{
if( m_modeCostStore[puIndex][i] <= thresholdSkipMode * m_bestModeCostStore[puIndex] )
{
uiRdModeList.push_back( m_savedRdModeList[puIndex][i] );
extendRefList.push_back(m_savedExtendRefList[puIndex][i]);
numModesForFullRD++;
}
}
}
else //this is necessary because we skip the candidates list calculation, since it was already obtained for the DCT-II. Now we load it
{
// Restore the modes to be checked with RD
numModesForFullRD = m_savedNumRdModes[puIndex];
uiRdModeList.resize( numModesForFullRD );
std::copy_n( m_savedRdModeList[puIndex], m_savedNumRdModes[puIndex], uiRdModeList.begin() );
CandCostList.resize(numModesForFullRD);
extendRefList.resize(numModesForFullRD);
std::copy_n(m_savedExtendRefList[puIndex], m_savedNumRdModes[puIndex], extendRefList.begin());
}
}
#endif
#if JVET_M0102_INTRA_SUBPARTITIONS
//对于ISP列表移除non-MPM
if( nOptionsForISP > 1 ) // we remove the non-MPMs from the ISP lists
{
static_vector< uint32_t, FAST_UDI_MAX_RDMODE_NUM > uiRdModeListCopyHor = m_rdModeListWithoutMrlHor;
m_rdModeListWithoutMrlHor.clear();//复制并清除ISP列表
static_vector< uint32_t, FAST_UDI_MAX_RDMODE_NUM > uiRdModeListCopyVer = m_rdModeListWithoutMrlVer;
m_rdModeListWithoutMrlVer.clear();
static_vector< uint32_t, FAST_UDI_MAX_RDMODE_NUM > *listPointerCopy, *listPointer;
for( int ispOptionIdx = 1; ispOptionIdx < nOptionsForISP; ispOptionIdx++ ){//遍历ISP模式
cu.ispMode = ispOptions[ispOptionIdx];
//we get the mpm cand list
const int numMPMs = NUM_MOST_PROBABLE_MODES;
unsigned uiPreds[numMPMs];
pu.multiRefIdx = 0;
PU::getIntraMPMs( pu, uiPreds );
//we copy only the ISP MPMs
listPointerCopy = &( cu.ispMode == HOR_INTRA_SUBPARTITIONS ? uiRdModeListCopyHor : uiRdModeListCopyVer );
listPointer = &( cu.ispMode == HOR_INTRA_SUBPARTITIONS ? m_rdModeListWithoutMrlHor : m_rdModeListWithoutMrlVer );
for( int k = 0; k < listPointerCopy->size(); k++ ) {
for( int q = 0; q < numMPMs; q++ ) {
if( listPointerCopy->at( k ) == uiPreds[q] ){//列表中存在MPM模式
listPointer->push_back( listPointerCopy->at( k ) );//存入ISP列表
break;
}
}
}
}
cu.ispMode = NOT_INTRA_SUBPARTITIONS;
}
#endif
CHECK( numModesForFullRD != uiRdModeList.size(), "Inconsistent state!" );
// after this point, don't use numModesForFullRD
// PBINTRA fast
#if JVET_M0464_UNI_MTS
if( m_pcEncCfg->getUsePbIntraFast() && !cs.slice->isIntra() && uiRdModeList.size() < numModesAvailable )
#else
if( m_pcEncCfg->getUsePbIntraFast() && !cs.slice->isIntra() && uiRdModeList.size() < numModesAvailable && emtUsageFlag != 2 )
#endif
{
if( CandHadList.size() < 3 || CandHadList[2] > cs.interHad * PBINTRA_RATIO )
{
uiRdModeList.resize( std::min<size_t>( uiRdModeList.size(), 2 ) );
#if JVET_M0102_INTRA_SUBPARTITIONS
extendRefList.resize( std::min<size_t>( extendRefList.size(), 2 ) );
if( nOptionsForISP > 1 )
{
m_rdModeListWithoutMrlHor.resize( std::min<size_t>( m_rdModeListWithoutMrlHor.size(), 2 ) );
m_rdModeListWithoutMrlVer.resize( std::min<size_t>( m_rdModeListWithoutMrlVer.size(), 2 ) );
}
#endif
}
if( CandHadList.size() < 2 || CandHadList[1] > cs.interHad * PBINTRA_RATIO )
{
uiRdModeList.resize( std::min<size_t>( uiRdModeList.size(), 1 ) );
#if JVET_M0102_INTRA_SUBPARTITIONS
extendRefList.resize( std::min<size_t>( extendRefList.size(), 1 ) );
if( nOptionsForISP > 1 )
{
m_rdModeListWithoutMrlHor.resize( std::min<size_t>( m_rdModeListWithoutMrlHor.size(), 1 ) );
m_rdModeListWithoutMrlVer.resize( std::min<size_t>( m_rdModeListWithoutMrlVer.size(), 1 ) );
}
#endif
}
if( CandHadList.size() < 1 || CandHadList[0] > cs.interHad * PBINTRA_RATIO )
{
cs.dist = std::numeric_limits<Distortion>::max();
cs.interHad = 0;
//===== reset context models =====
m_CABACEstimator->getCtx() = SubCtx(Ctx::IntraLumaMpmFlag, ctxStartIntraMode);
m_CABACEstimator->getCtx() = SubCtx( Ctx::MHIntraPredMode, ctxStartMHIntraMode );
m_CABACEstimator->getCtx() = SubCtx( Ctx::MultiRefLineIdx, ctxStartMrlIdx );
return;
}
}
#if JVET_M0102_INTRA_SUBPARTITIONS && JVET_M0464_UNI_MTS
if ( nOptionsForISP > 1 )
{
//we create a single full RD list that includes all intra modes using regular intra, MRL and ISP
auto* firstIspList = ispOptions[1] == HOR_INTRA_SUBPARTITIONS ? &m_rdModeListWithoutMrlHor : &m_rdModeListWithoutMrlVer;
auto* secondIspList = ispOptions[1] == HOR_INTRA_SUBPARTITIONS ? &m_rdModeListWithoutMrlVer : &m_rdModeListWithoutMrlHor;
if ( m_pcEncCfg->getUseFastISP() )
{
// find the first non-MRL mode
size_t indexFirstMode = std::find( extendRefList.begin(), extendRefList.end(), 0 ) - extendRefList.begin();
// if not found, just take the last mode
if( indexFirstMode >= extendRefList.size() ) indexFirstMode = extendRefList.size() - 1;
// move the mode indicated by indexFirstMode to the beginning
for( int idx = ((int)indexFirstMode) - 1; idx >= 0; idx-- )
{
std::swap( extendRefList[idx], extendRefList[idx + 1] );
std::swap( uiRdModeList [idx], uiRdModeList [idx + 1] );
}
//insert all ISP modes after the first non-mrl mode
uiRdModeList.insert( uiRdModeList.begin() + 1, secondIspList->begin(), secondIspList->end() );
uiRdModeList.insert( uiRdModeList.begin() + 1, firstIspList->begin() , firstIspList->end() );
extendRefList.insert( extendRefList.begin() + 1, secondIspList->size(), MRL_NUM_REF_LINES + ispOptions[2] );
extendRefList.insert( extendRefList.begin() + 1, firstIspList->size() , MRL_NUM_REF_LINES + ispOptions[1] );
}
else
{
//insert all ISP modes at the end of the current list
uiRdModeList.insert( uiRdModeList.end(), secondIspList->begin(), secondIspList->end() );
uiRdModeList.insert( uiRdModeList.end(), firstIspList->begin() , firstIspList->end() );
extendRefList.insert( extendRefList.end(), secondIspList->size(), MRL_NUM_REF_LINES + ispOptions[2] );
extendRefList.insert( extendRefList.end(), firstIspList->size() , MRL_NUM_REF_LINES + ispOptions[1] );
}
}
CHECKD(uiRdModeList.size() != extendRefList.size(),"uiRdModeList and extendRefList do not have the same size!");
#endif
//===== check modes (using r-d costs) =====
uint32_t uiBestPUMode = 0;//最佳角度模式
int bestExtendRef = 0;//最佳扩展索引
CodingStructure *csTemp = m_pTempCS[gp_sizeIdxInfo->idxFrom( cu.lwidth() )][gp_sizeIdxInfo->idxFrom( cu.lheight() )];
CodingStructure *csBest = m_pBestCS[gp_sizeIdxInfo->idxFrom( cu.lwidth() )][gp_sizeIdxInfo->idxFrom( cu.lheight() )];
csTemp->slice = cs.slice;
csBest->slice = cs.slice;
csTemp->initStructData();
csBest->initStructData();
// just to be sure
numModesForFullRD = ( int ) uiRdModeList.size();//全RDO模式数
#if JVET_M0102_INTRA_SUBPARTITIONS
PartSplit intraSubPartitionsProcOrder = TU_NO_ISP;//
int bestNormalIntraModeIndex = -1;//最佳帧内模式索引
uint8_t bestIspOption = NOT_INTRA_SUBPARTITIONS;//最佳ISP选项
TUIntraSubPartitioner subTuPartitioner( partitioner );//subTU划分器
#if JVET_M0464_UNI_MTS
bool ispHorAllZeroCbfs = false, ispVerAllZeroCbfs = false;
//********全RDO搜索*********//
for (uint32_t uiMode = 0; uiMode < numModesForFullRD; uiMode++)
{
// set luma prediction mode
uint32_t uiOrgMode = uiRdModeList[uiMode];//预测模式
cu.ispMode = extendRefList[uiMode] > MRL_NUM_REF_LINES ? extendRefList[uiMode] - MRL_NUM_REF_LINES : NOT_INTRA_SUBPARTITIONS;//ISP模式
#else
if ( !cu.ispMode && !cu.emtFlag )
{
m_modeCtrl->setEmtFirstPassNoIspCost( MAX_DOUBLE );
}
for( uint32_t ispOptionIdx = 0; ispOptionIdx < nOptionsForISP; ispOptionIdx++ )
{
cu.ispMode = ispOptions[ispOptionIdx];
int numModesForFullRDispOption = cu.ispMode == NOT_INTRA_SUBPARTITIONS ? numModesForFullRD : cu.ispMode == HOR_INTRA_SUBPARTITIONS ? (int)m_rdModeListWithoutMrlHor.size() : (int)m_rdModeListWithoutMrlVer.size();
for( uint32_t uiMode = 0; uiMode < numModesForFullRDispOption; uiMode++ )
{
// set luma prediction mode
uint32_t uiOrgMode = cu.ispMode == NOT_INTRA_SUBPARTITIONS ? uiRdModeList[uiMode] : cu.ispMode == HOR_INTRA_SUBPARTITIONS ? m_rdModeListWithoutMrlHor[uiMode] : m_rdModeListWithoutMrlVer[uiMode];
#endif
pu.intraDir[0] = uiOrgMode;
int multiRefIdx = 0;
pu.multiRefIdx = multiRefIdx;
if( cu.ispMode )//ISP模式
{
intraSubPartitionsProcOrder = CU::getISPType( cu, COMPONENT_Y );
bool tuIsDividedInRows = CU::divideTuInRows( cu );
#if !OPTIMAL_IN_ANG
#if JVET_M0464_UNI_MTS
if ( ( tuIsDividedInRows && ispHorAllZeroCbfs ) || ( !tuIsDividedInRows && ispVerAllZeroCbfs ) )
{
continue;
}
#endif
if( m_intraModeDiagRatio.at( bestNormalIntraModeIndex ) > 1.25 )
{
continue;
}
if( ( m_intraModeHorVerRatio.at( bestNormalIntraModeIndex ) > 1.25 && tuIsDividedInRows ) || ( m_intraModeHorVerRatio.at( bestNormalIntraModeIndex ) < 0.8 && !tuIsDividedInRows ) )
{
continue;
}
#endif
}
else//Non-ISP模式
{
multiRefIdx = extendRefList[uiMode];
pu.multiRefIdx = multiRefIdx;//扩展索引
CHECK( pu.multiRefIdx && ( pu.intraDir[0] == DC_IDX || pu.intraDir[0] == PLANAR_IDX ), "ERL" );
}
#else
for (uint32_t uiMode = 0; uiMode < numModesForFullRD; uiMode++)
{
// set luma prediction mode
uint32_t uiOrgMode = uiRdModeList[uiMode];
pu.intraDir[0] = uiOrgMode;
int multiRefIdx = extendRefList[uiMode];
pu.multiRefIdx = multiRefIdx;
CHECK(pu.multiRefIdx && (pu.intraDir[0] == DC_IDX || pu.intraDir[0] == PLANAR_IDX), "ERL");
#endif
// set context models
m_CABACEstimator->getCtx() = ctxStart;
// determine residual for partition
cs.initSubStructure( *csTemp, partitioner.chType, cs.area, true );
#if JVET_M0102_INTRA_SUBPARTITIONS
if( cu.ispMode ){//ISP
xRecurIntraCodingLumaQT( *csTemp, subTuPartitioner, bestCurrentCost, 0, intraSubPartitionsProcOrder );
}
else{//non-ISP
#if JVET_M0464_UNI_MTS
xRecurIntraCodingLumaQT( *csTemp, partitioner, bestIspOption ? bestCurrentCost : MAX_DOUBLE, -1, TU_NO_ISP, bestIspOption );//
#else
xRecurIntraCodingLumaQT( *csTemp, partitioner, MAX_DOUBLE, -1 );
#endif
}
if( cu.ispMode && !csTemp->cus[0]->firstTU->cbf[COMPONENT_Y] )
{
#if JVET_M0464_UNI_MTS
if ( cu.ispMode == HOR_INTRA_SUBPARTITIONS )
{
ispHorAllZeroCbfs |= ( m_pcEncCfg->getUseFastISP() && csTemp->tus[0]->lheight() > 2 && csTemp->cost >= bestCurrentCost );
}
else
{
ispVerAllZeroCbfs |= ( m_pcEncCfg->getUseFastISP() && csTemp->tus[0]->lwidth() > 2 && csTemp->cost >= bestCurrentCost );
}
#endif
csTemp->cost = MAX_DOUBLE;
#if JVET_M0428_ENC_DB_OPT
csTemp->costDbOffset = 0;
#endif
}
#else
xRecurIntraCodingLumaQT( *csTemp, partitioner );
#endif
#if !JVET_M0464_UNI_MTS
#if JVET_M0102_INTRA_SUBPARTITIONS
if (emtUsageFlag == 1 && m_pcEncCfg->getFastIntraEMT() && !cu.ispMode)
#else
if( emtUsageFlag == 1 && m_pcEncCfg->getFastIntraEMT() )
#endif
{
m_modeCostStore[puIndex][uiMode] = csTemp->cost; //cs.cost;
}
#endif
DTRACE( g_trace_ctx, D_INTRA_COST, "IntraCost T %f (%d) \n", csTemp->cost, uiOrgMode );
// check r-d cost
if( csTemp->cost < csBest->cost )
{
std::swap( csTemp, csBest );
uiBestPUMode = uiOrgMode;//最佳PU模式
bestExtendRef = multiRefIdx;//最佳扩展索引
#if JVET_M0102_INTRA_SUBPARTITIONS
bestIspOption = cu.ispMode;//最佳ISP选项
#endif
#if !JVET_M0464_UNI_MTS
#if JVET_M0102_INTRA_SUBPARTITIONS
if (emtUsageFlag == 1 && m_pcEncCfg->getFastIntraEMT() && !cu.ispMode)
#else
if( ( emtUsageFlag == 1 ) && m_pcEncCfg->getFastIntraEMT() )
#endif
{
m_bestModeCostStore[puIndex] = csBest->cost; //cs.cost;
}
#endif
#if JVET_M0102_INTRA_SUBPARTITIONS
if( csBest->cost < bestCurrentCost )
{
bestCurrentCost = csBest->cost;//保持当前代价最小
}
if( !cu.ispMode )
{
bestNormalIntraModeIndex = uiMode;//最佳模式索引
}
#endif
}
csTemp->releaseIntermediateData();
} // Mode loop
#if JVET_M0102_INTRA_SUBPARTITIONS
#if !JVET_M0464_UNI_MTS
if (!cu.ispMode && !cu.emtFlag)
{
m_modeCtrl->setEmtFirstPassNoIspCost(csBest->cost);
}
}
#endif
cu.ispMode = bestIspOption;
#endif
#if JVET_M0427_INLOOP_RESHAPER
cs.useSubStructure(*csBest, partitioner.chType, pu.singleChan(CHANNEL_TYPE_LUMA), true, true, keepResi, keepResi);
#else
cs.useSubStructure( *csBest, partitioner.chType, pu.singleChan( CHANNEL_TYPE_LUMA ), KEEP_PRED_AND_RESI_SIGNALS, true, keepResi, keepResi );
#endif
csBest->releaseIntermediateData();
//=== update PU data ====
pu.intraDir[0] = uiBestPUMode;
pu.multiRefIdx = bestExtendRef;
}
//===== reset context models =====
m_CABACEstimator->getCtx() = ctxStart;
}
454
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
