VTM1.0代码阅读：estIntraPredLumaQT函数

estIntraPredLumaQT在67种帧内亮度模式中，获取帧内亮度模式的最优模式。
函数流程比较简单：
先对67中模式进行SATD粗选择；
之后再加入MPM的帧内模式；
最后RDO得到最优的帧内模式。

Void IntraSearch::estIntraPredLumaQT( CodingUnit &cu, Partitioner &partitioner )
{
  CodingStructure       &cs            = *cu.cs;
  const SPS             &sps           = *cs.sps;
  const UInt             uiWidthBit    = cs.pcv->rectCUs ? g_aucLog2[partitioner.currArea().lwidth() ] : CU::getIntraSizeIdx(cu);
  const UInt             uiHeightBit   =                   g_aucLog2[partitioner.currArea().lheight()];

  // Lambda calculation at equivalent Qp of 4 is recommended because at that Qp, the quantization divisor is 1.
  const double sqrtLambdaForFirstPass = m_pcRdCost->getMotionLambda(cu.transQuantBypass) / double(1 << SCALE_BITS);


  //===== loop over partitions =====

  const TempCtx ctxStart          ( m_CtxCache, m_CABACEstimator->getCtx() );
  const TempCtx ctxStartIntraMode ( m_CtxCache, SubCtx( Ctx::IPredMode[CHANNEL_TYPE_LUMA],        m_CABACEstimator->getCtx() ) );

  CHECK( !cu.firstPU, "CU has no PUs" );
  const bool keepResi   = cs.pps->getPpsRangeExtension().getCrossComponentPredictionEnabledFlag() || KEEP_PRED_AND_RESI_SIGNALS;


  UInt extraModes = 0; // add two extra modes, which would be used after uiMode <= DC_IDX is removed for cu.nsstIdx == 3


  static_vector<UInt,   FAST_UDI_MAX_RDMODE_NUM> uiHadModeList;
  static_vector<Double, FAST_UDI_MAX_RDMODE_NUM> CandCostList;
  static_vector<Double, FAST_UDI_MAX_RDMODE_NUM> CandHadList;

  auto &pu = *cu.firstPU;
  {
    CandHadList.clear();
    CandCostList.clear();
    uiHadModeList.clear();

    CHECK(pu.cu != &cu, "PU is not contained in the CU");

    //===== determine set of modes to be tested (using prediction signal only) =====
    Int numModesAvailable = NUM_LUMA_MODE; // total number of Intra modes		帧内亮度模式数67
    static_vector< UInt, FAST_UDI_MAX_RDMODE_NUM > uiRdModeList;		//各帧内模式按照cost从小到大加入uiRdModeList

    Int numModesForFullRD = 3;
    if( cs.pcv->rectCUs )
    {
      numModesForFullRD = g_aucIntraModeNumFast_UseMPM_2D[uiWidthBit - MIN_CU_LOG2][uiHeightBit - MIN_CU_LOG2];
    }
    else
    {
      numModesForFullRD = m_pcEncCfg->getFastUDIUseMPMEnabled() ? g_aucIntraModeNumFast_UseMPM[uiWidthBit] : g_aucIntraModeNumFast_NotUseMPM[uiWidthBit];
    }


    {
      // this should always be true
      CHECK( !pu.Y().valid(), "PU is not valid" );

      //===== init pattern for luma prediction =====			//帧内预测的左侧和上侧的参考像素初始化，进行补全和滤波
      initIntraPatternChType( cu, pu.Y(), IntraPrediction::useFilteredIntraRefSamples( COMPONENT_Y, pu, false, pu ) );
      if( numModesForFullRD != numModesAvailable )			//首先SATD粗选择，最后进行RDO选最优
      {
        CHECK( numModesForFullRD >= numModesAvailable, "Too many modes for full RD search" );

        const CompArea &area = pu.Y();

        PelBuf piOrg         = cs.getOrgBuf(area);			//原始像素和预测像素
        PelBuf piPred        = cs.getPredBuf(area);

        DistParam distParam;								//由两个像素序列得失真的类

        const Bool bUseHadamard = cu.transQuantBypass == 0;	//是否使用哈达吗变换求失真
													//SATD通过orig和pred像素得失真，RDCheck通过orig和reco像素得失真
        m_pcRdCost->setDistParam(distParam, piOrg, piPred, sps.getBitDepth(CHANNEL_TYPE_LUMA), COMPONENT_Y, bUseHadamard);

        distParam.applyWeight = false;

        bool bSatdChecked[NUM_INTRA_MODE];
        memset( bSatdChecked, 0, sizeof( bSatdChecked ) );

        {
          for( Int modeIdx = 0; modeIdx < numModesAvailable; modeIdx++ )		//67种模式进行SATD
          {
            UInt       uiMode = modeIdx;
            Distortion uiSad  = 0;

            // Skip checking extended Angular modes in the first round of SATD
            if( uiMode > DC_IDX && ( uiMode & 1 ) )
            {
              continue;							//第一次循环只计算原来的35种模式，SATD之后添加选中模式的相邻模式再SATD
            }

            bSatdChecked[uiMode] = true;

            pu.intraDir[0] = modeIdx;

            if( useDPCMForFirstPassIntraEstimation( pu, uiMode ) )
            {
              encPredIntraDPCM( COMPONENT_Y, piOrg, piPred, uiMode );	//PCM模式
            }
            else
            { 															//按照帧内模式进行像素预测，得到预测像素pred
              predIntraAng( COMPONENT_Y, piPred, pu, IntraPrediction::useFilteredIntraRefSamples( COMPONENT_Y, pu, true, pu ) );
            }
            // use Hadamard transform here
            uiSad += distParam.distFunc(distParam);			//失真

            // NB xFracModeBitsIntra will not affect the mode for chroma that may have already been pre-estimated.
            m_CABACEstimator->getCtx() = SubCtx( Ctx::IPredMode[CHANNEL_TYPE_LUMA], ctxStartIntraMode );

            UInt64 fracModeBits = xFracModeBitsIntra(pu, uiMode, CHANNEL_TYPE_LUMA);

            Double cost = ( Double ) uiSad + ( Double ) fracModeBits * sqrtLambdaForFirstPass;		//cost

            DTRACE( g_trace_ctx, D_INTRA_COST, "IntraHAD: %u, %llu, %f (%d)\n", uiSad, fracModeBits, cost, uiMode );

            updateCandList( uiMode, cost,  uiRdModeList, CandCostList, numModesForFullRD + extraModes );	//按照cost从小到大将uiMode加入uiRdModeList中
            updateCandList( uiMode, uiSad, uiHadModeList, CandHadList, 3                 + extraModes );
          }
        } // NSSTFlag

        // forget the extra modes
        uiRdModeList.resize( numModesForFullRD );			//选取cost最小的numModesForFullRD个帧内模式

        if( m_pcEncCfg->getFastUDIUseMPMEnabled() )			//MPM
        {
          unsigned  numMPMs = pu.cs->pcv->numMPMs;			//mpm添加的模式数量
          unsigned *uiPreds = ( unsigned* ) alloca( numMPMs * sizeof( unsigned ) );

          const Int numCand = PU::getIntraMPMs( pu, uiPreds );		//获取mpm添加的帧内模式

          for( Int j = 0; j < numCand; j++ )
          {
            Bool mostProbableModeIncluded = false;
            Int  mostProbableMode         = uiPreds[j];


            for( Int i = 0; i < numModesForFullRD; i++ )		//mpm模式是否已经包含在uiRdModeList中
            {
              mostProbableModeIncluded |= ( mostProbableMode == uiRdModeList[i] );
            }
            if( !mostProbableModeIncluded )
            {
              numModesForFullRD++;
              uiRdModeList.push_back( mostProbableMode );		//mpm模式加入到uiRdModeList
            }
          }
        }
      }
      else
      {
        for( Int i = 0; i < numModesForFullRD; i++ )		//所有帧内模式都RDcheck
        {
          uiRdModeList.push_back( i );
        }
      }
    }


    CHECK( numModesForFullRD != uiRdModeList.size(), "Inconsistent state!" );

    // after this point, don't use numModesForFullRD

    // PBINTRA fast
    if( m_pcEncCfg->getUsePbIntraFast() && !cs.slice->isIntra() && cu.partSize == SIZE_2Nx2N && uiRdModeList.size() < numModesAvailable )
    {
      if( CandHadList.size() < 3 || CandHadList[2] > cs.interHad * PBINTRA_RATIO )
      {
        uiRdModeList.resize( std::min<size_t>( uiRdModeList.size(), 2 ) );
      }
      if( CandHadList.size() < 2 || CandHadList[1] > cs.interHad * PBINTRA_RATIO )
      {
        uiRdModeList.resize( std::min<size_t>( uiRdModeList.size(), 1 ) );
      }
      if( CandHadList.size() < 1 || CandHadList[0] > cs.interHad * PBINTRA_RATIO )
      {
        cs.dist     = MAX_UINT;
        cs.interHad = 0;

        //===== reset context models =====
        m_CABACEstimator->getCtx() = SubCtx( Ctx::IPredMode       [CHANNEL_TYPE_LUMA], ctxStartIntraMode );

        return;
      }
    }

    //===== check modes (using r-d costs) =====	RDO 率失真优化选择最优帧内模式
    UInt       uiBestPUMode  = 0;
												//csTemp用于各模式的RDCheck，之后将最优模式数据swap给csBest
    CodingStructure *csTemp = m_pTempCS[gp_sizeIdxInfo->idxFrom( cu.lwidth() )][gp_sizeIdxInfo->idxFrom( cu.lheight() )];
    CodingStructure *csBest = m_pBestCS[gp_sizeIdxInfo->idxFrom( cu.lwidth() )][gp_sizeIdxInfo->idxFrom( cu.lheight() )];

    csTemp->slice = cs.slice;
    csBest->slice = cs.slice;
    csTemp->initStructData();
    csBest->initStructData();

    // just to be sure
    numModesForFullRD = ( int ) uiRdModeList.size();
    for (UInt uiMode = 0; uiMode < numModesForFullRD; uiMode++)		//粗选择的模式，进行RDO选最优
    {
      // set luma prediction mode
      UInt uiOrgMode = uiRdModeList[uiMode];

      pu.intraDir[0] = uiOrgMode;


      // set context models
      m_CABACEstimator->getCtx() = ctxStart;

      // determine residual for partition
      cs.initSubStructure( *csTemp, partitioner.chType, cs.area, true );	//初始化csTemp

#if ENABLE_RQT_INTRA_SPEEDUP
      xRecurIntraCodingLumaQT( *csTemp, partitioner, true );
#else 															//预测得到各亮度模式的pred像素，从而得到残差，经过变换反变换得到reco像素
      xRecurIntraCodingLumaQT( *csTemp, partitioner );			//由orig和reco，可以得到真正的失真，从而得到cost
#endif

      DTRACE( g_trace_ctx, D_INTRA_COST, "IntraCost T %f (%d) \n", csTemp->cost, uiOrgMode );

      // check r-d cost
      if( csTemp->cost < csBest->cost )			//将cost最小的帧内模式存入csBest
      {
        std::swap( csTemp, csBest );

        uiBestPUMode  = uiOrgMode;

      }

      csTemp->releaseIntermediateData();		//csTemp清空初始化
    } // Mode loop

												//csBest中的最优亮度帧内模式的数据，存入cs，也就是xCheckRDCostIntra函数中的tempCS
    cs.useSubStructure( *csBest, partitioner.chType, pu.singleChan( CHANNEL_TYPE_LUMA ), KEEP_PRED_AND_RESI_SIGNALS, true, keepResi, keepResi );

    csBest->releaseIntermediateData();			//csBest清空初始化
    //=== update PU data ====

    pu.intraDir[0] = uiBestPUMode;
  }

  //===== reset context models =====
  m_CABACEstimator->getCtx() = ctxStart;
}