VVC代码阅读 帧内预测部分(1) xCheckRDCostIntra()函数（部分）

VVC参考软件VTM6.0中变换的相关代码讲解与注释（一）_sky_Ryota的博客-CSDN博客

VTM6变换算法小结（译自O2002）_Allen---Jiang的博客-CSDN博客

H.266/VVC代码学习：xCheckRDCostIntra函数_涵小呆的博客-CSDN博客

亮度预测函数前

bool EncCu::xCheckRDCostIntra(CodingStructure *&tempCS, CodingStructure *&bestCS, Partitioner &partitioner, const EncTestMode& encTestMode, bool adaptiveColorTrans)
{
  double          bestInterCost             = m_modeCtrl->getBestInterCost();//最佳帧间代价
  double          costSize2Nx2NmtsFirstPass = m_modeCtrl->getMtsSize2Nx2NFirstPassCost();//MTS不开启情况下的RDcost
  bool            skipSecondMtsPass         = m_modeCtrl->getSkipSecondMTSPass();//为true表示跳过测试除了第一个trGrp之外的情况
  const SPS&      sps                       = *tempCS->sps;

//为True表示测试MTS开启的情况
  const int       maxSizeMTS                = MTS_INTRA_MAX_CU_SIZE;//32
  uint8_t         considerMtsSecondPass     = ( sps.getUseIntraMTS() && isLuma( partitioner.chType ) && partitioner.currArea().lwidth() <= maxSizeMTS && partitioner.currArea().lheight() <= maxSizeMTS ) ? 1 : 0;
  //为1表示当前CU开启ISP
  bool   useIntraSubPartitions   = false;
  //色度分量最大允许RDcost
  double maxCostAllowedForChroma = MAX_DOUBLE;
  //当前CU的最佳模式
  const  CodingUnit *bestCU      = bestCS->getCU( partitioner.chType );
  Distortion interHad = m_modeCtrl->getInterHad();

  //一些参数设置
  double dct2Cost                =   MAX_DOUBLE;//不开启MTS且不开启LFNST情况下的RDcost
  double bestNonDCT2Cost         = MAX_DOUBLE;//开启MTS或开启LFNST情况下的最佳RDcost
  //如果对应的trGrp中有测试的模式成为最佳模式，此为对应的trGrp的最佳RDcost
  double trGrpBestCost     [ 4 ] = { MAX_DOUBLE, MAX_DOUBLE, MAX_DOUBLE, MAX_DOUBLE };
  double globalBestCost          =   MAX_DOUBLE;//如果有测试的模式成为最佳模式，此为最佳RDcost
  bool   bestSelFlag       [ 4 ] = { false, false, false, false };//如果对应的trGrp中有测试的模式成为最佳模式，此为true
  bool   trGrpCheck        [ 4 ] = { true, true, true, true };//如果对应的trGrp需要测试，此为true
  //对应trGrp要测试MTS中的哪些情况由下面两个变量决定
  int    startMTSIdx       [ 4 ] = { 0, 1, 2, 3 };
  int    endMTSIdx         [ 4 ] = { 0, 1, 2, 3 };
  //当前trGrp的最佳RDcost与不开启MTS且不开启LFNST情况下的RDcost的比例如果超过阈值，就会停止测试下一个trGrp。这个阈值就由trGrpStopThreshold决定
  double trGrpStopThreshold[ 3 ] = { 1.001, 1.001, 1.001 };
  int    bestMtsFlag             =   0;//如果有测试的模式成为最佳模式，此为最佳模式的mtsFlag
  int    bestLfnstIdx            =   0;//如果有测试的模式成为最佳模式，此为最佳模式的lfnstIdx
  
  //如果当前CU可以测试LFNST开启的情况，此为2否则就为0
  const int  maxLfnstIdx         = ( partitioner.isSepTree( *tempCS ) && partitioner.chType == CHANNEL_TYPE_CHROMA && ( partitioner.currArea().lwidth() < 8 || partitioner.currArea().lheight() < 8 ) )
                                   || ( partitioner.currArea().lwidth() > sps.getMaxTbSize() || partitioner.currArea().lheight() > sps.getMaxTbSize() ) ? 0 : 2;
  //为true表示跳过测试其余的LFNST的情况
  bool       skipOtherLfnst      = false;
  //要测试LFNST中的哪些情况由下面两个变量决定
  int        startLfnstIdx       = 0;
  int        endLfnstIdx         = sps.getUseLFNST() ? maxLfnstIdx : 0;
  
  //trGrp的数量
  int grpNumMax = sps.getUseLFNST() ? m_pcEncCfg->getMTSIntraMaxCand() : 1;
  m_modeCtrl->setISPWasTested(false);
  m_pcIntraSearch->invalidateBestModeCost();
  if (sps.getUseColorTrans() && !CS::isDualITree(*tempCS))
  {
    if ((m_pcEncCfg->getRGBFormatFlag() && adaptiveColorTrans) || (!m_pcEncCfg->getRGBFormatFlag() && !adaptiveColorTrans))
    {
      m_pcIntraSearch->invalidateBestRdModeFirstColorSpace();
    }
  }

  bool foundZeroRootCbf = false;/发现RootCbf为0的情况
  if (sps.getUseColorTrans())  //如果用了色彩转换
  {
    CHECK(tempCS->treeType != TREE_D || partitioner.treeType != TREE_D, "localtree should not be applied when adaptive color transform is enabled");
    CHECK(tempCS->modeType != MODE_TYPE_ALL || partitioner.modeType != MODE_TYPE_ALL, "localtree should not be applied when adaptive color transform is enabled");
    CHECK(adaptiveColorTrans && (CS::isDualITree(*tempCS) || partitioner.chType != CHANNEL_TYPE_LUMA), "adaptive color transform cannot be applied to dual-tree");
  }

  

MTS：多变换核选择

LFNST:低频不可分离变换

trGrp：推测应该是代表一个过程，测试MTS和LFNST在不同情况下的模式。有很多个trGrp

顶上第三个博客下评论：指的是MTS的DCT8和DST7的那四个组合，MTS_DST7_DST7、 MTS_DCT8_DST7、 MTS_DST7_DCT8、MTS_DCT8_DCT8。這四組變換都會對目前的CU block進行estIntraPredLumaQT函式，然後計算出最好的角度模式(0~67)

for循环分trGrp测试

for( int trGrpIdx = 0; trGrpIdx < grpNumMax; trGrpIdx++ )
  {
    //MTS的标志位设定
    const uint8_t startMtsFlag = trGrpIdx > 0;
    const uint8_t endMtsFlag   = sps.getUseLFNST() ? considerMtsSecondPass : 0;

    if( ( trGrpIdx == 0 || ( !skipSecondMtsPass && considerMtsSecondPass ) ) && trGrpCheck[ trGrpIdx ] )
    {
      for( int lfnstIdx = startLfnstIdx; lfnstIdx <= endLfnstIdx; lfnstIdx++ )
      {
        for( uint8_t mtsFlag = startMtsFlag; mtsFlag <= endMtsFlag; mtsFlag++ )
        {
          if (sps.getUseColorTrans() && !CS::isDualITree(*tempCS))
          {
            m_pcIntraSearch->setSavedRdModeIdx(trGrpIdx*(NUM_LFNST_NUM_PER_SET * 2) + lfnstIdx * 2 + mtsFlag);
          }
          if (mtsFlag > 0 && lfnstIdx > 0)
          {
            continue;
          }
          //3) if interHad is 0, only try further modes if some intra mode was already better than inter
          if( sps.getUseLFNST() && m_pcEncCfg->getUsePbIntraFast() && !tempCS->slice->isIntra() && bestCU && CU::isInter( *bestCS->getCU( partitioner.chType ) ) && interHad == 0 )
          {
            continue;
          }

          tempCS->initStructData( encTestMode.qp );
          //根据area信息创建一个CU，并在下面信息填充
          CodingUnit &cu      = tempCS->addCU( CS::getArea( *tempCS, tempCS->area, partitioner.chType ), partitioner.chType );
          //CU信息传递
          partitioner.setCUData( cu );
          cu.slice            = tempCS->slice;
          cu.tileIdx          = tempCS->pps->getTileIdx( tempCS->area.lumaPos() );
          cu.skip             = false;
          cu.mmvdSkip = false;
          cu.predMode         = MODE_INTRA;
          cu.chromaQpAdj      = m_cuChromaQpOffsetIdxPlus1;
          cu.qp               = encTestMode.qp;
          cu.lfnstIdx         = lfnstIdx;
          cu.mtsFlag          = mtsFlag;
          cu.ispMode          = NOT_INTRA_SUBPARTITIONS;
          cu.colorTransform = adaptiveColorTrans;
          //根据CU添加PU
          CU::addPUs( cu );

          tempCS->interHad    = interHad;
          为true表示更新了最佳模式
          m_bestModeUpdated = tempCS->useDbCost = bestCS->useDbCost = false;

          bool validCandRet = false;
          //判断是否亮度分量
          if( isLuma( partitioner.chType ) )
          {
            //ISP uses the value of the best cost so far (luma if it is the fast version) to avoid test non-necessary subpartitions
            double bestCostSoFar = partitioner.isSepTree(*tempCS) ? m_modeCtrl->getBestCostWithoutSplitFlags() : bestCU && bestCU->predMode == MODE_INTRA ? bestCS->lumaCost : bestCS->cost;
            if (partitioner.isSepTree(*tempCS) && encTestMode.maxCostAllowed < bestCostSoFar)
            {
              bestCostSoFar = encTestMode.maxCostAllowed;
            }
            validCandRet = m_pcIntraSearch->estIntraPredLumaQT(cu, partitioner, bestCostSoFar, mtsFlag, startMTSIdx[trGrpIdx], endMTSIdx[trGrpIdx], (trGrpIdx > 0), !cu.colorTransform ? bestCS : nullptr);

变换的部分先粗略看，可参考VTM10.0代码学习13：xCheckRDCostIntra()_柴门风雪夜的博客-CSDN博客

addPUs()可参考之前写的addCU()函数

m_bestModeUpdated：为true表示更新了最佳模式

bestCostSoFar：检测当前tempCs是否为分离树,如果为true，则值为m_ComprCUCtxList中的bestCostWithoutSplitFlags。如果为false，则判断bestCU是否存在且preMode是否为MODE_INTRA。为true,则值为bestCS->lumaCost.为false,值为bestCS->Cost

if语句：如果当前tempCs为分离树且当前测试模式的maxCostAllowed小于bestCostSoFar，则

bestCostSoFar = maxCostAllowed

然后就是亮度预测函数estIntraPredLumaQT()

后面的之后再看