H.266/VVC-VTM代码学习21-对划分模式进行RDO的函数xCheckModeSplit

H.266/VVC专栏传送

上一篇：H.266/VVC-VTM代码学习20-CU层进行RDO函数xCompressCU
下一篇：H.266/VVC-VTM代码学习22-partitioner类判断划分模式是否可用函数partitioner.canSplit

前言

VTM是H.266/VVC视频编码标准的参考软件，研究VTM代码给研究人员解释了VVC编码标准的详细标准规范与细节。

本文是笔者对VTM代码的一点学习记录，成文于笔者刚开始接触VVC期间，期间很多概念和理论框架还很不成熟，若文中存在错误欢迎批评指正，也欢迎广大视频编码学习者沟通交流、共同进步。

VTM代码的下载及编译请参考博文：
【视频编码学习】H.266/VVC参考软件VTM配置运行(VTM-6.0版本)

一、函数作用

函数 xCheckModeSplit 在 xCompressCU 中被调用，函数主要作用是进行 CU 划分，并对划分后的子 CU 调用 xCompressCU 进一步 RDO 选取最优模式。

xCompressCU 函数详解请查阅 H.266/VVC-VTM代码学习20-CU层进行RDO函数xCompressCU

建立 m_ComprCUCtxList 列表的过程请查阅 H.266/VVC-VTM代码学习19-CU层确定测试模式函数initCULevel

二、函数详解

void EncCu::xCheckModeSplit(CodingStructure *&tempCS, CodingStructure *&bestCS, Partitioner &partitioner, const EncTestMode& encTestMode, const ModeType modeTypeParent, bool &skipInterPass )
{
  // 当前测试QP
  const int qp                = encTestMode.qp;
  // 当前slice
  const Slice &slice          = *tempCS->slice;
  // 之前的QP
  const int oldPrevQp         = tempCS->prevQP[partitioner.chType];
  // 之前的LUT
  const auto oldMotionLut     = tempCS->motionLut;
#if ENABLE_QPA_SUB_CTU
  // PPS
  const PPS &pps              = *tempCS->pps;
  // 当前深度
  const uint32_t currDepth    = partitioner.currDepth;
#endif
  // 之前的PLT
  const auto oldPLT           = tempCS->prevPLT;
  // 获取当前测试的划分模式
  const PartSplit split = getPartSplit( encTestMode );
  // 子结点的模式类型
  const ModeType modeTypeChild = partitioner.modeType;

  CHECK( split == CU_DONT_SPLIT, "No proper split provided!" );
  // 初始化 CS（dist清零、cost取正无穷等）
  tempCS->initStructData( qp );
  // 获取上下文
  m_CABACEstimator->getCtx() = m_CurrCtx->start;
  // 定位上下文各部分起始位置
  const TempCtx ctxStartSP( m_CtxCache, SubCtx( Ctx::SplitFlag,   m_CABACEstimator->getCtx() ) );
  const TempCtx ctxStartQt( m_CtxCache, SubCtx( Ctx::SplitQtFlag, m_CABACEstimator->getCtx() ) );
  const TempCtx ctxStartHv( m_CtxCache, SubCtx( Ctx::SplitHvFlag, m_CABACEstimator->getCtx() ) );
  const TempCtx ctxStart12( m_CtxCache, SubCtx( Ctx::Split12Flag, m_CABACEstimator->getCtx() ) );
  const TempCtx ctxStartMC( m_CtxCache, SubCtx( Ctx::ModeConsFlag, m_CABACEstimator->getCtx() ) );
  // 重置上下文比特
  m_CABACEstimator->resetBits();
  // 设置划分 CU 的模式
  m_CABACEstimator->split_cu_mode( split, *tempCS, partitioner );
  // 设置模式限制
  m_CABACEstimator->mode_constraint( split, *tempCS, partitioner, modeTypeChild );
  // 计算预测 cost 的因子在当前 QP > 30 时为 1.1，否则为 1.075
  const double factor = ( tempCS->currQP[partitioner.chType] > 30 ? 1.1 : 1.075 );
  // 是否使用 DB cost
  tempCS->useDbCost = m_pcEncCfg->getUseEncDbOpt();
  if (!tempCS->useDbCost)
    CHECK(bestCS->costDbOffset != 0, "error");
  // 计算根据上下文预测的 cost
  const double cost   = m_pcRdCost->calcRdCost( uint64_t( m_CABACEstimator->getEstFracBits() + ( ( bestCS->fracBits ) / factor ) ), Distortion( bestCS->dist / factor ) ) + bestCS->costDbOffset / factor;

  // 定位上下文各部分
  m_CABACEstimator->getCtx() = SubCtx( Ctx::SplitFlag,   ctxStartSP );
  m_CABACEstimator->getCtx() = SubCtx( Ctx::SplitQtFlag, ctxStartQt );
  m_CABACEstimator->getCtx() = SubCtx( Ctx::SplitHvFlag, ctxStartHv );
  m_CABACEstimator->getCtx() = SubCtx( Ctx::Split12Flag, ctxStart12 );
  m_CABACEstimator->getCtx() = SubCtx( Ctx::ModeConsFlag, ctxStartMC );
  // 若上下文预测的 cost > bestCS cost，则重置上下文状态且确认 bestCS
  if (cost > bestCS->cost + bestCS->costDbOffset
#if ENABLE_QPA_SUB_CTU
    || (m_pcEncCfg->getUsePerceptQPA() && !m_pcEncCfg->getUseRateCtrl() && pps.getUseDQP() && (slice.getCuQpDeltaSubdiv() > 0) && (split == CU_HORZ_SPLIT || split == CU_VERT_SPLIT) &&
        (currDepth == 0)) // force quad-split or no split at CTU level
#endif
    )
  {
    xCheckBestMode( tempCS, bestCS, partitioner, encTestMode );
    return;
  }
  // 父结点所有模式均可尝试，且子结点模式类型为 can try intra, ibc, palette，则 chromaNotSplit
  const bool chromaNotSplit = modeTypeParent == MODE_TYPE_ALL && modeTypeChild == MODE_TYPE_INTRA ? true : false;
  // 若当前划分树类型不是 TREE_D
  // 则当前 tempCS 划分树类型设置为 TREE_L
  if( partitioner.treeType != TREE_D )
  {
    tempCS->treeType = TREE_L;
  }
  // 若当前划分树类型是 TREE_D
  else
  {
    // 若色度不划分
    // 则当前 tempCS 划分树类型设置为 TREE_L
    if( chromaNotSplit )
    {
      CHECK( partitioner.chType != CHANNEL_TYPE_LUMA, "chType must be luma" );
      tempCS->treeType = partitioner.treeType = TREE_L;
    }
    // 若色度划分
    // 则当前 tempCS 划分树类型设置为 TREE_D
    else
    {
      tempCS->treeType = partitioner.treeType = TREE_D;
    }
  }

  // 按划分模式划分当前区域（深度++）
  partitioner.splitCurrArea( split, *tempCS );
  // 确认 QG possible at children level
  bool qgEnableChildren = partitioner.currQgEnable(); // QG possible at children level

  m_CurrCtx++;

  // 将 tempCS 重建像素值置0
  tempCS->getRecoBuf().fill( 0 );
  // 将 tempCS 预测像素值置0
  tempCS->getPredBuf().fill(0);
  AffineMVInfo tmpMVInfo;
  bool isAffMVInfoSaved;
  m_pcInterSearch->savePrevAffMVInfo(0, tmpMVInfo, isAffMVInfoSaved);
  BlkUniMvInfo tmpUniMvInfo;
  bool         isUniMvInfoSaved = false;
  if (!tempCS->slice->isIntra())
  {
    m_pcInterSearch->savePrevUniMvInfo(tempCS->area.Y(), tmpUniMvInfo, isUniMvInfoSaved);
  }

  // 开始遍历划分区域
  do
  {
    // 获取划分后的子 CU 区域
    const auto &subCUArea  = partitioner.currArea();

    // 必须包含在当前 picture 中
    if( tempCS->picture->Y().contains( subCUArea.lumaPos() ) )
    {
      // 获取子 CU 在宽和高方向分别对应的索引
      const unsigned wIdx    = gp_sizeIdxInfo->idxFrom( subCUArea.lwidth () );
      const unsigned hIdx    = gp_sizeIdxInfo->idxFrom( subCUArea.lheight() );

      // 子 CU 对应的 tempCS 与 bestCS
      CodingStructure *tempSubCS = m_pTempCS[wIdx][hIdx];
      CodingStructure *bestSubCS = m_pBestCS[wIdx][hIdx];

      // 子 CU 的相关变量初始化
      tempCS->initSubStructure( *tempSubCS, partitioner.chType, subCUArea, false );
      tempCS->initSubStructure( *bestSubCS, partitioner.chType, subCUArea, false );
      // 将子 CU 的 tempCS、bestCS 的 best Parent 均设置为 bestCS
      tempSubCS->bestParent = bestSubCS->bestParent = bestCS;
      // 设置子 CU 的最大允许 cost
      double newMaxCostAllowed = isLuma(partitioner.chType) ? std::min(encTestMode.maxCostAllowed, bestCS->cost - m_pcRdCost->calcRdCost(tempCS->fracBits, tempCS->dist)) : MAX_DOUBLE;
      newMaxCostAllowed = std::max(0.0, newMaxCostAllowed);
      // 调用 xCompressCU 函数处理子 CU，获取子 CU 的最优模式
      xCompressCU(tempSubCS, bestSubCS, partitioner, newMaxCostAllowed);
      // 子 CU 的 bestParent 设置为空
      tempSubCS->bestParent = bestSubCS->bestParent = nullptr;

      // 若子 CU 的最优 cost 为正无穷
      if( bestSubCS->cost == MAX_DOUBLE )
      {
        CHECK( split == CU_QUAD_SPLIT, "Split decision reusing cannot skip quad split" );
        // 将当前 cost 设置为正无穷
        tempCS->cost = MAX_DOUBLE;
        tempCS->costDbOffset = 0;
        tempCS->useDbCost = m_pcEncCfg->getUseEncDbOpt();
        // 退回上下文
        m_CurrCtx--;
        // 退出当前划分
        partitioner.exitCurrSplit();
        // 检查当前模式是否为最佳模式，若为最佳模式则与 bestCS 交换
        xCheckBestMode( tempCS, bestCS, partitioner, encTestMode );
        // 若当前通道为 LUMA
        // 则还原 LUT
        if( partitioner.chType == CHANNEL_TYPE_LUMA )
        {
          tempCS->motionLut = oldMotionLut;
        }
        // 结束划分尝试
        return;
      }
      // 是否需要保存残差信号
      bool keepResi = KEEP_PRED_AND_RESI_SIGNALS;
      // 将子cu的最优模式信息copy到tempCS中
      tempCS->useSubStructure( *bestSubCS, partitioner.chType, CS::getArea( *tempCS, subCUArea, partitioner.chType ), KEEP_PRED_AND_RESI_SIGNALS, true, keepResi, keepResi, true );

      // 若当前 QG 可用
      // 则用子 CU prevQP 更新当前 tempCS 的 prevQP
      if( partitioner.currQgEnable() )
      {
        tempCS->prevQP[partitioner.chType] = bestSubCS->prevQP[partitioner.chType];
      }
      // 若当前限制尝试 Inter
      if( partitioner.isConsInter() )
      {
        // 循环遍历所有子 CU，检查预测模式是否均为 INTER
        for( int i = 0; i < bestSubCS->cus.size(); i++ )
        {
          CHECK( bestSubCS->cus[i]->predMode != MODE_INTER, "all CUs must be inter mode in an Inter coding region (SCIPU)" );
        }
      }
      // 若当前限制尝试 INTRA
      else if( partitioner.isConsIntra() )
      {
        // 循环遍历所有子 CU，检查预测模式是否均为 INTRA
        for( int i = 0; i < bestSubCS->cus.size(); i++ )
        {
          CHECK( bestSubCS->cus[i]->predMode == MODE_INTER, "all CUs must not be inter mode in an Intra coding region (SCIPU)" );
        }
      }

      // tempSubCS和bestSubCS的清空初始化
      tempSubCS->releaseIntermediateData();
      bestSubCS->releaseIntermediateData();
      // 若当前帧不是 I 帧，且当前限制只尝试 INTRA
      if( !tempCS->slice->isIntra() && partitioner.isConsIntra() )
      {
        // 计算 tempCS cost
        tempCS->cost = m_pcRdCost->calcRdCost( tempCS->fracBits, tempCS->dist );
        // 若当前 cost > 最佳 cost
        if( tempCS->cost > bestCS->cost )
        {
          tempCS->cost = MAX_DOUBLE;
          tempCS->costDbOffset = 0;
          tempCS->useDbCost = m_pcEncCfg->getUseEncDbOpt();
          m_CurrCtx--;
          // 划分的子cu区域，在partitioner中出栈
          partitioner.exitCurrSplit();
          // 若当前通道为 LUMA
          // 则还原 LUT
          if( partitioner.chType == CHANNEL_TYPE_LUMA )
          {
            tempCS->motionLut = oldMotionLut;
          }
          return;
        }
      }
    }
  } while( partitioner.nextPart( *tempCS ) );  // 所有划分子 CU 处理完成

  // 划分的子cu区域，在partitioner中出栈
  partitioner.exitCurrSplit();


  m_CurrCtx--;
  // 若色度不划分
  if( chromaNotSplit )
  {
    //Note: In local dual tree region, the chroma CU refers to the central luma CU's QP.
    //If the luma CU QP shall be predQP (no residual in it and before it in the QG), it must be revised to predQP before encoding the chroma CU
    //Otherwise, the chroma CU uses predQP+deltaQP in encoding but is decoded as using predQP, thus causing encoder-decoded mismatch on chroma qp.
    // 若使用 DQP
    if( tempCS->pps->getUseDQP() )
    {
      //find parent CS that including all coded CUs in the QG before this node
      CodingStructure* qgCS = tempCS;
      bool deltaQpCodedBeforeThisNode = false;
      // 若当前区域不是 QG 最上层区域（即有父结点）
      if( partitioner.currArea().lumaPos() != partitioner.currQgPos )
      {
        // 当前结点到 QGCS 经过的父结点数量
        int numParentNodeToQgCS = 0;
        // 将 qgCS 定位到 parent CS that including all coded CUs in the QG before this node
        while( qgCS->area.lumaPos() != partitioner.currQgPos )
        {
          CHECK( qgCS->parent == nullptr, "parent of qgCS shall exsit" );
          qgCS = qgCS->parent;
          numParentNodeToQgCS++;
        }

        //check whether deltaQP has been coded (in luma CU or luma&chroma CU) before this node
        CodingStructure* parentCS = tempCS->parent;
        // 循环逐层遍历当前结点所有父结点
        for( int i = 0; i < numParentNodeToQgCS; i++ )
        {
          //checking each parent
          // 循环遍历当前父结点的所有 CU，检查 deltaQP 在当前结点前是否已经编码
          CHECK( parentCS == nullptr, "parentCS shall exsit" );
          for( const auto &cu : parentCS->cus )
          {
            if( cu->rootCbf && !isChroma( cu->chType ) )
            {
              deltaQpCodedBeforeThisNode = true;
              break;
            }
          }
          parentCS = parentCS->parent;
        }
      }

      //revise luma CU qp before the first luma CU with residual in the SCIPU to predQP
      // 若 deltaQP 在当前结点前未编码
      if( !deltaQpCodedBeforeThisNode )
      {
        //get pred QP of the QG
        // 定位当前 QG 的第一个 CU
        const CodingUnit* cuFirst = qgCS->getCU( CHANNEL_TYPE_LUMA );
        CHECK( cuFirst->lumaPos() != partitioner.currQgPos, "First cu of the Qg is wrong" );
        // 获取当前 QG 的第一个 CU 的预测 QP
        int predQp = CU::predictQP( *cuFirst, qgCS->prevQP[CHANNEL_TYPE_LUMA] );

        //revise to predQP
        int firstCuHasResidual = (int)tempCS->cus.size();
        // 循环定位当前块下的 firstCuHasResidual
        for( int i = 0; i < tempCS->cus.size(); i++ )
        {
          if( tempCS->cus[i]->rootCbf )
          {
            firstCuHasResidual = i;
            break;
          }
        }
        // 将当前块 firstCuHasResidual 前的所有 CU QP 设为预测 QP
        for( int i = 0; i < firstCuHasResidual; i++ )
        {
          tempCS->cus[i]->qp = predQp;
        }
      }
    }  // use DQP
    assert( tempCS->treeType == TREE_L );
    uint32_t numCuPuTu[6];
    tempCS->picture->cs->getNumCuPuTuOffset( numCuPuTu );
    tempCS->picture->cs->useSubStructure( *tempCS, partitioner.chType, CS::getArea( *tempCS, partitioner.currArea(), partitioner.chType ), false, true, false, false, false );

    // 若 chroma 可用，对 chroma 进行处理
    if (isChromaEnabled(tempCS->pcv->chrFormat))
    {
    partitioner.chType = CHANNEL_TYPE_CHROMA;
    tempCS->treeType = partitioner.treeType = TREE_C;

    m_CurrCtx++;

    const unsigned wIdx = gp_sizeIdxInfo->idxFrom( partitioner.currArea().lwidth() );
    const unsigned hIdx = gp_sizeIdxInfo->idxFrom( partitioner.currArea().lheight() );
    CodingStructure *tempCSChroma = m_pTempCS2[wIdx][hIdx];
    CodingStructure *bestCSChroma = m_pBestCS2[wIdx][hIdx];
    tempCS->initSubStructure( *tempCSChroma, partitioner.chType, partitioner.currArea(), false );
    tempCS->initSubStructure( *bestCSChroma, partitioner.chType, partitioner.currArea(), false );
    tempCS->treeType = TREE_D;
    xCompressCU( tempCSChroma, bestCSChroma, partitioner );

    //attach chromaCS to luma CS and update cost
    bool keepResi = KEEP_PRED_AND_RESI_SIGNALS;
    //bestCSChroma->treeType = tempCSChroma->treeType = TREE_C;
    CHECK( bestCSChroma->treeType != TREE_C || tempCSChroma->treeType != TREE_C, "wrong treeType for chroma CS" );
    tempCS->useSubStructure( *bestCSChroma, partitioner.chType, CS::getArea( *bestCSChroma, partitioner.currArea(), partitioner.chType ), KEEP_PRED_AND_RESI_SIGNALS, true, keepResi, true, true );

    //release tmp resource
    tempCSChroma->releaseIntermediateData();
    bestCSChroma->releaseIntermediateData();
    //tempCS->picture->cs->releaseIntermediateData();
      m_CurrCtx--;
    }
    tempCS->picture->cs->clearCuPuTuIdxMap( partitioner.currArea(), numCuPuTu[0], numCuPuTu[1], numCuPuTu[2], numCuPuTu + 3 );


    //recover luma tree status
    partitioner.chType = CHANNEL_TYPE_LUMA;
    partitioner.treeType = TREE_D;
    partitioner.modeType = MODE_TYPE_ALL;
  }

  // Finally, generate split-signaling bits for RD-cost check
  // 最后，生成用于 RD cost check 的划分信号比特
  const PartSplit implicitSplit = partitioner.getImplicitSplit( *tempCS );

  {
    bool enforceQT = implicitSplit == CU_QUAD_SPLIT;

    // LARGE CTU bug
    if( m_pcEncCfg->getUseFastLCTU() )
    {
      unsigned minDepth = 0;
      unsigned maxDepth = floorLog2(tempCS->sps->getCTUSize()) - floorLog2(tempCS->sps->getMinQTSize(slice.getSliceType(), partitioner.chType));

      if( auto ad = dynamic_cast<AdaptiveDepthPartitioner*>( &partitioner ) )
      {
        ad->setMaxMinDepth( minDepth, maxDepth, *tempCS );
      }

      if( minDepth > partitioner.currQtDepth )
      {
        // enforce QT
        enforceQT = true;
      }
    }

    if( !enforceQT )
    {
      m_CABACEstimator->resetBits();

      m_CABACEstimator->split_cu_mode( split, *tempCS, partitioner );
      partitioner.modeType = modeTypeParent;
      m_CABACEstimator->mode_constraint( split, *tempCS, partitioner, modeTypeChild );
      tempCS->fracBits += m_CABACEstimator->getEstFracBits(); // split bits
    }
  }

  tempCS->cost = m_pcRdCost->calcRdCost( tempCS->fracBits, tempCS->dist );

  // Check Delta QP bits for splitted structure
  if( !qgEnableChildren ) // check at deepest QG level only
  xCheckDQP( *tempCS, partitioner, true );

  // If the configuration being tested exceeds the maximum number of bytes for a slice / slice-segment, then
  // a proper RD evaluation cannot be performed. Therefore, termination of the
  // slice/slice-segment must be made prior to this CTU.
  // This can be achieved by forcing the decision to be that of the rpcTempCU.
  // The exception is each slice / slice-segment must have at least one CTU.
  if (bestCS->cost != MAX_DOUBLE)
  {
  }
  else
  {
    bestCS->costDbOffset = 0;
  }
  tempCS->useDbCost = m_pcEncCfg->getUseEncDbOpt();
  if( tempCS->cus.size() > 0 && modeTypeParent == MODE_TYPE_ALL && modeTypeChild == MODE_TYPE_INTER )
  {
    int areaSizeNoResiCu = 0;
    for( int k = 0; k < tempCS->cus.size(); k++ )
    {
      areaSizeNoResiCu += (tempCS->cus[k]->rootCbf == false) ? tempCS->cus[k]->lumaSize().area() : 0;
    }
    if( areaSizeNoResiCu >= (tempCS->area.lumaSize().area() >> 1) )
    {
      skipInterPass = true;
    }
  }

  // RD check for sub partitioned coding structure.
  xCheckBestMode( tempCS, bestCS, partitioner, encTestMode );

  if (isAffMVInfoSaved)
    m_pcInterSearch->addAffMVInfo(tmpMVInfo);
  if (!tempCS->slice->isIntra() && isUniMvInfoSaved)
  {
    m_pcInterSearch->addUniMvInfo(tmpUniMvInfo);
  }

  tempCS->motionLut = oldMotionLut;

  tempCS->prevPLT   = oldPLT;

  tempCS->releaseIntermediateData();

  tempCS->prevQP[partitioner.chType] = oldPrevQp;
}

---

上一篇：H.266/VVC-VTM代码学习20-CU层进行RDO函数xCompressCU
下一篇：H.266/VVC-VTM代码学习22-partitioner类判断划分模式是否可用函数partitioner.canSplit