H.266/VVC代码学习10:xRecurIntraCodingLumaQT函数

1 VTM4.0版本

初始化后分为两种情况:1.可以继续划分,2.不可继续划分
1.可以继续划分时,根据cost的大小进行TU的划分(递归调用多次划分),设置cbf,获取上下文模型并保存代价
2.不可继续划分时,设置变换的模式(6种,决定是否开启及开启几个),对可以进行变换的模式进行后续残差计算、变换量化和反变换重建等操作,具体函数为xIntraCodingTUBlock(DCT2与其余5种不同),更新率失真信息。
可点击查看详情:H.266/VVC代码学习9:xIntraCodingTUBlock函数
最终获取缓存,确定代价。
具体代码和部分注释如下:

#if JVET_M0102_INTRA_SUBPARTITIONS
void IntraSearch::xRecurIntraCodingLumaQT( CodingStructure &cs, Partitioner &partitioner, const double bestCostSoFar, const int subTuIdx, const PartSplit ispType )
{
	/*************************** 初始化 **********************/
        int   subTuCounter = subTuIdx;
  const UnitArea &currArea = partitioner.currArea();
  const CodingUnit     &cu = *cs.getCU( currArea.lumaPos(), partitioner.chType );
        bool  earlySkipISP = false;
#else
void IntraSearch::xRecurIntraCodingLumaQT( CodingStructure &cs, Partitioner &partitioner )
{
  const UnitArea &currArea = partitioner.currArea();
#if !JVET_M0464_UNI_MTS
  const CodingUnit &cu     = *cs.getCU(currArea.lumaPos(), partitioner.chType);
#endif
#endif
  uint32_t currDepth       = partitioner.currTrDepth;
  const PPS &pps           = *cs.pps;
  const bool keepResi      = pps.getPpsRangeExtension().getCrossComponentPredictionEnabledFlag() || KEEP_PRED_AND_RESI_SIGNALS;
  bool bCheckFull          = true;	// 不能继续划分:当前PU可进行变换和量化
  bool bCheckSplit         = false;	// 是否进行分割:当前PU应继续划分
  bCheckFull               = !partitioner.canSplit( TU_MAX_TR_SPLIT, cs );
  bCheckSplit              = partitioner.canSplit( TU_MAX_TR_SPLIT, cs );

#if JVET_M0102_INTRA_SUBPARTITIONS
  if( cu.ispMode )
  {
    bCheckSplit = partitioner.canSplit( ispType, cs );
    bCheckFull = !bCheckSplit;
  }
#endif
  uint32_t    numSig           = 0;

#if JVET_M0464_UNI_MTS
  double     dSingleCost                        = MAX_DOUBLE;
  Distortion uiSingleDistLuma                   = 0;
  uint64_t   singleFracBits                     = 0;
  int        bestModeId[MAX_NUM_COMPONENT]      = { 0, 0, 0 };
#else
  bool    checkInitTrDepth = false, checkInitTrDepthTransformSkipWinner = false;

  double     dSingleCost                        = MAX_DOUBLE;
  Distortion uiSingleDistLuma                   = 0;
  uint64_t     singleFracBits                     = 0;
  bool       checkTransformSkip                 = pps.getUseTransformSkip();
  int        bestModeId[MAX_NUM_COMPONENT]      = {0, 0, 0};
  uint8_t      nNumTransformCands                 = cu.emtFlag ? 4 : 1; //4 is the number of transforms of emt
  bool       isAllIntra                         = m_pcEncCfg->getIntraPeriod() == 1;

  uint8_t numTransformIndexCands                  = nNumTransformCands;
#endif
	//处理上下文
  const TempCtx ctxStart  ( m_CtxCache, m_CABACEstimator->getCtx() );
  TempCtx       ctxBest   ( m_CtxCache );

  /************************** 创建进行操作的cs ********************/
  CodingStructure *csSplit = nullptr;
  CodingStructure *csFull  = nullptr;

  if( bCheckSplit )
  {
    csSplit = &cs;
  }
  else if( bCheckFull )
  {
    csFull = &cs;
  }

  /*************************** 如果PU划分完成的操作 ************************/
  if( bCheckFull )
  {
    csFull->cost = 0.0;

    TransformUnit &tu = csFull->addTU( CS::getArea( *csFull, currArea, partitioner.chType ), partitioner.chType );
    tu.depth = currDepth;

#if JVET_M0464_UNI_MTS
    const bool tsAllowed  = TU::isTSAllowed ( tu, COMPONENT_Y );//TS:变换跳过的模式,能否进行
    const bool mtsAllowed = TU::isMTSAllowed( tu, COMPONENT_Y );//MTS:选变换核的模式,能否进行
    uint8_t nNumTransformCands = 1 + ( tsAllowed ? 1 : 0 ) + ( mtsAllowed ? 4 : 0 ); // DCT + TS + 4 MTS = 6 tests
    std::vector<TrMode> trModes;
    trModes.push_back( TrMode( 0, true ) ); //0:DCT2直接加入
    if( tsAllowed )//1:如果可以进行TS,则加入
    {
      trModes.push_back( TrMode( 1, true ) );
    }
    if( mtsAllowed )//2~5:如果可以进行MTS,则依次加入
    {
      for( int i = 2; i < 6; i++ )
      {
        trModes.push_back( TrMode( i, true) );
      }
    }

    CHECK( !tu.Y().valid(), "Invalid TU" );
#else
    checkTransformSkip &= TU::hasTransformSkipFlag( *tu.cs, tu.Y() );
    checkTransformSkip &= !cu.transQuantBypass;
    checkTransformSkip &= !cu.emtFlag;
#if JVET_M0102_INTRA_SUBPARTITIONS
    checkTransformSkip &= !cu.ispMode;
#endif

    CHECK( !tu.Y().valid(), "Invalid TU" );

    //this prevents transformSkip from being checked because we already know it's not the best mode
    checkTransformSkip = ( checkInitTrDepth && !checkInitTrDepthTransformSkipWinner ) ? false : checkTransformSkip;


    CHECK( checkInitTrDepthTransformSkipWinner && !checkTransformSkip, "Transform Skip must be enabled if it was the winner in the previous call of xRecurIntraCodingLumaQT!" );
#endif

    CodingStructure &saveCS = *m_pSaveCS[0];

    TransformUnit *tmpTU = nullptr;

    Distortion singleDistTmpLuma = 0;
    uint64_t     singleTmpFracBits = 0;
    double     singleCostTmp     = 0;
    int        firstCheckId      = 0;//第一个模式的标号

#if JVET_M0464_UNI_MTS
    int       lastCheckId        = trModes[nNumTransformCands-1].first;//最后一个模式的标号
    bool isNotOnlyOneMode        = nNumTransformCands != 1;//变换模式不止一种
#else
    //we add the EMT candidates to the loop. TransformSkip will still be the last one to be checked (when modeId == lastCheckId) as long as checkTransformSkip is true
    int        lastCheckId       = numTransformIndexCands - ( firstCheckId + 1 ) + ( int ) checkTransformSkip;
    bool isNotOnlyOneMode        = lastCheckId != firstCheckId && !checkInitTrDepthTransformSkipWinner;
#endif

    if( isNotOnlyOneMode )//如除了DCT2还有其他模式的变换时,创建临时cs以保留数据
    {
      saveCS.pcv     = cs.pcv;
      saveCS.picture = cs.picture;
      saveCS.area.repositionTo(cs.area);
      saveCS.clearTUs();
      tmpTU = &saveCS.addTU(currArea, partitioner.chType);
    }

#if JVET_M0464_UNI_MTS
    bool    cbfDCT2  = true;
#else
    bool cbfBestMode = false;
#endif

#if JVET_M0464_UNI_MTS
    for( int modeId = firstCheckId; modeId < nNumTransformCands; modeId++ )//遍历变换模式
    {
      if( !cbfDCT2 || ( m_pcEncCfg->getUseTransformSkipFast() && bestModeId[COMPONENT_Y] == 1 ) )
      {
        break;
      }
      if( !trModes[modeId].second )//如果变换模式不开启,则跳过
      {
        continue;
      }
      tu.mtsIdx = trModes[modeId].first;//mtsIdx,变换方式的序号
#else
    for( int modeId = firstCheckId; modeId <= lastCheckId; modeId++ )
    {
      if( checkInitTrDepthTransformSkipWinner )
      {
        //If this is a full RQT call and the winner of the first call (checkFirst=true) was transformSkip, then we skip the first iteration of the loop, since transform skip always comes at the end
        if( modeId == firstCheckId )
        {
          continue;
        }
      }

      uint8_t transformIndex = modeId;


      if( ( transformIndex < lastCheckId ) || ( ( transformIndex == lastCheckId ) && !checkTransformSkip ) ) //we avoid this if the mode is transformSkip
      {
        // Skip checking other transform candidates if zero CBF is encountered and it is the best transform so far
        if( m_pcEncCfg->getFastIntraEMT() && isAllIntra && transformIndex && !cbfBestMode )
        {
          continue;
        }
      }
#endif

	  //如果不是第一种模式,并且不只有一种模式,则获取上下文
      if ((modeId != firstCheckId) && isNotOnlyOneMode)
      {
        m_CABACEstimator->getCtx() = ctxStart;
      }

      int default0Save1Load2 = 0;//模式重复使用标志
      singleDistTmpLuma = 0;

	  //如果是第一种模式,且有多种可用模式
#if JVET_M0464_UNI_MTS
      if( modeId == firstCheckId && nNumTransformCands > 1 )
#else
      if (modeId == firstCheckId && modeId != lastCheckId && !checkInitTrDepthTransformSkipWinner )
#endif
      {
        default0Save1Load2 = 1;
      }
	  //如果不是第一种模式(这时当然有多种可用模式)
      else if (modeId != firstCheckId)
      {
        default0Save1Load2 = 2;
      }
#if JVET_M0102_INTRA_SUBPARTITIONS
      if( cu.ispMode )
      {
        default0Save1Load2 = 0;
      }
#endif
	  /******** 进行残差变换重建等操作 ********/
#if JVET_M0464_UNI_MTS
      if( nNumTransformCands > 1 )//如果不止一种模式
      {
        xIntraCodingTUBlock( tu, COMPONENT_Y, false, singleDistTmpLuma, default0Save1Load2, &numSig, modeId == 0 ? &trModes : nullptr, true );
        if( modeId == 0 )//把可用模式的标号记下,并用lastCheckId存可用模式的数量
        {
          for( int i = 0; i < nNumTransformCands; i++ )
          {
            if( trModes[i].second )
            {
              lastCheckId = trModes[i].first;
            }
          }
        }
      }
      else//如果只有一种模式,即如果只做DCT变换,直接简单的变换即可
      {
        xIntraCodingTUBlock( tu, COMPONENT_Y, false, singleDistTmpLuma, default0Save1Load2, &numSig );
      }
#else
      if (cu.emtFlag)
      {
        tu.emtIdx = transformIndex;
      }
      if( !checkTransformSkip )
      {
        tu.transformSkip[COMPONENT_Y] = false;
      }
      else
      {
        tu.transformSkip[COMPONENT_Y] = modeId == lastCheckId;
      }

      xIntraCodingTUBlock( tu, COMPONENT_Y, false, singleDistTmpLuma, default0Save1Load2, &numSig );
#endif

      //----- determine rate and r-d cost 存储率失真信息 -----
#if JVET_M0464_UNI_MTS
      if( ( trModes[modeId].first != 0 && !TU::getCbfAtDepth( tu, COMPONENT_Y, currDepth ) ) )
#else
      //the condition (transformIndex != DCT2_EMT) seems to be irrelevant, since DCT2_EMT=7 and the highest value of transformIndex is 4
      if( ( modeId == lastCheckId && checkTransformSkip && !TU::getCbfAtDepth( tu, COMPONENT_Y, currDepth ) ) )
#endif
      {//为了在cbf为零时不对TS标志进行编码,禁止cbf为零的TS的情况。
        //In order not to code TS flag when cbf is zero, the case for TS with cbf being zero is forbidden.
        singleCostTmp = MAX_DOUBLE;
      }
      else//获取亮度比特值和代价信息
      {
#if JVET_M0102_INTRA_SUBPARTITIONS
        if( cu.ispMode && m_pcRdCost->calcRdCost( csFull->fracBits, csFull->dist + singleDistTmpLuma ) > bestCostSoFar )
        {
          earlySkipISP = true;
        }
        else
        {
          singleTmpFracBits = xGetIntraFracBitsQT( *csFull, partitioner, true, false, subTuCounter, ispType );
        }
#else
        singleTmpFracBits = xGetIntraFracBitsQT( *csFull, partitioner, true, false );
#endif
        singleCostTmp     = m_pcRdCost->calcRdCost( singleTmpFracBits, singleDistTmpLuma );
      }

	  /********** 更新最优率失真信息 ***********/
      if (singleCostTmp < dSingleCost)
      {
        dSingleCost       = singleCostTmp;
        uiSingleDistLuma  = singleDistTmpLuma;
        singleFracBits    = singleTmpFracBits;

#if JVET_M0464_UNI_MTS
        bestModeId[COMPONENT_Y] = trModes[modeId].first;//最佳变换模式
        if( trModes[modeId].first == 0 )//如果是DCT2最优,则获取cbf
        {
          cbfDCT2  = TU::getCbfAtDepth( tu, COMPONENT_Y, currDepth );
        }
#else
        bestModeId[COMPONENT_Y] = modeId;
        cbfBestMode       = TU::getCbfAtDepth( tu, COMPONENT_Y, currDepth );
#endif

        if( bestModeId[COMPONENT_Y] != lastCheckId )//如果还没循环结束,临时保存一下各种信息
        {
#if KEEP_PRED_AND_RESI_SIGNALS || JVET_M0427_INLOOP_RESHAPER
          saveCS.getPredBuf( tu.Y() ).copyFrom( csFull->getPredBuf( tu.Y() ) );
#endif
          saveCS.getRecoBuf( tu.Y() ).copyFrom( csFull->getRecoBuf( tu.Y() ) );

          if( keepResi )
          {
            saveCS.getResiBuf   ( tu.Y() ).copyFrom( csFull->getResiBuf   ( tu.Y() ) );
            saveCS.getOrgResiBuf( tu.Y() ).copyFrom( csFull->getOrgResiBuf( tu.Y() ) );
          }

          tmpTU->copyComponentFrom( tu, COMPONENT_Y );

          ctxBest = m_CABACEstimator->getCtx();
        }
      }
    }

    if( bestModeId[COMPONENT_Y] != lastCheckId )//如果还没循环结束,最终保存一下各种信息
    {
#if KEEP_PRED_AND_RESI_SIGNALS || JVET_M0427_INLOOP_RESHAPER
      csFull->getPredBuf( tu.Y() ).copyFrom( saveCS.getPredBuf( tu.Y() ) );
#endif
      csFull->getRecoBuf( tu.Y() ).copyFrom( saveCS.getRecoBuf( tu.Y() ) );

      if( keepResi )
      {
        csFull->getResiBuf   ( tu.Y() ).copyFrom( saveCS.getResiBuf   ( tu.Y() ) );
        csFull->getOrgResiBuf( tu.Y() ).copyFrom( saveCS.getOrgResiBuf( tu.Y() ) );
      }

      tu.copyComponentFrom( *tmpTU, COMPONENT_Y );

      if( !bCheckSplit )
      {
        m_CABACEstimator->getCtx() = ctxBest;
      }
    }
    else if( bCheckSplit )
    {
      ctxBest = m_CABACEstimator->getCtx();
    }
	/******** 更新率失真信息 *****/
    csFull->cost     += dSingleCost;
    csFull->dist     += uiSingleDistLuma;
    csFull->fracBits += singleFracBits;
#if LUMA_AND_CHROMA_BITS
	if (csFull->chType == 0) csFull->lumaBits += singleFracBits;
	if (csFull->chType == 1) csFull->chromaBits += singleFracBits;
#endif 
  }

  /*************************** 如果PU划分未完的操作 ************************/
  if( bCheckSplit )
  {
    //----- store full entropy coding status, load original entropy coding status 存储全熵编码状态,加载原始熵编码状态 -----
    if( bCheckFull )
    {
      m_CABACEstimator->getCtx() = ctxStart;
    }
    //----- code splitted block 分割编码 -----
    csSplit->cost = 0;

    bool uiSplitCbfLuma  = false;
    bool splitIsSelected = true;
    if( partitioner.canSplit( TU_MAX_TR_SPLIT, cs ) )
    {
      partitioner.splitCurrArea( TU_MAX_TR_SPLIT, cs );
    }

#if JVET_M0102_INTRA_SUBPARTITIONS
    if( cu.ispMode )
    {
      partitioner.splitCurrArea( ispType, *csSplit );
    }
#endif
    do
    {
#if JVET_M0102_INTRA_SUBPARTITIONS
      xRecurIntraCodingLumaQT( *csSplit, partitioner, bestCostSoFar, subTuCounter, ispType );//递归调用,继续划分
      subTuCounter += subTuCounter != -1 ? 1 : 0;//划分了几次
#else
      xRecurIntraCodingLumaQT( *csSplit, partitioner );
#endif

#if JVET_M0102_INTRA_SUBPARTITIONS
      if( !cu.ispMode )
      {
        csSplit->setDecomp( partitioner.currArea().Y() );
      }
      else if( CU::isISPFirst( cu, partitioner.currArea().Y(), COMPONENT_Y ) )
      {
        csSplit->setDecomp( cu.Y() );
      }
#else
      csSplit->setDecomp( partitioner.currArea().Y() );
#endif

#if JVET_M0102_INTRA_SUBPARTITIONS
      uiSplitCbfLuma |= TU::getCbfAtDepth( *csSplit->getTU( partitioner.currArea().lumaPos(), partitioner.chType, subTuCounter - 1 ), COMPONENT_Y, partitioner.currTrDepth );

      if( cu.ispMode )//splitIsSelected是默认为true的,表示要划分到什么成都。以下是令splitIsSelected = false的原因
      { //如果累计成本已经大于目前的最佳成本,则跳出循环条件(对RD性能没有影响)
        //exit condition if the accumulated cost is already larger than the best cost so far (no impact in RD performance)
        if( csSplit->cost > bestCostSoFar )//如果大于ISP临时的代价
        {
          earlySkipISP    = true;
          splitIsSelected = false;
          break;
        }
        else
        {
          //more restrictive exit condition 更严格的跳出循环条件
          bool tuIsDividedInRows = CU::divideTuInRows( cu );
          int nSubPartitions = tuIsDividedInRows ? cu.lheight() >> g_aucLog2[cu.firstTU->lheight()] : cu.lwidth() >> g_aucLog2[cu.firstTU->lwidth()];//短边
          double threshold = nSubPartitions == 2 ? 0.95 : subTuCounter == 1 ? 0.83 : 0.91;//阈值
          if( subTuCounter < nSubPartitions && csSplit->cost > bestCostSoFar*threshold )//如果大于ISP临时的代价*阈值
          {
            earlySkipISP    = true;
            splitIsSelected = false;
            break;
          }
        }
      }
#else
      uiSplitCbfLuma |= TU::getCbfAtDepth( *csSplit->getTU( partitioner.currArea().lumaPos(), partitioner.chType ), COMPONENT_Y, partitioner.currTrDepth );
#endif



    } while( partitioner.nextPart( *csSplit ) );//划分完成

    partitioner.exitCurrSplit();

    if( splitIsSelected )//splitIsSelected = true时(划分到底了)
    {
      for( auto &ptu : csSplit->tus )//将每个tu设置cbf
      {
        if( currArea.Y().contains( ptu->Y() ) )
        {
          TU::setCbfAtDepth( *ptu, COMPONENT_Y, currDepth, uiSplitCbfLuma ? 1 : 0 );
        }
      }

      //----- restore context states 恢复上下文 -----
      m_CABACEstimator->getCtx() = ctxStart;

      //----- determine rate and r-d cost 确定比特率和RDcost -----
#if JVET_M0102_INTRA_SUBPARTITIONS
      csSplit->fracBits = xGetIntraFracBitsQT( *csSplit, partitioner, true, false, cu.ispMode ? 0 : -1, ispType );
#if LUMA_AND_CHROMA_BITS
	  if (csSplit->chType == 0) csSplit->fracBits = xGetIntraFracBitsQT(*csSplit, partitioner, true, false, cu.ispMode ? 0 : -1, ispType);
	  if (csSplit->chType == 1) csSplit->fracBits = xGetIntraFracBitsQT(*csSplit, partitioner, true, false, cu.ispMode ? 0 : -1, ispType);
#endif 
#else
      csSplit->fracBits = xGetIntraFracBitsQT(*csSplit, partitioner, true, false);
#endif

      //--- update cost 更新代价 ---
      csSplit->cost     = m_pcRdCost->calcRdCost(csSplit->fracBits, csSplit->dist);
    }
  }

  if( csFull || csSplit )//获取缓存,确定代价
  {
    {
      // otherwise this would've happened in useSubStructure 否则这将发生在useSubStructure中
      cs.picture->getRecoBuf( currArea.Y() ).copyFrom( cs.getRecoBuf( currArea.Y() ) );
#if JVET_M0427_INLOOP_RESHAPER
      cs.picture->getPredBuf( currArea.Y() ).copyFrom( cs.getPredBuf( currArea.Y() ) );
#endif
    }

#if JVET_M0102_INTRA_SUBPARTITIONS
    if( cu.ispMode && earlySkipISP )
    {
      cs.cost = MAX_DOUBLE;
    }
    else
    {
      cs.cost = m_pcRdCost->calcRdCost( cs.fracBits, cs.dist );
    }
#else
    cs.cost = m_pcRdCost->calcRdCost( cs.fracBits, cs.dist );
#endif
  }
}

2 VTM7.0版本(20191128更新)

和4.0版本基本相同,具体步骤在代码中。

bool IntraSearch::xRecurIntraCodingLumaQT( CodingStructure &cs, Partitioner &partitioner, const double bestCostSoFar, const int subTuIdx, const PartSplit ispType, const bool ispIsCurrentWinner, bool mtsCheckRangeFlag, int mtsFirstCheckId, int mtsLastCheckId, bool moreProbMTSIdxFirst )
{
	/*********************************************************** 1.初始化 ********************************************************/
        int   subTuCounter = subTuIdx;
  const UnitArea &currArea = partitioner.currArea();
  const CodingUnit     &cu = *cs.getCU( currArea.lumaPos(), partitioner.chType );
        bool  earlySkipISP = false;//属于ISP模式时提前跳出
  uint32_t currDepth       = partitioner.currTrDepth;
  const SPS &sps           = *cs.sps;
  const PPS &pps           = *cs.pps;
  const bool keepResi      = pps.getPpsRangeExtension().getCrossComponentPredictionEnabledFlag() || KEEP_PRED_AND_RESI_SIGNALS;
  bool bCheckFull          = true;
  bool bCheckSplit         = false;
  bCheckFull               = !partitioner.canSplit( TU_MAX_TR_SPLIT, cs );//如果不可以继续划分为TU
  bCheckSplit              = partitioner.canSplit( TU_MAX_TR_SPLIT, cs );//如果可继续划分为TU

  if( cu.ispMode )
  {
    bCheckSplit = partitioner.canSplit( ispType, cs );//如果不可以继续划分为TU
    bCheckFull = !bCheckSplit;//如果可继续划分为TU
  }
  uint32_t    numSig           = 0;

  double     dSingleCost                        = MAX_DOUBLE;
  Distortion uiSingleDistLuma                   = 0;
  uint64_t   singleFracBits                     = 0;
  bool       checkTransformSkip                 = sps.getTransformSkipEnabledFlag();
  int        bestModeId[ MAX_NUM_COMPONENT ]    = { 0, 0, 0 };
  uint8_t    nNumTransformCands                 = cu.mtsFlag ? 4 : 1;
  uint8_t    numTransformIndexCands             = nNumTransformCands;

  const TempCtx ctxStart  ( m_CtxCache, m_CABACEstimator->getCtx() );
  TempCtx       ctxBest   ( m_CtxCache );

  CodingStructure *csSplit = nullptr;
  CodingStructure *csFull  = nullptr;

#if JVET_P1026_ISP_LFNST_COMBINATION
  CUCtx cuCtx;
  cuCtx.isDQPCoded = true;
  cuCtx.isChromaQpAdjCoded = true;
#endif

  if( bCheckSplit )
  {
    csSplit = &cs;
  }
  else if( bCheckFull )
  {
    csFull = &cs;
  }

  /*********************************************************** 2.不继续划分为TU的变换 ******************************************************/
  bool validReturnFull = false;
  if( bCheckFull )
  {
	/***************** 2.1 初始化要遍历的变换模式列表 ******************/
    csFull->cost = 0.0;//初始化代价

    TransformUnit &tu = csFull->addTU( CS::getArea( *csFull, currArea, partitioner.chType ), partitioner.chType );
    tu.depth = currDepth;

    const bool tsAllowed  = TU::isTSAllowed( tu, COMPONENT_Y );//是否允许TS
#if JVET_P1026_MTS_SIGNALLING
    const bool mtsAllowed = CU::isMTSAllowed( cu, COMPONENT_Y );//是否允许MTS
#else
    const bool mtsAllowed = TU::isMTSAllowed( tu, COMPONENT_Y );
#endif
    std::vector<TrMode> trModes;//创建要经过的变换模式

	/* 如果是二次变换,只测试DCT2和TS */
    if( sps.getUseLFNST() )
    {
      checkTransformSkip &= tsAllowed;
      checkTransformSkip &= !cu.mtsFlag;
      checkTransformSkip &= !cu.lfnstIdx;

      if( !cu.mtsFlag && checkTransformSkip )
      {
        trModes.push_back( TrMode( 0, true ) ); //DCT2
        trModes.push_back( TrMode( 1, true ) ); //TS
      }
    }
	/* 如果不是二次变换,测试DCT2和TS和4种MTS */
    else
    {
      nNumTransformCands = 1 + ( tsAllowed ? 1 : 0 ) + ( mtsAllowed ? 4 : 0 ); // DCT + TS + 4 MTS = 6 tests

      trModes.push_back( TrMode( 0, true ) ); //DCT2
      if( tsAllowed )
      {
        trModes.push_back( TrMode( 1, true ) );//TS
      }
      if( mtsAllowed )
      {
        for( int i = 2; i < 6; i++ )
        {
          trModes.push_back( TrMode( i, true ) );//MTS
        }
      }
    }

    CHECK( !tu.Y().valid(), "Invalid TU" );

	/************************ 2.2 进行各种模式的变换并比较 ************************/

	/*******************  2.2.1 初始化 ********************/
    CodingStructure &saveCS = *m_pSaveCS[0];

    TransformUnit *tmpTU = nullptr;

    Distortion singleDistTmpLuma = 0;//初始化当前部分的失真D
    uint64_t     singleTmpFracBits = 0;//初始化当前部分的比特数R
    double     singleCostTmp     = 0;//初始化临时代价

    int        firstCheckId      = ( sps.getUseLFNST() && mtsCheckRangeFlag && cu.mtsFlag ) ? mtsFirstCheckId : 0;//第一个变换模式序号

    //we add the MTS candidates to the loop. TransformSkip will still be the last one to be checked (when modeId == lastCheckId) as long as checkTransformSkip is true
    int        lastCheckId       = sps.getUseLFNST() ? ( ( mtsCheckRangeFlag && cu.mtsFlag ) ? ( mtsLastCheckId + ( int ) checkTransformSkip ) : 
									( numTransformIndexCands - ( firstCheckId + 1 ) + ( int ) checkTransformSkip ) ) :
                                   trModes[ nNumTransformCands - 1 ].first;		//最后一个变换模式序号
    bool isNotOnlyOneMode        = sps.getUseLFNST() ? lastCheckId != firstCheckId : nNumTransformCands != 1;

    if( isNotOnlyOneMode )//如果只有一种模式,直接存入这一模式
    {
      saveCS.pcv     = cs.pcv;
      saveCS.picture = cs.picture;
      saveCS.area.repositionTo(cs.area);
      saveCS.clearTUs();
      tmpTU = &saveCS.addTU(currArea, partitioner.chType);
    }

    bool    cbfBestMode      = false;
    bool    cbfBestModeValid = false;
    bool    cbfDCT2  = true;

    double bestDCT2cost = MAX_DOUBLE;
    double threshold = m_pcEncCfg->getUseFastISP() && !cu.ispMode && ispIsCurrentWinner && nNumTransformCands > 1 ? 1 + 1.4 / sqrt( cu.lwidth() * cu.lheight() ) : 1;//设一个阈值,可能是快速算法?

	/************  2.2.2 遍历变换模式进行变换操作 ***********/
    for( int modeId = firstCheckId; modeId <= ( sps.getUseLFNST() ? lastCheckId : ( nNumTransformCands - 1 ) ); modeId++ )
    {
      uint8_t transformIndex = modeId;

	  /******** 2.2.2.1 提前跳出本次循环条件 ********/

	  /** 2.2.2.1.1 如果可以进行二次变换 **/
      if( sps.getUseLFNST() )//如果可以使用二次变换但不是DCT2,需要跳过
      {
        if( ( transformIndex < lastCheckId ) || ( ( transformIndex == lastCheckId ) && !checkTransformSkip ) ) //we avoid this if the mode is transformSkip
        {//如果遇到零CBF,则跳过检查其他变换候选对象,这是迄今为止最好的变换
          // Skip checking other transform candidates if zero CBF is encountered and it is the best transform so far
          if( m_pcEncCfg->getUseFastLFNST() && transformIndex && !cbfBestMode && cbfBestModeValid )
          {
            continue;//如果进行二次变换但不是DCT2-DCT2,并且不是最佳模式
          }
        }
      }
	  /* 2.2.2.1.2 如果不进行二次变换 */
      else//TS跳过,当前模式不可用则跳过等
      {
#if JVET_AHG14_LOSSLESS
        if( !( m_pcEncCfg->getCostMode() == COST_LOSSLESS_CODING ) )//如果有残差才看
        {
#endif
#if JVET_P0058_CHROMA_TS
        if( !cbfDCT2 || ( m_pcEncCfg->getUseTransformSkipFast() && bestModeId[ COMPONENT_Y ] == MTS_SKIP))
#else
        if( !cbfDCT2 || ( m_pcEncCfg->getUseTransformSkipFast() && bestModeId[ COMPONENT_Y ] == 1 ) )
#endif
        {
          break;//如果最佳模式是skip,跳出循环
        }
        if( !trModes[ modeId ].second )
        {
          continue;//如果当前模式不进行,跳出本次循环
        }
        //we compare the DCT-II cost against the best ISP cost so far (except for TS)
#if JVET_P0058_CHROMA_TS
        if (m_pcEncCfg->getUseFastISP() && !cu.ispMode && ispIsCurrentWinner && trModes[modeId].first != MTS_DCT2_DCT2 && (trModes[modeId].first != MTS_SKIP || !tsAllowed) && bestDCT2cost > bestCostSoFar * threshold)
#else
        if( m_pcEncCfg->getUseFastISP() && !cu.ispMode && ispIsCurrentWinner && trModes[ modeId ].first != 0 && ( trModes[ modeId ].first != 1 || !tsAllowed ) && bestDCT2cost > bestCostSoFar * threshold )
#endif
        {
          continue;//如果满足一定条件,跳出本次循环
        }
#if JVET_AHG14_LOSSLESS
        }
#endif
#if JVET_P0058_CHROMA_TS
        tu.mtsIdx[COMPONENT_Y] = trModes[modeId].first;
#else
        tu.mtsIdx = trModes[ modeId ].first;
#endif
      }

	  /***** 2.2.2.2 确定变换方向,进行变换过程 *****/
      if ((modeId != firstCheckId) && isNotOnlyOneMode)
      {
        m_CABACEstimator->getCtx() = ctxStart;
      }

      int default0Save1Load2 = 0;
      singleDistTmpLuma = 0;

      if( modeId == firstCheckId && ( sps.getUseLFNST() ? ( modeId != lastCheckId ) : ( nNumTransformCands > 1 ) ) )
      {
        default0Save1Load2 = 1;//只有一个模式,则为1
      }
      else if (modeId != firstCheckId)
      {
        if( sps.getUseLFNST() && !cbfBestModeValid )
        {
          default0Save1Load2 = 1;//如果是二次变换,也相当于只有一个模式
        }
        else
        {
          default0Save1Load2 = 2;//如果有多个模式,则为2
        }
      }
      if( cu.ispMode )//如果是ISP模式,default0Save1Load2为0
      {
        default0Save1Load2 = 0; //如果继续划分,则为0
      }
	  

	  /** 2.2.2.2.1 如果可以进行二次变换 **/
      if( sps.getUseLFNST() )
      {
	    /** 如果是多核变换,进行多核变换与帧内模式对应 **/
        if( cu.mtsFlag )//多核变换的两个变换方向对应起来
        {
          if( moreProbMTSIdxFirst )
          {
            const ChannelType     chType      = toChannelType( COMPONENT_Y );
            const CompArea&       area        = tu.blocks[ COMPONENT_Y ];
            const PredictionUnit& pu          = *cs.getPU( area.pos(), chType );
            uint32_t              uiIntraMode = pu.intraDir[ chType ];

            if( transformIndex == 1 )//多核变换的两个方向
            {
#if JVET_P0058_CHROMA_TS
              tu.mtsIdx[COMPONENT_Y] = (uiIntraMode < 34) ? MTS_DST7_DCT8 : MTS_DCT8_DST7;
#else
              tu.mtsIdx = ( uiIntraMode < 34 ) ? MTS_DST7_DCT8 : MTS_DCT8_DST7;
#endif
            }
            else if( transformIndex == 2 )//多核变换的两个方向
            {
#if JVET_P0058_CHROMA_TS
              tu.mtsIdx[COMPONENT_Y] = (uiIntraMode < 34) ? MTS_DCT8_DST7 : MTS_DST7_DCT8;
#else
              tu.mtsIdx = ( uiIntraMode < 34 ) ? MTS_DCT8_DST7 : MTS_DST7_DCT8;
#endif
            }
            else//多核变换的两个方向
            {
#if JVET_P0058_CHROMA_TS
              tu.mtsIdx[COMPONENT_Y] = MTS_DST7_DST7 + transformIndex;
#else
              tu.mtsIdx = MTS_DST7_DST7 + transformIndex;
#endif
            }
          }
          else//多核变换的两个方向
          {
#if JVET_P0058_CHROMA_TS
            tu.mtsIdx[COMPONENT_Y] = MTS_DST7_DST7 + transformIndex;
#else
            tu.mtsIdx = MTS_DST7_DST7 + transformIndex;
#endif
          }
        }
		/** 如果不是多核变换 **/
        else
        {
#if JVET_P0058_CHROMA_TS
          tu.mtsIdx[COMPONENT_Y] = transformIndex;
#else
          tu.mtsIdx = transformIndex;
#endif
        }
		/** 如果是TS变换模式 **/
        if( !cu.mtsFlag && checkTransformSkip )
        {
          xIntraCodingTUBlock( tu, COMPONENT_Y, false, singleDistTmpLuma, default0Save1Load2, &numSig, modeId == 0 ? &trModes : nullptr, true );//进行多核变换
          if( modeId == 0 )
          {
            for( int i = 0; i < 2; i++ )//遍历DCT2和TS
            {
              if( trModes[ i ].second )//如果当前模式可用
              {
                lastCheckId = trModes[ i ].first;//最后一个检查的模式设置为当前模式
              }
            }
          }
        }
		/** 如果不是TS变换模式 **/
        else
        {
          xIntraCodingTUBlock( tu, COMPONENT_Y, false, singleDistTmpLuma, default0Save1Load2, &numSig );//进行普通变换
        }
      }
	  /* 2.2.2.2.2 如果不可以进行二次变换 */
      else
      {
		 /******************* 如果是TS或多核变换 ***************/
        if( nNumTransformCands > 1 )
        {
          xIntraCodingTUBlock( tu, COMPONENT_Y, false, singleDistTmpLuma, default0Save1Load2, &numSig, modeId == 0 ? &trModes : nullptr, true );//进行多核变换
          if( modeId == 0 )
          {
            for( int i = 0; i < nNumTransformCands; i++ )
            {
              if( trModes[ i ].second )
              {
                lastCheckId = trModes[ i ].first;
              }
            }
          }
        }
		/******************* 如果是正常变换 ***************/
        else
        {
          xIntraCodingTUBlock( tu, COMPONENT_Y, false, singleDistTmpLuma, default0Save1Load2, &numSig );//进行普通变换
        }
      }

	  /******* 2.2.2.3 通过RD决定最佳模式 *********/
      //----- determine rate and r-d cost -----

	  /** 2.2.2.3.1 如果进行二次变换 **/
      if( ( sps.getUseLFNST() ? ( modeId == lastCheckId && modeId != 0 && checkTransformSkip ) : ( trModes[ modeId ].first != 0 ) ) && !TU::getCbfAtDepth( tu, COMPONENT_Y, currDepth ) )
      {
        //In order not to code TS flag when cbf is zero, the case for TS with cbf being zero is forbidden.
        singleCostTmp = MAX_DOUBLE;
      }
	  /* 2.2.2.2.1 如果不进行二次变换 */
	  else
      {
        if( cu.ispMode && m_pcRdCost->calcRdCost( csFull->fracBits, csFull->dist + singleDistTmpLuma ) > bestCostSoFar )
        {
          earlySkipISP = true;
        }
        else
        {
#if JVET_P1026_ISP_LFNST_COMBINATION
          singleTmpFracBits = xGetIntraFracBitsQT( *csFull, partitioner, true, false, subTuCounter, ispType, &cuCtx );
#else
          singleTmpFracBits = xGetIntraFracBitsQT( *csFull, partitioner, true, false, subTuCounter, ispType );
#endif
        }
        singleCostTmp     = m_pcRdCost->calcRdCost( singleTmpFracBits, singleDistTmpLuma );
      }

	  
      if ( !cu.ispMode && nNumTransformCands > 1 && modeId == firstCheckId )
      {
        bestDCT2cost = singleCostTmp;
      }

	  /********* 2.2.2.4 比较最终决定代价 **********/
      if (singleCostTmp < dSingleCost)
      {
        dSingleCost       = singleCostTmp;
        uiSingleDistLuma  = singleDistTmpLuma;
        singleFracBits    = singleTmpFracBits;

        if( sps.getUseLFNST() )
        {
          bestModeId[ COMPONENT_Y ] = modeId;
          cbfBestMode = TU::getCbfAtDepth( tu, COMPONENT_Y, currDepth );
          cbfBestModeValid = true;
          validReturnFull = true;
        }
        else
        {
          bestModeId[ COMPONENT_Y ] = trModes[ modeId ].first;
          if( trModes[ modeId ].first == 0 )
          {
            cbfDCT2 = TU::getCbfAtDepth( tu, COMPONENT_Y, currDepth );
          }
        }

        if( bestModeId[COMPONENT_Y] != lastCheckId )
        {
          saveCS.getPredBuf( tu.Y() ).copyFrom( csFull->getPredBuf( tu.Y() ) );
          saveCS.getRecoBuf( tu.Y() ).copyFrom( csFull->getRecoBuf( tu.Y() ) );

          if( keepResi )
          {
            saveCS.getResiBuf   ( tu.Y() ).copyFrom( csFull->getResiBuf   ( tu.Y() ) );
            saveCS.getOrgResiBuf( tu.Y() ).copyFrom( csFull->getOrgResiBuf( tu.Y() ) );
          }

          tmpTU->copyComponentFrom( tu, COMPONENT_Y );

          ctxBest = m_CABACEstimator->getCtx();
        }
      }
    }

	/*********************** 2.3 返回代价 ************************/

    if( sps.getUseLFNST() && !validReturnFull )//如果是二次变换,但没有正确的返回,则不这样进行
    {
      csFull->cost = MAX_DOUBLE;

      if( bCheckSplit )
      {
        ctxBest = m_CABACEstimator->getCtx();
      }
    }
    else//其他情况是正常情况,获取代价
    {
      if( bestModeId[COMPONENT_Y] != lastCheckId )
      {
        csFull->getPredBuf( tu.Y() ).copyFrom( saveCS.getPredBuf( tu.Y() ) );
        csFull->getRecoBuf( tu.Y() ).copyFrom( saveCS.getRecoBuf( tu.Y() ) );

        if( keepResi )
        {
          csFull->getResiBuf   ( tu.Y() ).copyFrom( saveCS.getResiBuf   ( tu.Y() ) );
          csFull->getOrgResiBuf( tu.Y() ).copyFrom( saveCS.getOrgResiBuf( tu.Y() ) );
        }

        tu.copyComponentFrom( *tmpTU, COMPONENT_Y );

        if( !bCheckSplit )
        {
          m_CABACEstimator->getCtx() = ctxBest;
        }
      }
      else if( bCheckSplit )
      {
        ctxBest = m_CABACEstimator->getCtx();
      }

      csFull->cost     += dSingleCost;//代价RD
      csFull->dist     += uiSingleDistLuma;//失真D
      csFull->fracBits += singleFracBits;//比特数R
    }
  }

  /************************************************************ 3.继续划分为TU的变换 *******************************************************/
  bool validReturnSplit = false;
  if( bCheckSplit )//如果继续划分为TU,则递归调用xRecurIntraCodingLumaQT,对每个子块继续变换
  {
	/*************** 3.1 初始化上下文和划分 ***************/
    //----- store full entropy coding status, load original entropy coding status -----
    if( bCheckFull )
    {
      m_CABACEstimator->getCtx() = ctxStart;//存储完整的熵编码状态,加载原始熵编码状态(上下文信息)
    }
    //----- code splitted block -----
    csSplit->cost = 0;

    bool uiSplitCbfLuma  = false;
    bool splitIsSelected = true;
    if( partitioner.canSplit( TU_MAX_TR_SPLIT, cs ) )//如果TU可以划分,则继续划分
    {
      partitioner.splitCurrArea( TU_MAX_TR_SPLIT, cs );
    }

    if( cu.ispMode )//如果是ISP,则继续划分
    {
      partitioner.splitCurrArea( ispType, *csSplit );
    }

	/*************** 3.2 递归进行TU级的运算 ***************/
    do//递归调用xRecurIntraCodingLumaQT
    {
      bool tmpValidReturnSplit = xRecurIntraCodingLumaQT( *csSplit, partitioner, bestCostSoFar, subTuCounter, ispType, false, mtsCheckRangeFlag, mtsFirstCheckId, mtsLastCheckId );
      subTuCounter += subTuCounter != -1 ? 1 : 0;
      if( sps.getUseLFNST() && !tmpValidReturnSplit )
      {
        splitIsSelected = false;
        break;
      }

      if( !cu.ispMode )
      {
        csSplit->setDecomp( partitioner.currArea().Y() );
      }
      else if( CU::isISPFirst( cu, partitioner.currArea().Y(), COMPONENT_Y ) )
      {
        csSplit->setDecomp( cu.Y() );
      }

      uiSplitCbfLuma |= TU::getCbfAtDepth( *csSplit->getTU( partitioner.currArea().lumaPos(), partitioner.chType, subTuCounter - 1 ), COMPONENT_Y, partitioner.currTrDepth );
      if( cu.ispMode )//如果是ISP
      {
        //exit condition if the accumulated cost is already larger than the best cost so far (no impact in RD performance)

        if( csSplit->cost > bestCostSoFar )//并不十分严格的跳出条件:如果ISP到了当前块,代价已经超出了之前的最佳
        {
          earlySkipISP    = true;//就提前跳出MIP
          splitIsSelected = false;
          break;
        }
        else//快速算法,比较严格的跳出条件:如果ISP到了当前块,代价达到了一定阈值
        {
          //more restrictive exit condition 
          bool tuIsDividedInRows = CU::divideTuInRows( cu );
          int nSubPartitions = tuIsDividedInRows ? cu.lheight() >> floorLog2(cu.firstTU->lheight()) : cu.lwidth() >> floorLog2(cu.firstTU->lwidth());
          double threshold = nSubPartitions == 2 ? 0.95 : subTuCounter == 1 ? 0.83 : 0.91;
          if( subTuCounter < nSubPartitions && csSplit->cost > bestCostSoFar*threshold )
          {
            earlySkipISP    = true;
            splitIsSelected = false;
            break;
          }
        }
      }
    } while( partitioner.nextPart( *csSplit ) );

    partitioner.exitCurrSplit();

	/******** 3.3 若ISP没有提前跳出,算代价设标志 *******/
    if( splitIsSelected )//如果一直没有提前跳出
    {
      for( auto &ptu : csSplit->tus )//遍历每一个TU
      {
        if( currArea.Y().contains( ptu->Y() ) )
        {
          TU::setCbfAtDepth( *ptu, COMPONENT_Y, currDepth, uiSplitCbfLuma ? 1 : 0 );//设置CBF
        }
      }

      //----- restore context states -----
      m_CABACEstimator->getCtx() = ctxStart;

#if JVET_P1026_ISP_LFNST_COMBINATION
      cuCtx.violatesLfnstConstrained[CHANNEL_TYPE_LUMA] = false;
      cuCtx.violatesLfnstConstrained[CHANNEL_TYPE_CHROMA] = false;
      cuCtx.lfnstLastScanPos = false;
#if JVET_P1026_MTS_SIGNALLING
      cuCtx.violatesMtsCoeffConstraint = false;
#endif
#endif

      //----- determine rate and r-d cost -----
#if JVET_P1026_ISP_LFNST_COMBINATION
      csSplit->fracBits = xGetIntraFracBitsQT( *csSplit, partitioner, true, false, cu.ispMode ? 0 : -1, ispType, &cuCtx );//计算R
#else
      csSplit->fracBits = xGetIntraFracBitsQT( *csSplit, partitioner, true, false, cu.ispMode ? 0 : -1, ispType );
#endif

      //--- update cost ---
      csSplit->cost     = m_pcRdCost->calcRdCost(csSplit->fracBits, csSplit->dist);//更新RD

      validReturnSplit = true;//TU划分成功
    }
  }

  /******************************************************** 4.处理返回值是true还是false *****************************************************/
  bool retVal = false;
  if( csFull || csSplit )//本部分总结:返回true的条件:1.不使用二次变换或使用二次变换了但有合理返回,2.没有提前跳出isp
  {
    if( !sps.getUseLFNST() || validReturnFull || validReturnSplit )
    {
      {
        // otherwise this would've happened in useSubStructure
        cs.picture->getRecoBuf( currArea.Y() ).copyFrom( cs.getRecoBuf( currArea.Y() ) );
        cs.picture->getPredBuf( currArea.Y() ).copyFrom( cs.getPredBuf( currArea.Y() ) );
      }

      if( cu.ispMode && earlySkipISP )//这种情况依然返回false:isp模式提前跳出(用了ISP发现不好)
      {
        cs.cost = MAX_DOUBLE;
      }
      else
      {
        cs.cost = m_pcRdCost->calcRdCost( cs.fracBits, cs.dist );
        retVal = true;
      }
    }
  }
  return retVal;
}

  • 1
    点赞
  • 6
    收藏
    觉得还不错? 一键收藏
  • 0
    评论

“相关推荐”对你有帮助么?

  • 非常没帮助
  • 没帮助
  • 一般
  • 有帮助
  • 非常有帮助
提交
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值