局部光照补偿 (local illumination compensation,LIC)技术基于光照变化的线性模型,认为当前块和参考块之间存在线性关系,表示为α*p[x]+β,其中p[x]是当前块单向预测MV指向的参考块,α和β是分别是缩放因子和偏移项,通过当前块模板和参考块模板(下图蓝色区域)推导得到,如下图所示。
LIC的使用条件:
- 仅用于帧间单向预测
- 对亮度样本少于32的块禁用 LIC
- 参考块模板的样本是通过使用当前块MV的运动补偿生成的,无需将其舍入到整数像素精度
- 不和CIIP、IBC共同使用
- LIC用于亮度、色度分量
LIC的线性模型参数是通过当前块的模板像素(相邻重建像素,模板宽度为1)和参考块的模板像素(当前块的模板通过使用当前块的MV通过MC生成的)通过最小二乘法构造的,主要包括以下两种情况:
(1)对于非子块模式,LIC 参数推导是基于当前CU对应的模板块样本,如下图所示,LIC的参数是通过当前CU的模板样本(左图的蓝色像素点)和参考CU的模板样本(右图的黄色像素点)通过最小二乘法推导得到。
(2)对于子块模型(Affine),LIC 参数基于边界子块导出,导出的单个模型应用于整个块,如下图所示,由于每个子Pu的MV各不相同,因此参考块的模板样本是通过边界各个子PU的MV各自进行运动补偿后得到的样本拼接而成(右图黄色像素点)。
最小二乘法的计算公式如下图所示,其中ref指代参考模板样本,Cur指代的是当前模板样本
相关代码:
xLocalIlluComp函数是LIC技术的关键代码,主要包括:
- 调用xGetSublkTemplate函数获取重建模板和参考模板
- xGetLICParamGeneral函数计算线性参数alpha和beta
- linearTransform函数进行线性变换
void InterPrediction::xLocalIlluComp(const PredictionUnit& pu,
const ComponentID compID,
const Picture& refPic,
const Mv& mv,
const bool biPred,
PelBuf& dstBuf
)
{
Pel* refLeftTemplate = m_pcLICRefLeftTemplate; // 左侧参考模板
Pel* refAboveTemplate = m_pcLICRefAboveTemplate; // 上参考模板
Pel* recLeftTemplate = m_pcLICRecLeftTemplate; // 左相邻重建模板
Pel* recAboveTemplate = m_pcLICRecAboveTemplate; // 上相邻重建模板
int numTemplate[2] = { 0 , 0 }; // 0:Above 上侧模板可用像素数, 1:Left左侧模板可用像素数
// 获取重建模板和参考模板
xGetSublkTemplate(*pu.cu, compID, refPic, mv, pu.blocks[compID].width, pu.blocks[compID].height, 0, 0, numTemplate, refLeftTemplate, refAboveTemplate, recLeftTemplate, recAboveTemplate);
// 计算线性参数alpha和beta
int shift = 0, scale = 0, offset = 0;
xGetLICParamGeneral(*pu.cu, compID, numTemplate, refLeftTemplate, refAboveTemplate, recLeftTemplate, recAboveTemplate, shift, scale, offset);
const ClpRng& clpRng = pu.cu->cs->slice->clpRng(compID);
dstBuf.linearTransform(scale, shift, offset, true, clpRng);
}
xGetSublkTemplate函数用来重建模板和参考模板,参考模板是通过xGetPredBlkTpl函数,使用当前块的MV进行运动补偿得到的。
void InterPrediction::xGetSublkTemplate(const CodingUnit& cu,
const ComponentID compID,
const Picture& refPic,
const Mv& mv,
const int sublkWidth,
const int sublkHeight,
const int posW,
const int posH,
int* numTemplate,
Pel* refLeftTemplate,
Pel* refAboveTemplate,
Pel* recLeftTemplate,
Pel* recAboveTemplate)
{
const int bitDepth = cu.cs->sps->getBitDepth(toChannelType(compID));
const int precShift = std::max(0, bitDepth - 12);
const Picture& currPic = *cu.cs->picture; // 当前重建图
const CodingUnit* const cuAbove = cu.cs->getCU(cu.blocks[compID].pos().offset(0, -1), toChannelType(compID));
const CodingUnit* const cuLeft = cu.cs->getCU(cu.blocks[compID].pos().offset(-1, 0), toChannelType(compID));
const CPelBuf recBuf = cuAbove || cuLeft ? currPic.getRecoBuf(cu.cs->picture->blocks[compID]) : CPelBuf();
const CPelBuf refBuf = cuAbove || cuLeft ? refPic.getRecoBuf(refPic.blocks[compID]) : CPelBuf();
std::vector<Pel>& invLUT = m_pcReshape->getInvLUT();
// above 上侧模板
if (cuAbove && posH == 0)
{
// 通过运动补偿获取上参考模板
xGetPredBlkTpl<true>(cu, compID, refBuf, mv, posW, posH, sublkWidth, refAboveTemplate);
// 当前块相邻重建像素
const Pel* rec = recBuf.bufAt(cu.blocks[compID].pos().offset(0, -1));
for (int k = posW; k < posW + sublkWidth; k++)
{
int refVal = refAboveTemplate[k];
int recVal = rec[k];
if (isLuma(compID) && cu.cs->picHeader->getLmcsEnabledFlag() && m_pcReshape->getCTUFlag())
{
recVal = invLUT[recVal];
}
recVal >>= precShift;
refVal >>= precShift;
refAboveTemplate[k] = refVal;
recAboveTemplate[k] = recVal;
numTemplate[0]++; // 上侧模板计数 + 1
}
}
// left 左侧模板
if (cuLeft && posW == 0)
{
xGetPredBlkTpl<false>(cu, compID, refBuf, mv, posW, posH, sublkHeight, refLeftTemplate);
const Pel* rec = recBuf.bufAt(cu.blocks[compID].pos().offset(-1, 0));
for (int k = posH; k < posH + sublkHeight; k++)
{
int refVal = refLeftTemplate[k];
int recVal = rec[recBuf.stride * k];
if (isLuma(compID) && cu.cs->picHeader->getLmcsEnabledFlag() && m_pcReshape->getCTUFlag())
{
recVal = invLUT[recVal];
}
recVal >>= precShift;
refVal >>= precShift;
refLeftTemplate[k] = refVal;
recLeftTemplate[k] = recVal;
numTemplate[1]++; // 左侧模板计数 + 1
}
}
}
xGetLICParamGeneral函数计算线性参数α和β
void InterPrediction::xGetLICParamGeneral(const CodingUnit& cu,
const ComponentID compID,
int* numTemplate,
Pel* refLeftTemplate,
Pel* refAboveTemplate,
Pel* recLeftTemplate,
Pel* recAboveTemplate,
int& shift,
int& scale,
int& offset
)
{
const int cuWidth = cu.blocks[compID].width; // CU的宽度
const int cuHeight = cu.blocks[compID].height; // CU的高度
const int bitDepth = cu.cs->sps->getBitDepth(toChannelType(compID));
const int precShift = std::max(0, bitDepth - 12);
const int maxNumMinus1 = 30 - 2 * std::min(bitDepth, 12) - 1;
const int minDimBit = floorLog2(std::min(cuHeight, cuWidth));
const int minDim = 1 << minDimBit; // 最小的维度尺寸(宽度和高度较小的尺寸)
int minStepBit = minDim > 8 ? 1 : 0;
while (minDimBit > minStepBit + maxNumMinus1) { minStepBit++; } //make sure log2(2*minDim/tmpStep) + 2*min(bitDepth,12) <= 30
const int numSteps = minDim >> minStepBit; // 上模板和左侧模板选取相同数目的像素
const int dimShift = minDimBit - minStepBit;
//----- get correlation data -----
int x = 0, y = 0, xx = 0, xy = 0, cntShift = 0;
// above
if (numTemplate[0] != 0)
{
for (int k = 0; k < numSteps; k++)
{
CHECK(((k * cuWidth) >> dimShift) >= cuWidth, "Out of range");
int refVal = refAboveTemplate[((k * cuWidth) >> dimShift)];
int recVal = recAboveTemplate[((k * cuWidth) >> dimShift)];
x += refVal;
y += recVal;
xx += refVal * refVal;
xy += refVal * recVal;
}
cntShift = dimShift;
}
// left
if (numTemplate[1] != 0)
{
for (int k = 0; k < numSteps; k++)
{
CHECK(((k * cuHeight) >> dimShift) >= cuHeight, "Out of range");
int refVal = refLeftTemplate[((k * cuHeight) >> dimShift)];
int recVal = recLeftTemplate[((k * cuHeight) >> dimShift)];
x += refVal;
y += recVal;
xx += refVal * refVal;
xy += refVal * recVal;
}
cntShift += (cntShift ? 1 : dimShift);
}
//----- determine scale and offset -----
shift = m_LICShift;
if (cntShift == 0)
{
scale = (1 << shift);
offset = 0;
return;
}
const int cropShift = std::max(0, bitDepth - precShift + cntShift - 15);
const int xzOffset = (xx >> m_LICRegShift);
const int sumX = x << precShift;
const int sumY = y << precShift;
const int sumXX = ((xx + xzOffset) >> (cropShift << 1)) << cntShift;
const int sumXY = ((xy + xzOffset) >> (cropShift << 1)) << cntShift;
const int sumXsumX = (x >> cropShift) * (x >> cropShift);
const int sumXsumY = (x >> cropShift) * (y >> cropShift);
int a1 = sumXY - sumXsumY;
int a2 = sumXX - sumXsumX;
int scaleShiftA2 = getMSB(abs(a2)) - 6;
int scaleShiftA1 = scaleShiftA2 - m_LICShiftDiff;
scaleShiftA2 = std::max(0, scaleShiftA2);
scaleShiftA1 = std::max(0, scaleShiftA1);
const int scaleShiftA = scaleShiftA2 + 15 - shift - scaleShiftA1;
a1 = a1 >> scaleShiftA1;
a2 = Clip3(0, 63, a2 >> scaleShiftA2);
scale = int((int64_t(a1) * int64_t(m_LICMultApprox[a2])) >> scaleShiftA); // scale = a1 / a2,此处应该是为了避免除法将a2映射到小数了
scale = Clip3(0, 1 << (shift + 2), scale);
const int maxOffset = (1 << (bitDepth - 1)) - 1;
const int minOffset = -1 - maxOffset;
offset = (sumY - ((scale * sumX) >> shift) + ((1 << (cntShift)) >> 1)) >> cntShift;
offset = Clip3(minOffset, maxOffset, offset);
}