[MXNet]Lecture04批量归一化

从零开始实现：

from mxnet import ndarray as nd

def pure_batch_norm(X,gemma,beta,eps=1e-5):
	assert len(X.shape) in (2,4)
	if len(X.shape)==2:
		mean=X.mean(axis=0)
		variance=((X-mean)**2).mean(axis=0)

	else:
		mean=X.mean(axis=(0,2,3),keepdims=True)
		variance=((X-mean)**2).mean(axis=(0,2,3),keepdims=True)

	print("mean",mean)
	print("gemma",gemma)
	print("gemma.reshape",gemma.reshape(mean.shape))

	x_hat=(X-mean)/nd.sqrt(variance+eps)
	return gemma.reshape(mean.shape)*x_hat+beta.reshape(mean.shape)

X=nd.arange(6).reshape((3,2))
y=pure_batch_norm(X,gemma=nd.array([1,1]),beta=nd.array([0,0]))
print(y)
X2=nd.arange(36).reshape((1,4,3,3))
y2=pure_batch_norm(X2,gemma=nd.array([1,1,1,1]),beta=nd.array([0,0,0,0]))
print(y2)

对于全联接层来说，输入是二维，要针对每一个特征进行归一化，所以有几列（axis=1方向），gemma就有几个数。

对于卷积层来说，输入是四维，针对每一个channel进行归一化，所以有几个channel（axis=1方向），gemma就有几个数。

如果把测试时的情况考虑进来，可以用移动平均的方法：

from mxnet import ndarray as nd
def batch_norm(x,gemma,beta,istraining,moving_mean,moving_variance,moving_momentum,eps=1e-5):
	assert len(x.shape) in (2,4)
	if len(x.shape)==2:
		mean=x.mean(axis=0)
		variance=((x-mean)**2).mean(axis=0)
	else:
		mean=x.mean(axis=(0,2,3),keepdims=True)
		variacne=((x-mean)**2).mean(axis=(0,2,3),keepdims=True)
		moving_mean=moving_mean.reshape(mean.shape)
		moving_variance=moving_variance.reshape(variance.shape)
	print(x)

	if istraining:
		x_hat=(x-mean)/nd.sqrt(variance+eps)
		moving_mean[:]=moving_momentum*moving_mean+(1.-moving_momentum)*mean	
		moving_variance[:]=moving_momentum*moving_variance+(1.-moving_momentum)*variance

	else:
		x_hat=(x-moving_mean)/nd.sqrt(moving_variance+eps)
	print('moving_mean',moving_mean)
	print('moving_variacne',moving_variance)
	print(x_hat)
	return x_hat*gemma.reshape(mean.shape)+beta.reshape(mean.shape)
moving_mean=nd.zeros(2)
moving_variance=nd.zeros(2)

gluon实现