待更新…
第一阶段 自动微分
步骤1 作为箱子的变量
步骤3 函数的连续调用
步骤4 数值微分
步骤6 手动进行反向传播
import numpy as np
class Varialbe:
def __init__(self, data):
self.data = data
self.grad = None
class Fuction:
def __call__(self, input):
x = input.data
y = self.forward(x)
output = Varialbe(y)
self.input = input
return output
def forward(self, x):
raise NotImplementedError()
def backward(self, gy):
raise NotImplementedError()
class Square(Fuction):
def forward(self, x):
y = x ** 2
return y
def backward(self, gy):
x = self.input.data
gx = 2 * x * gy
return gx
class Exp(Fuction):
def forward(self, x):
y = np.exp(x)
return y
def backward(self, gy):
x = self.input.data
gx = np.exp(x) * gy
return gx
def numerical_deff(f, x, eps=1e-4):
x0 = Varialbe(x.data - eps)
x1 = Varialbe(x.data + eps)
y0 = f(x0)
y1 = f(x1)
return (y1.data - y0.data) / (2 * eps)
A = Square()
B = Exp()
C = Square()
x = Varialbe(np.array(0.5))
a = A(x)
b = B(a)
c = C(b)
y.grad = np.array(1.0)
b.grad = C.backward(y.grad)
a.grad = B.backward(b.grad)
x.garad = A.backward(a.grad)
print(x.grad)
步骤7 反向传播的自动化
import numpy as np
class Varialbe:
def __init__(self, data):
self.data = data
self.grad = None
self.creator = None
def set_creator(self, func):
self.creator = func
class Fuction:
def __call__(self, input):
x = input.data
y = self.forward(x)
output = Varialbe(y)
output.set_creator(self)
self.input = input
self.output = output
return output
def forward(self, x):
raise NotImplementedError()
def backward(self, gy):
raise NotImplementedError()
class Square(Fuction):
def forward(self, x):
y = x ** 2
return y
def backward(self, gy):
x = self.input.data
gx = 2 * x * gy
return gx
class Exp(Fuction):
def forward(self, x):
y = np.exp(x)
return y
def backward(self, gy):
x = self.input.data
gx = np.exp(x) * gy
return gx
def numerical_deff(f, x, eps=1e-4):
x0 = Varialbe(x.data - eps)
x1 = Varialbe(x.data + eps)
y0 = f(x0)
y1 = f(x1)
return (y1.data - y0.data) / (2 * eps)
A = Square()
B = Exp()
C = Square()
x = Varialbe(np.array(0.5))
a = A(x)
b = B(a)
y = C(b)
# assert y.creator == C
# assert y.creator.input == b
# assert y.creator.input.creator == B
# assert y.creator.input.creator.input == a
# assert y.creator.input.creator.input.creator == A
# assert y.creator.input.creator.input.creator.input == x
y.grad = np.array(1.0)
C = y.creator
b = C.input
b.grad = C.backward(y.grad)
B = b.creator
a = B.input
a.grad = B.backward(b.grad)
A = a.creator
x = A.input
x.grad = A.backward(a.grad)
print(x.grad)
import numpy as np
# 调用一个Function时，Function会记录左右的input和output，同时output会记录生成它的Func
class Varialbe:
def __init__(self, data):
self.data = data
self.grad = None
self.creator = None
def set_creator(self, func):
self.creator = func
def backward(self):
f = self.creator
if f is not None:
x = f.input
x.grad = f.backward(self.grad)
x.backward()
class Fuction:
def __call__(self, input):
x = input.data
y = self.forward(x)
output = Varialbe(y)
output.set_creator(self)
self.input = input
self.output = output
return output
def forward(self, x):
raise NotImplementedError()
def backward(self, gy):
raise NotImplementedError()
class Square(Fuction):
def forward(self, x):
y = x ** 2
return y
def backward(self, gy):
x = self.input.data
gx = 2 * x * gy
return gx
class Exp(Fuction):
def forward(self, x):
y = np.exp(x)
return y
def backward(self, gy):
x = self.input.data
gx = np.exp(x) * gy
return gx
def numerical_deff(f, x, eps=1e-4):
x0 = Varialbe(x.data - eps)
x1 = Varialbe(x.data + eps)
y0 = f(x0)
y1 = f(x1)
return (y1.data - y0.data) / (2 * eps)
A = Square()
B = Exp()
C = Square()
x = Varialbe(np.array(0.5))
a = A(x)
b = B(a)
y = C(b)
# assert y.creator == C
# assert y.creator.input == b
# assert y.creator.input.creator == B
# assert y.creator.input.creator.input == a
# assert y.creator.input.creator.input.creator == A
# assert y.creator.input.creator.input.creator.input == x
y.grad = np.array(1.0)
y.backward()
print(x.grad)
import numpy as np
# 调用一个Function时，Function会记录左右的input和output，同时output会记录生成它的Func
class Varialbe:
def __init__(self, data):
self.data = data
self.grad = None
self.creator = None
def set_creator(self, func):
self.creator = func
# def backward(self):
# f = self.creator
# if f is not None:
# x = f.input
# x.grad = f.backward(self.grad)
# x.backward()
def backward(self):
funcs = [self.creator]
while funcs:
f = funcs.pop()
x, y = f.input, f.output
x.grad = f.backward(y.grad)
if x.creator is not None:
funcs.append(x.creator)
class Fuction:
def __call__(self, input):
x = input.data
y = self.forward(x)
output = Varialbe(y)
output.set_creator(self)
self.input = input
self.output = output
return output
def forward(self, x):
raise NotImplementedError()
def backward(self, gy):
raise NotImplementedError()
class Square(Fuction):
def forward(self, x):
y = x ** 2
return y
def backward(self, gy):
x = self.input.data
gx = 2 * x * gy
return gx
class Exp(Fuction):
def forward(self, x):
y = np.exp(x)
return y
def backward(self, gy):
x = self.input.data
gx = np.exp(x) * gy
return gx
def numerical_deff(f, x, eps=1e-4):
x0 = Varialbe(x.data - eps)
x1 = Varialbe(x.data + eps)
y0 = f(x0)
y1 = f(x1)
return (y1.data - y0.data) / (2 * eps)
A = Square()
B = Exp()
C = Square()
x = Varialbe(np.array(0.5))
a = A(x)
b = B(a)
y = C(b)
# assert y.creator == C
# assert y.creator.input == b
# assert y.creator.input.creator == B
# assert y.creator.input.creator.input == a
# assert y.creator.input.creator.input.creator == A
# assert y.creator.input.creator.input.creator.input == x
# y.grad = np.array(1.0)
# C1 = y.creator
# b = C1.input
# b.grad = C1.backward(y.grad)
# B1 = b.creator
# a = B1.input
# a.grad = B1.backward(b.grad)
# A1 = a.creator
# x = A1.input
# x.grad = A1.backward(a.grad)
# print(x.grad)
y.grad = np.array(1.0)
y.backward()
print(x.grad)
import numpy as np
# 调用一个Function时，Function会记录左右的input和output，同时output会记录生成它的Func
class Varialbe:
def __init__(self, data):
self.data = data
self.grad = None
self.creator = None
def set_creator(self, func):
self.creator = func
# def backward(self):
# f = self.creator
# if f is not None:
# x = f.input
# x.grad = f.backward(self.grad)
# x.backward()
def backward(self):
if self.grad is None:
self.grad = np.ones_like(self.data)
funcs = [self.creator]
while funcs:
f = funcs.pop()
x, y = f.input, f.output
x.grad = f.backward(y.grad)
if x.creator is not None:
funcs.append(x.creator)
class Fuction:
def __call__(self, input):
x = input.data
y = self.forward(x)
output = Varialbe(y)
output.set_creator(self)
self.input = input
self.output = output
return output
def forward(self, x):
raise NotImplementedError()
def backward(self, gy):
raise NotImplementedError()
class Square(Fuction):
def forward(self, x):
y = x ** 2
return y
def backward(self, gy):
x = self.input.data
gx = 2 * x * gy
return gx
class Exp(Fuction):
def forward(self, x):
y = np.exp(x)
return y
def backward(self, gy):
x = self.input.data
gx = np.exp(x) * gy
return gx
def numerical_deff(f, x, eps=1e-4):
x0 = Varialbe(x.data - eps)
x1 = Varialbe(x.data + eps)
y0 = f(x0)
y1 = f(x1)
return (y1.data - y0.data) / (2 * eps)
def square(x):
f = Square()
return f(x)
def exp(x):
f = Exp()
return f(x)
x = Varialbe(np.array(0.5))
y = square(exp(square(x)))
y.backward()
print(x.grad)
import numpy as np
import unittest
class SquareTest(unittest.TestCase):
def test_forward(self):
x = Varialbe(np.array(2.0))
y = square(x)
expected = np.array(4.0)
self.assertEqual(y.data, expected)
def test_backward(self):
x = Varialbe(np.array(3.))
y = square(x)
y.backward()
expected = np.array(6.0)
self.assertEqual(x.grad, expected)
def test_gradient_check(self):
x = Varialbe(np.random.rand(1))
y = square(x)
y.backward()
num_grad = numerical_deff(square, x)
flg = np.allclose(x.grad, num_grad)
self.assertTrue(flg)
# 调用一个Function时，Function会记录左右的input和output，同时output会记录生成它的Func
class Varialbe:
def __init__(self, data):
if data is not None:
if not isinstance(data, np.ndarray):
raise TypeError('{} is not supported'.format(type(data)))
self.data = data
self.grad = None
self.creator = None
def set_creator(self, func):
self.creator = func
# def backward(self):
# f = self.creator
# if f is not None:
# x = f.input
# x.grad = f.backward(self.grad)
# x.backward()
def backward(self):
if self.grad is None:
self.grad = np.ones_like(self.data)
funcs = [self.creator]
while funcs:
f = funcs.pop()
x, y = f.input, f.output
x.grad = f.backward(y.grad)
if x.creator is not None:
funcs.append(x.creator)
class Fuction:
def __call__(self, input):
x = input.data
y = self.forward(x)
output = Varialbe(as_array(y))
output.set_creator(self)
self.input = input
self.output = output
return output
def forward(self, x):
raise NotImplementedError()
def backward(self, gy):
raise NotImplementedError()
class Square(Fuction):
def forward(self, x):
y = x ** 2
return y
def backward(self, gy):
x = self.input.data
gx = 2 * x * gy
return gx
class Exp(Fuction):
def forward(self, x):
y = np.exp(x)
return y
def backward(self, gy):
x = self.input.data
gx = np.exp(x) * gy
return gx
def numerical_deff(f, x, eps=1e-4):
x0 = Varialbe(x.data - eps)
x1 = Varialbe(x.data + eps)
y0 = f(x0)
y1 = f(x1)
return (y1.data - y0.data) / (2 * eps)
def square(x):
f = Square()
return f(x)
def exp(x):
f = Exp()
return f(x)
def as_array(x):
if np.isscalar(x):
return np.array(x)
return x
x = Varialbe(np.array(0.5))
y = square(exp(square(x)))
y.backward()
print(x.grad)
#步骤11 可变长参数
import numpy as np
import unittest
class SquareTest(unittest.TestCase):
def test_forward(self):
x = Varialbe(np.array(2.0))
y = square(x)
expected = np.array(4.0)
self.assertEqual(y.data, expected)
def test_backward(self):
x = Varialbe(np.array(3.))
y = square(x)
y.backward()
expected = np.array(6.0)
self.assertEqual(x.grad, expected)
def test_gradient_check(self):
x = Varialbe(np.random.rand(1))
y = square(x)
y.backward()
num_grad = numerical_deff(square, x)
flg = np.allclose(x.grad, num_grad)
self.assertTrue(flg)
# 调用一个Function时，Function会记录左右的input和output，同时output会记录生成它的Func
class Varialbe:
def __init__(self, data):
if data is not None:
if not isinstance(data, np.ndarray):
raise TypeError('{} is not supported'.format(type(data)))
self.data = data
self.grad = None
self.creator = None
self.generation = 0
def set_creator(self, func):
self.creator = func
self.generation = func.generation + 1
# def backward(self):
# f = self.creator
# if f is not None:
# x = f.input
# x.grad = f.backward(self.grad)
# x.backward()
def backward(self):
if self.grad is None:
self.grad = np.ones_like(self.data)
funcs = []
seen_set = set()
# 反向传播的时候，按照generation
def add_func(f):
if f not in seen_set:
funcs.append(f)
seen_set.add(f)
funcs.sort(key=lambda x: x.generation)
add_func(self.creator)
funcs = [self.creator]
while funcs:
f = funcs.pop()
gys = [output.grad for output in f.outputs]
gxs = f.backward(*gys)
if not isinstance(gxs, tuple):
gxs = (gxs, )
for x, gx in zip(f.inputs, gxs):
if x.grad is None: #为了节省内存，同一个Variable求完梯度后grad并不会清空，所以当新的公式也需要对该变量求导后，会导致grad累加，结果错误
x.grad = gx
else:
# 为了解决相同变量作为输入的问题x+x = y
x.grad = x.grad + gx
if x.creator is not None:
# funcs.append(x.creator)
add_func(x.creator)
def cleargrad(self):
self.grad = None
class Fuction:
def __call__(self, *inputs):
xs = [x.data for x in inputs]
ys = self.forward(*xs)
if not isinstance(ys, tuple):
ys = (ys, )
outputs = [Varialbe(as_array(y)) for y in ys]
self.generation = max([x.generation for x in inputs])
for output in outputs:
output.set_creator(self)
self.inputs = inputs
self.outputs = outputs
# print(type(self.inputs))
# print(type(self.outputs))
# print(type(ys))
return outputs if len(outputs) > 1 else outputs[0]
def forward(self, x):
raise NotImplementedError()
def backward(self, gy):
raise NotImplementedError()
class Square(Fuction):
def forward(self, x):
print(type(x))
y = x ** 2
return y
def backward(self, gy):
x = self.inputs[0].data # inputs是tuple类型变量，对于输入只有一个元素的tuple（x,)
gx = 2 * x * gy
return gx
class Exp(Fuction):
def forward(self, x):
y = np.exp(x)
return y
def backward(self, gy):
x = self.input.data
gx = np.exp(x) * gy
return
class Add(Fuction):
def forward(self, x0, x1):
y = x0 + x1
return y
def backward(self, gy):
return gy, gy
def numerical_deff(f, x, eps=1e-4):
x0 = Varialbe(x.data - eps)
x1 = Varialbe(x.data + eps)
y0 = f(x0)
y1 = f(x1)
return (y1.data - y0.data) / (2 * eps)
def square(x):
f = Square()
z = f(x)
return z
def exp(x):
f = Exp()
return f(x)
def as_array(x):
if np.isscalar(x):
return np.array(x)
return x
def add(x0, x1):
return Add()(x0, x1)
x = Varialbe(np.array(2.0))
a = square(x)
y = add(square(a), square(a))
y.backward()
print(y.data)
print(x.grad)
扫一扫
5116
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
