mse优化方法（二）——adagrad

import numpy as np
# 导入动画包
import matplotlib.animation as animation
data = np.array([
    [80,200],
    [95,230],
    [104,245],
    [112,247],
    [125,259],
    [135,262]
])

# 两个数组记录m和b的变化过程
mhistroy=[]
bhistroy=[]
# 记录mse的变化过程
msehistory=[]

Weight  =np.ones((2,1)) # m和b 采用矩阵的方式指定权重
ones = np.ones((len(data),1))
Feature = np.hstack((data[:,0:1],ones))
label = data[:,1:2]

learningrate = 2

#初始化cache 记录m和b的变化率
cache = np.zeros((2,1))

def gradentdecent1(): #采用矩阵的方式梯度下降
    global Weight,learningrate,cache
    # 计算的是m和b的梯度
    slop = np.dot(Feature.T,(np.dot(Feature,Weight)-label))
    mse = np.sum(np.square(np.dot(Feature,Weight)-label))
    msehistory.append(mse)

    ## 关键代码，考虑历史的梯度变化率
    ## 如果历史偏差大， 惩罚收敛。历史变化小，激励收敛
    cache = cache + slop**2
    Weight = Weight - learningrate*slop/np.sqrt(cache+0.0000000001)
    mhistroy.append(Weight[0][0])
    bhistroy.append(Weight[1][0])

for i in range(50000):
    gradentdecent1()  

#动态展示
%matplotlib notebook
import matplotlib.pyplot as plt
fig = plt.figure(figsize=(6,6),dpi=60)
plt.xlim(0,5)
plt.ylim(0,130)

axis_name, =  plt.plot(mhistroy[0:100],bhistroy[0:100],c='r')

plt.annotate("goal",xy=(1.0859,122.68), xytext=(+10, +15),
             textcoords='offset points', fontsize=12,
             arrowprops=dict(arrowstyle="->"))

def update(num):
    axis_name.set_data(mhistroy[0:num*100],bhistroy[0:num*100])

animation.FuncAnimation(fig,update,np.arange(0,501),interval=20,repeat=False)