scikit-learn 梯度提升下降 回归

import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline

f = lambda x : (x - 3)**2 + 2.5*x -7.5

求解导数令导数=0求解最小值

# 导数 = 梯度
2*(x - 3)*1 + 2.5 = 0
2*x - 3.5 = 0
x = 1.75

x = np.linspace(-2,5,100)
y = f(x)
plt.plot(x,y)

梯度下降求解最小值

# 导数函数
d = lambda x : 2*(x - 3) + 2.5

# 学习率，每次改变数值的时候，改变多少
learning_rate = 0.1

# min_value瞎蒙的值，方法，最快的速度找到最优解（梯度下降）
min_value = np.random.randint(-3,5,size = 1)[0]
print('-------------------',min_value)
# 记录数据更新了，原来的值，上一步的值,退出条件
min_value_last = min_value + 0.1
# tollerence容忍度，误差，在万分之一，任务结束
tol = 0.0001
count = 0
while True:
    if np.abs(min_value - min_value_last) < tol:
        break
#     梯度下降
    min_value_last = min_value
#     更新值:梯度下降
    min_value = min_value - learning_rate*d(min_value)
    count +=1
    print('+++++++++++++++++++++%d'%(count),min_value)
print('**********************',min_value)
#------------------- 3
#+++++++++++++++++++++1 2.75
#+++++++++++++++++++++2 2.55
#+++++++++++++++++++++3 2.3899999999999997
#...
#+++++++++++++++++++++36 1.7504056481920731
#+++++++++++++++++++++37 1.7503245185536584
#********************** 1.7503245185536584

f2 = lambda x : -(x - 3)**2 + 2.5*x -7.5
x = np.linspace(-2,10,100)
y = f2(x)
plt.plot(x,y)

# 梯度提升
# 导数函数

result = []

d2 = lambda x : -2*(x - 3) + 2.5

learning_rate = 10

# max_value瞎蒙的值，方法，最快的速度找到最优解（梯度下降）
# 梯度消失，梯度爆炸（因为学习率太大）
max_value = np.random.randint(2,8,size = 1)[0]
# max_value = 1000

result.append(max_value)

print('-------------------',max_value)
# 记录数据更新了，原来的值，上一步的值,退出条件
max_value_last = max_value + 0.001
# tollerence容忍度，误差，在万分之一，任务结束
# precision精确度，精度达到了万分之一，任务结束
precision = 0.0001
count = 0
while True:
    if count >3000:
        break
    if np.abs(max_value - max_value_last) < precision:
        break
#     梯度上升
    max_value_last = max_value
#     更新值:梯度上升
    max_value = max_value + learning_rate*d2(max_value)
    result.append(max_value)
    count +=1
    print('+++++++++++++++++++++%d'%(count),max_value)
print('**********************',max_value)
#------------------- 6
#+++++++++++++++++++++1 -29.0
#+++++++++++++++++++++2 636.0
#...
#+++++++++++++++++++++3000 nan
#+++++++++++++++++++++3001 nan
#********************** nan

# 更新规则
ret = ret - ret*step

plt.figure(figsize=(12,9))
x = np.linspace(4,8,100)
y = f2(x)
plt.plot(x,y)
result = np.asarray(result)
plt.plot(result,f2(result),'*')