梯度提升Gradient Boosting进行正弦函数回归&浅析损失函数和学习率影响

1. 正弦函数回归

# coding: utf-8
# - https://scikit-learn.org/stable/modules/ensemble.html#gradient-boosting
# [1] J. Friedman, Greedy Function Approximation: A Gradient Boosting Machine, The Annals of Statistics, Vol. 29, No. 5, 2001.
# [2] T. Hastie, R. Tibshirani and J. Friedman, The Elements of Statistical Learning, Second Edition, Section 10.13.2, Springer, 2009.
import numpy as np
from matplotlib import pyplot as plt
from sklearn.ensemble import GradientBoostingRegressor
from tqdm import tqdm
import utils


def fit_gradient_boosting_regressor(X: np.ndarray, n_samples: int, y: np.ndarray,
                                    learning_rate: float, max_depth: int, random_state: int, loss: str):
    """Fits gradient tree boosting regressor and returns the learned model and training errors."""
    gb = GradientBoostingRegressor(n_estimators=n_samples, learning_rate=learning_rate, max_depth=max_depth, random_state=random_state,loss=loss)
    gb.fit(X,y)
    train_errors = gb.train_score_
    return gb, train_errors

def plot_sine(X, y):
    plt.figure()
    plt.plot(X, y)
    plt.xlabel("$x$")
    plt.ylabel("$sin(x)$")
    plt.title("Sinus Data")
    plt.show()


def plot_loss(errors):
    plt.figure()
    plt.plot(errors)
    plt.xlabel("Iteration")
    plt.ylabel("Least Squares Loss")
    plt.title("Gradient Boosting: Loss over Iterations")
    plt.show()


def visualize_gradient_boosting_sine(X, y, axs, staged_y_preds, i, step):
    # Plot sine curve fitting
    ax = axs[i//step]
    ax.set_xlim(0, X.max())
    ax.set_ylim(-1, 1)
    ax.set_xlabel("$x$")
    ax.set_ylabel("$sin(x) / Prediction$")

    # Obtain lines
    error_lines = []
    for _ in range(X.shape[<