基于L1方案模型剪枝，以DNN为例实现剪枝及其前向传播

前言

前一篇博客Tensorflow模型剪枝写到了tensorflow自带的剪枝方案添加掩膜矩阵来实现剪枝，在实际操作中我们并不知道每层对最后输出结果影响有多大每层用固定的稀疏度显然不合理。因此我们可以借助L1范数剪裁weight初步观察我们每一层剪裁力度对最后输出结果的影响大小。

一、模型保存参数的读取

def search_file(root_dir, data_type):
    for root, dirs, files in os.walk(root_dir):
        for file in files:
            if os.path.splitext(file)[-1].lower() == data_type:
                file_name = root + file
    # print(file_name)
    return file

def return_parameter(model_dir):
    ckpt = search_file(model_dir, '.meta')
    ckpt_path = model_dir + ckpt.split('.')[0]
    reader = tf.train.NewCheckpointReader(ckpt_path)
    all_variables = reader.get_variable_to_shape_map()
    # print(all_variables)
    # loop save non-None data in txt
    parameter_dict = {}
    for key in all_variables.keys():
        # print(key,all_variables[key])
        parameter_data = reader.get_tensor(key)
        if 'generator' in key.split('_'):
            print('key', key, shape:',parameter_data.shape)
            parameter_dict[key] = parameter_data
    # print(parameter_dict.keys())
    return parameter_dict

这里需要注意，如果模型保存了很多个，可以把想要的模型单独拿出来，一共四个文件后缀分别.data, .index, .meta，和checkpoint放在一个文件夹下。用以上代码查看所有的参数key和data。因为我这里是用的GAN存在生成网络和对抗网络我只需要查看生成网络的参数加了判断参数key包含generator，其他网络结构自行设计。

二、基于L1范数权重参数剪枝

def weight_pruning(data_in, spares_rate):
    """
    weights pruning
    :param data_in: 2-D weight
    :param spares_rate: float
    :return: 2-D weight
    """
    [r, l] = data_in.shape
    data_sort = np.sort(abs(data_in.reshape([r*l])))
    data_min = data_sort[int(r*l*spares_rate)]
    data_in[abs(data_in) < data_min] = 0

    # count None-Zero
    none_z = np.nonzero(data_in)
    z_sizes = len(data_in[none_z])
    rate = z_sizes/(r*l)
    print(1-rate, "pruning rate")
    return data_in

主要思想是对每一层参数的绝对值进行排序，设定该层的剪裁率，将排序小于剪裁率的数据全部置零。

三、读取模型计算结果

def fully_connected(data, weight, biases, in_data_dim, out_data_dim, bias = True):
    """
    fully connected 2-D
    :param data: 1-D data in
    :param weight: 2-D [data_in ,data_out]
    :param biases: 1-D data_in
    :param in_data_dim: int
    :param out_data_dim: int
    :param bias: bool
    :return:1-D data_out
    """
    fully_out = []
    # print(data.shape, weight.shape,biases.shape,in_data_dim,out_data_dim,"data weight biases i_dim out_dim")
    if bias == False:
        for i in range(out_data_dim):
            out_temp = np.sum(data * weight[:,i])
            fully_out.append(out_temp)
    else:
        for i in range(out_data_dim):
            out_temp = np.sum(data * weight[:,i]) + biases[i]
            fully_out.append(out_temp)
    return fully_out

def model_prediction(data_in, parameter):
    [r, l, p, q] = data_in.shape
    data_in = np.reshape(data_in,[r, l*p])
    prediction_data = np.zeros((r,p))
    fcl_1_w = parameter['generator_model/fcl_1/w']
    fcl_1_b = parameter['generator_model/fcl_1/b']
    fcl_2_w = parameter['generator_model/fcl_2/w']
    fcl_2_b = parameter['generator_model/fcl_2/b']
    fcl_3_w = parameter['generator_model/fcl_3/w']
    fcl_3_b = parameter['generator_model/fcl_3/b']
    fcl_4_w = parameter['generator_model/fcl_4/w']
    fcl_4_b = parameter['generator_model/fcl_4/b']
    # weight pruning
    # fcl_1_w = weight_pruning(fcl_1_w, 0.6)
    # fcl_2_w = weight_pruning(fcl_2_w, 0.6)
    # fcl_3_w = weight_pruning(fcl_3_w, 0.6)
    # fcl_4_w = weight_pruning(fcl_4_w, 0.6)
    print("model parameter pruning finish")
    # print("fcl_1_w", fcl_1_w.shape,"fcl_2_w", fcl_2_w.shape, "fcl_3_w", fc1_3_w.shape,"fcl_4_w", fc1_4_w.shape,"fcl_1_b", fcl_1_b.shape)
    ##   model parameter config
    for i in range(r):
        data_inputs = np.reshape(data_in[i], [7*257])
        fcl_1 = fully_connected(data_inputs, fcl_1_w, fcl_1_b, 1799, 64)
        fcl_1 = relu_activate(fcl_1)
        fcl_2 = fully_connected(fcl_1, fcl_2_w, fcl_2_b, 64, 387)
        fcl_2 = relu_activate(fcl_2)
        fcl_3 = fully_connected(fcl_2, fcl_3_w, fcl_3_b, 387, 64)
        fcl_3 = relu_activate(fcl_3)
        fcl_4 = fully_connected(fcl_3, fcl_4_w, fcl_4_b, 64, 257)
        prediction_data[i] = fcl_4
        print("frame :", i)
    plt.imshow(prediction_data.T)
    plt.show()
    return prediction_data

我这里是按照多帧图输入，可以先把weight pruning注释掉测试一下结果的正确性，然后再设置每一层剪裁数据参数量设置不同的值查看对最后结果的影响。

四、代码

"""
!/usr/bin/env python
-*- coding:utf-8 -*-
Author: eric.lai
Created on 2019/9/18 16:32
"""
import os
import numpy as np
import tensorflow as tf
from audio_processing import *
from deeplearning_api import *
import matplotlib.pylab as plt
from scipy.signal import butter, lfilter


def search_file(root_dir, data_type):
    for root, dirs, files in os.walk(root_dir):
        for file in files:
            if os.path.splitext(file)[-1].lower() == data_type:
                file_name = root + file
    # print(file_name)
    return file

def find_file(root_dir, data_type):
    file_path = []
    for root, dirs, files in os.walk(root_dir):
        for file in files:
            if os.path.splitext(file)[-1].lower() == data_type:
                file_path.append(file)
    # print(file_path)
    return file_path

def search_key(keys,keys_key1, keys_key2):
    for key in keys:
        if keys_key1 in key.split('/') and keys_key2 in key.split('/'):
            return key
        else:
            continue

def scale_feature(data, mean, std):
    for i in range(data.shape[0]):

        if len(data.shape) == 3:
            for j in range(data.shape[1]):
                data[i, j, :] = (data[i, j, :] - mean) / std
        if len(data.shape) == 2:
            data[i, :] = (data[i, :] - mean) / std
    return data

def relu_activate(data_in):
    for i in range(len(data_in)):
        if data_in[i] < 0:
            data_in[i] = 0
    return data_in

def butter_bandpass(lowcut, highcut, fs, order=5):
    nyq = 0.5 * fs
    low = lowcut / nyq
    high = highcut / nyq
    b, a = butter(order, [low, high], btype='band')
    return b, a

def butter_bandpass_filter(data, lowcut, highcut, fs, order=5):
    b, a = butter_bandpass(lowcut, highcut, fs, order=order)
    y = lfilter(b, a, data)
    return y

def weight_pruning(data_in, spares_rate):
    """
    weights pruning
    :param data_in: 2-D weight
    :param spares_rate: float
    :return: 2-D weight
    """
    # [r, l] = data_in.shape
    # data_sort = np.sort(abs(data_in.reshape([r*l])))
    # data_zeros = data_sort[0:int(r*l*spares_rate)]
    # for i in range(r):
    #     for j in range(l):
    #         if abs(data_in[i][j]) <= data_zeros[-1]:
    #             data_in[i][j] = 0
    # return data_in

    [r, l] = data_in.shape
    data_sort = np.sort(abs(data_in.reshape([r*l])))
    data_min = data_sort[int(r*l*spares_rate)]
    data_in[abs(data_in) < data_min] = 0

    # count None-Zero
    none_z = np.nonzero(data_in)
    z_sizes = len(data_in[none_z])
    rate = z_sizes/(r*l)
    print(1-rate, "pruning rate")
    return data_in

def statistic_weight(modle_dir):
    parameter = return_parameter(model_dir)
    fcl_1_w = parameter['generator_model/fcl_1/w']
    fcl_1_b = parameter['generator_model/fcl_1/b']
    fcl_2_w = parameter['generator_model/fcl_2/w']
    fcl_2_b = parameter['generator_model/fcl_2/b']
    fcl_3_w = parameter['generator_model/fcl_3/w']
    fcl_3_b = parameter['generator_model/fcl_3/b']
    fcl_4_w = parameter['generator_model/fcl_4/w']
    fcl_4_b = parameter['generator_model/fcl_4/b']

    fcl_1_w_ = fcl_1_w.reshape([7*257*2048])
    fcl_2_w_ = fcl_2_w.reshape([2048*2048])
    fcl_3_w_ = fcl_3_w.reshape([2048*2048])
    fcl_4_w_ = fcl_4_w.reshape([2048*257])
    print(np.median(abs(fcl_1_w_)), np.median(abs(fcl_2_w_)),np.median(abs(fcl_3_w_)),np.median(abs(fcl_4_w_)),"all parameter mean")
    print(np.min(abs(fcl_1_w)), np.min(abs(fcl_2_w)),np.min(abs(fcl_3_w)),np.min(abs(fcl_4_w)),"mean")

    plt.figure(1)
    plt.subplot(141);plt.imshow(fcl_1_w)
    plt.subplot(142);plt.imshow(fcl_2_w)
    plt.subplot(143);plt.imshow(fcl_3_w)
    plt.subplot(144);plt.imshow(fcl_4_w)
    plt.colorbar()

    plt.figure(2)
    plt.subplot(221);plt.hist(fcl_1_w_,200)
    plt.subplot(222);plt.hist(fcl_2_w_,200)
    plt.subplot(223);plt.hist(fcl_3_w_,200)
    plt.subplot(224);plt.hist(fcl_4_w_,200)
    plt.show()

def fully_connected_1D(data, weight, biases, in_data_dim, out_data_dim, bias = True):
    """
    data [a*b*c]
    weight [a*b*c*out_data_dim]
    """
    fully_out = []
    out_temp = 0
    if bias == False:
        for i in range(out_data_dim):
            for j in range(in_data_dim):
                out_temp += data[j] * weight[i*in_data_dim+j]
            fully_out.append(out_temp)
    else:
        for i in range(out_data_dim):
            for j in range(in_data_dim):
                out_temp += data[j] * weight[i*in_data_dim+j]
            out_temp +=  biases[i]
            fully_out.append(out_temp)
    # print(fully_out)
    return fully_out


def fully_connected(data, weight, biases, in_data_dim, out_data_dim, bias = True):
    """
    fully connected 2-D
    :param data: 1-D data in
    :param weight: 2-D [data_in ,data_out]
    :param biases: 1-D data_in
    :param in_data_dim: int
    :param out_data_dim: int
    :param bias: bool
    :return:1-D data_out
    """
    fully_out = []
    # print(data.shape, weight.shape,biases.shape,in_data_dim,out_data_dim,"data weight biases i_dim out_dim")
    if bias == False:
        for i in range(out_data_dim):
            out_temp = np.sum(data * weight[:,i])
            fully_out.append(out_temp)
    else:
        for i in range(out_data_dim):
            out_temp = np.sum(data * weight[:,i]) + biases[i]
            fully_out.append(out_temp)
    return fully_out


def return_parameter(model_dir):
    ckpt = search_file(model_dir, '.meta')
    ckpt_path = model_dir + ckpt.split('.')[0]

    reader = tf.train.NewCheckpointReader(ckpt_path)
    all_variables = reader.get_variable_to_shape_map()
    # print(all_variables)
    # loop save non-None data in txt
    parameter_dict = {}
    for key in all_variables.keys():
        # print(key,all_variables[key])
        parameter_data = reader.get_tensor(key)
        if 'generator' in key.split('_'):
            print('**************** save', key ,' succeed******************* shape:',parameter_data.shape)
            parameter_dict[key] = parameter_data
    # print(parameter_dict.keys())
    return parameter_dict


def model_prediction(data_in, parameter):
    [r, l, p, q] = data_in.shape
    data_in = np.reshape(data_in,[r, l*p])
    prediction_data = np.zeros((r,p))
    fcl_1_w = parameter['generator_model/fcl_1/w']
    fcl_1_b = parameter['generator_model/fcl_1/b']
    fcl_2_w = parameter['generator_model/fcl_2/w']
    fcl_2_b = parameter['generator_model/fcl_2/b']
    fcl_3_w = parameter['generator_model/fcl_3/w']
    fcl_3_b = parameter['generator_model/fcl_3/b']
    fcl_4_w = parameter['generator_model/fcl_4/w']
    fcl_4_b = parameter['generator_model/fcl_4/b']
    # weight pruning
    # fcl_1_w = weight_pruning(fcl_1_w, 0.6)
    # fcl_2_w = weight_pruning(fcl_2_w, 0.6)
    # fcl_3_w = weight_pruning(fcl_3_w, 0.6)
    # fcl_4_w = weight_pruning(fcl_4_w, 0.6)
    print("model parameter pruning finish")
    # print("fcl_1_w", fcl_1_w.shape,"fcl_2_w", fcl_2_w.shape, "fcl_3_w", fc1_3_w.shape,"fcl_4_w", fc1_4_w.shape,"fcl_1_b", fcl_1_b.shape)
    ##   model parameter config
    for i in range(r):
        data_inputs = np.reshape(data_in[i], [7*257])
        fcl_1 = fully_connected(data_inputs, fcl_1_w, fcl_1_b, 1799, 64)
        fcl_1 = relu_activate(fcl_1)
        fcl_2 = fully_connected(fcl_1, fcl_2_w, fcl_2_b, 64, 387)
        fcl_2 = relu_activate(fcl_2)
        fcl_3 = fully_connected(fcl_2, fcl_3_w, fcl_3_b, 387, 64)
        fcl_3 = relu_activate(fcl_3)
        fcl_4 = fully_connected(fcl_3, fcl_4_w, fcl_4_b, 64, 257)
        prediction_data[i] = fcl_4
        print("frame :", i)
    plt.imshow(prediction_data.T)
    plt.show()
    return prediction_data


if __name__ == '__main__':
    model_dir = 'C:/Users/asus/Desktop/model_64/'
    wave_dir = 'C:/Users/asus/Desktop/mix/eval/test/'
    save_dir = 'C:/Users/asus/Desktop/mix/save_data_0.6/'
    scale_path = 'C:/Users/asus/Desktop/scale/scale.hdf5'
    prediction_audio(wave_dir, save_dir, scale_path, model_dir)