python-tensorflow框架学习 -2

本文接着 python-tensorflow框架学习 -1  继续写续集：主要针对上文末尾提到的一些复杂的部分的详细说明，另预告在下一续集中会把简单的网络结构 + 复杂的一些优化网络部分 ---> 构成稍微完善的tf框架。

part two : 简单的回归网络、简单的分类网络、保存/读取文件、优化器的选择、tensorboard运用、split data的切分数据集。

* simple regression

* simple classification 

* save model

* optimizers 

* tensorboard 

* split dataset

2.1 simple regression.py

# simple regression 简单的回归模型
import tensorflow as tf
import matplotlib.pyplot as plt
import numpy as np
# 0 随机种子
tf.set_random_seed(1)
np.random.seed(1)
# 1 create fake data
x = np.linspace(-1, 1, 100)[:, np.newaxis]  # shape:(100,1)
noise = np.random.normal(0, 0.1, size=x.shape)
y = np.power(x, 2)+noise
# 2 画出原始函数图像
# plt.scatter(x, y)
# plt.show()
# 3 开始定义占位符和相应的网络结构 loss和相应的优化器 建立会话并运行
tf_x = tf.placeholder(tf.float32, x.shape)
tf_y = tf.placeholder(tf.float32, y.shape)

l1 = tf.layers.dense(tf_x, 10, tf.nn.relu)
output = tf.layers.dense(l1, 1)

loss = tf.losses.mean_squared_error(tf_y, output)

optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.5)
train_op = optimizer.minimize(loss)

sess = tf.Session()
sess.run(tf.global_variables_initializer())

plt.ion()  # 区别于plt.show() 会一直处于绘画过程，画图过程是一个持续的操作。与 plt.ioff() 搭配一起使用
for step in range(100):
    _, l, pred = sess.run([train_op, loss, output], feed_dict={tf_x:x,tf_y:y})
    if step % 5 == 0:
        plt.cla()
        plt.scatter(x, y)
        plt.plot(x, pred, 'r-', lw=5)
        plt.text(0.5, 0, 'loss=%.4f'%l, fontdict={'size': 20, 'color': 'red'})
        plt.pause(0.1)
plt.ioff()  # 对应plt.ion()的关闭技术步骤
plt.show()

res 是一个的动态变化过程，最终结果如下：

2.2 simple classification.py 

# simple classification 简单的分类问题
import tensorflow as tf
import matplotlib.pyplot as plt
import numpy as np
# 0 随机种子
tf.set_random_seed(1)
np.random.seed(1)
# 1 create fake data
n_data = np.ones((100, 2))
x0 = np.random.normal(2*n_data, 1)      # class0 x shape=(100, 2)
y0 = np.zeros(100)                      # class0 y shape=(100, )
x1 = np.random.normal(-2*n_data, 1)     # class1 x shape=(100, 2)
y1 = np.ones(100)                       # class1 y shape=(100, )
x = np.vstack((x0, x1))  # shape (200, 2) + some noise
y = np.hstack((y0, y1))  # shape (200, )
# plot data
# plt.scatter(x[:, 0], x[:, 1], c=y, s=100, lw=0, cmap='RdYlGn')
# plt.show()

# 2 创建占位符、构建网络定义loss和acc 相应的优化器、创建会话初始化变量并运行
tf_x = tf.placeholder(tf.float32, x.shape)     # input x
tf_y = tf.placeholder(tf.int32, y.shape)     # input y
# neural network layers
l1 = tf.layers.dense(tf_x, 10, tf.nn.relu)          # hidden layer
output = tf.layers.dense(l1, 2)                     # output layer
# loss function and accuracy function
loss = tf.losses.sparse_softmax_cross_entropy(labels=tf_y, logits=output)           # compute cost
accuracy = tf.metrics.accuracy(          # return (acc, update_op), and create 2 local variables
    labels=tf.squeeze(tf_y), predictions=tf.argmax(output, axis=1),)[1]
# optimizer function
optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.05)
train_op = optimizer.minimize(loss)
# session and init
sess = tf.Session()                                                                 # control training and others
init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())
sess.run(init_op)     # initialize var in graph

plt.ion()   # something about plotting
for step in range(100):
    # train and net output
    _, acc, pred = sess.run([train_op, accuracy, output], {tf_x: x, tf_y: y})
    if step % 2 == 0:
        # plot and show learning process
        plt.cla()
        plt.scatter(x[:, 0], x[:, 1], c=pred.argmax(1), s=100, lw=0, cmap='RdYlGn')
        plt.text(1.5, -4, 'Accuracy=%.2f' % acc, fontdict={'size': 20, 'color': 'red'})
        plt.pause(0.1)

plt.ioff()
plt.show()

同样res是一个动态变化的过程，最终状态如下：

2.3 save model.py

# save model 部分的代码
import tensorflow as tf
import matplotlib.pyplot as plt
import numpy as np
tf.set_random_seed(1)
np.random.seed(1)
# 1 create fake data
x = np.linspace(-1, 1, 100)[:, np.newaxis]          # shape (100, 1)
noise = np.random.normal(0, 0.1, size=x.shape)
y = np.power(x, 2) + noise                          # shape (100, 1) + some noise
# 2 定义save函数 [only save train_op on here ]
def save():
    print('This is save')
    # build neural network
    tf_x = tf.placeholder(tf.float32, x.shape)  # input x
    tf_y = tf.placeholder(tf.float32, y.shape)  # input y
    hidden = tf.layers.dense(tf_x, 10, tf.nn.relu)          # hidden layer
    outputs = tf.layers.dense(hidden, 1)                     # output layer
    loss_val = tf.losses.mean_squared_error(tf_y, outputs)   # compute cost
    train_op = tf.train.GradientDescentOptimizer(learning_rate=0.5).minimize(loss_val)

    sess = tf.Session()
    sess.run(tf.global_variables_initializer())  # initialize var in graph
    ##########  以下便是保存save函数的核心
    saver = tf.train.Saver()
    for step in range(100):
        sess.run(train_op, feed_dict={tf_x:x, tf_y:y})
    saver.save(sess, '.logs303/3part_saver_params', write_meta_graph=False)  # meta_graph is not recommended
    print('saving model and param have finishing')
    # plotting
    pred,loos = sess.run([outputs,loss_val],feed_dict= {tf_x: x, tf_y: y})
    plt.figure(1, figsize=(10, 5))
    plt.subplot(121)
    plt.scatter(x, y)
    plt.plot(x, pred, 'r-', lw=5)
    plt.text(-1, 1.2, 'Save Loss=%.4f' % loos, fontdict={'size': 15, 'color': 'red'})

######## 重要的读取已经保存的模型函数，并将其模型中的参数应用于新的模型(注：读取文件之前，需要建立相应的同结构占位符和网络结构)
def reload():
    print('this is reload')
    tf_x = tf.placeholder(tf.float32, x.shape)
    tf_y = tf.placeholder(tf.float32, y.shape)
    hidden_ = tf.layers.dense(tf_x, 10, tf.nn.relu)
    outputs_ = tf.layers.dense(hidden_, 1)
    loss_val_ = tf.losses.mean_pairwise_squared_error(tf_y, outputs_)

    sess = tf.Session()
    # don't need init variable,just restoring trained variables
    saver = tf.train.Saver()  # define a saver for saving and restoring
    saver.restore(sess, '.logs303/3part_saver_params')
    # plotting
    pred_, loss_ = sess.run([outputs_, loss_val_], {tf_x: x, tf_y: y})
    plt.subplot(122)
    plt.scatter(x, y)
    plt.plot(x, pred_, 'r-', lw=5)
    plt.text(-1, 1.2, 'Reload Loss=%.4f' % loss_, fontdict={'size': 15, 'color': 'red'})
    plt.show()
save()
# destroy previous net
tf.reset_default_graph()
reload()

需要用已保存的模型文件，则需要新建立新的same结构的网络，但两者的结果十分接近或者一致，但不保证一定相等。如下接近

2.4 optimizers.py

# 比较不同optimizer：SGD Momentum RMSprop Adam 部分代码
import tensorflow as tf
import matplotlib.pyplot as plt
import numpy as np
# 0 随机种子和超参数
tf.set_random_seed(1)
np.random.seed(1)
LR = 0.01
BATCH_SIZE = 32

# 1 create fake data
x = np.linspace(-1, 1, 100)[:, np.newaxis]          # shape (100, 1)
noise = np.random.normal(0, 0.1, size=x.shape)
y = np.power(x, 2) + noise                          # shape (100, 1) + some noise
# plot dataset
# plt.scatter(x, y)
# plt.show()

# 2 define general network for different optimizers
class Net:
    def __init__(self, opt, **kwargs):  # 需要传入的参数是opt, x y
        self.x = tf.placeholder(tf.float32, [None, 1])
        self.y = tf.placeholder(tf.float32, [None, 1])

        l = tf.layers.dense(self.x, 20, tf.nn.relu)
        out = tf.layers.dense(l, 1)

        self.loss = tf.losses.mean_squared_error(self.y, out)
        self.train = opt(LR, **kwargs).minimize(self.loss)

# 3 根据上述通用的网络结构定义不同的网络 # different nets
net_SGD = Net(tf.train.GradientDescentOptimizer)
net_Momentum = Net(tf.train.MomentumOptimizer, momentum=0.9)
net_RMSprop = Net(tf.train.RMSPropOptimizer)
net_Adam = Net(tf.train.AdamOptimizer)
nets = [net_SGD, net_Momentum, net_RMSprop, net_Adam]

# 4 创建会话 并初始化一个大loss存储列表
sess = tf.Session()
sess.run(tf.global_variables_initializer())
losses_his = [[], [], [], []]   # record loss

# 5 具体训练training
for step in range(300):          # for each training step
    index = np.random.randint(0, x.shape[0], BATCH_SIZE)  # 随机batch化输入变量x
    b_x = x[index]
    b_y = y[index]

    for net, l_his in zip(nets, losses_his):   # 学习处理的方式
        _, l = sess.run([net.train, net.loss], {net.x: b_x, net.y: b_y})
        l_his.append(l)     # loss recoder

# 6 不同的网络保存的结果值进行分别绘制loss曲线：plot loss history
labels = ['SGD', 'Momentum', 'RMSprop', 'Adam']
for i, l_his in enumerate(losses_his):
    plt.plot(l_his, label=labels[i])
plt.legend(loc='best')
plt.xlabel('Steps')
plt.ylabel('Loss')
plt.ylim((0, 0.2))
plt.show()

对比四种不同优化函数的loss变化曲线：相比较而言，Adam在实验中，更为通用一些。

2.5 tensorboard.py

# 305 tensorboar部分的学习代码
import tensorflow as tf
import numpy as np

tf.set_random_seed(1)
np.random.seed(1)

# fake data
x = np.linspace(-1, 1, 100)[:, np.newaxis]          # shape (100, 1)
noise = np.random.normal(0, 0.1, size=x.shape)
y = np.power(x, 2) + noise                          # shape (100, 1) + some noise

with tf.variable_scope('Inputs'):
    tf_x = tf.placeholder(tf.float32, x.shape, name='x')
    tf_y = tf.placeholder(tf.float32, y.shape, name='y')

with tf.variable_scope('Net'):
    l1 = tf.layers.dense(tf_x, 10, tf.nn.relu, name='hidden_layer')
    output = tf.layers.dense(l1, 1, name='output_layer')

    # add to histogram summary
    tf.summary.histogram('h_out', l1)
    tf.summary.histogram('pred', output)

loss = tf.losses.mean_squared_error(tf_y, output, scope='loss')
train_op = tf.train.GradientDescentOptimizer(learning_rate=0.5).minimize(loss)
tf.summary.scalar('loss', loss)     # add loss to scalar summary

sess = tf.Session()
sess.run(tf.global_variables_initializer())

writer = tf.summary.FileWriter('./log305', sess.graph)     # write to file
merge_op = tf.summary.merge_all()                       # operation to merge all summary

for step in range(100):
    # train and net output
    _, result = sess.run([train_op, merge_op], {tf_x: x, tf_y: y})
    writer.add_summary(result, step)

# 备注：win10系统下的tensorboard可视化终于写通看到图了！！！！
# terminals中的命令行不能有引号，否则报错，写成：tensorboard --logdir=logs305/ 。然后运行terminal即可看到相应的events文件。
# 另外注意：tesorboard生成的文件在logs305目录下，而logs305与生成tensorboards记录的py文件属于同一层级目录。

另外注意区分
1) tf.variable_scope 和 tf.name_scope的区别 [推荐使用variable_scope，name_scope如果name有重复的，可能会报错。]

2) tf.summary.histogram 和 tf.summary.scalar 和 tf.summary.merge_all函数的区别

2.6 split dataset

# 306 根据全量data切分训练集和测试集
import tensorflow as tf
import numpy as np

# load your data or create your data on here
npx = np.random.uniform(-1, 1, (1000, 1))                           # x data
npy = np.power(npx, 2) + np.random.normal(0, 0.1, size=npx.shape)   # y data
npx_train, npx_test = np.split(npx, [800])                          # training and test data train:test = 80% : 20%
npy_train, npy_test = np.split(npy, [800])

# use placeholder, later you may need different data, pass the different data into placeholder
tfx = tf.placeholder(npx_train.dtype, npx_train.shape)
tfy = tf.placeholder(npy_train.dtype, npy_train.shape)

# create dataloader
dataset = tf.data.Dataset.from_tensor_slices((tfx, tfy))
dataset = dataset.shuffle(buffer_size=1000)   # choose data randomly from this buffer
dataset = dataset.batch(32)                   # batch size you will use
dataset = dataset.repeat(3)                   # repeat for 3 epochs
iterator = dataset.make_initializable_iterator()  # later we have to initialize this one

# your network
bx, by = iterator.get_next()                  # use batch to update
l1 = tf.layers.dense(bx, 10, tf.nn.relu)
out = tf.layers.dense(l1, npy.shape[1])
loss = tf.losses.mean_squared_error(by, out)
train = tf.train.GradientDescentOptimizer(0.1).minimize(loss)

sess = tf.Session()
# need to initialize the iterator in this case
sess.run([iterator.initializer, tf.global_variables_initializer()], feed_dict={tfx: npx_train, tfy: npy_train})

for step in range(201):
  try:
    _, trainl = sess.run([train, loss])                       # train
    if step % 10 == 0:
      testl = sess.run(loss, {bx: npx_test, by: npy_test})    # test
      print('step: %i/200' % step, '|train loss:', trainl, '|test loss:', testl)
  except tf.errors.OutOfRangeError:     # if training takes more than 3 epochs, training will be stopped
    print('Finish the last epoch.')
    break

结果：

step: 0/200 |train loss: 0.40924406 |test loss: 0.24913685
step: 10/200 |train loss: 0.069355614 |test loss: 0.09077467
step: 20/200 |train loss: 0.057845347 |test loss: 0.065595865
step: 30/200 |train loss: 0.039349467 |test loss: 0.053314663
step: 40/200 |train loss: 0.042040057 |test loss: 0.044973265
step: 50/200 |train loss: 0.035803095 |test loss: 0.039617945
step: 60/200 |train loss: 0.020598968 |test loss: 0.035423964
step: 70/200 |train loss: 0.031522855 |test loss: 0.03133628
Finish the last epoch.