配置阿里云docker Tensorflow镜像下载 dataset准备和地址 mnist_cnn.py例程

john@john-wang:~/tf2$ docker run -it --rm -v $PWD:/tmp -w /tmp tensorflow/tensorflow

https://account.aliyun.com/login/login.htm
https://cr.console.aliyun.com/cn-hangzhou/new
starwang119

https://cr.console.aliyun.com/cn-hangzhou/instances/mirrors

针对Docker客户端版本大于 1.10.0 的用户

您可以通过修改daemon配置文件/etc/docker/daemon.json来使用加速器

sudo mkdir -p /etc/docker
sudo tee /etc/docker/daemon.json <<-‘EOF’
{
“registry-mirrors”: [“https://rr0r4ivk.mirror.aliyuncs.com”]
}
EOF

john@john-wang:/$ sudo systemctl daemon-reload
john@john-wang:/$ sudo systemctl restart docker

john@john-virtual-machine:/etc/docker$ docker search nginx

docker pull xilinx/vitis-ai:latest
john@john-wang:/$ docker pull tensorflow/tensorflow:latest-gpu-jupyter

https://kitieeae.mirror.aliyuncs.com

1. 安装／升级Docker客户端
   推荐安装1.10.0以上版本的Docker客户端，参考文档 docker-ce

2. 配置镜像加速器
   针对Docker客户端版本大于 1.10.0 的用户

您可以通过修改daemon配置文件/etc/docker/daemon.json来使用加速器

# sudo mkdir -p /etc/docker
sudo vim /etc/docker/daemon.json 
{
  "registry-mirrors": ["https://kitieeae.mirror.aliyuncs.com"]
}
sudo systemctl daemon-reload
sudo systemctl restart docker

下载网页和镜像
https://www.tensorflow.org/install/docker#examples_using_cpu-only_images
john@john-wang:/$ docker run -it --rm tensorflow/tensorflow:latest-jupyter
john@john-wang:/$ docker run -it --rm tensorflow/tensorflow

启动镜像并把当前目前镜像设为工作目录：
To run a TensorFlow program developed on the host machine within a container, mount the host directory and change the container’s working directory (-v hostDir:containerDir -w workDir):
john@john-wang:~/tf2$ docker run -it --rm -v $PWD:/tmp -w /tmp tensorflow/tensorflow

忽略告警
python警告
import warnings
warnings.filterwarnings(“ignore”)

TF警告
python环境下
通过在python文件中添加如下两行代码，设置TensorFlow日志输出级别
import os
os.environ[“TF_CPP_MIN_LOG_LEVEL”] = “3”

TensorFlow的日志级别分为以下三种：
TF_CPP_MIN_LOG_LEVEL = 1 //默认设置，为显示所有信息
TF_CPP_MIN_LOG_LEVEL = 2 //只显示error和warining信息
TF_CPP_MIN_LOG_LEVEL = 3 //只显示error信息
所以，当TensorFlow出现警告信息，又不想让警告信息显示时，可进行如下设置：

dataset准备
# numpy arrays
x = np.arange(0, 10)
# create dataset objects from the arrays, convert to tensor
dx = tf.data.Dataset.from_tensor_slices(x)
# zip the two datasets together
dcomb = tf.data.Dataset.zip((dx, dy)).repeat().batch(3)
# iter
iterator = iter(dcomb)
# next(iter())
for i in range(15):
data = next(iterator)
以上程序成功地将numpy arrays转换成zip iter

mnist_cnn.py例程

import tensorflow as tf
import os
from tensorflow import keras
import numpy as np
import datetime as dt

# import tensorflow.contrib.eager as tfe
# tfe.enable_eager_execution()

(x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()
xlen = len(x_test)
x_test = tf.Variable(x_test)
x_test = tf.cast(x_test, tf.float32)
x_test = x_test / 255.0
# print(f"x_test {x_test.shape}")
# x_test = tf.reshape(x_test, (xlen, 28, 28, 1))
x_test = tf.reshape(x_test, (len(x_test), 28, 28, 1))

STORE_PATH = '/Users/andrewthomas/Adventures in ML/TensorFlowBook/TensorBoard'

def get_batch(x_data, y_data, batch_size):
    idxs = np.random.randint(0, len(y_data), batch_size)
#   print(f"idxs = {idxs}")
    return x_data[idxs,:,:], y_data[idxs]


class ConvLayer(tf.keras.layers.Layer):
    def __init__(self, activation, input_channels, output_channels, window_size, pool_size, filt_stride, pool_stride,
                 initializer=tf.keras.initializers.he_normal()):
        super(ConvLayer, self).__init__()
        self.initializer = initializer
        self.activation = activation
        self.input_channels = input_channels
        self.output_channels = output_channels
        self.window_size = window_size
        self.pool_size = pool_size
        self.filt_stride = filt_stride
        self.pool_stride = pool_stride
        self.w = self.add_weight(shape=(window_size[0], window_size[1], input_channels, output_channels),
                                 initializer=self.initializer,
                                 trainable=True)
        self.b = self.add_weight(shape=(output_channels,), initializer=tf.zeros_initializer, trainable=True)

    def call(self, inputs):
        filt_stride = [1, self.filt_stride[0], self.filt_stride[1], 1]
        out_layer = tf.nn.conv2d(inputs, self.w, filt_stride, padding='SAME')
        # add the bias
        out_layer += self.b
        out_layer = self.activation(out_layer)
        pool_shape = [1, self.pool_size[0], self.pool_size[1], 1]
        pool_strides = [1, self.pool_stride[0], self.pool_stride[1], 1]
        out_layer = tf.nn.max_pool(out_layer, ksize=pool_shape, strides=pool_strides, padding='SAME')
        return out_layer

def loss_fn(logits, labels):
    return tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=labels))

model = tf.keras.Sequential([
    ConvLayer(tf.nn.relu, 1, 32, [5, 5], [2, 2], [1, 1], [2, 2]),
    ConvLayer(tf.nn.relu, 32, 64, [5, 5], [2, 2], [1, 1], [2, 2]),
    tf.keras.layers.Flatten(),
    tf.keras.layers.Dense(300, activation=tf.nn.relu, kernel_initializer=tf.keras.initializers.he_normal()),
    tf.keras.layers.Dropout(0.5),
    tf.keras.layers.Dense(10, activation=None)
])

optimizer = tf.keras.optimizers.Adam()
iterations = 5 #5000
batch_size = 32
train_writer = tf.summary.create_file_writer(STORE_PATH + f"/MNIST_CNN_{dt.datetime.now().strftime('%d%m%Y%H%M')}")
for i in range(iterations):
    batch_x, batch_y = get_batch(x_train, y_train, batch_size=batch_size)
    # create tensors
    batch_x = tf.Variable(batch_x)
    batch_y = tf.Variable(batch_y)
    batch_y = tf.cast(batch_y, tf.int32)
    # get the images in the right format
    batch_x = tf.cast(batch_x, tf.float32)
    batch_x = batch_x / 255.0
#    print(f"batch_x = {batch_x.shape}")
    batch_x = tf.reshape(batch_x, (batch_size, 28, 28, 1))
    with tf.GradientTape() as tape:
        logits = model(batch_x)
#        print(f"logits = {logits[0]}")
        loss = loss_fn(logits, batch_y) # exp softmax matrix: p=exp/sigma_exp vector:softmax_cross_entropy_with_logits: label*lnp, sum, mean e.g. loss
    gradients = tape.gradient(loss, model.trainable_variables)
    optimizer.apply_gradients(zip(gradients, model.trainable_variables))
    if i % 1 == 0:
        max_idxs = tf.argmax(logits, axis=1)
        print(f"logits = {logits[0]}")
        print(f"logits = {logits.shape}")
        print(max_idxs)
        print(max_idxs.numpy())
        print(batch_y.numpy())
        train_acc = np.sum(max_idxs.numpy() == batch_y.numpy()) / len(batch_y)
        test_logits = model(x_test, training=False)
        max_idxs = tf.argmax(test_logits, axis=1)
        test_acc = np.sum(max_idxs.numpy() == y_test) / len(y_test)
        print(f"Iter: {i}, loss={loss:.3f}, train accuracy={train_acc * 100:.3f}%, test accuracy={test_acc * 100:.3f}%")
        with train_writer.as_default():
            tf.summary.scalar('loss', loss, step=i)
            tf.summary.scalar('train_accuracy', train_acc, step=i)
            tf.summary.scalar('test_accuracy', test_acc, step=i)

# determine the test accuracy
logits = model(x_test, training=False)
max_idxs = tf.argmax(logits, axis=1)
acc = np.sum(max_idxs.numpy() == y_test) / len(y_test)
print("Final test accuracy is {:.2f}%".format(acc * 100))

Splits and slicing
https://www.tensorflow.org/datasets/splits
(cat_train, cat_valid, cat_test), info = tfds.load(‘cats_vs_dogs’, split=[‘train[:80%]’, ‘train[80%:90%]’, ‘train[-10%:]’], with_info=True, as_supervised=True)
train_10_80pct_ds = tfds.load(‘mnist’, split=‘train[:10%]+train[-80%:]’)

dataset address
https://www.tensorflow.org/datasets/catalog/cats_vs_dogs
https://www.tensorflow.org/datasets/api_docs/python/tfds/load