dataset应用,生成csv与读取解析

另一篇：
首先先学习Dateset API、利用API读csv文件，读取tfrecord文件
API：tf.data.Dataset.from_tensor_slices
repeat, batch, interleave, map, shuffle, list_files
读取csv文件：
tf.data.TextLineDataset, tf.io.decode_csv(解析)
使用tfrecord这个tf独有的文件格式时：
tf.train.FloatList, tf.train.Int64List, tf.train.BytesList
tf.train.Feature, tf.train.Features, tf.train.Example
example.SerializeToString(序列化)
tf.io.ParseSingleExample

import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import sklearn
import pandas as pd
import os
import sys
import time
import tensorflow as tf

from tensorflow import keras
from tensorflow.keras.callbacks import TensorBoard
for module in [mpl, np, pd, sklearn, tf, keras]:
    print(module.__name__,module.__version__)
"""
dataset = tf.data.Dataset.from_tensor_slices(np.arange(10))
print(dataset)

for item in dataset:
    print(item)

# 1.repeat epoch
# 2.get batch
dataset = dataset.repeat(3).batch(7)
#从0-10，一个batch7个数，一共遍历三次
for item in dataset:
    print(item)


# interleave:把处理的结果合并起来形成新的数据集
# case: 文件dataset ->具体数据集

dataset2 = dataset.interleave(
    lambda v: tf.data.Dataset.from_tensor_slices(v),   #map_fn做一个什么样的变化
    cycle_length = 5,  #cycle_length 并行程度
    block_length = 5,  #block_lengh 每次取多少结果

)
for item in dataset2:
    print("第三次",item)
#元组
x = np.array([[1,2],[3,4],[5,6]])
y = np.array(['cat','dog','fox'])
dataset3 = tf.data.Dataset.from_tensor_slices((x,y))
print("第四次",dataset3)


for item_x, item_y in dataset3:
    print("第五次",item_x.numpy(),item_y.numpy())
"""
#字典
x = np.array([[1,2],[3,4],[5,6]])
y = np.array(['cat','dog','fox'])
dataset4 = tf.data.Dataset.from_tensor_slices(
    {
        "feature":x,"label":y
    }
)
for item in dataset4:
    print("第六次",item["feature"].numpy(),item["label"].numpy())

二：
生成csv文件：
首先生成一个文件夹（这个代码常用）

output_dir = "generate_csv"
if not os.path.exists(output_dir):
    os.mkdir(output_dir)

设计一个函数实现:

#因为对x,y分别生成csv文件，所以要用name_prefix做区分
#把之前数据集切割成多个文件进行存储
def save_to_csv(output_dir, data, name_prefix,header = None,n_parts = 10):
    #用format生成文件名，免得文件太多找不到，第一个中括号是name_prefix,就是train,valid还是test,后面中括号存的是两位的一个整数
    path_format = os.path.join(output_dir,"{}_{:02d}.csv")
    filenames = []

    for file_idx, row_indices in enumerate(
        np.array_split(np.arange(len(data)),n_parts)):
        #上面array_spli意思是把len(data)按照n_parts等分
        part_csv = path_format.format(name_prefix,file_idx)
        filenames.append(part_csv)
        with open(part_csv,"wt",encoding="utf-8") as f:
            if header is not None:
                f.write(header + "\n")
            for row_index in row_indices:
                f.write(",".join([repr(col) for col in data[row_index]]))
                f.write('\n')

    return filenames

由于刚刚定义函数只有data,而不是按x,y分别输入，所以要对x,y进行merge
np.c_()可以把数据按行进行merge

train_data = np.c_[x_train_scaled, y_train]
valid_data = np.c_[x_valid_scaled, y_valid]
test_data = np.c_[x_test_scaled, y_test]
header_cols = housing.feature_names + ["MidianHouseValue"]
header_str = ",".join(header_cols)

这就是生成CSV文件的所有步骤

下面是读取我们生成的CSV文件然后拿它进行训练：

# 1.filename ->dataset
# 2.read file->dataset ->datasets ->merge
# 3.prase csv 解析
n_readers = 5
filename_dataset = tf.data.Dataset.list_files(train_filenames)
#上面是把文件名变成一个dataset
#下面是使用interleave的API,用来对filename_dataset每一个元素进行操作，然后将结果合并
#TextLineDataset是专门用于读取文本文件的api，按行读取文本行成dataset
#skip(1)每次读取不读第一行就是head
dataset = filename_dataset.interleave(
    lambda filename:tf.data.TextLineDataset(filename).skip(1),
    cycle_length = n_readers#并行度
)
# 查看数据
for line in dataset.take(15):
    print(line.numpy())

现在就是一堆九个数字的dataset，我们要对它进行解析：

# tf.io.decode_csv(str,record_defaults)例子1解码，record_defaults定义各个字符串类型
"""sample_str = '1,2,3,4,5'
record_defaults = [tf.constant(0,dtype=tf.int32),
                   0,
                   np.nan,
                   "hello",
                   tf.constant([])
                   ]
parsed_fields = tf.io.decode_csv(sample_str,record_defaults)
print(parsed_fields)
"""
#有两个参数，第一个是输入的字符串，第二个是期望的field
def parse_csv_line(line,n_fields = 9):
    defs = [tf.constant(np.nan)]*n_fields#即record
    parsed_fields = tf.io.decode_csv(line,record_defaults=defs)
    #我们知道前八个是x后一个是y,想转化成向量就用tf.stack
    x = tf.stack(parsed_fields[0:-1])
    y = tf.stack(parsed_fields[-1:])
    return x,y


"""
#测试一下函数成功了没
print(parse_csv_line(b'-0.47966389100153284,1.874166156711919,0.0560470563410166,'
               b'-0.006849812286680542,0.02944600829038973,-0.12115399093152514,'
               b'1.0338979434143465,-1.3407723682739239,2.895',n_fields=9)
      )
"""
#接下来定义一个完整的函数
# 1.filename ->dataset
# 2.read file->dataset ->datasets ->merge
# 3.prase csv 解析
def csv_reader_dataset(filenames,n_readers=5,batch_size = 32,
                       n_parse_threads=5,shuffle_buffer_size=10000#d打乱
                       ):
    dataset = tf.data.Dataset.list_files(filenames)#把filename变成dataset
    dataset = dataset.repeat()#重复无限次
    dataset = dataset.interleave(
        lambda filename: tf.data.TextLineDataset(filename).skip(1),
        cycle_length = n_readers)#把文件名转成文本内容
    dataset.shuffle(shuffle_buffer_size)#混排
    dataset = dataset.map(parse_csv_line,num_parallel_calls=n_parse_threads)
    #由于我们传进去的参数不是一个一个的而是按batch传进的
    dataset = dataset.batch(batch_size)
    return dataset
"""
# 举例子
train_set = csv_reader_dataset(train_filenames,batch_size = 3)
for x_batch,y_batch in train_set.take(2):
    print("x:")
    pprint.pprint(x_batch)
    print("y:")
    pprint.pprint(y_batch)
"""
batch_size = 32
train_set = csv_reader_dataset(train_filenames,
                               batch_size = batch_size)
valid_set = csv_reader_dataset(valid_filenames,
                               batch_size = batch_size)
test_set = csv_reader_dataset(test_filenames,
                               batch_size = batch_size)

# 训练集做完，训练model，进行fit
model = keras.models.Sequential([
    keras.layers.Dense(30, activation='relu',
                       input_shape=[8]),
    keras.layers.Dense(1),
])

model.compile(loss="mean_squared_error", optimizer="sgd")
callbacks = [keras.callbacks.EarlyStopping(
    patience=5, min_delta=1e-2)]
history = model.fit(train_set,
                    validation_data = valid_set,
                    #dateset一直产生数据，所以告诉它多少次是一个epoch
                    steps_per_epoch=11160//batch_size,
                    validation_steps=3870//batch_size,
                    epochs = 100,
                    callbacks = callbacks)
model.evaluate(test_set,steps=5160//batch_size)