文章目录
![在这里插入图片描述](https://i-blog.csdnimg.cn/blog_migrate/8fdd97431b11d929041bda32947b30e4.png#pic_center)
1.数据处理
1)数据shuffle
2)TFRecord
3)TextLineDataset方式
4)from_generator方法
2.建模方法
例子
3.Baseline构建
数据处理
import json
import os
import datetime as dt
from tqdm import tqdm
import pandas as pd
import numpy as np
# label
def f2cat(filename: str) -> str:
return filename.split('.')[0]
# 读取文件,label变成list
class Simplified():
def __init__(self, input_path='./input'):
self.input_path = input_path
def list_all_categories(self):
files = os.listdir(self.input_path)
return sorted([f2cat(f) for f in files], key=str.lower)
def read_training_csv(self, category, nrows=None, usecols=None, drawing_transform=False):
df = pd.read_csv(os.path.join(self.input_path, category + '.csv'),
nrows=nrows, parse_dates=['timestamp'], usecols=usecols)
if drawing_transform:
df['drawing'] = df['drawing'].apply(json.loads)
return df
start = dt.datetime.now()
# 加载数据路径
s = Simplified('./data/')
NCSVS = 100
# 加载类别
categories = s.list_all_categories()
print(len(categories))
# tqdm显示进度
for y, cat in tqdm(enumerate(categories)):
df = s.read_training_csv(cat)
df['y'] = y
# 构建cv,知道数据属于哪个文件
df['cv'] = (df.key_id // 10 ** 7) % NCSVS # 100
# 将csv文件的1/100的数据写到里面,for循环
for k in range(NCSVS):
filename = './shuffle_data/train_k{}.csv'.format(k)
chunk = df[df.cv == k]
chunk = chunk.drop(['key_id'], axis=1)
if y == 0:
chunk.to_csv(filename, index=False)
else:
chunk.to_csv(filename, mode='a', header=False, index=False)
#对csv数据打乱,进行压缩
for k in tqdm(range(NCSVS)):
filename = './shuffle_data/train_k{}.csv'.format(k)
if os.path.exists(filename):
df = pd.read_csv(filename)
df['rnd'] = np.random.rand(len(df))
df = df.sort_values(by='rnd').drop('rnd', axis=1)
df.to_csv('./shuffle_data_gzip/train_k{}.csv.gz'.format(k), compression='gzip', index=False)
os.remove(filename)
print(df.shape)
end = dt.datetime.now()
print('Latest run {}.\nTotal time {}s'.format(end, (end - start).seconds))
数据读取
import glob
import tensorflow as tf
import numpy as np
import pandas as pd
import os
import cv2
import json
import matplotlib.pyplot as plt
os.environ['CUDA_VISIBLE_DEVICES']='1'
# 每一个csv.gz文件读取 -> y已