版本工具
Anaconda Navigator : 1.9.12
Spyder : 4.1.4
Python : 3.8
下载数据集
从keras内置直接载入
通过keras内置数据集的包直接载入即可:
#numpy 支持多维数组与矩阵运算
#pandas 基于numpy 为解决数据分析的创建
import random
import numpy as np
import pandas as pd
#import tensorflow as tf
from tensorflow.python.keras.utils.np_utils import to_categorical
import tensorflow.keras as keras
from matplotlib import pyplot as plt
#(x,y)测试集,(x_t,y_t)训练集
#(x_train_image,x_train_lable)=(6000,6000)
#(x_test_image,x_test_lable)=(1000,1000)
(x,y),(x_t,y_t) = keras.datasets.mnist.load_data()
# =============================================================================
# print("训练集:x=%s,y=%s"%(x.shape,y.shape))
# print("测试集:x_t=%s,y_t=%s"%(x_t.shape,y_t.shape))
#
# =============================================================================
查看数据集基本性质
基本操作,仅仅只是查看数据集的属性,此处定义了一个随机数和绘图函数
#输出随机pos开始的连续图像
# =============================================================================
# pos =random.randint(1,59990)
# def plot_image_labels(images,labels,idx,num=5):
# fig=plt.gcf()
#
# fig.set_size_inches(12,14)
#
# if num>5:
# num=5
# for i in range(0,num):
# ax=plt.subplot(1,5,1+i)
# ax.imshow(x[idx],cmap='binary')
# title='label='+str(y[idx])
# ax.set_title(title,fontsize=10)
# ax.set_xticks([])
# ax.set_yticks([])
# idx+=1
# plt.show()
# #训练集随机5张连续图像
# #测试集随机5张连续图像
# plot_image_labels(x,y,pos)
# plot_image_labels(x_t,y_t,pos)
# =============================================================================
预处理
降维、归一
#将二维图像转换为一维并归一化
x_train_image=x.reshape(60000,784).astype('float')
x_test_image=x_t.reshape(10000,784).astype('float')
x_train_image=x_train_image/255.0
x_test_image=x_test_image/255.0
print("原训练集大小:",x.shape)
print("现训练集大小:",x_train_image.shape)
print("原训练集样例:",x[0])
print("原训练集样例:",x_train_image[0])
标签独热编码
#将标签转化为独热编码向量组
y_trainOnehot =to_categorical(y)
y_testOnehot =to_categorical(y_t)
print("原训练标签大小",y.shape)
print("现训练标签大小",y_trainOnehot.shape)
print("原训练标签:",y[:10])
print("现训练标签:",y_trainOnehot[:10])
前半部分代码如下:
代码部分注释掉 请自己消化理解后分块取消注释即可实现
# -*- coding: utf-8 -*-
"""
Spyder Editor
"""
#numpy 支持多维数组与矩阵运算
#pandas 基于numpy 为解决数据分析的创建
import random
import numpy as np
import pandas as pd
#import tensorflow as tf
from tensorflow.python.keras.utils.np_utils import to_categorical
import tensorflow.keras as keras
from matplotlib import pyplot as plt
#(x,y)测试集,(x_t,y_t)训练集
#(x_train_image,x_train_lable)=(6000,6000)
#(x_test_image,x_test_lable)=(1000,1000)
(x,y),(x_t,y_t) = keras.datasets.mnist.load_data()
# =============================================================================
# print("训练集:x=%s,y=%s"%(x.shape,y.shape))
# print("测试集:x_t=%s,y_t=%s"%(x_t.shape,y_t.shape))
#
# =============================================================================
#输出随机pos开始的连续图像
# =============================================================================
# pos =random.randint(1,59990)
# def plot_image_labels(images,labels,idx,num=5):
# fig=plt.gcf()
#
# fig.set_size_inches(12,14)
#
# if num>5:
# num=5
# for i in range(0,num):
# ax=plt.subplot(1,5,1+i)
# ax.imshow(x[idx],cmap='binary')
# title='label='+str(y[idx])
# ax.set_title(title,fontsize=10)
# ax.set_xticks([])
# ax.set_yticks([])
# idx+=1
# plt.show()
# #训练集随机5张连续图像
# #测试集随机5张连续图像
# plot_image_labels(x,y,pos)
# plot_image_labels(x_t,y_t,pos)
# =============================================================================
#将二维图像转换为一维并归一化
x_train_image=x.reshape(60000,784).astype('float')
x_test_image=x_t.reshape(10000,784).astype('float')
x_train_image=x_train_image/255.0
x_test_image=x_test_image/255.0
print("原训练集大小:",x.shape)
print("现训练集大小:",x_train_image.shape)
print("原训练集样例:",x[0])
print("原训练集样例:",x_train_image[0])
#将标签转化为独热编码向量组
y_trainOnehot =to_categorical(y)
y_testOnehot =to_categorical(y_t)
print("原训练标签大小",y.shape)
print("现训练标签大小",y_trainOnehot.shape)
print("原训练标签:",y[:10])
print("现训练标签:",y_trainOnehot[:10])
数据集增强
通过对原数据的图像的像素点进行上下左右移动来扩大数据集
支持文件:因为需要对本地下载好的数据集进行操作,所以还需要自行下载mnist.pkl.gz文件夹
下载地址: https://pan.baidu.com/s/1o96x05xUUTs8c-8i1_YKFw
提取码: 4h9m
下载后,将文件放置在你的增强数据集python文件的同级目录,便于查找
后半部分代码如下
# -*- coding: utf-8 -*-
"""
Created on Mon Dec 21 20:01:03 2020
"""
from __future__ import print_function
import pickle
import gzip
import os.path
import random
import numpy as np
print("Expanding the MNIST training set")
if os.path.exists("../data/mnist_expanded.pkl.gz"):
print("The expanded training set already exists. Exiting.")
else:
f = gzip.open("./mnist.pkl.gz", 'rb')
training_data, validation_data, test_data = pickle.load(f,encoding="bytes")
f.close()
expanded_training_pairs = []
j = 0
for x, y in zip(training_data[0], training_data[1]):
expanded_training_pairs.append((x, y))
image = np.reshape(x, (-1, 28))
j += 1
if j % 1000 == 0: print("Expanding image number", j)
for d, axis, index_position, index in [
(1, 0, "first", 0),
(-1, 0, "first", 27),
(1, 1, "last", 0),
(-1, 1, "last", 27)]:
new_img = np.roll(image, d, axis)
if index_position == "first":
new_img[index, :] = np.zeros(28)
else:
new_img[:, index] = np.zeros(28)
expanded_training_pairs.append((np.reshape(new_img, 784), y))
random.shuffle(expanded_training_pairs)
expanded_training_data = [list(d) for d in zip(*expanded_training_pairs)]
print("Saving expanded data. This may take a few minutes.")
f = gzip.open("./mnist_expanded.pkl.gz", "w")
pickle.dump((expanded_training_data, validation_data, test_data), f)
f.close()
效果如下:
图片取样
通过tensorflow模块下载数据集后,取前20张图片并保存在MNIST_data/raw/ 文件夹下
# -*- coding: utf-8 -*-
"""
Created on Mon Dec 21 19:39:31 2020
"""
from PIL import Image
from tensorflow.examples.tutorials.mnist import input_data
import os
#读取MNIST数据集。如果不存在会事先下载。
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
# 我们把原始图片保存在MNIST_data/raw/文件夹下
# 如果没有这个文件夹会自动创建
save_dir = 'MNIST_data/raw/'
if os.path.exists(save_dir) is False:
os.makedirs(save_dir)
# 保存前20张图片
for i in range(20):
# 请注意,mnist.train.images[i, :]就表示第i张图片(序号从0开始)
image_array = mnist.train.images[i, :]
# TensorFlow中的MNIST图片是一个784维的向量,我们重新把它还原为28x28维的图像。
image_array = image_array.reshape(28, 28)
# 保存文件的格式为 mnist_train_0.jpg, mnist_train_1.jpg, ... ,mnist_train_19.jpg
filename = save_dir + 'mnist_train_%d.jpg' % i
# 将image_array保存为图片
# 先用scipy.misc.toimage转换为图像,再调用save直接保存。
Image.fromarray((image_array*255).astype('uint8'), mode='L').convert('RGB').save(filename)
#scipy.misc.toimage(image_array, cmin=0.0, cmax=1.0).save(filename)
print('Please check: %s ' % save_dir)
取样效果如下
图片处理原理
利用PIL下的Image/ImageEnhance处理
# -*- coding: utf-8 -*-
"""
Created on Mon Dec 21 21:59:17 2020
"""
from PIL import Image
from PIL import ImageEnhance
# =============================================================================
# enhancer =ImageEnhance.Sharpness()
# for i in range(8):
# factor =i/4.0
# enhancer.enhance(factor).show("Sharpness %f"% factor)
#
# =============================================================================
#图片路径前面添加r,避免python与正则表达式语法的冲突!
#原始图像
image = Image.open(r'C:\Users\dell\.spyder-py3\MNIST_data\raw\mnist_train_0.jpg',mode="r")
image.show()
#亮度增强
enh_bri = ImageEnhance.Brightness(image)
brightness = 1.5
image_brightened = enh_bri.enhance(brightness)
image_brightened.show()
#色度增强
enh_col = ImageEnhance.Color(image)
color = 1.5
image_colored = enh_col.enhance(color)
image_colored.show()
#对比度增强
enh_con = ImageEnhance.Contrast(image)
contrast = 1.5
image_contrasted = enh_con.enhance(contrast)
image_contrasted.show()
#锐度增强
enh_sha = ImageEnhance.Sharpness(image)
sharpness = 3.0
image_sharped = enh_sha.enhance(sharpness)
image_sharped.show()
图像亮度增强
定义存储路径字符串即可,其余部分都类似于前面存储图片部分
srcloc = r"C:\Users\dell\.spyder-py3\MNIST_data\raw\mnist_train_"+str(i)+".jpg"
# -*- coding: utf-8 -*-
"""
Created on Mon Dec 21 22:28:49 2020
"""
from PIL import Image
from PIL import ImageEnhance
from tensorflow.examples.tutorials.mnist import input_data
import os
import numpy as np
#读取MNIST数据集。如果不存在会事先下载。
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
# 我们把原始图片保存在MNIST_data/raw/文件夹下
# 如果没有这个文件夹会自动创建
save_dir = 'MNIST_data/brightness/'
if os.path.exists(save_dir) is False:
os.makedirs(save_dir)
# 保存前20张图片
for i in range(20):
srcloc = r"C:\Users\dell\.spyder-py3\MNIST_data\birghtness\mnist_train_"+str(i)+".jpg"
image_array =Image.open(srcloc,mode="r")
#增强亮度
enh_bri = ImageEnhance.Brightness(image_array)
brightness = 1.5
image_brightened = enh_bri.enhance(brightness)
a = np.asarray(image_brightened)
# 保存文件的格式为 mnist_trainbrightness_0.jpg, mnist_trainbrightness_1.jpg, ... ,mnist_trainbrightness_19.jpg
filename = save_dir + 'mnist_trainbrightness_%d.jpg' % i
# 将image_array保存为图片
# 先用scipy.misc.toimage转换为图像,再调用save直接保存。
Image.fromarray(a).save(filename)
print('Please check: %s ' % save_dir)
亮度增强效果
图像色度增强
# -*- coding: utf-8 -*-
"""
Created on Mon Dec 21 23:22:42 2020
"""
from PIL import Image
from PIL import ImageEnhance
from tensorflow.examples.tutorials.mnist import input_data
import os
import numpy as np
#读取MNIST数据集。如果不存在会事先下载。
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
# 我们把原始图片保存在MNIST_data/raw/文件夹下
# 如果没有这个文件夹会自动创建
save_dir = 'MNIST_data/color/'
if os.path.exists(save_dir) is False:
os.makedirs(save_dir)
# 保存前20张图片
for i in range(20):
srcloc = r"C:\Users\dell\.spyder-py3\MNIST_data\raw\mnist_train_"+str(i)+".jpg"
image_array =Image.open(srcloc,mode="r")
#色度增强
enh_col = ImageEnhance.Color(image_array)
color = 1.5
image_colored = enh_col.enhance(color)
a = np.asarray(image_colored)
filename = save_dir + 'mnist_traincolor_%d.jpg' % i
# 将image_array保存为图片
# 先用scipy.misc.toimage转换为图像,再调用save直接保存。
Image.fromarray(a).save(filename)
print('Please check: %s ' % save_dir)
色度增强效果
图像对比度增强
# -*- coding: utf-8 -*-
"""
Created on Mon Dec 21 23:22:42 2020
"""
from PIL import Image
from PIL import ImageEnhance
from tensorflow.examples.tutorials.mnist import input_data
import os
import numpy as np
#读取MNIST数据集。如果不存在会事先下载。
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
# 我们把原始图片保存在MNIST_data/raw/文件夹下
# 如果没有这个文件夹会自动创建
save_dir = 'MNIST_data/contrast/'
if os.path.exists(save_dir) is False:
os.makedirs(save_dir)
# 保存前20张图片
for i in range(20):
srcloc = r"C:\Users\dell\.spyder-py3\MNIST_data\raw\mnist_train_"+str(i)+".jpg"
image_array =Image.open(srcloc,mode="r")
#对比度增强
enh_con = ImageEnhance.Contrast(image_array)
contrast = 1.5
image_contrasted = enh_con.enhance(contrast)
a = np.asarray(image_contrasted)
filename = save_dir + 'mnist_traincontrast_%d.jpg' % i
# 将image_array保存为图片
# 先用scipy.misc.toimage转换为图像,再调用save直接保存。
Image.fromarray(a).save(filename)
print('Please check: %s ' % save_dir)
对比度增强效果
图像锐度增强
# -*- coding: utf-8 -*-
"""
Created on Tue Dec 22 00:11:08 2020
"""
from PIL import Image
from PIL import ImageEnhance
from tensorflow.examples.tutorials.mnist import input_data
import os
import numpy as np
#读取MNIST数据集。如果不存在会事先下载。
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
# 我们把原始图片保存在MNIST_data/raw/文件夹下
# 如果没有这个文件夹会自动创建
save_dir = 'MNIST_data/sharpness/'
if os.path.exists(save_dir) is False:
os.makedirs(save_dir)
# 保存前20张图片
for i in range(20):
srcloc = r"C:\Users\dell\.spyder-py3\MNIST_data\raw\mnist_train_"+str(i)+".jpg"
image_array =Image.open(srcloc,mode="r")
#锐度增强
enh_sha = ImageEnhance.Sharpness(image_array)
sharpness = 3.0
image_sharped = enh_sha.enhance(sharpness)
a = np.asarray(image_sharped)
filename = save_dir + 'mnist_trainsharpness_%d.jpg' % i
# 将image_array保存为图片
# 先用scipy.misc.toimage转换为图像,再调用save直接保存。
Image.fromarray(a).save(filename)
print('Please check: %s ' % save_dir)
锐度增强效果
踩坑点:很多函数及包由于版本不同,存在包名及函数名不同的问题,不兼容,需要更改!!!
参考文献
一个处女座的程序猿:https://blog.csdn.net/qq_41185868/article/details/79094752
DemonHunter211:
https://blog.csdn.net/kwame211/article/details/85999357