#!/usr/bin/env python
# coding: utf-8
# In[1]:
import os
os.environ["CUDA_VISIBLE_DEVICES"] = '0'
# In[2]:
import tensorflow as tf
# In[3]:
keras = tf.keras
layers = tf.keras.layers
# In[4]:
# 2.0.0-alpha0
# In[5]:
import matplotlib.pyplot as plt
# In[6]:
get_ipython().run_line_magic('matplotlib', 'inline')
# In[7]:
import numpy as np
# In[8]:
import glob
# In[9]:
import os
# In[10]:
print("Tensorflow version:{}".format(tf.__version__))
# In[11]:
total_image = glob.glob("./data/*")
# In[12]:
# 过滤不能加载的图片
# 单进程
# total_image_ = []
# l = len(total_image)
# # l = 1000
# for idx in range(l):
# try:
# image = tf.io.read_file(total_image[idx])
# tf.image.decode_jpeg(image, channels=3)
# total_image_.append(total_image[idx])
# except:
# print(idx)
# print(total_image[idx])
# In[13]:
# from multiprocessing import Process
# from multiprocessing import Queue
# q = Queue()
from multiprocessing import Pool
alist = []
def one_work(a, b):
alist = []
for idx in range(a,b):
if not total_image[idx].endswith(".jpg"):
continue
try:
image = tf.io.read_file(total_image[idx])
tf.image.decode_jpeg(image, channels=3)
alist.append(total_image[idx])
except:
pass
return alist
pool = Pool(processes=16)
l = len(total_image)
one = l//32 + 1
results = []
for idx in range(32):
results.append(pool.apply_async(one_work, (idx*one, min(idx*one+one,l))))
pool.close()
pool.join()
# In[14]:
len(results)
# In[15]:
total_image_ = []
for idx in range(len(results)):
result = results[idx]
total_image_ += result.get()
# In[16]:
len(total_image_)
# In