opencv-keras停车位识别

最近在学习opencv，在学习唐宇迪老师的课程，将其中的一个停车位识别做一下笔记，该项目用到了opencv做图像处理，还用到了keras的cnn网络来学习图片做分类，我觉得是一个很与代表的小项目，故做一个完整的笔记。
首先，我们应该下来明确一下思路，我们要做什么（目的），目的是为了什么，然后怎么做（方法），我们下来看一下停车场的图片

可以看到这个类似于卫星图的照片，首先我们看到停车场是个多边形的，里面有很多停车位，停车位是由很多横线组成的。有的体内停车位是空着的，有的停车位是停了车的，我们需要将空着的停车位做出标记，这样就可以很快找到可用的停车位置，直接过去停车就行了，停车场一般都很大，不可能在车里靠我们的眼睛就能观察完整个停车场，这样，我们就不用一排一排的找了，我们所清楚了目的，那么我们就要明确该怎么做。

1. 图像预处理（opencv）

在拿到停车场图片的时候，我们首要的任务就是知道停车场的情况，比如，

• 停车场的轮廓
• 停车场有多少停车位
• 这些停车位怎么布局
• 停车位的位置信息

1.1 图像背景过滤

停车场的图像是彩色的，里面除了停车场的信息，还有很多无用的信息，还有很多背景，首先我们需要将背景过滤掉
我们将用到cv2.inRange 和 cv2.bitwise_and函数
代码如下

导入资源

import matplotlib.pyplot as plt
import cv2
import os, glob
import numpy as np
import pickle

图像显示函数

def show_images(images, cmap=None):
    cols = 2
    rows = (len(images)+1)//cols
    plt.figure(figsize=(15, 12))
    for i, image in enumerate(images):
        plt.subplot(rows, cols, i+1)
        cmap = "gray" if len(image.shape) == 2 else cmap
        plt.imshow(image, cmap=cmap)
        plt.xticks([])
        plt.yticks([])
    # 自动调整子图之间的参数（距离，标签距离）
    plt.tight_layout(pad=0, h_pad=0, w_pad=0)
    plt.show()
    
def cv_show(name, img):
    cv2.imshow(name, img)
    cv2.waitKey(0)
    cv2.destroyWindow(name)

def select_rgb_white_yellow(image):
    # 过滤背景
    lower = np.uint8([120, 120, 120])
    upper = np.uint8([255, 255, 255])
    # 低于lower_red 和高于uppper_red的部分都变成0， 之间的数字变成255，相当于过滤掉背景
    white_mask = cv2.inRange(image, lower, upper)
    cv_show("white_mask", white_mask)
    
    masked = cv2.bitwise_and(image, image, mask = white_mask)
    cv_show("masked", masked)
    return masked

# 获取停车场图像
test_images = [plt.imread(path) for path in glob.glob("test_images/*.jpg")]
show_images(test_images)

white_yellow_images = list(map(select_rgb_white_yellow, test_images))
show_images(white_yellow_images)

效果如下

1.2 图片灰度处理

接下来我们需要将彩色图片转换为灰度图片，

def convert_gray_scale(image):
    return cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)
    
gray_images = list(map(convert_gray_scale, white_yellow_images))
show_images(gray_images)

1.3 边缘检测

我们要找到停车位和停车张轮廓，必须得先找到边缘，只有找到边缘了，才好确定轮廓

def detect_edges(image, low_threshold=50, high_threshold=200):
   return cv2.Canny(image, low_threshold, high_threshold)
edge_images = list(map(lambda image: detect_edges(image), gray_images))
show_images(edge_images)
cv_show("dege", edge_images[0])

效果如下，当然效果还可以优化

1.4 裁剪停车场有效区域

在找到了停车场的边缘后，我们这里确定停车场的轮廓，去除一些其他区域，毕竟，停车场区域才是我们关心的，在代码里，可以通过手动找到停车场的定点坐标，根据坐标来画出轮廓，然后跟原图进行与操作，就得到了停车场轮廓

def filter_region(image, vertices):
    mask = np.zeros_like(image)
    if len(mask.shape) == 2:
    	//根据vertices对全0矩阵填充255，形成多边形
        cv2.fillPoly(mask, vertices, 255)
        cv_show("mask", mask)
    return cv2.bitwise_and(image, mask)
def select_region(image):
    rows, cols = image.shape[:2]
    print("image shape is rows = %d, cols = %d" %(rows, cols))
    pt_1  = [cols*0.05, rows*0.90]
    pt_2 = [cols*0.05, rows*0.70]
    pt_3 = [cols*0.30, rows*0.55]
    pt_4 = [cols*0.6, rows*0.15]
    pt_5 = [cols*0.90, rows*0.15] 
    pt_6 = [cols*0.90, rows*0.90]
    vertices = np.array([[pt_1, pt_2, pt_3, pt_4, pt_5, pt_6]], dtype=np.int32)
    print(vertices)                   
    point_img = image.copy()
    point_img = cv2.cvtColor(point_img, cv2.COLOR_GRAY2RGB)
    for point in vertices[0]:
        cv2.circle(point_img, (point[0], point[1]), 5, (0,0,255), 4)
    cv_show("point_image", point_img)
    return filter_region(image, vertices)

roi_images = list(map(select_region, edge_images))
show_images(roi_images)

效果如下

1.5 查找直线（霍夫变换）

在得到停车场的轮廓和边缘后，我们需要将边缘的像素点连成线，这就用到了霍夫变换，cv2.HoughLinesP

def hough_lines(image):
    """
    输入的图像是边沿检测过的结果
    minLineLengh: 线的最短长度，比这个短的要忽略掉
    maxLineCap: 两条线直线的最大间隔，小于这个值，认为是一条线
    rho：距离精度
    theta：角度精度
    threshod：超过设定阀值才被检测出线段
    """
    return cv2.HoughLinesP(image, rho=0.1, theta=np.pi/10, threshold=15, minLineLength=9, maxLineGap=4)

def draw_lines(image, lines, color=[255, 0, 0], thickness=2, make_copy=True):
    # 过滤霍夫检测到的直线
    if make_copy:
        image = np.copy(image)
    cleaned = []
    for line in lines:
        for x1, y1, x2, y2 in line:
            if abs(y2-y1) <=1 and abs(x2-x1) >=25 and abs(x2-x1) <=55:
                cleaned.append(( x1, y1, x2, y2))
                cv2.line(image, (x1, y1), (x2, y2), color, thickness)
    print("lines detected: ", len(cleaned))
    
    return image
# 找线段
list_of_lines = list(map(hough_lines, roi_images))
# 画线段
line_images = []
for image, lines in zip(test_images, list_of_lines):
    line_images.append(draw_lines(image, lines))
show_images(line_images)
cv_show("line", line_images[1])

效果如下

1.6 停车区域划分

在找到停车位的线后，我们可以观察到停车场的车位是一列一列排序，我们要确定每一个列的位置
我们将画出每一列的方框，现将所有直线排序，然后根据每一个横线的左边的x1为依据分类，因为属于同一列的x1相差不大，而分别属于两列的x1就相差很大了。方框的左上角的y坐标就是每一列中第一条直线的y1坐标，右下角的y坐标就是每一列最后一条直线的y2坐标

def identify_blocks(image, lines, make_copy=True):
    if make_copy:
        new_image = np.copy(image)
        # step 1: 过滤部分直线
        cleaned = []
        for line in lines:
             for x1, y1, x2, y2 in line:
                if abs(y2-y1) <=1 and abs(x2-x1) >=25 and abs(x2-x1) <=55:
                    cleaned.append(( x1, y1, x2, y2))
        # Step 2: 对直线按照x1排序
        import operator
        list1 = sorted(cleaned, key=operator.itemgetter(0, 1))
        
        # Step3: 找到多个列，相当于每列是一排车
        clusers = {}
        dIndex = 0
        # 不同类之间两条线的距离
        clus_dist = 10
        for i in range(len(list1) -1):
            distance = abs(list1[i+1][0] - list1[i][0])
            if distance <= clus_dist:
                if not dIndex in clusers.keys() :clusers[dIndex] = []
                clusers[dIndex].append(list1[i])
                clusers[dIndex].append(list1[i +1])
            else:
                dIndex += 1
                
        # Step 4: 得到坐标
        rects = {}
        i = 0
        for key in clusers:
            all_list = clusers[key]
            cleaned = list(set(all_list))
            if len(cleaned) > 5:
                cleaned = sorted(cleaned, key=lambda tup: tup[1])
                avg_y1 = cleaned[0][1]
                avg_y2 = cleaned[-1][1]
                avg_x1 = 0
                avg_x2 = 0
                for tup in cleaned:
                    avg_x1 += tup[0]
                    avg_x2 += tup[2]
                avg_x1 = avg_x1/len(cleaned)
                avg_x2 = avg_x2/len(cleaned)
                rects[i] = (avg_x1, avg_y1, avg_x2, avg_y2)
                i +=1
                
        print("num Parking Lanes: ", len(rects))
        # Step 5: 画出矩形框
        buff = 7
        for key in rects:
            tup_topLeft = (int(rects[key][0] - buff), int(rects[key][1]))
            tup_botRight = (int(rects[key][2] + buff), int(rects[key][3]))
            cv2.rectangle(new_image, tup_topLeft, tup_botRight, (0,255,0), 3)
            
      return new_image, rects
      
rect_images = []
rect_coords = []
for image, lines in zip(test_images, list_of_lines):
    new_image, rects = identify_blocks(image, lines)
    rect_images.append(new_image)
    rect_coords.append(rects)
show_images(rect_images)

效果如下，第二张效果要好一写

1.7 画出停车位（获取每个停车位坐标）

在确定了停车的列，然后就要确定每个停车位的坐标

def draw_parking(image, rects, make_copy = True, color=[255, 0, 0], thickness=2, save = True):
        if make_copy:
            new_image = np.copy(image)
        gap = 15.5
        spot_dict = {} # 字典：一个车位对应一个位置
        tot_spots = 0
        #微调
        adj_y1 = {0: 20, 1:-10, 2:0, 3:-11, 4:28, 5:5, 6:-15, 7:-15, 8:-10, 9:-30, 10:9, 11:-32}
        adj_y2 = {0: 30, 1: 50, 2:15, 3:10, 4:-15, 5:15, 6:15, 7:-20, 8:15, 9:15, 10:0, 11:30}
        
        adj_x1 = {0: -8, 1:-15, 2:-15, 3:-15, 4:-15, 5:-15, 6:-15, 7:-15, 8:-10, 9:-10, 10:-10, 11:0}
        adj_x2 = {0: 0, 1: 15, 2:15, 3:15, 4:15, 5:15, 6:15, 7:15, 8:10, 9:10, 10:10, 11:0}
        for key in rects:
            tup = rects[key]
            x1 = int(tup[0]+ adj_x1[key])
            x2 = int(tup[2]+ adj_x2[key])
            y1 = int(tup[1] + adj_y1[key])
            y2 = int(tup[3] + adj_y2[key])
            cv2.rectangle(new_image, (x1, y1),(x2,y2),(0,255,0),2)
            num_splits = int(abs(y2-y1)//gap)
            for i in range(0, num_splits+1):
                y = int(y1 + i*gap)
                cv2.line(new_image, (x1, y), (x2, y), color, thickness)
            if key > 0 and key < len(rects) -1 :        
                #竖直线
                x = int((x1 + x2)/2)
                cv2.line(new_image, (x, y1), (x, y2), color, thickness)
            # 计算数量
            if key == 0 or key == (len(rects) -1):
                tot_spots += num_splits +1
            else:
                tot_spots += 2*(num_splits +1)
                
            # 字典对应好
            if key == 0 or key == (len(rects) -1):
                for i in range(0, num_splits+1):
                    cur_len = len(spot_dict)
                    y = int(y1 + i*gap)
                    spot_dict[(x1, y, x2, y+gap)] = cur_len +1        
            else:
                for i in range(0, num_splits+1):
                    cur_len = len(spot_dict)
                    y = int(y1 + i*gap)
                    x = int((x1 + x2)/2)
                    spot_dict[(x1, y, x, y+gap)] = cur_len +1
                    spot_dict[(x, y, x2, y+gap)] = cur_len +2   
        
        print("total parking spaces: ", tot_spots, cur_len)
        if save:
            filename = 'with_parking.jpg'
            cv2.imwrite(filename, new_image)
        return new_image, spot_dict

delineated = []
spot_pos = []
for image, rects in zip(test_images, rect_coords):
    new_image, spot_dict = draw_parking(image, rects)
    delineated.append(new_image)
    spot_pos.append(spot_dict)

show_images(delineated)
cv_show("parking", delineated[0])
final_spot_dict = spot_pos[0]
print(len(final_spot_dict))  
print(final_spot_dict)

效果如下，第一张效果要好一些

1.8 保存车位坐标

# 保存
 with open('spot_dict.pickle', 'wb') as handle:
    pickle.dump(final_spot_dict, handle, protocol=pickle.HIGHEST_PROTOCOL)
# 读取
final_spot_dict = {}
with open('spot_dict.pickle', 'rb') as fb:
    final_spot_dict = pickle.load(fb)  
print(final_spot_dict)

1.9 保存CNN训练图片

def save_images_for_cnn(image, spot_dict, folder_name = 'cnn_data'):
    for spot in spot_dict.keys():
        (x1, y1, x2, y2) = spot
        (x1, y1, x2, y2) = (int(x1), int(y1), int(x2), int(y2))
        
        # 图片裁剪
        spot_img = image[y1:y2, x1:x2]
        # 放大两倍
        spot_img = cv2.resize(spot_img, (0,0), fx=2.0, fy = 2.0)
        spot_id= spot_dict[spot]
        filename = 'spot'+str(spot_id)+'.jpg'
        print(spot_img.shape, filename, (x1, x2, y1, y2))
        cv2.imwrite(os.path.join(folder_name, filename), spot_img)

save_images_for_cnn(test_images[0], final_spot_dict)

2.图像学习（keras）

这一步我们使用keras来训练模型，我们已经有了每个停车位的坐标，利用这些坐标来截取输入图片，然后传入模型中进行预测该车位是占用的还是空闲的

import numpy as np
import os
from keras import applications
from keras.preprocessing.image import ImageDataGenerator
from keras import optimizers
from keras.models import Model
from keras.callbacks import ModelCheckpoint, EarlyStopping
from keras.layers.core import Flatten
from keras.layers.core import Dense
from keras.models import load_model

2.1 加载数据

files_train = 0
files_validation = 0

# 返回当前进程的工作目录
cwd = os.getcwd()
folder = 'train_data/train'
for sub_folder in os.listdir(folder):
    path, dirs, files = next(os.walk(os.path.join(folder, sub_folder)))
    files_train += len(files)
    
folder = 'train_data/test'
for sub_folder in os.listdir(folder):
    path, dirs, files = next(os.walk(os.path.join(folder,sub_folder)))
    files_validation += len(files)
print(files_train, files_validation)

2.2加载模型

img_width, img_height = 48, 48
train_data_dir = "train_data/train"
validation_data_dir = "train_data/test"
nb_train_samples = files_train
nb_validation_samples = files_validation

batch_size = 32
epochs  = 15
num_classes = 2

model = applications.VGG16(weights='imagenet', include_top=False, input_shape = (img_width, img_height, 3))
for layer in model.layers[:10]:
    layer.trainable = False
    
x = model.output
x = Flatten()(x)
predictions = Dense(num_classes, activation="softmax")(x)
model_final = Model(input=model.input, output=predictions)
model_final.compile(loss = "categorical_crossentropy", 
                    optimizer = optimizers.SGD(lr=0.0001, momentum=0.9), 
                    metrics=["accuracy"]) 

2.3数据增强

train_datagen = ImageDataGenerator(
rescale = 1./255,
horizontal_flip = True,
fill_mode = "nearest",
zoom_range = 0.1,
width_shift_range = 0.1,
height_shift_range=0.1,
rotation_range=5)

test_datagen = ImageDataGenerator(
rescale = 1./255,
horizontal_flip = True,
fill_mode = "nearest",
zoom_range = 0.1,
width_shift_range = 0.1,
height_shift_range=0.1,
rotation_range=5)

train_generator = train_datagen.flow_from_directory(
train_data_dir,
target_size = (img_height, img_width),
batch_size = batch_size,
class_mode = "categorical")

validation_generator = test_datagen.flow_from_directory(
validation_data_dir,
target_size = (img_height, img_width),
class_mode = "categorical")

2.4添加回调函数

checkpoint = ModelCheckpoint("car1.h5", monitor='val_acc', verbose=1, save_best_only=True, 
                             save_weights_only=False, mode='auto', period=1)
early = EarlyStopping(monitor='val_acc', min_delta=0, patience=10, verbose=1, mode='auto')

2.4训练数据

history_object = model_final.fit_generator(train_generator,
	samples_per_epoch = nb_train_samples,
	epochs = epochs,
	validation_data = validation_generator,
	nb_val_samples = nb_validation_samples,
	callbacks = [checkpoint, early])

2.5预测

def make_prediction(image, model, class_dictionary):
    img = image/255.
    image = np.expand_dims(img, axis=0)
    class_predicted = model.predict(image)
    inID = np.argmax(class_predicted[0])
    label = class_dictionary[inID]
    
    return label
def keras_model(weights_path):
    model = load_model(weights_path)
    return model

weights_path = 'car1.h5'
video_name = 'parking_video.mp4'
class_dictionary = {}
class_dictionary[0] = 'empty'
class_dictionary[1] = 'occupied'

model = keras_model(weights_path)


def predict_on_video(video_name,final_spot_dict, model,class_dictionary,ret=True): 
    cap = cv2.VideoCapture(video_name)
    count = 0
    while ret:
        ret, image = cap.read()
        count +=1
        if count == 5:
            count = 0
            new_image = np.copy(image)
            overlay = np.copy(image)
            cnt_empty = 0
            all_spots = 0
            color = [0, 255, 255]
            alpha = 0.5
            for spot in final_spot_dict.keys():
                all_spots += 1
                (x1, y1, x2, y2) = spot
                (x1, y1, x2, y2) = (int(x1), int(y1), int(x2), int(y2))
                spot_img = image[y1:y2, x1:x2]
                spot_img = cv2.resize(spot_img, (48, 48))
                label = make_prediction(spot_img,model,class_dictionary)
                if label == 'empty':
                    cv2.rectangle(overlay, (int(x1),int(y1)), (int(x2),int(y2)), color, -1)
                    cnt_empty += 1
            # 叠加图片
            cv2.addWeighted(overlay, alpha, new_image, 1 - alpha, 0, new_image)
            cv2.putText(new_image, "Available: %d spots" %cnt_empty, (30, 95), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255, 255, 255), 2)
            cv2.putText(new_image, "Total: %d spots" %all_spots, (30, 125),cv2.FONT_HERSHEY_SIMPLEX,0.7, (255, 255, 255), 2)
            cv2.imshow('frame', new_image)
            if cv2.waitKey(10) & 0xFF == ord('q'):
                break
    cv2.destroyAllWindows()
    cap.release() 

def video_test(video_name, final_spot_dict, model, class_dictionary):
    name = video_name
    #cap = cv2.VideoCapture(name)
    predict_on_video(name,final_spot_dict,model, class_dictionary)

video_test(video_name,final_spot_dict,model,class_dictionary)