.h5文件转.pb文件(windows+Keras+Yolov3训练自己数据集)细节处理。

一、权重与网络结构转换的环境:

1.win10+cuda9.0

2.tensorflow-gpu==1.10;tensorflow==1.12

3.keras==2.2;python==3.5

二、Keras保存权重有两种方式:

#方式一:权重的参数和网络结构分开保存
    1,保存网络结构到json文件
     model_json=model.to_json()

        with open("mosel.json") as file:
            file.write(model_json)
    2,保存权重参数到h5文件
     model.save_weights('trained.h5')

#方式二:权重参数和网络结构一起保存到h5文件
    model.save('trained.h5')
     

   项目源码参考:https://github.com/qqwweee/keras-yolo3,权重保存方式修改在train.py文件中,个人倾向使用save()方式,第一种方式model.to_json()保存尚未解决;

三、.h5文件转成.pb文件:

       方式1:.h5转weights,具体代码参考:h5_to_weights.py ,参考链接:https://github.com/amir-abdi/keras_to_tensorflow.git

import argparse
import numpy
import numpy as np
import keras
from keras.models import load_model
from keras import backend as K
#从yolo3中导入网络结构
from yolo3.model import yolo_head,yolo_body,box_iou
import tensorflow as tf

def parser():
    parser = argparse.ArgumentParser(description="Darknet\'s yolov3.cfg and yolov3.weights \
                                      converted into Keras\'s yolov3.h5!")
    # 修改成自己文件的路径
    parser.add_argument('-cfg_path', default="./yolov3.cfg",help='yolov3.cfg')
    parser.add_argument('-h5_path', default="./trained.h5",help='yolov3.h5')
    parser.add_argument('-output_path', default="./train.weights",help='yolov3.weights')
    return parser.parse_args()


class WeightSaver(object):

    def __init__(self, h5_path, output_path):
        #self.load=yolo_body(Input(shape=(None, None, 3)), 3, num_classes)
        # 网络结构加入
        self.model = load_model(h5_path,custom_objects={"yolo_head":yolo_head,'yolo_body':yolo_body,'tf': tf,'box_iou':box_iou},'<lambda>': lambda y_true, output: output)
        # 如果要读取keras调用save_weights的h5文件,可以先读取一次save的h5,
        # 然后取消下面的注释,读取save_weights的h5
        #        self.model.load_weights('text.h5')
        self.layers = {weight.name: weight for weight in self.model.weights}
        self.sess = K.get_session()
        self.fhandle = open(output_path, 'wb')
        self._write_head()

    def _write_head(self):
        numpy_data = numpy.ndarray(shape=(3,),
                                   dtype='int32',
                                   buffer=np.array([0, 2, 0], dtype='int32'))
        self.save(numpy_data)
        numpy_data = numpy.ndarray(shape=(1,),
                                   dtype='int64',
                                   buffer=np.array([320000], dtype='int64'))
        self.save(numpy_data)

    def get_bn_layername(self, num):
        layer_name = 'batch_normalization_{num}'.format(num=num)
        bias = self.layers['{0}/beta:0'.format(layer_name)]
        scale = self.layers['{0}/gamma:0'.format(layer_name)]
        mean = self.layers['{0}/moving_mean:0'.format(layer_name)]
        var = self.layers['{0}/moving_variance:0'.format(layer_name)]

        bias_np = self.get_numpy(bias)
        scale_np = self.get_numpy(scale)
        mean_np = self.get_numpy(mean)
        var_np = self.get_numpy(var)
        return bias_np, scale_np, mean_np, var_np

    def get_convbias_layername(self, num):
        layer_name = 'conv2d_{num}'.format(num=num)
        bias = self.layers['{0}/bias:0'.format(layer_name)]

        bias_np = self.get_numpy(bias)
        return bias_np

    def get_conv_layername(self, num):
        layer_name = 'conv2d_{num}'.format(num=num)
        conv = self.layers['{0}/kernel:0'.format(layer_name)]

        conv_np = self.get_numpy(conv)
        return conv_np

    def get_numpy(self, layer_name):
        numpy_data = self.sess.run(layer_name)
        return numpy_data

    def save(self, numpy_data):
        bytes_data = numpy_data.tobytes()
        self.fhandle.write(bytes_data)
        self.fhandle.flush()

    def close(self):
        self.fhandle.close()


class KerasParser(object):

    def __init__(self, cfg_path, h5_path, output_path):
        self.block_gen = self._get_block(cfg_path)
        self.weights_saver = WeightSaver(h5_path, output_path)
        self.count_conv = 0
        self.count_bn = 0

    def _get_block(self, cfg_path):

        block = {}
        with open(cfg_path, 'r', encoding='utf-8') as fr:
            for line in fr:
                line = line.strip()
                if '[' in line and ']' in line:
                    if block:
                        yield block
                    block = {}
                    block['type'] = line.strip(' []')
                elif not line or '#' in line:
                    continue
                else:
                    key, val = line.strip().replace(' ', '').split('=')
                    key, val = key.strip(), val.strip()
                    block[key] = val

            yield block

    def close(self):
        self.weights_saver.close()

    def conv(self, block):
        self.count_conv += 1
        batch_normalize = 'batch_normalize' in block
        print('handing.. ', self.count_conv)

        # 如果bn存在,则先处理bn,顺序为bias,scale,mean,var
        if batch_normalize:
            bias, scale, mean, var = self.bn()
            self.weights_saver.save(bias)

            scale = scale.reshape(1, -1)
            mean = mean.reshape(1, -1)
            var = var.reshape(1, -1)
            remain = np.concatenate([scale, mean, var], axis=0)
            self.weights_saver.save(remain)

        # 否则,先处理biase
        else:
            conv_bias = self.weights_saver.get_convbias_layername(self.count_conv)
            self.weights_saver.save(conv_bias)

        # 接着处理weights
        conv_weights = self.weights_saver.get_conv_layername(self.count_conv)
        # 需要将(height, width, in_dim, out_dim)转换成(out_dim, in_dim, height, width)
        conv_weights = np.transpose(conv_weights, [3, 2, 0, 1])
        self.weights_saver.save(conv_weights)

    def bn(self):
        self.count_bn += 1
        bias, scale, mean, var = self.weights_saver.get_bn_layername(self.count_bn)
        return bias, scale, mean, var


def main():
    args = parser()
    keras_loader = KerasParser(args.cfg_path, args.h5_path, args.output_path)

    for block in keras_loader.block_gen:
        if 'convolutional' in block['type']:
            keras_loader.conv(block)
    keras_loader.close()


if __name__ == "__main__":
    main()

注意点:将文件放置在keras_yolo3项目下,遇到问题点1:“yolo_head is not defind”,或者是“tf is not defind ”...在

self.model = load_model(h5_path,custom_objects={"yolo_head":yolo_head,'yolo_body':yolo_body,'tf': tf,'box_iou':box_iou,'<lambda>': lambda y_true, output: output})中把缺少的东西加进去。

然后运行:python h5_to_weights.py

之后生成的.weights文件大概200多兆: buffer is too small for requested array

未完待续........

要将 YOLOv3 的权重文件换为 Keras 可以使用的 .h5 文件,需要进行以下步骤: 1. 下载并安装 `keras` 和 `tensorflow` 库: ``` pip install keras tensorflow ``` 2. 下载 YOLOv3 的权重文件 `yolov3.weights` 和配置文件 `yolov3.cfg`。 3. 使用 `yolo_weights_convert.py` 脚本将权重文件换为 Keras 模型: ``` python yolo_weights_convert.py yolov3.cfg yolov3.weights model_data/yolo.h5 ``` 其中,`yolov3.cfg` 是 YOLOv3 的配置文件路径,`yolov3.weights` 是权重文件路径,`model_data/yolo.h5` 是换后的 Keras 模型保存路径。 以下是 `yolo_weights_convert.py` 的代码: ```python import argparse import numpy as np import struct import os from keras.layers import Conv2D, Input, ZeroPadding2D, BatchNormalization, LeakyReLU, UpSampling2D from keras.layers.merge import add, concatenate from keras.models import Model from keras.engine.topology import Layer from keras import backend as K class YoloLayer(Layer): def __init__(self, anchors, max_grid, batch_size, warmup_batches, ignore_thresh, grid_scale, obj_scale, noobj_scale, xywh_scale, class_scale, **kwargs): self.ignore_thresh = ignore_thresh self.warmup_batches = warmup_batches self.anchors = anchors self.grid_scale = grid_scale self.obj_scale = obj_scale self.noobj_scale = noobj_scale self.xywh_scale = xywh_scale self.class_scale = class_scale self.batch_size = batch_size self.true_boxes = K.placeholder(shape=(self.batch_size, 1, 1, 1, 50, 4)) super(YoloLayer, self).__init__(**kwargs) def build(self, input_shape): super(YoloLayer, self).build(input_shape) def get_grid_size(self, net_h, net_w): return net_h // 32, net_w // 32 def call(self, x): input_image, y_pred, y_true = x self.net_h, self.net_w = input_image.shape.as_list()[1:3] self.grid_h, self.grid_w = self.get_grid_size(self.net_h, self.net_w) # adjust the shape of the y_predict [batch, grid_h, grid_w, 3, 4+1+80] y_pred = K.reshape(y_pred, (self.batch_size, self.grid_h, self.grid_w, 3, 4 + 1 + 80)) # convert the coordinates to absolute coordinates box_xy = K.sigmoid(y_pred[..., :2]) box_wh = K.exp(y_pred[..., 2:4]) box_confidence = K.sigmoid(y_pred[..., 4:5]) box_class_probs = K.softmax(y_pred[..., 5:]) # adjust the shape of the y_true [batch, 50, 4+1] object_mask = y_true[..., 4:5] true_class_probs = y_true[..., 5:] # true_boxes[..., 0:2] = center, true_boxes[..., 2:4] = wh true_boxes = self.true_boxes[..., 0:4] # shape=[batch, 50, 4] true_xy = true_boxes[..., 0:2] * [self.grid_w, self.grid_h] # shape=[batch, 50, 2] true_wh = true_boxes[..., 2:4] * [self.net_w, self.net_h] # shape=[batch, 50, 2] true_wh_half = true_wh / 2. true_mins = true_xy - true_wh_half true_maxes = true_xy + true_wh_half # calculate the Intersection Over Union (IOU) pred_xy = K.expand_dims(box_xy, 4) pred_wh = K.expand_dims(box_wh, 4) pred_wh_half = pred_wh / 2. pred_mins = pred_xy - pred_wh_half pred_maxes = pred_xy + pred_wh_half intersect_mins = K.maximum(pred_mins, true_mins) intersect_maxes = K.minimum(pred_maxes, true_maxes) intersect_wh = K.maximum(intersect_maxes - intersect_mins, 0.) intersect_areas = intersect_wh[..., 0] * intersect_wh[..., 1] pred_areas = pred_wh[..., 0] * pred_wh[..., 1] true_areas = true_wh[..., 0] * true_wh[..., 1] union_areas = pred_areas + true_areas - intersect_areas iou_scores = intersect_areas / union_areas # calculate the best IOU, set the object mask and update the class probabilities best_ious = K.max(iou_scores, axis=4) object_mask_bool = K.cast(best_ious >= self.ignore_thresh, K.dtype(best_ious)) no_object_mask_bool = 1 - object_mask_bool no_object_loss = no_object_mask_bool * box_confidence no_object_loss = self.noobj_scale * K.mean(no_object_loss) true_box_class = true_class_probs * object_mask true_box_confidence = object_mask true_box_xy = true_boxes[..., 0:2] * [self.grid_w, self.grid_h] - pred_mins true_box_wh = K.log(true_boxes[..., 2:4] * [self.net_w, self.net_h] / pred_wh) true_box_wh = K.switch(object_mask, true_box_wh, K.zeros_like(true_box_wh)) # avoid log(0)=-inf true_box_xy = K.switch(object_mask, true_box_xy, K.zeros_like(true_box_xy)) # avoid log(0)=-inf box_loss_scale = 2 - true_boxes[..., 2:3] * true_boxes[..., 3:4] xy_loss = object_mask * box_loss_scale * K.binary_crossentropy(true_box_xy, box_xy) wh_loss = object_mask * box_loss_scale * 0.5 * K.square(true_box_wh - box_wh) confidence_loss = true_box_confidence * K.binary_crossentropy(box_confidence, true_box_confidence) \ + (1 - true_box_confidence) * K.binary_crossentropy(box_confidence, true_box_confidence) \ * no_object_mask_bool class_loss = object_mask * K.binary_crossentropy(true_box_class, box_class_probs) xy_loss = K.mean(K.sum(xy_loss, axis=[1, 2, 3, 4])) wh_loss = K.mean(K.sum(wh_loss, axis=[1, 2, 3, 4])) confidence_loss = K.mean(K.sum(confidence_loss, axis=[1, 2, 3, 4])) class_loss = K.mean(K.sum(class_loss, axis=[1, 2, 3, 4])) loss = self.grid_scale * (xy_loss + wh_loss) + confidence_loss * self.obj_scale + no_object_loss \ + class_loss * self.class_scale # warm up training batch_no = K.cast(self.batch_size / 2, dtype=K.dtype(object_mask)) warmup_steps = self.warmup_batches warmup_lr = batch_no / warmup_steps batch_no = K.cast(K.minimum(warmup_steps, batch_no), dtype=K.dtype(object_mask)) lr = self.batch_size / (batch_no * warmup_steps) warmup_decay = (1 - batch_no / warmup_steps) ** 4 lr = lr * (1 - warmup_decay) + warmup_lr * warmup_decay self.add_loss(loss) self.add_metric(loss, name='loss', aggregation='mean') self.add_metric(xy_loss, name='xy_loss', aggregation='mean') self.add_metric(wh_loss, name='wh_loss', aggregation='mean') self.add_metric(confidence_loss, name='confidence_loss', aggregation='mean') self.add_metric(class_loss, name='class_loss', aggregation='mean') self.add_metric(lr, name='lr', aggregation='mean') return y_pred def compute_output_shape(self, input_shape): return input_shape[1] def get_config(self): config = { 'ignore_thresh': self.ignore_thresh, 'warmup_batches': self.warmup_batches, 'anchors': self.anchors, 'grid_scale': self.grid_scale, 'obj_scale': self.obj_scale, 'noobj_scale': self.noobj_scale, 'xywh_scale': self.xywh_scale, 'class_scale': self.class_scale } base_config = super(YoloLayer, self).get_config() return dict(list(base_config.items()) + list(config.items())) def _conv_block(inp, convs, skip=True): x = inp count = 0 for conv in convs: if count == (len(convs) - 2) and skip: skip_connection = x count += 1 if conv['stride'] > 1: x = ZeroPadding2D(((1, 0), (1, 0)))(x) # unlike tensorflow darknet prefer left and top paddings x = Conv2D(conv['filter'], conv['kernel'], strides=conv['stride'], padding='valid' if conv['stride'] > 1 else 'same', # unlike tensorflow darknet prefer left and top paddings name='conv_' + str(conv['layer_idx']), use_bias=False if conv['bnorm'] else True)(x) if conv['bnorm']: x = BatchNormalization(epsilon=0.001, name='bnorm_' + str(conv['layer_idx']))(x) if conv['leaky']: x = LeakyReLU(alpha=0.1, name='leaky_' + str(conv['layer_idx']))(x) return add([skip_connection, x]) if skip else x def make_yolov3_model(): input_image = Input(shape=(None, None, 3)) true_boxes = Input(shape=(1, 1, 1, 50, 4)) # Layer 0 => 4 x = _conv_block(input_image, [{'filter': 32, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True}, {'filter': 64, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True}, {'filter': 32, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True}, {'filter': 64, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True}]) # Layer 5 => 8 x = _conv_block(x, [{'filter': 128, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True}, {'filter': 64, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True}, {'filter': 128, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True}]) # Layer 9 => 11 x = _conv_block(x, [{'filter': 64, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True}, {'filter': 128, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True}]) # Layer 12 => 15 x = _conv_block(x, [{'filter': 256, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True}, {'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True}, {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True}]) # Layer 16 => 36 for i in range(7): x = _conv_block(x, [{'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True}, {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True}]) skip_36 = x # Layer 37 => 40 x = _conv_block(x, [{'filter': 512, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True}, {'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True}, {'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True}]) # Layer 41 => 61 for i in range(7): x = _conv_block(x, [{'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True}, {'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True}]) skip_61 = x # Layer 62 => 65 x = _conv_block(x, [{'filter': 1024, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True}, {'filter': 512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True}, {'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True}, {'filter': 512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True}, {'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True}]) # Layer 66 => 74 for i in range(3): x = _conv_block(x, [{'filter': 512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True}, {'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True}]) # Layer 75 => 79 x = _conv_block(x, [{'filter': 512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True}, {'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True}, {'filter': 512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True}, {'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True}, {'filter': 512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True}]) # Layer 80 => 82 yolo_82 = _conv_block(x, [{'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True}, {'filter': 255, 'kernel': 1, 'stride': 1, 'bnorm': False, 'leaky': False}], skip=False) # Layer 83 => 86 x = _conv_block(x, [{'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True}], skip=False) x = UpSampling2D(2)(x) x = concatenate([x, skip_61]) # Layer 87 => 91 x = _conv_block(x, [{'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True}, {'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True}, {'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True}, {'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True}, {'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True}], skip=False) # Layer 92 => 94 yolo_94 = _conv_block(x, [{'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True}, {'filter': 255, 'kernel': 1, 'stride': 1, 'bnorm': False, 'leaky': False}], skip=False) # Layer 95 => 98 x = _conv_block(x, [{'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True}], skip=False) x = UpSampling2D(2)(x) x = concatenate([x, skip_36]) # Layer 99 => 106 yolo_106 = _conv_block(x, [{'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True}, {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True}, {'filter
评论 5
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值