caffe如何自定义网络以及自定义层(python)(二)

继续熟悉前一篇的东西。

Load pretrained parameters to classify an image

首先从gist下载模型脚本:

./scripts/download_model_from_gist.sh <gist_id>

#!/usr/bin/env sh
#coding=utf-8
#终端后面输入的第一个参数
GIST=$1
#终端后面输入的第二个参数
DIRNAME=${2:-./models}
#-z检测字符串长度是否为0,为0返回 true。
if [ -z $GIST ]; then
  echo "usage: download_model_from_gist.sh <gist_id> <dirname>"
  exit
fi

GIST_DIR=$(echo $GIST | tr '/' '-')
MODEL_DIR="$DIRNAME/$GIST_DIR"
#-d检测文件是否是目录,如果是,则返回 true。
if [ -d $MODEL_DIR ]; then
    echo "$MODEL_DIR already exists! Please make sure you're not overwriting anything important!"
    exit
fi

echo "Downloading Caffe model info to $MODEL_DIR ..."
#-p检测文件是否是具名管道,如果是,则返回 true。
mkdir -p $MODEL_DIR
wget https://gist.github.com/$GIST/download -O $MODEL_DIR/gist.zip
unzip -j $MODEL_DIR/gist.zip -d $MODEL_DIR
rm $MODEL_DIR/gist.zip
echo "Done"
脚本二,./scripts/download_model_binary.py <dirname>

#!/usr/bin/env python
#coding=utf-8
import os
import sys
import time
import yaml
import urllib
import hashlib
import argparse
"""
argparse模块使得编写用户友好的命令行接口非常容易。程序只需定义好它要求的参数,
然后argparse将负责如何从sys.argv中解析出这些参数。argparse模块还会自动生成帮
助和使用信息并且当用户赋给程序非法的参数时产生错误信息。
"""
required_keys = ['caffemodel', 'caffemodel_url', 'sha1']


def reporthook(count, block_size, total_size):
    """
    From http://blog.moleculea.com/2012/10/04/urlretrieve-progres-indicator/
    """
    global start_time
    if count == 0:
        start_time = time.time()
        return
    duration = (time.time() - start_time) or 0.01
    progress_size = int(count * block_size)
    speed = int(progress_size / (1024 * duration))
    percent = int(count * block_size * 100 / total_size)
    sys.stdout.write("\r...%d%%, %d MB, %d KB/s, %d seconds passed" %
                    (percent, progress_size / (1024 * 1024), speed, duration))
    sys.stdout.flush()


def parse_readme_frontmatter(dirname):
    readme_filename = os.path.join(dirname, 'readme.md')
    with open(readme_filename) as f:
#line.strip()默认删除空白符(包括'\n', '\r',  '\t',  ' ')
        lines = [line.strip() for line in f.readlines()]
#lines为列表,找'---'的位置
    top = lines.index('---')
    bottom = lines.index('---', top + 1)#top + 1结束索引
#要通过YAML描述与Python的对应关系,从而方便读者了解YAML的层次及结构,
#最常见的是映射到Python中的列表(List)、字典(Dictionary)两种对象类型
    frontmatter = yaml.load('\n'.join(lines[top + 1:bottom]))
    assert all(key in frontmatter for key in required_keys)
    return dirname, frontmatter


def valid_dirname(dirname):
    try:
        return parse_readme_frontmatter(dirname)
    except Exception as e:
        print('ERROR: {}'.format(e))
        raise argparse.ArgumentTypeError(
            'Must be valid Caffe model directory with a correct readme.md')


if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description='Download trained model binary.')
    parser.add_argument('dirname', type=valid_dirname)#输入的参数
    args = parser.parse_args()

    # A tiny hack: the dirname validator also returns readme YAML frontmatter.
    dirname = args.dirname[0]
    frontmatter = args.dirname[1]
#这个就是下载的模型
    model_filename = os.path.join(dirname, frontmatter['caffemodel'])

    # Closure-d function for checking SHA1.
#对于MD5如果需要16位(bytes)的值那么调用对象的digest()而hexdigest()默认是32位(bytes),
#同理Sha1的digest()和hexdigest()分别产生20位(bytes)和40位(bytes)的hash值
    def model_checks_out(filename=model_filename, sha1=frontmatter['sha1']):
        with open(filename, 'rb') as f:
            return hashlib.sha1(f.read()).hexdigest() == sha1
#判断是否存在
    if os.path.exists(model_filename) and model_checks_out():
        print("Model already exists.")
        sys.exit(0)
#下载模型并且核实模型
    urllib.urlretrieve(
        frontmatter['caffemodel_url'], model_filename, reporthook)
    if not model_checks_out():
        print('ERROR: model did not download correctly! Run this again.')
        sys.exit(1)
gist上默认的路径在models下面,用上面的脚本下载一个caffenet的模型和label:

./scripts/download_model_binary.py models/bvlc_reference_caffenet
./data/ilsvrc12/get_ilsvrc_aux.sh
#have a look at the model
python python/draw_net.py models/bvlc_reference_caffenet/deploy.prototxt caffenet.png
open caffenet.png

首先将deploy.prototxt的输入层里面的10改为1,然后运行下面的代码:


#!/usr/bin/env python
#coding=utf-8
import numpy as np
import matplotlib.pyplot as plt
from PIL import Image

caffe_root = '/home/x/git/caffe/'
import sys
sys.path.insert(0, caffe_root + 'python')
import caffe

#caffe.set_mode_cpu()

caffe.set_device(0)
caffe.set_mode_gpu()

#load the model
net = caffe.Net('/home/x/git/caffe/models/bvlc_reference_caffenet/deploy.prototxt',
                '/home/x/git/caffe/models/bvlc_reference_caffenet/bvlc_reference_caffenet.caffemodel',
                caffe.TEST)

# load input and configure preprocessing
transformer = caffe.io.Transformer({'data': net.blobs['data'].data.shape})
transformer.set_mean('data', np.load('/home/x/git/caffe/python/caffe/imagenet/ilsvrc_2012_mean.npy').mean(1).mean(1))
transformer.set_transpose('data', (2,0,1))
transformer.set_channel_swap('data', (2,1,0))
transformer.set_raw_scale('data', 255.0)

#note we can change the batch size on-the-fly
#since we classify only one image, we change batch size from 10 to 1
net.blobs['data'].reshape(1,3,227,227)

#load the image in the data layer
im = caffe.io.load_image('/home/x/git/caffe/examples/images/cat.jpg')
net.blobs['data'].data[...] = transformer.preprocess('data', im)

#compute
out = net.forward()

# other possibility : out = net.forward_all(data=np.asarray([transformer.preprocess('data', im)]))

#predicted predicted class
print out['prob'].argmax()
print '========================'
#print predicted labels
labels = np.loadtxt("/home/x/git/caffe/data/ilsvrc12/synset_words.txt", str, delimiter='\t')
top_k = net.blobs['prob'].data[0].flatten().argsort()[-1:-6:-1]
print labels[top_k]

官网给出的Surgery,http://nbviewer.jupyter.org/github/BVLC/caffe/blob/master/examples/net_surgery.ipynb

import numpy as np
import matplotlib.pyplot as plt
#matplotlib inline
import pylab
#matplotlib.use('TkAgg')
#import matplotlib.rcsetup as rcsetup
#print(rcsetup.all_backends)

# Make sure that caffe is on the python path:
caffe_root = '/home/x/git/caffe/'  # this file is expected to be in {caffe_root}/examples
import sys
sys.path.insert(0, caffe_root + 'python')

import caffe

# configure plotting
plt.rcParams['figure.figsize'] = (10, 10)
plt.rcParams['image.interpolation'] = 'nearest'
plt.rcParams['image.cmap'] = 'gray'
# Load the net, list its data and params, and filter an example image.
caffe.set_mode_cpu()
net = caffe.Net('/home/x/git/caffe/examples/net_surgery/conv.prototxt', caffe.TEST)
print("blobs {}\nparams {}".format(net.blobs.keys(), net.params.keys()))

# load image and prepare as a single input batch for Caffe
im = np.array(caffe.io.load_image('/home/x/git/caffe/examples/images/cat_gray.jpg', color=False)).squeeze()
plt.title("original image")
plt.imshow(im)
plt.axis('off')

im_input = im[np.newaxis, np.newaxis, :, :]
net.blobs['data'].reshape(*im_input.shape)
net.blobs['data'].data[...] = im_input
# helper show filter outputs
def show_filters(net):
    net.forward()
    plt.figure()
    filt_min, filt_max = net.blobs['conv'].data.min(), net.blobs['conv'].data.max()
    for i in range(3):
        plt.subplot(1,4,i+2)
        plt.title("filter #{} output".format(i))
        plt.imshow(net.blobs['conv'].data[0, i], vmin=filt_min, vmax=filt_max)
        plt.tight_layout()
        plt.axis('off')
# filter the image with initial
show_filters(net)
# pick first filter output
conv0 = net.blobs['conv'].data[0, 0]
print("pre-surgery output mean {:.2f}".format(conv0.mean()))
# set first filter bias to 1
net.params['conv'][1].data[0] = 1.
net.forward()
print("post-surgery output mean {:.2f}".format(conv0.mean()))
ksize = net.params['conv'][0].data.shape[2:]
# make Gaussian blur
sigma = 1.
y, x = np.mgrid[-ksize[0]//2 + 1:ksize[0]//2 + 1, -ksize[1]//2 + 1:ksize[1]//2 + 1]
g = np.exp(-((x**2 + y**2)/(2.0*sigma**2)))
gaussian = (g / g.sum()).astype(np.float32)
net.params['conv'][0].data[0] = gaussian
# make Sobel operator for edge detection
net.params['conv'][0].data[1:] = 0.
sobel = np.array((-1, -2, -1, 0, 0, 0, 1, 2, 1), dtype=np.float32).reshape((3,3))
net.params['conv'][0].data[1, 0, 1:-1, 1:-1] = sobel  # horizontal
net.params['conv'][0].data[2, 0, 1:-1, 1:-1] = sobel.T  # vertical
show_filters(net)
pylab.show()
#print(matplotlib.backends.backend)
print 'aaaaaaaaaaaaa'
下面将caffenet的全链接层转化为全卷积。

#!/usr/bin/env python
#coding=utf-8
import numpy as np
import pylab
import matplotlib.pyplot as plt
from PIL import Image
caffe_root = '/home/x/git/caffe/'
import sys
sys.path.insert(0, caffe_root + 'python')
import caffe

#caffe.set_mode_cpu()

caffe.set_device(0)
caffe.set_mode_gpu()

# Load the original network and extract the fully connected layers' parameters.
net = caffe.Net('/home/x/git/caffe/models/bvlc_reference_caffenet/deploy.prototxt',
                '/home/x/git/caffe/models/bvlc_reference_caffenet/bvlc_reference_caffenet.caffemodel',
                caffe.TEST)
params = ['fc6', 'fc7', 'fc8']
# fc_params = {name: (weights, biases)}
fc_params = {pr: (net.params[pr][0].data, net.params[pr][1].data) for pr in params}
for fc in params:
    print '{} weights are {} dimensional and biases are {} dimensional'.format(fc, fc_params[fc][0].shape, fc_params[fc][1].shape)

# Load the fully convolutional network to transplant the parameters.
net_full_conv = caffe.Net('/home/x/git/caffe/examples/net_surgery/bvlc_caffenet_full_conv.prototxt', 
                          '/home/x/git/caffe/models/bvlc_reference_caffenet/bvlc_reference_caffenet.caffemodel',
                          caffe.TEST)
params_full_conv = ['fc6-conv', 'fc7-conv', 'fc8-conv']
# conv_params = {name: (weights, biases)}
conv_params = {pr: (net_full_conv.params[pr][0].data, net_full_conv.params[pr][1].data) for pr in params_full_conv}

for conv in params_full_conv:
    print '{} weights are {} dimensional and biases are {} dimensional'.format(conv, conv_params[conv][0].shape, conv_params[conv][1].shape)
for pr, pr_conv in zip(params, params_full_conv):
    conv_params[pr_conv][0].flat = fc_params[pr][0].flat  # flat unrolls the arrays
    conv_params[pr_conv][1][...] = fc_params[pr][1]

net_full_conv.save('/home/x/git/caffe/examples/net_surgery/bvlc_caffenet_full_conv.caffemodel')


# load input and configure preprocessing
im = caffe.io.load_image('/home/x/git/caffe/examples/images/cat.jpg')
transformer = caffe.io.Transformer({'data': net_full_conv.blobs['data'].data.shape})
transformer.set_mean('data', np.load('/home/x/git/caffe/python/caffe/imagenet/ilsvrc_2012_mean.npy').mean(1).mean(1))
transformer.set_transpose('data', (2,0,1))
transformer.set_channel_swap('data', (2,1,0))
transformer.set_raw_scale('data', 255.0)
# make classification map by forward and print prediction indices at each location
out = net_full_conv.forward_all(data=np.asarray([transformer.preprocess('data', im)]))
print out['prob'][0].argmax(axis=0)
# show net input and confidence map (probability of the top prediction at each location)
plt.subplot(1, 2, 1)
plt.imshow(transformer.deprocess('data', net_full_conv.blobs['data'].data[0]))
plt.subplot(1, 2, 2)
plt.imshow(out['prob'][0,281])
pylab.show()

顺便跑跑特征可视化的代码:

#!/usr/bin/env python
#coding=utf-8
import numpy as np
import pylab
import matplotlib.pyplot as plt
from PIL import Image
caffe_root = '/home/x/git/caffe/'
import sys
sys.path.insert(0, caffe_root + 'python')
import caffe

#caffe.set_mode_cpu()

caffe.set_device(0)
caffe.set_mode_gpu()
# If you get "No module named _caffe", either you have not built pycaffe or you have the wrong path.
import os
if os.path.isfile(caffe_root + 'models/bvlc_reference_caffenet/bvlc_reference_caffenet.caffemodel'):
    print 'CaffeNet found.'
else:
    print 'Downloading pre-trained CaffeNet model...'
#../scripts/download_model_binary.py ../models/bvlc_reference_caffenet

model_def = caffe_root + 'models/bvlc_reference_caffenet/deploy.prototxt'
model_weights = caffe_root + 'models/bvlc_reference_caffenet/bvlc_reference_caffenet.caffemodel'

net = caffe.Net(model_def,      # defines the structure of the model
                model_weights,  # contains the trained weights
                caffe.TEST)     # use test mode (e.g., don't perform dropout)
# load the mean ImageNet image (as distributed with Caffe) for subtraction
mu = np.load(caffe_root + 'python/caffe/imagenet/ilsvrc_2012_mean.npy')
mu = mu.mean(1).mean(1)  # average over pixels to obtain the mean (BGR) pixel values
print 'mean-subtracted values:', zip('BGR', mu)

# create transformer for the input called 'data'
transformer = caffe.io.Transformer({'data': net.blobs['data'].data.shape})

transformer.set_transpose('data', (2,0,1))  # move image channels to outermost dimension
transformer.set_mean('data', mu)            # subtract the dataset-mean value in each channel
transformer.set_raw_scale('data', 255)      # rescale from [0, 1] to [0, 255]
transformer.set_channel_swap('data', (2,1,0))  # swap channels from RGB to BGR
# set the size of the input (we can skip this if we're happy
#  with the default; we can also change it later, e.g., for different batch sizes)
net.blobs['data'].reshape(50,        # batch size
                          3,         # 3-channel (BGR) images
                          227, 227)  # image size is 227x227
image = caffe.io.load_image(caffe_root + 'examples/images/cat.jpg')
transformed_image = transformer.preprocess('data', image)
plt.imshow(image)
# copy the image data into the memory allocated for the net
net.blobs['data'].data[...] = transformed_image

### perform classification
output = net.forward()

output_prob = output['prob'][0]  # the output probability vector for the first image in the batch

print 'predicted class is:', output_prob.argmax()


# load ImageNet labels
labels_file = caffe_root + 'data/ilsvrc12/synset_words.txt'
#if not os.path.exists(labels_file):
#    !../data/ilsvrc12/get_ilsvrc_aux.sh
    
labels = np.loadtxt(labels_file, str, delimiter='\t')

print 'output label:', labels[output_prob.argmax()]

# sort top five predictions from softmax output
top_inds = output_prob.argsort()[::-1][:5]  # reverse sort and take five largest items

print 'probabilities and labels:'
zip(output_prob[top_inds], labels[top_inds])
# for each layer, show the output shape
for layer_name, blob in net.blobs.iteritems():
    print layer_name + '\t' + str(blob.data.shape)
for layer_name, param in net.params.iteritems():
    print layer_name + '\t' + str(param[0].data.shape), str(param[1].data.shape)
def vis_square(data):
    """Take an array of shape (n, height, width) or (n, height, width, 3)
       and visualize each (height, width) thing in a grid of size approx. sqrt(n) by sqrt(n)"""
    
    # normalize data for display
    data = (data - data.min()) / (data.max() - data.min())
    
    # force the number of filters to be square
    n = int(np.ceil(np.sqrt(data.shape[0])))
    padding = (((0, n ** 2 - data.shape[0]),
               (0, 1), (0, 1))                 # add some space between filters
               + ((0, 0),) * (data.ndim - 3))  # don't pad the last dimension (if there is one)
    data = np.pad(data, padding, mode='constant', constant_values=1)  # pad with ones (white)
    
    # tile the filters into an image
    data = data.reshape((n, n) + data.shape[1:]).transpose((0, 2, 1, 3) + tuple(range(4, data.ndim + 1)))
    data = data.reshape((n * data.shape[1], n * data.shape[3]) + data.shape[4:])
    plt.imshow(data); plt.axis('off')
    pylab.show()
# the parameters are a list of [weights, biases]
filters = net.params['conv1'][0].data
vis_square(filters.transpose(0, 2, 3, 1))

feat = net.blobs['conv1'].data[0, :36]
vis_square(feat)

feat = net.blobs['pool5'].data[0]
vis_square(feat)
feat = net.blobs['fc6'].data[0]
plt.subplot(2, 1, 1)
plt.plot(feat.flat)
plt.subplot(2, 1, 2)
_ = plt.hist(feat.flat[feat.flat > 0], bins=100)
feat = net.blobs['prob'].data[0]
plt.figure(figsize=(15, 3))
plt.plot(feat.flat)
pylab.show()

上图是the first layer filters,conv1

上图是The first layer output,conv1

上图是The fifth layer after pooling,pool5


上图是The first fully connected layer,fc6


上图是The final probability output,prob

知识小插曲:

diff命令:

功能:比较两个文件的差异,并把不同地方的信息显示出来。默认diff格式的信息。

diff比较两个文件或文件集合的差异,并记录下来,生成一个diff文件,这也是我们常说的补丁文件。也使用patch命令对相应的文件打补丁。differential [ˌdɪfəˈrenʃəl]

语法:diff    [options]   FILES

FILES的格式:

  • FILE1 FILE2 :源是一个文件,目标也是文件。这两个文件必须是文本文件。以逐行的方式,比较文本文件的异同处。
  • DIR1 DIR2   :源是一个目录,目标是目录。diff 命令会比较两个目录下名字相同的文本文件,依照字母次序排序,列出不同的二进制文件,列出公共子目录,列出只在一个目录出现的文件。
  • FILE DIR     :源是一个文件,目标是目录。diff命令把源文件与目标目录下的同名文件比较。
  • DIR FILE     :源是一个目录,目标是文件(不是目录)。源目录下所有文件中与目标文件同名的文件,将用来与目标文件比较。

FILE可以是“-”,代表由标准输入设备读入的文本。DIR不能是"-"。
被指定的文件不能是标准的输入。

 

短选项 长选项 含义
-i --ignore-case 忽略文件内容大小写的区别
  --ignore-file-name-case 忽略文件名大小写的区别
  --no-ignore-file-name-case 不忽略文件名大小写的区别
-E --ignore-tab-expansion 忽略由制表符宽度造成的差异
-b --ignore-space-change 忽略由空格数不同造成的差异
-w --ignore-all-space 忽略所有空格
-B --ignore-blank-lines 忽略任何因空行而造成的差异
-I --ignore-matching-lines=RE 如果某行匹配正则表达式,则忽略由该行造成的差异
-a --text 所有文件都以文本方式处理
  --strip-trailing-cr 去除输入内容每行末端的 carriage return 字符
-c 或 或 -C 行数 --context[=行数] 显示指定<行数>(默认 3 行)copied 格式的上下文
-u  或 -U 行数 --unified[=行数] 显示指定<行数>(默认 3 行)unified 格式的上下文(合并的方式)
  --label 标识 使用<标识>代替文件名称
-p --show-c-function 显示和每个差异有关的 C 函数名称
-F  --show-function-line=RE 显示最接近而符合<正则表示式>的一行
-q --brief 只显示文件是否不同
-e --ed 以 ed script 方式输出
  --normal 以正常的 diff 方式输出
-n --rcs 以 RCS diff 格式输出
-y --side-by-side 以两列并排的方式显示
-W --width=NUM 每行显示最多 NUM (默认 130) 个字符
  --left-column 当有两行相同时只显示左边的一行
  --suppress-common-lines 当有两行相同时不会显示
-D --ifdef=NAME 输出的内容以‘#ifdef NAME’方式标明差异
  --GTYPE-group-format=GFMT 效果类似,但会以 GFMT 格式处理 GTYPE 输入的行
可以是LTYPE的选择或是‘changed’
  --line-format=LFMT 效果类似,但会以 LFMT 格式处理每一行资料
  --LTYPE-line-format=LFMT 效果类似,但会以 LFMT 格式处理 LTYPE 输入的行
LTYPE 可以是‘old’、‘new’或‘unchanged’。
-l --paginate 将输出送至‘pr’指令来分页
-t --expand-tabs 将输出中的 tab 转换成空格
-T --initial-tab 每行先加上 tab 字符,使 tab 字符可以对齐
  --tabsize=NUM 定位字符 (tab) 的宽度,默认为 8 个空格宽
  --suppress-blank-empty suppress space or tab before empty output lines
-r --recursive 递归比较子目录中的文件
-N --new-file 不存在的文件以空文件方式处理
  --unidirectional-new-file 若第一文件案不存在,以空文件处理
-s --report-identical-files 文件相同则报告,否则无任何提示
-x --exclude=PAT 排除匹配 PAT 的文件
-X --exclude-from=FILE 排除所有匹配在 FILE 中列出的模式的文件
-S --starting-file=FILE 当比较目录時,由 FILE 开始比较
  --from-file=FILE1 将 FILE1 和操作数中的所有文件/目录作比较。FILE1 可以是目录
  --to-file=FILE2 将操作数中的所有文件/目录和 FILE2 作比较。FILE2 可以是目录
  --horizon-lines=NUM keep NUM lines of the common prefix and suffix
-d --minimal 尽可能找出最小的差异
  --speed-large-files 假设文件十分大而且其中含有许多微小的差异

diff输出格式(默认):
n1 a n3,n4 表示在文件1的n1行后面添加n3到n4行
n1,n2 d n3 表示在n1到n2行之间删除n3行
n1,n2 c n3,n4 表示把n1,n2行用n3,n4行替换掉
字母a:表示附加(add)
字符c:表示修改(change)
字符d:表示删除(delete)
字母前的是源文件,字母后是目标文件。Nx表示行号。
以”<”打头的行属于第一个文件,以”>”打头的行属于第二个文件。

比较文件

log2014.log内容 2013-01 2013-02 2014-03 2013-04 2013-05 2013-06 2013-07 2013-07 2013-09 2013-10 2013-11 2013-12

log2013.log内容 2013-01 2013-02 2013-03 2013-04 2013-05 2013-06 2013-07 2013-08 2013-09 2013-10

比较两个文件 [root@localhost test3]# diff log2014.log log2013.log 3c3 < 2014-03 --- > 2013-03 8c8 < 2013-07 --- > 2013-08 11,12d10 < 2013-11 < 2013-12 上面的“3c3”和“8c8”表示log2014.log和log20143log文件在3行和第8行内容有所不同;"11,12d10"表示第一个文件比第二个文件多了第11和12行。 以”<”打头的行属于第一个文件,以”>”打头的行属于第二个文件。 并排格式输出 [root@localhost test3]# diff log2014.log log2013.log -y -W 50 2013-01 2013-01 2013-02 2013-02 2014-03 | 2013-03 2013-04 2013-04 2013-05 2013-05 2013-06 2013-06 2013-07 2013-07 2013-07 | 2013-08 2013-09 2013-09 2013-10 2013-10 2013-11 < 2013-12 < [root@localhost test3]# diff log2013.log log2014.log -y -W 50 2013-01 2013-01 2013-02 2013-02 2013-03 | 2014-03 2013-04 2013-04 2013-05 2013-05 2013-06 2013-06 2013-07 2013-07 2013-08 | 2013-07 2013-09 2013-09 2013-10 2013-10 > 2013-11 > 2013-12 说明: “|”表示前后2个文件内容有不同 “<”表示后面文件比前面文件少了1行内容 “>”表示后面文件比前面文件多了1行内容 上下文输出格式 [root@localhost test3]# diff log2013.log log2014.log -c *** log2013.log 2012-12-07 16:36:26.000000000 +0800 --- log2014.log 2012-12-07 18:01:54.000000000 +0800 *************** *** 1,10 **** 2013-01 2013-02 ! 2013-03 2013-04 2013-05 2013-06 2013-07 ! 2013-08 2013-09 2013-10 --- 1,12 ---- 2013-01 2013-02 ! 2014-03 2013-04 2013-05 2013-06 2013-07 ! 2013-07 2013-09 2013-10 + 2013-11 + 2013-12 [root@localhost test3]# diff log2014.log log2013.log -c *** log2014.log 2012-12-07 18:01:54.000000000 +0800 --- log2013.log 2012-12-07 16:36:26.000000000 +0800 *************** *** 1,12 **** 2013-01 2013-02 ! 2014-03 2013-04 2013-05 2013-06 2013-07 ! 2013-07 2013-09 2013-10 - 2013-11 - 2013-12 --- 1,10 ---- 2013-01 2013-02 ! 2013-03 2013-04 2013-05 2013-06 2013-07 ! 2013-08 2013-09 2013-10 [root@localhost test3]# 这种方式在开头两行作了比较文件的说明,这里有三中特殊字符: “+” 比较的文件的后者比前着多一行 “-” 比较的文件的后者比前着少一行 “!” 比较的文件两者有差别的行 统一格式输出 [root@localhost test3]# diff log2014.log log2013.log -u --- log2014.log 2012-12-07 18:01:54.000000000 +0800 +++ log2013.log 2012-12-07 16:36:26.000000000 +0800 @@ -1,12 +1,10 @@ 2013-01 2013-02 -2014-03 +2013-03 2013-04 2013-05 2013-06 2013-07 -2013-07 +2013-08 2013-09 2013-10 -2013-11 -2013-12 第一部分,也是文件的基本信息: --- log2014.log 2012-12-07 18:01:54.000000000 +0800 +++ log2013.log 2012-12-07 16:36:26.000000000 +0800 "---"表示变动前的文件,"+++"表示变动后的文件。 第二部分,变动的位置用两个@作为起首和结束 @@ -1,12 +1,10 @@ 前面的"-1,12"分成三个部分:减号表示第一个文件(即log2014.log),"1"表示第1行,"12"表示连续12行。意思:第一个文件从第1行开始的连续12行。 同样的,"+1,10" 表示第二个文件从第1行开始的连续10行。

 

比较文件夹

[root@localhost test]# diff test3 test6 
Only in test6: linklog.log 
Only in test6: log2012.log 
 
[root@localhost test]# diff test3/log2013.log test6/log2013.log 
1,10c1,3 
< 2013-01 
< 2013-02 
< 2013-03 
< 2013-04 
< 2013-05 
< 2013-06 
< 2013-07 
< 2013-08 
< 2013-09 
< 2013-10 
--- 
> hostnamebaidu=baidu.com 
> hostnamesina=sina.com 
> hostnames=true 
 
[root@localhost test]# diff test3/log2014.log test6/log2014.log 
1,12d0 
< 2013-01 
< 2013-02 
< 2014-03 
< 2013-04 
< 2013-05 
< 2013-06 
< 2013-07 
< 2013-07 
< 2013-09 
< 2013-10 
< 2013-11 
< 2013-12 
Only in test6: log2015.log 
Only in test6: log2016.log 
Only in test6: log2017.log 
[root@localhost test]# 

 

比较两个文件不同,并生产补丁

[root@localhost test3]# diff -ruN log2013.log log2014.log >patch.log 
[root@localhost test3]# ll 
总计 12 
-rw-r--r-- 2 root root  80 12-07 16:36 log2013.log 
-rw-r--r-- 1 root root  96 12-07 18:01 log2014.log 
-rw-r--r-- 1 root root 248 12-07 21:33 patch.log 
 
[root@localhost test3]# cat patch.log 
--- log2013.log 2012-12-07 16:36:26.000000000 +0800 
+++ log2014.log 2012-12-07 18:01:54.000000000 +0800 
@@ -1,10 +1,12 @@ 
 2013-01 
 2013-02 
-2013-03 
+2014-03 
 2013-04 
 2013-05 
 2013-06 
 2013-07 
-2013-08 
+2013-07 
 2013-09 
 2013-10 
+2013-11 
+2013-12 
[root@localhost test3]#

 

打补丁

[root@localhost test3]# cat log2013.log 
2013-01 
2013-02 
2013-03 
2013-04 
2013-05 
2013-06 
2013-07 
2013-08 
2013-09 
2013-10 
 
[root@localhost test3]# patch log2013.log patch.log  使用上面产生的patch.log文件打补丁 
patching file log2013.log 
 
[root@localhost test3]# cat log2013.log 
2013-01 
2013-02 
2014-03 
2013-04 
2013-05 
2013-06 
2013-07 
2013-07 
2013-09 
2013-10 
2013-11 
2013-12 
[root@localhost test3]# 

 

实例

实例:有这样两个文件。程序清单1,程序清单2 
程序清单1 :hello.c 
#include 
int main(void) 
{ 
    char msg[] = "Hello world!"; 
    
    puts(msg); 
    printf("Welcome to use diff commond.n"); 
    
    return 0;  
} 
 
程序清单2:hello_diff.c 
#include 
#include  
int main(void) 
{ 
    char msg[] = "Hello world,fome hello_diff.c"; 
    
    puts(msg); 
    printf("hello_diff.c says,'Here you are,using diff.'n"); 
    
    return 0; 
} 
 
 

我们使用diff命令来查看这两个文件的不同之处,有一下几种方便的方法:

1、普通格式输出: [root@localhost diff]# diff hello.c hello_diff.c 1a2 > #include 5c6 char msg[] = "Hello world,fome hello_diff.c"; 8c9 printf("hello_diff.c says,'Here you are,using diff.'n"); [root@localhost diff]# 上面的“1a2”表示后面的一个文件"hello_diff.c"比前面的一个文件"hello.c"多了一行 "5c6"表示第一个文件的第5行与第二个文件的第6行有区别 2、并排格式输出 [root@localhost diff]# diff hello.c hello_diff.c -y -W 130 -W选择可以指定输出列的宽度,这里指定输出列宽为130 #include #include > #include int main(void) int main(void) { { char msg[] = "Hello world!"; | char msg[] = "Hello world,fome hello_diff.c"; puts(msg); puts(msg); printf("Welcome to use diff commond.n"); | printf("hello_diff.c says,'Here you are,using diff.' return 0; return 0; } } [root@localhost diff]# 这种并排格式的对比一目了然,可以快速找到不同的地方。 3、上下文输出格式 [root@localhost diff]# diff hello.c hello_diff.c -c *** hello.c 2007-09-25 17:54:51.000000000 +0800 --- hello_diff.c 2007-09-25 17:56:00.000000000 +0800 *************** *** 1,11 **** #include int main(void) { ! char msg[] = "Hello world!"; puts(msg); ! printf("Welcome to use diff commond.n"); return 0; } --- 1,12 ---- #include + #include int main(void) { ! char msg[] = "Hello world,fome hello_diff.c"; puts(msg); ! printf("hello_diff.c says,'Here you are,using diff.'n"); return 0; } [root@localhost diff]# 这种方式在开头两行作了比较文件的说明,这里有三中特殊字符: + 比较的文件的后者比前着多一行 - 比较的文件的后者比前着少一行 ! 比较的文件两者有差别的行 4、统一输出格式 [root@localhost diff]# diff hello.c hello_diff.c -u --- hello.c 2007-09-25 17:54:51.000000000 +0800 +++ hello_diff.c 2007-09-25 17:56:00.000000000 +0800 @@ -1,11 +1,12 @@ #include +#include int main(void) { - char msg[] = "Hello world!"; + char msg[] = "Hello world,fome hello_diff.c"; puts(msg); - printf("Welcome to use diff commond.n"); + printf("hello_diff.c says,'Here you are,using diff.'n"); return 0; } [root@localhost diff]# 5、其他 [root@localhost diff]# diff hello.c hello_diff.c -q 假如你想查看两个文件是否不同又不想显示差异之处的话,可以加上-q选项 Files hello.c and hello_diff.c differ [root@localhost diff]# [root@localhost diff]# diff hello.c hello_diff.c -c -I include 这里通过“ -I include”选项来忽略带有“ include”字样的行。另外你还可以提供一些匹配规则来忽略某中差别,可以用 -I regexp *** hello.c 2007-09-25 17:54:51.000000000 +0800 --- hello_diff.c 2007-09-25 17:56:00.000000000 +0800 *************** *** 2,11 **** int main(void) { ! char msg[] = "Hello world!"; puts(msg); ! printf("Welcome to use diff commond.n"); return 0; } --- 3,12 ---- int main(void) { ! char msg[] = "Hello world,fome hello_diff.c"; puts(msg); ! printf("hello_diff.c says,'Here you are,using diff.'n"); return 0; } [root@localhost diff]# 下面是我对上面两个网络配置文件对比:

caffePath=/home/x/git/caffe
example=$caffePath/examples/net_surgery
modle=$caffePath/models
test_lb=$example/../test_layer_lb/test_python
diff $example/bvlc_caffenet_full_conv.prototxt $modle/bvlc_reference_caffenet/deploy.prototxt 2>&1 | tee $test_lb/log.txt
输出log.txt:
1,2c1
< # Fully convolutional network version of CaffeNet.
< name: "CaffeNetConv"
---
> name: "CaffeNet"
7,11c6
<   input_param {
<     # initial shape for a fully convolutional network:
<     # the shape can be set for each input by reshape.
<     shape: { dim: 1 dim: 3 dim: 451 dim: 451 }
<   }
---
>   input_param { shape: { dim: 1 dim: 3 dim: 227 dim: 227 } }
157,158c152,153
<   name: "fc6-conv"
<   type: "Convolution"
---
>   name: "fc6"
>   type: "InnerProduct"
160,161c155,156
<   top: "fc6-conv"
<   convolution_param {
---
>   top: "fc6"
>   inner_product_param {
163d157
<     kernel_size: 6
169,170c163,164
<   bottom: "fc6-conv"
<   top: "fc6-conv"
---
>   bottom: "fc6"
>   top: "fc6"
175,176c169,170
<   bottom: "fc6-conv"
<   top: "fc6-conv"
---
>   bottom: "fc6"
>   top: "fc6"
182,186c176,180
<   name: "fc7-conv"
<   type: "Convolution"
<   bottom: "fc6-conv"
<   top: "fc7-conv"
<   convolution_param {
---
>   name: "fc7"
>   type: "InnerProduct"
>   bottom: "fc6"
>   top: "fc7"
>   inner_product_param {
188d181
<     kernel_size: 1
194,195c187,188
<   bottom: "fc7-conv"
<   top: "fc7-conv"
---
>   bottom: "fc7"
>   top: "fc7"
200,201c193,194
<   bottom: "fc7-conv"
<   top: "fc7-conv"
---
>   bottom: "fc7"
>   top: "fc7"
207,211c200,204
<   name: "fc8-conv"
<   type: "Convolution"
<   bottom: "fc7-conv"
<   top: "fc8-conv"
<   convolution_param {
---
>   name: "fc8"
>   type: "InnerProduct"
>   bottom: "fc7"
>   top: "fc8"
>   inner_product_param {
213d205
<     kernel_size: 1
219c211
<   bottom: "fc8-conv"
---
>   bottom: "fc8"

发布了347 篇原创文章 · 获赞 607 · 访问量 261万+
展开阅读全文

没有更多推荐了,返回首页

©️2019 CSDN 皮肤主题: 技术黑板 设计师: CSDN官方博客

分享到微信朋友圈

×

扫一扫,手机浏览