这里是引用其他博主：
keras CNN卷积核可视化,热度图
链接：https://blog.csdn.net/DLW__/article/details/99599397
可视化卷及神经网络热力图
链接：https://blog.csdn.net/Einstellung/article/details/82858974
一、卷积核可视化
代码如下（示例）：
import matplotlib.pyplot as plt
import numpy as np
from keras import backend as K
from keras.models import load_model
from keras.applications.vgg16 import VGG16
# 将浮点图像转换成有效图像
def deprocess_image(x):
# 对张量进行规范化
x -= x.mean()
x /= (x.std() + 1e-5)
x *= 0.1
x += 0.5
x = np.clip(x, 0, 1)
# 转化到RGB数组
x *= 255
x = np.clip(x, 0, 255).astype('uint8')
return x
# 可视化滤波器
def kernelvisual(model, layer_target=1, num_iterate=5):
# 图像尺寸和通道
img_height, img_width, num_channels = K.int_shape(model.input)[1:4]
num_out = K.int_shape(model.layers[layer_target].output)[-1]
plt.suptitle('[%s] convnet filters visualizing' % model.layers[layer_target].name)
print('第%d层有%d个通道' % (layer_target, num_out))
for i_kernal in range(num_out):
input_img = model.input
# 构建一个损耗函数，使所考虑的层的第n个滤波器的激活最大化，-1层softmax层
if layer_target == -1:
loss = K.mean(model.output[:, i_kernal])
else:
loss = K.mean(model.layers[layer_target].output[:, :, :, i_kernal]) # m*28*28*128
# 计算图像对损失函数的梯度
grads = K.gradients(loss, input_img)[0]
# 效用函数通过其L2范数标准化张量
grads /= (K.sqrt(K.mean(K.square(grads))) + 1e-5)
# 此函数返回给定输入图像的损耗和梯度
iterate = K.function([input_img], [loss, grads])
# 从带有一些随机噪声的灰色图像开始
np.random.seed(0)
# 随机图像
# input_img_data = np.random.randint(0, 255, (1, img_height, img_width, num_channels)) # 随机
# input_img_data = np.zeros((1, img_height, img_width, num_channels)) # 零值
input_img_data = np.random.random((1, img_height, img_width, num_channels)) * 20 + 128. # 随机灰度
input_img_data = np.array(input_img_data, dtype=float)
failed = False
# 运行梯度上升
print('####################################', i_kernal + 1)
loss_value_pre = 0
# 运行梯度上升num_iterate步
for i in range(num_iterate):
loss_value, grads_value = iterate([input_img_data])
if i % int(num_iterate/5) == 0:
print('Iteration %d/%d, loss: %f' % (i, num_iterate, loss_value))
print('Mean grad: %f' % np.mean(grads_value))
if all(np.abs(grads_val) < 0.000001 for grads_val in grads_value.flatten()):
failed = True
print('Failed')
break
if loss_value_pre != 0 and loss_value_pre > loss_value:
break
if loss_value_pre == 0:
loss_value_pre = loss_value
# if loss_value > 0.99:
# break
input_img_data += grads_value * 1 # e-3
img_re = deprocess_image(input_img_data[0])
if num_channels == 1:
img_re = np.reshape(img_re, (img_height, img_width))
else:
img_re = np.reshape(img_re, (img_height, img_width, num_channels))
plt.subplot(np.ceil(np.sqrt(num_out)), np.ceil(np.sqrt(num_out)), i_kernal + 1)
plt.imshow(img_re) # , cmap='gray'
plt.axis('off')
plt.show()
#model = load_model('train3.h5')
model = VGG16(weights='imagenet')
kernelvisual(model,-1) # 对最终输出可视化
#kernelvisual(model,1)
kernelvisual(model,6) # 对第二个卷积层可视化
运行结果
训练1000次：最后一层输出可视化为(第-1层有1000个通道)：
第6层输出可视化为(第6层有128个通道):
二、热度图可视化
代码如下（示例）：
import cv2
import matplotlib.pyplot as plt
import numpy as np
from keras import backend as K
from keras.preprocessing import image
def heatmap(model, data_img, layer_idx, img_show=None, pred_idx=None):
# 图像处理
if data_img.shape.__len__() != 4:
# 由于用作输入的img需要预处理,用作显示的img需要原图,因此分开两个输入
if img_show is None:
img_show = data_img
# 缩放
input_shape = K.int_shape(model.input)[1:3] # (28,28)
data_img = image.img_to_array(image.array_to_img(data_img).resize(input_shape))
# 添加一个维度->(1, 224, 224, 3)
data_img = np.expand_dims(data_img, axis=0)
if pred_idx is None:
# 预测
preds = model.predict(data_img)
# 获取最高预测项的index
pred_idx = np.argmax(preds[0])
# 目标输出估值
target_output = model.output[:, pred_idx]
# 目标层的输出代表各通道关注的位置
last_conv_layer_output = model.layers[layer_idx].output
# 求最终输出对目标层输出的导数(优化目标层输出),代表目标层输出对结果的影响
grads = K.gradients(target_output, last_conv_layer_output)[0]
# 将每个通道的导数取平均,值越高代表该通道影响越大
pooled_grads = K.mean(grads, axis=(0, 1, 2))
iterate = K.function([model.input], [pooled_grads, last_conv_layer_output[0]])
pooled_grads_value, conv_layer_output_value = iterate([data_img])
# 将各通道关注的位置和各通道的影响乘起来
for i in range(conv_layer_output_value.shape[-1]):
conv_layer_output_value[:, :, i] *= pooled_grads_value[i]
# 对各通道取平均得图片位置对结果的影响
heatmap = np.mean(conv_layer_output_value, axis=-1)
# 规范化
heatmap = np.maximum(heatmap, 0)
heatmap /= np.max(heatmap)
# plt.matshow(heatmap)
# plt.show()
# 叠加图片
# 缩放成同等大小
heatmap = cv2.resize(heatmap, (img_show.shape[1], img_show.shape[0]))
heatmap = np.uint8(255 * heatmap)
# 将热图应用于原始图像.由于opencv热度图为BGR,需要转RGB
superimposed_img = img_show + cv2.applyColorMap(heatmap, cv2.COLORMAP_JET)[:,:,::-1] * 0.4
# 截取转uint8
superimposed_img = np.minimum(superimposed_img, 255).astype('uint8')
# 显示图片
#plt.imshow(superimposed_img)
#plt.show()
# 保存为文件
#superimposed_img = img_show + cv2.applyColorMap(heatmap, cv2.COLORMAP_JET) * 0.4
#cv2.imwrite('ele.png', superimposed_img)
return superimposed_img, heatmap
# 生成所有卷积层的热度图
def heatmaps(model, data_img, img_show=None):
if img_show is None:
img_show = np.array(data_img)
# Resize
input_shape = K.int_shape(model.input)[1:3] # (28,28,1)
data_img = image.img_to_array(image.array_to_img(data_img).resize(input_shape))
# 添加一个维度->(1, 224, 224, 3)
data_img = np.expand_dims(data_img, axis=0)
# 预测
preds = model.predict(data_img)
# 获取最高预测项的index
pred_idx = np.argmax(preds[0])
print("预测为:%d(%f)" % (pred_idx, preds[0][pred_idx]))
indexs = []
for i in range(model.layers.__len__()):
if 'conv' in model.layers[i].name:
indexs.append(i)
print('模型共有%d个卷积层' % indexs.__len__())
plt.suptitle('heatmaps for each conv')
for i in range(indexs.__len__()):
ret = heatmap(model, data_img, indexs[i], img_show=img_show, pred_idx=pred_idx)
plt.subplot(np.ceil(np.sqrt(indexs.__len__()*2)), np.ceil(np.sqrt(indexs.__len__()*2)), i*2 + 1)\
.set_title(model.layers[indexs[i]].name)
plt.imshow(ret[0])
cv2.imwrite('%d.png'%(i), ret[0])
#plt.axis('off')
plt.subplot(np.ceil(np.sqrt(indexs.__len__()*2)), np.ceil(np.sqrt(indexs.__len__()*2)), i*2 + 2)\
.set_title(model.layers[indexs[i]].name)
plt.imshow(ret[1])
cv2.imwrite('%d.png'%(i+100), ret[1])
#plt.axis('off')
plt.show()
from keras.applications.vgg16 import VGG16
from keras.applications.vgg16 import preprocess_input
#from keras.applications.inception_resnet_v2 import InceptionResNetV2
model = VGG16(weights='imagenet')
#model = InceptionResNetV2(weights='imagenet')
data_img = image.img_to_array(image.load_img('monkey.JPG'))
# VGG16预处理:RGB转BGR,并对每一个颜色通道去均值中心化
data_img = preprocess_input(data_img)
img_show = image.img_to_array(image.load_img('monkey.JPG'))
heatmaps(model, data_img, img_show)
运行结果
最后一层卷积层可视化结果保存为：
三、可视化CNN热力图
代码如下（示例）：
from keras.applications.vgg16 import VGG16
from keras import backend as K
from keras.preprocessing import image
from keras.applications.vgg16 import preprocess_input, decode_predictions
import numpy as np
import matplotlib.pyplot as plt
import cv2
K.clear_session()
model = VGG16(weights='imagenet')
img_path = 'elephant.jpg'
img = image.load_img(img_path, target_size=(224, 224)) # 大小为224*224的Python图像库图像
x = image.img_to_array(img) # 形状为（224， 224， 3）的float32格式Numpy数组
x = np.expand_dims(x, axis=0) # 添加一个维度，将数组转化为（1， 224， 224， 3）的形状批量
x = preprocess_input(x) #按批量进行预处理（按通道颜色进行标准化）
preds = model.predict(x)
print('Predicted:', decode_predictions(preds, top=3)[0])
print('网络认为预测向量中最大激活的元素对应是“非洲象”类别的元素，索引编号:',np.argmax(preds[0]))
#为了展示图像中哪些部分最像非洲象，我们使用Grad-CAM算法：
african_elephant_output = model.output[:, 386] # 预测向量中的非洲象元素
last_conv_layer = model.get_layer('block5_conv3') # block5_conv3层的输出特征图，它是VGG16的最后一个卷积层
grads = K.gradients(african_elephant_output, last_conv_layer.output)[0] # 非洲象类别相对于block5_conv3输出特征图的梯度
pooled_grads = K.mean(grads, axis=(0, 1, 2)) # 形状是（512， ）的向量，每个元素是特定特征图通道的梯度平均大小
iterate = K.function([model.input], [pooled_grads, last_conv_layer.output[0]]) # 这个函数允许我们获取刚刚定义量的值：对于给定样本图像，pooled_grads和block5_conv3层的输出特征图
pooled_grads_value, conv_layer_output_value = iterate([x]) # 给我们两个大象样本图像，这两个量都是Numpy数组
for i in range(512):
conv_layer_output_value[:, :, i] *= pooled_grads_value[i] # 将特征图数组的每个通道乘以这个通道对大象类别重要程度
heatmap = np.mean(conv_layer_output_value, axis=-1) # 得到的特征图的逐通道的平均值即为类激活的热力图
heatmap = np.maximum(heatmap, 0)
heatmap /= np.max(heatmap)
plt.matshow(heatmap)
plt.show()
img = cv2.imread(img_path) # 用cv2加载原始图像
heatmap = cv2.resize(heatmap, (img.shape[1], img.shape[0])) # 将热力图的大小调整为与原始图像相同
heatmap = np.uint8(255 * heatmap) # 将热力图转换为RGB格式
heatmap = cv2.applyColorMap(heatmap, cv2.COLORMAP_JET) # 将热力图应用于原始图像
superimposed_img = heatmap * 0.4 + img # 这里的0.4是热力图强度因子
#plt.imshow(superimposed_img)
#plt.show()
cv2.imwrite('elephant-cam.png', superimposed_img) # 将图像保存到硬盘
运行结果
四、grad-cam可视化(pytorch)
代码如下（示例）：
import argparse
import cv2
import numpy as np
import torch
from torch.autograd import Function
from torchvision import models
class FeatureExtractor():
""" Class for extracting activations and
registering gradients from targetted intermediate layers """
def __init__(self, model, target_layers):
self.model = model
self.target_layers = target_layers
self.gradients = []
def save_gradient(self, grad):
self.gradients.append(grad)
def __call__(self, x):
outputs = []
self.gradients = []
for name, module in self.model._modules.items():
x = module(x)
if name in self.target_layers:
x.register_hook(self.save_gradient)
outputs += [x]
return outputs, x
class ModelOutputs():
""" Class for making a forward pass, and getting:
1. The network output.
2. Activations from intermeddiate targetted layers.
3. Gradients from intermeddiate targetted layers. """
def __init__(self, model, feature_module, target_layers):
self.model = model
self.feature_module = feature_module
self.feature_extractor = FeatureExtractor(self.feature_module, target_layers)
def get_gradients(self):
return self.feature_extractor.gradients
def __call__(self, x):
target_activations = []
for name, module in self.model._modules.items():
if module == self.feature_module:
target_activations, x = self.feature_extractor(x)
elif "avgpool" in name.lower():
x = module(x)
x = x.view(x.size(0),-1)
else:
x = module(x)
return target_activations, x
def preprocess_image(img):
means = [0.485, 0.456, 0.406]
stds = [0.229, 0.224, 0.225]
preprocessed_img = img.copy()[:, :, ::-1]
for i in range(3):
preprocessed_img[:, :, i] = preprocessed_img[:, :, i] - means[i]
preprocessed_img[:, :, i] = preprocessed_img[:, :, i] / stds[i]
preprocessed_img = \
np.ascontiguousarray(np.transpose(preprocessed_img, (2, 0, 1)))
preprocessed_img = torch.from_numpy(preprocessed_img)
preprocessed_img.unsqueeze_(0)
input = preprocessed_img.requires_grad_(True)
return input
def show_cam_on_image(img, mask):
heatmap = cv2.applyColorMap(np.uint8(255 * mask), cv2.COLORMAP_JET)
heatmap = np.float32(heatmap) / 255
cam = heatmap + np.float32(img)
cam = cam / np.max(cam)
cv2.imwrite("cam.jpg", np.uint8(255 * cam))
class GradCam:
def __init__(self, model, feature_module, target_layer_names, use_cuda):
self.model = model
self.feature_module = feature_module
self.model.eval()
self.cuda = use_cuda
if self.cuda:
self.model = model.cuda()
self.extractor = ModelOutputs(self.model, self.feature_module, target_layer_names)
def forward(self, input):
return self.model(input)
def __call__(self, input, index=None):
if self.cuda:
features, output = self.extractor(input.cuda())
else:
features, output = self.extractor(input)
if index == None:
index = np.argmax(output.cpu().data.numpy())
one_hot = np.zeros((1, output.size()[-1]), dtype=np.float32)
one_hot[0][index] = 1
one_hot = torch.from_numpy(one_hot).requires_grad_(True)
if self.cuda:
one_hot = torch.sum(one_hot.cuda() * output)
else:
one_hot = torch.sum(one_hot * output)
self.feature_module.zero_grad()
self.model.zero_grad()
one_hot.backward(retain_graph=True)
grads_val = self.extractor.get_gradients()[-1].cpu().data.numpy()
target = features[-1]
target = target.cpu().data.numpy()[0, :]
weights = np.mean(grads_val, axis=(2, 3))[0, :]
cam = np.zeros(target.shape[1:], dtype=np.float32)
for i, w in enumerate(weights):
cam += w * target[i, :, :]
cam = np.maximum(cam, 0)
cam = cv2.resize(cam, input.shape[2:])
cam = cam - np.min(cam)
cam = cam / np.max(cam)
return cam
class GuidedBackpropReLU(Function):
@staticmethod
def forward(self, input):
positive_mask = (input > 0).type_as(input)
output = torch.addcmul(torch.zeros(input.size()).type_as(input), input, positive_mask)
self.save_for_backward(input, output)
return output
@staticmethod
def backward(self, grad_output):
input, output = self.saved_tensors
grad_input = None
positive_mask_1 = (input > 0).type_as(grad_output)
positive_mask_2 = (grad_output > 0).type_as(grad_output)
grad_input = torch.addcmul(torch.zeros(input.size()).type_as(input),
torch.addcmul(torch.zeros(input.size()).type_as(input), grad_output,
positive_mask_1), positive_mask_2)
return grad_input
class GuidedBackpropReLUModel:
def __init__(self, model, use_cuda):
self.model = model
self.model.eval()
self.cuda = use_cuda
if self.cuda:
self.model = model.cuda()
def recursive_relu_apply(module_top):
for idx, module in module_top._modules.items():
recursive_relu_apply(module)
if module.__class__.__name__ == 'ReLU':
module_top._modules[idx] = GuidedBackpropReLU.apply
# replace ReLU with GuidedBackpropReLU
recursive_relu_apply(self.model)
def forward(self, input):
return self.model(input)
def __call__(self, input, index=None):
if self.cuda:
output = self.forward(input.cuda())
else:
output = self.forward(input)
if index == None:
index = np.argmax(output.cpu().data.numpy())
one_hot = np.zeros((1, output.size()[-1]), dtype=np.float32)
one_hot[0][index] = 1
one_hot = torch.from_numpy(one_hot).requires_grad_(True)
if self.cuda:
one_hot = torch.sum(one_hot.cuda() * output)
else:
one_hot = torch.sum(one_hot * output)
# self.model.features.zero_grad()
# self.model.classifier.zero_grad()
one_hot.backward(retain_graph=True)
output = input.grad.cpu().data.numpy()
output = output[0, :, :, :]
return output
def get_args():
parser = argparse.ArgumentParser()
parser.add_argument('--use-cuda', action='store_true', default=False,
help='Use NVIDIA GPU acceleration')
parser.add_argument('--image-path', type=str, default='./examples/both.png',
help='Input image path')
args = parser.parse_args()
args.use_cuda = args.use_cuda and torch.cuda.is_available()
if args.use_cuda:
print("Using GPU for acceleration")
else:
print("Using CPU for computation")
return args
def deprocess_image(img):
""" see https://github.com/jacobgil/keras-grad-cam/blob/master/grad-cam.py#L65 """
img = img - np.mean(img)
img = img / (np.std(img) + 1e-5)
img = img * 0.1
img = img + 0.5
img = np.clip(img, 0, 1)
return np.uint8(img*255)
if __name__ == '__main__':
""" python grad_cam.py <path_to_image>
1. Loads an image with opencv.
2. Preprocesses it for VGG19 and converts to a pytorch variable.
3. Makes a forward pass to find the category index with the highest score,
and computes intermediate activations.
Makes the visualization. """
args = get_args()
# Can work with any model, but it assumes that the model has a
# feature method, and a classifier method,
# as in the VGG models in torchvision.
model = models.resnet50(pretrained=True)
grad_cam = GradCam(model=model, feature_module=model.layer4, \
target_layer_names=["2"], use_cuda=args.use_cuda)
img = cv2.imread(args.image_path, 1)
img = np.float32(cv2.resize(img, (224, 224))) / 255
input = preprocess_image(img)
# If None, returns the map for the highest scoring category.
# Otherwise, targets the requested index.
target_index = None
mask = grad_cam(input, target_index)
show_cam_on_image(img, mask)
gb_model = GuidedBackpropReLUModel(model=model, use_cuda=args.use_cuda)
print(model._modules.items())
gb = gb_model(input, index=target_index)
gb = gb.transpose((1, 2, 0))
cam_mask = cv2.merge([mask, mask, mask])
cam_gb = deprocess_image(cam_mask*gb)
gb = deprocess_image(gb)
cv2.imwrite('gb.jpg', gb)
cv2.imwrite('cam_gb.jpg', cam_gb)
运行结果
总结
提示：这里对文章进行总结：
关于上述图片及相关代码附链接(包括vgg16_weights.h5权重文件)：
可视化-VGG16模型(20210527).zip
扫一扫
551
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
