CNN可视化-Guided Grad-CAM

原文:http://openaccess.thecvf.com/content_ICCV_2017/papers/Selvaraju_Grad-CAM_Visual_Explanations_ICCV_2017_paper.pdf
在这里插入图片描述
(b,g)Guided Backprop 提供了高像素的可视化效果对有贡献的特征, (c,h)Grad-CAM定位class-discriminative区域。(d,i)Guided Backprop 结合Grad-CAM获得了高像素可视化和class-discriminative。

CAM

首先recap一下CAM(class activation mapping)。对于global average pooling前有 K K K个feature maps A k ∈ R u × v A^{k}\in R^{u\times v} AkRu×v, u u u代表宽度, v v v代表高度。这些feature maps通过global average pooling和线性转换得到类别 c c c的分数:
在这里插入图片描述
为了产生类别 c c c的llocalization map L C A M c ∈ R u × v L_{CAM}^{c}\in R^{u\times v} LCAMcRu×v,使用已经学习到的FC参数来线性组合所有的feature maps:

在这里插入图片描述
之后, L C A M c ∈ R u × v L_{CAM}^{c}\in R^{u\times v} LCAMcRu×v在spatial上进行归一化到[0,1]。

Grad-CAM

Gradient-weighted CAM,使用梯度来计算权重。对于类别的分数 y c y^{c} yc,设某个卷积层之后的feature maps A A A,计算关于 y c y^{c} yc的梯度 ∂ y c ∂ A i j k \frac{\partial y^{c}}{\partial A_{ij}^{k}} Aijkyc。使用global average pooling来整合梯度值来obtain每个feature map的权重 α k c \alpha_{k}^{c} αkc:

在这里插入图片描述
加权结合所有feature maps,并且加入ReLU函数:
在这里插入图片描述
得到的 L C A M c ∈ R u × v L_{CAM}^{c}\in R^{u\times v} LCAMcRu×v与卷积feature maps的尺寸一致,比如VGGNet和AlexNet的最后一个卷积层大小为 14 × 14 14\times 14 14×14 L C A M c ∈ R u × v L_{CAM}^{c}\in R^{u\times v} LCAMcRu×v还要进行归一化到[0,1]。值的一提的是使用ReLU,目的是为了只对feature maps中的positive影响的pixels有兴趣。没有ReLU,localization maps有时的强调区域大于期望的类别并且获得更差的localization表现。直觉上的,负值表明属于其他的类别,使用ReLU可以阻挡他们。
可视化不同卷积层后的feature maps的localizations :
在这里插入图片描述
从图中可以看出,越浅的层localizations表现越差。

Guided Grad-CAM

Grad-CAM可视化是class-discriminative并且较好地localize图像区域,但是缺乏像pixel-space梯度可视化方法(如deconvolution和Guided Backpropagation)那样展示细粒度重要性。因此将Guided Backpropagation或deconvolution结合到Grad-CAM通过point-wise multiplicatipon(这里 L C A M c L_{CAM}^{c} LCAMc需要升采样到原始的输入图像大小通过bi-linear interpolation),结合之后,可视化的feature maps同时配备high-resolution和class-discriminative。作者认为Guided Backpropagation比deconvolution具有更小的噪声,所以选择使用Guided Backpropagation。过程总览如下:
在这里插入图片描述
下面展示了Guided Backpropagation和deconvolution在通过ReLU层的backward梯度更新区别。论文地址:https://arxiv.org/pdf/1412.6806.pdf
在这里插入图片描述

  • 普通的backward,那些被置0的位置的反向梯度为0.
  • deconvnet的backward:那些负值的梯度置0
  • guided-backpropgation:以上两者的结合

代码实现

完整的代码可参考:https://github.com/kazuto1011/grad-cam-pytorch
导入包:

from collections import OrderedDict, Sequence
import numpy as np
import torch
import torch.nn as nn
from torch.nn import functional as F
from tqdm import tqdm

创建一个基类,用于实现forward计算概率,backward传播class-specific的梯度,删除hook函数等功能。

class _BaseWrapper(object):
    """
    Please modify forward() and backward() according to your task.
    """

    def __init__(self, model):
        super(_BaseWrapper, self).__init__()
        self.device = next(model.parameters()).device
        self.model = model
        self.handlers = []  # a set of hook function handlers

    def _encode_one_hot(self, ids):
        one_hot = torch.zeros_like(self.logits).to(self.device)
        one_hot.scatter_(1, ids, 1.0)
        return one_hot

    def forward(self, image):
        """
        计算各类的概率并返回排名信息
        """
        self.model.zero_grad()
        self.logits = self.model(image)
        self.probs = F.softmax(self.logits, dim=1)
        return self.probs.sort(dim=1, descending=True)

    def backward(self, ids):
        """
        Class-specific backpropagation
        Either way works:
        1. self.logits.backward(gradient=one_hot, retain_graph=True)
        2. (self.logits * one_hot).sum().backward(retain_graph=True)
        """

        one_hot = self._encode_one_hot(ids)
        # retain_graph表明一直保留计算图中的梯度
        # 因为多个类别要分别计算的class-specific梯度,进行多次backward
        self.logits.backward(gradient=one_hot, retain_graph=True)

    def generate(self):
        raise NotImplementedError

    def remove_hook(self):
        """
        Remove all the forward/backward hook functions
        """
        for handle in self.handlers:
            handle.remove()

back-propagation类用以计算image数据的梯度:

class BackPropagation(_BaseWrapper):
    def forward(self, image):
        self.image = image.requires_grad_()
        return super(BackPropagation, self).forward(self.image)

    def generate(self):
        gradient = self.image.grad.clone()
        # 清空梯度,防止下一次backward时累积
        self.image.grad.zero_()
        return gradient

实现GuidedBackPropagation,在朴素BP的基础上将输入ReLU前image的<0的梯度置0。

class GuidedBackPropagation(BackPropagation):
    def __init__(self, model):
        super(GuidedBackPropagation, self).__init__(model)

        def backward_hook(module, grad_in, grad_out):
            # Cut off negative gradients
            if isinstance(module, nn.ReLU):
                return (torch.clamp(grad_in[0], min=0.0),)

        for module in self.model.named_modules():
            self.handlers.append(module[1].register_backward_hook(backward_hook))

不执行朴素的BP,输入ReLU前image的梯度:ReLU输出image的梯度中<0的梯度置0

class Deconvnet(BackPropagation):
    def __init__(self, model):
        super(Deconvnet, self).__init__(model)

        def backward_hook(module, grad_in, grad_out):
            # Cut off negative gradients and ignore ReLU
            if isinstance(module, nn.ReLU):
                return (torch.clamp(grad_out[0], min=0.0),)

        for module in self.model.named_modules():
            self.handlers.append(module[1].register_backward_hook(backward_hook))

Grad-CAM

class GradCAM(_BaseWrapper):
    def __init__(self, model, candidate_layers=None):
        super(GradCAM, self).__init__(model)
        self.fmap_pool = OrderedDict()  # 保存每一层之后的feature maps
        self.grad_pool = OrderedDict()  # 保存每一层之后的feature maps的梯度
        self.candidate_layers = candidate_layers  # list,指定需要哪些层的featuremaps值和梯度信息

        def forward_hook(key):
            def forward_hook_(module, input, output):
                # Save featuremaps
                self.fmap_pool[key] = output.detach()

            return forward_hook_

        def backward_hook(key):
            def backward_hook_(module, grad_in, grad_out):
                # Save the gradients correspond to the featuremaps
                self.grad_pool[key] = grad_out[0].detach()

            return backward_hook_

        # If any candidates are not specified, the hook is registered to all the layers.
        for name, module in self.model.named_modules():
            if self.candidate_layers is None or name in self.candidate_layers:
                self.handlers.append(module.register_forward_hook(forward_hook(name)))
                self.handlers.append(module.register_backward_hook(backward_hook(name)))

	# 获取某一层后的feature map的值和梯度信息
    def _find(self, pool, target_layer):
        if target_layer in pool.keys():
            return pool[target_layer]
        else:
            raise ValueError("Invalid layer name: {}".format(target_layer))

    def _compute_grad_weights(self, grads):
        return F.adaptive_avg_pool2d(grads, 1)

    def forward(self, image):
        self.image_shape = image.shape[2:]
        return super(GradCAM, self).forward(image)
	
	# 生成heatmap
    def generate(self, target_layer):
        fmaps = self._find(self.fmap_pool, target_layer)
        grads = self._find(self.grad_pool, target_layer)
        weights = self._compute_grad_weights(grads)

        gcam = torch.mul(fmaps, weights).sum(dim=1, keepdim=True)
        gcam = F.relu(gcam)
		
		# 双线性插值升采样
        gcam = F.interpolate(
            gcam, self.image_shape, mode="bilinear", align_corners=False
        )

		# 对生成mask进行归一化
        B, C, H, W = gcam.shape
        gcam = gcam.view(B, -1)
        gcam -= gcam.min(dim=1, keepdim=True)[0]
        gcam /= gcam.max(dim=1, keepdim=True)[0]
        gcam = gcam.view(B, C, H, W)

        return gcam

主函数调用:

"""
    Common usage:
    1. Wrap your model with visualization classes defined in grad_cam.py
    2. Run forward() with images
    3. Run backward() with a list of specific classes
    4. Run generate() to export results
    """

    # =========================================================================
    print("Vanilla Backpropagation:")

    bp = BackPropagation(model=model)
    probs, ids = bp.forward(images)

	# 选取前topk个类求class-specific的梯度
    for i in range(topk):
        # In this example, we specify the high confidence classes
        bp.backward(ids=ids[:, [i]])
        gradients = bp.generate()

        # Save results as image files
        for j in range(len(images)):
            print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j, i]))

            save_gradient(
                filename=osp.join(
                    output_dir,
                    "{}-{}-vanilla-{}.png".format(j, arch, classes[ids[j, i]]),
                ),
                gradient=gradients[j],
            )

    # Remove all the hook function in the "model"
    bp.remove_hook()

    # =========================================================================
    print("Deconvolution:")

    deconv = Deconvnet(model=model)
    _ = deconv.forward(images)

    for i in range(topk):
        deconv.backward(ids=ids[:, [i]])
        gradients = deconv.generate()

        for j in range(len(images)):
            print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j, i]))

            save_gradient(
                filename=osp.join(
                    output_dir,
                    "{}-{}-deconvnet-{}.png".format(j, arch, classes[ids[j, i]]),
                ),
                gradient=gradients[j],
            )

    deconv.remove_hook()

    # =========================================================================
    print("Grad-CAM/Guided Backpropagation/Guided Grad-CAM:")

    gcam = GradCAM(model=model)
    _ = gcam.forward(images)

    gbp = GuidedBackPropagation(model=model)
    _ = gbp.forward(images)

    for i in range(topk):
        # Guided Backpropagation
        gbp.backward(ids=ids[:, [i]])
        gradients = gbp.generate()

        # Grad-CAM
        gcam.backward(ids=ids[:, [i]])
        regions = gcam.generate(target_layer=target_layer)

        for j in range(len(images)):
            print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j, i]))

            # Guided Backpropagation
            save_gradient(
                filename=osp.join(
                    output_dir,
                    "{}-{}-guided-{}.png".format(j, arch, classes[ids[j, i]]),
                ),
                gradient=gradients[j],
            )

            # Grad-CAM
            save_gradcam(
                filename=osp.join(
                    output_dir,
                    "{}-{}-gradcam-{}-{}.png".format(
                        j, arch, target_layer, classes[ids[j, i]]
                    ),
                ),
                gcam=regions[j, 0],
                raw_image=raw_images[j],
            )

            # Guided Grad-CAM
            save_gradient(
                filename=osp.join(
                    output_dir,
                    "{}-{}-guided_gradcam-{}-{}.png".format(
                        j, arch, target_layer, classes[ids[j, i]]
                    ),
                ),
                gradient=torch.mul(regions, gradients)[j],
            )

下面展示一个整体可执行的程序。
使用Pytorch预训练的ResNet-50遍历ImageNet验证集生成top-1预测结果的热图。

from collections.abc import Sequence

import numpy as np
import torch
import torch.nn as nn
from torch.nn import functional as F

import os
import copy
import os.path as osp

import click
import cv2
import matplotlib.cm as cm
import numpy as np
import torch
import torch.nn.functional as F
from torchvision import models, transforms
import matplotlib.cm as cm
from PIL import Image


class _BaseWrapper(object):
    def __init__(self, model):
        super(_BaseWrapper, self).__init__()
        self.device = next(model.parameters()).device
        self.model = model
        self.handlers = []  # a set of hook function handlers

    def _encode_one_hot(self, ids):
        one_hot = torch.zeros_like(self.logits).to(self.device)
        one_hot.scatter_(1, ids, 1.0)
        return one_hot

    def forward(self, image):
        self.image_shape = image.size()[2:]
        self.logits = self.model(image)
        self.probs = F.softmax(self.logits, dim=1)
        return self.probs.sort(dim=1, descending=True)  # ordered results

    def backward(self, ids):
        """
        Class-specific backpropagation
        """
        one_hot = self._encode_one_hot(ids)
        self.model.zero_grad()
        self.logits.backward(gradient=one_hot, retain_graph=True)

    def generate(self):
        raise NotImplementedError

    def remove_hook(self):
        """
        Remove all the forward/backward hook functions
        """
        for handle in self.handlers:
            handle.remove()


class BackPropagation(_BaseWrapper):
    def forward(self, image):
        self.image = image.requires_grad_()
        return super(BackPropagation, self).forward(self.image)

    def generate(self):
        gradient = self.image.grad.clone()
        self.image.grad.zero_()
        return gradient


class GradCAM(_BaseWrapper):
    """
    "Grad-CAM: Visual Explanations from Deep Networks via Gradient-based Localization"
    https://arxiv.org/pdf/1610.02391.pdf
    Look at Figure 2 on page 4
    """

    def __init__(self, model, candidate_layers=None):
        super(GradCAM, self).__init__(model)
        self.fmap_pool = {}
        self.grad_pool = {}
        self.candidate_layers = candidate_layers  # list

        def save_fmaps(key):
            def forward_hook(module, input, output):
                self.fmap_pool[key] = output.detach()

            return forward_hook

        def save_grads(key):
            def backward_hook(module, grad_in, grad_out):
                self.grad_pool[key] = grad_out[0].detach()

            return backward_hook

        # If any candidates are not specified, the hook is registered to all the layers.
        for name, module in self.model.named_modules():
            if self.candidate_layers is None or name in self.candidate_layers:
                self.handlers.append(module.register_forward_hook(save_fmaps(name)))
                self.handlers.append(module.register_backward_hook(save_grads(name)))

    def _find(self, pool, target_layer):
        if target_layer in pool.keys():
            return pool[target_layer]
        else:
            raise ValueError("Invalid layer name: {}".format(target_layer))

    def generate(self, target_layer):
        fmaps = self._find(self.fmap_pool, target_layer)
        grads = self._find(self.grad_pool, target_layer)
        weights = F.adaptive_avg_pool2d(grads, 1)

        gcam = torch.mul(fmaps, weights).sum(dim=1, keepdim=True)
        gcam = F.relu(gcam)
        gcam = F.interpolate(
            gcam, self.image_shape, mode="bilinear", align_corners=False
        )

        B, C, H, W = gcam.shape
        gcam = gcam.view(B, -1)
        gcam -= gcam.min(dim=1, keepdim=True)[0]
        gcam /= gcam.max(dim=1, keepdim=True)[0]
        gcam = gcam.view(B, C, H, W)

        return gcam


def save_gradcam(filename, gcam, raw_image, paper_cmap=False):
    gcam = gcam.cpu().numpy()
    cmap = cm.jet(gcam)[..., :3] * 255.0
    if paper_cmap:
        alpha = gcam[..., None]
        gcam = alpha * cmap + (1 - alpha) * raw_image
    else:
        gcam = (cmap.astype(np.float) + raw_image.astype(np.float)) / 2
    gcam = np.uint8(gcam)
    im=Image.fromarray(gcam) 
    im.save(filename)


os.environ["CUDA_VISIBLE_DEVICES"] = '1'

image_dir = '/dev/shm/ImageNet/val/'
dirs = []
for dir in os.listdir(image_dir):
    dirs.append(dir)
dirs.sort()



net1 = models.__dict__['resnet50'](pretrained=True)
net1.cuda()
net1.eval()


bp = BackPropagation(model=net1)
gcam = GradCAM(model=net1)

global_id = 0
label = -1  # label是图像的ground-truth label
for dir in dirs:
    cur_dir = os.path.join(image_dir, dir)
    label = label + 1
    for jpeg in os.listdir(cur_dir):
        raw_image = cv2.imread(os.path.join(cur_dir, jpeg))
        raw_image = cv2.resize(raw_image, (224,) * 2)
        image = transforms.Compose(
            [
                transforms.ToTensor(),
                transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
            ])(raw_image[..., ::-1].copy())
        image = torch.unsqueeze(image, 0).cuda()

        probs, ids = bp.forward(image)
        _ = gcam.forward(image)
        # ids[:, [0]]这里0表示top-1的预测类别
        gcam.backward(ids=ids[:, [0]])
        regions = gcam.generate(target_layer='layer4')
        save_gradcam(
        filename=osp.join(
            '/home/user/hhd/winycg/self-kd/heatmap/', str(global_id))+'.jpg',
            gcam=regions[0, 0],
            raw_image=raw_image,
        )

        global_id += 1
已标记关键词 清除标记
相关推荐
©️2020 CSDN 皮肤主题: 编程工作室 设计师:CSDN官方博客 返回首页