gradcam.py

# import the necessary packages
from tensorflow.keras.models import Model
from tensorflow.keras import Input
import tensorflow as tf
import numpy as np
import cv2


class GradCAM:
    def __init__(self, model, classIdx, layerName=None):
        # store the model, the class index used to measure the class
        # activation map, and the layer to be used when visualizing
        # the class activation map
        self.model = model
        self.classIdx = classIdx
        self.layerName = layerName

        # if the layer name is None, attempt to automatically find
        # the target output layer
        if self.layerName is None:
            self.layerName = self.find_target_layer()

    def find_target_layer(self):
        # attempt to find the final convolutional layer in the network
        # by looping over the layers of the network in reverse order
        for layer in reversed(self.model.layers[0].layers):
            # check to see if the layer has a 4D output
            if len(layer.output_shape) == 4:
                return layer.name

        # otherwise, we could not find a 4D layer so the GradCAM
        # algorithm cannot be applied
        raise ValueError("Could not find 4D layer. Cannot apply GradCAM.")

    def heatmap(self, image, convOutputs, grads, eps=1e-8):
        # compute the guided gradients
        castConvOutputs = tf.cast(convOutputs > 0, "float32")
        castGrads = tf.cast(grads > 0, "float32")
        guidedGrads = castConvOutputs * castGrads * grads

        # the convolution and guided gradients have a batch dimension
        # (which we don't need) so let's grab the volume itself and
        # discard the batch
        convOutputs = convOutputs[0]
        guidedGrads = guidedGrads[0]

        # compute the average of the gradient values, and using them
        # as weights, compute the ponderation of the filters with
        # respect to the weights
        weights = tf.reduce_mean(guidedGrads, axis=(0, 1))
        cam = tf.reduce_sum(tf.multiply(weights, convOutputs), axis=-1)

        # grab the spatial dimensions of the input image and resize
        # the output class activation map to match the input image
        # dimensions
        (w, h) = (image.shape[2], image.shape[1])
        heatmap = cv2.resize(cam.numpy(), (w, h))

        # normalize the heatmap such that all values lie in the range
        # [0, 1], scale the resulting values to the range [0, 255],
        # and then convert to an unsigned 8-bit integer
        numer = heatmap - np.min(heatmap)
        denom = (heatmap.max() - heatmap.min()) + eps
        heatmap = numer / denom
        heatmap = (heatmap * 255).astype("uint8")

        # return the resulting heatmap to the calling function
        return heatmap

    def compute_heatmap(self, images, eps=1e-8):
        # construct our gradient model by supplying (1) the inputs
        # to our pre-trained model, (2) the output of the (presumably)
        # final 4D layer in the network, and (3) the output of the
        # softmax activations from the model

        visModel1 = Model(self.model.layers[0].input, [self.model.layers[0].get_layer(self.layerName).output, self.model.layers[0].output])
        visModel2 = Model(self.model.layers[1].input, [self.model.layers[1].get_layer(self.layerName).output, self.model.layers[1].output])
        classifier = Model(self.model.layers[2].input, self.model.layers[2].output)
        concat_dimension = 1
        # record operations for automatic differentiation
        with tf.GradientTape(persistent=True) as tape:
            # cast the image tensor to a float-32 data type, pass the
            # image through the gradient model, and grab the loss
            # associated with the specific class index
            input1 = tf.cast(images[0], tf.float32)
            input2 = tf.cast(images[1], tf.float32)

            convOutputs1, visFeat1= visModel1(input1)
            convOutputs2, visFeat2= visModel2(input2)
            features = tf.concat([visFeat1, visFeat2], concat_dimension)
            predictions = classifier(features)

            loss = predictions[:, self.classIdx]

        # use automatic differentiation to compute the gradients
        grads1 = tape.gradient(loss, convOutputs1)
        heatmap1 = self.heatmap(images[0],convOutputs1,grads1)
        grads2 = tape.gradient(loss, convOutputs2)
        heatmap2 = self.heatmap(images[1],convOutputs2,grads2)
        del tape
        return [heatmap1, heatmap2]
    def overlay_heatmap(self, heatmap, image, alpha=0.5,
                        colormap=cv2.COLORMAP_VIRIDIS):
        # apply the supplied color map to the heatmap and then
        # overlay the heatmap on the input image
        heatmap = cv2.applyColorMap(heatmap, colormap)
        output = cv2.addWeighted(image, alpha, heatmap, 1 - alpha, 0)

        # return a 2-tuple of the color mapped heatmap and the output,
        # overlaid image
        return (heatmap, output)