i am trying to generate a heatmap for image depending on weights from last conv layer in pretrained densenet121, but when i try to multiply the weights by the output of model in gives me this error:
“ValueError: operands could not be broadcast together with shapes (1024,) (7,7)”
this is the code :
import os
import torch
import urllib
from PIL import Image
from torchvision import transforms
import numpy as np
import cv2
# load the model
model = torch.hub.load('pytorch/vision:v0.9.0', 'densenet121', pretrained=True)
# or any of these variants
# model = torch.hub.load('pytorch/vision:v0.9.0', 'densenet169', pretrained=True)
# model = torch.hub.load('pytorch/vision:v0.9.0', 'densenet201', pretrained=True)
# model = torch.hub.load('pytorch/vision:v0.9.0', 'densenet161', pretrained=True)
model.eval()
filename = 'dog.jpg'
# get input image and apply preprocessing
input_image = Image.open(filename)
preprocess = transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
])
input_tensor = preprocess(input_image)
input_batch = input_tensor.unsqueeze(0)
# output = model(input_batch)
# print(output.shape)
# The output has unnormalized scores. To get probabilities, you can run a softmax on it.
# probabilities = torch.nn.functional.softmax(output[0], dim=0)
# print(probabilities)
named_layers = dict(model.named_modules())
# for layer in named_layers:
# print(layer)
# print(named_layers['features.denseblock4.denselayer16'])
class densenet_last_layer(torch.nn.Module):
def __init__(self, model):
super(densenet_last_layer, self).__init__()
self.features = torch.nn.Sequential(
*list(model.children())[:-1]
)
def forward(self, x):
x = self.features(x)
x = torch.nn.functional.relu(x, inplace=True)
return x
conv_model = densenet_last_layer(model)
# print(conv_model)
input_batch = torch.autograd.Variable(input_batch)
conv_output = conv_model(input_batch)
# print(conv_output.shape)
conv_output = conv_output.cpu().data.numpy()
# conv_output = np.squeeze(conv_output)
# for state in model.state_dict():
# print(state)
weights = model.state_dict()['classifier.weight']
weights = weights.cpu().numpy()
bias = model.state_dict()['classifier.bias']
bias = bias.cpu().numpy()
# print(conv_output.shape)
heatmap = None
for i in range(0, len(weights)):
map = conv_output[0, i, :, :]
if i == 0:
heatmap = weights[i] * map
else:
heatmap += weights[i] * map
# ---- Blend original and heatmap
npHeatmap = heatmap.cpu().data.numpy()
transCrop = 224
imgOriginal = cv2.imread(filename, 1)
imgOriginal = cv2.resize(imgOriginal, (transCrop, transCrop))
cam = npHeatmap / np.max(npHeatmap)
cam = cv2.resize(cam, (transCrop, transCrop))
heatmap = cv2.applyColorMap(np.uint8(255 * cam), cv2.COLORMAP_JET)
img = heatmap * 0.5 + imgOriginal
cv2.imwrite(os.getcwd(), img)
print(model.state_dict()['classifier.weight'])
Based on the error message the heatmap and imgOriginal are two numpy arrays in different shapes, which cannot be accumulated:
heatmap = np.random.randn(1024,)
imgOriginal = np.random.randn(7, 7)
img = heatmap * 0.5 + imgOriginal
> ValueError: operands could not be broadcast together with shapes (1024,) (7,7)
so you would have to make sure the heatmap has the same shape as imgOriginal (or can be broadcasted).
I’m also unsure, what exactly is happening in these lines of code:
heatmap = weights[i] * map
as it seems you are indexing the weight matrix of the last linear layer and are multiplying it with a channel of an output activation of a conv layer.
i am trying to generate heatmap but always have corrupted images…i am not sure about the logic of generating CAM but i will explain the steps i do so that if you can please help me…
1-) the first thing i load my trained model then remove the last classification layer from it to get the last conv layer.
2-) then i convert the image to tensor so i can feed it to the last convolutional layer and get the output feature map.
3-) i loop through the weights of classification layer of my trained model and then multiply the classification weights by output feature map.
that was the logic i do…i am not pretty sure it is right because i am new to deep learning and don’t have a lot of experience with it.
i would be very thankful for you help.
here is the code :
import os
import numpy as np
import time
import sys
from PIL import Image
import heatmap_test as ht
import cv2
import torch
import torch.nn as nn
import torch.backends.cudnn as cudnn
import torchvision
import torchvision.transforms as transforms
from densenet import densenet121
from densenet import densenet169
from densenet import densenet201
from densenet import DenseNet
from torchvision.transforms import ToTensor
class densenet_last_layer(torch.nn.Module):
def __init__(self, model):
super(densenet_last_layer, self).__init__()
# remove last classification layer
self.features = torch.nn.Sequential(
*list(model.children())[:-1]
)
def forward(self, x):
x = self.features(x)
x = torch.nn.functional.relu(x, inplace=True)
return x
class DrawHeatmap():
def __init__(self, pathModel, img, pathOutputFile):
model = densenet121(False)
modelCheckpoint = torch.load(pathModel, map_location=torch.device('cpu'))
model.load_state_dict(modelCheckpoint['state_dict'], strict=False)
named_layers = dict(model.named_modules())
model.eval()
weights = None
bias = None
for name, params in model.named_parameters():
print(name)
if name == 'classifier.weight':
weights = params
if name == 'classifier.bias':
bias = params
conv_model = densenet_last_layer(model)
# ---- Initialize the image transform - resize + normalize
normalize = transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
transformList = []
# transformList.append(transforms.Resize(transCrop))
transformList.append(transforms.ToTensor())
# transformList.append(normalize)
transformSequence = transforms.Compose(transformList)
imageData = Image.open(img).convert('RGB')
imageData = transformSequence(imageData)
imageData = imageData.unsqueeze(0)
input_img = torch.autograd.Variable(imageData)
output = conv_model(input_img)
# print(output.shape)
#
# output = output.cpu().data.numpy()
# output = np.squeeze(output)
print(output.shape)
heatmap = None
for i in range(0, len(weights)):
map = output[0, i, :, :]
if i == 0:
heatmap = weights[i] * map
else:
heatmap += weights[i] * map
npHeatmap = heatmap.cpu().data.numpy()
imgOriginal = cv2.imread(img, 1)
imgOriginal = cv2.resize(imgOriginal, (224, 224))
cam = npHeatmap / np.max(npHeatmap)
cam = cv2.resize(cam, (224, 224))
heatmap = cv2.applyColorMap(np.uint8(255 * cam), cv2.COLORMAP_JET)
img = heatmap * 0.5 + imgOriginal
cv2.imwrite(pathOutputFile, img)
pathInputImage = 'C:/Users/mahmo/Desktop/best projects/Stanford Code/cxr-code/00000001_000.png'
pathModel = 'C:/Users/mahmo/Desktop/best projects/Stanford Code/cxr-code/m-25012018-123527.pth.tar'
pathOutputImage = 'C:/Users/mahmo/Desktop/best projects/Stanford Code/cxr-code/heatmap_test.png'
h = DrawHeatmap(pathModel, pathInputImage, pathOutputImage)
# ht.draw_CAM(pathModel, pathInputImage, True)