0

I have two views of one object and using multi view cnn from .https://github.com/SAMY-ER/Multi-View-Image-Classification. This works great and I am getting better results. Though how can I apply grad cam to it. any help is appreciated . I want to see the cam on both views to see what parts in two images contributing to classification This is my model

MULTI-VIEW CONVOLUTIONAL NEURAL NETWORK (MVCNN) ARCHITECTURE

class MVCNN(nn.Module):
   def __init__(self, num_classes=1000, pretrained=True):
    super(MVCNN, self).__init__()
resnet = models.resnet50(pretrained = pretrained)

fc_in_features = resnet.fc.in_features
self.features = nn.Sequential(*list(resnet.children())[:-1])
self.classifier = nn.Sequential(
    nn.Dropout(),
    ## multiplying by 2 to take care of two views
    nn.Linear(fc_in_features * 2, 2048),
    nn.ReLU(inplace=True),
    nn.Dropout(),
    nn.Linear(2048, 2048),
    nn.ReLU(inplace=True),
    nn.Linear(2048, num_classes)
)

def forward(self, inputs): # inputs.shape = samples x views x height x width x channels inputs = inputs.transpose(0, 1) view_features = [] for view_batch in inputs: view_batch = self.features(view_batch) view_batch = view_batch.view(view_batch.shape[0], view_batch.shape[1:].numel()) view_features.append(view_batch)

concat_views = torch.cat(view_features,-1)
#pooled_views, _ = torch.max(torch.stack(view_features), 0)
#outputs = self.classifier(pooled_views)
outputs = self.classifier(concat_views)
return outputs

0 Answers0