I am making use of domain adversarial learning with a gradient reverse layer. The job of the domain discriminator in my case is to classify the domain of genuine data alone.
class GradientReversalF(torch.autograd.Function):
@staticmethod
def forward(ctx, x, alpha):
"let the input unchaged"
ctx.save_for_backward(alpha)
return x
@staticmethod
def backward(ctx, grad_output):
"reverse the gradient by multipling -alpha"
alpha = ctx.saved_tensors[0]
if ctx.needs_input_grad[0]:
grad_output = (grad_output * (-alpha))
return (grad_output, None)
class GradientReverse(nn.Module):
def __init__(self, alpha, *args, **kwargs):
"Reverse GR layer hook"
super().__init__(*args, **kwargs)
self.alpha = torch.tensor(alpha, requires_grad=False)
assert alpha > 0, 'alpha must be > 0'
print(f"The gradient will be multiplied by: {-alpha}")
def forward(self, x):
return GradientReversalF.apply(x, self.alpha)
class Discriminator(nn.Module):
def __init__(self,block,alpha):
super(Discriminator, self).__init__()
self.fc1 = nn.Linear(128 * block.expansion, 128 * block.expansion)
self.fc1.weight.data.normal_(0, 0.01)
self.fc1.bias.data.fill_(0.0)
self.fc2 = nn.Linear(128 * block.expansion, 3)
self.fc2.weight.data.normal_(0, 0.3)
self.fc2.bias.data.fill_(0.0)
self.relu = nn.ReLU()
self.dropout = nn.Dropout(0.3)
self.grl_layer = GradientReverse(alpha)
def forward(self, feature):
feature = self.grl_layer.forward(feature)
feature = self.fc1(feature)
feature = self.relu(feature)
feature = self.dropout(feature)
feature = self.fc2(feature)
return feature
def domain_disc():
torch.manual_seed(0)
model =Discriminator(ECABasicBlock,1)
return model
What is confusing me is that the first tsne plot, where genuine data is clustered together ( domain discriminator did a bad job ) gives me higher accuracy( 0.32) for the domain discriminator than the second and third tsne plots where genuine data seems more separated.( accuracies 0.26 for second and 0.02 for third respectively). Does this mean the tsne plot is not exactly reflective of how well the domain discriminator works because the tsne plot has brought down dimensions to 2 to plot?
