Hi !
I need to build a tiny Deep Learning project for school and I am using pytorch to compare all of my modules.
Most module returns to me the same gradient and output as pytorch and I am happy with that.
But I fail all my tests involving my linear Module (wx + b)
Here is my test :
TestLinearModule(unittest.TestCase):
def __init__(self, *args, **kwargs):
super(TestLinearModule, self).__init__(*args, **kwargs)
self.batch = 2
self.feature_in = 3
self.feature_out = 3
def test_linear_forward_random_init(self):
"""
test the linear module forward pass with with specific weights
"""
x = np.random.randn(self.batch, self.feature_in)
_x = torch.tensor(x, dtype=torch.float32)
weights = np.random.randn(self.feature_in, self.feature_out).astype(np.float32)
_weights = torch.tensor(weights, dtype=torch.float32)
bias = np.zeros((self.feature_out, ))
_bias = torch.tensor(bias, dtype=torch.float32)
linear = Linear(self.feature_in,self.feature_out, weights, bias)
oracle_linear = torch.nn.Linear(self.feature_in, self.feature_out, bias=False)
oracle_linear.weight = torch.nn.parameter.Parameter(_weights, requires_grad=False)
oracle_linear.bias = torch.nn.parameter.Parameter(_bias, requires_grad=False)
print("weights : ", weights)
print("_weights : ", oracle_linear.weight)
print("bias : ", bias)
print("_bias : ", oracle_linear.bias)
output = linear.forward(x)
oracle_output = oracle_linear(_x)
# test output shape
self.assertEqual(output.shape, (self.batch, self.feature_out))
# test output value
np.testing.assert_array_equal(output, oracle_output.detach())
Here is the code of my linear class :
class Linear(Module):
"""
A class implementing a linear module
linear operation : f(x) = XW
"""
def __init__(self,features_in,features_out,weights=None, bias=None):
"""
Args:
features_in: features_in size
features_out: features_out size
weights: the initialized weights
bias : the initilized bais
"""
super(Linear, self).__init__()
self.features_in = features_in
self.features_out = features_out
if not weights is None:
assert weights.shape[0] == features_in
assert weights.shape[1] == features_out
self._parameters = weights
if not bias is None:
assert bias.shape[0] == features_out
self._bias = bias
else:
# TODO : add initialization strategies
# using random initialization by default
self._parameters = np.random.randn(features_in, features_out)
# zero-init for bias
self._bias = np.zeros((features_out, ))
self.zero_grad()
def forward(self, data):
"""
Linear operation : f_w(x) = XW
Args:
data: (batch,features_in)
Out:
(batch,features_out)
"""
assert self._parameters.shape[0] == data.shape[1]
return np.matmul(data,self._parameters) + self._bias
Please, if you have any idea, let me know 
Thx !