I’m trying to train my model, and although it’s working, I’m getting the warning:
TracerWarning: Trace had nondeterministic nodes. Did you forget call .eval() on your model? Nodes:
%input.22 : Float(100:210, 210:1, requires_grad=1, device=cuda:0) = aten::dropout(%input.21, %261, %262) # /home/wilson/anaconda3/envs/cv/lib/python3.6/site-packages/torch/nn/functional.py:983:0
%input.27 : Float(100:52, 52:1, requires_grad=1, device=cuda:0) = aten::dropout(%input.26, %274, %275) # /home/wilson/anaconda3/envs/cv/lib/python3.6/site-packages/torch/nn/functional.py:983:0
This may cause errors in trace checking. To disable trace checking, pass check_trace=False to torch.jit.trace()
_module_class,
/home/wilson/anaconda3/envs/cv/lib/python3.6/site-packages/torch/jit/_trace.py:966: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error:
With rtol=1e-05 and atol=1e-05, found 698 element(s) (out of 700) whose difference(s) exceeded the margin of error (including 0 nan comparisons). The greatest difference was 0.040440917015075684 (-1.8279550075531006 vs. -1.787514090538025), which occurred at index (93, 1).
_module_class,
To fix it, I’ve already tried model.eval() (recommended by other similar posts), although not sure why I’d set it to evaluation model during training.
My model:
class Net(nn.Module):
'''
Neural network for binary classification with 4 classes
Params:
batch_size: -
'''
def __init__(self, batch_size):
super(Net, self).__init__()
# Convolutional Layers
stride = 2
padding = 5
image_size = 720
conv_kernel_size = 5
pool_kernel_size = 2
num_kernels = [3,
15, 30, 45,
60, 75, 90,
105, 120, 135, 150]
output_size = 7
conv1_shape = self.conv_layer_shape("conv1_shape", batch_size, image_size, num_kernels[1], conv_kernel_size, padding, stride)
conv2_shape = self.conv_layer_shape("conv2_shape", batch_size, conv1_shape[2], num_kernels[2], conv_kernel_size, padding, stride)
conv3_shape = self.conv_layer_shape("conv3_shape", batch_size, conv2_shape[2], num_kernels[3], conv_kernel_size, padding, stride)
conv4_shape = self.conv_layer_shape("conv4_shape", batch_size, conv3_shape[2], num_kernels[4], conv_kernel_size, padding, stride)
pool4_shape = self.pool_layer_shape('pool4_shape', batch_size, conv4_shape[2], num_kernels[4], pool_kernel_size, stride)
conv5_shape = self.conv_layer_shape("conv5_shape", batch_size, pool4_shape[2], num_kernels[5], conv_kernel_size, padding, stride)
pool5_shape = self.pool_layer_shape('pool5_shape', batch_size, conv5_shape[2], num_kernels[5], pool_kernel_size, stride)
conv6_shape = self.conv_layer_shape("conv6_shape", batch_size, pool5_shape[2], num_kernels[6], conv_kernel_size, padding, stride)
pool6_shape = self.pool_layer_shape('pool6_shape', batch_size, conv6_shape[2], num_kernels[6], pool_kernel_size, stride)
conv7_shape = self.conv_layer_shape("conv7_shape", batch_size, pool6_shape[2], num_kernels[7], conv_kernel_size, padding, stride)
pool7_shape = self.pool_layer_shape('pool7_shape', batch_size, conv7_shape[2], num_kernels[7], pool_kernel_size, stride)
self.conv1 = nn.Conv2d(num_kernels[0], num_kernels[1], conv_kernel_size, stride, padding)
self.conv2 = nn.Conv2d(num_kernels[1], num_kernels[2], conv_kernel_size, stride, padding)
self.conv3 = nn.Conv2d(num_kernels[2], num_kernels[3], conv_kernel_size, stride, padding)
self.conv4 = nn.Conv2d(num_kernels[3], num_kernels[4], conv_kernel_size, stride, padding)
self.conv5 = nn.Conv2d(num_kernels[4], num_kernels[5], conv_kernel_size, stride, padding)
self.conv6 = nn.Conv2d(num_kernels[5], num_kernels[6], conv_kernel_size, stride, padding)
self.conv7 = nn.Conv2d(num_kernels[6], num_kernels[7], conv_kernel_size, stride, padding)
self.max_pool = nn.MaxPool2d(pool_kernel_size, stride, padding=0)
# Fully Connected Layers
fc1_size = pool7_shape[1] * pool7_shape[2] * pool7_shape[3]
print('fc1_size:', fc1_size)
fc2_size = fc1_size // 2
print('fc2_size:', fc2_size)
fc3_size = fc2_size // 2
print('fc3_size:', fc3_size)
fc4_size = fc3_size // 2
print('fc4_size:', fc4_size)
fc5_size = fc4_size // 2
print('fc5_size:', fc5_size)
print('output_size:', output_size)
self.fc1 = nn.Linear(fc1_size, fc2_size)
self.fc2 = nn.Linear(fc2_size, fc3_size)
self.fc3 = nn.Linear(fc3_size, fc4_size)
self.fc4 = nn.Linear(fc4_size, fc5_size)
self.fc5 = nn.Linear(fc5_size, output_size)
def conv_layer_shape(self, layer_name, batch_size, w_in, num_filters, kernel_size, padding, stride):
'''
Returns shape of a convolutional layer
Parameters:
layer_name: Name of layer
batch_size: Batch size
w_in: Width/Height of Previous Layer
num_filters: Number of Filters
kernel_size: Filter/Kernel Size
padding: Padding
stride: Stride
Returns:
shape: Shape of convolutional layer
'''
w_out = int((w_in - kernel_size + 2*padding)/stride + 1)
shape = (batch_size, num_filters, w_out, w_out)
print('{}: {}'.format(layer_name, shape))
return shape
def pool_layer_shape(self, layer_name, batch_size, w_in, num_filters, kernel_size, stride):
'''
Returns shape of a pooling layer
Params:
batch_size: Batch size
w_in: Width/Height of previous layer
num_filters: Number of filers
kernel_size: Filter/Kernel size
stride: Stride
Returns:
shape: Shape of pooling layer
'''
w_out = int((w_in * (kernel_size-1) - 1) / stride)
shape = (batch_size, num_filters, w_out, w_out)
print('{}: {}'.format(layer_name, shape))
return shape
def forward(self, x):
'''
Feed-forward
Params:
x: Batch of images
'''
x = F.relu(self.conv1(x))
x = F.relu(self.conv2(x))
x = F.relu(self.conv3(x))
x = self.max_pool(F.relu(self.conv4(x)))
x = self.max_pool(F.relu(self.conv5(x)))
x = self.max_pool(F.relu(self.conv6(x)))
x = self.max_pool(F.relu(self.conv7(x)))
x = x.view(x.size(0), -1)
x = F.dropout(F.relu(self.fc1(x)))
x = F.relu(self.fc2(x))
x = F.dropout(F.relu(self.fc3(x)))
x = F.relu(self.fc4(x))
x = self.fc5(x)
x = F.log_softmax(x, dim=1)
return x
Edit:
The warning has disappeared after removing writer.add_graph(net, images) from SummaryWritter (TensorBoard)
How can I add a graph to TensorBoard and remove these warnings?