Hi,
I’m currently experimenting with some custom layers (based on numpy). For this I tried to make a reference conv2d function, to know for sure that I correctly understand the inner workings. However, I ran into some issue that some output elements are not completely identical. Is there a way to fix this?
I attached some sample code that explains the problem I’m facing. I would really appreciate any feedback/help.
Kind regards,
Barry
import numpy as np
import torch
import torch.nn.functional as F
from torch.nn.parameter import Parameter
from torch.autograd import Function
def DConv2dFunction(input, param, bias, stride, padding, dilation=1, groups=1):
"""
Desc: conv2d reference routine.
"""
assert(dilation == 1) # no support for higher dilations
assert(groups == 1) # no support for larger groups
# Convert input, weights and bias to numpy.
d = input.data.cpu().numpy()
d_pad = np.pad(d
, pad_width=((0,),(0,),(padding,),(padding,))
, mode='constant', constant_values=0)
p = param.data.cpu().numpy()
b = bias.data.cpu().numpy()
# Dimensions.
N, Kc, H, W = d.shape
C, Kc, Kh, Kw = p.shape
W_out = int(np.floor((W + 2*padding - dilation*(Kw-1) - 1) / stride + 1))
H_out = int(np.floor((H + 2*padding - dilation*(Kh-1) - 1) / stride + 1))
# Compute output samples.
res = np.zeros((N, C, H_out, W_out), dtype=np.float32)
for n in range(N):
for to in range(C):
for y in range(H_out):
for x in range(W_out):
for ti in range(Kc):
for ky in range(Kh):
for kx in range(Kw):
res[n,to,y,x] += p[to,ti,ky,kx] * d_pad[n,ti,(y*stride)+ky,(x*stride)+kx]
res[n,to,y,x] += b[to]
return Parameter(torch.from_numpy(res)).type_as(input)
# Layer dimensions
# N; batch size, C; output channels, H/W; feature map dimensions
# Kc; input channels, Kh; kernel height, Kw; kernel width
N, C, H, W = 1, 8, 28, 28
Kc, Kh, Kw = 16, 5, 5
padding=2
stride=1
dilation=1
groups=1
# Create random input, weights, and biases
torch.manual_seed(99)
if True: # set to false for integer values
d = Parameter(torch.randn(N, Kc, H, W).type(torch.FloatTensor))
p = Parameter(torch.randn(C, Kc, Kh, Kw).type(torch.FloatTensor))
b = Parameter(torch.randn(C).type(torch.FloatTensor))
else:
d = Parameter(torch.randn(N, Kc, H, W).mul(255).round().type(torch.FloatTensor))
p = Parameter(torch.randn(C, Kc, Kh, Kw).mul(255).round().type(torch.FloatTensor))
b = Parameter(torch.randn(C).mul(255).round().type(torch.FloatTensor))
# Compute result using PyTorch routine
res1 = F.conv2d(d, p, b, stride, padding, dilation, groups)
out1 = res1.cpu().data.numpy().flatten()
# Compute result using Reference routine
res2 = DConv2dFunction(d, p, b, stride, padding, dilation, groups)
out2 = res2.cpu().data.numpy().flatten()
# Check approximate equality per element
max_relative_difference = 1e-4
z = np.isclose(out1, out2, rtol=max_relative_difference, atol=0)
print("Number of elements that exceed margin: {}/{}".format(sum(~z), z.size))
if sum(~z) > 0:
print("expected -> actual")
for pytorch, reference in zip(out1[~z], out2[~z]):
print("{:.7f} -> {:.7f}".format(pytorch, reference))
Which outputs:
Number of elements that exceed margin: 10/6272
expected -> actual
-0.0515766 -> -0.0515714
0.0141878 -> 0.0141795
-0.0277182 -> -0.0277148
-0.0202761 -> -0.0202698
0.0075884 -> 0.0075939
-0.0075113 -> -0.0075098
-0.0163350 -> -0.0163383
-0.0050793 -> -0.0050831
0.0368888 -> 0.0368946
0.0233984 -> 0.0234014
Which implies that 10 out of 6272 output samples are slightly different, as can be seen in the printout.
