Here is a problem about how to implement model-parallel on pytorch.
Since I have read tuturial that it uses very simple instructions, here are codes
class Model(nn.Module):
def __init__(self):
super().__init__()
self.model_a = Model_A().to('cuda:0')
self.model_b = Model_B().to('cuda:1')
def forward(self, x):
x = self.model_a(x.to('cuda:0'))
x = self.model_b(x.to('cuda:1'))
return x
However, I do not have many GPUs. I want to use torch.distribute.send and torch.distribute.recv in forward and backward to build model-parallel. But how can I do it? Is there any tutoral?
class Net(nn.Module):
def __init__(self):
super(Net,self).__init__()
self.conv1 = nn.Conv2d(1,32,3,padding = 1)
self.pool1 = nn.MaxPool2d(2,2)
self.conv2 = nn.Conv2d(32,64,3,padding = 1)
self.pool2 = nn.MaxPool2d(2,2)
self.conv3 = nn.Conv2d(64,128,3,padding = 1)
self.pool3 = nn.MaxPool2d(2,2)
self.fc1 = nn.Linear(128*3*3,625)
self.fc2 = nn.Linear(625,10)
def forward(self,x,rank):
if rank == 0:
x = self.pool1(F.relu(self.conv1(x)))
x = self.pool2(F.relu(self.conv2(x)))
x = self.pool3(F.relu(self.conv3(x)))
dist.send(x,1)
elif rank == 1:
dist.recv(x,0)
x = x.view(-1,128*3*3)
x = F.relu(self.fc1(x))
x = self.fc2(x)
return x
I could use rank and send and save in forward. But how can I change backward?