How to use Dataparallel when there is ‘for’ cycle in the network?

I have a server with two GPUs, if I use one GPU I need more than 10 days to finish 1000 epoch. But when I tried to use Dataparallel, the programm didn’t work. It seems because there is a ‘for’ cycle in my network. So how can I use Dataparallel in this case. Or is there any other solution to speed up training?

Can you post the error that shows while using ‘for’ cycle in your network ?

class WaveNet( nn.Module ):    
    def __init__(self, mu, n_residue, n_skip, dilation_depth, n_repeat):
        # mu: audio quantization size
        # n_residue: residue channels
        # n_skip: skip channels
        # dilation_depth & n_repeat: dilation layer setup
        self.mu = mu
        super( WaveNet, self ).__init__()
        self.dilation_depth = dilation_depth
        dilations = self.dilations = [2 ** i for i in range( dilation_depth )] * n_repeat
        self.one_hot = One_Hot( mu )
        self.from_input = nn.Conv1d( in_channels=mu, out_channels=n_residue, kernel_size=1 )
        self.from_input = nn.DataParallel(self.from_input)
        self.conv_sigmoid = nn.ModuleList(
            [nn.Conv1d( in_channels=n_residue, out_channels=n_residue, kernel_size=2, dilation=d )
             for d in dilations] )
        self.conv_sigmoid = nn.DataParallel(self.conv_sigmoid)
        self.conv_tanh = nn.ModuleList(
            [nn.Conv1d( in_channels=n_residue, out_channels=n_residue, kernel_size=2, dilation=d )
             for d in dilations] )
        self.conv_tanh = nn.DataParallel(self.conv_tanh)
        self.skip_scale = nn.ModuleList( [nn.Conv1d( in_channels=n_residue, out_channels=n_skip, kernel_size=1 )
                                          for d in dilations] )
        self.skip_scale = nn.DataParallel(self.skip_scale)
        self.residue_scale = nn.ModuleList( [nn.Conv1d( in_channels=n_residue, out_channels=n_residue, kernel_size=1 )
                                             for d in dilations] )
        self.residue_scale = nn.DataParallel(self.residue_scale)
        self.conv_post_1 = nn.Conv1d( in_channels=n_skip, out_channels=n_skip, kernel_size=1 )
        self.conv_post_1 = nn.DataParallel(self.conv_post_1)
        self.conv_post_2 = nn.Conv1d( in_channels=n_skip, out_channels=mu, kernel_size=1 )
        self.conv_post_2 = nn.DataParallel(self.conv_post_2)

    def forward(self, input, train=True):
        output = self.preprocess( input, train )
        skip_connections = []  # save for generation purposes
        for s, t, skip_scale, residue_scale in zip( self.conv_sigmoid, self.conv_tanh, self.skip_scale,
                                                    self.residue_scale ):
            output, skip = self.residue_forward( output, s, t, skip_scale, residue_scale )
            skip_connections.append( skip )
        # sum up skip connections
        output = sum( [s[:, :, -output.size( 2 ):] for s in skip_connections] )
        output = self.postprocess( output, train )
        return output

TypeError: zip argument #1 must support iteration

where does your code throws this error TypeError: zip argument #1 must support iteration ? you can post the error trace as well

  from ._conv import register_converters as _register_converters
Traceback (most recent call last):
  File "WaveNet3.py", line 301, in <module>
    logits = net(s_data, train=True)
  File "/home/hit/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py", line 325, in __call__
    result = self.forward(*input, **kwargs)
  File "WaveNet3.py", line 181, in forward
    self.residue_scale ):
TypeError: zip argument #1 must support iteration

@ashwin.raju93