The task consists in regressing steering angles from this Udacity dataset.
Preprocessing:
- get grayscale version of images in [0,1] interval and crop them to 200x200 size with the following procedure:
half_the_width = int(img.shape[1] / 2)
img = img[img.shape[0] - crop_heigth: img.shape[0], half_the_width - int(crop_width / 2):
half_the_width + int(crop_width / 2)]
Thus I have input images with size 200x200 and target of dimension 1 (steering angle).
Loss used is MSEloss
This is the Keras model that I want to emulate, it uses Adam with learning rate decay 1e-5 and is taken from DroNet repo:
def resnet8(img_width, img_height, img_channels, output_dim):
"""
Define model architecture.
# Arguments
img_width: Target image widht.
img_height: Target image height.
img_channels: Target image channels.
output_dim: Dimension of model output.
# Returns
model: A Model instance.
"""
# Input
img_input = Input(shape=(img_height, img_width, img_channels))
x1 = Conv2D(32, (5, 5), strides=[2,2], padding='same')(img_input)
x1 = MaxPooling2D(pool_size=(3, 3), strides=[2,2])(x1)
# First residual block
x2 = keras.layers.normalization.BatchNormalization()(x1)
x2 = Activation('relu')(x2)
x2 = Conv2D(32, (3, 3), strides=[2,2], padding='same',
kernel_initializer="he_normal",
kernel_regularizer=regularizers.l2(1e-4))(x2)
x2 = keras.layers.normalization.BatchNormalization()(x2)
x2 = Activation('relu')(x2)
x2 = Conv2D(32, (3, 3), padding='same',
kernel_initializer="he_normal",
kernel_regularizer=regularizers.l2(1e-4))(x2)
x1 = Conv2D(32, (1, 1), strides=[2,2], padding='same')(x1)
x3 = add([x1, x2])
# Second residual block
x4 = keras.layers.normalization.BatchNormalization()(x3)
x4 = Activation('relu')(x4)
x4 = Conv2D(64, (3, 3), strides=[2,2], padding='same',
kernel_initializer="he_normal",
kernel_regularizer=regularizers.l2(1e-4))(x4)
x4 = keras.layers.normalization.BatchNormalization()(x4)
x4 = Activation('relu')(x4)
x4 = Conv2D(64, (3, 3), padding='same',
kernel_initializer="he_normal",
kernel_regularizer=regularizers.l2(1e-4))(x4)
x3 = Conv2D(64, (1, 1), strides=[2,2], padding='same')(x3)
x5 = add([x3, x4])
# Third residual block
x6 = keras.layers.normalization.BatchNormalization()(x5)
x6 = Activation('relu')(x6)
x6 = Conv2D(128, (3, 3), strides=[2,2], padding='same',
kernel_initializer="he_normal",
kernel_regularizer=regularizers.l2(1e-4))(x6)
x6 = keras.layers.normalization.BatchNormalization()(x6)
x6 = Activation('relu')(x6)
x6 = Conv2D(128, (3, 3), padding='same',
kernel_initializer="he_normal",
kernel_regularizer=regularizers.l2(1e-4))(x6)
x5 = Conv2D(128, (1, 1), strides=[2,2], padding='same')(x5)
x7 = add([x5, x6])
x = Flatten()(x7)
x = Activation('relu')(x)
x = Dropout(0.5)(x)
# Steering channel
steer = Dense(output_dim)(x)
# Collision channel
coll = Dense(output_dim)(x)
coll = Activation('sigmoid')(coll)
# Define steering-collision model
model = Model(inputs=[img_input], outputs=[steer, coll])
print(model.summary())
return model
In PyTorch I am trying to implement only steering angle prediction, the paper mentions that steering angles and collision predictions are uncorrelated.
This is my PyTorch implementation:
MODEL
def init_kernel(m):
if isinstance(m, nn.Conv2d):
# Initialize kernels of Conv2d layers as kaiming normal
nn.init.kaiming_normal_(m.weight, mode='fan_in', nonlinearity='relu')
# Initialize biases of Conv2d layers at 0
nn.init.zeros_(m.bias)
def __init__(self, img_channels, in_height, in_width, output_dim):
super(resnet8, self).__init__()
self.layer1 = nn.Sequential(
nn.Conv2d(in_channels=img_channels,out_channels=32,
kernel_size=[5,5], stride=[2,2], padding=[5//2,5//2]),
nn.MaxPool2d(kernel_size=[3,3], stride=[2,2]))
self.residual_block_1a = nn.Sequential(
nn.BatchNorm2d(32),
nn.ReLU(),
nn.Conv2d(in_channels=32,out_channels=32, kernel_size=[3,3],
stride=[2,2], padding=[3//2,3//2]),
nn.BatchNorm2d(32),
nn.ReLU(),
nn.Conv2d(in_channels=32,out_channels=32, kernel_size=[3,3],
padding=[3//2,3//2]))
self.parallel_conv_1 = nn.Conv2d(in_channels=32,out_channels=32,
kernel_size=[1,1], stride=[2,2],
padding=[1//2,1//2])
self.residual_block_2a = nn.Sequential(
nn.BatchNorm2d(32),
nn.ReLU(),
nn.Conv2d(in_channels=32,out_channels=64, kernel_size=[3,3],
stride=[2,2], padding=[3//2,3//2]),
nn.BatchNorm2d(64),
nn.ReLU(),
nn.Conv2d(in_channels=64,out_channels=64, kernel_size=[3,3],
padding=[3//2,3//2]))
self.parallel_conv_2 = nn.Conv2d(in_channels=32,out_channels=64,
kernel_size=[1,1], stride=[2,2],
padding=[1//2,1//2])
self.residual_block_3a = nn.Sequential(
nn.BatchNorm2d(64),
nn.ReLU(),
nn.Conv2d(in_channels=64,out_channels=128, kernel_size=[3,3],
stride=[2,2], padding=[3//2,3//2]),
nn.BatchNorm2d(128),
nn.ReLU(),
nn.Conv2d(in_channels=128,out_channels=128, kernel_size=[3,3],
padding=[3//2,3//2]))
self.parallel_conv_3 = nn.Conv2d(in_channels=64,out_channels=128,
kernel_size=[1,1], stride=[2,2],
padding=[1//2,1//2])
self.output_dim = output_dim
self.last_block = nn.Sequential(
nn.ReLU(),
nn.Dropout2d(),
nn.Linear(6272,self.output_dim))
# Initialize layers exactly as in Keras
for m in self.modules():
if isinstance(m, nn.Conv2d) or isinstance(m, nn.Linear):
nn.init.xavier_uniform_(m.weight, gain=nn.init.calculate_gain('relu'))
nn.init.zeros_(m.bias)
elif isinstance(m, nn.BatchNorm2d):
# Initialize kernels of Conv2d layers as kaiming normal
nn.init.constant_(m.weight, 1)
nn.init.constant_(m.bias, 0)
self.residual_block_1a.apply(init_kernel)
self.residual_block_2a.apply(init_kernel)
self.residual_block_3a.apply(init_kernel)
def forward(self, x):
x1 = self.layer1(x)
# First residual block
x2 = self.residual_block_1a(x1)
x1 = self.parallel_conv_1(x1)
x3 = x1.add(x2)
# Second residual block
x4 = self.residual_block_2a(x3)
x3 = self.parallel_conv_2(x3)
x5 = x3.add(x4)
# Third residual block
x6 = self.residual_block_3a(x5)
x5 = self.parallel_conv_3(x5)
x7 = x5.add(x6)
out = x7.view(x7.size(0), -1) # Flatten
out = self.last_block(out)
return out
TRAINING LOOP
def compute_l2_reg(model,model_name):
# Function that sets weight_decay only for weights and not biases and only
# for conv layers inside residual layers
lambda_ = FLAGS.weight_decay
params_dict = dict(model.named_parameters())
l2_reg=[]
if model_name == 'resnet8':
for key, value in params_dict.items():
if ((key[-8:] == '2.weight' or key[-8:] == '5.weight') and key[0:8]=='residual'):
l2_reg += [lambda_*torch.norm(value.view(value.size(0),-1),2)]
l2_reg = sum(l2_reg)
return l2_reg
def train_model(model, num_epochs, learning_rate, train_loader, valid_loader,
patience, model_name):
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
# To track the training loss as the model trains
train_losses = []
# To track the validation loss as the model trains
valid_losses = []
# To track the average training loss per epoch as the model trains
avg_train_losses = []
# To track the average validation loss per epoch as the model trains
avg_valid_losses = []
# Initialize the early_stopping object
early_stopping = EarlyStopping(patience=patience, verbose=True)
# Training loop
decay = FLAGS.decay # Default 1e-5
fcn = lambda step: 1./(1. + decay*step)
scheduler = LambdaLR(optimizer, lr_lambda=fcn)
for epoch in range(1, num_epochs+1):
###################
# TRAIN the model #
###################
model.train() # prep model for training
for batch, (images, targets) in enumerate(train_loader, 1):
# Load images and targets to device
images = images.to(device)
targets = targets.to(device)
# Clear gradients
optimizer.zero_grad()
# Forward pass
outputs = model(images)
# Calculate loss
l2_reg = compute_l2_reg(model,model_name)
loss = F.mse_loss(outputs, targets) + l2_reg
# Backward pass
loss.backward()
# Update weights
optimizer.step()
# Decay Learning Rate
scheduler.step()
# Record training loss
train_losses.append(loss.item())
######################
# VALIDATE the model #
######################
model.eval() # prep model for evaluation
for images, targets in valid_loader:
images = images.to(device)
targets = targets.to(device)
# Forward pass:
outputs = model(images)
# Calculate loss
loss = F.mse_loss(outputs, targets)
# Record validation loss
valid_losses.append(loss.item())
# Print training/validation statistics
# Calculate average loss over an epoch
train_loss = np.average(train_losses)
valid_loss = np.average(valid_losses)
avg_train_losses.append(train_loss)
avg_valid_losses.append(valid_loss)
epoch_len = len(str(num_epochs))
print_msg = (f'[{epoch:>{epoch_len}}/{num_epochs:>{epoch_len}}] ' +
f'train_loss: {train_loss:.5f} ' +
f'valid_loss: {valid_loss:.5f}')
print(print_msg)
# Clear lists to track next epoch
train_losses = []
valid_losses = []
# Early_stopping needs the validation loss to check if it has decresed,
# and if it has, it will make a checkpoint of the current model
early_stopping(valid_loss, model)
if early_stopping.early_stop:
print("Early stopping")
break
# Load the last checkpoint with the best model
# (returned by early_stopping call)
model.load_state_dict(torch.load('checkpoint.pt'))
print('Training completed and model saved.')
return model, avg_train_losses, avg_valid_losses
where weight_decay = 1e-4 and Early Stop is a function that checks for validation loss and stores the model with least validation error.
I tried to emulate in PyTorch same manual and default initializations as in Keras, you can see it in my Model code.
I also tried to exactly reproduce the same regularization as in Keras kernel regularizer.