Low accuracy on a big dataset

I trained a resnet18 for classifying photos into “good” and "bad categories on 2 different image data sets. One contains 4000 train photos and 200 validation photos and the other one is bigger, containing 15000 training photos and 3000 validation photos.
I got around 80% accuracy on the first smaller dataset but when I train the model on the bigger dataset, I only have 50% accuracy.
This is the pytorch code:

    def model_training(res_model, criterion, optimizer, scheduler, number_epochs=25):
        since = time.time()
        best_resmodel_wts = copy.deepcopy(res_model.state_dict())
        best_accuracy = 0.0
        for epochs in range(number_epochs):
            print('Epoch {}/{}'.format(epochs, number_epochs - 1))
            print('-' * 10)
            for phase in ['train', 'val']: 
                if phase == 'train':
                running_loss = 0.0
                running_corrects = 0
                for inputs, labels in loaders_data[phase]:
                    inputs = inputs.to(device)
                    labels = labels.to(device)
                    with torch.set_grad_enabled(phase == 'train'): ## forwarding si tracking 
                        outputs = res_model(inputs)
                        _, preds = torch.max(outputs, 1)
                        loss = criterion(outputs, labels)
                        if phase == 'train':# backward si optimizare doar in faza de antrenare 
                    running_loss += loss.item() * inputs.size(0)
                    running_corrects += torch.sum(preds == labels.data)
                if phase == 'train':
                epoch_loss = running_loss / sizes_datasets[phase]
                epoch_acc = running_corrects.double() / sizes_datasets[phase]
                print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss, epoch_acc))
                if phase == 'val' and epoch_acc > best_accuracy: 
                    best_accuracy = epoch_acc
                    best_resmodel_wts = copy.deepcopy(res_model.state_dict())
        time_elapsed = time.time() - since
        print('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))
        print('Best val Acc: {:4f}'.format(best_accuracy))
        return res_model
    finetune_model = models.resnet18(pretrained=True)
    num_ftrs = finetune_model.fc.in_features

    finetune_model.fc = nn.Linear(num_ftrs, 2)
    finetune_model = finetune_model.to(device)
    criterion = nn.CrossEntropyLoss()
    finetune_optim = optim.SGD(finetune_model.parameters(), lr=0.001, momentum=0.9)
    exp_lr_scheduler = lr_scheduler.StepLR(finetune_optim, step_size=7, gamma=0.1)
    finetune_model = model_training(finetune_model, criterion, finetune_optim, exp_lr_scheduler,

Is there any way to improve the accuracy when working on the bigger data set?
Also is cross entropy loss good for this image classification?