Acc is not stable

I set up a simple CNN network, trained through the MNIST data set.But accuracy varies from 99 percent to 10 percent.I don’t know what went wrong.
deepconvnet.py

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.utils.data as Data
import torch.optim as optim
import numpy as np
import matplotlib.pyplot as plt
import os
from torchvision import datasets, transforms


class DeepConvNet(nn.Module):
    def __init__(self, train_set_x, train_set_y, test_set_x, test_set_y,
                DROPOUT = 0.5, BATH_SIZE = 1000, EPOCH = 20, LEARNING_RATE = 0.01):
        super(DeepConvNet, self).__init__()
        self.conv = nn.Sequential(
            # nn.Conv2d(in_channels = 1, out_channels = 16, kernel_size = 3, padding = 1, padding_mode = 'zeros'),
            nn.Conv2d(in_channels = 1, out_channels = 16, kernel_size = 3, padding = 1),
            nn.ReLU(),
            # nn.Conv2d(in_channels = 16, out_channels = 16, kernel_size = 3, padding = 1, padding_mode = 'zeros'),
            nn.Conv2d(in_channels = 16, out_channels = 16, kernel_size = 3, padding = 1),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size = 2, stride = 2),
            nn.Conv2d(in_channels = 16, out_channels = 32, kernel_size = 3, padding = 1, padding_mode = 'zeros'),
            nn.ReLU(),
            nn.Conv2d(in_channels = 32, out_channels = 32, kernel_size = 3, padding = 2, padding_mode = 'zeros'),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size = 2, stride = 2),
            nn.Conv2d(in_channels = 32, out_channels = 64, kernel_size = 3, padding = 1, padding_mode = 'zeros'),
            nn.ReLU(),
            nn.Conv2d(in_channels = 64, out_channels = 64, kernel_size = 3, padding = 1, padding_mode = 'zeros'),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size = 2, stride = 2),
        )
        for m in self.conv:
            if(isinstance(m, nn.Conv2d)):
                nn.init.kaiming_normal_(m.weight, mode='fan_in')
        self.fc = nn.Sequential(
            nn.Linear(in_features = 1024, out_features = 50),
            nn.ReLU(),
            # nn.Dropout(DROPOUT),
            nn.Linear(in_features = 50, out_features = 10),
            nn.ReLU(),
            # nn.Dropout(DROPOUT)
        )
        for m in self.conv:
            if(isinstance(m, nn.Linear)):
                nn.init.kaiming_normal_(m.weight, mode='fan_in')
                m.parameters = m.parameters / 10
        self.train_dataset = Data.TensorDataset(train_set_x, train_set_y)
        self.test_dataset = Data.TensorDataset(test_set_x, test_set_y)
        self.BATH_SIZE = BATH_SIZE
        self.loader_train = Data.DataLoader(
            dataset = self.train_dataset,
            batch_size = BATH_SIZE,
            shuffle = True,
        )
        self.loader_test = Data.DataLoader(
            dataset = self.test_dataset,
            batch_size = len(test_set_y),
            shuffle = True
        )
        self.optimizer = optim.Adam(self.parameters(), lr = LEARNING_RATE, betas = (0.9, 0.999), eps = 1e-8)
        # self.optimizer = optim.SGD(self.parameters(), lr = LEARNING_RATE)
        # self.optimizer = optim.RMSprop(self.parameters(),lr = LEARNING_RATE)
        self.EPOCH = EPOCH

    def forward(self, x):
        conv_out = self.conv(x)
        conv_out = conv_out.view(conv_out.size()[0], -1)
        out = self.fc(conv_out)
        return out
        
    def loss(self, pre_y, label_y):
        # loss = nn.CrossEntropyLoss()
        # Loss = loss(pre_y, label_y)
        # return Loss
        loss = F.log_softmax(pre_y)
        loss = -torch.mean((loss[range(pre_y.shape[0]), label_y]))
        return loss
        
    def train_net(self):
        for eporch in range(self.EPOCH):
            running_loss = 0.0
            for i, data in enumerate(self.loader_train):
                x, y = data
                # print(x.view(1000,-1)[1])
                # print(y.shape)
                # print(i)
                # img = np.transpose(np.array(x[0].numpy()), (1, 2, 0))
                # print(img.shape)
                # plt.imshow(img)
                # plt.show()
                
                self.optimizer.zero_grad()
                # params = list(self.named_parameters())
                # print(params)
                # (name, param) = params[0]
                # print(name)
                # print(param.shape)
                # print(y)
                # os.system("pause")
                outputs = self.forward(x)
                # print(outputs.shape)
                Loss = self.loss(outputs, y)
                Loss.backward()
                # print(param.grad)
                self.optimizer.step()
                running_loss += Loss.item()
                # if (i+1) % self.BATH_SIZE == 0:
                prediction = torch.argmax(outputs, 1)
                correct = (prediction == y).sum().float()
                total = float(len(prediction))
                acc = correct / total
                print(acc)
                print('Epoch: %d, iter: %5d, Loss: %3f Acc: %3f' %(eporch+1, (i+1)*self.BATH_SIZE, running_loss, acc))
                # print(outputs.shape)
                # print(y)


                running_loss = 0.0
                
        torch.save(self.state_dict(),'parameters.pth')            
        print('Finish Train')

    def test(self):
        self.eval()
        for data in self.loader_test:
            # print(data)
            x, y = data
        outputs = self.forward(x)
        # print(outputs)
        Loss = self.loss(outputs, y)
        prediction = torch.argmax(outputs, 1)
        # print(prediction)
        correct = (prediction == y).sum().float()
        total = float(len(prediction))
        acc = correct / total
        print("On test set, the loss is %.3f, Acc is %.3f" %(Loss, acc))            

traindeepnet.py

import sys, os
sys.path.append(os.pardir)
import numpy as np
import matplotlib.pyplot as plt
from mnist import load_mnist
import torch
from deepconvnet import DeepConvNet

(x_train,t_train), (x_test,t_test) = load_mnist(flatten=False)

x_train = torch.Tensor(x_train)
t_train = torch.Tensor(t_train).long()
# x_train = torch.Tensor(x_train)[:10000]
# t_train = torch.Tensor(t_train).long()[:10000]
x_test = torch.Tensor(x_test)
t_test = torch.Tensor(t_test).long()


print(x_train.shape)
print(t_train.shape)                                            


Net = DeepConvNet(x_train, t_train, x_test, t_test)
Net.test()
Net.train_net()
Net.test()

mnist.py

# coding: utf-8
try:
    import urllib.request
except ImportError:
    raise ImportError('You should use Python 3.x')
import os.path
import gzip
import pickle
import os
import numpy as np


url_base = 'http://yann.lecun.com/exdb/mnist/'
key_file = {
    'train_img':'train-images-idx3-ubyte.gz',
    'train_label':'train-labels-idx1-ubyte.gz',
    'test_img':'t10k-images-idx3-ubyte.gz',
    'test_label':'t10k-labels-idx1-ubyte.gz'
}

dataset_dir = os.path.dirname(os.path.abspath(__file__))
save_file = dataset_dir + "/mnist.pkl"

train_num = 60000
test_num = 10000
img_dim = (1, 28, 28)
img_size = 784


def _download(file_name):
    file_path = dataset_dir + "/" + file_name
    
    if os.path.exists(file_path):
        return

    print("Downloading " + file_name + " ... ")
    urllib.request.urlretrieve(url_base + file_name, file_path)
    print("Done")
    
def download_mnist():
    for v in key_file.values():
       _download(v)
        
def _load_label(file_name):
    file_path = dataset_dir + "/" + file_name
    
    print("Converting " + file_name + " to NumPy Array ...")
    with gzip.open(file_path, 'rb') as f:
            labels = np.frombuffer(f.read(), np.uint8, offset=8)
    print("Done")
    
    return labels

def _load_img(file_name):
    file_path = dataset_dir + "/" + file_name
    
    print("Converting " + file_name + " to NumPy Array ...")    
    with gzip.open(file_path, 'rb') as f:
            data = np.frombuffer(f.read(), np.uint8, offset=16)
    data = data.reshape(-1, img_size)
    print("Done")
    
    return data
    
def _convert_numpy():
    dataset = {}
    dataset['train_img'] =  _load_img(key_file['train_img'])
    dataset['train_label'] = _load_label(key_file['train_label'])    
    dataset['test_img'] = _load_img(key_file['test_img'])
    dataset['test_label'] = _load_label(key_file['test_label'])
    
    return dataset

def init_mnist():
    download_mnist()
    dataset = _convert_numpy()
    print("Creating pickle file ...")
    with open(save_file, 'wb') as f:
        pickle.dump(dataset, f, -1)
    print("Done!")

def _change_one_hot_label(X):
    T = np.zeros((X.size, 10))
    for idx, row in enumerate(T):
        row[X[idx]] = 1
        
    return T
    

def load_mnist(normalize=True, flatten=True, one_hot_label=False):
    """读入MNIST数据集
    
    Parameters
    ----------
    normalize : 将图像的像素值正规化为0.0~1.0
    one_hot_label : 
        one_hot_label为True的情况下,标签作为one-hot数组返回
        one-hot数组是指[0,0,1,0,0,0,0,0,0,0]这样的数组
    flatten : 是否将图像展开为一维数组
    
    Returns
    -------
    (训练图像, 训练标签), (测试图像, 测试标签)
    """
    if not os.path.exists(save_file):
        init_mnist()
        
    with open(save_file, 'rb') as f:
        dataset = pickle.load(f)
    
    if normalize:
        for key in ('train_img', 'test_img'):
            dataset[key] = dataset[key].astype(np.float32)
            dataset[key] /= 255.0
            
    if one_hot_label:
        dataset['train_label'] = _change_one_hot_label(dataset['train_label'])
        dataset['test_label'] = _change_one_hot_label(dataset['test_label'])
    
    if not flatten:
         for key in ('train_img', 'test_img'):
            dataset[key] = dataset[key].reshape(-1, 1, 28, 28)

    return (dataset['train_img'], dataset['train_label']), (dataset['test_img'], dataset['test_label']) 


if __name__ == '__main__':
    init_mnist()

If your accuracy is already really high (~99%), further training (especially with a high learning rate) might push the model out of the “good” loss landscape and your model would have to be retrained.