[FIXED] Why the accuracy of my neural network does not increase?

Issue

I tried to implement in python using pytorch from scratch a convolutional neural network based on the structure of AlexNet using the CIFAR10 dataset but my accuracy is very very low (10%). How can I improve my accuracy? Is there a structural problem or I have only to change the hyperparameters?I am sorry to there are trivial errors but I am a beginner in neural networks.

import matplotlib.pyplot as plt
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import torchvision

from collections import OrderedDict

import torchvision.datasets as datasets
import torchvision.transforms as transforms
from torchvision.datasets import CIFAR10
import os

class Config:
  def __init__(self):
    self.random_seed = 42
    self.n_epochs = 50
    self.batch_size_train = 256
    self.batch_size_test = 1000
    self.learning_rate = 0.0001
    self.momentum = 0.5
    self.log_interval = 500
    self.dropout_probability = 0.5


conf = Config()

torch.manual_seed(conf.random_seed)
torch.cuda.manual_seed(conf.random_seed)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
np.random.seed(conf.random_seed)

train_loader = torch.utils.data.DataLoader(
    torchvision.datasets.CIFAR10('/files/', train=True, download=True,
                               transform=torchvision.transforms.Compose([
        transforms.ToTensor(),
        transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
    ])),
                               batch_size=conf.batch_size_train,
                               shuffle=True
)

test_loader = torch.utils.data.DataLoader(
    torchvision.datasets.CIFAR10('/files/', train=False, download=True,
                               transform=transforms.Compose([
        transforms.ToTensor(),
        transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)
                             )
    ])),
                               batch_size=conf.batch_size_test,
                               shuffle=True
)
examples = enumerate(test_loader)
batch_idx, (example_data, example_targets) = next(examples)

example_data.shape
class AlexNet(nn.Module):
  def __init__(self):
    super(AlexNet,self).__init__()
    self.feature_extraction = nn.Sequential(
        nn.Conv2d(3, 96, kernel_size=11,stride = 4,padding = 2),
        nn.ReLU(),
        nn.MaxPool2d(kernel_size=3,stride=2),
        nn.Conv2d(96, 256, kernel_size = 5, padding=2),
        nn.ReLU(),
        nn.MaxPool2d(kernel_size=3,stride=2),
        nn.Conv2d(256, 384, kernel_size = 3, padding=1),
        nn.ReLU(),
        nn.Conv2d(384, 384, kernel_size = 3, padding=1),
        nn.ReLU(),
        nn.Conv2d(384, 256, kernel_size = 3, padding=1),
        nn.ReLU(),
        nn.MaxPool2d(kernel_size=3,stride=2)
        )
    self.classifier = nn.Sequential (
        nn.Dropout(p=0.5, inplace=True),
        nn.Linear(3072, out_features=4096),
        nn.ReLU(),
        nn.Dropout(p=0.5, inplace=True),
        nn.Linear(in_features=4096, out_features=4096),
        nn.ReLU(),
        nn.Linear(in_features=4096, out_features=1000)
    )
  def forward(self,x):
      x = x.view(x.size(0),-1)
      return self.classifier(x)
network = AlexNet()

model_parameters = filter(lambda p: p.requires_grad, network.parameters())
params = sum([np.prod(p.size()) for p in model_parameters])
print("The model has {} parameters.".format(f'{params:,}'))

optimizer = optim.Adam(
    network.parameters(),
    lr=conf.learning_rate
)

train_losses = []
train_counter = []
test_losses = []
test_counter = [i*len(train_loader.dataset) for i in range(conf.n_epochs + 1)]

for epoch in range(2):  # loop over the dataset multiple times

    running_loss = 0.0
    for i, data in enumerate(train_loader, 0):
        # get the inputs; data is a list of [inputs, labels]
        inputs, labels = data

        # zero the parameter gradients
        optimizer.zero_grad()

        # forward + backward + optimize
        outputs = network(inputs)
        loss = F.nll_loss(outputs, labels)
        loss.backward()
        optimizer.step()
        running_loss += loss.item()
        if i % 256 == 255:    # print every 2000 mini-batches
            print('[%d, %5d] loss: %.3f' %
                  (epoch + 1, i + 1, running_loss / 256))
            running_loss = 0.0

print('Finished Training')

correct = 0
total = 0
# since we're not training, we don't need to calculate the gradients for our outputs
with torch.no_grad():
    for data in test_loader:
        images, labels = data
        # calculate outputs by running images through the network
        outputs = network(images)
        # the class with the highest energy is what we choose as prediction
        _, predicted = torch.max(outputs.data, 1)
        total += labels.size(0)
        correct += (predicted == labels).sum().item()

print('Accuracy of the network on the 10000 test images: %d %%' % (
    100 * correct / total))

Solution

As you may have noticed there are plenty of loss functions in any deep learning package. You must choose the appropriate one based on problem criteria, such as multiclass/binary, multilabel/simple, log_logits, already softmaxed logits, and ... . nll_loss often used in with log_softmax logits, but you have used it with raw logits. Based on what was said adding log_softmax to forward path would do the work. So the model would change to this:

  def forward(self,x):
      x = x.view(x.size(0),-1)
      x = self.classifier(x)
      return torch.nn.functional.log_softmax(x, 1)

In this way I have got Accuracy of the network on the 10000 test images: 43 % after one epoch.

Answered By - meti