{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "pytorch_nn.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/gmihaila/machine_learning_toolbox/blob/master/pytorch_nn.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Y8f-8KN0CB46",
        "colab_type": "text"
      },
      "source": [
        "## SImple NN\n",
        "\n",
        "1 hiddent layer NN"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iddJzHDZCARo",
        "colab_type": "code",
        "outputId": "27576a6b-3369-481e-e403-2c2e19261cbf",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 153
        }
      },
      "source": [
        "import torch\n",
        "\n",
        "n_input, n_hidden, n_output = 5, 3, 1\n",
        "\n",
        "## initialize tensor for inputs, and outputs \n",
        "X = torch.randn((1, n_input))\n",
        "y = torch.rand((1,n_output)) \n",
        "\n",
        "\n",
        "print(x.size())\n",
        "print(y.size())\n",
        "print()\n",
        "\n",
        "## initialize tensor variables for weights \n",
        "w1 = torch.rand((n_input, n_hidden))\n",
        "w2 = torch.rand((n_hidden, n_output))\n",
        "\n",
        "print(w1.size())\n",
        "print(w2.size())\n",
        "print()\n",
        "\n",
        "## initialize tensor variables for bias terms \n",
        "b1 = torch.rand((1,n_hidden))\n",
        "b2 = torch.rand((1,n_output))\n",
        "\n",
        "print(b1.size())\n",
        "print(b2.size())\n",
        "print()\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.Size([1, 5])\n",
            "torch.Size([1, 1])\n",
            "\n",
            "torch.Size([5, 3])\n",
            "torch.Size([3, 1])\n",
            "\n",
            "torch.Size([1, 3])\n",
            "torch.Size([1, 1])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RzKELrKSD-t5",
        "colab_type": "text"
      },
      "source": [
        "1. Forward Propagation\n",
        "2. Loss computation\n",
        "3. Backpropagation\n",
        "4. Updating the parameters"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dHNZuYc-EEWb",
        "colab_type": "code",
        "outputId": "4eb66562-2ba9-4f66-8c3f-6557a921d0c8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        }
      },
      "source": [
        "## sigmoid activation function using pytorch\n",
        "def sigmoid_activation(z):\n",
        "  return 1 / (1 + torch.exp(-z))\n",
        "\n",
        "## activation of hidden layer \n",
        "z1 = torch.mm(X,w1) + b1\n",
        "a1 = sigmoid_activation(z1)\n",
        "\n",
        "print(z1)\n",
        "print(a1)\n",
        "print()\n",
        "\n",
        "## activation (output) of final layer \n",
        "z2 = torch.mm(a1, w2) + b2\n",
        "a2 = output = sigmoid_activation(z2)\n",
        "\n",
        "print(z2)\n",
        "print(output)\n",
        "print()\n",
        "\n",
        "loss = y - output\n",
        "\n",
        "print(loss)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[-0.0190, -1.0797, -0.6849]])\n",
            "tensor([[0.4952, 0.2536, 0.3352]])\n",
            "\n",
            "tensor([[0.8171]])\n",
            "tensor([[0.6936]])\n",
            "\n",
            "tensor([[0.0193]])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Y4oRU8O9JTOG",
        "colab_type": "text"
      },
      "source": [
        "### Backprop\n",
        "\n",
        "* loss gets multiplied by weights to penalize more of the bad weights\n",
        "* some weights contirbute more to the output. If the error is large, their loss will be more"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XbfW0NhaEkM_",
        "colab_type": "code",
        "outputId": "869f88a9-78a2-4112-a37d-d9ee024b19b9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        }
      },
      "source": [
        "## function to calculate the derivative of activation\n",
        "def sigmoid_delta(x):\n",
        "  return x * (1 - x)\n",
        "\n",
        "## compute derivative of error terms\n",
        "delta_output = sigmoid_delta(output)\n",
        "delta_hidden = sigmoid_delta(a1)\n",
        "\n",
        "print(delta_output)\n",
        "print(delta_hidden)\n",
        "print()\n",
        "\n",
        "\n",
        "## backpass the changes to previous layers \n",
        "d_output = loss * delta_output\n",
        "loss_h = torch.mm(d_output, w2.t())\n",
        "d_hidden = loss_h * delta_hidden\n",
        "\n",
        "print(d_output)\n",
        "print(loss_h)\n",
        "print(d_hidden)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[0.2125]])\n",
            "tensor([[0.2500, 0.1893, 0.2228]])\n",
            "\n",
            "tensor([[0.0041]])\n",
            "tensor([[0.0022, 0.0008, 0.0002]])\n",
            "tensor([[5.4456e-04, 1.5298e-04, 3.9821e-05]])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EbWmAwXtJ4xj",
        "colab_type": "text"
      },
      "source": [
        "### Update Parameters"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YHRwQNWyJ-KG",
        "colab_type": "code",
        "outputId": "982ea3e1-fe8f-4936-9aab-9b811b1f005d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "learning_rate = 0.1\n",
        "\n",
        "w2 += torch.mm(a1.t(), d_output) * learning_rate\n",
        "w1 += torch.mm(X.t(), d_hidden) * learning_rate\n",
        "\n",
        "\n",
        "print(w2)\n",
        "print(w1)\n",
        "print()\n",
        "\n",
        "b1 += d_output.sum() * learning_rate\n",
        "b2 += d_hidden.sum() * learning_rate\n",
        "\n",
        "print(b1)\n",
        "print(b2)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[0.5323],\n",
            "        [0.1976],\n",
            "        [0.0441]])\n",
            "tensor([[0.4660, 0.4263, 0.1392],\n",
            "        [0.5357, 0.0522, 0.0833],\n",
            "        [0.0660, 0.7443, 0.5947],\n",
            "        [0.1371, 0.1337, 0.8427],\n",
            "        [0.7600, 0.7110, 0.8661]])\n",
            "\n",
            "tensor([[0.7842, 0.3612, 0.9707]])\n",
            "tensor([[0.4894]])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vdo5seVgay4f",
        "colab_type": "text"
      },
      "source": [
        "## MNIST -NN"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "65a63f53-5470-42b0-a60e-7c1185648297",
        "id": "ZSRKe2UUm9-K",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "import torch\n",
        "from torch import optim\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from torchvision import transforms\n",
        "from torchvision.datasets import MNIST\n",
        "from torch.utils.data import DataLoader\n",
        "from torch.utils.data.sampler import SubsetRandomSampler\n",
        "import numpy as np\n",
        "\n",
        "# transform the raw dataset into tensors and normalize them in a fixed range\n",
        "_tasks = transforms.Compose([transforms.ToTensor(),transforms.Normalize((0.5,), (0.5,))]) #pass mean 0.5 and std 0.5\n",
        "\n",
        "## Load MNIST Dataset and apply transformations\n",
        "mnist = MNIST(\"data\", download=True, train=True, transform=_tasks)\n",
        "\n",
        "## create training and validation split \n",
        "split = int(0.8 * len(mnist))\n",
        "index_list = list(range(len(mnist)))\n",
        "train_idx, valid_idx = index_list[:split], index_list[split:]\n",
        "\n",
        "## create sampler objects using SubsetRandomSampler\n",
        "tr_sampler = SubsetRandomSampler(train_idx)\n",
        "val_sampler = SubsetRandomSampler(valid_idx)\n",
        "\n",
        "## create iterator objects for train and valid datasets\n",
        "trainloader = DataLoader(mnist, batch_size=256, sampler=tr_sampler)\n",
        "validloader = DataLoader(mnist, batch_size=256, sampler=val_sampler)\n",
        "\n",
        "\n",
        "for data, label in trainloader:\n",
        "  print(np.shape(data))\n",
        "  # Flatten MNIST images into a 784 long vector\n",
        "  # data = data.view(data.shape[0], -1)\n",
        "  # print(data.shape)\n",
        "\n",
        "  data = torch.flatten(data, start_dim=1)\n",
        "  print(data.shape)\n",
        "  break"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.Size([256, 1, 28, 28])\n",
            "torch.Size([256, 784])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8RFHs8bda0xx",
        "colab_type": "text"
      },
      "source": [
        "### NN"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0coSr0H-a4yR",
        "colab_type": "code",
        "outputId": "120060f9-1187-49b9-fb4d-f984df8546e8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        }
      },
      "source": [
        "import torch\n",
        "from torch import optim\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from torchvision import transforms\n",
        "from torchvision.datasets import MNIST\n",
        "from torch.utils.data import DataLoader\n",
        "from torch.utils.data.sampler import SubsetRandomSampler\n",
        "import numpy as np\n",
        "\n",
        "# transform the raw dataset into tensors and normalize them in a fixed range\n",
        "_tasks = transforms.Compose([transforms.ToTensor(),transforms.Normalize((0.5,), (0.5,))])\n",
        "\n",
        "## Load MNIST Dataset and apply transformations\n",
        "mnist = MNIST(\"data\", download=True, train=True, transform=_tasks)\n",
        "\n",
        "## create training and validation split \n",
        "split = int(0.8 * len(mnist))\n",
        "index_list = list(range(len(mnist)))\n",
        "train_idx, valid_idx = index_list[:split], index_list[split:]\n",
        "\n",
        "## create sampler objects using SubsetRandomSampler\n",
        "tr_sampler = SubsetRandomSampler(train_idx)\n",
        "val_sampler = SubsetRandomSampler(valid_idx)\n",
        "\n",
        "## create iterator objects for train and valid datasets\n",
        "trainloader = DataLoader(mnist, batch_size=256, sampler=tr_sampler)\n",
        "validloader = DataLoader(mnist, batch_size=256, sampler=val_sampler)\n",
        "\n",
        "## Build class of model\n",
        "class Model(nn.Module):\n",
        "    def __init__(self):\n",
        "        super().__init__()\n",
        "        self.hidden = nn.Linear(784, 128)\n",
        "        self.output = nn.Linear(128, 10)\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = self.hidden(x)\n",
        "        x = torch.sigmoid(x)\n",
        "        x = self.output(x)\n",
        "        return x\n",
        "\n",
        "model = Model()\n",
        "\n",
        "loss_function = nn.CrossEntropyLoss()\n",
        "optimizer = optim.SGD(model.parameters(), lr=0.01, weight_decay=1e-6, momentum=0.9, nesterov=True)\n",
        "\n",
        "for epoch in range(1,11):\n",
        "\n",
        "  train_loss, valid_loss = [], []\n",
        "  model.train() # activates training mod\n",
        "\n",
        "  ## Training on 1 epoch\n",
        "  for data, target in trainloader:\n",
        "\n",
        "    data = torch.flatten(data, start_dim=1)\n",
        "\n",
        "    optimizer.zero_grad() #clears gradients of all optimized classes\n",
        "\n",
        "    ## forward pass\n",
        "    output = model(data)\n",
        "\n",
        "    ## loss calc\n",
        "    loss = loss_function(output, target)\n",
        "\n",
        "    ## backeard propagation\n",
        "    loss.backward()\n",
        "\n",
        "    ## weight optimization\n",
        "    optimizer.step() #performs a single optimization step\n",
        "    train_loss.append(loss.item())\n",
        "\n",
        "  ### Evaluation on 1 epoch\n",
        "  for data, target in validloader:\n",
        "\n",
        "    data = torch.flatten(data, start_dim=1)\n",
        "\n",
        "    output = model(data)\n",
        "    loss = loss_function(output, target)\n",
        "    valid_loss.append(loss.item())\n",
        "  \n",
        "  print(\"Epoch:\", epoch, \"Training Loss: \", np.mean(train_loss), \"Valid Loss: \", np.mean(valid_loss))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch: 1 Training Loss:  1.4278296362212364 Valid Loss:  0.7039066375570094\n",
            "Epoch: 2 Training Loss:  0.5753547265491588 Valid Loss:  0.446183591446978\n",
            "Epoch: 3 Training Loss:  0.43230179998468843 Valid Loss:  0.37309577490421053\n",
            "Epoch: 4 Training Loss:  0.3773977352266616 Valid Loss:  0.3384805096590773\n",
            "Epoch: 5 Training Loss:  0.34632741073344614 Valid Loss:  0.3146806723893957\n",
            "Epoch: 6 Training Loss:  0.3253439431019286 Valid Loss:  0.299487607593232\n",
            "Epoch: 7 Training Loss:  0.3091939938987823 Valid Loss:  0.28539510832187975\n",
            "Epoch: 8 Training Loss:  0.2957918224658104 Valid Loss:  0.27507571083434085\n",
            "Epoch: 9 Training Loss:  0.28438072516880136 Valid Loss:  0.2652054229315291\n",
            "Epoch: 10 Training Loss:  0.27375956244291144 Valid Loss:  0.2566439474516727\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j_5G0MzHewg8",
        "colab_type": "text"
      },
      "source": [
        "### Validation NN"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o7DHUA_wbkYI",
        "colab_type": "code",
        "outputId": "7e164fe9-4ec5-4ca7-eb4c-565d9cd4cfa7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        }
      },
      "source": [
        "## dataloader for validation dataset \n",
        "dataiter = iter(validloader)\n",
        "data, labels = dataiter.next()\n",
        "data = torch.flatten(data, start_dim=1)\n",
        "output = model(data)\n",
        "\n",
        "print(output.shape)\n",
        "print(output[0])\n",
        "\n",
        "_, pred_tensor = torch.max(output, 1)\n",
        "\n",
        "print(pred_tensor.shape)\n",
        "print(pred_tensor[0])\n",
        "\n",
        "preds = np.squeeze(pred_tensor.numpy())\n",
        "\n",
        "print(\"Actual: \", labels[:10])\n",
        "print(\"Predic: \", preds[:10])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.Size([256, 10])\n",
            "tensor([-3.2789, -1.0494,  0.6679,  0.9565, -1.4610,  1.6791, -3.1327, -2.5092,\n",
            "         8.5172,  0.7382], grad_fn=<SelectBackward>)\n",
            "torch.Size([256])\n",
            "tensor(8)\n",
            "Actual:  tensor([8, 8, 7, 9, 6, 8, 3, 7, 7, 5])\n",
            "Predic:  [8 5 7 9 6 8 8 7 7 5]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iY07nwY8rcDg",
        "colab_type": "text"
      },
      "source": [
        "## MNIST - NN [1 GPU]"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ha61G2ngrbiT",
        "colab_type": "code",
        "outputId": "3b9e9339-8045-4bd8-d041-20f1e8f9ed5a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "import torch\n",
        "from torch import optim\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from torchvision import transforms\n",
        "from torchvision.datasets import MNIST\n",
        "from torch.utils.data import DataLoader\n",
        "from torch.utils.data.sampler import SubsetRandomSampler\n",
        "import numpy as np\n",
        "from torch.backends import cudnn\n",
        "\n",
        "cudnn.benchmark = True\n",
        "\n",
        "# transform the raw dataset into tensors and normalize them in a fixed range\n",
        "_tasks = transforms.Compose([transforms.ToTensor(),transforms.Normalize((0.5,), (0.5,))])\n",
        "\n",
        "## Load MNIST Dataset and apply transformations\n",
        "mnist = MNIST(\"data\", download=True, train=True, transform=_tasks)\n",
        "\n",
        "## create training and validation split \n",
        "split = int(0.8 * len(mnist))\n",
        "index_list = list(range(len(mnist)))\n",
        "train_idx, valid_idx = index_list[:split], index_list[split:]\n",
        "\n",
        "## create sampler objects using SubsetRandomSampler\n",
        "tr_sampler = SubsetRandomSampler(train_idx)\n",
        "val_sampler = SubsetRandomSampler(valid_idx)\n",
        "\n",
        "## create iterator objects for train and valid datasets\n",
        "trainloader = DataLoader(mnist, batch_size=256, sampler=tr_sampler, num_workers=2)\n",
        "validloader = DataLoader(mnist, batch_size=256, sampler=val_sampler, num_workers=2)\n",
        "\n",
        "## GPU\n",
        "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "## Build class of model\n",
        "class Model(nn.Module):\n",
        "    def __init__(self):\n",
        "        super().__init__()\n",
        "        self.hidden = nn.Linear(784, 128)\n",
        "        self.output = nn.Linear(128, 10)\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = self.hidden(x)\n",
        "        x = torch.sigmoid(x)\n",
        "        x = self.output(x)\n",
        "        return x\n",
        "\n",
        "model = Model()\n",
        "\n",
        "model.to(device)\n",
        "\n",
        "loss_function = nn.CrossEntropyLoss()\n",
        "optimizer = optim.SGD(model.parameters(), lr=0.01, weight_decay=1e-6, momentum=0.9, nesterov=True)\n",
        "\n",
        "for epoch in range(1,11):\n",
        "\n",
        "  train_loss, valid_loss = [], []\n",
        "  model.train() # activates training mod\n",
        "\n",
        "  ## Training on 1 epoch\n",
        "  for data, target in trainloader:\n",
        "\n",
        "    data = torch.flatten(data.to(device), start_dim=1)\n",
        "\n",
        "    optimizer.zero_grad() #clears gradients of all optimized classes\n",
        "\n",
        "    ## forward pass\n",
        "    output = model(data.to(device))\n",
        "\n",
        "    ## loss calc\n",
        "    loss = loss_function(output.to(device), target.to(device))\n",
        "\n",
        "    ## backeard propagation\n",
        "    loss.backward()\n",
        "\n",
        "    ## weight optimization\n",
        "    optimizer.step() #performs a single optimization step\n",
        "    train_loss.append(loss.item())\n",
        "\n",
        "  ### Evaluation on 1 epoch\n",
        "  for data, target in validloader:\n",
        "\n",
        "    data = torch.flatten(data, start_dim=1)\n",
        "\n",
        "    output = model(data.to(device))\n",
        "    loss = loss_function(output.to(device), target.to(device))\n",
        "    valid_loss.append(loss.item())\n",
        "  \n",
        "  print(\"Epoch:\", epoch, \"Training Loss: \", np.mean(train_loss), \"Valid Loss: \", np.mean(valid_loss))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "cuda:0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "scUaqYhqvfyP",
        "colab_type": "text"
      },
      "source": [
        "## MNIST - NN [Multy GPU - Multy Core]\n",
        "\n",
        "Specify certain GPUs"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vfdi1HANvi7f",
        "colab_type": "code",
        "outputId": "b0dec8ee-d09e-4d4f-f54d-3cf6f28c5c32",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "import os\n",
        "# os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0,1,2\" # number of gpu devices\n",
        "import torch\n",
        "from torch import optim\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from torchvision import transforms\n",
        "from torchvision.datasets import MNIST\n",
        "from torch.utils.data import DataLoader\n",
        "from torch.utils.data.sampler import SubsetRandomSampler\n",
        "import numpy as np\n",
        "from torch.backends import cudnn\n",
        "import multiprocessing\n",
        "\n",
        "cudnn.benchmark = True\n",
        "\n",
        "n_cores = multiprocessing.cpu_count()\n",
        "\n",
        "# transform the raw dataset into tensors and normalize them in a fixed range\n",
        "_tasks = transforms.Compose([transforms.ToTensor(),transforms.Normalize((0.5,), (0.5,))])\n",
        "\n",
        "## Load MNIST Dataset and apply transformations\n",
        "mnist = MNIST(\"data\", download=True, train=True, transform=_tasks)\n",
        "\n",
        "## create training and validation split \n",
        "split = int(0.8 * len(mnist))\n",
        "index_list = list(range(len(mnist)))\n",
        "train_idx, valid_idx = index_list[:split], index_list[split:]\n",
        "\n",
        "## create sampler objects using SubsetRandomSampler\n",
        "tr_sampler = SubsetRandomSampler(train_idx)\n",
        "val_sampler = SubsetRandomSampler(valid_idx)\n",
        "\n",
        "## create iterator objects for train and valid datasets\n",
        "trainloader = DataLoader(mnist, batch_size=256, sampler=tr_sampler, num_workers=n_cores)\n",
        "validloader = DataLoader(mnist, batch_size=256, sampler=val_sampler, num_workers=n_cores)\n",
        "\n",
        "## GPU\n",
        "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "## Build class of model\n",
        "class Model(nn.Module):\n",
        "    def __init__(self):\n",
        "        super().__init__()\n",
        "        self.hidden = nn.Linear(784, 128)\n",
        "        self.output = nn.Linear(128, 10)\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = self.hidden(x)\n",
        "        x = torch.sigmoid(x)\n",
        "        x = self.output(x)\n",
        "        return x\n",
        "\n",
        "model = Model()\n",
        "\n",
        "## Multi GPU\n",
        "if torch.cuda.device_count() > 1:\n",
        "  print(\"We can use\", torch.cuda.device_count(), \"GPUs\")\n",
        "  model = nn.DataParallel(model, device_ids=[1]) # device_ids=[0,1,2] depending on the # of gpus\n",
        "\n",
        "model.to(device)\n",
        "\n",
        "loss_function = nn.CrossEntropyLoss()\n",
        "optimizer = optim.SGD(model.parameters(), lr=0.01, weight_decay=1e-6, momentum=0.9, nesterov=True)\n",
        "\n",
        "for epoch in range(1,11):\n",
        "\n",
        "  train_loss, valid_loss = [], []\n",
        "  model.train() # activates training mod\n",
        "\n",
        "  ## Training on 1 epoch\n",
        "  for data, target in trainloader:\n",
        "\n",
        "    data = torch.flatten(data.to(device), start_dim=1)\n",
        "\n",
        "    optimizer.zero_grad() #clears gradients of all optimized classes\n",
        "\n",
        "    ## forward pass\n",
        "    output = model(data.to(device))\n",
        "\n",
        "    ## loss calc\n",
        "    loss = loss_function(output.to(device), target.to(device))\n",
        "\n",
        "    ## backeard propagation\n",
        "    loss.backward()\n",
        "\n",
        "    ## weight optimization\n",
        "    optimizer.step() #performs a single optimization step\n",
        "    train_loss.append(loss.item())\n",
        "\n",
        "  ### Evaluation on 1 epoch\n",
        "  for data, target in validloader:\n",
        "\n",
        "    data = torch.flatten(data, start_dim=1)\n",
        "\n",
        "    output = model(data.to(device))\n",
        "    loss = loss_function(output.to(device), target.to(device))\n",
        "    valid_loss.append(loss.item())\n",
        "  \n",
        "  print(\"Epoch:\", epoch, \"Training Loss: \", np.mean(train_loss), \"Valid Loss: \", np.mean(valid_loss))"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n",
            "torch.Size([256, 10])\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-36-5fa9db7f8f35>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     70\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     71\u001b[0m   \u001b[0;31m## Training on 1 epoch\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 72\u001b[0;31m   \u001b[0;32mfor\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtrainloader\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     73\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     74\u001b[0m     \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mflatten\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstart_dim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    566\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrcvd_idx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreorder_dict\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    567\u001b[0m             \u001b[0mbatch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreorder_dict\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrcvd_idx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 568\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_process_next_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    569\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    570\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatches_outstanding\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m_process_next_batch\u001b[0;34m(self, batch)\u001b[0m\n\u001b[1;32m    599\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_process_next_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    600\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrcvd_idx\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 601\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_put_indices\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    602\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_utils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mExceptionWrapper\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    603\u001b[0m             \u001b[0;31m# make multiline KeyError msg readable by working around\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m_put_indices\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    589\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_put_indices\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    590\u001b[0m         \u001b[0;32massert\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatches_outstanding\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0;36m2\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnum_workers\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 591\u001b[0;31m         \u001b[0mindices\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msample_iter\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    592\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mindices\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    593\u001b[0m             \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/utils/data/sampler.py\u001b[0m in \u001b[0;36m__iter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    170\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__iter__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    171\u001b[0m         \u001b[0mbatch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 172\u001b[0;31m         \u001b[0;32mfor\u001b[0m \u001b[0midx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msampler\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    173\u001b[0m             \u001b[0mbatch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    174\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/utils/data/sampler.py\u001b[0m in \u001b[0;36m<genexpr>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m     94\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     95\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__iter__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 96\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindices\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandperm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindices\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     97\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     98\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__len__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8Wy3zgHCy7Ot",
        "colab_type": "text"
      },
      "source": [
        "## MNIST CNN [Multy GPU, CPU]"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uJUm3qjvBfFe",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 170
        },
        "outputId": "e4fdfb90-566e-43a4-e4c5-424dce012a09"
      },
      "source": [
        "import os\n",
        "import torch\n",
        "from torch import optim\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from torchvision import transforms\n",
        "from torchvision.datasets import MNIST\n",
        "from torch.utils.data import DataLoader\n",
        "from torch.utils.data.sampler import SubsetRandomSampler\n",
        "from torch.backends import cudnn\n",
        "import numpy as np\n",
        "import multiprocessing\n",
        "\n",
        "cudnn.benchmark = True\n",
        "\n",
        "num_cores = multiprocessing.cpu_count()\n",
        "\n",
        "# transform the raw dataset into tensors and normalize them in a fixed range\n",
        "_tasks = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (0.5,))])\n",
        "\n",
        "## Load MNIST Dataset and apply transformations\n",
        "mnist = MNIST(\"data\", download=True, train=True, transform=_tasks)\n",
        "\n",
        "## create training and validation split \n",
        "split = int(0.8 * len(mnist))\n",
        "index_list = list(range(len(mnist)))\n",
        "train_idx, valid_idx = index_list[:split], index_list[split:]\n",
        "\n",
        "## create sampler objects using SubsetRandomSampler\n",
        "tr_sampler = SubsetRandomSampler(train_idx)\n",
        "val_sampled = SubsetRandomSampler(valid_idx)\n",
        "\n",
        "## create iterator objects for train and valid datasets\n",
        "trainloader = DataLoader(mnist, batch_size=256, sampler=tr_sampler, num_workers=num_cores)\n",
        "validloader = DataLoader(mnist, batch_size=256, sampler=val_sampler, num_workers=num_cores)\n",
        "\n",
        "## GPU\n",
        "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "## Build class of model\n",
        "class Model(nn.Module):\n",
        "  def __init__(self):\n",
        "    super(Model, self).__init__()\n",
        "    \n",
        "    ## define layers\n",
        "    self.conv1 = nn.Conv2d(1, 16, 3, padding=1)\n",
        "    self.conv2 = nn.Conv2d(16, 32, 3, padding=1)\n",
        "    self.conv3 = nn.Conv2d(32, 64, 3, padding=1)\n",
        "    self.pool = nn.MaxPool2d(kernel_size=2, stride=2)\n",
        "    self.linear1 = nn.Linear(64*3*3, 512)\n",
        "    self.linear2 = nn.Linear(512,10)\n",
        "    \n",
        "    return\n",
        "\n",
        "  def forward(self, x):\n",
        "    x = self.pool(F.relu(self.conv1(x)))\n",
        "    x = self.pool(F.relu(self.conv2(x)))\n",
        "    x = self.pool(F.relu(self.conv3(x)))\n",
        "    x = x.view(-1,64*3*3) #torch.flatten(x, start_dim=1) ## reshaping\n",
        "    x = F.relu(self.linear1(x))\n",
        "    x = self.linear2(x)\n",
        "\n",
        "    return x\n",
        "\n",
        "## create model\n",
        "model = Model()\n",
        "\n",
        "## in case of multi gpu\n",
        "if torch.cuda.device_count() > 1:\n",
        "  print(\"Using\", torch.cuda.device_count(), \"GPUs\")\n",
        "  model = nn.DataParallel(model, device_ids=[1]) # [0,1,2,3]\n",
        "\n",
        "## put model on gpu\n",
        "model.to(device)\n",
        "\n",
        "## loss fucntion\n",
        "loss_function = nn.CrossEntropyLoss()\n",
        "## optimizer\n",
        "optimizer = optim.SGD(model.parameters(), lr=0.01, weight_decay=1e-6, momentum=0.9, nesterov=True)\n",
        "## run for n epochs\n",
        "for epoch in range(1,11):\n",
        "  train_loss , valid_loss = [], []\n",
        "\n",
        "  ## train part\n",
        "  model.train()\n",
        "  for data, target in trainloader:\n",
        "    ## gradients acumulate. need to clear them on each example\n",
        "    optimizer.zero_grad()\n",
        "    output = model(data.to(device))\n",
        "    loss = loss_function(output.to(device), target.to(device))\n",
        "    loss.backward()\n",
        "    optimizer.step()\n",
        "    train_loss.append(loss.item())\n",
        "\n",
        "  ## evaluation part on validation\n",
        "  model.eval() ##set model in evaluation mode\n",
        "  for data, target in validloader:\n",
        "    output = model(data.to(device))\n",
        "    loss = loss_function(output.to(device), target.to(device))\n",
        "    valid_loss.append(loss.item())\n",
        "\n",
        "  print(\"Epoch:\", epoch, \"Training Loss: \", np.mean(train_loss), \"Valid Loss: \", np.mean(valid_loss))"
      ],
      "execution_count": 53,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch: 1 Training Loss:  1.3867575062557738 Valid Loss:  0.20830699516103623\n",
            "Epoch: 2 Training Loss:  0.14508407129014425 Valid Loss:  0.10438944716402825\n",
            "Epoch: 3 Training Loss:  0.0902247162773571 Valid Loss:  0.07995569214541862\n",
            "Epoch: 4 Training Loss:  0.06947706440622185 Valid Loss:  0.08476778730115991\n",
            "Epoch: 5 Training Loss:  0.05636777369146968 Valid Loss:  0.06578631869497452\n",
            "Epoch: 6 Training Loss:  0.048184677641442485 Valid Loss:  0.05531598358078206\n",
            "Epoch: 7 Training Loss:  0.04294469940694089 Valid Loss:  0.05248951709809455\n",
            "Epoch: 8 Training Loss:  0.038696830844546254 Valid Loss:  0.048144756558727714\n",
            "Epoch: 9 Training Loss:  0.03434643990043154 Valid Loss:  0.04841103820883213\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hJZJLvwgsUrq",
        "colab_type": "text"
      },
      "source": [
        "#### Evaluation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eSRiJuEPsW0x",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}