{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "tf2.1_multi_gpu.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyMRddhUPGWwiOs+/AnbXfNH",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/gmihaila/machine_learning_things/blob/master/learning_tensorflow/tf2_1_multi_gpu.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nkifbROOs6Kj",
        "colab_type": "text"
      },
      "source": [
        "Tutorial https://www.tensorflow.org/guide/distributed_training"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ylcvufTPjOw2",
        "colab_type": "text"
      },
      "source": [
        "### Imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UY9Ov-QkE-qw",
        "colab_type": "code",
        "outputId": "26a79df9-acc0-47af-b977-45e7f0240a03",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        }
      },
      "source": [
        "import os\n",
        "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n",
        "# Import TensorFlow\n",
        "from __future__ import absolute_import, division, print_function, unicode_literals\n",
        "try:\n",
        "  # %tensorflow_version only exists in Colab.\n",
        "  %tensorflow_version 2.x\n",
        "except Exception:\n",
        "  pass\n",
        "import tensorflow as tf\n",
        "\n",
        "  \n",
        "print(\"tf version running \", tf.version.VERSION)\n",
        "print(\"Num GPUs Available: \", len(tf.config.experimental.list_physical_devices('GPU')))\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tf version running  2.1.0\n",
            "Num GPUs Available:  1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vuZFV3KvjgEK",
        "colab_type": "text"
      },
      "source": [
        "### Compare GPU - CPU"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3dSgF0ESLpbF",
        "colab_type": "code",
        "outputId": "de24428f-5233-4b3b-920c-640891106094",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 125
        }
      },
      "source": [
        "device_name = tf.test.gpu_device_name()\n",
        "if device_name != '/device:GPU:0':\n",
        "  print(\n",
        "      '\\n\\nThis error most likely means that this notebook is not '\n",
        "      'configured to use a GPU.  Change this in Notebook Settings via the '\n",
        "      'command palette (cmd/ctrl-shift-P) or the Edit menu.\\n\\n')\n",
        "  raise SystemError('GPU device not found')\n",
        "\n",
        "def cpu():\n",
        "  with tf.device('/cpu:0'):\n",
        "    random_image_cpu = tf.random.normal((100, 100, 100, 3))\n",
        "    net_cpu = tf.keras.layers.Conv2D(32, 7)(random_image_cpu)\n",
        "    return tf.math.reduce_sum(net_cpu)\n",
        "\n",
        "def gpu():\n",
        "  with tf.device('/device:GPU:0'):\n",
        "    random_image_gpu = tf.random.normal((100, 100, 100, 3))\n",
        "    net_gpu = tf.keras.layers.Conv2D(32, 7)(random_image_gpu)\n",
        "    return tf.math.reduce_sum(net_gpu)\n",
        "  \n",
        "# We run each op once to warm up; see: https://stackoverflow.com/a/45067900\n",
        "cpu()\n",
        "gpu()\n",
        "\n",
        "# Run the op several times.\n",
        "print('Time (s) to convolve 32x7x7x3 filter over random 100x100x100x3 images '\n",
        "      '(batch x height x width x channel). Sum of ten runs.')\n",
        "print('CPU (s):')\n",
        "cpu_time = timeit.timeit('cpu()', number=10, setup=\"from __main__ import cpu\")\n",
        "print(cpu_time)\n",
        "print('GPU (s):')\n",
        "gpu_time = timeit.timeit('gpu()', number=10, setup=\"from __main__ import gpu\")\n",
        "print(gpu_time)\n",
        "print('GPU speedup over CPU: {}x'.format(int(cpu_time/gpu_time)))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Time (s) to convolve 32x7x7x3 filter over random 100x100x100x3 images (batch x height x width x channel). Sum of ten runs.\n",
            "CPU (s):\n",
            "3.4301189010000144\n",
            "GPU (s):\n",
            "0.1132346699999971\n",
            "GPU speedup over CPU: 30x\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qJDBbtKOjlSj",
        "colab_type": "text"
      },
      "source": [
        "### Multi GPU"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kgK4EkMkjmkJ",
        "colab_type": "code",
        "outputId": "aa4f5e74-2617-46a7-d6fe-d3078a8b7f20",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 161
        }
      },
      "source": [
        "mirrored_strategy = tf.distribute.MirroredStrategy()\n",
        "with mirrored_strategy.scope():\n",
        "  model = tf.keras.Sequential([tf.keras.layers.Dense(1, input_shape=(1,))])\n",
        "  model.compile(loss='mse', optimizer='sgd')\n",
        "\n",
        "\n",
        "dataset = tf.data.Dataset.from_tensors(([1.], [1.])).repeat(100).batch(10)\n",
        "model.fit(dataset, epochs=2)\n",
        "model.evaluate(dataset)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Using MirroredStrategy with devices ('/job:localhost/replica:0/task:0/device:GPU:0',)\n",
            "Train for 10 steps\n",
            "Epoch 1/2\n",
            "10/10 [==============================] - 2s 244ms/step - loss: 0.0091\n",
            "Epoch 2/2\n",
            "10/10 [==============================] - 0s 3ms/step - loss: 0.0040\n",
            "10/10 [==============================] - 1s 109ms/step - loss: 0.0025\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.002487475983798504"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pam4WbWyz3Dq",
        "colab_type": "text"
      },
      "source": [
        "### Talon Script"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ms6wM_PMz4xt",
        "colab_type": "text"
      },
      "source": [
        "### Single GPU"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "trBYYf8oz6LG",
        "colab_type": "code",
        "outputId": "2d87e942-470c-42e8-ec5f-43dcfdfc0fae",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 253
        }
      },
      "source": [
        "import sys\n",
        "import os\n",
        "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n",
        "import tensorflow as tf\n",
        "import timeit\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "  if tf.test.is_gpu_available():\n",
        "    print(\"\\nGPU detected yeeey!\\n\")\n",
        "    print(\"tf version running \", tf.version.VERSION)\n",
        "    print(\"\\nNum GPUs Available: \", len(tf.config.experimental.list_physical_devices('GPU')))\n",
        "\n",
        "    device_name = tf.test.gpu_device_name()\n",
        "    if device_name != '/device:GPU:0':\n",
        "      print(\n",
        "          '\\n\\nThis error most likely means that this script is not '\n",
        "          'configured to use a GPU.  Change this in Notebook Settings via the '\n",
        "          'command palette (cmd/ctrl-shift-P) or the Edit menu.\\n\\n')\n",
        "      raise SystemError('GPU device not found')\n",
        "\n",
        "    def cpu():\n",
        "      with tf.device('/cpu:0'):\n",
        "        random_image_cpu = tf.random.normal((100, 100, 100, 3))\n",
        "        net_cpu = tf.keras.layers.Conv2D(32, 7)(random_image_cpu)\n",
        "        return tf.math.reduce_sum(net_cpu)\n",
        "\n",
        "    def gpu():\n",
        "      with tf.device('/device:GPU:0'):\n",
        "        random_image_gpu = tf.random.normal((100, 100, 100, 3))\n",
        "        net_gpu = tf.keras.layers.Conv2D(32, 7)(random_image_gpu)\n",
        "        return tf.math.reduce_sum(net_gpu)\n",
        "      \n",
        "    # We run each op once to warm up; see: https://stackoverflow.com/a/45067900\n",
        "    cpu()\n",
        "    gpu()\n",
        "\n",
        "    # Run the op several times.\n",
        "    print('Time (s) to convolve 32x7x7x3 filter over random 100x100x100x3 images '\n",
        "          '(batch x height x width x channel). Sum of ten runs.')\n",
        "    print('CPU (s):')\n",
        "    cpu_time = timeit.timeit('cpu()', number=10, setup=\"from __main__ import cpu\")\n",
        "    print(cpu_time)\n",
        "    print('GPU (s):')\n",
        "    gpu_time = timeit.timeit('gpu()', number=10, setup=\"from __main__ import gpu\")\n",
        "    print(gpu_time)\n",
        "    print('GPU speedup over CPU: {}x'.format(int(cpu_time/gpu_time)))\n",
        "\n",
        "  else:\n",
        "    print(\"No GPU detected!\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "GPU detected yeeey!\n",
            "tf version running  1.15.0\n",
            "Num GPUs Available:  1\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "If using Keras pass *_constraint arguments to layers.\n",
            "Time (s) to convolve 32x7x7x3 filter over random 100x100x100x3 images (batch x height x width x channel). Sum of ten runs.\n",
            "CPU (s):\n",
            "0.1697342129999697\n",
            "GPU (s):\n",
            "0.1691328770000382\n",
            "GPU speedup over CPU: 1x\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PGT8OBru21tX",
        "colab_type": "text"
      },
      "source": [
        "### Multi GPU"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V7f0R8TK23bv",
        "colab_type": "code",
        "outputId": "9801c190-fe12-458c-fcba-8f7c1690c061",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 539
        }
      },
      "source": [
        "import sys\n",
        "import os\n",
        "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n",
        "import tensorflow as tf\n",
        "import timeit\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "  if tf.test.is_gpu_available():\n",
        "    print(\"\\nGPU detected yeeey!\\n\")\n",
        "    print(\"tf version running \", tf.version.VERSION)\n",
        "    print(\"\\nNum GPUs Available: \", len(tf.config.experimental.list_physical_devices('GPU')))\n",
        "    mirrored_strategy = tf.distribute.MirroredStrategy()\n",
        "    with mirrored_strategy.scope():\n",
        "      model = tf.keras.Sequential([tf.keras.layers.Dense(1, input_shape=(1,))])\n",
        "      model.compile(loss='mse', optimizer='sgd')\n",
        "\n",
        "\n",
        "    dataset = tf.data.Dataset.from_tensors(([1.], [1.])).repeat(50000).batch(50)\n",
        "    model.fit(dataset, epochs=10)\n",
        "    model.evaluate(dataset)\n",
        "\n",
        "  else:\n",
        "    print(\"No GPU detected!\")\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "GPU detected yeeey!\n",
            "\n",
            "tf version running  1.15.0\n",
            "\n",
            "Num GPUs Available:  1\n",
            "WARNING:tensorflow:Expected a shuffled dataset but input dataset `x` is not shuffled. Please invoke `shuffle()` on input dataset.\n",
            "Train on 1000 steps\n",
            "Epoch 1/10\n",
            "1000/1000 [==============================] - 2s 2ms/step - loss: 0.0245\n",
            "Epoch 2/10\n",
            "1000/1000 [==============================] - 2s 2ms/step - loss: 5.1159e-13\n",
            "Epoch 3/10\n",
            "1000/1000 [==============================] - 2s 2ms/step - loss: 5.1159e-13\n",
            "Epoch 4/10\n",
            "1000/1000 [==============================] - 2s 2ms/step - loss: 5.1159e-13\n",
            "Epoch 5/10\n",
            "1000/1000 [==============================] - 2s 2ms/step - loss: 5.1159e-13\n",
            "Epoch 6/10\n",
            "1000/1000 [==============================] - 2s 2ms/step - loss: 5.1159e-13\n",
            "Epoch 7/10\n",
            "1000/1000 [==============================] - 2s 2ms/step - loss: 5.1159e-13\n",
            "Epoch 8/10\n",
            "1000/1000 [==============================] - 2s 2ms/step - loss: 5.1159e-13\n",
            "Epoch 9/10\n",
            "1000/1000 [==============================] - 2s 2ms/step - loss: 5.1159e-13\n",
            "Epoch 10/10\n",
            "1000/1000 [==============================] - 2s 2ms/step - loss: 5.1159e-13\n",
            "1000/1000 [==============================] - 1s 1ms/step - loss: 5.1159e-13\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}