alihan/ml_things
1
0
Fork 0
mirror of https://github.com/gmihaila/ml_things.git synced 2021-10-04 01:29:04 +03:00
Code Issues Projects Releases Wiki Activity

Created using Colaboratory

Browse Source
This commit is contained in:
George Mihaila
2020-02-08 18:49:40 -06:00
parent 6ac9ca6e02
commit 6a9dfe4cda
1 changed files with 412 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

412
learning_tensorflow/tf2_1_multi_gpu.ipynb Normal file
View File

@@ -0,0 +1,412 @@
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"name": "tf2.1_multi_gpu.ipynb",
"provenance": [],
"collapsed_sections": [],
"authorship_tag": "ABX9TyMRddhUPGWwiOs+/AnbXfNH",
"include_colab_link": true
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"accelerator": "GPU"
},
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "view-in-github",
"colab_type": "text"
},
"source": [
"<a href=\"https://colab.research.google.com/github/gmihaila/machine_learning_things/blob/master/learning_tensorflow/tf2_1_multi_gpu.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "nkifbROOs6Kj",
"colab_type": "text"
},
"source": [
"Tutorial https://www.tensorflow.org/guide/distributed_training"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "ylcvufTPjOw2",
"colab_type": "text"
},
"source": [
"### Imports"
]
},
{
"cell_type": "code",
"metadata": {
"id": "UY9Ov-QkE-qw",
"colab_type": "code",
"outputId": "26a79df9-acc0-47af-b977-45e7f0240a03",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 53
}
},
"source": [
"import os\n",
"os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n",
"# Import TensorFlow\n",
"from __future__ import absolute_import, division, print_function, unicode_literals\n",
"try:\n",
" # %tensorflow_version only exists in Colab.\n",
" %tensorflow_version 2.x\n",
"except Exception:\n",
" pass\n",
"import tensorflow as tf\n",
"\n",
" \n",
"print(\"tf version running \", tf.version.VERSION)\n",
"print(\"Num GPUs Available: \", len(tf.config.experimental.list_physical_devices('GPU')))\n"
],
"execution_count": 0,
"outputs": [
{
"output_type": "stream",
"text": [
"tf version running 2.1.0\n",
"Num GPUs Available: 1\n"
],
"name": "stdout"
}
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "vuZFV3KvjgEK",
"colab_type": "text"
},
"source": [
"### Compare GPU - CPU"
]
},
{
"cell_type": "code",
"metadata": {
"id": "3dSgF0ESLpbF",
"colab_type": "code",
"outputId": "de24428f-5233-4b3b-920c-640891106094",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 125
}
},
"source": [
"device_name = tf.test.gpu_device_name()\n",
"if device_name != '/device:GPU:0':\n",
" print(\n",
" '\\n\\nThis error most likely means that this notebook is not '\n",
" 'configured to use a GPU. Change this in Notebook Settings via the '\n",
" 'command palette (cmd/ctrl-shift-P) or the Edit menu.\\n\\n')\n",
" raise SystemError('GPU device not found')\n",
"\n",
"def cpu():\n",
" with tf.device('/cpu:0'):\n",
" random_image_cpu = tf.random.normal((100, 100, 100, 3))\n",
" net_cpu = tf.keras.layers.Conv2D(32, 7)(random_image_cpu)\n",
" return tf.math.reduce_sum(net_cpu)\n",
"\n",
"def gpu():\n",
" with tf.device('/device:GPU:0'):\n",
" random_image_gpu = tf.random.normal((100, 100, 100, 3))\n",
" net_gpu = tf.keras.layers.Conv2D(32, 7)(random_image_gpu)\n",
" return tf.math.reduce_sum(net_gpu)\n",
" \n",
"# We run each op once to warm up; see: https://stackoverflow.com/a/45067900\n",
"cpu()\n",
"gpu()\n",
"\n",
"# Run the op several times.\n",
"print('Time (s) to convolve 32x7x7x3 filter over random 100x100x100x3 images '\n",
" '(batch x height x width x channel). Sum of ten runs.')\n",
"print('CPU (s):')\n",
"cpu_time = timeit.timeit('cpu()', number=10, setup=\"from __main__ import cpu\")\n",
"print(cpu_time)\n",
"print('GPU (s):')\n",
"gpu_time = timeit.timeit('gpu()', number=10, setup=\"from __main__ import gpu\")\n",
"print(gpu_time)\n",
"print('GPU speedup over CPU: {}x'.format(int(cpu_time/gpu_time)))"
],
"execution_count": 0,
"outputs": [
{
"output_type": "stream",
"text": [
"Time (s) to convolve 32x7x7x3 filter over random 100x100x100x3 images (batch x height x width x channel). Sum of ten runs.\n",
"CPU (s):\n",
"3.4301189010000144\n",
"GPU (s):\n",
"0.1132346699999971\n",
"GPU speedup over CPU: 30x\n"
],
"name": "stdout"
}
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "qJDBbtKOjlSj",
"colab_type": "text"
},
"source": [
"### Multi GPU"
]
},
{
"cell_type": "code",
"metadata": {
"id": "kgK4EkMkjmkJ",
"colab_type": "code",
"outputId": "aa4f5e74-2617-46a7-d6fe-d3078a8b7f20",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 161
}
},
"source": [
"mirrored_strategy = tf.distribute.MirroredStrategy()\n",
"with mirrored_strategy.scope():\n",
" model = tf.keras.Sequential([tf.keras.layers.Dense(1, input_shape=(1,))])\n",
" model.compile(loss='mse', optimizer='sgd')\n",
"\n",
"\n",
"dataset = tf.data.Dataset.from_tensors(([1.], [1.])).repeat(100).batch(10)\n",
"model.fit(dataset, epochs=2)\n",
"model.evaluate(dataset)"
],
"execution_count": 0,
"outputs": [
{
"output_type": "stream",
"text": [
"INFO:tensorflow:Using MirroredStrategy with devices ('/job:localhost/replica:0/task:0/device:GPU:0',)\n",
"Train for 10 steps\n",
"Epoch 1/2\n",
"10/10 [==============================] - 2s 244ms/step - loss: 0.0091\n",
"Epoch 2/2\n",
"10/10 [==============================] - 0s 3ms/step - loss: 0.0040\n",
"10/10 [==============================] - 1s 109ms/step - loss: 0.0025\n"
],
"name": "stdout"
},
{
"output_type": "execute_result",
"data": {
"text/plain": [
"0.002487475983798504"
]
},
"metadata": {
"tags": []
},
"execution_count": 10
}
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "pam4WbWyz3Dq",
"colab_type": "text"
},
"source": [
"### Talon Script"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "ms6wM_PMz4xt",
"colab_type": "text"
},
"source": [
"### Single GPU"
]
},
{
"cell_type": "code",
"metadata": {
"id": "trBYYf8oz6LG",
"colab_type": "code",
"outputId": "2d87e942-470c-42e8-ec5f-43dcfdfc0fae",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 253
}
},
"source": [
"import sys\n",
"import os\n",
"os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n",
"import tensorflow as tf\n",
"import timeit\n",
"\n",
"if __name__ == \"__main__\":\n",
" if tf.test.is_gpu_available():\n",
" print(\"\\nGPU detected yeeey!\\n\")\n",
" print(\"tf version running \", tf.version.VERSION)\n",
" print(\"\\nNum GPUs Available: \", len(tf.config.experimental.list_physical_devices('GPU')))\n",
"\n",
" device_name = tf.test.gpu_device_name()\n",
" if device_name != '/device:GPU:0':\n",
" print(\n",
" '\\n\\nThis error most likely means that this script is not '\n",
" 'configured to use a GPU. Change this in Notebook Settings via the '\n",
" 'command palette (cmd/ctrl-shift-P) or the Edit menu.\\n\\n')\n",
" raise SystemError('GPU device not found')\n",
"\n",
" def cpu():\n",
" with tf.device('/cpu:0'):\n",
" random_image_cpu = tf.random.normal((100, 100, 100, 3))\n",
" net_cpu = tf.keras.layers.Conv2D(32, 7)(random_image_cpu)\n",
" return tf.math.reduce_sum(net_cpu)\n",
"\n",
" def gpu():\n",
" with tf.device('/device:GPU:0'):\n",
" random_image_gpu = tf.random.normal((100, 100, 100, 3))\n",
" net_gpu = tf.keras.layers.Conv2D(32, 7)(random_image_gpu)\n",
" return tf.math.reduce_sum(net_gpu)\n",
" \n",
" # We run each op once to warm up; see: https://stackoverflow.com/a/45067900\n",
" cpu()\n",
" gpu()\n",
"\n",
" # Run the op several times.\n",
" print('Time (s) to convolve 32x7x7x3 filter over random 100x100x100x3 images '\n",
" '(batch x height x width x channel). Sum of ten runs.')\n",
" print('CPU (s):')\n",
" cpu_time = timeit.timeit('cpu()', number=10, setup=\"from __main__ import cpu\")\n",
" print(cpu_time)\n",
" print('GPU (s):')\n",
" gpu_time = timeit.timeit('gpu()', number=10, setup=\"from __main__ import gpu\")\n",
" print(gpu_time)\n",
" print('GPU speedup over CPU: {}x'.format(int(cpu_time/gpu_time)))\n",
"\n",
" else:\n",
" print(\"No GPU detected!\")"
],
"execution_count": 0,
"outputs": [
{
"output_type": "stream",
"text": [
"GPU detected yeeey!\n",
"tf version running 1.15.0\n",
"Num GPUs Available: 1\n",
"WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
"Instructions for updating:\n",
"If using Keras pass *_constraint arguments to layers.\n",
"Time (s) to convolve 32x7x7x3 filter over random 100x100x100x3 images (batch x height x width x channel). Sum of ten runs.\n",
"CPU (s):\n",
"0.1697342129999697\n",
"GPU (s):\n",
"0.1691328770000382\n",
"GPU speedup over CPU: 1x\n"
],
"name": "stdout"
}
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "PGT8OBru21tX",
"colab_type": "text"
},
"source": [
"### Multi GPU"
]
},
{
"cell_type": "code",
"metadata": {
"id": "V7f0R8TK23bv",
"colab_type": "code",
"outputId": "9801c190-fe12-458c-fcba-8f7c1690c061",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 539
}
},
"source": [
"import sys\n",
"import os\n",
"os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n",
"import tensorflow as tf\n",
"import timeit\n",
"\n",
"if __name__ == \"__main__\":\n",
" if tf.test.is_gpu_available():\n",
" print(\"\\nGPU detected yeeey!\\n\")\n",
" print(\"tf version running \", tf.version.VERSION)\n",
" print(\"\\nNum GPUs Available: \", len(tf.config.experimental.list_physical_devices('GPU')))\n",
" mirrored_strategy = tf.distribute.MirroredStrategy()\n",
" with mirrored_strategy.scope():\n",
" model = tf.keras.Sequential([tf.keras.layers.Dense(1, input_shape=(1,))])\n",
" model.compile(loss='mse', optimizer='sgd')\n",
"\n",
"\n",
" dataset = tf.data.Dataset.from_tensors(([1.], [1.])).repeat(50000).batch(50)\n",
" model.fit(dataset, epochs=10)\n",
" model.evaluate(dataset)\n",
"\n",
" else:\n",
" print(\"No GPU detected!\")\n"
],
"execution_count": 0,
"outputs": [
{
"output_type": "stream",
"text": [
"\n",
"GPU detected yeeey!\n",
"\n",
"tf version running 1.15.0\n",
"\n",
"Num GPUs Available: 1\n",
"WARNING:tensorflow:Expected a shuffled dataset but input dataset `x` is not shuffled. Please invoke `shuffle()` on input dataset.\n",
"Train on 1000 steps\n",
"Epoch 1/10\n",
"1000/1000 [==============================] - 2s 2ms/step - loss: 0.0245\n",
"Epoch 2/10\n",
"1000/1000 [==============================] - 2s 2ms/step - loss: 5.1159e-13\n",
"Epoch 3/10\n",
"1000/1000 [==============================] - 2s 2ms/step - loss: 5.1159e-13\n",
"Epoch 4/10\n",
"1000/1000 [==============================] - 2s 2ms/step - loss: 5.1159e-13\n",
"Epoch 5/10\n",
"1000/1000 [==============================] - 2s 2ms/step - loss: 5.1159e-13\n",
"Epoch 6/10\n",
"1000/1000 [==============================] - 2s 2ms/step - loss: 5.1159e-13\n",
"Epoch 7/10\n",
"1000/1000 [==============================] - 2s 2ms/step - loss: 5.1159e-13\n",
"Epoch 8/10\n",
"1000/1000 [==============================] - 2s 2ms/step - loss: 5.1159e-13\n",
"Epoch 9/10\n",
"1000/1000 [==============================] - 2s 2ms/step - loss: 5.1159e-13\n",
"Epoch 10/10\n",
"1000/1000 [==============================] - 2s 2ms/step - loss: 5.1159e-13\n",
"1000/1000 [==============================] - 1s 1ms/step - loss: 5.1159e-13\n"
],
"name": "stdout"
}
]
}
]
}