mirror of
https://github.com/fchollet/deep-learning-with-python-notebooks.git
synced 2021-07-27 01:28:40 +03:00
465 lines
10 KiB
Plaintext
465 lines
10 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"This is a companion notebook for the book [Deep Learning with Python, Second Edition](https://www.manning.com/books/deep-learning-with-python-second-edition?a_aid=keras&a_bid=76564dff). For readability, it only contains runnable code blocks and section titles, and omits everything else in the book: text paragraphs, figures, and pseudocode.\n\n**If you want to be able to follow what's going on, I recommend reading the notebook side by side with your copy of the book.**\n\nThis notebook was generated for TensorFlow 2.6."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"# Best practices for the real world"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"## Getting the most our of your models"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"### Hyperparameter optimization"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"#### Using KerasTuner"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"!pip install keras-tuner -q"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"**A KerasTuner model-building function**"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"from tensorflow import keras\n",
|
|
"from tensorflow.keras import layers\n",
|
|
"\n",
|
|
"def build_model(hp):\n",
|
|
" units = hp.Int(name=\"units\", min_value=16, max_value=64, step=16)\n",
|
|
" model = keras.Sequential([\n",
|
|
" layers.Dense(units, activation=\"relu\"),\n",
|
|
" layers.Dense(10, activation=\"softmax\")\n",
|
|
" ])\n",
|
|
" optimizer = hp.Choice(name=\"optimizer\", values=[\"rmsprop\", \"adam\"])\n",
|
|
" model.compile(\n",
|
|
" optimizer=optimizer,\n",
|
|
" loss=\"sparse_categorical_crossentropy\",\n",
|
|
" metrics=[\"accuracy\"])\n",
|
|
" return model"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"**A KerasTuner HyperModel**"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"import kerastuner as kt\n",
|
|
"\n",
|
|
"class SimpleMLP(kt.HyperModel):\n",
|
|
" def __init__(self, num_classes):\n",
|
|
" self.num_classes = num_classes\n",
|
|
"\n",
|
|
" def build(self, hp):\n",
|
|
" units = hp.Int(name=\"units\", min_value=16, max_value=64, step=16)\n",
|
|
" model = keras.Sequential([\n",
|
|
" layers.Dense(units, activation=\"relu\"),\n",
|
|
" layers.Dense(self.num_classes, activation=\"softmax\")\n",
|
|
" ])\n",
|
|
" optimizer = hp.Choice(name=\"optimizer\", values=[\"rmsprop\", \"adam\"])\n",
|
|
" model.compile(\n",
|
|
" optimizer=optimizer,\n",
|
|
" loss=\"sparse_categorical_crossentropy\",\n",
|
|
" metrics=[\"accuracy\"])\n",
|
|
" return model\n",
|
|
"\n",
|
|
"hypermodel = SimpleMLP(num_classes=10)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"tuner = kt.BayesianOptimization(\n",
|
|
" build_model,\n",
|
|
" objective=\"val_accuracy\",\n",
|
|
" max_trials=100,\n",
|
|
" executions_per_trial=2,\n",
|
|
" directory=\"mnist_kt_test\",\n",
|
|
" overwrite=True,\n",
|
|
")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"tuner.search_space_summary()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"(x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()\n",
|
|
"x_train = x_train.reshape((-1, 28 * 28)).astype(\"float32\") / 255\n",
|
|
"x_test = x_test.reshape((-1, 28 * 28)).astype(\"float32\") / 255\n",
|
|
"x_train_full = x_train[:]\n",
|
|
"y_train_full = y_train[:]\n",
|
|
"num_val_samples = 10000\n",
|
|
"x_train, x_val = x_train[:-num_val_samples], x_train[-num_val_samples:]\n",
|
|
"y_train, y_val = y_train[:-num_val_samples], y_train[-num_val_samples:]\n",
|
|
"callbacks = [\n",
|
|
" keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=5),\n",
|
|
"]\n",
|
|
"tuner.search(\n",
|
|
" x_train, y_train,\n",
|
|
" batch_size=128,\n",
|
|
" epochs=100,\n",
|
|
" validation_data=(x_val, y_val),\n",
|
|
" callbacks=callbacks,\n",
|
|
" verbose=2,\n",
|
|
")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"**Querying the best hyperparameter configurations**"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"top_n = 4\n",
|
|
"best_hps = tuner.get_best_hyperparameters(top_n)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"def get_best_epoch(hp):\n",
|
|
" model = build_model(hp)\n",
|
|
" callbacks=[\n",
|
|
" keras.callbacks.EarlyStopping(\n",
|
|
" monitor=\"val_loss\", mode=\"min\", patience=10)\n",
|
|
" ]\n",
|
|
" history = model.fit(\n",
|
|
" x_train, y_train,\n",
|
|
" validation_data=(x_val, y_val),\n",
|
|
" epochs=100,\n",
|
|
" batch_size=128,\n",
|
|
" callbacks=callbacks)\n",
|
|
" val_loss_per_epoch = history.history[\"val_loss\"]\n",
|
|
" best_epoch = val_loss_per_epoch.index(min(val_loss_per_epoch)) + 1\n",
|
|
" print(f\"Best epoch: {best_epoch}\")\n",
|
|
" return best_epoch"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"def get_best_trained_model(hp):\n",
|
|
" best_epoch = get_best_epoch(hp)\n",
|
|
" model.fit(\n",
|
|
" x_train_full, y_train_full,\n",
|
|
" batch_size=128, epochs=int(best_epoch * 1.2))\n",
|
|
" return model\n",
|
|
"\n",
|
|
"best_models = []\n",
|
|
"for hp in best_hps:\n",
|
|
" model = get_best_trained_model(hp)\n",
|
|
" model.evaluate(x_test, y_test)\n",
|
|
" best_models.append(model)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"best_models = tuner.get_best_models(top_n)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"#### The art of crafting the right search space"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"#### The future of hyperparameter tuning: automated machine learning"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"### Model ensembling"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"## Scaling up model training"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"### Speeding up training on GPU with Mixed Precision"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"#### Understanding floating-point precision"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"import tensorflow as tf\n",
|
|
"import numpy as np\n",
|
|
"np_array = np.zeros((2, 2))\n",
|
|
"tf_tensor = tf.convert_to_tensor(np_array)\n",
|
|
"tf_tensor.dtype"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"np_array = np.zeros((2, 2))\n",
|
|
"tf_tensor = tf.convert_to_tensor(np_array, dtype=\"float32\")\n",
|
|
"tf_tensor.dtype"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"#### Mixed-precision training in practice"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"from tensorflow import keras\n",
|
|
"keras.mixed_precision.set_global_policy(\"mixed_float16\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"### Multi-GPU training"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"#### Getting your hands on two or more GPUs"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"#### Single-host, multi-device synchronous training"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"### TPU training"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"#### Using a TPU via Google Colab"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"#### Leveraging step fusing to improve TPU utilization"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"## Chapter summary"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"colab": {
|
|
"collapsed_sections": [],
|
|
"name": "chapter13_best-practices-for-the-real-world.i",
|
|
"private_outputs": false,
|
|
"provenance": [],
|
|
"toc_visible": true
|
|
},
|
|
"kernelspec": {
|
|
"display_name": "Python 3",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.7.0"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 0
|
|
} |