{ "cells": [ { "cell_type": "markdown", "metadata": { "colab_type": "text" }, "source": [ "This is a companion notebook for the book [Deep Learning with Python, Second Edition](https://www.manning.com/books/deep-learning-with-python-second-edition?a_aid=keras&a_bid=76564dff). For readability, it only contains runnable code blocks and section titles, and omits everything else in the book: text paragraphs, figures, and pseudocode.\n\n**If you want to be able to follow what's going on, I recommend reading the notebook side by side with your copy of the book.**\n\nThis notebook was generated for TensorFlow 2.6." ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text" }, "source": [ "# Deep learning for timeseries" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text" }, "source": [ "## Different kinds of timeseries tasks" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text" }, "source": [ "## A temperature forecasting example" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab_type": "code" }, "outputs": [], "source": [ "!wget https://s3.amazonaws.com/keras-datasets/jena_climate_2009_2016.csv.zip\n", "!unzip jena_climate_2009_2016.csv.zip" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text" }, "source": [ "**Inspecting the data of the Jena weather dataset**" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab_type": "code" }, "outputs": [], "source": [ "import os\n", "fname = os.path.join(\"jena_climate_2009_2016.csv\")\n", "\n", "with open(fname) as f:\n", " data = f.read()\n", "\n", "lines = data.split(\"\\n\")\n", "header = lines[0].split(\",\")\n", "lines = lines[1:]\n", "print(header)\n", "print(len(lines))" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text" }, "source": [ "**Parsing the data**" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab_type": "code" }, "outputs": [], "source": [ "import numpy as np\n", "temperature = np.zeros((len(lines),))\n", "raw_data = np.zeros((len(lines), len(header) - 1))\n", "for i, line in enumerate(lines):\n", " values = [float(x) for x in line.split(\",\")[1:]]\n", " temperature[i] = values[1]\n", " raw_data[i, :] = values[:]" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text" }, "source": [ "**Plotting the temperature timeseries**" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab_type": "code" }, "outputs": [], "source": [ "from matplotlib import pyplot as plt\n", "plt.plot(range(len(temperature)), temperature)" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text" }, "source": [ "**Plotting the first 10 days of the temperature timeseries**" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab_type": "code" }, "outputs": [], "source": [ "plt.plot(range(1440), temperature[:1440])" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text" }, "source": [ "**Computing the number of samples we'll use for each data split.**" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab_type": "code" }, "outputs": [], "source": [ "num_train_samples = int(0.5 * len(raw_data))\n", "num_val_samples = int(0.25 * len(raw_data))\n", "num_test_samples = len(raw_data) - num_train_samples - num_val_samples\n", "print(\"num_train_samples:\", num_train_samples)\n", "print(\"num_val_samples:\", num_val_samples)\n", "print(\"num_test_samples:\", num_test_samples)" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text" }, "source": [ "### Preparing the data" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text" }, "source": [ "**Normalizing the data**" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab_type": "code" }, "outputs": [], "source": [ "mean = raw_data[:num_train_samples].mean(axis=0)\n", "raw_data -= mean\n", "std = raw_data[:num_train_samples].std(axis=0)\n", "raw_data /= std" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab_type": "code" }, "outputs": [], "source": [ "import numpy as np\n", "from tensorflow import keras\n", "int_sequence = np.arange(10)\n", "dummy_dataset = keras.preprocessing.timeseries_dataset_from_array(\n", " data=int_sequence[:-3],\n", " targets=int_sequence[3:],\n", " sequence_length=3,\n", " batch_size=2,\n", ")\n", "\n", "for inputs, targets in dummy_dataset:\n", " for i in range(inputs.shape[0]):\n", " print([int(x) for x in inputs[i]], int(targets[i]))" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text" }, "source": [ "**Instantiating Datasets for training, validation, and testing.**" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab_type": "code" }, "outputs": [], "source": [ "sampling_rate = 6\n", "sequence_length = 120\n", "delay = sampling_rate * (sequence_length + 24 - 1)\n", "batch_size = 256\n", "\n", "train_dataset = keras.preprocessing.timeseries_dataset_from_array(\n", " raw_data[:-delay],\n", " targets=temperature[delay:],\n", " sampling_rate=sampling_rate,\n", " sequence_length=sequence_length,\n", " shuffle=True,\n", " batch_size=batch_size,\n", " start_index=0,\n", " end_index=num_train_samples)\n", "\n", "val_dataset = keras.preprocessing.timeseries_dataset_from_array(\n", " raw_data[:-delay],\n", " targets=temperature[delay:],\n", " sampling_rate=sampling_rate,\n", " sequence_length=sequence_length,\n", " shuffle=True,\n", " batch_size=batch_size,\n", " start_index=num_train_samples,\n", " end_index=num_train_samples + num_val_samples)\n", "\n", "test_dataset = keras.preprocessing.timeseries_dataset_from_array(\n", " raw_data[:-delay],\n", " targets=temperature[delay:],\n", " sampling_rate=sampling_rate,\n", " sequence_length=sequence_length,\n", " shuffle=True,\n", " batch_size=batch_size,\n", " start_index=num_train_samples + num_val_samples)" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text" }, "source": [ "**Inspecting the output of one of our Datasets.**" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab_type": "code" }, "outputs": [], "source": [ "for samples, targets in train_dataset:\n", " print(\"samples shape:\", samples.shape)\n", " print(\"targets shape:\", targets.shape)\n", " break" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text" }, "source": [ "### A common-sense, non-machine-learning baseline" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text" }, "source": [ "**Computing the common-sense baseline MAE**" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab_type": "code" }, "outputs": [], "source": [ "def evaluate_naive_method(dataset):\n", " total_abs_err = 0.\n", " samples_seen = 0\n", " for samples, targets in dataset:\n", " preds = samples[:, -1, 1] * std[1] + mean[1]\n", " total_abs_err += np.sum(np.abs(preds - targets))\n", " samples_seen += samples.shape[0]\n", " return total_abs_err / samples_seen\n", "\n", "print(f\"Validation MAE: {evaluate_naive_method(val_dataset):.2f}\")\n", "print(f\"Test MAE: {evaluate_naive_method(test_dataset):.2f}\")" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text" }, "source": [ "### Let's try a basic machine learning model" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text" }, "source": [ "**Training and evaluating a densely connected model**" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab_type": "code" }, "outputs": [], "source": [ "from tensorflow import keras\n", "from tensorflow.keras import layers\n", "\n", "inputs = keras.Input(shape=(sequence_length, raw_data.shape[-1]))\n", "x = layers.Flatten()(inputs)\n", "x = layers.Dense(16, activation=\"relu\")(x)\n", "outputs = layers.Dense(1)(x)\n", "model = keras.Model(inputs, outputs)\n", "\n", "callbacks = [\n", " keras.callbacks.ModelCheckpoint(\"jena_dense.keras\",\n", " save_best_only=True)\n", "]\n", "model.compile(optimizer=\"rmsprop\", loss=\"mse\", metrics=[\"mae\"])\n", "history = model.fit(train_dataset,\n", " epochs=10,\n", " validation_data=val_dataset,\n", " callbacks=callbacks)\n", "\n", "model = keras.models.load_model(\"jena_dense.keras\")\n", "print(f\"Test MAE: {model.evaluate(test_dataset)[1]:.2f}\")" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text" }, "source": [ "**Plotting results**" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab_type": "code" }, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", "loss = history.history[\"mae\"]\n", "val_loss = history.history[\"val_mae\"]\n", "epochs = range(1, len(loss) + 1)\n", "plt.figure()\n", "plt.plot(epochs, loss, \"bo\", label=\"Training MAE\")\n", "plt.plot(epochs, val_loss, \"b\", label=\"Validation MAE\")\n", "plt.title(\"Training and validation MAE\")\n", "plt.legend()\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text" }, "source": [ "### Let's try a 1D convolutional model" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab_type": "code" }, "outputs": [], "source": [ "inputs = keras.Input(shape=(sequence_length, raw_data.shape[-1]))\n", "x = layers.Conv1D(8, 24, activation=\"relu\")(inputs)\n", "x = layers.MaxPooling1D(2)(x)\n", "x = layers.Conv1D(8, 12, activation=\"relu\")(x)\n", "x = layers.MaxPooling1D(2)(x)\n", "x = layers.Conv1D(8, 6, activation=\"relu\")(x)\n", "x = layers.GlobalAveragePooling1D()(x)\n", "outputs = layers.Dense(1)(x)\n", "model = keras.Model(inputs, outputs)\n", "\n", "callbacks = [\n", " keras.callbacks.ModelCheckpoint(\"jena_conv.keras\",\n", " save_best_only=True)\n", "]\n", "model.compile(optimizer=\"rmsprop\", loss=\"mse\", metrics=[\"mae\"])\n", "history = model.fit(train_dataset,\n", " epochs=10,\n", " validation_data=val_dataset,\n", " callbacks=callbacks)\n", "\n", "model = keras.models.load_model(\"jena_conv.keras\")\n", "print(f\"Test MAE: {model.evaluate(test_dataset)[1]:.2f}\")" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text" }, "source": [ "### A first recurrent baseline" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text" }, "source": [ "**A simple LSTM-based model**" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab_type": "code" }, "outputs": [], "source": [ "inputs = keras.Input(shape=(sequence_length, raw_data.shape[-1]))\n", "x = layers.LSTM(16)(inputs)\n", "outputs = layers.Dense(1)(x)\n", "model = keras.Model(inputs, outputs)\n", "\n", "callbacks = [\n", " keras.callbacks.ModelCheckpoint(\"jena_lstm.keras\",\n", " save_best_only=True)\n", "]\n", "model.compile(optimizer=\"rmsprop\", loss=\"mse\", metrics=[\"mae\"])\n", "history = model.fit(train_dataset,\n", " epochs=10,\n", " validation_data=val_dataset,\n", " callbacks=callbacks)\n", "\n", "model = keras.models.load_model(\"jena_lstm.keras\")\n", "print(\"Test MAE: {model.evaluate(test_dataset)[1]:.2f}\")" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text" }, "source": [ "## Understanding recurrent neural networks" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text" }, "source": [ "**NumPy implementation of a simple RNN**" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab_type": "code" }, "outputs": [], "source": [ "import numpy as np\n", "timesteps = 100\n", "input_features = 32\n", "output_features = 64\n", "inputs = np.random.random((timesteps, input_features))\n", "state_t = np.zeros((output_features,))\n", "W = np.random.random((output_features, input_features))\n", "U = np.random.random((output_features, output_features))\n", "b = np.random.random((output_features,))\n", "successive_outputs = []\n", "for input_t in inputs:\n", " output_t = np.tanh(np.dot(W, input_t) + np.dot(U, state_t) + b)\n", " successive_outputs.append(output_t)\n", " state_t = output_t\n", "final_output_sequence = np.concatenate(successive_outputs, axis=0)" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text" }, "source": [ "### A recurrent layer in Keras" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text" }, "source": [ "**A RNN layer that can process sequences of any length**" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab_type": "code" }, "outputs": [], "source": [ "num_features = 14\n", "inputs = keras.Input(shape=(None, num_features))\n", "outputs = layers.SimpleRNN(16)(inputs)" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text" }, "source": [ "**A RNN layer that returns only its last output step**" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab_type": "code" }, "outputs": [], "source": [ "num_features = 14\n", "steps = 120\n", "inputs = keras.Input(shape=(steps, num_features))\n", "outputs = layers.SimpleRNN(16, return_sequences=False)(inputs)\n", "print(outputs.shape)" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text" }, "source": [ "**A RNN layer that returns its full output sequence**" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab_type": "code" }, "outputs": [], "source": [ "num_features = 14\n", "steps = 120\n", "inputs = keras.Input(shape=(steps, num_features))\n", "outputs = layers.SimpleRNN(16, return_sequences=True)(inputs)\n", "print(outputs.shape)" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text" }, "source": [ "**Stacking RNN layers**" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab_type": "code" }, "outputs": [], "source": [ "inputs = keras.Input(shape=(steps, num_features))\n", "x = layers.SimpleRNN(16, return_sequences=True)(inputs)\n", "x = layers.SimpleRNN(16, return_sequences=True)(x)\n", "outputs = layers.SimpleRNN(16)(x)" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text" }, "source": [ "## Advanced use of recurrent neural networks" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text" }, "source": [ "### Using recurrent dropout to fight overfitting" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text" }, "source": [ "**Training and evaluating a dropout-regularized LSTM**" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab_type": "code" }, "outputs": [], "source": [ "inputs = keras.Input(shape=(sequence_length, raw_data.shape[-1]))\n", "x = layers.LSTM(32, recurrent_dropout=0.25)(inputs)\n", "x = layers.Dropout(0.5)(x)\n", "outputs = layers.Dense(1)(x)\n", "model = keras.Model(inputs, outputs)\n", "\n", "callbacks = [\n", " keras.callbacks.ModelCheckpoint(\"jena_lstm_dropout.keras\",\n", " save_best_only=True)\n", "]\n", "model.compile(optimizer=\"rmsprop\", loss=\"mse\", metrics=[\"mae\"])\n", "history = model.fit(train_dataset,\n", " epochs=50,\n", " validation_data=val_dataset,\n", " callbacks=callbacks)" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab_type": "code" }, "outputs": [], "source": [ "inputs = keras.Input(shape=(sequence_length, num_features))\n", "x = layers.LSTM(32, recurrent_dropout=0.2, unroll=True)(inputs)" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text" }, "source": [ "### Stacking recurrent layers" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text" }, "source": [ "**Training and evaluating a dropout-regularized, stacked GRU model**" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab_type": "code" }, "outputs": [], "source": [ "inputs = keras.Input(shape=(sequence_length, raw_data.shape[-1]))\n", "x = layers.GRU(32, recurrent_dropout=0.5, return_sequences=True)(inputs)\n", "x = layers.GRU(32, recurrent_dropout=0.5)(x)\n", "x = layers.Dropout(0.5)(x)\n", "outputs = layers.Dense(1)(x)\n", "model = keras.Model(inputs, outputs)\n", "\n", "callbacks = [\n", " keras.callbacks.ModelCheckpoint(\"jena_stacked_gru_dropout.keras\",\n", " save_best_only=True)\n", "]\n", "model.compile(optimizer=\"rmsprop\", loss=\"mse\", metrics=[\"mae\"])\n", "history = model.fit(train_dataset,\n", " epochs=50,\n", " validation_data=val_dataset,\n", " callbacks=callbacks)\n", "model = keras.models.load_model(\"jena_stacked_gru_dropout.keras\")\n", "print(f\"Test MAE: {model.evaluate(test_dataset)[1]:.2f}\")" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text" }, "source": [ "### Using bidirectional RNNs" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text" }, "source": [ "**Training and evaluating a bidirectional LSTM**" ] }, { "cell_type": "code", "execution_count": 0, "metadata": { "colab_type": "code" }, "outputs": [], "source": [ "inputs = keras.Input(shape=(sequence_length, raw_data.shape[-1]))\n", "x = layers.Bidirectional(layers.LSTM(16))(inputs)\n", "outputs = layers.Dense(1)(x)\n", "model = keras.Model(inputs, outputs)\n", "\n", "model.compile(optimizer=\"rmsprop\", loss=\"mse\", metrics=[\"mae\"])\n", "history = model.fit(train_dataset,\n", " epochs=10,\n", " validation_data=val_dataset)" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text" }, "source": [ "### *_Going even further_*" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text" }, "source": [ "## Chapter summary" ] } ], "metadata": { "colab": { "collapsed_sections": [], "name": "chapter10_dl-for-timeseries.i", "private_outputs": false, "provenance": [], "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.0" } }, "nbformat": 4, "nbformat_minor": 0 }