mirror of
https://github.com/fchollet/deep-learning-with-python-notebooks.git
synced 2021-07-27 01:28:40 +03:00
1419 lines
35 KiB
Plaintext
1419 lines
35 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"This is a companion notebook for the book [Deep Learning with Python, Second Edition](https://www.manning.com/books/deep-learning-with-python-second-edition?a_aid=keras&a_bid=76564dff). For readability, it only contains runnable code blocks and section titles, and omits everything else in the book: text paragraphs, figures, and pseudocode.\n\n**If you want to be able to follow what's going on, I recommend reading the notebook side by side with your copy of the book.**\n\nThis notebook was generated for TensorFlow 2.6."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"# Working with Keras: a deep dive"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"## A spectrum of workflows"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"## Different ways to build Keras models"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"### The Sequential model"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"**The Sequential class**"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"from tensorflow import keras\n",
|
|
"from tensorflow.keras import layers\n",
|
|
"\n",
|
|
"model = keras.Sequential([\n",
|
|
" layers.Dense(64, activation=\"relu\"),\n",
|
|
" layers.Dense(10, activation=\"softmax\")\n",
|
|
"])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"**Incrementally building a Sequential model**"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"model = keras.Sequential()\n",
|
|
"model.add(layers.Dense(64, activation=\"relu\"))\n",
|
|
"model.add(layers.Dense(10, activation=\"softmax\"))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"**Calling a model for the first time to build it**"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"model.build(input_shape=(None, 3))\n",
|
|
"model.weights"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"**The summary method**"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"model.summary()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"**Naming models and layers with the `name` argument**"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"model = keras.Sequential(name=\"my_example_model\")\n",
|
|
"model.add(layers.Dense(64, activation=\"relu\", name=\"my_first_layer\"))\n",
|
|
"model.add(layers.Dense(10, activation=\"softmax\", name=\"my_last_layer\"))\n",
|
|
"model.build((None, 3))\n",
|
|
"model.summary()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"**Specifying the input shape of your model in advance**"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"model = keras.Sequential()\n",
|
|
"model.add(keras.Input(shape=(3,)))\n",
|
|
"model.add(layers.Dense(64, activation=\"relu\"))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"model.summary()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"model.add(layers.Dense(10, activation=\"softmax\"))\n",
|
|
"model.summary()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"### The Functional API"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"#### A simple example"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"**A simple Functional model with two Dense layers**"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"inputs = keras.Input(shape=(3,), name=\"my_input\")\n",
|
|
"features = layers.Dense(64, activation=\"relu\")(inputs)\n",
|
|
"outputs = layers.Dense(10, activation=\"softmax\")(features)\n",
|
|
"model = keras.Model(inputs=inputs, outputs=outputs)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"inputs = keras.Input(shape=(3,), name=\"my_input\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"inputs.shape"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"inputs.dtype"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"features = layers.Dense(64, activation=\"relu\")(inputs)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"features.shape"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"outputs = layers.Dense(10, activation=\"softmax\")(features)\n",
|
|
"model = keras.Model(inputs=inputs, outputs=outputs)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"model.summary()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"#### Multi-input, multi-output models"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"**A multi-input, multi-output Functional model**"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"vocabulary_size = 10000\n",
|
|
"num_tags = 100\n",
|
|
"num_departments = 4\n",
|
|
"\n",
|
|
"title = keras.Input(shape=(vocabulary_size,), name=\"title\")\n",
|
|
"text_body = keras.Input(shape=(vocabulary_size,), name=\"text_body\")\n",
|
|
"tags = keras.Input(shape=(num_tags,), name=\"tags\")\n",
|
|
"\n",
|
|
"features = layers.Concatenate()([title, text_body, tags])\n",
|
|
"features = layers.Dense(64, activation=\"relu\")(features)\n",
|
|
"\n",
|
|
"priority = layers.Dense(1, activation=\"sigmoid\", name=\"priority\")(features)\n",
|
|
"department = layers.Dense(\n",
|
|
" num_departments, activation=\"softmax\", name=\"department\")(features)\n",
|
|
"\n",
|
|
"model = keras.Model(inputs=[title, text_body, tags], outputs=[priority, department])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"#### Training a multi-input, multi-output model"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"**Training a model by providing lists of input & target arrays**"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"import numpy as np\n",
|
|
"\n",
|
|
"num_samples = 1280\n",
|
|
"\n",
|
|
"title_data = np.random.randint(0, 2, size=(num_samples, vocabulary_size))\n",
|
|
"text_body_data = np.random.randint(0, 2, size=(num_samples, vocabulary_size))\n",
|
|
"tags_data = np.random.randint(0, 2, size=(num_samples, num_tags))\n",
|
|
"\n",
|
|
"priority_data = np.random.random(size=(num_samples, 1))\n",
|
|
"department_data = np.random.randint(0, 2, size=(num_samples, num_departments))\n",
|
|
"\n",
|
|
"model.compile(optimizer=\"adam\",\n",
|
|
" loss=[\"mean_squared_error\", \"categorical_crossentropy\"],\n",
|
|
" metrics=[[\"mean_absolute_error\"], [\"accuracy\"]])\n",
|
|
"model.fit([title_data, text_body_data, tags_data],\n",
|
|
" [priority_data, department_data],\n",
|
|
" epochs=1)\n",
|
|
"model.evaluate([title_data, text_body_data, tags_data],\n",
|
|
" [priority_data, department_data])\n",
|
|
"priority_preds, department_preds = model.predict([title_data, text_body_data, tags_data])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"**Training a model by providing dicts of input & target arrays**"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"model.compile(optimizer=\"adam\",\n",
|
|
" loss={\"priority\": \"mean_squared_error\", \"department\": \"categorical_crossentropy\"},\n",
|
|
" metrics={\"priority\": [\"mean_absolute_error\"], \"department\": [\"accuracy\"]})\n",
|
|
"model.fit({\"title\": title_data, \"text_body\": text_body_data, \"tags\": tags_data},\n",
|
|
" {\"priority\": priority_data, \"department\": department_data},\n",
|
|
" epochs=1)\n",
|
|
"model.evaluate({\"title\": title_data, \"text_body\": text_body_data, \"tags\": tags_data},\n",
|
|
" {\"priority\": priority_data, \"department\": department_data})\n",
|
|
"priority_preds, department_preds = model.predict(\n",
|
|
" {\"title\": title_data, \"text_body\": text_body_data, \"tags\": tags_data})"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"#### The power of the Functional API: access to layer connectivity"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"##### Plotting layer connectivity"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"keras.utils.plot_model(model, \"ticket_classifier.png\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"keras.utils.plot_model(model, \"ticket_classifier_with_shape_info.png\", show_shapes=True)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"##### Feature extraction with a Functional model"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"**Retrieving the inputs or outputs of a layer in a Functional model**"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"model.layers"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"model.layers[3].input"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"model.layers[3].output"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"**Creating a new model by reusing intermediate layer outputs**"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"features = model.layers[4].output\n",
|
|
"difficulty = layers.Dense(3, activation=\"softmax\", name=\"difficulty\")(features)\n",
|
|
"\n",
|
|
"new_model = keras.Model(\n",
|
|
" inputs=[title, text_body, tags],\n",
|
|
" outputs=[priority, department, difficulty])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"keras.utils.plot_model(model, \"updated_ticket_classifier.png\", show_shapes=True)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"### Subclassing the `Model` class"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"#### Rewriting our previous example as a subclassed model"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"**A simple subclassed model**"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"class CustomerTicketModel(keras.Model):\n",
|
|
"\n",
|
|
" def __init__(self, num_departments):\n",
|
|
" super().__init__()\n",
|
|
" self.concat_layer = layers.Concatenate()\n",
|
|
" self.mixing_layer = layers.Dense(64, activation=\"relu\")\n",
|
|
" self.priority_scorer = layers.Dense(1, activation=\"sigmoid\")\n",
|
|
" self.department_classifier = layers.Dense(\n",
|
|
" num_departments, activation=\"softmax\")\n",
|
|
"\n",
|
|
" def call(self, inputs):\n",
|
|
" title = inputs[\"title\"]\n",
|
|
" text_body = inputs[\"text_body\"]\n",
|
|
" tags = inputs[\"tags\"]\n",
|
|
"\n",
|
|
" features = self.concat_layer([title, text_body, tags])\n",
|
|
" features = self.mixing_layer(features)\n",
|
|
" priority = self.priority_scorer(features)\n",
|
|
" department = self.department_classifier(features)\n",
|
|
" return priority, department"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"model = CustomerTicketModel(num_departments=4)\n",
|
|
"\n",
|
|
"priority, department = model(\n",
|
|
" {\"title\": title_data, \"text_body\": text_body_data, \"tags\": tags_data})"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"model.compile(optimizer=\"adam\",\n",
|
|
" loss=[\"mean_squared_error\", \"categorical_crossentropy\"],\n",
|
|
" metrics=[[\"mean_absolute_error\"], [\"accuracy\"]])\n",
|
|
"model.fit({\"title\": title_data, \"text_body\": text_body_data, \"tags\": tags_data},\n",
|
|
" [priority_data, department_data],\n",
|
|
" epochs=1)\n",
|
|
"model.evaluate({\"title\": title_data, \"text_body\": text_body_data, \"tags\": tags_data},\n",
|
|
" [priority_data, department_data])\n",
|
|
"priority_preds, department_preds = model.predict(\n",
|
|
" {\"title\": title_data, \"text_body\": text_body_data, \"tags\": tags_data})"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"#### Beware: what subclassed models don't support"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"### Mixing and matching different components"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"**Creating a Functional model that includes a subclassed model**"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"class Classifier(keras.Model):\n",
|
|
"\n",
|
|
" def __init__(self, num_classes=2):\n",
|
|
" super().__init__()\n",
|
|
" if num_classes == 2:\n",
|
|
" num_units = 1\n",
|
|
" activation = \"sigmoid\"\n",
|
|
" else:\n",
|
|
" num_units = num_classes\n",
|
|
" activation = \"softmax\"\n",
|
|
" self.dense = layers.Dense(num_units, activation=activation)\n",
|
|
"\n",
|
|
" def call(self, inputs):\n",
|
|
" return self.dense(inputs)\n",
|
|
"\n",
|
|
"inputs = keras.Input(shape=(3,))\n",
|
|
"features = layers.Dense(64, activation=\"relu\")(inputs)\n",
|
|
"outputs = Classifier(num_classes=10)(features)\n",
|
|
"model = keras.Model(inputs=inputs, outputs=outputs)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"**Creating a subclassed model that includes a Functional model**"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"inputs = keras.Input(shape=(64,))\n",
|
|
"outputs = layers.Dense(1, activation=\"sigmoid\")(inputs)\n",
|
|
"binary_classifier = keras.Model(inputs=inputs, outputs=outputs)\n",
|
|
"\n",
|
|
"class MyModel(keras.Model):\n",
|
|
"\n",
|
|
" def __init__(self, num_classes=2):\n",
|
|
" super().__init__()\n",
|
|
" self.dense = layers.Dense(64, activation=\"relu\")\n",
|
|
" self.classifier = binary_classifier\n",
|
|
"\n",
|
|
" def call(self, inputs):\n",
|
|
" features = self.dense(inputs)\n",
|
|
" return self.classifier(features)\n",
|
|
"\n",
|
|
"model = MyModel()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"### Remember: use the right tool for the job"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"## Using built-in training and evaluation loops"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"**The standard workflow: `compile()` / `fit()` / `evaluate()` / `predict()`**"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"from tensorflow.keras.datasets import mnist\n",
|
|
"\n",
|
|
"def get_mnist_model():\n",
|
|
" inputs = keras.Input(shape=(28 * 28,))\n",
|
|
" features = layers.Dense(512, activation=\"relu\")(inputs)\n",
|
|
" features = layers.Dropout(0.5)(features)\n",
|
|
" outputs = layers.Dense(10, activation=\"softmax\")(features)\n",
|
|
" model = keras.Model(inputs, outputs)\n",
|
|
" return model\n",
|
|
"\n",
|
|
"(images, labels), (test_images, test_labels) = mnist.load_data()\n",
|
|
"images = images.reshape((60000, 28 * 28)).astype(\"float32\") / 255\n",
|
|
"test_images = test_images.reshape((10000, 28 * 28)).astype(\"float32\") / 255\n",
|
|
"train_images, val_images = images[10000:], images[:10000]\n",
|
|
"train_labels, val_labels = labels[10000:], labels[:10000]\n",
|
|
"\n",
|
|
"model = get_mnist_model()\n",
|
|
"model.compile(optimizer=\"rmsprop\",\n",
|
|
" loss=\"sparse_categorical_crossentropy\",\n",
|
|
" metrics=[\"accuracy\"])\n",
|
|
"model.fit(train_images, train_labels,\n",
|
|
" epochs=3,\n",
|
|
" validation_data=(val_images, val_labels))\n",
|
|
"test_metrics = model.evaluate(test_images, test_labels)\n",
|
|
"predictions = model.predict(test_images)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"### Writing your own metrics"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"**Implementing a custom metric by subclassing the `Metric` class**"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"import tensorflow as tf\n",
|
|
"\n",
|
|
"class RootMeanSquaredError(keras.metrics.Metric):\n",
|
|
"\n",
|
|
" def __init__(self, name=\"rmse\", **kwargs):\n",
|
|
" super().__init__(name=name, **kwargs)\n",
|
|
" self.mse_sum = self.add_weight(\n",
|
|
" name=\"mse_sum\", initializer=\"zeros\")\n",
|
|
" self.total_samples = self.add_weight(\n",
|
|
" name=\"total_samples\", initializer=\"zeros\", dtype=\"int32\")\n",
|
|
"\n",
|
|
" def update_state(self, y_true, y_pred, sample_weight=None):\n",
|
|
" y_true = tf.one_hot(y_true, depth=tf.shape(y_pred)[1])\n",
|
|
" mse = tf.reduce_sum(tf.square(y_true - y_pred))\n",
|
|
" self.mse_sum.assign_add(mse)\n",
|
|
" num_samples = tf.shape(y_pred)[0]\n",
|
|
" self.total_samples.assign_add(num_samples)\n",
|
|
"\n",
|
|
" def result(self):\n",
|
|
" return tf.sqrt(self.mse_sum / tf.cast(self.total_samples, tf.float32))\n",
|
|
"\n",
|
|
" def reset_states(self):\n",
|
|
" self.mse_sum.assign(0.)\n",
|
|
" self.total_samples.assign(0)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"model = get_mnist_model()\n",
|
|
"model.compile(optimizer=\"rmsprop\",\n",
|
|
" loss=\"sparse_categorical_crossentropy\",\n",
|
|
" metrics=[\"accuracy\", RootMeanSquaredError()])\n",
|
|
"model.fit(train_images, train_labels,\n",
|
|
" epochs=3,\n",
|
|
" validation_data=(val_images, val_labels))\n",
|
|
"test_metrics = model.evaluate(test_images, test_labels)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"### Using Callbacks"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"#### The `EarlyStopping` and `ModelCheckpoint` callbacks"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"**Using the `callbacks` argument in the `fit()` method**"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"callbacks_list = [\n",
|
|
" keras.callbacks.EarlyStopping(\n",
|
|
" monitor=\"accuracy\",\n",
|
|
" patience=1,\n",
|
|
" ),\n",
|
|
" keras.callbacks.ModelCheckpoint(\n",
|
|
" filepath=\"my_checkpoint_path\",\n",
|
|
" monitor=\"val_loss\",\n",
|
|
" save_best_only=True,\n",
|
|
" )\n",
|
|
"]\n",
|
|
"model = get_mnist_model()\n",
|
|
"model.compile(optimizer=\"rmsprop\",\n",
|
|
" loss=\"sparse_categorical_crossentropy\",\n",
|
|
" metrics=[\"accuracy\"])\n",
|
|
"model.fit(train_images, train_labels,\n",
|
|
" epochs=10,\n",
|
|
" callbacks=callbacks_list,\n",
|
|
" validation_data=(val_images, val_labels))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"model = keras.models.load_model(\"my_checkpoint_path\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"### Writing your own callbacks"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"**Creating a custom callback by subclassing the `Callback` class**"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"from matplotlib import pyplot as plt\n",
|
|
"\n",
|
|
"class LossHistory(keras.callbacks.Callback):\n",
|
|
" def on_train_begin(self, logs):\n",
|
|
" self.per_batch_losses = []\n",
|
|
"\n",
|
|
" def on_batch_end(self, batch, logs):\n",
|
|
" self.per_batch_losses.append(logs.get(\"loss\"))\n",
|
|
"\n",
|
|
" def on_epoch_end(self, epoch, logs):\n",
|
|
" plt.clf()\n",
|
|
" plt.plot(range(len(self.per_batch_losses)), self.per_batch_losses,\n",
|
|
" label=\"Training loss for each batch\")\n",
|
|
" plt.xlabel(f\"Batch (epoch {epoch})\")\n",
|
|
" plt.ylabel(\"Loss\")\n",
|
|
" plt.legend()\n",
|
|
" plt.savefig(f\"plot_at_epoch_{epoch}\")\n",
|
|
" self.per_batch_losses = []"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"model = get_mnist_model()\n",
|
|
"model.compile(optimizer=\"rmsprop\",\n",
|
|
" loss=\"sparse_categorical_crossentropy\",\n",
|
|
" metrics=[\"accuracy\"])\n",
|
|
"model.fit(train_images, train_labels,\n",
|
|
" epochs=10,\n",
|
|
" callbacks=[LossHistory()],\n",
|
|
" validation_data=(val_images, val_labels))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"### Monitoring and visualization with TensorBoard"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"## Writing your own training and evaluation loops"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"### Training versus inference"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"### Low-level usage of metrics"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"metric = keras.metrics.SparseCategoricalAccuracy()\n",
|
|
"targets = [0, 1, 2]\n",
|
|
"predictions = [[1, 0, 0], [0, 1, 0], [0, 0, 1]]\n",
|
|
"metric.update_state(targets, predictions)\n",
|
|
"current_result = metric.result()\n",
|
|
"print(f\"result: {current_result:.2f}\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"values = [0, 1, 2, 3, 4]\n",
|
|
"mean_tracker = keras.metrics.Mean()\n",
|
|
"for value in values:\n",
|
|
" mean_tracker.update_state(value)\n",
|
|
"print(f\"Mean of values: {mean_tracker.result():.2f}\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"### A complete training and evaluation loop"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"**Writing a step-by-step training loop: the training step function**"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"model = get_mnist_model()\n",
|
|
"\n",
|
|
"loss_fn = keras.losses.SparseCategoricalCrossentropy()\n",
|
|
"optimizer = keras.optimizers.RMSprop()\n",
|
|
"metrics = [keras.metrics.SparseCategoricalAccuracy()]\n",
|
|
"loss_tracking_metric = keras.metrics.Mean()\n",
|
|
"\n",
|
|
"def train_step(inputs, targets):\n",
|
|
" with tf.GradientTape() as tape:\n",
|
|
" predictions = model(inputs, training=True)\n",
|
|
" loss = loss_fn(targets, predictions)\n",
|
|
" gradients = tape.gradient(loss, model.trainable_weights)\n",
|
|
" optimizer.apply_gradients(zip(gradients, model.trainable_weights))\n",
|
|
"\n",
|
|
" logs = {}\n",
|
|
" for metric in metrics:\n",
|
|
" metric.update_state(targets, predictions)\n",
|
|
" logs[metric.name] = metric.result()\n",
|
|
"\n",
|
|
" loss_tracking_metric.update_state(loss)\n",
|
|
" logs[\"loss\"] = loss_tracking_metric.result()\n",
|
|
" return logs"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"**Writing a step-by-step training loop: resetting the metrics**"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"def reset_metrics():\n",
|
|
" for metric in metrics:\n",
|
|
" metric.reset_states()\n",
|
|
" loss_tracking_metric.reset_states()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"**Writing a step-by-step training loop: the loop itself**"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"training_dataset = tf.data.Dataset.from_tensor_slices((train_images, train_labels))\n",
|
|
"training_dataset = training_dataset.batch(32)\n",
|
|
"epochs = 3\n",
|
|
"for epoch in range(epochs):\n",
|
|
" reset_metrics()\n",
|
|
" for inputs_batch, targets_batch in training_dataset:\n",
|
|
" logs = train_step(inputs_batch, targets_batch)\n",
|
|
" print(f\"Results at the end of epoch {epoch}\")\n",
|
|
" for key, value in logs.items():\n",
|
|
" print(f\"...{key}: {value:.4f}\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"**Writing a step-by-step evaluation loop**"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"def test_step(inputs, targets):\n",
|
|
" predictions = model(inputs, training=False)\n",
|
|
" loss = loss_fn(targets, predictions)\n",
|
|
"\n",
|
|
" logs = {}\n",
|
|
" for metric in metrics:\n",
|
|
" metric.update_state(targets, predictions)\n",
|
|
" logs[\"val_\" + metric.name] = metric.result()\n",
|
|
"\n",
|
|
" loss_tracking_metric.update_state(loss)\n",
|
|
" logs[\"val_loss\"] = loss_tracking_metric.result()\n",
|
|
" return logs\n",
|
|
"\n",
|
|
"val_dataset = tf.data.Dataset.from_tensor_slices((val_images, val_labels))\n",
|
|
"val_dataset = val_dataset.batch(32)\n",
|
|
"reset_metrics()\n",
|
|
"for inputs_batch, targets_batch in val_dataset:\n",
|
|
" logs = test_step(inputs_batch, targets_batch)\n",
|
|
"print(\"Evaluation results:\")\n",
|
|
"for key, value in logs.items():\n",
|
|
" print(f\"...{key}: {value:.4f}\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"### Make it fast with `tf.function`"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"**Adding a `tf.function` decorator to our evaluation step function**"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"@tf.function\n",
|
|
"def test_step(inputs, targets):\n",
|
|
" predictions = model(inputs, training=False)\n",
|
|
" loss = loss_fn(targets, predictions)\n",
|
|
"\n",
|
|
" logs = {}\n",
|
|
" for metric in metrics:\n",
|
|
" metric.update_state(targets, predictions)\n",
|
|
" logs[\"val_\" + metric.name] = metric.result()\n",
|
|
"\n",
|
|
" loss_tracking_metric.update_state(loss)\n",
|
|
" logs[\"val_loss\"] = loss_tracking_metric.result()\n",
|
|
" return logs\n",
|
|
"\n",
|
|
"val_dataset = tf.data.Dataset.from_tensor_slices((val_images, val_labels))\n",
|
|
"val_dataset = val_dataset.batch(32)\n",
|
|
"reset_metrics()\n",
|
|
"for inputs_batch, targets_batch in val_dataset:\n",
|
|
" logs = test_step(inputs_batch, targets_batch)\n",
|
|
"print(\"Evaluation results:\")\n",
|
|
"for key, value in logs.items():\n",
|
|
" print(f\"...{key}: {value:.4f}\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"### Leveraging `fit()` with a custom training loop"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"**Implementing a custom training step to use with `fit()`**"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"loss_fn = keras.losses.SparseCategoricalCrossentropy()\n",
|
|
"loss_tracker = keras.metrics.Mean(name=\"loss\")\n",
|
|
"\n",
|
|
"class CustomModel(keras.Model):\n",
|
|
" def train_step(self, data):\n",
|
|
" inputs, targets = data\n",
|
|
" with tf.GradientTape() as tape:\n",
|
|
" predictions = self(inputs, training=True)\n",
|
|
" loss = loss_fn(targets, predictions)\n",
|
|
" gradients = tape.gradient(loss, model.trainable_weights)\n",
|
|
" optimizer.apply_gradients(zip(gradients, model.trainable_weights))\n",
|
|
"\n",
|
|
" loss_tracker.update_state(loss)\n",
|
|
" return {\"loss\": loss_tracker.result()}\n",
|
|
"\n",
|
|
" @property\n",
|
|
" def metrics(self):\n",
|
|
" return [loss_tracker]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"inputs = keras.Input(shape=(28 * 28,))\n",
|
|
"features = layers.Dense(512, activation=\"relu\")(inputs)\n",
|
|
"features = layers.Dropout(0.5)(features)\n",
|
|
"outputs = layers.Dense(10, activation=\"softmax\")(features)\n",
|
|
"model = CustomModel(inputs, outputs)\n",
|
|
"\n",
|
|
"model.compile(optimizer=keras.optimizers.RMSprop())\n",
|
|
"model.fit(train_images, train_labels, epochs=3)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"class CustomModel(keras.Model):\n",
|
|
" def train_step(self, data):\n",
|
|
" inputs, targets = data\n",
|
|
" with tf.GradientTape() as tape:\n",
|
|
" predictions = self(inputs, training=True)\n",
|
|
" loss = self.compiled_loss(targets, predictions)\n",
|
|
" gradients = tape.gradient(loss, model.trainable_weights)\n",
|
|
" optimizer.apply_gradients(zip(gradients, model.trainable_weights))\n",
|
|
" self.compiled_metrics.update_state(targets, predictions)\n",
|
|
" return {m.name: m.result() for m in self.metrics}"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 0,
|
|
"metadata": {
|
|
"colab_type": "code"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"inputs = keras.Input(shape=(28 * 28,))\n",
|
|
"features = layers.Dense(512, activation=\"relu\")(inputs)\n",
|
|
"features = layers.Dropout(0.5)(features)\n",
|
|
"outputs = layers.Dense(10, activation=\"softmax\")(features)\n",
|
|
"model = CustomModel(inputs, outputs)\n",
|
|
"\n",
|
|
"model.compile(optimizer=keras.optimizers.RMSprop(),\n",
|
|
" loss=keras.losses.SparseCategoricalCrossentropy(),\n",
|
|
" metrics=[keras.metrics.SparseCategoricalAccuracy()])\n",
|
|
"model.fit(train_images, train_labels, epochs=3)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"colab_type": "text"
|
|
},
|
|
"source": [
|
|
"## Chapter summary"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"colab": {
|
|
"collapsed_sections": [],
|
|
"name": "chapter07_working-with-keras.i",
|
|
"private_outputs": false,
|
|
"provenance": [],
|
|
"toc_visible": true
|
|
},
|
|
"kernelspec": {
|
|
"display_name": "Python 3",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.7.0"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 0
|
|
} |