{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Info\n",
    "\n",
    "\n",
    "\n",
    "## Materials:\n",
    "\n",
    "* [review_my_code_dqn_for_gym_frozenlake](https://www.reddit.com/r/learnmachinelearning/comments/9vasqm/review_my_code_dqn_for_gym_frozenlake/)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2.2.0'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gym\n",
    "import numpy as np \n",
    "import random\n",
    "from collections import deque\n",
    "import tensorflow as tf\n",
    "\n",
    "tf.compat.v1.disable_eager_execution()\n",
    "\n",
    "tf.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def one_hot_state(state,state_space):\n",
    "    state_m=np.zeros((1,state_space))\n",
    "    state_m[0][state]=1\n",
    "    return state_m\n",
    "\n",
    "def experience_replay():\n",
    "    # Sample minibatch from the memory\n",
    "    minibatch = random.sample(memory, batch_size)\n",
    "    # Extract informations from each memory\n",
    "    for state, action, reward, next_state, done in minibatch:\n",
    "        # if done, make our target reward\n",
    "        target = reward\n",
    "        if not done:\n",
    "          # predict the future discounted reward\n",
    "          target = reward + gamma * \\\n",
    "                   np.max(model.predict(next_state))\n",
    "        # make the agent to approximately map\n",
    "        # the current state to future discounted reward\n",
    "        # We'll call that target_f\n",
    "        target_f = model.predict(state)\n",
    "        target_f[0][action] = target\n",
    "        # Train the Neural Net with the state and target_f\n",
    "        model.fit(state, target_f, epochs=1, verbose=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /users/yby1/.local/lib/python3.7/site-packages/tensorflow/python/ops/resource_variable_ops.py:1666: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n",
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense (Dense)                (None, 16)                272       \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 24)                408       \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 4)                 100       \n",
      "=================================================================\n",
      "Total params: 780\n",
      "Trainable params: 780\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Episode 10 Total Reward: 0.0 Reward Rate 0.00 Epsilon 0.990\n",
      "Episode 20 Total Reward: 0.0 Reward Rate 0.00 Epsilon 0.980\n",
      "Episode 30 Total Reward: 0.0 Reward Rate 0.00 Epsilon 0.970\n",
      "Episode 40 Total Reward: 0.0 Reward Rate 0.00 Epsilon 0.961\n",
      "Episode 50 Total Reward: 0.0 Reward Rate 0.00 Epsilon 0.951\n",
      "Episode 60 Total Reward: 0.0 Reward Rate 0.00 Epsilon 0.942\n",
      "Episode 70 Total Reward: 0.0 Reward Rate 0.00 Epsilon 0.932\n",
      "Episode 80 Total Reward: 0.0 Reward Rate 0.00 Epsilon 0.923\n",
      "Episode 90 Total Reward: 0.0 Reward Rate 0.00 Epsilon 0.914\n",
      "Episode 100 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.905\n",
      "Episode 110 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.896\n",
      "Episode 120 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.887\n",
      "Episode 130 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.878\n",
      "Episode 140 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.869\n",
      "Episode 150 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.861\n",
      "Episode 160 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.852\n",
      "Episode 170 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.844\n",
      "Episode 180 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.835\n",
      "Episode 190 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.827\n",
      "Episode 200 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.819\n",
      "Episode 210 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.811\n",
      "Episode 220 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.803\n",
      "Episode 230 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.795\n",
      "Episode 240 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.787\n",
      "Episode 250 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.779\n",
      "Episode 260 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.771\n",
      "Episode 270 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.764\n",
      "Episode 280 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.756\n",
      "Episode 290 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.749\n",
      "Episode 300 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.741\n",
      "Episode 310 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.734\n",
      "Episode 320 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.726\n",
      "Episode 330 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.719\n",
      "Episode 340 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.712\n",
      "Episode 350 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.705\n",
      "Episode 360 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.698\n",
      "Episode 370 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.691\n",
      "Episode 380 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.684\n",
      "Episode 390 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.677\n",
      "Episode 400 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.671\n",
      "Episode 410 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.664\n",
      "Episode 420 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.657\n",
      "Episode 430 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.651\n",
      "Episode 440 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.644\n",
      "Episode 450 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.638\n",
      "Episode 460 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.632\n",
      "Episode 470 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.625\n",
      "Episode 480 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.619\n",
      "Episode 490 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.613\n",
      "Episode 500 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.607\n",
      "Episode 510 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.601\n",
      "Episode 520 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.595\n",
      "Episode 530 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.589\n",
      "Episode 540 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.583\n",
      "Episode 550 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.577\n",
      "Episode 560 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.572\n",
      "Episode 570 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.566\n",
      "Episode 580 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.560\n",
      "Episode 590 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.555\n",
      "Episode 600 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.549\n",
      "Episode 610 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.544\n",
      "Episode 620 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.538\n",
      "Episode 630 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.533\n",
      "Episode 640 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.528\n",
      "Episode 650 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.523\n",
      "Episode 660 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.517\n",
      "Episode 670 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.512\n",
      "Episode 680 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.507\n",
      "Episode 690 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.502\n",
      "Episode 700 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.497\n",
      "Episode 710 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.492\n",
      "Episode 720 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.487\n",
      "Episode 730 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.482\n",
      "Episode 740 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.478\n",
      "Episode 750 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.473\n",
      "Episode 760 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.468\n",
      "Episode 770 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.464\n",
      "Episode 780 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.459\n",
      "Episode 790 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.454\n",
      "Episode 800 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.450\n",
      "Episode 810 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.445\n",
      "Episode 820 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.441\n",
      "Episode 830 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.437\n",
      "Episode 840 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.432\n",
      "Episode 850 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.428\n",
      "Episode 860 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.424\n",
      "Episode 870 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.420\n",
      "Episode 880 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.415\n",
      "Episode 890 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.411\n",
      "Episode 900 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.407\n",
      "Episode 910 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.403\n",
      "Episode 920 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.399\n",
      "Episode 930 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.395\n",
      "Episode 940 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.391\n",
      "Episode 950 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.387\n",
      "Episode 960 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.384\n",
      "Episode 970 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.380\n",
      "Episode 980 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.376\n",
      "Episode 990 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.372\n",
      "Episode 1000 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.369\n",
      "Episode 1010 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.365\n",
      "Episode 1020 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.361\n",
      "Episode 1030 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.358\n",
      "Episode 1040 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.354\n",
      "Episode 1050 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.351\n",
      "Episode 1060 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.347\n",
      "Episode 1070 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.344\n",
      "Episode 1080 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.340\n",
      "Episode 1090 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.337\n",
      "Episode 1100 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.334\n",
      "Episode 1110 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.330\n",
      "Episode 1120 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.327\n",
      "Episode 1130 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.324\n",
      "Episode 1140 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.320\n",
      "Episode 1150 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.317\n",
      "Episode 1160 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.314\n",
      "Episode 1170 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.311\n",
      "Episode 1180 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.308\n",
      "Episode 1190 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.305\n",
      "Episode 1200 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.302\n",
      "Episode 1210 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.299\n",
      "Episode 1220 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.296\n",
      "Episode 1230 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.293\n",
      "Episode 1240 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.290\n",
      "Episode 1250 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.287\n",
      "Episode 1260 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.284\n",
      "Episode 1270 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.282\n",
      "Episode 1280 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.279\n",
      "Episode 1290 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.276\n",
      "Episode 1300 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.273\n",
      "Episode 1310 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.271\n",
      "Episode 1320 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.268\n",
      "Episode 1330 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.265\n",
      "Episode 1340 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.263\n",
      "Episode 1350 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.260\n",
      "Episode 1360 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.257\n",
      "Episode 1370 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.255\n",
      "Episode 1380 Total Reward: 0.0 Reward Rate 0.01 Epsilon 0.252\n",
      "Episode 1390 Total Reward: 1.0 Reward Rate 0.02 Epsilon 0.250\n",
      "Episode 1400 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.247\n",
      "Episode 1410 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.245\n",
      "Episode 1420 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.242\n",
      "Episode 1430 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.240\n",
      "Episode 1440 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.238\n",
      "Episode 1450 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.235\n",
      "Episode 1460 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.233\n",
      "Episode 1470 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.231\n",
      "Episode 1480 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.228\n",
      "Episode 1490 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.226\n",
      "Episode 1500 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.224\n",
      "Episode 1510 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.222\n",
      "Episode 1520 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.219\n",
      "Episode 1530 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.217\n",
      "Episode 1540 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.215\n",
      "Episode 1550 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.213\n",
      "Episode 1560 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.211\n",
      "Episode 1570 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.209\n",
      "Episode 1580 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.207\n",
      "Episode 1590 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.205\n",
      "Episode 1600 Total Reward: 1.0 Reward Rate 0.02 Epsilon 0.203\n",
      "Episode 1610 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.201\n",
      "Episode 1620 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.199\n",
      "Episode 1630 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.197\n",
      "Episode 1640 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.195\n",
      "Episode 1650 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.193\n",
      "Episode 1660 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.191\n",
      "Episode 1670 Total Reward: 1.0 Reward Rate 0.02 Epsilon 0.189\n",
      "Episode 1680 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.187\n",
      "Episode 1690 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.185\n",
      "Episode 1700 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.184\n",
      "Episode 1710 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.182\n",
      "Episode 1720 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.180\n",
      "Episode 1730 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.178\n",
      "Episode 1740 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.176\n",
      "Episode 1750 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.175\n",
      "Episode 1760 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.173\n",
      "Episode 1770 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.171\n",
      "Episode 1780 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.169\n",
      "Episode 1790 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.168\n",
      "Episode 1800 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.166\n",
      "Episode 1810 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.164\n",
      "Episode 1820 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.163\n",
      "Episode 1830 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.161\n",
      "Episode 1840 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.160\n",
      "Episode 1850 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.158\n",
      "Episode 1860 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.157\n",
      "Episode 1870 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.155\n",
      "Episode 1880 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.153\n",
      "Episode 1890 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.152\n",
      "Episode 1900 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.150\n",
      "Episode 1910 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.149\n",
      "Episode 1920 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.147\n",
      "Episode 1930 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.146\n",
      "Episode 1940 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.145\n",
      "Episode 1950 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.143\n",
      "Episode 1960 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.142\n",
      "Episode 1970 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.140\n",
      "Episode 1980 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.139\n",
      "Episode 1990 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.138\n",
      "Episode 2000 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.136\n",
      "Episode 2010 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.135\n",
      "Episode 2020 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.134\n",
      "Episode 2030 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.132\n",
      "Episode 2040 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.131\n",
      "Episode 2050 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.130\n",
      "Episode 2060 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.128\n",
      "Episode 2070 Total Reward: 1.0 Reward Rate 0.02 Epsilon 0.127\n",
      "Episode 2080 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.126\n",
      "Episode 2090 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.125\n",
      "Episode 2100 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.123\n",
      "Episode 2110 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.122\n",
      "Episode 2120 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.121\n",
      "Episode 2130 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.120\n",
      "Episode 2140 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.119\n",
      "Episode 2150 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.117\n",
      "Episode 2160 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.116\n",
      "Episode 2170 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.115\n",
      "Episode 2180 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.114\n",
      "Episode 2190 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.113\n",
      "Episode 2200 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.112\n",
      "Episode 2210 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.111\n",
      "Episode 2220 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.110\n",
      "Episode 2230 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.108\n",
      "Episode 2240 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.107\n",
      "Episode 2250 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.106\n",
      "Episode 2260 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.105\n",
      "Episode 2270 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.104\n",
      "Episode 2280 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.103\n",
      "Episode 2290 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.102\n",
      "Episode 2300 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.101\n",
      "Episode 2310 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.100\n",
      "Episode 2320 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.099\n",
      "Episode 2330 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.098\n",
      "Episode 2340 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.097\n",
      "Episode 2350 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.096\n",
      "Episode 2360 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.095\n",
      "Episode 2370 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.094\n",
      "Episode 2380 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.093\n",
      "Episode 2390 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.093\n",
      "Episode 2400 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.092\n",
      "Episode 2410 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.091\n",
      "Episode 2420 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.090\n",
      "Episode 2430 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.089\n",
      "Episode 2440 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.088\n",
      "Episode 2450 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.087\n",
      "Episode 2460 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.086\n",
      "Episode 2470 Total Reward: 1.0 Reward Rate 0.02 Epsilon 0.086\n",
      "Episode 2480 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.085\n",
      "Episode 2490 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.084\n",
      "Episode 2500 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.083\n",
      "Episode 2510 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.082\n",
      "Episode 2520 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.081\n",
      "Episode 2530 Total Reward: 1.0 Reward Rate 0.02 Epsilon 0.081\n",
      "Episode 2540 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.080\n",
      "Episode 2550 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.079\n",
      "Episode 2560 Total Reward: 1.0 Reward Rate 0.02 Epsilon 0.078\n",
      "Episode 2570 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.077\n",
      "Episode 2580 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.077\n",
      "Episode 2590 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.076\n",
      "Episode 2600 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.075\n",
      "Episode 2610 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.074\n",
      "Episode 2620 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.074\n",
      "Episode 2630 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.073\n",
      "Episode 2640 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.072\n",
      "Episode 2650 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.072\n",
      "Episode 2660 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.071\n",
      "Episode 2670 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.070\n",
      "Episode 2680 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.069\n",
      "Episode 2690 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.069\n",
      "Episode 2700 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.068\n",
      "Episode 2710 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.067\n",
      "Episode 2720 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.067\n",
      "Episode 2730 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.066\n",
      "Episode 2740 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.066\n",
      "Episode 2750 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.065\n",
      "Episode 2760 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.064\n",
      "Episode 2770 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.064\n",
      "Episode 2780 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.063\n",
      "Episode 2790 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.062\n",
      "Episode 2800 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.062\n",
      "Episode 2810 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.061\n",
      "Episode 2820 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.061\n",
      "Episode 2830 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.060\n",
      "Episode 2840 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.059\n",
      "Episode 2850 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.059\n",
      "Episode 2860 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.058\n",
      "Episode 2870 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.058\n",
      "Episode 2880 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.057\n",
      "Episode 2890 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.057\n",
      "Episode 2900 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.056\n",
      "Episode 2910 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.055\n",
      "Episode 2920 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.055\n",
      "Episode 2930 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.054\n",
      "Episode 2940 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.054\n",
      "Episode 2950 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.053\n",
      "Episode 2960 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.053\n",
      "Episode 2970 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.052\n",
      "Episode 2980 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.052\n",
      "Episode 2990 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.051\n",
      "Episode 3000 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.051\n",
      "Episode 3010 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.050\n",
      "Episode 3020 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.050\n",
      "Episode 3030 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.049\n",
      "Episode 3040 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.049\n",
      "Episode 3050 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.048\n",
      "Episode 3060 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.048\n",
      "Episode 3070 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.047\n",
      "Episode 3080 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.047\n",
      "Episode 3090 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.046\n",
      "Episode 3100 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.046\n",
      "Episode 3110 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.046\n",
      "Episode 3120 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.045\n",
      "Episode 3130 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.045\n",
      "Episode 3140 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.044\n",
      "Episode 3150 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.044\n",
      "Episode 3160 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.043\n",
      "Episode 3170 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.043\n",
      "Episode 3180 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.043\n",
      "Episode 3190 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.042\n",
      "Episode 3200 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.042\n",
      "Episode 3210 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.041\n",
      "Episode 3220 Total Reward: 1.0 Reward Rate 0.02 Epsilon 0.041\n",
      "Episode 3230 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.041\n",
      "Episode 3240 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.040\n",
      "Episode 3250 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.040\n",
      "Episode 3260 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.039\n",
      "Episode 3270 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.039\n",
      "Episode 3280 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.039\n",
      "Episode 3290 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.038\n",
      "Episode 3300 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.038\n",
      "Episode 3310 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.037\n",
      "Episode 3320 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.037\n",
      "Episode 3330 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.037\n",
      "Episode 3340 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.036\n",
      "Episode 3350 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.036\n",
      "Episode 3360 Total Reward: 0.0 Reward Rate 0.02 Epsilon 0.036\n",
      "Episode 3370 Total Reward: 1.0 Reward Rate 0.03 Epsilon 0.035\n",
      "Episode 3380 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.035\n",
      "Episode 3390 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.035\n",
      "Episode 3400 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.034\n",
      "Episode 3410 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.034\n",
      "Episode 3420 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.034\n",
      "Episode 3430 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.033\n",
      "Episode 3440 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.033\n",
      "Episode 3450 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.033\n",
      "Episode 3460 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.032\n",
      "Episode 3470 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.032\n",
      "Episode 3480 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.032\n",
      "Episode 3490 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.031\n",
      "Episode 3500 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.031\n",
      "Episode 3510 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.031\n",
      "Episode 3520 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.031\n",
      "Episode 3530 Total Reward: 1.0 Reward Rate 0.03 Epsilon 0.030\n",
      "Episode 3540 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.030\n",
      "Episode 3550 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.030\n",
      "Episode 3560 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.029\n",
      "Episode 3570 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.029\n",
      "Episode 3580 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.029\n",
      "Episode 3590 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.029\n",
      "Episode 3600 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.028\n",
      "Episode 3610 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.028\n",
      "Episode 3620 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.028\n",
      "Episode 3630 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.027\n",
      "Episode 3640 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.027\n",
      "Episode 3650 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.027\n",
      "Episode 3660 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.027\n",
      "Episode 3670 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.026\n",
      "Episode 3680 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.026\n",
      "Episode 3690 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.026\n",
      "Episode 3700 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.026\n",
      "Episode 3710 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.025\n",
      "Episode 3720 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.025\n",
      "Episode 3730 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.025\n",
      "Episode 3740 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.025\n",
      "Episode 3750 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.024\n",
      "Episode 3760 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.024\n",
      "Episode 3770 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.024\n",
      "Episode 3780 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.024\n",
      "Episode 3790 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.024\n",
      "Episode 3800 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.023\n",
      "Episode 3810 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.023\n",
      "Episode 3820 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.023\n",
      "Episode 3830 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.023\n",
      "Episode 3840 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.022\n",
      "Episode 3850 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.022\n",
      "Episode 3860 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.022\n",
      "Episode 3870 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.022\n",
      "Episode 3880 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.022\n",
      "Episode 3890 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.021\n",
      "Episode 3900 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.021\n",
      "Episode 3910 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.021\n",
      "Episode 3920 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.021\n",
      "Episode 3930 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.021\n",
      "Episode 3940 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.020\n",
      "Episode 3950 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.020\n",
      "Episode 3960 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.020\n",
      "Episode 3970 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.020\n",
      "Episode 3980 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.020\n",
      "Episode 3990 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.019\n",
      "Episode 4000 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.019\n",
      "Episode 4010 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.019\n",
      "Episode 4020 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.019\n",
      "Episode 4030 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.019\n",
      "Episode 4040 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.019\n",
      "Episode 4050 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.018\n",
      "Episode 4060 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.018\n",
      "Episode 4070 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.018\n",
      "Episode 4080 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.018\n",
      "Episode 4090 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.018\n",
      "Episode 4100 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.018\n",
      "Episode 4110 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.017\n",
      "Episode 4120 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.017\n",
      "Episode 4130 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.017\n",
      "Episode 4140 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.017\n",
      "Episode 4150 Total Reward: 1.0 Reward Rate 0.03 Epsilon 0.017\n",
      "Episode 4160 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.017\n",
      "Episode 4170 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.016\n",
      "Episode 4180 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.016\n",
      "Episode 4190 Total Reward: 1.0 Reward Rate 0.03 Epsilon 0.016\n",
      "Episode 4200 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.016\n",
      "Episode 4210 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.016\n",
      "Episode 4220 Total Reward: 1.0 Reward Rate 0.03 Epsilon 0.016\n",
      "Episode 4230 Total Reward: 1.0 Reward Rate 0.03 Epsilon 0.016\n",
      "Episode 4240 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.015\n",
      "Episode 4250 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.015\n",
      "Episode 4260 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.015\n",
      "Episode 4270 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.015\n",
      "Episode 4280 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.015\n",
      "Episode 4290 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.015\n",
      "Episode 4300 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.015\n",
      "Episode 4310 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.014\n",
      "Episode 4320 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.014\n",
      "Episode 4330 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.014\n",
      "Episode 4340 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.014\n",
      "Episode 4350 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.014\n",
      "Episode 4360 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.014\n",
      "Episode 4370 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.014\n",
      "Episode 4380 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.014\n",
      "Episode 4390 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.013\n",
      "Episode 4400 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.013\n",
      "Episode 4410 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.013\n",
      "Episode 4420 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.013\n",
      "Episode 4430 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.013\n",
      "Episode 4440 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.013\n",
      "Episode 4450 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.013\n",
      "Episode 4460 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.013\n",
      "Episode 4470 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.012\n",
      "Episode 4480 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.012\n",
      "Episode 4490 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.012\n",
      "Episode 4500 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.012\n",
      "Episode 4510 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.012\n",
      "Episode 4520 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.012\n",
      "Episode 4530 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.012\n",
      "Episode 4540 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.012\n",
      "Episode 4550 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.012\n",
      "Episode 4560 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.011\n",
      "Episode 4570 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.011\n",
      "Episode 4580 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.011\n",
      "Episode 4590 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.011\n",
      "Episode 4600 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.011\n",
      "Episode 4610 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.011\n",
      "Episode 4620 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.011\n",
      "Episode 4630 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.011\n",
      "Episode 4640 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.011\n",
      "Episode 4650 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.011\n",
      "Episode 4660 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.010\n",
      "Episode 4670 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.010\n",
      "Episode 4680 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.010\n",
      "Episode 4690 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.010\n",
      "Episode 4700 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.010\n",
      "Episode 4710 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.010\n",
      "Episode 4720 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.010\n",
      "Episode 4730 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.010\n",
      "Episode 4740 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.010\n",
      "Episode 4750 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.010\n",
      "Episode 4760 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.010\n",
      "Episode 4770 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.009\n",
      "Episode 4780 Total Reward: 1.0 Reward Rate 0.03 Epsilon 0.009\n",
      "Episode 4790 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.009\n",
      "Episode 4800 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.009\n",
      "Episode 4810 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.009\n",
      "Episode 4820 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.009\n",
      "Episode 4830 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.009\n",
      "Episode 4840 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.009\n",
      "Episode 4850 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.009\n",
      "Episode 4860 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.009\n",
      "Episode 4870 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.009\n",
      "Episode 4880 Total Reward: 1.0 Reward Rate 0.03 Epsilon 0.009\n",
      "Episode 4890 Total Reward: 1.0 Reward Rate 0.03 Epsilon 0.009\n",
      "Episode 4900 Total Reward: 1.0 Reward Rate 0.03 Epsilon 0.008\n",
      "Episode 4910 Total Reward: 1.0 Reward Rate 0.03 Epsilon 0.008\n",
      "Episode 4920 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.008\n",
      "Episode 4930 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.008\n",
      "Episode 4940 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.008\n",
      "Episode 4950 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.008\n",
      "Episode 4960 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.008\n",
      "Episode 4970 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.008\n",
      "Episode 4980 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.008\n",
      "Episode 4990 Total Reward: 0.0 Reward Rate 0.03 Epsilon 0.008\n",
      "Episode 5000 Total Reward: 1.0 Reward Rate 0.03 Epsilon 0.008\n"
     ]
    }
   ],
   "source": [
    "# 1. Parameters of Q-leanring    \n",
    "gamma = .9\n",
    "learning_rate=0.002\n",
    "episode = 5001\n",
    "capacity=64\n",
    "batch_size=32\n",
    "\n",
    "# Exploration parameters\n",
    "epsilon = 1.0           # Exploration rate\n",
    "max_epsilon = 1.0             # Exploration probability at start\n",
    "min_epsilon = 0.001            # Minimum exploration probability \n",
    "decay_rate = 0.001             # Exponential decay rate for exploration prob\n",
    "\n",
    "# 2. Load Environment \n",
    "env = gym.make(\"FrozenLake-v0\")\n",
    "\n",
    "# env.obeservation.n, env.action_space.n gives number of states and action in env loaded\n",
    "state_space=env.observation_space.n\n",
    "action_space=env.action_space.n\n",
    "\n",
    "#Neural network model for DQN\n",
    "model = tf.keras.Sequential()\n",
    "model.add(tf.keras.layers.Dense(state_space, input_dim=state_space, activation='relu'))\n",
    "model.add(tf.keras.layers.Dense(24, activation='relu'))\n",
    "model.add(tf.keras.layers.Dense(action_space, activation='linear'))\n",
    "model.compile(loss='mse',\n",
    "              optimizer=tf.keras.optimizers.Adam(lr=learning_rate))\n",
    "\n",
    "model.summary()\n",
    "\n",
    "\n",
    "reward_array=[]\n",
    "memory = deque([], maxlen=capacity)\n",
    "for i in range(episode):\n",
    "    state=env.reset()    \n",
    "    total_reward=0\n",
    "    done=False\n",
    "    while not done:        \n",
    "        state1_one_hot=one_hot_state(state,state_space)\n",
    "        # 3. Choose an action a in the current world state (s)\n",
    "        ## First we randomize a number\n",
    "        exp_exp_tradeoff = np.random.uniform()\n",
    "        \n",
    "        ## If this number > greater than epsilon --> exploitation (taking the biggest Q value for this state)\n",
    "        if exp_exp_tradeoff > epsilon:\n",
    "            action = np.argmax(model.predict(state1_one_hot))\n",
    "        # Else doing a random choice --> exploration\n",
    "        else:\n",
    "            action = env.action_space.sample()\n",
    "        \n",
    "        #Training without experience replay\n",
    "        state2,reward,done,info=env.step(action)\n",
    "        state2_one_hot=one_hot_state(state2,state_space)\n",
    "        target = (reward + gamma *\n",
    "                      np.max(model.predict(state2_one_hot)))\n",
    "                      \n",
    "        target_f=model.predict(state1_one_hot)\n",
    "        target_f[0][action]=target\n",
    "        model.fit(state1_one_hot,target_f,epochs=1,verbose=0)        \n",
    "        total_reward +=reward\n",
    "                        \n",
    "        state=state2\n",
    "        \n",
    "        #Training with experience replay\n",
    "        #appending to memory\n",
    "        memory.append((state1_one_hot, action, reward, state2_one_hot, done))\n",
    "        #experience replay\n",
    "    if i>batch_size:\n",
    "        experience_replay()\n",
    "            \n",
    "    reward_array.append(total_reward)\n",
    "    # Reduce epsilon (because we need less and less exploration)\n",
    "    epsilon = min_epsilon + (max_epsilon - min_epsilon)*np.exp(-decay_rate*i) \n",
    "    \n",
    "    if i % 10==0 and i !=0:\n",
    "        print('Episode {} Total Reward: {} Reward Rate {:.2f} Epsilon {:.3f}'.format(i,total_reward,(sum(reward_array)/i), epsilon))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
