{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "Now let's get to the fun part -- training a model. I'll start by installing the dependencies." ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Requirement already satisfied: peft==0.5.0 in /usr/local/lib/python3.10/dist-packages (0.5.0)\n", "\u001b[31mERROR: Could not find a version that satisfies the requirement python-dotenv==2.0.0 (from versions: 0.1.0, 0.1.2, 0.1.3, 0.1.5, 0.2.0, 0.3.0, 0.4.0, 0.5.0, 0.5.1, 0.6.0, 0.6.1, 0.6.2, 0.6.3, 0.6.4, 0.6.5, 0.7.0, 0.7.1, 0.8.0, 0.8.1, 0.8.2, 0.9.0, 0.9.1, 0.10.0, 0.10.1, 0.10.2, 0.10.3, 0.10.4, 0.10.5, 0.11.0, 0.12.0, 0.13.0, 0.14.0, 0.15.0, 0.16.0, 0.17.0, 0.17.1, 0.18.0, 0.19.0, 0.19.1, 0.19.2, 0.20.0, 0.21.0, 0.21.1, 1.0.0)\u001b[0m\u001b[31m\n", "\u001b[0m\u001b[31mERROR: No matching distribution found for python-dotenv==2.0.0\u001b[0m\u001b[31m\n", "\u001b[0m\n", "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.1.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.2.1\u001b[0m\n", "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3.10 -m pip install --upgrade pip\u001b[0m\n", "Note: you may need to restart the kernel to use updated packages.\n", "fatal: destination path 'axolotl' already exists and is not an empty directory.\n", "Obtaining file:///workspace/OpenPipe/examples/classify-recipes/axolotl\n", " Preparing metadata (setup.py) ... \u001b[?25ldone\n", "\u001b[?25hCollecting transformers@ git+https://github.com/huggingface/transformers.git (from axolotl==0.1)\n", " Cloning https://github.com/huggingface/transformers.git to /tmp/pip-install-ckp96ans/transformers_783779e09ad546a5be81c173eca5fd38\n", " Running command git clone --filter=blob:none --quiet https://github.com/huggingface/transformers.git /tmp/pip-install-ckp96ans/transformers_783779e09ad546a5be81c173eca5fd38\n", " Resolved https://github.com/huggingface/transformers.git to commit f26099e7b5cf579f99a42bab6ddd371bf2c8d548\n", " Installing build dependencies ... \u001b[?25ldone\n", "\u001b[?25h Getting requirements to build wheel ... \u001b[?25ldone\n", "\u001b[?25h Preparing metadata (pyproject.toml) ... \u001b[?25ldone\n", "\u001b[?25hCollecting accelerate@ git+https://github.com/huggingface/accelerate@2a289f6108e77a77a4efffb3f6316bc98538413b (from axolotl==0.1)\n", " Using cached accelerate-0.22.0.dev0-py3-none-any.whl\n", "Requirement already satisfied: bitsandbytes>=0.41.1 in /usr/local/lib/python3.10/dist-packages (from axolotl==0.1) (0.41.1)\n", "Requirement already satisfied: addict in /usr/local/lib/python3.10/dist-packages (from axolotl==0.1) (2.4.0)\n", "Requirement already satisfied: fire in /usr/local/lib/python3.10/dist-packages (from axolotl==0.1) (0.5.0)\n", "Requirement already satisfied: PyYAML==6.0 in /usr/local/lib/python3.10/dist-packages (from axolotl==0.1) (6.0)\n", "Requirement already satisfied: datasets in /usr/local/lib/python3.10/dist-packages (from axolotl==0.1) (2.14.4)\n", "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.10/dist-packages (from axolotl==0.1) (0.1.99)\n", "Requirement already satisfied: wandb in /usr/local/lib/python3.10/dist-packages (from axolotl==0.1) (0.15.8)\n", "Requirement already satisfied: einops in /usr/local/lib/python3.10/dist-packages (from axolotl==0.1) (0.6.1)\n", "Requirement already satisfied: xformers in /usr/local/lib/python3.10/dist-packages (from axolotl==0.1) (0.0.21)\n", "Requirement already satisfied: optimum in /usr/local/lib/python3.10/dist-packages (from axolotl==0.1) (1.11.2)\n", "Requirement already satisfied: hf_transfer in /usr/local/lib/python3.10/dist-packages (from axolotl==0.1) (0.1.3)\n", "Requirement already satisfied: colorama in /usr/local/lib/python3.10/dist-packages (from axolotl==0.1) (0.4.6)\n", "Requirement already satisfied: numba in /usr/local/lib/python3.10/dist-packages (from axolotl==0.1) (0.57.1)\n", "Requirement already satisfied: numpy==1.24.4 in /usr/local/lib/python3.10/dist-packages (from axolotl==0.1) (1.24.4)\n", "Requirement already satisfied: bert-score==0.3.13 in /usr/local/lib/python3.10/dist-packages (from axolotl==0.1) (0.3.13)\n", "Requirement already satisfied: evaluate==0.4.0 in /usr/local/lib/python3.10/dist-packages (from axolotl==0.1) (0.4.0)\n", "Requirement already satisfied: rouge-score==0.1.2 in /usr/local/lib/python3.10/dist-packages (from axolotl==0.1) (0.1.2)\n", "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from axolotl==0.1) (1.11.2)\n", "Requirement already satisfied: scikit-learn==1.2.2 in /usr/local/lib/python3.10/dist-packages (from axolotl==0.1) (1.2.2)\n", "Requirement already satisfied: pynvml in /usr/local/lib/python3.10/dist-packages (from axolotl==0.1) (11.5.0)\n", "Requirement already satisfied: flash-attn==2.0.8 in /usr/local/lib/python3.10/dist-packages (from axolotl==0.1) (2.0.8)\n", "Requirement already satisfied: torch>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from bert-score==0.3.13->axolotl==0.1) (2.0.1+cu118)\n", "Requirement already satisfied: pandas>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from bert-score==0.3.13->axolotl==0.1) (2.0.3)\n", "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from bert-score==0.3.13->axolotl==0.1) (2.28.1)\n", "Requirement already satisfied: tqdm>=4.31.1 in /usr/local/lib/python3.10/dist-packages (from bert-score==0.3.13->axolotl==0.1) (4.66.1)\n", "Requirement already satisfied: matplotlib in /usr/local/lib/python3.10/dist-packages (from bert-score==0.3.13->axolotl==0.1) (3.7.2)\n", "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from bert-score==0.3.13->axolotl==0.1) (23.1)\n", "Requirement already satisfied: dill in /usr/local/lib/python3.10/dist-packages (from evaluate==0.4.0->axolotl==0.1) (0.3.7)\n", "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from evaluate==0.4.0->axolotl==0.1) (3.3.0)\n", "Requirement already satisfied: multiprocess in /usr/local/lib/python3.10/dist-packages (from evaluate==0.4.0->axolotl==0.1) (0.70.15)\n", "Requirement already satisfied: fsspec[http]>=2021.05.0 in /usr/local/lib/python3.10/dist-packages (from evaluate==0.4.0->axolotl==0.1) (2023.6.0)\n", "Requirement already satisfied: huggingface-hub>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from evaluate==0.4.0->axolotl==0.1) (0.16.4)\n", "Requirement already satisfied: responses<0.19 in /usr/local/lib/python3.10/dist-packages (from evaluate==0.4.0->axolotl==0.1) (0.18.0)\n", "Requirement already satisfied: ninja in /usr/local/lib/python3.10/dist-packages (from flash-attn==2.0.8->axolotl==0.1) (1.11.1)\n", "Requirement already satisfied: absl-py in /usr/local/lib/python3.10/dist-packages (from rouge-score==0.1.2->axolotl==0.1) (1.4.0)\n", "Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (from rouge-score==0.1.2->axolotl==0.1) (3.8.1)\n", "Requirement already satisfied: six>=1.14.0 in /usr/lib/python3/dist-packages (from rouge-score==0.1.2->axolotl==0.1) (1.16.0)\n", "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn==1.2.2->axolotl==0.1) (1.3.2)\n", "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn==1.2.2->axolotl==0.1) (3.2.0)\n", "Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets->axolotl==0.1) (12.0.1)\n", "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets->axolotl==0.1) (3.8.5)\n", "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers@ git+https://github.com/huggingface/transformers.git->axolotl==0.1) (3.9.0)\n", "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers@ git+https://github.com/huggingface/transformers.git->axolotl==0.1) (2023.8.8)\n", "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /usr/local/lib/python3.10/dist-packages (from transformers@ git+https://github.com/huggingface/transformers.git->axolotl==0.1) (0.13.3)\n", "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from transformers@ git+https://github.com/huggingface/transformers.git->axolotl==0.1) (0.3.2)\n", "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate@ git+https://github.com/huggingface/accelerate@2a289f6108e77a77a4efffb3f6316bc98538413b->axolotl==0.1) (5.9.5)\n", "Requirement already satisfied: termcolor in /usr/local/lib/python3.10/dist-packages (from fire->axolotl==0.1) (2.3.0)\n", "Requirement already satisfied: llvmlite<0.41,>=0.40.0dev0 in /usr/local/lib/python3.10/dist-packages (from numba->axolotl==0.1) (0.40.1)\n", "Requirement already satisfied: coloredlogs in /usr/local/lib/python3.10/dist-packages (from optimum->axolotl==0.1) (15.0.1)\n", "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from optimum->axolotl==0.1) (1.11.1)\n", "Requirement already satisfied: Click!=8.0.0,>=7.1 in /usr/local/lib/python3.10/dist-packages (from wandb->axolotl==0.1) (8.1.7)\n", "Requirement already satisfied: GitPython!=3.1.29,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from wandb->axolotl==0.1) (3.1.32)\n", "Requirement already satisfied: sentry-sdk>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from wandb->axolotl==0.1) (1.29.2)\n", "Requirement already satisfied: docker-pycreds>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from wandb->axolotl==0.1) (0.4.0)\n", "Requirement already satisfied: pathtools in /usr/local/lib/python3.10/dist-packages (from wandb->axolotl==0.1) (0.1.2)\n", "Requirement already satisfied: setproctitle in /usr/local/lib/python3.10/dist-packages (from wandb->axolotl==0.1) (1.3.2)\n", "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from wandb->axolotl==0.1) (68.0.0)\n", "Requirement already satisfied: appdirs>=1.4.3 in /usr/local/lib/python3.10/dist-packages (from wandb->axolotl==0.1) (1.4.4)\n", "Requirement already satisfied: protobuf!=4.21.0,<5,>=3.19.0 in /usr/local/lib/python3.10/dist-packages (from wandb->axolotl==0.1) (4.24.1)\n", "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (4.7.1)\n", "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (3.0)\n", "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (3.1.2)\n", "Requirement already satisfied: triton==2.0.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (2.0.0)\n", "Requirement already satisfied: cmake in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (3.25.0)\n", "Requirement already satisfied: lit in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (15.0.7)\n", "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->axolotl==0.1) (23.1.0)\n", "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->axolotl==0.1) (2.1.1)\n", "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->axolotl==0.1) (6.0.4)\n", "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->axolotl==0.1) (4.0.3)\n", "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->axolotl==0.1) (1.9.2)\n", "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->axolotl==0.1) (1.4.0)\n", "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->axolotl==0.1) (1.3.1)\n", "Requirement already satisfied: gitdb<5,>=4.0.1 in /usr/local/lib/python3.10/dist-packages (from GitPython!=3.1.29,>=1.0.0->wandb->axolotl==0.1) (4.0.10)\n", "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas>=1.0.1->bert-score==0.3.13->axolotl==0.1) (2.8.2)\n", "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas>=1.0.1->bert-score==0.3.13->axolotl==0.1) (2023.3)\n", "Requirement already satisfied: tzdata>=2022.1 in /usr/local/lib/python3.10/dist-packages (from pandas>=1.0.1->bert-score==0.3.13->axolotl==0.1) (2023.3)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->bert-score==0.3.13->axolotl==0.1) (3.4)\n", "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->bert-score==0.3.13->axolotl==0.1) (1.26.13)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->bert-score==0.3.13->axolotl==0.1) (2022.12.7)\n", "Requirement already satisfied: humanfriendly>=9.1 in /usr/local/lib/python3.10/dist-packages (from coloredlogs->optimum->axolotl==0.1) (10.0)\n", "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->bert-score==0.3.13->axolotl==0.1) (1.1.0)\n", "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib->bert-score==0.3.13->axolotl==0.1) (0.11.0)\n", "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->bert-score==0.3.13->axolotl==0.1) (4.42.1)\n", "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->bert-score==0.3.13->axolotl==0.1) (1.4.4)\n", "Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->bert-score==0.3.13->axolotl==0.1) (9.3.0)\n", "Requirement already satisfied: pyparsing<3.1,>=2.3.1 in /usr/lib/python3/dist-packages (from matplotlib->bert-score==0.3.13->axolotl==0.1) (2.4.7)\n", "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->optimum->axolotl==0.1) (1.2.1)\n", "Requirement already satisfied: smmap<6,>=3.0.1 in /usr/local/lib/python3.10/dist-packages (from gitdb<5,>=4.0.1->GitPython!=3.1.29,>=1.0.0->wandb->axolotl==0.1) (5.0.0)\n", "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.0.0->bert-score==0.3.13->axolotl==0.1) (2.1.2)\n", "Installing collected packages: axolotl\n", " Attempting uninstall: axolotl\n", " Found existing installation: axolotl 0.1\n", " Uninstalling axolotl-0.1:\n", " Successfully uninstalled axolotl-0.1\n", " Running setup.py develop for axolotl\n", "Successfully installed axolotl\n", "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n", "\u001b[0m\n", "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.1.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.2.1\u001b[0m\n", "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3.10 -m pip install --upgrade pip\u001b[0m\n", "Note: you may need to restart the kernel to use updated packages.\n" ] } ], "source": [ "%pip install peft==0.5.0 python-dotenv==2.0.0\n", "\n", "!git clone https://github.com/OpenAccess-AI-Collective/axolotl\n", "%pip install -e \"./axolotl[flash-attn]\"" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Note to the reader: since we'll be basing our fine-tuned model on Meta's Llama 2, you need to apply for access to the weights (which will be automatically granted). Follow the steps on [HuggingFace](https://huggingface.co/meta-llama/Llama-2-7b-hf), then create a read-only access token [here](https://huggingface.co/settings/tokens) and copy it into your .env file." ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Hugging Face token set: True\n" ] } ], "source": [ "import dotenv\n", "import os\n", "\n", "dotenv.load_dotenv()\n", "\n", "has_token = os.getenv(\"HUGGING_FACE_HUB_TOKEN\") is not None\n", "\n", "print(f\"Hugging Face token set: {has_token}\")\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "I'll use the [axolotl](https://github.com/OpenAccess-AI-Collective/axolotl) library to manage this training run. It includes a lot of neat tricks that speed up training without sacrificing quality.\n", "\n", "In this case I'm using 8-bit training to use less GPU RAM, and sample packing to maximize GPU utilization. You can read more about the available options at https://github.com/OpenAccess-AI-Collective/axolotl.\n", "\n", "The training run options are defined in [training-config.yaml](./training-config.yaml)." ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "The following values were not passed to `accelerate launch` and had defaults used instead:\n", "\t`--num_processes` was set to a value of `1`\n", "\t`--num_machines` was set to a value of `1`\n", "\t`--mixed_precision` was set to a value of `'no'`\n", "\t`--dynamo_backend` was set to a value of `'no'`\n", "To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`.\n", "\n", " dP dP dP\n", " 88 88 88\n", ".d8888b. dP. .dP .d8888b. 88 .d8888b. d8888P 88\n", "88' `88 `8bd8' 88' `88 88 88' `88 88 88\n", "88. .88 .d88b. 88. .88 88 88. .88 88 88\n", "`88888P8 dP' `dP `88888P' dP `88888P' dP dP\n", "\n", "[2023-08-24 20:18:54,867] [INFO] [axolotl.normalize_config:72] [PID:125016] GPU memory usage baseline: 0.000GB (+0.674GB misc)\u001b[39m\n", "[2023-08-24 20:18:54,867] [INFO] [axolotl.scripts.train:189] [PID:125016] loading tokenizer... meta-llama/Llama-2-7b-hf\u001b[39m\n", "[2023-08-24 20:18:55,078] [DEBUG] [axolotl.load_tokenizer:64] [PID:125016] EOS: 2 / \u001b[39m\n", "[2023-08-24 20:18:55,078] [DEBUG] [axolotl.load_tokenizer:65] [PID:125016] BOS: 1 / \u001b[39m\n", "[2023-08-24 20:18:55,078] [DEBUG] [axolotl.load_tokenizer:66] [PID:125016] PAD: 0 / [PAD]\u001b[39m\n", "[2023-08-24 20:18:55,078] [DEBUG] [axolotl.load_tokenizer:67] [PID:125016] UNK: 0 / \u001b[39m\n", "[2023-08-24 20:18:55,079] [INFO] [axolotl.load_tokenized_prepared_datasets:126] [PID:125016] Unable to find prepared dataset in data/last_run_prepared/82cd9d58e34e0db98296199248c92d0d\u001b[39m\n", "[2023-08-24 20:18:55,079] [INFO] [axolotl.load_tokenized_prepared_datasets:127] [PID:125016] Loading raw datasets...\u001b[39m\n", "[2023-08-24 20:18:55,079] [INFO] [axolotl.load_tokenized_prepared_datasets:132] [PID:125016] No seed provided, using default seed of 42\u001b[39m\n", "/usr/local/lib/python3.10/dist-packages/datasets/load.py:2072: FutureWarning: 'use_auth_token' was deprecated in favor of 'token' in version 2.14.0 and will be removed in 3.0.0.\n", "You can remove this warning by passing 'token=None' instead.\n", " warnings.warn(\n", "Downloading data files: 100%|███████████████████| 1/1 [00:00<00:00, 1909.97it/s]\n", "Extracting data files: 100%|█████████████████████| 1/1 [00:00<00:00, 130.16it/s]\n", "Generating train split: 4501 examples [00:00, 72594.78 examples/s]\n", "Map (num_proc=64): 100%|███████████| 4501/4501 [00:01<00:00, 3465.17 examples/s]\n", "[2023-08-24 20:18:58,085] [INFO] [axolotl.load_tokenized_prepared_datasets:330] [PID:125016] merging datasets\u001b[39m\n", "[2023-08-24 20:18:58,092] [INFO] [axolotl.load_tokenized_prepared_datasets:337] [PID:125016] Saving merged prepared dataset to disk... data/last_run_prepared/82cd9d58e34e0db98296199248c92d0d\u001b[39m\n", "Saving the dataset (1/1 shards): 100%|█| 4501/4501 [00:00<00:00, 63380.02 exampl\n", "Filter (num_proc=255): 100%|███████| 4275/4275 [00:01<00:00, 3385.29 examples/s]\n", "Filter (num_proc=226): 100%|██████████| 226/226 [00:01<00:00, 196.38 examples/s]\n", "Map (num_proc=255): 100%|██████████| 4275/4275 [00:02<00:00, 1480.29 examples/s]\n", "Map (num_proc=226): 100%|██████████████| 226/226 [00:05<00:00, 44.33 examples/s]\n", "[2023-08-24 20:19:33,527] [INFO] [axolotl.calculate_total_num_steps:346] [PID:125016] calculating total_num_tokens\u001b[39m\n", "[2023-08-24 20:19:33,536] [INFO] [axolotl.calculate_total_num_steps:353] [PID:125016] 📝 UPDATE CONFIG WITH: `total_num_tokens: 1514815`\u001b[39m\n", "[2023-08-24 20:19:33,552] [INFO] [axolotl.utils.dataloader.generate_batches:181] [PID:125016] generating packed batches\u001b[39m\n", "[2023-08-24 20:19:33,590] [INFO] [axolotl.utils.dataloader.generate_batches:187] [PID:125016] 2ae1e19cb9bd6022bcc024ba552b1341f4c424a75595ff3419969cc2f838c2ba\u001b[39m\n", "[2023-08-24 20:19:40,094] [INFO] [axolotl.utils.dataloader.len_w_stats:293] [PID:125016] packing_efficiency_estimate: 1.0 actual packing efficiency: 0.9732312654194079\u001b[39m\n", "[2023-08-24 20:19:40,094] [INFO] [axolotl.utils.dataloader._len_est:262] [PID:125016] packing_efficiency_estimate: 1.0 total_num_tokens per device: 1514815\u001b[39m\n", "[2023-08-24 20:19:40,094] [INFO] [axolotl.calculate_total_num_steps:393] [PID:125016] data_loader_len: 182\u001b[39m\n", "[2023-08-24 20:19:40,094] [INFO] [axolotl.calculate_total_num_steps:402] [PID:125016] 📝 UPDATE CONFIG WITH: `sample_packing_eff_est: 0.98`\u001b[39m\n", "[2023-08-24 20:19:40,094] [INFO] [axolotl.calculate_total_num_steps:410] [PID:125016] total_num_steps: 227\u001b[39m\n", "[2023-08-24 20:19:40,094] [INFO] [axolotl.scripts.train:211] [PID:125016] loading model and (optionally) peft_config...\u001b[39m\n", "[2023-08-24 20:19:40,114] [INFO] [axolotl.load_model:106] [PID:125016] patching with flash attention\u001b[39m\n", "[2023-08-24 20:19:40,117] [INFO] [axolotl.load_model:147] [PID:125016] patching _expand_mask\u001b[39m\n", "Loading checkpoint shards: 100%|██████████████████| 2/2 [00:17<00:00, 8.60s/it]\n", "\u001b[33m[2023-08-24 20:19:58,136] [WARNING] [axolotl.load_model:337] [PID:125016] increasing model.config.max_position_embeddings to 4096\u001b[39m\n", "[2023-08-24 20:19:58,136] [INFO] [axolotl.load_model:343] [PID:125016] GPU memory usage after model load: 6.681GB (+0.364GB cache, +1.159GB misc)\u001b[39m\n", "[2023-08-24 20:19:58,136] [INFO] [axolotl.load_model:349] [PID:125016] converting PEFT model w/ prepare_model_for_kbit_training\u001b[39m\n", "[2023-08-24 20:19:58,146] [INFO] [axolotl.load_lora:473] [PID:125016] found linear modules: ['k_proj', 'q_proj', 'o_proj', 'up_proj', 'down_proj', 'gate_proj', 'v_proj']\u001b[39m\n", "trainable params: 79,953,920 || all params: 6,818,369,536 || trainable%: 1.172625208678628\n", "[2023-08-24 20:20:53,348] [INFO] [axolotl.load_model:394] [PID:125016] GPU memory usage after adapters: 6.830GB (+1.365GB cache, +1.159GB misc)\u001b[39m\n", "[2023-08-24 20:20:53,380] [INFO] [axolotl.scripts.train:267] [PID:125016] Compiling torch model\u001b[39m\n", "[2023-08-24 20:20:53,544] [INFO] [axolotl.scripts.train:272] [PID:125016] Pre-saving adapter config to ./models/run1\u001b[39m\n", "[2023-08-24 20:20:53,548] [INFO] [axolotl.scripts.train:288] [PID:125016] Starting trainer...\u001b[39m\n", "[2023-08-24 20:20:53,747] [INFO] [axolotl.utils.dataloader._len_est:262] [PID:125016] packing_efficiency_estimate: 0.98 total_num_tokens per device: 1514815\u001b[39m\n", "[2023-08-24 20:20:53,747] [INFO] [axolotl.utils.dataloader._len_est:262] [PID:125016] packing_efficiency_estimate: 0.98 total_num_tokens per device: 1514815\u001b[39m\n", "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mopenpipe\u001b[0m (\u001b[33mopenpipe-team\u001b[0m). Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n", "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.15.8\n", "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m/workspace/OpenPipe/examples/classify-recipes/wandb/run-20230824_202055-run1\u001b[0m\n", "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\n", "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33mrun1\u001b[0m\n", "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/openpipe-team/classify-recipes\u001b[0m\n", "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/openpipe-team/classify-recipes/runs/run1\u001b[0m\n", " 0%| | 0/230 [00:00