From 3f68e950976eca364943d80ca515b0196ef5be13 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 18 Apr 2023 01:29:27 -0400 Subject: [PATCH 01/31] Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index e95b655..315a95a 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit e95b6554b493e71a0275764342e09bd5784a7026 +Subproject commit 315a95a4d30db726fb7d244dd3b9e90a83fb1616 From 35abf89552c6167cec6b110fc6981585970147cd Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 18 Apr 2023 01:30:04 -0400 Subject: [PATCH 02/31] Add bindings for LoRA adapters. Closes #88 --- llama_cpp/llama_cpp.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 811f69a..cad9030 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -114,7 +114,9 @@ LLAMA_FTYPE_ALL_F32 = ctypes.c_int(0) LLAMA_FTYPE_MOSTLY_F16 = ctypes.c_int(1) # except 1d tensors LLAMA_FTYPE_MOSTLY_Q4_0 = ctypes.c_int(2) # except 1d tensors LLAMA_FTYPE_MOSTLY_Q4_1 = ctypes.c_int(3) # except 1d tensors -LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16 = ctypes.c_int(4) # tok_embeddings.weight and output.weight are F16 +LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16 = ctypes.c_int( + 4 +) # tok_embeddings.weight and output.weight are F16 # Functions @@ -175,6 +177,22 @@ _lib.llama_model_quantize.argtypes = [c_char_p, c_char_p, c_int] _lib.llama_model_quantize.restype = c_int +# Apply a LoRA adapter to a loaded model +# path_base_model is the path to a higher quality model to use as a base for +# the layers modified by the adapter. Can be NULL to use the current loaded model. +# The model needs to be reloaded before applying a new adapter, otherwise the adapter +# will be applied on top of the previous one +# Returns 0 on success +def llama_apply_lora_from_file( + ctx: llama_context_p, path_lora: bytes, path_base_model: bytes, n_threads: c_int +) -> c_int: + return _lib.llama_apply_lora_from_file(ctx, path_lora, path_base_model, n_threads) + + +_lib.llama_apply_lora_from_file.argtypes = [llama_context_p, c_char_p, c_char_p, c_int] +_lib.llama_apply_lora_from_file.restype = c_int + + # Returns the KV cache that will contain the context for the # ongoing prediction with the model. def llama_get_kv_cache(ctx: llama_context_p): From eb7f278cc645ad85ad641713d423bc8193016fd2 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 18 Apr 2023 01:43:44 -0400 Subject: [PATCH 03/31] Add lora_path parameter to Llama model --- llama_cpp/llama.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index edd2eef..931d0ff 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -39,6 +39,7 @@ class Llama: n_threads: Optional[int] = None, n_batch: int = 8, last_n_tokens_size: int = 64, + lora_path: Optional[str] = None, verbose: bool = True, ): """Load a llama.cpp model from `model_path`. @@ -57,6 +58,7 @@ class Llama: n_threads: Number of threads to use. If None, the number of threads is automatically determined. n_batch: Maximum number of prompt tokens to batch together when calling llama_eval. last_n_tokens_size: Maximum number of tokens to keep in the last_n_tokens deque. + lora_path: Path to a LoRA file to apply to the model. verbose: Print verbose output to stderr. Raises: @@ -108,6 +110,17 @@ class Llama: self.model_path.encode("utf-8"), self.params ) + self.lora_path = None + if lora_path: + self.lora_path = lora_path + if llama_cpp.llama_apply_lora_from_file( + self.ctx, + self.lora_path.encode("utf-8"), + self.model_path.encode("utf-8"), + llama_cpp.c_int(self.n_threads), + ): + raise RuntimeError(f"Failed to apply LoRA from path: {self.lora_path}") + if self.verbose: print(llama_cpp.llama_print_system_info().decode("utf-8"), file=sys.stderr) @@ -802,6 +815,7 @@ class Llama: last_n_tokens_size=self.last_n_tokens_size, n_batch=self.n_batch, n_threads=self.n_threads, + lora_path=self.lora_path, ) def __setstate__(self, state): @@ -819,6 +833,7 @@ class Llama: n_threads=state["n_threads"], n_batch=state["n_batch"], last_n_tokens_size=state["last_n_tokens_size"], + lora_path=state["lora_path"], verbose=state["verbose"], ) From b2d44aa6339c24ccdcda6922371cafb10ce92d40 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 18 Apr 2023 02:22:35 -0400 Subject: [PATCH 04/31] Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 315a95a..4274722 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 315a95a4d30db726fb7d244dd3b9e90a83fb1616 +Subproject commit 42747220b4cac548b6e3059b66b3e960b517cfa4 From 453e517fd54c5f2a882199629beb0f01002e0b40 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 18 Apr 2023 10:20:46 -0400 Subject: [PATCH 05/31] Add seperate lora_base path for applying LoRA to quantized models using original unquantized model weights. --- llama_cpp/llama.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 931d0ff..5f09e4d 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -39,6 +39,7 @@ class Llama: n_threads: Optional[int] = None, n_batch: int = 8, last_n_tokens_size: int = 64, + lora_base: Optional[str] = None, lora_path: Optional[str] = None, verbose: bool = True, ): @@ -58,6 +59,7 @@ class Llama: n_threads: Number of threads to use. If None, the number of threads is automatically determined. n_batch: Maximum number of prompt tokens to batch together when calling llama_eval. last_n_tokens_size: Maximum number of tokens to keep in the last_n_tokens deque. + lora_base: Optional path to base model, useful if using a quantized base model and you want to apply LoRA to an f16 model. lora_path: Path to a LoRA file to apply to the model. verbose: Print verbose output to stderr. @@ -110,16 +112,21 @@ class Llama: self.model_path.encode("utf-8"), self.params ) + self.lora_base = None self.lora_path = None if lora_path: + self.lora_base = lora_base + # Use lora_base if set otherwise revert to using model_path. + lora_base = lora_base if lora_base is not None else model_path + self.lora_path = lora_path if llama_cpp.llama_apply_lora_from_file( self.ctx, - self.lora_path.encode("utf-8"), - self.model_path.encode("utf-8"), + lora_path.encode("utf-8"), + lora_base.encode("utf-8"), llama_cpp.c_int(self.n_threads), ): - raise RuntimeError(f"Failed to apply LoRA from path: {self.lora_path}") + raise RuntimeError(f"Failed to apply LoRA from lora path: {lora_path} to base path: {lora_base}") if self.verbose: print(llama_cpp.llama_print_system_info().decode("utf-8"), file=sys.stderr) @@ -815,6 +822,7 @@ class Llama: last_n_tokens_size=self.last_n_tokens_size, n_batch=self.n_batch, n_threads=self.n_threads, + lora_base=self.lora_base, lora_path=self.lora_path, ) @@ -833,6 +841,7 @@ class Llama: n_threads=state["n_threads"], n_batch=state["n_batch"], last_n_tokens_size=state["last_n_tokens_size"], + lora_base=state["lora_base"], lora_path=state["lora_path"], verbose=state["verbose"], ) From 95c0dc134ebef73af5c3b88b246085dcd8e51492 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 18 Apr 2023 23:44:46 -0400 Subject: [PATCH 06/31] Update type signature to allow for null pointer to be passed. --- llama_cpp/llama_cpp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index cad9030..78d8e1f 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -184,7 +184,7 @@ _lib.llama_model_quantize.restype = c_int # will be applied on top of the previous one # Returns 0 on success def llama_apply_lora_from_file( - ctx: llama_context_p, path_lora: bytes, path_base_model: bytes, n_threads: c_int + ctx: llama_context_p, path_lora: ctypes.c_char_p, path_base_model: ctypes.c_char_p, n_threads: c_int ) -> c_int: return _lib.llama_apply_lora_from_file(ctx, path_lora, path_base_model, n_threads) From 0df4d69c205a9b9ca509854ae9840598b894259b Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 18 Apr 2023 23:45:25 -0400 Subject: [PATCH 07/31] If lora base is not set avoid re-loading the model by passing NULL --- llama_cpp/llama.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 5f09e4d..ea9f0ff 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -112,21 +112,20 @@ class Llama: self.model_path.encode("utf-8"), self.params ) - self.lora_base = None - self.lora_path = None - if lora_path: - self.lora_base = lora_base - # Use lora_base if set otherwise revert to using model_path. - lora_base = lora_base if lora_base is not None else model_path - - self.lora_path = lora_path + self.lora_base = lora_base + self.lora_path = lora_path + if self.lora_path: if llama_cpp.llama_apply_lora_from_file( self.ctx, - lora_path.encode("utf-8"), - lora_base.encode("utf-8"), + llama_cpp.c_char_p(self.lora_path.encode("utf-8")), + llama_cpp.c_char_p(self.lora_base.encode("utf-8")) + if self.lora_base is not None + else llama_cpp.c_char_p(0), llama_cpp.c_int(self.n_threads), ): - raise RuntimeError(f"Failed to apply LoRA from lora path: {lora_path} to base path: {lora_base}") + raise RuntimeError( + f"Failed to apply LoRA from lora path: {self.lora_path} to base path: {self.lora_base}" + ) if self.verbose: print(llama_cpp.llama_print_system_info().decode("utf-8"), file=sys.stderr) From 207ebbc8dc2e1fe94fa329ac5393d093d30f449e Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 19 Apr 2023 14:02:11 -0400 Subject: [PATCH 08/31] Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 4274722..884e7d7 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 42747220b4cac548b6e3059b66b3e960b517cfa4 +Subproject commit 884e7d7a2bfd7325b107442d6758983f5886ed3d From e4647c75ec49e21fa2146844c6b91faba58c6699 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 19 Apr 2023 15:57:46 -0400 Subject: [PATCH 09/31] Add use_mmap flag to server --- llama_cpp/server/__main__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/llama_cpp/server/__main__.py b/llama_cpp/server/__main__.py index 48481c6..b2ec4de 100644 --- a/llama_cpp/server/__main__.py +++ b/llama_cpp/server/__main__.py @@ -29,9 +29,10 @@ class Settings(BaseSettings): model: str n_ctx: int = 2048 n_batch: int = 8 - n_threads: int = ((os.cpu_count() or 2) // 2) or 1 + n_threads: int = max((os.cpu_count() or 2) // 2, 1) f16_kv: bool = True use_mlock: bool = False # This causes a silent failure on platforms that don't support mlock (e.g. Windows) took forever to figure out... + use_mmap: bool = True embedding: bool = True last_n_tokens_size: int = 64 logits_all: bool = False @@ -54,6 +55,7 @@ llama = llama_cpp.Llama( settings.model, f16_kv=settings.f16_kv, use_mlock=settings.use_mlock, + use_mmap=settings.use_mmap, embedding=settings.embedding, logits_all=settings.logits_all, n_threads=settings.n_threads, From 3d290623f5473929aaa59d80f9cb19d1c84c32bc Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 20 Apr 2023 01:08:15 -0400 Subject: [PATCH 10/31] Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 884e7d7..02d6988 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 884e7d7a2bfd7325b107442d6758983f5886ed3d +Subproject commit 02d6988121510c067e06d498a273a351a888f5b9 From 207adbdf1319302bb7bd6e137ca34d4605b36ec0 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 20 Apr 2023 01:48:24 -0400 Subject: [PATCH 11/31] Bump version --- pyproject.toml | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index aeb5579..80a354c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "llama_cpp_python" -version = "0.1.34" +version = "0.1.35" description = "Python bindings for the llama.cpp library" authors = ["Andrei Betlen "] license = "MIT" diff --git a/setup.py b/setup.py index b0ff844..3a7ee1c 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ setup( description="A Python wrapper for llama.cpp", long_description=long_description, long_description_content_type="text/markdown", - version="0.1.34", + version="0.1.35", author="Andrei Betlen", author_email="abetlen@gmail.com", license="MIT", From ba3959eafd38080f3bf3028746406f350a8ef793 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 20 Apr 2023 05:15:31 -0400 Subject: [PATCH 12/31] Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 02d6988..c8c2c52 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 02d6988121510c067e06d498a273a351a888f5b9 +Subproject commit c8c2c524827be8fd681a63f0e5a697b0bf4c587b From 1eb130a6b2445f4f9a41424362a64c26f3424529 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 21 Apr 2023 17:40:27 -0400 Subject: [PATCH 13/31] Update llama.cpp --- llama_cpp/llama_cpp.py | 9 ++++++--- vendor/llama.cpp | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 78d8e1f..97c6565 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -117,6 +117,8 @@ LLAMA_FTYPE_MOSTLY_Q4_1 = ctypes.c_int(3) # except 1d tensors LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16 = ctypes.c_int( 4 ) # tok_embeddings.weight and output.weight are F16 +LLAMA_FTYPE_MOSTLY_Q4_2 = ctypes.c_int(5) # except 1d tensors +LLAMA_FTYPE_MOSTYL_Q4_3 = ctypes.c_int(6) # except 1d tensors # Functions @@ -169,11 +171,12 @@ _lib.llama_free.restype = None # TODO: not great API - very likely to change # Returns 0 on success -def llama_model_quantize(fname_inp: bytes, fname_out: bytes, itype: c_int) -> c_int: - return _lib.llama_model_quantize(fname_inp, fname_out, itype) +# nthread - how many threads to use. If <=0, will use std::thread::hardware_concurrency(), else the number given +def llama_model_quantize(fname_inp: bytes, fname_out: bytes, ftype: c_int, nthread: c_int) -> c_int: + return _lib.llama_model_quantize(fname_inp, fname_out, ftype, nthread) -_lib.llama_model_quantize.argtypes = [c_char_p, c_char_p, c_int] +_lib.llama_model_quantize.argtypes = [c_char_p, c_char_p, c_int, c_int] _lib.llama_model_quantize.restype = c_int diff --git a/vendor/llama.cpp b/vendor/llama.cpp index c8c2c52..50cb666 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit c8c2c524827be8fd681a63f0e5a697b0bf4c587b +Subproject commit 50cb666b8a2e35a49b08c0f6bc81138c8f6f2ac1 From 643b73e15571ca005f54f81e22120707931408b9 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 21 Apr 2023 19:38:54 -0400 Subject: [PATCH 14/31] Bump version --- pyproject.toml | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 80a354c..c47ab7a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "llama_cpp_python" -version = "0.1.35" +version = "0.1.36" description = "Python bindings for the llama.cpp library" authors = ["Andrei Betlen "] license = "MIT" diff --git a/setup.py b/setup.py index 3a7ee1c..624e12e 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ setup( description="A Python wrapper for llama.cpp", long_description=long_description, long_description_content_type="text/markdown", - version="0.1.35", + version="0.1.36", author="Andrei Betlen", author_email="abetlen@gmail.com", license="MIT", From e99caedbbd59dae4c7d913fd3d1fb6a4998cdb7f Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sat, 22 Apr 2023 19:50:28 -0400 Subject: [PATCH 15/31] Update llama.cpp --- llama_cpp/llama_cpp.py | 39 +++++++++++++++++++++++++++++++++++++-- vendor/llama.cpp | 2 +- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 97c6565..2ffc4c5 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -172,7 +172,9 @@ _lib.llama_free.restype = None # TODO: not great API - very likely to change # Returns 0 on success # nthread - how many threads to use. If <=0, will use std::thread::hardware_concurrency(), else the number given -def llama_model_quantize(fname_inp: bytes, fname_out: bytes, ftype: c_int, nthread: c_int) -> c_int: +def llama_model_quantize( + fname_inp: bytes, fname_out: bytes, ftype: c_int, nthread: c_int +) -> c_int: return _lib.llama_model_quantize(fname_inp, fname_out, ftype, nthread) @@ -187,7 +189,10 @@ _lib.llama_model_quantize.restype = c_int # will be applied on top of the previous one # Returns 0 on success def llama_apply_lora_from_file( - ctx: llama_context_p, path_lora: ctypes.c_char_p, path_base_model: ctypes.c_char_p, n_threads: c_int + ctx: llama_context_p, + path_lora: ctypes.c_char_p, + path_base_model: ctypes.c_char_p, + n_threads: c_int, ) -> c_int: return _lib.llama_apply_lora_from_file(ctx, path_lora, path_base_model, n_threads) @@ -235,6 +240,36 @@ _lib.llama_set_kv_cache.argtypes = [llama_context_p, POINTER(c_uint8), c_size_t, _lib.llama_set_kv_cache.restype = None +# Returns the size in bytes of the state (rng, logits, embedding and kv_cache) +def llama_get_state_size(ctx: llama_context_p) -> c_size_t: + return _lib.llama_get_state_size(ctx) + + +_lib.llama_get_state_size.argtypes = [llama_context_p] +_lib.llama_get_state_size.restype = c_size_t + + +# Copies the state to the specified destination address. +# Destination needs to have allocated enough memory. +# Returns the number of bytes copied +def llama_copy_state_data(ctx: llama_context_p, dest) -> c_size_t: + return _lib.llama_copy_state_data(ctx, dest) + + +_lib.llama_copy_state_data.argtypes = [llama_context_p, POINTER(c_uint8)] +_lib.llama_copy_state_data.restype = c_size_t + + +# Set the state reading from the specified address +# Returns the number of bytes read +def llama_set_state_data(ctx: llama_context_p, src) -> c_size_t: + return _lib.llama_set_state_data(ctx, src) + + +_lib.llama_set_state_data.argtypes = [llama_context_p, POINTER(c_uint8)] +_lib.llama_set_state_data.restype = c_size_t + + # Run the llama inference to obtain the logits and probabilities for the next token. # tokens + n_tokens is the provided batch of new tokens to process # n_past is the number of tokens to use from previous eval calls diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 50cb666..0e018fe 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 50cb666b8a2e35a49b08c0f6bc81138c8f6f2ac1 +Subproject commit 0e018fe008eacebdbcfa2d61b6c988c245c961cd From 723059959302d109f3468f2426b442af1469542e Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sun, 23 Apr 2023 14:53:17 -0400 Subject: [PATCH 16/31] Disable mmap when applying lora weights. Closes #107 --- llama_cpp/llama.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index ea9f0ff..70dcea9 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -79,7 +79,7 @@ class Llama: self.params.f16_kv = f16_kv self.params.logits_all = logits_all self.params.vocab_only = vocab_only - self.params.use_mmap = use_mmap + self.params.use_mmap = use_mmap if lora_path is None else False self.params.use_mlock = use_mlock self.params.embedding = embedding From aa12d8a81f5b2cf6d9b7a037fa69bdec6ca036b1 Mon Sep 17 00:00:00 2001 From: eiery <19350831+eiery@users.noreply.github.com> Date: Sun, 23 Apr 2023 20:56:40 -0400 Subject: [PATCH 17/31] Update llama.py update n_batch default to 512 to match upstream llama.cpp --- llama_cpp/llama.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index ea9f0ff..a414a1c 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -37,7 +37,7 @@ class Llama: use_mlock: bool = False, embedding: bool = False, n_threads: Optional[int] = None, - n_batch: int = 8, + n_batch: int = 512, last_n_tokens_size: int = 64, lora_base: Optional[str] = None, lora_path: Optional[str] = None, From 02cf88131781f4aec6754be1468095ba2c9ea730 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 24 Apr 2023 09:30:10 -0400 Subject: [PATCH 18/31] Update llama.cpp --- llama_cpp/llama_cpp.py | 30 ------------------------------ vendor/llama.cpp | 2 +- 2 files changed, 1 insertion(+), 31 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 2ffc4c5..2b5af66 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -201,25 +201,6 @@ _lib.llama_apply_lora_from_file.argtypes = [llama_context_p, c_char_p, c_char_p, _lib.llama_apply_lora_from_file.restype = c_int -# Returns the KV cache that will contain the context for the -# ongoing prediction with the model. -def llama_get_kv_cache(ctx: llama_context_p): - return _lib.llama_get_kv_cache(ctx) - - -_lib.llama_get_kv_cache.argtypes = [llama_context_p] -_lib.llama_get_kv_cache.restype = POINTER(c_uint8) - - -# Returns the size of the KV cache -def llama_get_kv_cache_size(ctx: llama_context_p) -> c_size_t: - return _lib.llama_get_kv_cache_size(ctx) - - -_lib.llama_get_kv_cache_size.argtypes = [llama_context_p] -_lib.llama_get_kv_cache_size.restype = c_size_t - - # Returns the number of tokens in the KV cache def llama_get_kv_cache_token_count(ctx: llama_context_p) -> c_int: return _lib.llama_get_kv_cache_token_count(ctx) @@ -229,17 +210,6 @@ _lib.llama_get_kv_cache_token_count.argtypes = [llama_context_p] _lib.llama_get_kv_cache_token_count.restype = c_int -# Sets the KV cache containing the current context for the model -def llama_set_kv_cache( - ctx: llama_context_p, kv_cache, n_size: c_size_t, n_token_count: c_int -): - return _lib.llama_set_kv_cache(ctx, kv_cache, n_size, n_token_count) - - -_lib.llama_set_kv_cache.argtypes = [llama_context_p, POINTER(c_uint8), c_size_t, c_int] -_lib.llama_set_kv_cache.restype = None - - # Returns the size in bytes of the state (rng, logits, embedding and kv_cache) def llama_get_state_size(ctx: llama_context_p) -> c_size_t: return _lib.llama_get_state_size(ctx) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 0e018fe..c4fe84f 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 0e018fe008eacebdbcfa2d61b6c988c245c961cd +Subproject commit c4fe84fb0d28851a5c10e5a633f82ae2ba3b7fae From 86f8e5ad9162a57a72d3af598e477d0971e89eb7 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 24 Apr 2023 15:47:54 -0400 Subject: [PATCH 19/31] Refactor internal state for Llama class --- llama_cpp/llama.py | 63 +++++++++++++++++----------------------------- 1 file changed, 23 insertions(+), 40 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 70dcea9..f7a6e9e 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -84,16 +84,9 @@ class Llama: self.params.embedding = embedding self.last_n_tokens_size = last_n_tokens_size - self.last_n_tokens_data = deque( - [llama_cpp.llama_token(0)] * self.last_n_tokens_size, - maxlen=self.last_n_tokens_size, - ) - self.tokens_consumed = 0 - self.tokens: List[llama_cpp.llama_token] = [] self.n_batch = min(n_ctx, n_batch) - self.n_tokens = 0 - self.n_past = 0 - self.all_logits: List[List[float]] = [] # TODO: Use an array instead of a list. + self.eval_tokens: deque[llama_cpp.llama_token] = deque(maxlen=n_ctx) + self.eval_logits: deque[List[float]] = deque(maxlen=n_ctx) ### HACK: This is a hack to work around the fact that the llama.cpp API does not yet support ### saving and restoring state, this allows us to continue a completion if the last @@ -181,14 +174,8 @@ class Llama: def reset(self): """Reset the model state.""" - self.last_n_tokens_data.extend( - [llama_cpp.llama_token(0)] * self.last_n_tokens_size - ) - self.tokens_consumed = 0 - self.tokens.clear() - self.n_tokens = 0 - self.n_past = 0 - self.all_logits.clear() + self.eval_tokens.clear() + self.eval_logits.clear() def eval(self, tokens: Sequence[llama_cpp.llama_token]): """Evaluate a list of tokens. @@ -200,32 +187,25 @@ class Llama: n_ctx = int(llama_cpp.llama_n_ctx(self.ctx)) for i in range(0, len(tokens), self.n_batch): batch = tokens[i : min(len(tokens), i + self.n_batch)] - self.n_past = min(n_ctx - len(batch), self.tokens_consumed) - self.n_tokens = len(batch) + n_past = min(n_ctx - len(batch), len(self.eval_tokens)) + n_tokens = len(batch) return_code = llama_cpp.llama_eval( ctx=self.ctx, tokens=(llama_cpp.llama_token * len(batch))(*batch), - n_tokens=llama_cpp.c_int(self.n_tokens), - n_past=llama_cpp.c_int(self.n_past), + n_tokens=llama_cpp.c_int(n_tokens), + n_past=llama_cpp.c_int(n_past), n_threads=llama_cpp.c_int(self.n_threads), ) if int(return_code) != 0: raise RuntimeError(f"llama_eval returned {return_code}") - self.tokens.extend(batch) - self.last_n_tokens_data.extend(batch) - self.tokens_consumed += len(batch) + self.eval_tokens.extend(batch) if self.params.logits_all: - self.all_logits.extend(self._logits()) - - def _logits(self) -> List[List[float]]: - """Return the logits from the last call to llama_eval.""" - assert self.ctx is not None - n_vocab = llama_cpp.llama_n_vocab(self.ctx) - cols = int(n_vocab) - rows = self.n_tokens if self.params.logits_all else 1 - logits_view = llama_cpp.llama_get_logits(self.ctx) - logits = [[logits_view[i * cols + j] for j in range(cols)] for i in range(rows)] - return logits + n_vocab = llama_cpp.llama_n_vocab(self.ctx) + cols = int(n_vocab) + rows = n_tokens + logits_view = llama_cpp.llama_get_logits(self.ctx) + logits = [[logits_view[i * cols + j] for j in range(cols)] for i in range(rows)] + self.eval_logits.extend(logits) def sample( self, @@ -246,10 +226,13 @@ class Llama: The sampled token. """ assert self.ctx is not None + last_n_tokens_data = [llama_cpp.llama_token(0)] * max( + 0, self.last_n_tokens_size - len(self.eval_tokens) + ) + list(self.eval_tokens)[-self.last_n_tokens_size :] return llama_cpp.llama_sample_top_p_top_k( ctx=self.ctx, last_n_tokens_data=(llama_cpp.llama_token * self.last_n_tokens_size)( - *self.last_n_tokens_data + *last_n_tokens_data ), last_n_tokens_size=llama_cpp.c_int(self.last_n_tokens_size), top_k=llama_cpp.c_int(top_k), @@ -293,13 +276,13 @@ class Llama: if ( reset and self._cache - and len(self.tokens) > 0 - and self.tokens == tokens[: len(self.tokens)] + and len(self.eval_tokens) > 0 + and self.eval_tokens == tokens[: len(self.eval_tokens)] ): if self.verbose: print("generate cache hit", file=sys.stderr) reset = False - tokens = tokens[len(self.tokens) :] + tokens = tokens[len(self.eval_tokens) :] ### if reset: self.reset() @@ -537,7 +520,7 @@ class Llama: ] all_logprobs = [ [Llama.logit_to_logprob(logit) for logit in row] - for row in self.all_logits + for row in self.eval_logits ] for token, token_str, logprobs_token in zip( all_tokens, all_token_strs, all_logprobs From 280a047dd66cf6356690de0e881ddd7a9c88a51a Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 24 Apr 2023 15:52:24 -0400 Subject: [PATCH 20/31] Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index c4fe84f..8a0f867 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit c4fe84fb0d28851a5c10e5a633f82ae2ba3b7fae +Subproject commit 8a0f8673ba1cdc6aa6df27a9fbc698431ca70e8d From c4c332fc51f9dd0fbad813997b8dab4213812b87 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 24 Apr 2023 17:42:09 -0400 Subject: [PATCH 21/31] Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 8a0f867..54bb60e 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 8a0f8673ba1cdc6aa6df27a9fbc698431ca70e8d +Subproject commit 54bb60e26858be251a0eb3cb70f80322aff804a0 From 197cf80601a3ac4efe2ec3dfe17cdc1397b68975 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 24 Apr 2023 17:51:25 -0400 Subject: [PATCH 22/31] Add save/load state api for Llama class --- llama_cpp/llama.py | 43 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index f7a6e9e..c857bbe 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -4,7 +4,7 @@ import uuid import time import math import multiprocessing -from typing import List, Optional, Union, Generator, Sequence, Iterator +from typing import List, Optional, Union, Generator, Sequence, Iterator, Deque from collections import deque from . import llama_cpp @@ -20,6 +20,18 @@ class LlamaCache: pass +class LlamaState: + def __init__( + self, + eval_tokens: Deque[llama_cpp.llama_token], + eval_logits: Deque[List[float]], + llama_state, + ): + self.eval_tokens = eval_tokens + self.eval_logits = eval_logits + self.llama_state = llama_state + + class Llama: """High-level Python wrapper for a llama.cpp model.""" @@ -85,8 +97,8 @@ class Llama: self.last_n_tokens_size = last_n_tokens_size self.n_batch = min(n_ctx, n_batch) - self.eval_tokens: deque[llama_cpp.llama_token] = deque(maxlen=n_ctx) - self.eval_logits: deque[List[float]] = deque(maxlen=n_ctx) + self.eval_tokens: Deque[llama_cpp.llama_token] = deque(maxlen=n_ctx) + self.eval_logits: Deque[List[float]] = deque(maxlen=n_ctx) ### HACK: This is a hack to work around the fact that the llama.cpp API does not yet support ### saving and restoring state, this allows us to continue a completion if the last @@ -204,7 +216,10 @@ class Llama: cols = int(n_vocab) rows = n_tokens logits_view = llama_cpp.llama_get_logits(self.ctx) - logits = [[logits_view[i * cols + j] for j in range(cols)] for i in range(rows)] + logits = [ + [logits_view[i * cols + j] for j in range(cols)] + for i in range(rows) + ] self.eval_logits.extend(logits) def sample( @@ -828,6 +843,26 @@ class Llama: verbose=state["verbose"], ) + def save_state(self) -> LlamaState: + assert self.ctx is not None + state_size = llama_cpp.llama_get_state_size(self.ctx) + llama_state = (llama_cpp.c_uint8 * int(state_size))() + if llama_cpp.llama_copy_state_data(self.ctx, llama_state) != state_size: + raise RuntimeError("Failed to copy llama state data") + return LlamaState( + eval_tokens=self.eval_tokens.copy(), + eval_logits=self.eval_logits.copy(), + llama_state=llama_state, + ) + + def load_state(self, state: LlamaState) -> None: + assert self.ctx is not None + self.eval_tokens = state.eval_tokens.copy() + self.eval_logits = state.eval_logits.copy() + state_size = llama_cpp.llama_get_state_size(self.ctx) + if llama_cpp.llama_set_state_data(self.ctx, state.llama_state) != state_size: + raise RuntimeError("Failed to set llama state data") + @staticmethod def token_eos() -> llama_cpp.llama_token: """Return the end-of-sequence token.""" From cbe95bbb75ba72cbb39308ee645d3bf1e5507a86 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 24 Apr 2023 19:54:41 -0400 Subject: [PATCH 23/31] Add cache implementation using llama state --- llama_cpp/llama.py | 64 +++++++++++++++++++--------------------------- 1 file changed, 26 insertions(+), 38 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index c2d9d10..0a69b2c 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -12,12 +12,22 @@ from .llama_types import * class LlamaCache: - """Cache for a llama.cpp model. + """Cache for a llama.cpp model.""" - NOTE: This implementation currently only tells the Llama class to avoid reprocessing bytes and continue from the last - completion. It does not actually cache the results.""" + def __init__(self): + self.cache_state: Dict[Sequence[llama_cpp.llama_token], "LlamaState"] = dict() - pass + def __getitem__( + self, key: Sequence[llama_cpp.llama_token] + ) -> Optional["LlamaState"]: + return self.cache_state.get(tuple(key), None) + + def __contains__(self, key: Sequence[llama_cpp.llama_token]) -> bool: + return tuple(key) in self.cache_state + + def __setitem__(self, key: Sequence[llama_cpp.llama_token], value: "LlamaState"): + self.cache_state = dict() # NOTE: Currently limit to one cache entry. + self.cache_state[tuple(key)] = value class LlamaState: @@ -100,13 +110,7 @@ class Llama: self.eval_tokens: Deque[llama_cpp.llama_token] = deque(maxlen=n_ctx) self.eval_logits: Deque[List[float]] = deque(maxlen=n_ctx) - ### HACK: This is a hack to work around the fact that the llama.cpp API does not yet support - ### saving and restoring state, this allows us to continue a completion if the last - ### completion_bytes is a prefix to the prompt passed in. However this is actually incorrect - ### because it does not take into account stop tokens which have been processed by the model. - self._completion_bytes: List[bytes] = [] - self._cache: Optional[LlamaCache] = None - ### + self.cache: Optional[LlamaCache] = None self.n_threads = n_threads or max(multiprocessing.cpu_count() // 2, 1) @@ -182,7 +186,7 @@ class Llama: Args: cache: The cache to set. """ - self._cache = cache + self.cache = cache def reset(self): """Reset the model state.""" @@ -287,10 +291,9 @@ class Llama: The generated tokens. """ assert self.ctx is not None - ### HACK + if ( reset - and self._cache and len(self.eval_tokens) > 0 and self.eval_tokens == tokens[: len(self.eval_tokens)] ): @@ -298,7 +301,7 @@ class Llama: print("generate cache hit", file=sys.stderr) reset = False tokens = tokens[len(self.eval_tokens) :] - ### + if reset: self.reset() while True: @@ -415,20 +418,10 @@ class Llama: "logprobs is not supported for models created with logits_all=False" ) - ### HACK - reset: bool = True - _prompt: bytes = prompt.encode("utf-8") - _completion: bytes = b"".join(self._completion_bytes) - if len(_completion) and self._cache and _prompt.startswith(_completion): + if self.cache and prompt_tokens in self.cache: if self.verbose: - print("completion cache hit", file=sys.stderr) - reset = False - _prompt = _prompt[len(_completion) :] - prompt_tokens = self.tokenize(b" " + _prompt) - self._completion_bytes.append(_prompt) - else: - self._completion_bytes = [prompt.encode("utf-8")] - ### + print("cache hit", file=sys.stderr) + self.load_state(self.cache[prompt_tokens]) finish_reason = "length" for token in self.generate( @@ -437,12 +430,16 @@ class Llama: top_p=top_p, temp=temperature, repeat_penalty=repeat_penalty, - reset=reset, ): if token == llama_cpp.llama_token_eos(): text = self.detokenize(completion_tokens) finish_reason = "stop" break + + if self.cache and len(completion_tokens) == 0: + if prompt_tokens not in self.cache: + self.cache[prompt_tokens] = self.save_state() + completion_tokens.append(token) all_text = self.detokenize(completion_tokens) @@ -467,9 +464,6 @@ class Llama: break text = all_text[: len(all_text) - longest] returned_characters += len(text[start:]) - ### HACK - self._completion_bytes.append(text[start:]) - ### yield { "id": completion_id, "object": "text_completion", @@ -491,9 +485,6 @@ class Llama: break if stream: - ### HACK - self._completion_bytes.append(text[returned_characters:]) - ### yield { "id": completion_id, "object": "text_completion", @@ -510,9 +501,6 @@ class Llama: } return - ### HACK - self._completion_bytes.append(text) - ### text_str = text.decode("utf-8") if echo: From b75fa96bf7995e0f252ee7295262fb745f3d8290 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 24 Apr 2023 19:56:57 -0400 Subject: [PATCH 24/31] Update docs --- docs/index.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/index.md b/docs/index.md index 5424e26..c36adff 100644 --- a/docs/index.md +++ b/docs/index.md @@ -105,12 +105,16 @@ python3 setup.py develop - __call__ - create_chat_completion - set_cache + - save_state + - load_state - token_bos - token_eos show_root_heading: true ::: llama_cpp.LlamaCache +::: llama_cpp.LlamaState + ::: llama_cpp.llama_cpp options: show_if_no_docstring: true From d484c5634eed2b65cd6de2b3ff1e606031c1f67b Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 24 Apr 2023 22:18:54 -0400 Subject: [PATCH 25/31] Bugfix: Check cache keys as prefix to prompt tokens --- llama_cpp/llama.py | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 0a69b2c..487f44d 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -4,7 +4,7 @@ import uuid import time import math import multiprocessing -from typing import List, Optional, Union, Generator, Sequence, Iterator, Deque +from typing import List, Optional, Union, Generator, Sequence, Iterator, Deque, Tuple from collections import deque from . import llama_cpp @@ -15,15 +15,34 @@ class LlamaCache: """Cache for a llama.cpp model.""" def __init__(self): - self.cache_state: Dict[Sequence[llama_cpp.llama_token], "LlamaState"] = dict() + self.cache_state: Dict[Tuple[llama_cpp.llama_token, ...], "LlamaState"] = dict() + + def _sorted_keys(self) -> List[Tuple[llama_cpp.llama_token, ...]]: + return [ + key + for _, key in sorted( + ((len(key), key) for key in self.cache_state.keys()), reverse=True + ) + ] + + def _find_key( + self, key: Tuple[llama_cpp.llama_token, ...] + ) -> Optional[Tuple[llama_cpp.llama_token, ...]]: + for k in self._sorted_keys(): + if key[: len(k)] == k: + return k + return None def __getitem__( self, key: Sequence[llama_cpp.llama_token] ) -> Optional["LlamaState"]: - return self.cache_state.get(tuple(key), None) + _key = self._find_key(tuple(key)) + if _key is None: + return None + return self.cache_state[_key] def __contains__(self, key: Sequence[llama_cpp.llama_token]) -> bool: - return tuple(key) in self.cache_state + return self._find_key(tuple(key)) is not None def __setitem__(self, key: Sequence[llama_cpp.llama_token], value: "LlamaState"): self.cache_state = dict() # NOTE: Currently limit to one cache entry. @@ -295,7 +314,7 @@ class Llama: if ( reset and len(self.eval_tokens) > 0 - and self.eval_tokens == tokens[: len(self.eval_tokens)] + and tuple(self.eval_tokens) == tuple(tokens[: len(self.eval_tokens)]) ): if self.verbose: print("generate cache hit", file=sys.stderr) @@ -438,6 +457,8 @@ class Llama: if self.cache and len(completion_tokens) == 0: if prompt_tokens not in self.cache: + if self.verbose: + print("cache miss", file=sys.stderr) self.cache[prompt_tokens] = self.save_state() completion_tokens.append(token) From 9dddb3a607b522b18d254d3bf77e391f290e819e Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 25 Apr 2023 00:19:44 -0400 Subject: [PATCH 26/31] Bump version --- pyproject.toml | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c47ab7a..773d0c2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "llama_cpp_python" -version = "0.1.36" +version = "0.1.37" description = "Python bindings for the llama.cpp library" authors = ["Andrei Betlen "] license = "MIT" diff --git a/setup.py b/setup.py index 624e12e..ed3b48e 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ setup( description="A Python wrapper for llama.cpp", long_description=long_description, long_description_content_type="text/markdown", - version="0.1.36", + version="0.1.37", author="Andrei Betlen", author_email="abetlen@gmail.com", license="MIT", From 848c83dfd0e116b94d595d9303cac5e9e444669a Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 25 Apr 2023 01:36:37 -0400 Subject: [PATCH 27/31] Add FORCE_CMAKE option --- CMakeLists.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 27e06ac..bda2388 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,7 +2,11 @@ cmake_minimum_required(VERSION 3.4...3.22) project(llama_cpp) -if (UNIX) +option(FORCE_CMAKE "Force CMake build of Python bindings" OFF) + +set(FORCE_CMAKE $ENV{FORCE_CMAKE}) + +if (UNIX AND NOT FORCE_CMAKE) add_custom_command( OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp/libllama.so COMMAND make libllama.so From 996e31d861ce5c8cfbefe6af52a3da25cf484454 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 25 Apr 2023 01:37:07 -0400 Subject: [PATCH 28/31] Bump version --- pyproject.toml | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 773d0c2..3e416d0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "llama_cpp_python" -version = "0.1.37" +version = "0.1.38" description = "Python bindings for the llama.cpp library" authors = ["Andrei Betlen "] license = "MIT" diff --git a/setup.py b/setup.py index ed3b48e..20db9a7 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ setup( description="A Python wrapper for llama.cpp", long_description=long_description, long_description_content_type="text/markdown", - version="0.1.37", + version="0.1.38", author="Andrei Betlen", author_email="abetlen@gmail.com", license="MIT", From cc706fb94448f7d9c0db89eebd7188d738c6831d Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 25 Apr 2023 09:00:53 -0400 Subject: [PATCH 29/31] Add ctx check and re-order __init__. Closes #112 --- llama_cpp/llama.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 487f44d..df9a491 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -133,6 +133,9 @@ class Llama: self.n_threads = n_threads or max(multiprocessing.cpu_count() // 2, 1) + self.lora_base = lora_base + self.lora_path = lora_path + if not os.path.exists(model_path): raise ValueError(f"Model path does not exist: {model_path}") @@ -140,8 +143,8 @@ class Llama: self.model_path.encode("utf-8"), self.params ) - self.lora_base = lora_base - self.lora_path = lora_path + assert self.ctx is not None + if self.lora_path: if llama_cpp.llama_apply_lora_from_file( self.ctx, From 3cab3ef4cb1ae39ad19ffe2b58cdf6671dd82e43 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 25 Apr 2023 09:11:32 -0400 Subject: [PATCH 30/31] Update n_batch for server --- llama_cpp/server/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama_cpp/server/__main__.py b/llama_cpp/server/__main__.py index b2ec4de..af6cc38 100644 --- a/llama_cpp/server/__main__.py +++ b/llama_cpp/server/__main__.py @@ -28,7 +28,7 @@ from sse_starlette.sse import EventSourceResponse class Settings(BaseSettings): model: str n_ctx: int = 2048 - n_batch: int = 8 + n_batch: int = 512 n_threads: int = max((os.cpu_count() or 2) // 2, 1) f16_kv: bool = True use_mlock: bool = False # This causes a silent failure on platforms that don't support mlock (e.g. Windows) took forever to figure out... From cbd26fdcc116dc692308f2d262083dfd1ddaa142 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 25 Apr 2023 19:03:41 -0400 Subject: [PATCH 31/31] Update llama.cpp --- llama_cpp/llama_cpp.py | 1 + vendor/llama.cpp | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 2b5af66..1097d74 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -119,6 +119,7 @@ LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16 = ctypes.c_int( ) # tok_embeddings.weight and output.weight are F16 LLAMA_FTYPE_MOSTLY_Q4_2 = ctypes.c_int(5) # except 1d tensors LLAMA_FTYPE_MOSTYL_Q4_3 = ctypes.c_int(6) # except 1d tensors +LLAMA_FTYPE_MOSTYL_Q8_0 = ctypes.c_int(7) # except 1d tensors # Functions diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 54bb60e..4afcc37 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 54bb60e26858be251a0eb3cb70f80322aff804a0 +Subproject commit 4afcc378698e057fcde64e23eb664e5af8dd6956