alihan/llama-cpp-python
1
0
Fork 0
mirror of https://github.com/abetlen/llama-cpp-python.git synced 2023-09-07 17:34:22 +03:00
Code Issues Packages Projects Releases Wiki Activity

Temporarily disable cache until save state bug is fixed.

Browse Source
This commit is contained in:
Andrei Betlen
2023-06-09 11:10:24 -04:00
parent be0403da98
commit 0da655b3be
2 changed files with 7 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
llama_cpp/llama.py
View File

@@ -831,7 +831,9 @@ class Llama:
"logprobs is not supported for models created with logits_all=False"
)
if self.cache:
# Temporarily disable usage of the cache
# See: https://github.com/abetlen/llama-cpp-python/issues/348#issuecomment-1583072408
if self.cache and False:
try:
cache_item = self.cache[prompt_tokens]
cache_prefix_len = Llama.longest_token_prefix(
@@ -1069,14 +1071,14 @@ class Llama:
}
],
}
if self.cache:
if self.cache and False:
if self.verbose:
print("Llama._create_completion: cache save", file=sys.stderr)
self.cache[prompt_tokens + completion_tokens] = self.save_state()
print("Llama._create_completion: cache saved", file=sys.stderr)
return
if self.cache:
if self.cache and False:
if self.verbose:
print("Llama._create_completion: cache save", file=sys.stderr)
self.cache[prompt_tokens + completion_tokens] = self.save_state()