alihan/llama-cpp-python
1
0
Fork 0
mirror of https://github.com/abetlen/llama-cpp-python.git synced 2023-09-07 17:34:22 +03:00
Code Issues Packages Projects Releases Wiki Activity

Added cache size to settins object.

Browse Source
This commit is contained in:
Andrei Betlen
2023-05-07 19:33:17 -04:00
parent 0e94a70de1
commit 14da46f16e
1 changed files with 5 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
llama_cpp/server/app.py
View File

@@ -45,6 +45,10 @@ class Settings(BaseSettings):
default=False, default=False,
description="Use a cache to reduce processing times for evaluated prompts.", description="Use a cache to reduce processing times for evaluated prompts.",
) )
cache_size: int = Field(
default=2 << 30,
description="The size of the cache in bytes. Only used if cache is True.",
)
vocab_only: bool = Field( vocab_only: bool = Field(
default=False, description="Whether to only return the vocabulary." default=False, description="Whether to only return the vocabulary."
) )
@@ -89,7 +93,7 @@ def create_app(settings: Optional[Settings] = None):
verbose=settings.verbose, verbose=settings.verbose,
) )
if settings.cache: if settings.cache:
cache = llama_cpp.LlamaCache() cache = llama_cpp.LlamaCache(capacity_bytes=settings.cache_size)
llama.set_cache(cache) llama.set_cache(cache)
return app return app