tests: simple test for server module

This commit is contained in:
Lucas Doyle
2023-04-28 23:26:07 -07:00
parent 468377b0e2
commit 6d8db9d017
4 changed files with 117 additions and 2 deletions

View File

@@ -24,6 +24,7 @@ class Settings(BaseSettings):
last_n_tokens_size: int = 64
logits_all: bool = False
cache: bool = False # WARNING: This is an experimental feature
vocab_only: bool = False
app = FastAPI(
@@ -49,6 +50,7 @@ llama = llama_cpp.Llama(
n_batch=settings.n_batch,
n_ctx=settings.n_ctx,
last_n_tokens_size=settings.last_n_tokens_size,
vocab_only=settings.vocab_only,
)
if settings.cache:
cache = llama_cpp.LlamaCache()