alihan/llama-cpp-python
1
0
Fork 0
mirror of https://github.com/abetlen/llama-cpp-python.git synced 2023-09-07 17:34:22 +03:00
Code Issues Packages Projects Releases Wiki Activity

Implement logprobs parameter for text completion. Closes #2

Browse Source
This commit is contained in:
Andrei Betlen
2023-04-12 14:05:11 -04:00
parent 2a60eb820f
commit b3805bb9cc
2 changed files with 111 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
llama_cpp/server/__main__.py
View File

@@ -33,6 +33,7 @@ class Settings(BaseSettings):
use_mlock: bool = False # This causes a silent failure on platforms that don't support mlock (e.g. Windows) took forever to figure out...
embedding: bool = True
last_n_tokens_size: int = 64
logits_all: bool = False
app = FastAPI(
@@ -52,6 +53,7 @@ llama = llama_cpp.Llama(
f16_kv=settings.f16_kv,
use_mlock=settings.use_mlock,
embedding=settings.embedding,
logits_all=settings.logits_all,
n_threads=settings.n_threads,
n_batch=settings.n_batch,
n_ctx=settings.n_ctx,