alihan/llama-cpp-python
1
0
Fork 0
mirror of https://github.com/abetlen/llama-cpp-python.git synced 2023-09-07 17:34:22 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' of github.com:abetlen/llama_cpp_python into main

Browse Source
This commit is contained in:
Andrei Betlen
2023-04-06 21:07:38 -04:00
parent 55279b679d c2e690b326
commit 930db37dd2
4 changed files with 8 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
llama_cpp/server/__main__.py
View File

@@ -27,10 +27,10 @@ from sse_starlette.sse import EventSourceResponse
class Settings(BaseSettings):
model: str
n_ctx: int = 2048
n_batch: int = 2048
n_threads: int = os.cpu_count() or 1
n_batch: int = 8
n_threads: int = int(os.cpu_count() / 2) or 1
f16_kv: bool = True
use_mlock: bool = True
use_mlock: bool = False # This causes a silent failure on platforms that don't support mlock (e.g. Windows) took forever to figure out...
embedding: bool = True
last_n_tokens_size: int = 64