21 lines
577 B
YAML
21 lines
577 B
YAML
version: '3.8'
|
|
services:
|
|
llamacpp:
|
|
container_name: llamacpp
|
|
restart: unless-stopped
|
|
image: ghcr.io/ggerganov/llama.cpp:server-cuda # artifactory.turkcell.com.tr/local-docker-dist-dev/com/turkcell/sensai/pandasai/pandas-platform-base:0.0.3
|
|
runtime: nvidia
|
|
command: [
|
|
"-m", "models/Meta-Llama-3-8B.fp16.gguf",
|
|
"-c", "4096",
|
|
"--host", "0.0.0.0",
|
|
"--port", "8000",
|
|
"--n-gpu-layers", "99"
|
|
]
|
|
volumes:
|
|
- /media/raid/llms/llama3:/models
|
|
ports:
|
|
- "8000:8000"
|
|
environment:
|
|
NVIDIA_VISIBLE_DEVICES: all
|