From 84e723653ca99d51a74b454984acf2c077468561 Mon Sep 17 00:00:00 2001 From: FK Date: Wed, 13 Sep 2023 08:50:46 +0200 Subject: speculative: add --n-gpu-layers-draft option (#3063) --- examples/speculative/speculative.cpp | 1 + 1 file changed, 1 insertion(+) (limited to 'examples') diff --git a/examples/speculative/speculative.cpp b/examples/speculative/speculative.cpp index 822d7b52..2cd153f9 100644 --- a/examples/speculative/speculative.cpp +++ b/examples/speculative/speculative.cpp @@ -42,6 +42,7 @@ int main(int argc, char ** argv) { // load the draft model params.model = params.model_draft; + params.n_gpu_layers = params.n_gpu_layers_draft; std::tie(model_dft, ctx_dft) = llama_init_from_gpt_params(params); // tokenize the prompt -- cgit v1.2.3