Reset schedule earlier to allow overlap with ggml graph computation on device (#6933)

* Reset schedule earlier to allow overlap with graph computation on device

1 files changed, 4 insertions, 0 deletions

diff --git a/llama.cpp b/llama.cpp
index dd8b1f26..49f2b559 100644
--- a/llama.cpp
+++ b/llama.cpp
@@ -11473,6 +11473,10 @@ static int llama_decode_internal(
         }
     }
 
+    // Reset state for the next token before backend sync, to allow the CPU activities in the reset to
+    // overlap with device computation.
+    ggml_backend_sched_reset(lctx.sched);
+
     return 0;
 }