From 68e210b3543e0cc71268bee0920441747679ee13 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Fri, 22 Mar 2024 13:08:28 +0200 Subject: server : enable continuous batching by default (#6231) --- common/common.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'common/common.h') diff --git a/common/common.h b/common/common.h index 8dd8a3ed..31fd401b 100644 --- a/common/common.h +++ b/common/common.h @@ -139,7 +139,7 @@ struct gpt_params { bool interactive_first = false; // wait for user input immediately bool multiline_input = false; // reverse the usage of `\` bool simple_io = false; // improves compatibility with subprocesses and limited consoles - bool cont_batching = false; // insert new sequences for decoding on-the-fly + bool cont_batching = true; // insert new sequences for decoding on-the-fly bool input_prefix_bos = false; // prefix BOS to user inputs, preceding input_prefix bool ignore_eos = false; // ignore generated EOS tokens -- cgit v1.2.3