diff options
| author | Georgi Gerganov <ggerganov@gmail.com> | 2024-03-22 13:08:28 +0200 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2024-03-22 13:08:28 +0200 |
| commit | 68e210b3543e0cc71268bee0920441747679ee13 (patch) | |
| tree | a6c4d546faad2bf5b2921978f5f583fdb52cdd19 /common/common.h | |
| parent | b3e94f26bab8e3b5338b5902e5af5bb01894cb4a (diff) | |
server : enable continuous batching by default (#6231)
Diffstat (limited to 'common/common.h')
| -rw-r--r-- | common/common.h | 2 |
1 files changed, 1 insertions, 1 deletions
diff --git a/common/common.h b/common/common.h index 8dd8a3ed..31fd401b 100644 --- a/common/common.h +++ b/common/common.h @@ -139,7 +139,7 @@ struct gpt_params { bool interactive_first = false; // wait for user input immediately bool multiline_input = false; // reverse the usage of `\` bool simple_io = false; // improves compatibility with subprocesses and limited consoles - bool cont_batching = false; // insert new sequences for decoding on-the-fly + bool cont_batching = true; // insert new sequences for decoding on-the-fly bool input_prefix_bos = false; // prefix BOS to user inputs, preceding input_prefix bool ignore_eos = false; // ignore generated EOS tokens |