diff options
| author | Kerfuffle <44031344+KerfuffleV2@users.noreply.github.com> | 2023-11-18 14:48:17 -0700 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2023-11-18 14:48:17 -0700 |
| commit | 28a2e6e7d476717881be6eb9e2d3331342cec57b (patch) | |
| tree | 19c9ea98490818264fd3c585492002eaca9eadae /examples/tokenize/tokenize.cpp | |
| parent | 0b5c3b04572a05f80163a365070fb377a837ac27 (diff) | |
tokenize example: Respect normal add BOS token behavior (#4126)
Allow building with Makefile
Diffstat (limited to 'examples/tokenize/tokenize.cpp')
| -rw-r--r-- | examples/tokenize/tokenize.cpp | 2 |
1 files changed, 1 insertions, 1 deletions
diff --git a/examples/tokenize/tokenize.cpp b/examples/tokenize/tokenize.cpp index 17166836..4ff8e3fa 100644 --- a/examples/tokenize/tokenize.cpp +++ b/examples/tokenize/tokenize.cpp @@ -26,7 +26,7 @@ int main(int argc, char ** argv) { llama_context_params ctx_params = llama_context_default_params(); llama_context * ctx = llama_new_context_with_model(model, ctx_params); - const bool add_bos = true; + const bool add_bos = llama_should_add_bos_token(model); std::vector<llama_token> tokens; |