Fix for #3454 (#3455)

Fix: `sentencepiece` tokenizers with added tokens failed with an incorrect assertion

1 files changed, 6 insertions, 2 deletions

diff --git a/llama.cpp b/llama.cpp
index 79ea2b23..89dc403b 100644
--- a/llama.cpp
+++ b/llama.cpp
@@ -8200,7 +8200,9 @@ int llama_token_to_piece(const struct llama_model * model, llama_token token, ch
                 buf[0] = llama_token_to_byte(model->vocab, token);
                 return 1;
             } else {
-                GGML_ASSERT(false);
+                // TODO: for now we accept all unsupported token types,
+                // suppressing them like CONTROL tokens.
+                // GGML_ASSERT(false);
             }
             break;
         }
@@ -8216,7 +8218,9 @@ int llama_token_to_piece(const struct llama_model * model, llama_token token, ch
             } else if (llama_is_control_token(model->vocab, token)) {
                 ;
             } else {
-                GGML_ASSERT(false);
+                // TODO: for now we accept all unsupported token types,
+                // suppressing them like CONTROL tokens.
+                // GGML_ASSERT(false);
             }
             break;
         }