convert : add BPE pre-tokenization for DBRX (#7132)

* Add BPE pre-tokenization for DBRX.

* Add vocab GGUFs.

* Remove test.

* Remove GGUFs.

1 files changed, 1 insertions, 0 deletions

diff --git a/llama.h b/llama.h
index 5761cacf..388839f3 100644
--- a/llama.h
+++ b/llama.h
@@ -82,6 +82,7 @@ extern "C" {
         LLAMA_VOCAB_PRE_TYPE_REFACT         = 8,
         LLAMA_VOCAB_PRE_TYPE_COMMAND_R      = 9,
         LLAMA_VOCAB_PRE_TYPE_OLMO           = 10,
+        LLAMA_VOCAB_PRE_TYPE_DBRX           = 11,
     };
 
     // note: these values should be synchronized with ggml_rope