From 4cd621c26de2095cd7c4464bdec5fe2e696ef3f3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?DAN=E2=84=A2?= <dranger003@gmail.com>
Date: Wed, 8 May 2024 06:43:23 -0400
Subject: convert : add BPE pre-tokenization for DBRX (#7132)

* Add BPE pre-tokenization for DBRX.

* Add vocab GGUFs.

* Remove test.

* Remove GGUFs.
---
 llama.h | 1 +
 1 file changed, 1 insertion(+)

(limited to 'llama.h')

diff --git a/llama.h b/llama.h
index 5761cacf..388839f3 100644
--- a/llama.h
+++ b/llama.h
@@ -82,6 +82,7 @@ extern "C" {
         LLAMA_VOCAB_PRE_TYPE_REFACT         = 8,
         LLAMA_VOCAB_PRE_TYPE_COMMAND_R      = 9,
         LLAMA_VOCAB_PRE_TYPE_OLMO           = 10,
+        LLAMA_VOCAB_PRE_TYPE_DBRX           = 11,
     };
 
     // note: these values should be synchronized with ggml_rope
-- 
cgit v1.2.3