From 911b3900dded9a1cfe0f0e41b82c7a29baf3a217 Mon Sep 17 00:00:00 2001
From: Johan <JohanAR@users.noreply.github.com>
Date: Wed, 8 May 2024 14:27:58 +0200
Subject: server : add_special option for tokenize endpoint (#7059)

---
 examples/server/server.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'examples/server/server.cpp')

diff --git a/examples/server/server.cpp b/examples/server/server.cpp
index 85ae1ad9..06c0be56 100644
--- a/examples/server/server.cpp
+++ b/examples/server/server.cpp
@@ -3647,7 +3647,8 @@ int main(int argc, char ** argv) {
 
         std::vector<llama_token> tokens;
         if (body.count("content") != 0) {
-            tokens = ctx_server.tokenize(body["content"], false);
+            const bool add_special = json_value(body, "add_special", false);
+            tokens = ctx_server.tokenize(body["content"], add_special);
         }
         const json data = format_tokenizer_response(tokens);
         return res.set_content(data.dump(), "application/json; charset=utf-8");
-- 
cgit v1.2.3