From 52399254b3bceda279b4ea9111a983e32310166e Mon Sep 17 00:00:00 2001
From: Georgi Gerganov <ggerganov@gmail.com>
Date: Sun, 16 Jun 2024 14:51:40 +0300
Subject: unicode : avoid char32_t (#7957)

ggml-ci
---
 llama.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'llama.cpp')

diff --git a/llama.cpp b/llama.cpp
index 3bf9b668..bd4f8ec1 100644
--- a/llama.cpp
+++ b/llama.cpp
@@ -13246,7 +13246,7 @@ struct llm_tokenizer_wpm {
         const std::vector<uint32_t> cpts_nfd = unicode_cpts_normalize_nfd(unicode_cpts_from_utf8(text));
         std::vector<std::string> words(1, "");
 
-        for (const char32_t cpt : cpts_nfd) {
+        for (const uint32_t cpt : cpts_nfd) {
             const auto flags = unicode_cpt_flags(cpt);
 
             if (flags.is_whitespace) {
-- 
cgit v1.2.3