summaryrefslogtreecommitdiff
path: root/src/unicode.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'src/unicode.cpp')
-rw-r--r--src/unicode.cpp179
1 files changed, 109 insertions, 70 deletions
diff --git a/src/unicode.cpp b/src/unicode.cpp
index cfffde0d..a57456ea 100644
--- a/src/unicode.cpp
+++ b/src/unicode.cpp
@@ -18,6 +18,7 @@
#include <vector>
#include <locale>
#include <codecvt>
+#include <iostream>
size_t unicode_len_utf8(char src) {
const size_t lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4 };
@@ -25,7 +26,7 @@ size_t unicode_len_utf8(char src) {
return lookup[highbits];
}
-static std::string unicode_cpts_to_utf8(const std::vector<uint32_t> & cps) {
+static std::string unicode_cpts_to_utf8(const std::vector<uint32_t>& cps) {
std::string result;
for (size_t i = 0; i < cps.size(); ++i) {
result.append(unicode_cpt_to_utf8(cps[i]));
@@ -33,7 +34,7 @@ static std::string unicode_cpts_to_utf8(const std::vector<uint32_t> & cps) {
return result;
}
-uint32_t unicode_cpt_from_utf8(const std::string & utf8, size_t & offset) {
+uint32_t unicode_cpt_from_utf8(const std::string& utf8, size_t& offset) {
assert(offset < utf8.size());
if (!(utf8[offset + 0] & 0x80)) {
auto result = utf8[offset + 0];
@@ -44,7 +45,7 @@ uint32_t unicode_cpt_from_utf8(const std::string & utf8, size_t & offset) {
throw std::invalid_argument("invalid character");
}
if (!(utf8[offset + 0] & 0x20)) {
- if (offset + 1 >= utf8.size() || ! ((utf8[offset + 1] & 0xc0) == 0x80)) {
+ if (offset + 1 >= utf8.size() || !((utf8[offset + 1] & 0xc0) == 0x80)) {
throw std::invalid_argument("invalid character");
}
auto result = ((utf8[offset + 0] & 0x1f) << 6) | (utf8[offset + 1] & 0x3f);
@@ -52,7 +53,7 @@ uint32_t unicode_cpt_from_utf8(const std::string & utf8, size_t & offset) {
return result;
}
if (!(utf8[offset + 0] & 0x10)) {
- if (offset + 2 >= utf8.size() || ! ((utf8[offset + 1] & 0xc0) == 0x80) || ! ((utf8[offset + 2] & 0xc0) == 0x80)) {
+ if (offset + 2 >= utf8.size() || !((utf8[offset + 1] & 0xc0) == 0x80) || !((utf8[offset + 2] & 0xc0) == 0x80)) {
throw std::invalid_argument("invalid character");
}
auto result = ((utf8[offset + 0] & 0x0f) << 12) | ((utf8[offset + 1] & 0x3f) << 6) | (utf8[offset + 2] & 0x3f);
@@ -60,7 +61,7 @@ uint32_t unicode_cpt_from_utf8(const std::string & utf8, size_t & offset) {
return result;
}
if (!(utf8[offset + 0] & 0x08)) {
- if (offset + 3 >= utf8.size() || ! ((utf8[offset + 1] & 0xc0) == 0x80) || ! ((utf8[offset + 2] & 0xc0) == 0x80) || !((utf8[offset + 3] & 0xc0) == 0x80)) {
+ if (offset + 3 >= utf8.size() || !((utf8[offset + 1] & 0xc0) == 0x80) || !((utf8[offset + 2] & 0xc0) == 0x80) || !((utf8[offset + 3] & 0xc0) == 0x80)) {
throw std::invalid_argument("invalid character");
}
auto result = ((utf8[offset + 0] & 0x07) << 18) | ((utf8[offset + 1] & 0x3f) << 12) | ((utf8[offset + 2] & 0x3f) << 6) | (utf8[offset + 3] & 0x3f);
@@ -122,10 +123,10 @@ uint32_t unicode_cpt_from_utf8(const std::string & utf8, size_t & offset) {
static std::vector<codepoint_flags> unicode_cpt_flags_array() {
std::vector<codepoint_flags> cpt_flags(MAX_CODEPOINTS, codepoint_flags::UNDEFINED);
- assert (unicode_ranges_flags.front().first == 0);
- assert (unicode_ranges_flags.back().first == MAX_CODEPOINTS);
+ assert(unicode_ranges_flags.front().first == 0);
+ assert(unicode_ranges_flags.back().first == MAX_CODEPOINTS);
for (size_t i = 1; i < unicode_ranges_flags.size(); ++i) {
- const auto range_ini = unicode_ranges_flags[i-1]; // codepoint_ini, flags
+ const auto range_ini = unicode_ranges_flags[i - 1]; // codepoint_ini, flags
const auto range_end = unicode_ranges_flags[i]; // codepoint_end, flags
for (uint32_t cpt = range_ini.first; cpt < range_end.first; ++cpt) {
cpt_flags[cpt] = range_ini.second;
@@ -144,7 +145,7 @@ static std::vector<codepoint_flags> unicode_cpt_flags_array() {
cpt_flags[p.second].is_uppercase = true;
}
- for (auto &range : unicode_ranges_nfd) { // start, last, nfd
+ for (auto& range : unicode_ranges_nfd) { // start, last, nfd
cpt_flags[range.nfd].is_nfd = true;
}
@@ -199,22 +200,55 @@ static std::unordered_map<std::string, uint8_t> unicode_utf8_to_byte_map() {
return map;
}
-static inline std::wstring unicode_wstring_from_utf8(const std::string & s) {
+static inline bool is_valid_utf8(const std::string& str) {
+ int remaining_bytes = 0; // 当前多字节字符剩余的字节数
+ for (unsigned char c : str) {
+ if (remaining_bytes == 0) {
+ if ((c & 0x80) == 0x00) continue; // 1字节字符
+ else if ((c & 0xE0) == 0xC0) remaining_bytes = 1; // 2字节
+ else if ((c & 0xF0) == 0xE0) remaining_bytes = 2; // 3字节
+ else if ((c & 0xF8) == 0xF0) remaining_bytes = 3; // 4字节
+ else return false; // 非法起始字节
+ }
+ else {
+ // 检查后续字节是否为10xxxxxx
+ if ((c & 0xC0) != 0x80)
+ {
+ return false;
+ }
+ remaining_bytes--;
+ }
+ }
+ return (remaining_bytes == 0); // 确保多字节字符完整
+}
+
+static inline std::wstring unicode_wstring_from_utf8(const std::string& s) {
+#if defined(__clang__)
+ // disable C++17 deprecation warning for std::codecvt_utf8
+# pragma clang diagnostic push
+# pragma clang diagnostic ignored "-Wdeprecated-declarations"
+#endif
+ bool isvalid = is_valid_utf8(s);
std::wstring_convert<std::codecvt_utf8<wchar_t>> conv;
+
+#if defined(__clang__)
+# pragma clang diagnostic pop
+#endif
+
return conv.from_bytes(s);
}
-static std::vector<std::string> unicode_byte_encoding_process(const std::vector<std::string> & bpe_words) {
+static std::vector<std::string> unicode_byte_encoding_process(const std::vector<std::string>& bpe_words) {
std::vector<std::string> bpe_encoded_words;
- for (const auto & word : bpe_words) {
+ for (const auto& word : bpe_words) {
std::string text_utf;
- auto utf_word = unicode_cpts_from_utf8(word);
+ auto utf_word = unicode_cpts_from_utf8(word);
for (size_t i = 0; i < utf_word.size(); ++i) {
text_utf += unicode_cpt_to_utf8(utf_word[i]);
}
std::string encoded_token;
- for (char & c : text_utf) {
+ for (char& c : text_utf) {
encoded_token += unicode_byte_to_utf8(c);
}
bpe_encoded_words.emplace_back(encoded_token);
@@ -223,7 +257,7 @@ static std::vector<std::string> unicode_byte_encoding_process(const std::vector<
}
// GPT2 system regex: 's|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+
-static std::vector<size_t> unicode_regex_split_custom_gpt2(const std::string & text, const std::vector<size_t> & offsets) {
+static std::vector<size_t> unicode_regex_split_custom_gpt2(const std::string& text, const std::vector<size_t>& offsets) {
std::vector<size_t> bpe_offsets; // store the offset of each word
bpe_offsets.reserve(offsets.size()); // Reserve memory for the approximate size
@@ -237,16 +271,16 @@ static std::vector<size_t> unicode_regex_split_custom_gpt2(const std::string & t
start = offset_end;
static const uint32_t OUT_OF_RANGE = 0xFFFFFFFF;
- auto _get_cpt = [&] (const size_t pos) -> uint32_t {
+ auto _get_cpt = [&](const size_t pos) -> uint32_t {
return (offset_ini <= pos && pos < offset_end) ? cpts[pos] : OUT_OF_RANGE;
};
- auto _get_flags = [&] (const size_t pos) -> codepoint_flags {
+ auto _get_flags = [&](const size_t pos) -> codepoint_flags {
return (offset_ini <= pos && pos < offset_end) ? unicode_cpt_flags(cpts[pos]) : codepoint_flags{};
};
size_t _prev_end = offset_ini;
- auto _add_token = [&] (const size_t end) -> size_t {
+ auto _add_token = [&](const size_t end) -> size_t {
assert(_prev_end <= end && end <= offset_end);
size_t len = end - _prev_end;
if (len > 0) {
@@ -262,29 +296,29 @@ static std::vector<size_t> unicode_regex_split_custom_gpt2(const std::string & t
return len;
};
- for (size_t pos = offset_ini; pos < offset_end; /*pos++*/ ) {
+ for (size_t pos = offset_ini; pos < offset_end; /*pos++*/) {
const uint32_t cpt = _get_cpt(pos);
const auto flags = _get_flags(pos);
// regex: 's|'t|'re|'ve|'m|'ll|'d
- if (cpt == '\'' && pos+1 < offset_end) {
- uint32_t cpt_next = _get_cpt(pos+1);
+ if (cpt == '\'' && pos + 1 < offset_end) {
+ uint32_t cpt_next = _get_cpt(pos + 1);
if (cpt_next == 's' || cpt_next == 't' || cpt_next == 'm' || cpt_next == 'd') {
- pos += _add_token(pos+2);
+ pos += _add_token(pos + 2);
continue;
}
- if (pos+2 < offset_end) {
- uint32_t cpt_next_next = _get_cpt(pos+2);
+ if (pos + 2 < offset_end) {
+ uint32_t cpt_next_next = _get_cpt(pos + 2);
if ((cpt_next == 'r' && cpt_next_next == 'e') ||
(cpt_next == 'v' && cpt_next_next == 'e') ||
(cpt_next == 'l' && cpt_next_next == 'l')) {
- pos += _add_token(pos+3);
+ pos += _add_token(pos + 3);
continue;
}
}
}
- auto flags2 = (cpt == ' ' ? _get_flags(pos+1) : flags);
+ auto flags2 = (cpt == ' ' ? _get_flags(pos + 1) : flags);
// regex: <space>?\p{L}+
if (flags2.is_letter) {
pos += (cpt == ' ');
@@ -314,12 +348,12 @@ static std::vector<size_t> unicode_regex_split_custom_gpt2(const std::string & t
}
size_t num_whitespaces = 0;
- while (_get_flags(pos+num_whitespaces).is_whitespace) {
+ while (_get_flags(pos + num_whitespaces).is_whitespace) {
num_whitespaces++;
}
// regex: \s+(?!\S)
- if (num_whitespaces > 1 && _get_cpt(pos+num_whitespaces) != OUT_OF_RANGE) {
+ if (num_whitespaces > 1 && _get_cpt(pos + num_whitespaces) != OUT_OF_RANGE) {
pos += num_whitespaces - 1;
_add_token(pos);
continue;
@@ -341,7 +375,7 @@ static std::vector<size_t> unicode_regex_split_custom_gpt2(const std::string & t
}
// LLAMA3 system regex: "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+"
-static std::vector<size_t> unicode_regex_split_custom_llama3(const std::string & text, const std::vector<size_t> & offsets) {
+static std::vector<size_t> unicode_regex_split_custom_llama3(const std::string& text, const std::vector<size_t>& offsets) {
std::vector<size_t> bpe_offsets; // store the offset of each word
bpe_offsets.reserve(offsets.size()); // Reserve memory for the approximate size
@@ -355,16 +389,16 @@ static std::vector<size_t> unicode_regex_split_custom_llama3(const std::string &
start = offset_end;
static const uint32_t OUT_OF_RANGE = 0xFFFFFFFF;
- auto _get_cpt = [&] (const size_t pos) -> uint32_t {
+ auto _get_cpt = [&](const size_t pos) -> uint32_t {
return (offset_ini <= pos && pos < offset_end) ? cpts[pos] : OUT_OF_RANGE;
};
- auto _get_flags = [&] (const size_t pos) -> codepoint_flags {
+ auto _get_flags = [&](const size_t pos) -> codepoint_flags {
return (offset_ini <= pos && pos < offset_end) ? unicode_cpt_flags(cpts[pos]) : codepoint_flags{};
};
size_t _prev_end = offset_ini;
- auto _add_token = [&] (const size_t end) -> size_t {
+ auto _add_token = [&](const size_t end) -> size_t {
assert(_prev_end <= end && end <= offset_end);
size_t len = end - _prev_end;
if (len > 0) {
@@ -380,23 +414,23 @@ static std::vector<size_t> unicode_regex_split_custom_llama3(const std::string &
return len;
};
- for (size_t pos = offset_ini; pos < offset_end; /*pos++*/ ) {
+ for (size_t pos = offset_ini; pos < offset_end; /*pos++*/) {
const uint32_t cpt = _get_cpt(pos);
const auto flags = _get_flags(pos);
// regex: (?i:'s|'t|'re|'ve|'m|'ll|'d) // case insensitive
- if (cpt == '\'' && pos+1 < offset_end) {
- uint32_t cpt_next = unicode_tolower(_get_cpt(pos+1));
+ if (cpt == '\'' && pos + 1 < offset_end) {
+ uint32_t cpt_next = unicode_tolower(_get_cpt(pos + 1));
if (cpt_next == 's' || cpt_next == 't' || cpt_next == 'm' || cpt_next == 'd') {
- pos += _add_token(pos+2);
+ pos += _add_token(pos + 2);
continue;
}
- if (pos+2 < offset_end) {
- uint32_t cpt_next_next = unicode_tolower(_get_cpt(pos+2));
+ if (pos + 2 < offset_end) {
+ uint32_t cpt_next_next = unicode_tolower(_get_cpt(pos + 2));
if ((cpt_next == 'r' && cpt_next_next == 'e') ||
(cpt_next == 'v' && cpt_next_next == 'e') ||
(cpt_next == 'l' && cpt_next_next == 'l')) {
- pos += _add_token(pos+3);
+ pos += _add_token(pos + 3);
continue;
}
}
@@ -404,7 +438,7 @@ static std::vector<size_t> unicode_regex_split_custom_llama3(const std::string &
// regex: [^\r\n\p{L}\p{N}]?\p{L}+
if (!(cpt == '\r' || cpt == '\n' || flags.is_number)) {
- if (flags.is_letter || _get_flags(pos+1).is_letter) { // one or more letters
+ if (flags.is_letter || _get_flags(pos + 1).is_letter) { // one or more letters
pos++;
while (_get_flags(pos).is_letter) {
pos++;
@@ -418,7 +452,7 @@ static std::vector<size_t> unicode_regex_split_custom_llama3(const std::string &
if (flags.is_number) {
size_t ini = pos;
while (_get_flags(pos).is_number) {
- if (++pos - ini >= 3 ) {
+ if (++pos - ini >= 3) {
_add_token(pos);
ini = pos;
}
@@ -428,7 +462,7 @@ static std::vector<size_t> unicode_regex_split_custom_llama3(const std::string &
}
// regex: <space>?[^\s\p{L}\p{N}]+[\r\n]*
- auto flags2 = (cpt == ' ' ? _get_flags(pos+1) : flags);
+ auto flags2 = (cpt == ' ' ? _get_flags(pos + 1) : flags);
if (!(flags2.is_whitespace | flags2.is_letter | flags2.is_number) && flags.as_uint()) {
pos += (cpt == ' ');
while (!(flags2.is_whitespace | flags2.is_letter | flags2.is_number) && flags2.as_uint()) {
@@ -444,8 +478,8 @@ static std::vector<size_t> unicode_regex_split_custom_llama3(const std::string &
size_t num_whitespaces = 0;
size_t last_end_r_or_n = 0;
- while (_get_flags(pos+num_whitespaces).is_whitespace) {
- uint32_t cpt2 = _get_cpt(pos+num_whitespaces);
+ while (_get_flags(pos + num_whitespaces).is_whitespace) {
+ uint32_t cpt2 = _get_cpt(pos + num_whitespaces);
if (cpt2 == '\r' || cpt2 == '\n') {
last_end_r_or_n = pos + num_whitespaces + 1;
}
@@ -460,7 +494,7 @@ static std::vector<size_t> unicode_regex_split_custom_llama3(const std::string &
}
// regex: \s+(?!\S)
- if (num_whitespaces > 1 && _get_cpt(pos+num_whitespaces) != OUT_OF_RANGE) {
+ if (num_whitespaces > 1 && _get_cpt(pos + num_whitespaces) != OUT_OF_RANGE) {
pos += num_whitespaces - 1;
_add_token(pos);
continue;
@@ -482,7 +516,7 @@ static std::vector<size_t> unicode_regex_split_custom_llama3(const std::string &
}
// use std::wregex to split the text
-static std::vector<size_t> unicode_regex_split_stl(const std::wstring & wtext, const std::wstring & regex_expr, const std::vector<size_t> & offsets) {
+static std::vector<size_t> unicode_regex_split_stl(const std::wstring& wtext, const std::wstring& regex_expr, const std::vector<size_t>& offsets) {
std::wregex expr(regex_expr);
std::vector<size_t> bpe_offsets; // store the offset of each word
bpe_offsets.reserve(offsets.size()); // Reserve memory for the approximate size
@@ -502,7 +536,7 @@ static std::vector<size_t> unicode_regex_split_stl(const std::wstring & wtext, c
++it;
}
- if (start_idx < (int64_t) offset) {
+ if (start_idx < (int64_t)offset) {
bpe_offsets.emplace_back(offset - start_idx);
}
start += offset;
@@ -512,7 +546,7 @@ static std::vector<size_t> unicode_regex_split_stl(const std::wstring & wtext, c
}
// use std::regex to split the text
-static std::vector<size_t> unicode_regex_split_stl(const std::string & text, const std::string & regex_expr, const std::vector<size_t> & offsets) {
+static std::vector<size_t> unicode_regex_split_stl(const std::string& text, const std::string& regex_expr, const std::vector<size_t>& offsets) {
std::regex expr(regex_expr);
std::vector<size_t> bpe_offsets; // store the offset of each word
bpe_offsets.reserve(offsets.size()); // Reserve memory for the approximate size
@@ -532,7 +566,7 @@ static std::vector<size_t> unicode_regex_split_stl(const std::string & text, con
++it;
}
- if (start_idx < (int64_t) offset) {
+ if (start_idx < (int64_t)offset) {
bpe_offsets.emplace_back(offset - start_idx);
}
start += offset;
@@ -541,14 +575,15 @@ static std::vector<size_t> unicode_regex_split_stl(const std::string & text, con
return bpe_offsets;
}
-static std::vector<size_t> unicode_regex_split_custom(const std::string & text, const std::string & regex_expr, const std::vector<size_t> & offsets) {
+static std::vector<size_t> unicode_regex_split_custom(const std::string& text, const std::string& regex_expr, const std::vector<size_t>& offsets) {
std::vector<size_t> bpe_offsets;
if (regex_expr == "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)") {
bpe_offsets = unicode_regex_split_custom_gpt2(text, offsets);
- } else if (
- regex_expr == "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" ||
- regex_expr == "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+") {
+ }
+ else if (
+ regex_expr == "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" ||
+ regex_expr == "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+") {
bpe_offsets = unicode_regex_split_custom_llama3(text, offsets);
}
@@ -589,8 +624,8 @@ std::string unicode_cpt_to_utf8(uint32_t cp) {
throw std::invalid_argument("invalid codepoint");
}
-std::vector<uint32_t> unicode_cpts_normalize_nfd(const std::vector<uint32_t> & cpts) {
- auto comp = [] (const uint32_t cpt, const range_nfd & range) {
+std::vector<uint32_t> unicode_cpts_normalize_nfd(const std::vector<uint32_t>& cpts) {
+ auto comp = [](const uint32_t cpt, const range_nfd& range) {
return cpt < range.first;
};
std::vector<uint32_t> result(cpts.size());
@@ -602,7 +637,7 @@ std::vector<uint32_t> unicode_cpts_normalize_nfd(const std::vector<uint32_t> & c
return result;
}
-std::vector<uint32_t> unicode_cpts_from_utf8(const std::string & utf8) {
+std::vector<uint32_t> unicode_cpts_from_utf8(const std::string& utf8) {
std::vector<uint32_t> result;
result.reserve(utf8.size());
size_t offset = 0;
@@ -618,7 +653,7 @@ codepoint_flags unicode_cpt_flags(const uint32_t cp) {
return cp < cpt_flags.size() ? cpt_flags[cp] : undef;
}
-codepoint_flags unicode_cpt_flags(const std::string & utf8) {
+codepoint_flags unicode_cpt_flags(const std::string& utf8) {
static const codepoint_flags undef(codepoint_flags::UNDEFINED);
if (utf8.empty()) {
return undef; // undefined
@@ -632,7 +667,7 @@ std::string unicode_byte_to_utf8(uint8_t byte) {
return map.at(byte);
}
-uint8_t unicode_utf8_to_byte(const std::string & utf8) {
+uint8_t unicode_utf8_to_byte(const std::string& utf8) {
static std::unordered_map<std::string, uint8_t> map = unicode_utf8_to_byte_map();
return map.at(utf8);
}
@@ -642,7 +677,7 @@ uint32_t unicode_tolower(uint32_t cp) {
return it == unicode_map_lowercase.end() ? cp : it->second;
}
-std::vector<std::string> unicode_regex_split(const std::string & text, const std::vector<std::string> & regex_exprs) {
+std::vector<std::string> unicode_regex_split(const std::string& text, const std::vector<std::string>& regex_exprs) {
// unicode categories
static const std::map<std::string, int> k_ucat_enum = {
{ "\\p{N}", codepoint_flags::NUMBER },
@@ -671,9 +706,9 @@ std::vector<std::string> unicode_regex_split(const std::string & text, const std
// compute collapsed codepoints only if needed by at least one regex
bool need_collapse = false;
- for (auto & regex_expr : regex_exprs) {
+ for (auto& regex_expr : regex_exprs) {
// search for unicode categories
- for (const auto & ucat : k_ucat_enum) {
+ for (const auto& ucat : k_ucat_enum) {
if (std::string::npos != regex_expr.find(ucat.first)) {
need_collapse = true;
break;
@@ -702,18 +737,20 @@ std::vector<std::string> unicode_regex_split(const std::string & text, const std
if (flags.is_whitespace) {
//NOTE: C++ std::regex \s does not mach 0x85, Rust and Python regex does.
//text_collapsed[i] = (char) 0x85; // <Next Line> as whitespace fallback
- text_collapsed[i] = (char) 0x0B; // <vertical tab> as whitespace fallback
- } else if (k_ucat_cpt.find(flags.category_flag()) != k_ucat_cpt.end()) {
+ text_collapsed[i] = (char)0x0B; // <vertical tab> as whitespace fallback
+ }
+ else if (k_ucat_cpt.find(flags.category_flag()) != k_ucat_cpt.end()) {
text_collapsed[i] = k_ucat_cpt.at(flags.category_flag());
- } else {
- text_collapsed[i] = (char) 0xD0; // fallback
+ }
+ else {
+ text_collapsed[i] = (char)0xD0; // fallback
}
}
}
std::vector<size_t> bpe_offsets = { cpts.size() };
- for (auto & regex_expr : regex_exprs) {
+ for (auto& regex_expr : regex_exprs) {
// first, see if we have an efficient custom regex implementation
auto tmp = unicode_regex_split_custom(text, regex_expr, bpe_offsets);
@@ -727,7 +764,7 @@ std::vector<std::string> unicode_regex_split(const std::string & text, const std
// if a unicode category is used in the regex, we use the collapsed text and replace the unicode category
// with the corresponding collapsed representation
bool use_collapsed = false;
- for (auto & ucat : k_ucat_enum) {
+ for (auto& ucat : k_ucat_enum) {
if (std::string::npos != regex_expr.find(ucat.first)) {
use_collapsed = true;
break;
@@ -786,7 +823,8 @@ std::vector<std::string> unicode_regex_split(const std::string & text, const std
//printf("text_collapsed: %s\n", text_collapsed.c_str());
//printf("regex_expr_collapsed: %s\n", regex_expr_collapsed.c_str());
bpe_offsets = unicode_regex_split_stl(text_collapsed, regex_expr_collapsed, bpe_offsets);
- } else {
+ }
+ else {
// no unicode category used, we can use std::wregex directly
const std::wstring wregex_expr = unicode_wstring_from_utf8(regex_expr);
@@ -802,7 +840,8 @@ std::vector<std::string> unicode_regex_split(const std::string & text, const std
//printf("regex_expr: %s\n", regex_expr.c_str());
bpe_offsets = unicode_regex_split_stl(wtext, wregex_expr, bpe_offsets);
}
- } catch (std::regex_error & e) {
+ }
+ catch (std::regex_error& e) {
fprintf(stderr, "Failed to process regex: '%s'\n", regex_expr.c_str());
fprintf(stderr, "Regex error: %s\n", e.what());
throw std::runtime_error("Failed to process regex");
@@ -813,7 +852,7 @@ std::vector<std::string> unicode_regex_split(const std::string & text, const std
bpe_words.reserve(bpe_offsets.size()); // reserve memory for the approximate size
size_t start = 0;
- for (size_t & offset : bpe_offsets) {
+ for (size_t& offset : bpe_offsets) {
bpe_words.emplace_back();
for (size_t i = start; i < start + offset; ++i) {
bpe_words.back() += unicode_cpt_to_utf8(cpts[i]);
generated by cgit v1.2.3 (git 2.25.1) at 2025-08-15 14:27:51 +0000