diff options
Diffstat (limited to 'src/unicode.cpp')
| -rw-r--r-- | src/unicode.cpp | 179 |
1 files changed, 70 insertions, 109 deletions
diff --git a/src/unicode.cpp b/src/unicode.cpp index a57456ea..cfffde0d 100644 --- a/src/unicode.cpp +++ b/src/unicode.cpp @@ -18,7 +18,6 @@ #include <vector> #include <locale> #include <codecvt> -#include <iostream> size_t unicode_len_utf8(char src) { const size_t lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4 }; @@ -26,7 +25,7 @@ size_t unicode_len_utf8(char src) { return lookup[highbits]; } -static std::string unicode_cpts_to_utf8(const std::vector<uint32_t>& cps) { +static std::string unicode_cpts_to_utf8(const std::vector<uint32_t> & cps) { std::string result; for (size_t i = 0; i < cps.size(); ++i) { result.append(unicode_cpt_to_utf8(cps[i])); @@ -34,7 +33,7 @@ static std::string unicode_cpts_to_utf8(const std::vector<uint32_t>& cps) { return result; } -uint32_t unicode_cpt_from_utf8(const std::string& utf8, size_t& offset) { +uint32_t unicode_cpt_from_utf8(const std::string & utf8, size_t & offset) { assert(offset < utf8.size()); if (!(utf8[offset + 0] & 0x80)) { auto result = utf8[offset + 0]; @@ -45,7 +44,7 @@ uint32_t unicode_cpt_from_utf8(const std::string& utf8, size_t& offset) { throw std::invalid_argument("invalid character"); } if (!(utf8[offset + 0] & 0x20)) { - if (offset + 1 >= utf8.size() || !((utf8[offset + 1] & 0xc0) == 0x80)) { + if (offset + 1 >= utf8.size() || ! ((utf8[offset + 1] & 0xc0) == 0x80)) { throw std::invalid_argument("invalid character"); } auto result = ((utf8[offset + 0] & 0x1f) << 6) | (utf8[offset + 1] & 0x3f); @@ -53,7 +52,7 @@ uint32_t unicode_cpt_from_utf8(const std::string& utf8, size_t& offset) { return result; } if (!(utf8[offset + 0] & 0x10)) { - if (offset + 2 >= utf8.size() || !((utf8[offset + 1] & 0xc0) == 0x80) || !((utf8[offset + 2] & 0xc0) == 0x80)) { + if (offset + 2 >= utf8.size() || ! ((utf8[offset + 1] & 0xc0) == 0x80) || ! ((utf8[offset + 2] & 0xc0) == 0x80)) { throw std::invalid_argument("invalid character"); } auto result = ((utf8[offset + 0] & 0x0f) << 12) | ((utf8[offset + 1] & 0x3f) << 6) | (utf8[offset + 2] & 0x3f); @@ -61,7 +60,7 @@ uint32_t unicode_cpt_from_utf8(const std::string& utf8, size_t& offset) { return result; } if (!(utf8[offset + 0] & 0x08)) { - if (offset + 3 >= utf8.size() || !((utf8[offset + 1] & 0xc0) == 0x80) || !((utf8[offset + 2] & 0xc0) == 0x80) || !((utf8[offset + 3] & 0xc0) == 0x80)) { + if (offset + 3 >= utf8.size() || ! ((utf8[offset + 1] & 0xc0) == 0x80) || ! ((utf8[offset + 2] & 0xc0) == 0x80) || !((utf8[offset + 3] & 0xc0) == 0x80)) { throw std::invalid_argument("invalid character"); } auto result = ((utf8[offset + 0] & 0x07) << 18) | ((utf8[offset + 1] & 0x3f) << 12) | ((utf8[offset + 2] & 0x3f) << 6) | (utf8[offset + 3] & 0x3f); @@ -123,10 +122,10 @@ uint32_t unicode_cpt_from_utf8(const std::string& utf8, size_t& offset) { static std::vector<codepoint_flags> unicode_cpt_flags_array() { std::vector<codepoint_flags> cpt_flags(MAX_CODEPOINTS, codepoint_flags::UNDEFINED); - assert(unicode_ranges_flags.front().first == 0); - assert(unicode_ranges_flags.back().first == MAX_CODEPOINTS); + assert (unicode_ranges_flags.front().first == 0); + assert (unicode_ranges_flags.back().first == MAX_CODEPOINTS); for (size_t i = 1; i < unicode_ranges_flags.size(); ++i) { - const auto range_ini = unicode_ranges_flags[i - 1]; // codepoint_ini, flags + const auto range_ini = unicode_ranges_flags[i-1]; // codepoint_ini, flags const auto range_end = unicode_ranges_flags[i]; // codepoint_end, flags for (uint32_t cpt = range_ini.first; cpt < range_end.first; ++cpt) { cpt_flags[cpt] = range_ini.second; @@ -145,7 +144,7 @@ static std::vector<codepoint_flags> unicode_cpt_flags_array() { cpt_flags[p.second].is_uppercase = true; } - for (auto& range : unicode_ranges_nfd) { // start, last, nfd + for (auto &range : unicode_ranges_nfd) { // start, last, nfd cpt_flags[range.nfd].is_nfd = true; } @@ -200,55 +199,22 @@ static std::unordered_map<std::string, uint8_t> unicode_utf8_to_byte_map() { return map; } -static inline bool is_valid_utf8(const std::string& str) { - int remaining_bytes = 0; // 当前多字节字符剩余的字节数 - for (unsigned char c : str) { - if (remaining_bytes == 0) { - if ((c & 0x80) == 0x00) continue; // 1字节字符 - else if ((c & 0xE0) == 0xC0) remaining_bytes = 1; // 2字节 - else if ((c & 0xF0) == 0xE0) remaining_bytes = 2; // 3字节 - else if ((c & 0xF8) == 0xF0) remaining_bytes = 3; // 4字节 - else return false; // 非法起始字节 - } - else { - // 检查后续字节是否为10xxxxxx - if ((c & 0xC0) != 0x80) - { - return false; - } - remaining_bytes--; - } - } - return (remaining_bytes == 0); // 确保多字节字符完整 -} - -static inline std::wstring unicode_wstring_from_utf8(const std::string& s) { -#if defined(__clang__) - // disable C++17 deprecation warning for std::codecvt_utf8 -# pragma clang diagnostic push -# pragma clang diagnostic ignored "-Wdeprecated-declarations" -#endif - bool isvalid = is_valid_utf8(s); +static inline std::wstring unicode_wstring_from_utf8(const std::string & s) { std::wstring_convert<std::codecvt_utf8<wchar_t>> conv; - -#if defined(__clang__) -# pragma clang diagnostic pop -#endif - return conv.from_bytes(s); } -static std::vector<std::string> unicode_byte_encoding_process(const std::vector<std::string>& bpe_words) { +static std::vector<std::string> unicode_byte_encoding_process(const std::vector<std::string> & bpe_words) { std::vector<std::string> bpe_encoded_words; - for (const auto& word : bpe_words) { + for (const auto & word : bpe_words) { std::string text_utf; - auto utf_word = unicode_cpts_from_utf8(word); + auto utf_word = unicode_cpts_from_utf8(word); for (size_t i = 0; i < utf_word.size(); ++i) { text_utf += unicode_cpt_to_utf8(utf_word[i]); } std::string encoded_token; - for (char& c : text_utf) { + for (char & c : text_utf) { encoded_token += unicode_byte_to_utf8(c); } bpe_encoded_words.emplace_back(encoded_token); @@ -257,7 +223,7 @@ static std::vector<std::string> unicode_byte_encoding_process(const std::vector< } // GPT2 system regex: 's|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+ -static std::vector<size_t> unicode_regex_split_custom_gpt2(const std::string& text, const std::vector<size_t>& offsets) { +static std::vector<size_t> unicode_regex_split_custom_gpt2(const std::string & text, const std::vector<size_t> & offsets) { std::vector<size_t> bpe_offsets; // store the offset of each word bpe_offsets.reserve(offsets.size()); // Reserve memory for the approximate size @@ -271,16 +237,16 @@ static std::vector<size_t> unicode_regex_split_custom_gpt2(const std::string& te start = offset_end; static const uint32_t OUT_OF_RANGE = 0xFFFFFFFF; - auto _get_cpt = [&](const size_t pos) -> uint32_t { + auto _get_cpt = [&] (const size_t pos) -> uint32_t { return (offset_ini <= pos && pos < offset_end) ? cpts[pos] : OUT_OF_RANGE; }; - auto _get_flags = [&](const size_t pos) -> codepoint_flags { + auto _get_flags = [&] (const size_t pos) -> codepoint_flags { return (offset_ini <= pos && pos < offset_end) ? unicode_cpt_flags(cpts[pos]) : codepoint_flags{}; }; size_t _prev_end = offset_ini; - auto _add_token = [&](const size_t end) -> size_t { + auto _add_token = [&] (const size_t end) -> size_t { assert(_prev_end <= end && end <= offset_end); size_t len = end - _prev_end; if (len > 0) { @@ -296,29 +262,29 @@ static std::vector<size_t> unicode_regex_split_custom_gpt2(const std::string& te return len; }; - for (size_t pos = offset_ini; pos < offset_end; /*pos++*/) { + for (size_t pos = offset_ini; pos < offset_end; /*pos++*/ ) { const uint32_t cpt = _get_cpt(pos); const auto flags = _get_flags(pos); // regex: 's|'t|'re|'ve|'m|'ll|'d - if (cpt == '\'' && pos + 1 < offset_end) { - uint32_t cpt_next = _get_cpt(pos + 1); + if (cpt == '\'' && pos+1 < offset_end) { + uint32_t cpt_next = _get_cpt(pos+1); if (cpt_next == 's' || cpt_next == 't' || cpt_next == 'm' || cpt_next == 'd') { - pos += _add_token(pos + 2); + pos += _add_token(pos+2); continue; } - if (pos + 2 < offset_end) { - uint32_t cpt_next_next = _get_cpt(pos + 2); + if (pos+2 < offset_end) { + uint32_t cpt_next_next = _get_cpt(pos+2); if ((cpt_next == 'r' && cpt_next_next == 'e') || (cpt_next == 'v' && cpt_next_next == 'e') || (cpt_next == 'l' && cpt_next_next == 'l')) { - pos += _add_token(pos + 3); + pos += _add_token(pos+3); continue; } } } - auto flags2 = (cpt == ' ' ? _get_flags(pos + 1) : flags); + auto flags2 = (cpt == ' ' ? _get_flags(pos+1) : flags); // regex: <space>?\p{L}+ if (flags2.is_letter) { pos += (cpt == ' '); @@ -348,12 +314,12 @@ static std::vector<size_t> unicode_regex_split_custom_gpt2(const std::string& te } size_t num_whitespaces = 0; - while (_get_flags(pos + num_whitespaces).is_whitespace) { + while (_get_flags(pos+num_whitespaces).is_whitespace) { num_whitespaces++; } // regex: \s+(?!\S) - if (num_whitespaces > 1 && _get_cpt(pos + num_whitespaces) != OUT_OF_RANGE) { + if (num_whitespaces > 1 && _get_cpt(pos+num_whitespaces) != OUT_OF_RANGE) { pos += num_whitespaces - 1; _add_token(pos); continue; @@ -375,7 +341,7 @@ static std::vector<size_t> unicode_regex_split_custom_gpt2(const std::string& te } // LLAMA3 system regex: "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+" -static std::vector<size_t> unicode_regex_split_custom_llama3(const std::string& text, const std::vector<size_t>& offsets) { +static std::vector<size_t> unicode_regex_split_custom_llama3(const std::string & text, const std::vector<size_t> & offsets) { std::vector<size_t> bpe_offsets; // store the offset of each word bpe_offsets.reserve(offsets.size()); // Reserve memory for the approximate size @@ -389,16 +355,16 @@ static std::vector<size_t> unicode_regex_split_custom_llama3(const std::string& start = offset_end; static const uint32_t OUT_OF_RANGE = 0xFFFFFFFF; - auto _get_cpt = [&](const size_t pos) -> uint32_t { + auto _get_cpt = [&] (const size_t pos) -> uint32_t { return (offset_ini <= pos && pos < offset_end) ? cpts[pos] : OUT_OF_RANGE; }; - auto _get_flags = [&](const size_t pos) -> codepoint_flags { + auto _get_flags = [&] (const size_t pos) -> codepoint_flags { return (offset_ini <= pos && pos < offset_end) ? unicode_cpt_flags(cpts[pos]) : codepoint_flags{}; }; size_t _prev_end = offset_ini; - auto _add_token = [&](const size_t end) -> size_t { + auto _add_token = [&] (const size_t end) -> size_t { assert(_prev_end <= end && end <= offset_end); size_t len = end - _prev_end; if (len > 0) { @@ -414,23 +380,23 @@ static std::vector<size_t> unicode_regex_split_custom_llama3(const std::string& return len; }; - for (size_t pos = offset_ini; pos < offset_end; /*pos++*/) { + for (size_t pos = offset_ini; pos < offset_end; /*pos++*/ ) { const uint32_t cpt = _get_cpt(pos); const auto flags = _get_flags(pos); // regex: (?i:'s|'t|'re|'ve|'m|'ll|'d) // case insensitive - if (cpt == '\'' && pos + 1 < offset_end) { - uint32_t cpt_next = unicode_tolower(_get_cpt(pos + 1)); + if (cpt == '\'' && pos+1 < offset_end) { + uint32_t cpt_next = unicode_tolower(_get_cpt(pos+1)); if (cpt_next == 's' || cpt_next == 't' || cpt_next == 'm' || cpt_next == 'd') { - pos += _add_token(pos + 2); + pos += _add_token(pos+2); continue; } - if (pos + 2 < offset_end) { - uint32_t cpt_next_next = unicode_tolower(_get_cpt(pos + 2)); + if (pos+2 < offset_end) { + uint32_t cpt_next_next = unicode_tolower(_get_cpt(pos+2)); if ((cpt_next == 'r' && cpt_next_next == 'e') || (cpt_next == 'v' && cpt_next_next == 'e') || (cpt_next == 'l' && cpt_next_next == 'l')) { - pos += _add_token(pos + 3); + pos += _add_token(pos+3); continue; } } @@ -438,7 +404,7 @@ static std::vector<size_t> unicode_regex_split_custom_llama3(const std::string& // regex: [^\r\n\p{L}\p{N}]?\p{L}+ if (!(cpt == '\r' || cpt == '\n' || flags.is_number)) { - if (flags.is_letter || _get_flags(pos + 1).is_letter) { // one or more letters + if (flags.is_letter || _get_flags(pos+1).is_letter) { // one or more letters pos++; while (_get_flags(pos).is_letter) { pos++; @@ -452,7 +418,7 @@ static std::vector<size_t> unicode_regex_split_custom_llama3(const std::string& if (flags.is_number) { size_t ini = pos; while (_get_flags(pos).is_number) { - if (++pos - ini >= 3) { + if (++pos - ini >= 3 ) { _add_token(pos); ini = pos; } @@ -462,7 +428,7 @@ static std::vector<size_t> unicode_regex_split_custom_llama3(const std::string& } // regex: <space>?[^\s\p{L}\p{N}]+[\r\n]* - auto flags2 = (cpt == ' ' ? _get_flags(pos + 1) : flags); + auto flags2 = (cpt == ' ' ? _get_flags(pos+1) : flags); if (!(flags2.is_whitespace | flags2.is_letter | flags2.is_number) && flags.as_uint()) { pos += (cpt == ' '); while (!(flags2.is_whitespace | flags2.is_letter | flags2.is_number) && flags2.as_uint()) { @@ -478,8 +444,8 @@ static std::vector<size_t> unicode_regex_split_custom_llama3(const std::string& size_t num_whitespaces = 0; size_t last_end_r_or_n = 0; - while (_get_flags(pos + num_whitespaces).is_whitespace) { - uint32_t cpt2 = _get_cpt(pos + num_whitespaces); + while (_get_flags(pos+num_whitespaces).is_whitespace) { + uint32_t cpt2 = _get_cpt(pos+num_whitespaces); if (cpt2 == '\r' || cpt2 == '\n') { last_end_r_or_n = pos + num_whitespaces + 1; } @@ -494,7 +460,7 @@ static std::vector<size_t> unicode_regex_split_custom_llama3(const std::string& } // regex: \s+(?!\S) - if (num_whitespaces > 1 && _get_cpt(pos + num_whitespaces) != OUT_OF_RANGE) { + if (num_whitespaces > 1 && _get_cpt(pos+num_whitespaces) != OUT_OF_RANGE) { pos += num_whitespaces - 1; _add_token(pos); continue; @@ -516,7 +482,7 @@ static std::vector<size_t> unicode_regex_split_custom_llama3(const std::string& } // use std::wregex to split the text -static std::vector<size_t> unicode_regex_split_stl(const std::wstring& wtext, const std::wstring& regex_expr, const std::vector<size_t>& offsets) { +static std::vector<size_t> unicode_regex_split_stl(const std::wstring & wtext, const std::wstring & regex_expr, const std::vector<size_t> & offsets) { std::wregex expr(regex_expr); std::vector<size_t> bpe_offsets; // store the offset of each word bpe_offsets.reserve(offsets.size()); // Reserve memory for the approximate size @@ -536,7 +502,7 @@ static std::vector<size_t> unicode_regex_split_stl(const std::wstring& wtext, co ++it; } - if (start_idx < (int64_t)offset) { + if (start_idx < (int64_t) offset) { bpe_offsets.emplace_back(offset - start_idx); } start += offset; @@ -546,7 +512,7 @@ static std::vector<size_t> unicode_regex_split_stl(const std::wstring& wtext, co } // use std::regex to split the text -static std::vector<size_t> unicode_regex_split_stl(const std::string& text, const std::string& regex_expr, const std::vector<size_t>& offsets) { +static std::vector<size_t> unicode_regex_split_stl(const std::string & text, const std::string & regex_expr, const std::vector<size_t> & offsets) { std::regex expr(regex_expr); std::vector<size_t> bpe_offsets; // store the offset of each word bpe_offsets.reserve(offsets.size()); // Reserve memory for the approximate size @@ -566,7 +532,7 @@ static std::vector<size_t> unicode_regex_split_stl(const std::string& text, cons ++it; } - if (start_idx < (int64_t)offset) { + if (start_idx < (int64_t) offset) { bpe_offsets.emplace_back(offset - start_idx); } start += offset; @@ -575,15 +541,14 @@ static std::vector<size_t> unicode_regex_split_stl(const std::string& text, cons return bpe_offsets; } -static std::vector<size_t> unicode_regex_split_custom(const std::string& text, const std::string& regex_expr, const std::vector<size_t>& offsets) { +static std::vector<size_t> unicode_regex_split_custom(const std::string & text, const std::string & regex_expr, const std::vector<size_t> & offsets) { std::vector<size_t> bpe_offsets; if (regex_expr == "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)") { bpe_offsets = unicode_regex_split_custom_gpt2(text, offsets); - } - else if ( - regex_expr == "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" || - regex_expr == "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+") { + } else if ( + regex_expr == "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" || + regex_expr == "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+") { bpe_offsets = unicode_regex_split_custom_llama3(text, offsets); } @@ -624,8 +589,8 @@ std::string unicode_cpt_to_utf8(uint32_t cp) { throw std::invalid_argument("invalid codepoint"); } -std::vector<uint32_t> unicode_cpts_normalize_nfd(const std::vector<uint32_t>& cpts) { - auto comp = [](const uint32_t cpt, const range_nfd& range) { +std::vector<uint32_t> unicode_cpts_normalize_nfd(const std::vector<uint32_t> & cpts) { + auto comp = [] (const uint32_t cpt, const range_nfd & range) { return cpt < range.first; }; std::vector<uint32_t> result(cpts.size()); @@ -637,7 +602,7 @@ std::vector<uint32_t> unicode_cpts_normalize_nfd(const std::vector<uint32_t>& cp return result; } -std::vector<uint32_t> unicode_cpts_from_utf8(const std::string& utf8) { +std::vector<uint32_t> unicode_cpts_from_utf8(const std::string & utf8) { std::vector<uint32_t> result; result.reserve(utf8.size()); size_t offset = 0; @@ -653,7 +618,7 @@ codepoint_flags unicode_cpt_flags(const uint32_t cp) { return cp < cpt_flags.size() ? cpt_flags[cp] : undef; } -codepoint_flags unicode_cpt_flags(const std::string& utf8) { +codepoint_flags unicode_cpt_flags(const std::string & utf8) { static const codepoint_flags undef(codepoint_flags::UNDEFINED); if (utf8.empty()) { return undef; // undefined @@ -667,7 +632,7 @@ std::string unicode_byte_to_utf8(uint8_t byte) { return map.at(byte); } -uint8_t unicode_utf8_to_byte(const std::string& utf8) { +uint8_t unicode_utf8_to_byte(const std::string & utf8) { static std::unordered_map<std::string, uint8_t> map = unicode_utf8_to_byte_map(); return map.at(utf8); } @@ -677,7 +642,7 @@ uint32_t unicode_tolower(uint32_t cp) { return it == unicode_map_lowercase.end() ? cp : it->second; } -std::vector<std::string> unicode_regex_split(const std::string& text, const std::vector<std::string>& regex_exprs) { +std::vector<std::string> unicode_regex_split(const std::string & text, const std::vector<std::string> & regex_exprs) { // unicode categories static const std::map<std::string, int> k_ucat_enum = { { "\\p{N}", codepoint_flags::NUMBER }, @@ -706,9 +671,9 @@ std::vector<std::string> unicode_regex_split(const std::string& text, const std: // compute collapsed codepoints only if needed by at least one regex bool need_collapse = false; - for (auto& regex_expr : regex_exprs) { + for (auto & regex_expr : regex_exprs) { // search for unicode categories - for (const auto& ucat : k_ucat_enum) { + for (const auto & ucat : k_ucat_enum) { if (std::string::npos != regex_expr.find(ucat.first)) { need_collapse = true; break; @@ -737,20 +702,18 @@ std::vector<std::string> unicode_regex_split(const std::string& text, const std: if (flags.is_whitespace) { //NOTE: C++ std::regex \s does not mach 0x85, Rust and Python regex does. //text_collapsed[i] = (char) 0x85; // <Next Line> as whitespace fallback - text_collapsed[i] = (char)0x0B; // <vertical tab> as whitespace fallback - } - else if (k_ucat_cpt.find(flags.category_flag()) != k_ucat_cpt.end()) { + text_collapsed[i] = (char) 0x0B; // <vertical tab> as whitespace fallback + } else if (k_ucat_cpt.find(flags.category_flag()) != k_ucat_cpt.end()) { text_collapsed[i] = k_ucat_cpt.at(flags.category_flag()); - } - else { - text_collapsed[i] = (char)0xD0; // fallback + } else { + text_collapsed[i] = (char) 0xD0; // fallback } } } std::vector<size_t> bpe_offsets = { cpts.size() }; - for (auto& regex_expr : regex_exprs) { + for (auto & regex_expr : regex_exprs) { // first, see if we have an efficient custom regex implementation auto tmp = unicode_regex_split_custom(text, regex_expr, bpe_offsets); @@ -764,7 +727,7 @@ std::vector<std::string> unicode_regex_split(const std::string& text, const std: // if a unicode category is used in the regex, we use the collapsed text and replace the unicode category // with the corresponding collapsed representation bool use_collapsed = false; - for (auto& ucat : k_ucat_enum) { + for (auto & ucat : k_ucat_enum) { if (std::string::npos != regex_expr.find(ucat.first)) { use_collapsed = true; break; @@ -823,8 +786,7 @@ std::vector<std::string> unicode_regex_split(const std::string& text, const std: //printf("text_collapsed: %s\n", text_collapsed.c_str()); //printf("regex_expr_collapsed: %s\n", regex_expr_collapsed.c_str()); bpe_offsets = unicode_regex_split_stl(text_collapsed, regex_expr_collapsed, bpe_offsets); - } - else { + } else { // no unicode category used, we can use std::wregex directly const std::wstring wregex_expr = unicode_wstring_from_utf8(regex_expr); @@ -840,8 +802,7 @@ std::vector<std::string> unicode_regex_split(const std::string& text, const std: //printf("regex_expr: %s\n", regex_expr.c_str()); bpe_offsets = unicode_regex_split_stl(wtext, wregex_expr, bpe_offsets); } - } - catch (std::regex_error& e) { + } catch (std::regex_error & e) { fprintf(stderr, "Failed to process regex: '%s'\n", regex_expr.c_str()); fprintf(stderr, "Regex error: %s\n", e.what()); throw std::runtime_error("Failed to process regex"); @@ -852,7 +813,7 @@ std::vector<std::string> unicode_regex_split(const std::string& text, const std: bpe_words.reserve(bpe_offsets.size()); // reserve memory for the approximate size size_t start = 0; - for (size_t& offset : bpe_offsets) { + for (size_t & offset : bpe_offsets) { bpe_words.emplace_back(); for (size_t i = start; i < start + offset; ++i) { bpe_words.back() += unicode_cpt_to_utf8(cpts[i]); |