diff options
| author | Miwa / Ensan <63481257+ensan-hcl@users.noreply.github.com> | 2023-12-04 22:43:45 +0900 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2023-12-04 15:43:45 +0200 |
| commit | 5c9f90cba1cc6b0a2a7d19ee5dcb73cad6331d30 (patch) | |
| tree | a4b49982b86c0a09d87b4f6eeed6af944ca482f8 | |
| parent | 4fa44e84adb4c78e1885694cc3513982d4af2b08 (diff) | |
swift : fix prompt tokenization logic (#4321)
| -rw-r--r-- | examples/batched.swift/Sources/main.swift | 5 | ||||
| -rw-r--r-- | examples/llama.swiftui/llama.cpp.swift/LibLlama.swift | 5 |
2 files changed, 6 insertions, 4 deletions
diff --git a/examples/batched.swift/Sources/main.swift b/examples/batched.swift/Sources/main.swift index ce9d80d9..4d000534 100644 --- a/examples/batched.swift/Sources/main.swift +++ b/examples/batched.swift/Sources/main.swift @@ -215,9 +215,10 @@ print("decoded \(n_decode) tokens in \(String(format: "%.2f", Double(t_main_end llama_print_timings(context) private func tokenize(text: String, add_bos: Bool) -> [llama_token] { - let n_tokens = text.count + (add_bos ? 1 : 0) + let utf8Count = text.utf8.count + let n_tokens = utf8Count + (add_bos ? 1 : 0) let tokens = UnsafeMutablePointer<llama_token>.allocate(capacity: n_tokens) - let tokenCount = llama_tokenize(model, text, Int32(text.count), tokens, Int32(n_tokens), add_bos, /*special tokens*/ false) + let tokenCount = llama_tokenize(model, text, Int32(utf8Count), tokens, Int32(n_tokens), add_bos, /*special tokens*/ false) var swiftTokens: [llama_token] = [] for i in 0 ..< tokenCount { swiftTokens.append(tokens[Int(i)]) diff --git a/examples/llama.swiftui/llama.cpp.swift/LibLlama.swift b/examples/llama.swiftui/llama.cpp.swift/LibLlama.swift index 09b36d9e..f828106f 100644 --- a/examples/llama.swiftui/llama.cpp.swift/LibLlama.swift +++ b/examples/llama.swiftui/llama.cpp.swift/LibLlama.swift @@ -147,9 +147,10 @@ actor LlamaContext { } private func tokenize(text: String, add_bos: Bool) -> [llama_token] { - let n_tokens = text.count + (add_bos ? 1 : 0) + let utf8Count = text.utf8.count + let n_tokens = utf8Count + (add_bos ? 1 : 0) let tokens = UnsafeMutablePointer<llama_token>.allocate(capacity: n_tokens) - let tokenCount = llama_tokenize(model, text, Int32(text.count), tokens, Int32(n_tokens), add_bos, false) + let tokenCount = llama_tokenize(model, text, Int32(utf8Count), tokens, Int32(n_tokens), add_bos, false) var swiftTokens: [llama_token] = [] for i in 0..<tokenCount { |