From a316a425d04027453dc0fd45f003b647c12f66f9 Mon Sep 17 00:00:00 2001
From: Georgi Gerganov <ggerganov@gmail.com>
Date: Sat, 25 Mar 2023 20:26:40 +0200
Subject: Overhaul the examples structure

- main -> examples
- utils -> examples (renamed to "common")
- quantize -> examples
- separate tools for "perplexity" and "embedding"

Hope I didn't break something !
---
 tests/test-tokenizer-0.cpp | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

(limited to 'tests/test-tokenizer-0.cpp')

diff --git a/tests/test-tokenizer-0.cpp b/tests/test-tokenizer-0.cpp
index 49bc232b..38205532 100644
--- a/tests/test-tokenizer-0.cpp
+++ b/tests/test-tokenizer-0.cpp
@@ -1,9 +1,9 @@
-#include "utils.h"
 #include "llama.h"
 
 #include <cstdio>
 #include <string>
 #include <map>
+#include <vector>
 
 static const std::map<std::string, std::vector<llama_token>> k_tests = {
     { "Hello World",        { 1,  10994,   2787, }, },
@@ -48,7 +48,9 @@ int main(int argc, char **argv) {
     }
 
     for (const auto & test_kv : k_tests) {
-        const auto res = ::llama_tokenize(ctx, test_kv.first, true);
+        std::vector<llama_token> res(test_kv.first.size());
+        const int n = llama_tokenize(ctx, test_kv.first.c_str(), res.data(), res.size(), true);
+        res.resize(n);
 
         bool correct = res.size() == test_kv.second.size();
 
-- 
cgit v1.2.3