Overhaul the examples structure

- main -> examples
- utils -> examples (renamed to "common")
- quantize -> examples
- separate tools for "perplexity" and "embedding"

Hope I didn't break something !

1 files changed, 4 insertions, 2 deletions

diff --git a/tests/test-tokenizer-0.cpp b/tests/test-tokenizer-0.cpp
index 49bc232b..38205532 100644
--- a/tests/test-tokenizer-0.cpp
+++ b/tests/test-tokenizer-0.cpp
@@ -1,9 +1,9 @@
-#include "utils.h"
 #include "llama.h"
 
 #include <cstdio>
 #include <string>
 #include <map>
+#include <vector>
 
 static const std::map<std::string, std::vector<llama_token>> k_tests = {
     { "Hello World",        { 1,  10994,   2787, }, },
@@ -48,7 +48,9 @@ int main(int argc, char **argv) {
     }
 
     for (const auto & test_kv : k_tests) {
-        const auto res = ::llama_tokenize(ctx, test_kv.first, true);
+        std::vector<llama_token> res(test_kv.first.size());
+        const int n = llama_tokenize(ctx, test_kv.first.c_str(), res.data(), res.size(), true);
+        res.resize(n);
 
         bool correct = res.size() == test_kv.second.size();