From 94d1b3b4119209efcdd08df0dceaecbd1fe7f85c Mon Sep 17 00:00:00 2001
From: Jared Van Bortel <jared@nomic.ai>
Date: Sat, 23 Mar 2024 18:48:02 -0400
Subject: use _wfopen instead of fopen on Windows (#6248)

also fix missing #defines before windows.h, and BPE LF token on MSVC
---
 llama.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'llama.cpp')

diff --git a/llama.cpp b/llama.cpp
index b385ae36..9614cdb1 100644
--- a/llama.cpp
+++ b/llama.cpp
@@ -1065,7 +1065,7 @@ struct llama_file {
     size_t size;
 
     llama_file(const char * fname, const char * mode) {
-        fp = std::fopen(fname, mode);
+        fp = ggml_fopen(fname, mode);
         if (fp == NULL) {
             throw std::runtime_error(format("failed to open %s: %s", fname, strerror(errno)));
         }
@@ -4006,7 +4006,7 @@ static void llm_load_vocab(
     } else if (vocab.type == LLAMA_VOCAB_TYPE_WPM) {
         vocab.linefeed_id = vocab.special_pad_id;
     } else {
-        const std::vector<int> ids = llama_tokenize_internal(vocab, "\u010A", false);
+        const std::vector<int> ids = llama_tokenize_internal(vocab, "\xC4\x8A", false); // U+010A
         GGML_ASSERT(!ids.empty() && "model vocab missing newline token");
         vocab.linefeed_id = ids[0];
     }
-- 
cgit v1.2.3