diff options
| author | Georgi Gerganov <ggerganov@gmail.com> | 2023-10-03 21:04:01 +0300 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2023-10-03 21:04:01 +0300 |
| commit | ac2219fef34eb5b713c286c34c6e4162c39c8f3b (patch) | |
| tree | 00d1bdad46d4995188ff084ae6178b9ff55f7431 /examples/chat-persistent.sh | |
| parent | 48be797ffbd80b062f55778e09e97180eb25d2ab (diff) | |
llama : fix session saving/loading (#3400)
* llama : fix session saving/loading
* llama : temp fix for clearing "future" tokens from the KV cache
* llama : fix handling of "future" tokens when loading sessions
* llama : fix comments for llama_kv_cache API
Diffstat (limited to 'examples/chat-persistent.sh')
| -rwxr-xr-x | examples/chat-persistent.sh | 8 |
1 files changed, 4 insertions, 4 deletions
diff --git a/examples/chat-persistent.sh b/examples/chat-persistent.sh index e0c251e5..22f5b83d 100755 --- a/examples/chat-persistent.sh +++ b/examples/chat-persistent.sh @@ -9,7 +9,7 @@ if [[ -z "${PROMPT_CACHE_FILE+x}" || -z "${CHAT_SAVE_DIR+x}" ]]; then exit 1 fi -MODEL="${MODEL:-./models/13B/ggml-model-q4_0.bin}" +MODEL="${MODEL:-./models/llama-13b/ggml-model-q4_0.gguf}" PROMPT_TEMPLATE="${PROMPT_TEMPLATE:-./prompts/chat.txt}" USER_NAME="${USER_NAME:-User}" AI_NAME="${AI_NAME:-ChatLLaMa}" @@ -61,9 +61,9 @@ fi if [[ ! -e "$PROMPT_CACHE_FILE" ]]; then echo 'Prompt cache does not exist, building...' - # Default batch_size to 8 here for better user feedback during initial prompt processing + # Default batch_size to 64 here for better user feedback during initial prompt processing ./main 2>>"$LOG" \ - --batch_size 8 \ + --batch_size 64 \ "${OPTS[@]}" \ --prompt-cache "$PROMPT_CACHE_FILE" \ --file "$CUR_PROMPT_FILE" \ @@ -132,7 +132,7 @@ while read -e line; do # HACK get num tokens from debug message # TODO get both messages in one go if ! session_size_msg="$(tail -n30 "$LOG" | grep -oE "$SESSION_SIZE_MSG_PATTERN")" || - ! sample_time_msg="$( tail -n10 "$LOG" | grep -oE "$SAMPLE_TIME_MSG_PATTERN")"; then + ! sample_time_msg="$(tail -n10 "$LOG" | grep -oE "$SAMPLE_TIME_MSG_PATTERN")"; then echo >&2 "Couldn't get number of tokens from ./main output!" exit 1 fi |