diff options
| author | Georgi Gerganov <ggerganov@gmail.com> | 2023-10-28 14:23:11 +0300 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2023-10-28 14:23:11 +0300 |
| commit | ee1a0ec9cb367ba41d138134795cbbbe93d2bf1c (patch) | |
| tree | 04111cbd66f34c6cc986e70bb47b96b39828e0f2 /common | |
| parent | 177461104b454163473dced2a5038f4e016cdb7e (diff) | |
llama : add option for greedy sampling with probs (#3813)
* llama : add option for greedy sampling with probs
* llama : add comment about llama_sample_token_greedy() missing probs
* sampling : temp == 0.0 -> no probs, temp < 0.0 -> probs
Diffstat (limited to 'common')
| -rw-r--r-- | common/common.cpp | 1 | ||||
| -rw-r--r-- | common/sampling.cpp | 8 |
2 files changed, 7 insertions, 2 deletions
diff --git a/common/common.cpp b/common/common.cpp index c0d4924e..f81f4d35 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -224,6 +224,7 @@ bool gpt_params_parse(int argc, char ** argv, gpt_params & params) { break; } sparams.temp = std::stof(argv[i]); + sparams.temp = std::max(sparams.temp, 0.0f); } else if (arg == "--tfs") { if (++i >= argc) { invalid_param = true; diff --git a/common/sampling.cpp b/common/sampling.cpp index 5258d4e8..c4996c98 100644 --- a/common/sampling.cpp +++ b/common/sampling.cpp @@ -167,8 +167,12 @@ llama_token llama_sampling_sample( llama_sample_grammar(ctx_main, &cur_p, ctx_sampling->grammar); } - if (temp <= 0) { - // greedy sampling + if (temp < 0.0) { + // greedy sampling, with probs + llama_sample_softmax(ctx_main, &cur_p); + id = cur_p.data[0].id; + } else if (temp == 0.0) { + // greedy sampling, no probs id = llama_sample_token_greedy(ctx_main, &cur_p); } else { if (mirostat == 1) { |