diff options
| author | Jhen-Jie Hong <iainst0409@gmail.com> | 2023-10-06 07:44:24 -0500 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2023-10-06 15:44:24 +0300 |
| commit | 97af49fa395df77e4c18af0e1655b2fee67c9686 (patch) | |
| tree | 94ffe1bcd9ab427a5c84cc054c8f2f04b49a2445 /common/common.cpp | |
| parent | 16820a5a0d885113f21021ce934f0b0027b9d69a (diff) | |
server : reuse llama_sample_token common util (#3494)
* server : reuse llama_sample_token common function
* common : use n_probs for temperature sampling
Diffstat (limited to 'common/common.cpp')
| -rw-r--r-- | common/common.cpp | 9 |
1 files changed, 5 insertions, 4 deletions
diff --git a/common/common.cpp b/common/common.cpp index 6b9b4695..186f5b26 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1020,10 +1020,11 @@ llama_token llama_sample_token( id = llama_sample_token_mirostat_v2(ctx, &cur_p, mirostat_tau, mirostat_eta, &mirostat_mu); } else { // Temperature sampling - llama_sample_top_k (ctx, &cur_p, top_k, 1); - llama_sample_tail_free (ctx, &cur_p, tfs_z, 1); - llama_sample_typical (ctx, &cur_p, typical_p, 1); - llama_sample_top_p (ctx, &cur_p, top_p, 1); + size_t min_keep = std::max(1, params.n_probs); + llama_sample_top_k (ctx, &cur_p, top_k, min_keep); + llama_sample_tail_free (ctx, &cur_p, tfs_z, min_keep); + llama_sample_typical (ctx, &cur_p, typical_p, min_keep); + llama_sample_top_p (ctx, &cur_p, top_p, min_keep); llama_sample_temp(ctx, &cur_p, temp); { |