From ccb265c01676aad9ae5860ba50e74e61dfcd1cf8 Mon Sep 17 00:00:00 2001 From: Kawrakow Date: Tue, 3 Jun 2025 11:32:03 +0300 Subject: Adding the XTC sampler (#486) Co-authored-by: Iwan Kawrakow --- src/llama-sampling.cpp | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) (limited to 'src/llama-sampling.cpp') diff --git a/src/llama-sampling.cpp b/src/llama-sampling.cpp index 8910f6d6..06f44b02 100644 --- a/src/llama-sampling.cpp +++ b/src/llama-sampling.cpp @@ -434,6 +434,40 @@ void llama_sample_temp_impl(struct llama_sampling * smpl, llama_token_data_array } } +void llama_sample_xtc_impl(struct llama_sampling * smpl, llama_token_data_array * candidates, float probability, float threshold, size_t min_keep) { + if (probability < 0 || threshold > 0.5f || candidates->size < 2) { + return; + } + GGML_ASSERT(smpl); + const int64_t t_start_sample_us = ggml_time_us(); + if (probability < 1) { + std::uniform_real_distribution distribution(0.0f, 1.0f); + float chance = distribution(smpl->rng); + if (chance > probability) return; + } + + llama_sample_softmax_impl(nullptr, candidates); + + auto cur_size = candidates->size; + + int pos_last = 0; + + for (size_t i = 0; i < candidates->size; ++i) { + if (candidates->data[i].p >= threshold) { + pos_last = i; + } else break; + } + + if (candidates->size - pos_last >= min_keep && pos_last > 0) { + candidates->data += pos_last; + candidates->size -= pos_last; + } + + smpl->t_sample_us += ggml_time_us() - t_start_sample_us; + smpl->n_sample++; + +} + void llama_sample_repetition_penalties_impl( struct llama_sampling * smpl, llama_token_data_array * candidates, -- cgit v1.2.3