ExternalVendorCode/whisper.cpp
1
0
Fork 0
mirror of https://github.com/ggerganov/whisper.cpp.git synced 2024-12-19 04:37:51 +00:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

whisper : clear kv cache when using whisper_decode API

Browse Source
This commit is contained in:
Georgi Gerganov 2023-11-14 21:04:33 +02:00
parent b2123cb463
commit d77603578b
No known key found for this signature in database
GPG Key ID: 449E073F9DC10735
1 changed files with 4 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
whisper.cpp
View File

@ -3516,6 +3516,8 @@ int whisper_encode(struct whisper_context * ctx, int offset, int n_threads) {
int whisper_decode_with_state(struct whisper_context * ctx, struct whisper_state * state, const whisper_token * tokens, int n_tokens, int n_past, int n_threads) { int whisper_decode_with_state(struct whisper_context * ctx, struct whisper_state * state, const whisper_token * tokens, int n_tokens, int n_past, int n_threads) {
whisper_batch_prep_legacy(state->batch, tokens, n_tokens, n_past, 0); whisper_batch_prep_legacy(state->batch, tokens, n_tokens, n_past, 0);
whisper_kv_cache_seq_rm(ctx->state->kv_self, 0, n_past, -1);
if (!whisper_decode_internal(*ctx, *state, state->batch, n_threads, nullptr, nullptr)) { if (!whisper_decode_internal(*ctx, *state, state->batch, n_threads, nullptr, nullptr)) {
WHISPER_LOG_ERROR("%s: failed to eval\n", __func__); WHISPER_LOG_ERROR("%s: failed to eval\n", __func__);
return 1; return 1;
@ -3530,6 +3532,8 @@ int whisper_decode(struct whisper_context * ctx, const whisper_token * tokens, i
return false; return false;
} }
whisper_kv_cache_seq_rm(ctx->state->kv_self, 0, n_past, -1);
whisper_batch_prep_legacy(ctx->state->batch, tokens, n_tokens, n_past, 0); whisper_batch_prep_legacy(ctx->state->batch, tokens, n_tokens, n_past, 0);
if (!whisper_decode_internal(*ctx, *ctx->state, ctx->state->batch, n_threads, nullptr, nullptr)) { if (!whisper_decode_internal(*ctx, *ctx->state, ctx->state->batch, n_threads, nullptr, nullptr)) {