diff --git a/whisper.cpp b/whisper.cpp
index 289d7039..f6011974 100644
--- a/whisper.cpp
+++ b/whisper.cpp
@@ -1680,6 +1680,8 @@ static struct ggml_cgraph * whisper_build_graph_conv(
         wstate.embd_enc = cur;
     }
 
+    ggml_set_output(cur);
+
     ggml_build_forward_expand(gf, cur);
 
     ggml_free(ctx0);