From 22b6598cc9f1454567efa0d816fdc57637243999 Mon Sep 17 00:00:00 2001 From: goldwaving <77494627+goldwaving@users.noreply.github.com> Date: Sun, 28 Apr 2024 15:06:12 -0230 Subject: [PATCH] Remove unnecessary memory reallocation in fft (#2080) fft_out needs to be twice the frame_size, not the frame_step. It is resized in fft() anyway, but this change prevents an unnecessary reallocation. n_fft must match the mel filter size, so it is best not to calculate it from the framesize. We only need to get the magnitudes for half the spectrum since the other half is a mirror and not used in the mel filter loop later. --- whisper.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/whisper.cpp b/whisper.cpp index 1a6d889a..f31309ed 100644 --- a/whisper.cpp +++ b/whisper.cpp @@ -2900,11 +2900,13 @@ static void log_mel_spectrogram_worker_thread(int ith, const std::vector int n_samples, int frame_size, int frame_step, int n_threads, const whisper_filters & filters, whisper_mel & mel) { std::vector fft_in(frame_size, 0.0); - std::vector fft_out(2 * frame_step); - // make sure n_fft == 1 + (WHISPER_N_FFT / 2), bin_0 to bin_nyquist - int n_fft = 1 + (frame_size / 2); + std::vector fft_out(2 * frame_size); + int n_fft = filters.n_fft; int i = ith; + // make sure n_fft == 1 + (WHISPER_N_FFT / 2), bin_0 to bin_nyquist + assert( n_fft == 1 + (frame_size / 2) ); + // calculate FFT only when fft_in are not all zero for (; i < std::min(n_samples / frame_step + 1, mel.n_len); i += n_threads) { const int offset = i * frame_step; @@ -2923,7 +2925,7 @@ static void log_mel_spectrogram_worker_thread(int ith, const std::vector // Calculate modulus^2 of complex numbers // Use pow(fft_out[2 * j + 0], 2) + pow(fft_out[2 * j + 1], 2) causes inference quality problem? Interesting. - for (int j = 0; j < frame_size; j++) { + for (int j = 0; j < n_fft; j++) { fft_out[j] = (fft_out[2 * j + 0] * fft_out[2 * j + 0] + fft_out[2 * j + 1] * fft_out[2 * j + 1]); }