ref #4 : added transcription timestamps

Can be turned off with "-nt" argument.
Performance has also improved.

README.md

@@ -31,7 +31,7 @@ $ make base.en
 
 gcc -pthread -O3 -mavx -mavx2 -mfma -mf16c -c ggml.c
 g++ -pthread -O3 -std=c++11 -c main.cpp
-g++ -o main ggml.o main.o
+g++ -pthread -o main ggml.o main.o
 ./main -h
 
 usage: ./main [options]
@@ -40,22 +40,17 @@ options:
   -h,       --help           show this help message and exit
   -s SEED,  --seed SEED      RNG seed (default: -1)
   -t N,     --threads N      number of threads to use during computation (default: 4)
-  -T N,     --tokens N       maximum number of tokens to generate per iteration (default: 64)
   -v,       --verbose        verbose output
             --translate      translate from source language to english
   -ps,      --print_special  print special tokens
+  -nt,      --no_timestamps  do not print timestamps
   -l LANG,  --language LANG  spoken language (default: en)
   -m FNAME, --model FNAME    model path (default: models/ggml-base.en.bin)
   -f FNAME, --file FNAME     input WAV file path (default: samples/jfk.wav)
 
 bash ./download-ggml-model.sh base.en
 Downloading ggml model base.en ...
-models/ggml-base.en.bin         100%[=====================================>] 141.11M  7.84MB/s    in 18s
+Model base.en already exists. Skipping download.
-Done! Model 'base.en' saved in 'models/ggml-base.en.bin'
-You can now use it like this:
-
-  $ ./main -m models/ggml-base.en.bin -f samples/jfk.wav
-
 
 ===============================================
 Running base.en on all samples in ./samples ...
@@ -86,16 +81,17 @@ whisper_model_load: model size  =   140.54 MB
 log_mel_spectrogram: n_sample = 176000, n_len = 1100
 log_mel_spectrogram: recording length: 11.000000 s
 
-main: processing 176000 samples (11.0 sec), 4 threads, lang = english, task = transcribe ...
+main: processing 176000 samples (11.0 sec), 4 threads, lang = english, task = transcribe, timestamps = 1 ...
 
- And so my fellow Americans ask not what your country can do for you. Ask what you can do for your country.
+[00:00.000 --> 00:11.000]   And so my fellow Americans ask not what your country can do for you. Ask what you can do for your country.
 
-main:     load time =    71.89 ms
+
-main:      mel time =    36.95 ms
+main:     load time =    61.78 ms
+main:      mel time =    41.74 ms
 main:   sample time =     2.10 ms
-main:   encode time =   700.94 ms / 116.82 ms per layer
+main:   encode time =   718.60 ms / 119.77 ms per layer
-main:   decode time =    86.14 ms
+main:   decode time =    83.55 ms
-main:    total time =   898.72 ms
+main:    total time =   908.15 ms
 ```
 
 The command downloads the `base.en` model converted to custom `ggml` format and runs the inference on all `.wav` samples in the folder `samples`.
@@ -131,10 +127,73 @@ For example, you can use `ffmpeg` like this:
 ffmpeg -i input.mp3 -ar 16000 -ac 1 -c:a pcm_s16le output.wav
 ```
 
+Here is another example of transcribing a [3:24 min speech](https://upload.wikimedia.org/wikipedia/commons/1/1f/George_W_Bush_Columbia_FINAL.ogg) in less than a minute, using `medium.en` model:
+
+```bash
+$ ./main -m models/ggml-medium.en.bin -f samples/gb1.wav -t 8
+whisper_model_load: loading model from 'models/ggml-medium.en.bin'
+whisper_model_load: n_vocab       = 51864
+whisper_model_load: n_audio_ctx   = 1500
+whisper_model_load: n_audio_state = 1024
+whisper_model_load: n_audio_head  = 16
+whisper_model_load: n_audio_layer = 24
+whisper_model_load: n_text_ctx    = 448
+whisper_model_load: n_text_state  = 1024
+whisper_model_load: n_text_head   = 16
+whisper_model_load: n_text_layer  = 24
+whisper_model_load: n_mels        = 80
+whisper_model_load: f16           = 1
+whisper_model_load: type          = 4
+whisper_model_load: mem_required  = 2786.00 MB
+whisper_model_load: adding 1607 extra tokens
+whisper_model_load: ggml ctx size = 1644.97 MB
+whisper_model_load: memory size =   182.62 MB
+whisper_model_load: model size  =  1462.12 MB
+log_mel_spectrogram: n_sample = 3179750, n_len = 19873
+log_mel_spectrogram: recording length: 198.734375 s
+
+main: processing 3179750 samples (198.7 sec), 8 threads, lang = english, task = transcribe, timestamps = 1 ...
+
+[00:00.000 --> 00:08.000]   My fellow Americans, this day has brought terrible news and great sadness to our country.
+[00:08.000 --> 00:17.000]   At 9 o'clock this morning, Mission Control in Houston lost contact with our Space Shuttle Columbia.
+[00:17.000 --> 00:24.000]   A short time later, debris was seen falling from the skies above Texas.
+[00:24.000 --> 00:29.000]   The Columbia's lost. There are no survivors.
+[00:29.000 --> 00:32.000]   On board was a crew of seven.
+[00:32.000 --> 00:43.000]   Colonel Rick Husband, Lieutenant Colonel Michael Anderson, Commander Laurel Clark, Captain David Brown, Commander William McCool,
+[00:43.000 --> 00:52.000]   Dr. Kultner Aschavla, and Elon Ramon, a Colonel in the Israeli Air Force.
+[00:52.000 --> 00:58.000]   These men and women assumed great risk in the service to all humanity.
+[00:58.000 --> 01:06.000]   In an age when space flight has come to seem almost routine, it is easy to overlook the dangers of travel by rocket
+[01:06.000 --> 01:12.000]   and the difficulties of navigating the fierce outer atmosphere of the Earth.
+[01:12.000 --> 01:22.000]   These astronauts knew the dangers, and they faced them willingly, knowing they had a high and noble purpose in life.
+[01:22.000 --> 01:30.000]   Because of their courage, endearing, and idealism, we will miss them all the more.
+[01:30.000 --> 01:40.000]   All Americans today are thinking as well of the families of these men and women who have been given this sudden shock and grief.
+[01:40.000 --> 01:45.000]   You're not alone. Our entire nation agrees with you.
+[01:45.000 --> 01:52.000]   And those you love will always have the respect and gratitude of this country.
+[01:52.000 --> 01:56.000]   The cause in which they died will continue.
+[01:56.000 --> 02:07.000]   Mankind is led into the darkness beyond our world by the inspiration of discovery and the longing to understand.
+[02:07.000 --> 02:11.000]   Our journey into space will go on.
+[02:11.000 --> 02:16.000]   In the skies today, we saw destruction and tragedy.
+[02:16.000 --> 02:22.000]   Yet farther than we can see, there is comfort and hope.
+[02:22.000 --> 02:31.000]   In the words of the prophet Isaiah, "Lift your eyes and look to the heavens who created all these.
+[02:31.000 --> 02:39.000]   He who brings out the starry hosts one by one and calls them each by name."
+[02:39.000 --> 02:46.000]   Because of his great power and mighty strength, not one of them is missing.
+[02:46.000 --> 02:55.000]   The same creator who names the stars also knows the names of the seven souls we mourn today.
+[02:55.000 --> 03:05.000]   The crew of the shuttle Columbia did not return safely to Earth, yet we can pray that all are safely home.
+[03:05.000 --> 03:14.000]   May God bless the grieving families and may God continue to bless America.
+[03:14.000 --> 03:24.000]   [Music]
+
+
+main:     load time =   438.55 ms
+main:      mel time =   440.22 ms
+main:   sample time =    32.23 ms
+main:   encode time = 42329.63 ms / 1763.73 ms per layer
+main:   decode time = 15190.00 ms
+main:    total time = 58444.63 ms
+```
+
 ## Limitations
 
 - Very basic greedy sampling scheme - always pick up the top token
-- No timestamps
 - Inference only
 - Runs on the CPU
 - Only mono-channel 16-bit WAV is supported

main.cpp

@@ -206,6 +206,7 @@ struct whisper_vocab {
     id token_sot  = 50257;
     id token_prev = 50360;
     id token_solm = 50361; // ??
+    id token_not  = 50362; // no timestamps
     id token_beg  = 50363;
 
     // available tasks
@@ -217,17 +218,20 @@ struct whisper_vocab {
     }
 };
 
+struct whisper_result {
+    whisper_vocab::id id;
+    int64_t t;
+};
+
 // command-line parameters
 struct whisper_params {
     int32_t seed      = -1; // RNG seed, not used currently
     int32_t n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
 
-    // sampling parameter - used for the greedy strategy
-    int32_t max_tokens_per_iter = 64;
-
     bool verbose              = false;
     bool translate            = false;
     bool print_special_tokens = false;
+    bool no_timestamps        = false;
 
     std::string language  = "en";
     std::string model     = "models/ggml-base.en.bin";
@@ -244,8 +248,6 @@ bool whisper_params_parse(int argc, char ** argv, whisper_params & params) {
             params.seed = std::stoi(argv[++i]);
         } else if (arg == "-t" || arg == "--threads") {
             params.n_threads = std::stoi(argv[++i]);
-        } else if (arg == "-T" || arg == "--tokens") {
-            params.max_tokens_per_iter = std::stoi(argv[++i]);
         } else if (arg == "-v" || arg == "--verbose") {
             params.verbose = true;
         } else if (arg == "--translate") {
@@ -259,6 +261,8 @@ bool whisper_params_parse(int argc, char ** argv, whisper_params & params) {
             }
         } else if (arg == "-ps" || arg == "--print_special") {
             params.print_special_tokens = true;
+        } else if (arg == "-nt" || arg == "--no_timestamps") {
+            params.no_timestamps = true;
         } else if (arg == "-m" || arg == "--model") {
             params.model = argv[++i];
         } else if (arg == "-f" || arg == "--file") {
@@ -284,10 +288,10 @@ void whisper_print_usage(int argc, char ** argv, const whisper_params & params)
     fprintf(stderr, "  -h,       --help           show this help message and exit\n");
     fprintf(stderr, "  -s SEED,  --seed SEED      RNG seed (default: -1)\n");
     fprintf(stderr, "  -t N,     --threads N      number of threads to use during computation (default: %d)\n", params.n_threads);
-    fprintf(stderr, "  -T N,     --tokens N       maximum number of tokens to generate per iteration (default: %d)\n", params.max_tokens_per_iter);
     fprintf(stderr, "  -v,       --verbose        verbose output\n");
     fprintf(stderr, "            --translate      translate from source language to english\n");
     fprintf(stderr, "  -ps,      --print_special  print special tokens\n");
+    fprintf(stderr, "  -nt,      --no_timestamps  do not print timestamps\n");
     fprintf(stderr, "  -l LANG,  --language LANG  spoken language (default: %s)\n", params.language.c_str());
     fprintf(stderr, "  -m FNAME, --model FNAME    model path (default: %s)\n", params.model.c_str());
     fprintf(stderr, "  -f FNAME, --file FNAME     input WAV file path (default: %s)\n", params.fname_inp.c_str());
@@ -591,6 +595,7 @@ bool whisper_model_load(const std::string & fname, whisper_model & model, whispe
             vocab.token_sot++;
             vocab.token_prev++;
             vocab.token_solm++;
+            vocab.token_not++;
             vocab.token_beg++;
         }
 
@@ -605,6 +610,8 @@ bool whisper_model_load(const std::string & fname, whisper_model & model, whispe
                     word = "[_SOT_]";
                 } else if (i == vocab.token_prev) {
                     word = "[_PREV_]";
+                } else if (i == vocab.token_not) {
+                    word = "[_NOT_]";
                 } else if (i == vocab.token_beg) {
                     word = "[_BEG_]";
                 } else {
@@ -1842,15 +1849,13 @@ bool whisper_decode(
 // TODO: temperature
 whisper_vocab::id whisper_sample_best(
         const whisper_vocab & vocab,
-        const float * probs,
+        const float * probs) {
-        double temp,
-        int offset = 0) {
     int n_logits = vocab.id_to_token.size();
 
     std::vector<std::pair<double, whisper_vocab::id>> probs_id;
     probs_id.reserve(n_logits);
 
-    for (int i = offset; i < n_logits; i++) {
+    for (int i = 0; i < n_logits; i++) {
         probs_id.push_back(std::make_pair(probs[i], i));
     }
 
@@ -1872,13 +1877,49 @@ whisper_vocab::id whisper_sample_best(
     //}
 
     int res = 0;
-    while (probs_id[res].second == vocab.token_solm && res < (int) probs_id.size() - 1) {
+    while ((probs_id[res].second == vocab.token_sot ||
+            probs_id[res].second == vocab.token_solm ||
+            probs_id[res].second == vocab.token_not) &&
+            res < (int) probs_id.size() - 1) {
         res++;
     }
 
     return probs_id[res].second;
 }
 
+// samples only from the timestamps tokens
+whisper_vocab::id whisper_sample_timestamp(
+        const whisper_vocab & vocab,
+        const float * probs) {
+    int n_logits = vocab.id_to_token.size();
+
+    std::vector<std::pair<double, whisper_vocab::id>> probs_id;
+    probs_id.reserve(n_logits);
+
+    for (int i = vocab.token_beg + 1; i < n_logits; i++) {
+        probs_id.push_back(std::make_pair(probs[i], i));
+    }
+
+    const int top_k = 10;
+
+    // find the top K tokens
+    std::partial_sort(
+            probs_id.begin(),
+            probs_id.begin() + top_k, probs_id.end(),
+            [](const std::pair<double, whisper_vocab::id> & a, const std::pair<double, whisper_vocab::id> & b) {
+        return a.first > b.first;
+    });
+
+    probs_id.resize(top_k);
+
+    //printf("\n");
+    //for (int i = 0; i < (int) probs_id.size(); i++) {
+    //    printf("%d: '%s' %f, %d\n", i, vocab.id_to_token.at(probs_id[i].second).c_str(), probs_id[i].first, probs_id[i].second);
+    //}
+
+    return probs_id[0].second;
+}
+
 // Cooley-Tukey FFT
 // poor man's implmentation - use something better
 // input is real-valued
@@ -2032,6 +2073,20 @@ bool log_mel_spectrogram(
     return true;
 }
 
+//  500 -> 00:05.000
+// 6000 -> 01:00.000
+std::string to_timestamp(int64_t t) {
+    int64_t sec = t/100;
+    int64_t msec = t - sec*100;
+    int64_t min = sec/60;
+    sec = sec - min*60;
+
+    char buf[32];
+    snprintf(buf, sizeof(buf), "%02d:%02d.%03d", (int) min, (int) sec, (int) msec);
+
+    return std::string(buf);
+}
+
 int main(int argc, char ** argv) {
     const int64_t t_main_start_us = ggml_time_us();
 
@@ -2128,10 +2183,12 @@ int main(int argc, char ** argv) {
                 printf("%s: WARNING: model is not multilingual, ignoring language and translation options\n", __func__);
             }
         }
-        printf("%s: processing %d samples (%.1f sec), %d threads, lang = %s, task = %s ...\n",
+        printf("%s: processing %d samples (%.1f sec), %d threads, lang = %s, task = %s, timestamps = %d ...\n",
                 __func__, int(pcmf32.size()), float(pcmf32.size())/SAMPLE_RATE, params.n_threads,
                 g_lang.at(params.language).second.c_str(),
-                params.translate ? "translate" : "transcribe");
+                params.translate ? "translate" : "transcribe",
+                params.no_timestamps ? 0 : 1);
+        printf("\n");
     }
 
     // the accumulated text context so far
@@ -2148,6 +2205,9 @@ int main(int argc, char ** argv) {
         }
     }
 
+    // the generated text including timestamps
+    std::vector<whisper_result> result_all;
+
     // main loop
     int seek = 0;
     while (true) {
@@ -2165,7 +2225,7 @@ int main(int argc, char ** argv) {
                 return 1;
             }
 
-            t_encode_us = ggml_time_us() - t_start_us;
+            t_encode_us += ggml_time_us() - t_start_us;
         }
 
         std::vector<float> probs;
@@ -2192,11 +2252,16 @@ int main(int argc, char ** argv) {
         int seek_delta = 100*CHUNK_SIZE;
         whisper_vocab::id last_id = 0;
 
+        //printf("\n\n");
         //for (int i = 0; i < prompt.size(); i++) {
         //    printf("%s: prompt[%d] = %s\n", __func__, i, vocab.id_to_token[prompt[i]].c_str());
         //}
+        //printf("\n\n");
+
+        // the accumulated transcription in the current interation
+        int result_len = 0;
+        std::vector<whisper_result> result_cur;
 
-        printf("\n");
         for (int i = 0; i < model.hparams.n_text_ctx/2; ++i) {
             // decode
             if (prompt.size() > 0) {
@@ -2216,63 +2281,118 @@ int main(int argc, char ** argv) {
             // very basic greedy sampling strategy:
             //
             //   - always take the most probable token
-            //   - if we have accumulated more than 'params.max_tokens_per_iter' -> pick most probable timestamp token
-            //     and advance the sliding window by that amount
-            //   - in the meantime, if we encounter 2 consecutive timestamp tokens, we advance the sliding window too
             //
             // more sophisticated sampling strategies could be implemented here, but we keep it simple
             // feel free to experiment!
             //
             {
-                // sample next token
-                const float temp  = 1.0; // TODO
-
                 const int n_vocab = model.hparams.n_vocab;
 
                 whisper_vocab::id id  = 0;
+                whisper_vocab::id tid = vocab.token_beg;
 
                 {
                     const int64_t t_start_sample_us = ggml_time_us();
 
-                    id = whisper_sample_best(vocab, probs.data() + (probs.size() - n_vocab), temp, i > params.max_tokens_per_iter ? vocab.token_beg : 0);
+                    id = whisper_sample_best(vocab, probs.data() + (probs.size() - n_vocab));
+                    if (i > 0) {
+                        tid = whisper_sample_timestamp(vocab, probs.data() + (probs.size() - n_vocab));
+                    }
 
                     t_sample_us += ggml_time_us() - t_start_sample_us;
                 }
 
-                // end of text token
+                // update sliding window
-                if (id == vocab.token_eot) {
+                if (id > vocab.token_beg) {
-                    break;
-                }
-
-                // 2 consecutive time tokens
-                if (id > vocab.token_beg && last_id > vocab.token_beg) {
                     seek_delta = 2*(id - vocab.token_beg);
-                    done = true;
+                    result_len = i + 1;
                 }
                 last_id = id;
 
                 // add it to the context
                 prompt.push_back(id);
-                prompt_past.push_back(id);
+                result_cur.push_back({ id, seek + 2*(tid - vocab.token_beg) });
-            }
 
-            // display text
+                // end of text token
-            for (auto id : prompt) {
+                if (id == vocab.token_eot) {
-                if (params.print_special_tokens == false && id >= vocab.token_eot) {
+                    break;
-                    continue;
                 }
-                printf("%s", vocab.id_to_token[id].c_str());
             }
-            fflush(stdout);
 
             if (done) {
                 break;
             }
         }
 
+        result_cur.resize(result_len);
+        result_all.insert(result_all.end(), result_cur.begin(), result_cur.end());
+
+        for (const auto & r : result_cur) {
+            prompt_past.push_back(r.id);
+        }
+
+        // print the text from this iteration
+        if (result_cur.size() > 0) {
+            auto t0 = result_cur.front().t;
+
+            std::string text = "";
+            for (int i = 0; i < result_cur.size(); i++) {
+                if (params.print_special_tokens == false && result_cur[i].id >= vocab.token_eot) {
+                } else {
+                    text += vocab.id_to_token[result_cur[i].id];
+                }
+                if (result_cur[i].id > vocab.token_beg) {
+                    const auto t1 = result_cur[i].t;
+                    if (!text.empty()) {
+                        if (params.no_timestamps) {
+                            printf ("%s", text.c_str());
+                            fflush(stdout);
+                        } else {
+                            printf ("[%s --> %s]  %s\n", to_timestamp(t0).c_str(), to_timestamp(t1).c_str(), text.c_str());
+                        }
+                    }
+                    text = "";
+                    while (result_cur[i].id > vocab.token_beg && i < result_cur.size()) {
+                        i++;
+                    }
+                    i--;
+                    t0 = result_cur[i].t;
+                }
+            }
+
+            if (!text.empty()) {
+                printf ("[%s --> %s]  %s\n", to_timestamp(t0).c_str(), to_timestamp(seek + seek_delta).c_str(), text.c_str());
+            }
+        }
+
         seek += seek_delta;
     }
 
+    // WIP: attempt for per-token timestamps
+    //if (!params.no_timestamps && result_all.size() > 0) {
+    //    const int64_t dt = 500; // 5 second intervals
+
+    //    int i0 = 0;
+
+    //    int64_t t0 = result_all[0].t;
+    //    int64_t t1 = t0;
+
+    //    printf("\n\n");
+    //    for (int i = 0; i < result_all.size(); ++i) {
+    //        printf("'%s' -> %lld\n", vocab.id_to_token[result_all[i].id].c_str(), result_all[i].t);
+    //        if (result_all[i].t - t0 > dt) {
+    //            t1 = result_all[i - 1].t;
+    //            printf("[%s --> %s] ", to_timestamp(t0).c_str(), to_timestamp(t1).c_str());
+    //            for (int j = i0; j < i; ++j) {
+    //                printf("%s", vocab.id_to_token.at(result_all[j].id).c_str());
+    //            }
+    //            printf("\n");
+    //            i0 = i;
+    //            t0 = result_all[i].t;
+    //        }
+    //    }
+    //}
+
     // report timing
     {
         const int64_t t_main_end_us = ggml_time_us();