ExternalVendorCode/whisper.cpp
1
0
Fork 0
mirror of https://github.com/ggerganov/whisper.cpp.git synced 2024-12-18 20:27:53 +00:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

ref #4 : added transcription timestamps

Browse Source 
Can be turned off with "-nt" argument.
Performance has also improved.
This commit is contained in:
Georgi Gerganov 2022-09-29 23:09:04 +03:00
parent 8d4041c31f
commit 5877c3578e
No known key found for this signature in database
GPG Key ID: 449E073F9DC10735
2 changed files with 235 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

91
README.md
View File

@ -31,7 +31,7 @@ $ make base.en
gcc -pthread -O3 -mavx -mavx2 -mfma -mf16c -c ggml.c
g++ -pthread -O3 -std=c++11 -c main.cpp
g++ -o main ggml.o main.o
g++ -pthread -o main ggml.o main.o
./main -h
usage: ./main [options]
@ -40,22 +40,17 @@ options:
-h, --help show this help message and exit
-s SEED, --seed SEED RNG seed (default: -1)
-t N, --threads N number of threads to use during computation (default: 4)
-T N, --tokens N maximum number of tokens to generate per iteration (default: 64)
-v, --verbose verbose output
--translate translate from source language to english
-ps, --print_special print special tokens
-nt, --no_timestamps do not print timestamps
-l LANG, --language LANG spoken language (default: en)
-m FNAME, --model FNAME model path (default: models/ggml-base.en.bin)
-f FNAME, --file FNAME input WAV file path (default: samples/jfk.wav)
bash ./download-ggml-model.sh base.en
Downloading ggml model base.en ...
models/ggml-base.en.bin 100%[=====================================>] 141.11M 7.84MB/s in 18s
Done! Model 'base.en' saved in 'models/ggml-base.en.bin'
You can now use it like this:
$ ./main -m models/ggml-base.en.bin -f samples/jfk.wav
Model base.en already exists. Skipping download.
===============================================
Running base.en on all samples in ./samples ...
@ -86,16 +81,17 @@ whisper_model_load: model size = 140.54 MB
log_mel_spectrogram: n_sample = 176000, n_len = 1100
log_mel_spectrogram: recording length: 11.000000 s
main: processing 176000 samples (11.0 sec), 4 threads, lang = english, task = transcribe ...
main: processing 176000 samples (11.0 sec), 4 threads, lang = english, task = transcribe, timestamps = 1 ...
And so my fellow Americans ask not what your country can do for you. Ask what you can do for your country.
[00:00.000 --> 00:11.000] And so my fellow Americans ask not what your country can do for you. Ask what you can do for your country.
main: load time = 71.89 ms
main: mel time = 36.95 ms
main: load time = 61.78 ms
main: mel time = 41.74 ms
main: sample time = 2.10 ms
main: encode time = 700.94 ms / 116.82 ms per layer
main: decode time = 86.14 ms
main: total time = 898.72 ms
main: encode time = 718.60 ms / 119.77 ms per layer
main: decode time = 83.55 ms
main: total time = 908.15 ms
```
The command downloads the `base.en` model converted to custom `ggml` format and runs the inference on all `.wav` samples in the folder `samples`.
@ -131,10 +127,73 @@ For example, you can use `ffmpeg` like this:
ffmpeg -i input.mp3 -ar 16000 -ac 1 -c:a pcm_s16le output.wav
```
Here is another example of transcribing a [3:24 min speech](https://upload.wikimedia.org/wikipedia/commons/1/1f/George_W_Bush_Columbia_FINAL.ogg) in less than a minute, using `medium.en` model:
```bash
$ ./main -m models/ggml-medium.en.bin -f samples/gb1.wav -t 8
whisper_model_load: loading model from 'models/ggml-medium.en.bin'
whisper_model_load: n_vocab = 51864
whisper_model_load: n_audio_ctx = 1500
whisper_model_load: n_audio_state = 1024
whisper_model_load: n_audio_head = 16
whisper_model_load: n_audio_layer = 24
whisper_model_load: n_text_ctx = 448
whisper_model_load: n_text_state = 1024
whisper_model_load: n_text_head = 16
whisper_model_load: n_text_layer = 24
whisper_model_load: n_mels = 80
whisper_model_load: f16 = 1
whisper_model_load: type = 4
whisper_model_load: mem_required = 2786.00 MB
whisper_model_load: adding 1607 extra tokens
whisper_model_load: ggml ctx size = 1644.97 MB
whisper_model_load: memory size = 182.62 MB
whisper_model_load: model size = 1462.12 MB
log_mel_spectrogram: n_sample = 3179750, n_len = 19873
log_mel_spectrogram: recording length: 198.734375 s
main: processing 3179750 samples (198.7 sec), 8 threads, lang = english, task = transcribe, timestamps = 1 ...
[00:00.000 --> 00:08.000] My fellow Americans, this day has brought terrible news and great sadness to our country.
[00:08.000 --> 00:17.000] At 9 o'clock this morning, Mission Control in Houston lost contact with our Space Shuttle Columbia.
[00:17.000 --> 00:24.000] A short time later, debris was seen falling from the skies above Texas.
[00:24.000 --> 00:29.000] The Columbia's lost. There are no survivors.
[00:29.000 --> 00:32.000] On board was a crew of seven.
[00:32.000 --> 00:43.000] Colonel Rick Husband, Lieutenant Colonel Michael Anderson, Commander Laurel Clark, Captain David Brown, Commander William McCool,
[00:43.000 --> 00:52.000] Dr. Kultner Aschavla, and Elon Ramon, a Colonel in the Israeli Air Force.
[00:52.000 --> 00:58.000] These men and women assumed great risk in the service to all humanity.
[00:58.000 --> 01:06.000] In an age when space flight has come to seem almost routine, it is easy to overlook the dangers of travel by rocket
[01:06.000 --> 01:12.000] and the difficulties of navigating the fierce outer atmosphere of the Earth.
[01:12.000 --> 01:22.000] These astronauts knew the dangers, and they faced them willingly, knowing they had a high and noble purpose in life.
[01:22.000 --> 01:30.000] Because of their courage, endearing, and idealism, we will miss them all the more.
[01:30.000 --> 01:40.000] All Americans today are thinking as well of the families of these men and women who have been given this sudden shock and grief.
[01:40.000 --> 01:45.000] You're not alone. Our entire nation agrees with you.
[01:45.000 --> 01:52.000] And those you love will always have the respect and gratitude of this country.
[01:52.000 --> 01:56.000] The cause in which they died will continue.
[01:56.000 --> 02:07.000] Mankind is led into the darkness beyond our world by the inspiration of discovery and the longing to understand.
[02:07.000 --> 02:11.000] Our journey into space will go on.
[02:11.000 --> 02:16.000] In the skies today, we saw destruction and tragedy.
[02:16.000 --> 02:22.000] Yet farther than we can see, there is comfort and hope.
[02:22.000 --> 02:31.000] In the words of the prophet Isaiah, "Lift your eyes and look to the heavens who created all these.
[02:31.000 --> 02:39.000] He who brings out the starry hosts one by one and calls them each by name."
[02:39.000 --> 02:46.000] Because of his great power and mighty strength, not one of them is missing.
[02:46.000 --> 02:55.000] The same creator who names the stars also knows the names of the seven souls we mourn today.
[02:55.000 --> 03:05.000] The crew of the shuttle Columbia did not return safely to Earth, yet we can pray that all are safely home.
[03:05.000 --> 03:14.000] May God bless the grieving families and may God continue to bless America.
[03:14.000 --> 03:24.000] [Music]
main: load time = 438.55 ms
main: mel time = 440.22 ms
main: sample time = 32.23 ms
main: encode time = 42329.63 ms / 1763.73 ms per layer
main: decode time = 15190.00 ms
main: total time = 58444.63 ms
```
## Limitations
- Very basic greedy sampling scheme - always pick up the top token
- No timestamps
- Inference only
- Runs on the CPU
- Only mono-channel 16-bit WAV is supported

196
main.cpp
View File

@ -206,6 +206,7 @@ struct whisper_vocab {
id token_sot = 50257;
id token_prev = 50360;
id token_solm = 50361; // ??
id token_not = 50362; // no timestamps
id token_beg = 50363;
// available tasks
@ -217,17 +218,20 @@ struct whisper_vocab {
}
};
struct whisper_result {
whisper_vocab::id id;
int64_t t;
};
// command-line parameters
struct whisper_params {
int32_t seed = -1; // RNG seed, not used currently
int32_t n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
// sampling parameter - used for the greedy strategy
int32_t max_tokens_per_iter = 64;
bool verbose = false;
bool translate = false;
bool print_special_tokens = false;
bool no_timestamps = false;
std::string language = "en";
std::string model = "models/ggml-base.en.bin";
@ -244,8 +248,6 @@ bool whisper_params_parse(int argc, char ** argv, whisper_params & params) {
params.seed = std::stoi(argv[++i]);
} else if (arg == "-t" || arg == "--threads") {
params.n_threads = std::stoi(argv[++i]);
} else if (arg == "-T" || arg == "--tokens") {
params.max_tokens_per_iter = std::stoi(argv[++i]);
} else if (arg == "-v" || arg == "--verbose") {
params.verbose = true;
} else if (arg == "--translate") {
@ -259,6 +261,8 @@ bool whisper_params_parse(int argc, char ** argv, whisper_params & params) {
}
} else if (arg == "-ps" || arg == "--print_special") {
params.print_special_tokens = true;
} else if (arg == "-nt" || arg == "--no_timestamps") {
params.no_timestamps = true;
} else if (arg == "-m" || arg == "--model") {
params.model = argv[++i];
} else if (arg == "-f" || arg == "--file") {
@ -284,10 +288,10 @@ void whisper_print_usage(int argc, char ** argv, const whisper_params & params)
fprintf(stderr, " -h, --help show this help message and exit\n");
fprintf(stderr, " -s SEED, --seed SEED RNG seed (default: -1)\n");
fprintf(stderr, " -t N, --threads N number of threads to use during computation (default: %d)\n", params.n_threads);
fprintf(stderr, " -T N, --tokens N maximum number of tokens to generate per iteration (default: %d)\n", params.max_tokens_per_iter);
fprintf(stderr, " -v, --verbose verbose output\n");
fprintf(stderr, " --translate translate from source language to english\n");
fprintf(stderr, " -ps, --print_special print special tokens\n");
fprintf(stderr, " -nt, --no_timestamps do not print timestamps\n");
fprintf(stderr, " -l LANG, --language LANG spoken language (default: %s)\n", params.language.c_str());
fprintf(stderr, " -m FNAME, --model FNAME model path (default: %s)\n", params.model.c_str());
fprintf(stderr, " -f FNAME, --file FNAME input WAV file path (default: %s)\n", params.fname_inp.c_str());
@ -591,6 +595,7 @@ bool whisper_model_load(const std::string & fname, whisper_model & model, whispe
vocab.token_sot++;
vocab.token_prev++;
vocab.token_solm++;
vocab.token_not++;
vocab.token_beg++;
}
@ -605,6 +610,8 @@ bool whisper_model_load(const std::string & fname, whisper_model & model, whispe
word = "[_SOT_]";
} else if (i == vocab.token_prev) {
word = "[_PREV_]";
} else if (i == vocab.token_not) {
word = "[_NOT_]";
} else if (i == vocab.token_beg) {
word = "[_BEG_]";
} else {
@ -1842,15 +1849,13 @@ bool whisper_decode(
// TODO: temperature
whisper_vocab::id whisper_sample_best(
const whisper_vocab & vocab,
const float * probs,
double temp,
int offset = 0) {
const float * probs) {
int n_logits = vocab.id_to_token.size();
std::vector<std::pair<double, whisper_vocab::id>> probs_id;
probs_id.reserve(n_logits);
for (int i = offset; i < n_logits; i++) {
for (int i = 0; i < n_logits; i++) {
probs_id.push_back(std::make_pair(probs[i], i));
}
@ -1872,13 +1877,49 @@ whisper_vocab::id whisper_sample_best(
//}
int res = 0;
while (probs_id[res].second == vocab.token_solm && res < (int) probs_id.size() - 1) {
while ((probs_id[res].second == vocab.token_sot ||
probs_id[res].second == vocab.token_solm ||
probs_id[res].second == vocab.token_not) &&
res < (int) probs_id.size() - 1) {
res++;
}
return probs_id[res].second;
}
// samples only from the timestamps tokens
whisper_vocab::id whisper_sample_timestamp(
const whisper_vocab & vocab,
const float * probs) {
int n_logits = vocab.id_to_token.size();
std::vector<std::pair<double, whisper_vocab::id>> probs_id;
probs_id.reserve(n_logits);
for (int i = vocab.token_beg + 1; i < n_logits; i++) {
probs_id.push_back(std::make_pair(probs[i], i));
}
const int top_k = 10;
// find the top K tokens
std::partial_sort(
probs_id.begin(),
probs_id.begin() + top_k, probs_id.end(),
[](const std::pair<double, whisper_vocab::id> & a, const std::pair<double, whisper_vocab::id> & b) {
return a.first > b.first;
});
probs_id.resize(top_k);
//printf("\n");
//for (int i = 0; i < (int) probs_id.size(); i++) {
// printf("%d: '%s' %f, %d\n", i, vocab.id_to_token.at(probs_id[i].second).c_str(), probs_id[i].first, probs_id[i].second);
//}
return probs_id[0].second;
}
// Cooley-Tukey FFT
// poor man's implmentation - use something better
// input is real-valued
@ -2032,6 +2073,20 @@ bool log_mel_spectrogram(
return true;
}
// 500 -> 00:05.000
// 6000 -> 01:00.000
std::string to_timestamp(int64_t t) {
int64_t sec = t/100;
int64_t msec = t - sec*100;
int64_t min = sec/60;
sec = sec - min*60;
char buf[32];
snprintf(buf, sizeof(buf), "%02d:%02d.%03d", (int) min, (int) sec, (int) msec);
return std::string(buf);
}
int main(int argc, char ** argv) {
const int64_t t_main_start_us = ggml_time_us();
@ -2128,10 +2183,12 @@ int main(int argc, char ** argv) {
printf("%s: WARNING: model is not multilingual, ignoring language and translation options\n", __func__);
}
}
printf("%s: processing %d samples (%.1f sec), %d threads, lang = %s, task = %s ...\n",
printf("%s: processing %d samples (%.1f sec), %d threads, lang = %s, task = %s, timestamps = %d ...\n",
__func__, int(pcmf32.size()), float(pcmf32.size())/SAMPLE_RATE, params.n_threads,
g_lang.at(params.language).second.c_str(),
params.translate ? "translate" : "transcribe");
params.translate ? "translate" : "transcribe",
params.no_timestamps ? 0 : 1);
printf("\n");
}
// the accumulated text context so far
@ -2148,6 +2205,9 @@ int main(int argc, char ** argv) {
}
}
// the generated text including timestamps
std::vector<whisper_result> result_all;
// main loop
int seek = 0;
while (true) {
@ -2165,7 +2225,7 @@ int main(int argc, char ** argv) {
return 1;
}
t_encode_us = ggml_time_us() - t_start_us;
t_encode_us += ggml_time_us() - t_start_us;
}
std::vector<float> probs;
@ -2192,11 +2252,16 @@ int main(int argc, char ** argv) {
int seek_delta = 100*CHUNK_SIZE;
whisper_vocab::id last_id = 0;
//printf("\n\n");
//for (int i = 0; i < prompt.size(); i++) {
// printf("%s: prompt[%d] = %s\n", __func__, i, vocab.id_to_token[prompt[i]].c_str());
//}
//printf("\n\n");
// the accumulated transcription in the current interation
int result_len = 0;
std::vector<whisper_result> result_cur;
printf("\n");
for (int i = 0; i < model.hparams.n_text_ctx/2; ++i) {
// decode
if (prompt.size() > 0) {
@ -2216,63 +2281,118 @@ int main(int argc, char ** argv) {
// very basic greedy sampling strategy:
//
// - always take the most probable token
// - if we have accumulated more than 'params.max_tokens_per_iter' -> pick most probable timestamp token
// and advance the sliding window by that amount
// - in the meantime, if we encounter 2 consecutive timestamp tokens, we advance the sliding window too
//
// more sophisticated sampling strategies could be implemented here, but we keep it simple
// feel free to experiment!
//
{
// sample next token
const float temp = 1.0; // TODO
const int n_vocab = model.hparams.n_vocab;
whisper_vocab::id id = 0;
whisper_vocab::id tid = vocab.token_beg;
{
const int64_t t_start_sample_us = ggml_time_us();
id = whisper_sample_best(vocab, probs.data() + (probs.size() - n_vocab), temp, i > params.max_tokens_per_iter ? vocab.token_beg : 0);
id = whisper_sample_best(vocab, probs.data() + (probs.size() - n_vocab));
if (i > 0) {
tid = whisper_sample_timestamp(vocab, probs.data() + (probs.size() - n_vocab));
}
t_sample_us += ggml_time_us() - t_start_sample_us;
}
// end of text token
if (id == vocab.token_eot) {
break;
}
// 2 consecutive time tokens
if (id > vocab.token_beg && last_id > vocab.token_beg) {
// update sliding window
if (id > vocab.token_beg) {
seek_delta = 2*(id - vocab.token_beg);
done = true;
result_len = i + 1;
}
last_id = id;
// add it to the context
prompt.push_back(id);
prompt_past.push_back(id);
}
result_cur.push_back({ id, seek + 2*(tid - vocab.token_beg) });
// display text
for (auto id : prompt) {
if (params.print_special_tokens == false && id >= vocab.token_eot) {
continue;
// end of text token
if (id == vocab.token_eot) {
break;
}
printf("%s", vocab.id_to_token[id].c_str());
}
fflush(stdout);
if (done) {
break;
}
}
result_cur.resize(result_len);
result_all.insert(result_all.end(), result_cur.begin(), result_cur.end());
for (const auto & r : result_cur) {
prompt_past.push_back(r.id);
}
// print the text from this iteration
if (result_cur.size() > 0) {
auto t0 = result_cur.front().t;
std::string text = "";
for (int i = 0; i < result_cur.size(); i++) {
if (params.print_special_tokens == false && result_cur[i].id >= vocab.token_eot) {
} else {
text += vocab.id_to_token[result_cur[i].id];
}
if (result_cur[i].id > vocab.token_beg) {
const auto t1 = result_cur[i].t;
if (!text.empty()) {
if (params.no_timestamps) {
printf ("%s", text.c_str());
fflush(stdout);
} else {
printf ("[%s --> %s] %s\n", to_timestamp(t0).c_str(), to_timestamp(t1).c_str(), text.c_str());
}
}
text = "";
while (result_cur[i].id > vocab.token_beg && i < result_cur.size()) {
i++;
}
i--;
t0 = result_cur[i].t;
}
}
if (!text.empty()) {
printf ("[%s --> %s] %s\n", to_timestamp(t0).c_str(), to_timestamp(seek + seek_delta).c_str(), text.c_str());
}
}
seek += seek_delta;
}
// WIP: attempt for per-token timestamps
//if (!params.no_timestamps && result_all.size() > 0) {
// const int64_t dt = 500; // 5 second intervals
// int i0 = 0;
// int64_t t0 = result_all[0].t;
// int64_t t1 = t0;
// printf("\n\n");
// for (int i = 0; i < result_all.size(); ++i) {
// printf("'%s' -> %lld\n", vocab.id_to_token[result_all[i].id].c_str(), result_all[i].t);
// if (result_all[i].t - t0 > dt) {
// t1 = result_all[i - 1].t;
// printf("[%s --> %s] ", to_timestamp(t0).c_str(), to_timestamp(t1).c_str());
// for (int j = i0; j < i; ++j) {
// printf("%s", vocab.id_to_token.at(result_all[j].id).c_str());
// }
// printf("\n");
// i0 = i;
// t0 = result_all[i].t;
// }
// }
//}
// report timing
{
const int64_t t_main_end_us = ggml_time_us();