coreml : set convert_to="mlprogram" in convert

* coreml : skip model load in convert-whisper-to-coreml.py

This commit updates the conversion process for Whisper models to use the
"mlprogram" format instead of "neuralnetwork".

The motivation for this change is that when using the "neuralnetwork"
format the underlying model produced is based on protobuf and my
understanding is that there are limitations to this format, such as
sizes of strings and the complexity of the model.

Currently when trying to convert larger models such as large-v3 the
conversion fails but succeeds for smaller models.

The "mlprogram" format is a more recent addition to CoreML and is
designed to be more flexible and powerful, allowing for more complex
models and larger data types. This seems to work for larger and smaller
models alike and unless I'm there are considerations that I'm not aware
of I think this is what we should be using moving forward.
The error that is generated for large models is the following:
```console
Running MIL backend_neuralnetwork pipeline: 100%|█████████| 9/9 [00:00<00:00, 35.44 passes/s]
Translating MIL ==> NeuralNetwork Ops: 100%|███████████| 5641/5641 [03:31<00:00, 26.65 ops/s]
Traceback (most recent call last):
  File "/Users/danbev/work/ai/whisper-work/models/convert-whisper-to-coreml.py", line 322, in <module>
    encoder = convert_encoder(hparams, encoder, quantize=args.quantize)
              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/danbev/work/ai/whisper-work/models/convert-whisper-to-coreml.py", line 255, in convert_encoder
    model = ct.convert(
            ^^^^^^^^^^^
  File "/Users/danbev/work/ai/whisper-work/venv/lib/python3.11/site-packages/coremltools/converters/_converters_entry.py", line 635, in convert
    mlmodel = mil_convert(
              ^^^^^^^^^^^^
  File "/Users/danbev/work/ai/whisper-work/venv/lib/python3.11/site-packages/coremltools/converters/mil/converter.py", line 186, in mil_convert
    return _mil_convert(
           ^^^^^^^^^^^^^
  File "/Users/danbev/work/ai/whisper-work/venv/lib/python3.11/site-packages/coremltools/converters/mil/converter.py", line 245, in _mil_convert
    return modelClass(
           ^^^^^^^^^^^
  File "/Users/danbev/work/ai/whisper-work/venv/lib/python3.11/site-packages/coremltools/models/model.py", line 489, in __init__
    self.__proxy__, self._spec, self._framework_error = self._get_proxy_and_spec(
                                                        ^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/danbev/work/ai/whisper-work/venv/lib/python3.11/site-packages/coremltools/models/model.py", line 550, in _get_proxy_and_spec
    _MLModelProxy(
ValueError: basic_string
```

Refs: https://github.com/ggml-org/whisper.cpp/issues/3012

.devops/cublas.Dockerfile

@@ -12,7 +12,7 @@ FROM ${BASE_CUDA_DEV_CONTAINER} as build
 ARG CUDA_DOCKER_ARCH=all
 
 RUN apt-get update && \
-    apt-get install -y build-essential git cmake libsdl2-dev wget
+    apt-get install -y build-essential git cmake libsdl2-dev wget git
 
 WORKDIR /app
 

.devops/main-cuda.Dockerfile

@@ -17,7 +17,7 @@ ENV CUDA_DOCKER_ARCH=${CUDA_DOCKER_ARCH}
 ENV GGML_CUDA=1
 
 RUN apt-get update && \
-    apt-get install -y build-essential libsdl2-dev wget cmake \
+    apt-get install -y build-essential libsdl2-dev wget cmake git \
     && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
 
 # Ref: https://stackoverflow.com/a/53464012
@@ -33,7 +33,7 @@ ENV LD_LIBRARY_PATH /usr/local/cuda-${CUDA_MAIN_VERSION}/compat:$LD_LIBRARY_PATH
 WORKDIR /app
 
 RUN apt-get update && \
-  apt-get install -y curl ffmpeg wget cmake \
+  apt-get install -y curl ffmpeg wget cmake git \
   && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
 
 COPY --from=build /app /app

.devops/main.Dockerfile

@@ -2,7 +2,7 @@ FROM ubuntu:22.04 AS build
 WORKDIR /app
 
 RUN apt-get update && \
-  apt-get install -y build-essential wget cmake \
+  apt-get install -y build-essential wget cmake git \
   && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
 
 COPY .. .
@@ -12,7 +12,7 @@ FROM ubuntu:22.04 AS runtime
 WORKDIR /app
 
 RUN apt-get update && \
-  apt-get install -y curl ffmpeg libsdl2-dev wget cmake \
+  apt-get install -y curl ffmpeg libsdl2-dev wget cmake git \
   && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
 
 COPY --from=build /app /app

.github/workflows/bindings-go.yml

@@ -10,8 +10,8 @@ on:
       - whisper.h
 
 jobs:
-  ubuntu-latest:
+  ubuntu-22:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/setup-go@v5
         with:

.github/workflows/bindings-ruby.yml

@@ -1,55 +1,21 @@
 name: Bindings Tests (Ruby)
+
 on:
   push:
-    paths:
+    branches:
-      - bindings/ruby/**
+      - master
-      - src/**/*.c
-      - src/**/*.cpp
-      - src/**/*.h
-      - src/**/*.m
-      - src/**/*.metal
-      - include/**/*.c
-      - include/**/*.cpp
-      - include/**/*.h
-      - include/**/*.m
-      - include/**/*.metal
-      - ggml/**/*.c
-      - ggml/**/*.cpp
-      - ggml/**/*.h
-      - ggml/**/*.m
-      - ggml/**/*.metal
-      - scripts/get-flags.mk
-      - examples/dr_wav.h
   pull_request:
-    paths:
+    types: [opened, synchronize, reopened]
-      - bindings/ruby/**
-      - src/**/*.c
-      - src/**/*.cpp
-      - src/**/*.h
-      - src/**/*.m
-      - src/**/*.metal
-      - include/**/*.c
-      - include/**/*.cpp
-      - include/**/*.h
-      - include/**/*.m
-      - include/**/*.metal
-      - ggml/**/*.c
-      - ggml/**/*.cpp
-      - ggml/**/*.h
-      - ggml/**/*.m
-      - ggml/**/*.metal
-      - scripts/get-flags.mk
-      - examples/dr_wav.h
 
 jobs:
-  ubuntu-latest:
+  ubuntu-22:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     defaults:
       run:
         working-directory: bindings/ruby
     steps:
       - uses: ruby/setup-ruby@v1
         with:
-          ruby-version: '3.1'
+          ruby-version: '3.2'
       - uses: actions/checkout@v4
       - run: rake test

.github/workflows/build.yml

@@ -1,18 +1,92 @@
 name: CI
-on: [push, pull_request]
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    types: [opened, synchronize, reopened]
+  workflow_dispatch:
+    inputs:
+      create_release:
+        description: 'Create new release'
+        required: true
+        type: boolean
+      pre_release_tag:
+        description: 'Pre-release tag name'
+        required: false
+        type: string
+      run_type:
+        description: 'Workflow type to run'
+        required: true
+        type: choice
+        options:
+          - full-ci
+          - release-only
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
+  cancel-in-progress: true
+
+permissions:
+  contents: write  # for creating release
 
 env:
+  BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
   ubuntu_image: "ubuntu:22.04"
   VCPKG_BINARY_SOURCES: "clear;x-gha,readwrite"
 
 jobs:
-  ubuntu-latest:
+  determine-tag:
     runs-on: ubuntu-latest
+    outputs:
+      tag_name: ${{ steps.tag.outputs.name }}
+
+    steps:
+      - name: Checkout with full history
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Determine tag name
+        id: tag
+        shell: bash
+        run: |
+          BUILD_NUMBER=$(git rev-list --count HEAD)
+          SHORT_HASH=$(git rev-parse --short=7 HEAD)
+          CUSTOM_TAG="${{ github.event.inputs.pre_release_tag }}"
+
+          echo "Raw values:"
+          echo "BUILD_NUMBER: $BUILD_NUMBER"
+          echo "SHORT_HASH: $SHORT_HASH"
+          echo "BRANCH_NAME: ${{ env.BRANCH_NAME }}"
+          echo "CUSTOM_TAG: $CUSTOM_TAG"
+
+          # Use custom tag if provided
+          if [[ -n "$CUSTOM_TAG" ]]; then
+            echo "Using custom tag"
+            TAG_NAME="${CUSTOM_TAG}"
+          elif [[ "${{ env.BRANCH_NAME }}" == "master" ]]; then
+            echo "Using master branch format"
+            TAG_NAME="b${BUILD_NUMBER}"
+          else
+            echo "Using non-master branch format"
+            SAFE_NAME=$(echo "${{ env.BRANCH_NAME }}" | tr '/' '-')
+            TAG_NAME="${SAFE_NAME}-b${BUILD_NUMBER}-${SHORT_HASH}"
+          fi
+
+          echo "Final tag name: $TAG_NAME"
+          echo "name=$TAG_NAME" >> $GITHUB_OUTPUT
+
+  ubuntu-22:
+    if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' ||
+            github.event.inputs.run_type == 'full-ci' }}
+    runs-on: ubuntu-22.04
 
     strategy:
       fail-fast: false
       matrix:
-        arch: [linux/amd64, linux/arm64, linux/arm/v7, linux/ppc64le]
+        arch: [linux/amd64, linux/ppc64le]
 
     steps:
       - name: Clone
@@ -28,17 +102,86 @@ jobs:
             -w /workspace ${{ env.ubuntu_image }} /bin/sh -c '
             set -e
             apt update
-            apt install -y build-essential libsdl2-dev cmake
+            apt install -y build-essential libsdl2-dev cmake git
             cmake -B build
             cmake --build build --config Release -j $(nproc)'
 
-  macOS-latest:
+  ubuntu-22-arm64:
-    runs-on: macOS-latest
+    if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' ||
+            github.event.inputs.run_type == 'full-ci' }}
+    runs-on: ubuntu-22.04
+
+    strategy:
+      fail-fast: false
+      matrix:
+        arch: [linux/arm64]
 
     steps:
       - name: Clone
         uses: actions/checkout@v4
 
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Build ${{ matrix.arch }}
+        run: |
+          docker run --platform ${{ matrix.arch }} --rm \
+            -v ${{ github.workspace }}:/workspace \
+            -w /workspace ${{ env.ubuntu_image }} /bin/sh -c '
+            set -e
+            apt update
+            apt install -y build-essential libsdl2-dev cmake git
+            cmake -B build -DGGML_NATIVE=OFF -DGGML_CPU_ARM_ARCH=armv8-a
+            cmake --build build --config Release -j $(nproc)'
+
+  ubuntu-22-arm-v7:
+    if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' ||
+            github.event.inputs.run_type == 'full-ci' }}
+    runs-on: ubuntu-22.04
+
+    strategy:
+      fail-fast: false
+      matrix:
+        arch: [linux/arm/v7]
+
+    steps:
+      - name: Clone
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Build ${{ matrix.arch }}
+        run: |
+          docker run --platform ${{ matrix.arch }} --rm \
+            -v ${{ github.workspace }}:/workspace \
+            -w /workspace ${{ env.ubuntu_image }} /bin/sh -c '
+            set -e
+            apt update
+            apt install -y build-essential libsdl2-dev cmake git
+            cmake -B build -DGGML_NATIVE=OFF -DGGML_CPU_ARM_ARCH=armv7-a+fp
+            cmake --build build --config Release -j $(nproc)'
+
+  macOS-latest:
+    if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' ||
+            github.event.inputs.run_type == 'full-ci' }}
+    runs-on: macOS-latest
+
+    strategy:
+      matrix:
+        destination: ['generic/platform=macOS', 'generic/platform=iOS', 'generic/platform=tvOS']
+
+    steps:
+      - name: Clone
+        id: checkout
+        uses: actions/checkout@v4
+
+      - name: ccache
+        uses: hendrikmuhs/ccache-action@v1.2.16
+        with:
+          key: macOS-latest-swift
+          evict-old-files: 1d
+
       - name: Dependencies
         run: |
           brew update
@@ -46,35 +189,45 @@ jobs:
 
       - name: Build
         run: |
-          cmake -B build
+          sysctl -a
-          cmake --build build --config Release
+          cmake -B build -G Xcode \
+            -DGGML_METAL_USE_BF16=ON \
+            -DGGML_METAL_EMBED_LIBRARY=ON \
+            -DWHISPER_BUILD_EXAMPLES=OFF \
+            -DWHISPER_BUILD_TESTS=OFF \
+            -DWHISPER_BUILD_SERVER=OFF \
+            -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64"
+          cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
+
 
 #  freeBSD-latest:
-#    runs-on: macos-12
+#    runs-on: macos-13
 #
 #    steps:
 #      - name: Clone
 #        uses: actions/checkout@v4
 #
 #      - name: Build
-#        uses: cross-platform-actions/action@v0.24.0
+#        uses: cross-platform-actions/action@v0.27.0
 #        with:
 #          operating_system: freebsd
-#          version: '13.3'
+#          version: '14.2'
 #          run: |
 #            sudo pkg update
-#            sudo pkg install -y gmake sdl2 cmake
+#            sudo pkg install -y gmake sdl2 cmake git
 #            cmake -B build
 #            cmake --build build --config Release
 
-  ubuntu-latest-gcc:
+  ubuntu-22-gcc:
-    runs-on: ubuntu-latest
+    if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' ||
+            github.event.inputs.run_type == 'full-ci' }}
+    runs-on: ubuntu-22.04
 
     strategy:
       fail-fast: false
       matrix:
         build: [Debug, Release]
-        arch: [linux/amd64, linux/arm64, linux/arm/v7, linux/ppc64le]
+        arch: [linux/amd64, linux/ppc64le]
 
     steps:
       - name: Clone
@@ -90,13 +243,75 @@ jobs:
             -w /workspace ${{ env.ubuntu_image }} /bin/sh -c '
             set -e
             apt update
-            apt install -y build-essential cmake libsdl2-dev
+            apt install -y build-essential cmake libsdl2-dev git
             cmake . -DWHISPER_SDL2=ON -DCMAKE_BUILD_TYPE=${{ matrix.build }}
             make
             ctest -L gh --output-on-failure'
 
-  ubuntu-latest-clang:
+  ubuntu-22-gcc-arm64:
-    runs-on: ubuntu-latest
+    if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' ||
+            github.event.inputs.run_type == 'full-ci' }}
+    runs-on: ubuntu-22.04
+
+    strategy:
+      fail-fast: false
+      matrix:
+        build: [Debug, Release]
+        arch: [linux/arm64]
+
+    steps:
+      - name: Clone
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Build ${{ matrix.arch }}
+        run: |
+          docker run --platform ${{ matrix.arch }} --rm \
+            -v ${{ github.workspace }}:/workspace \
+            -w /workspace ${{ env.ubuntu_image }} /bin/sh -c '
+            set -e
+            apt update
+            apt install -y build-essential cmake libsdl2-dev git
+            cmake . -DWHISPER_SDL2=ON -DCMAKE_BUILD_TYPE=${{ matrix.build }} -DGGML_NATIVE=OFF -DGGML_CPU_ARM_ARCH=armv8-a
+            make
+            ctest -L gh --output-on-failure'
+
+  ubuntu-22-gcc-arm-v7:
+    if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' ||
+            github.event.inputs.run_type == 'full-ci' }}
+    runs-on: ubuntu-22.04
+
+    strategy:
+      fail-fast: false
+      matrix:
+        build: [Debug, Release]
+        arch: [linux/arm/v7]
+
+    steps:
+      - name: Clone
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Build ${{ matrix.arch }}
+        run: |
+          docker run --platform ${{ matrix.arch }} --rm \
+            -v ${{ github.workspace }}:/workspace \
+            -w /workspace ${{ env.ubuntu_image }} /bin/sh -c '
+            set -e
+            apt update
+            apt install -y build-essential cmake libsdl2-dev git
+            cmake . -DWHISPER_SDL2=ON -DCMAKE_BUILD_TYPE=${{ matrix.build }} -DGGML_NATIVE=OFF -DGGML_CPU_ARM_ARCH=armv7-a+fp
+            make
+            ctest -L gh --output-on-failure'
+
+  ubuntu-22-clang:
+    if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' ||
+            github.event.inputs.run_type == 'full-ci' }}
+    runs-on: ubuntu-22.04
 
     strategy:
       fail-fast: false
@@ -121,13 +336,15 @@ jobs:
             -w /workspace ${{ env.ubuntu_image }} /bin/sh -c '
             set -e
             apt update
-            apt install -y clang build-essential cmake libsdl2-dev
+            apt install -y clang build-essential cmake libsdl2-dev git
             cmake . -DWHISPER_SDL2=ON -DCMAKE_BUILD_TYPE=${{ matrix.build }} -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_C_COMPILER=clang
             make
             ctest -L gh --output-on-failure'
 
-  ubuntu-latest-gcc-sanitized:
+  ubuntu-22-gcc-sanitized:
-    runs-on: ubuntu-latest
+    if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' ||
+            github.event.inputs.run_type == 'full-ci' }}
+    runs-on: ubuntu-22.04
 
     strategy:
       fail-fast: false
@@ -149,12 +366,16 @@ jobs:
             -w /workspace ${{ env.ubuntu_image }} /bin/sh -c '
             set -e
             apt update
-            apt install -y build-essential cmake
+            apt install -y build-essential cmake git
-            cmake . -DCMAKE_BUILD_TYPE=Debug -DWHISPER_SANITIZE_${{ matrix.sanitizer }}=ON
+            cmake . -DCMAKE_BUILD_TYPE=Debug \
+              -DWHISPER_SANITIZE_${{ matrix.sanitizer }}=ON \
+              -DGGML_OPENMP=OFF
             make
             ctest -L gh --output-on-failure'
 
   ubuntu-22-cmake-sycl:
+    if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' ||
+            github.event.inputs.run_type == 'full-ci' }}
     runs-on: ubuntu-22.04
 
     strategy:
@@ -184,12 +405,12 @@ jobs:
         shell: bash
         run: |
           sudo apt update
-          sudo apt install intel-oneapi-compiler-dpcpp-cpp
+          sudo apt install intel-oneapi-compiler-dpcpp-cpp git
 
       - name: install oneAPI MKL library
         shell: bash
         run: |
-          sudo apt install intel-oneapi-mkl-devel
+          sudo apt install intel-oneapi-mkl-devel git
 
       - name: Clone
         id: checkout
@@ -205,6 +426,8 @@ jobs:
           cmake --build . --config Release -j $(nproc)
 
   ubuntu-22-cmake-sycl-fp16:
+    if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' ||
+            github.event.inputs.run_type == 'full-ci' }}
     runs-on: ubuntu-22.04
 
     strategy:
@@ -234,7 +457,7 @@ jobs:
         shell: bash
         run: |
           sudo apt update
-          sudo apt install intel-oneapi-compiler-dpcpp-cpp
+          sudo apt install intel-oneapi-compiler-dpcpp-cpp git
 
       - name: install oneAPI MKL library
         shell: bash
@@ -255,6 +478,8 @@ jobs:
           cmake --build . --config Release -j $(nproc)
 
   windows-msys2:
+    if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' ||
+            github.event.inputs.run_type == 'full-ci' }}
     runs-on: windows-latest
 
     strategy:
@@ -275,6 +500,7 @@ jobs:
           msystem: ${{matrix.sys}}
           install: >-
             base-devel
+            git
             mingw-w64-${{matrix.env}}-toolchain
             mingw-w64-${{matrix.env}}-cmake
             mingw-w64-${{matrix.env}}-SDL2
@@ -298,6 +524,8 @@ jobs:
             cmake --build build --config ${{ matrix.build }} -j $(nproc)
 
   windows:
+    if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' ||
+            github.event.inputs.run_type == 'full-ci' }}
     runs-on: windows-latest
 
     strategy:
@@ -358,6 +586,8 @@ jobs:
           path: build/bin/${{ matrix.build }}
 
   windows-blas:
+    if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' ||
+            github.event.inputs.run_type == 'full-ci' }}
     runs-on: windows-latest
 
     strategy:
@@ -430,75 +660,204 @@ jobs:
           name: whisper-blas-bin-${{ matrix.arch }}
           path: build/bin/${{ matrix.build }}
 
-# TODO: fix and re-enable
+  windows-cublas:
-#  windows-cublas:
+    if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' ||
-#    runs-on: windows-2019
+            github.event.inputs.run_type == 'full-ci' }}
-#
+    runs-on: windows-2019
-#    strategy:
+    strategy:
-#      matrix:
+      matrix:
-#        build: [Release]
+        build: [Release]
-#        arch: [x64]
+        arch: [x64]
-#        cublas: [ON]
+        cublas: [ON]
-#        sdl2: [ON]
+        sdl2: [ON]
-#        cuda-toolkit: [12.2.0, 11.8.0]
+        cuda-toolkit: [12.2.0, 11.8.0]
-#        include:
+        include:
-#          - arch: x64
+          - arch: x64
-#            s2arc: x64
+            sdl2: ON
-#          - sdl2: ON
+            sdl2_ver: 2.28.5
-#            s2ver: 2.28.5
+    steps:
-#
+      - name: Clone repository
-#    steps:
+        uses: actions/checkout@v4
-#      - name: Clone
+
-#        uses: actions/checkout@v4
+      - name: Install Ninja
-#
+        id: install_ninja
-#      - name: Add msbuild to PATH
+        run: |
-#        uses: microsoft/setup-msbuild@v2
+          choco install ninja
-#
+
-#      - name: Install CUDA Toolkit
+      - name: Install ccache
-#        id: cuda-toolkit
+        uses: hendrikmuhs/ccache-action@v1.2.16
-#        uses: Jimver/cuda-toolkit@v0.2.15
+        with:
-#        with:
+          key: ${{ github.job }}-${{ matrix.cuda-toolkit }}-${{ matrix.build }}
-#          cuda: '${{ matrix.cuda-toolkit }}'
+          variant: sccache
-#
+          evict-old-files: 5d
-#      - name: Fetch SDL2 and set SDL2_DIR
+
-#        if: matrix.sdl2 == 'ON'
+      - name: Install Cuda Toolkit 11.8.0
-#        run: |
+        if: ${{ matrix.cuda-toolkit == '11.8.0' }}
-#          C:/msys64/usr/bin/wget.exe -qO sdl2.zip https://github.com/libsdl-org/SDL/releases/download/release-${{ matrix.s2ver }}/SDL2-devel-${{ matrix.s2ver }}-VC.zip
+        run: |
-#          7z x sdl2.zip
+          $CUDA_VERSION = ${{ matrix.cuda-toolkit }}
-#          echo "SDL2_DIR=$env:GITHUB_WORKSPACE/SDL2-${{ matrix.s2ver }}/cmake" >> $env:GITHUB_ENV
+          $CUDA_TOOLKIT_DIR = "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v$CUDA_VERSION"
-#
+          $CUDA_DOWNLOAD = "https://developer.download.nvidia.com/compute/cuda/redist"
-#      - name: Configure
+
-#        run: >
+          # Components versions
-#          cmake -S . -B ./build -A ${{ matrix.arch }}
+          $CUDART_VER = "11.8.89"
-#          -DCMAKE_BUILD_TYPE=${{ matrix.build }}
+          $NVCC_VER   = "11.8.89"
-#          -DGGML_CUDA=${{ matrix.cublas }}
+          $NVRTC_VER  = "11.8.89"
-#          -DWHISPER_SDL2=${{ matrix.sdl2 }}
+          $CUBLAS_VER = "11.8.1.74"
-#
+          $NVTX_VER   = "11.8.86"
-#      - name: Build ${{ matrix.cuda-toolkit }}
+          $VS_VER     = "11.8.86"
-#        run: |
+          $NVPROF_VER = "11.8.87"
-#          cd ./build
+          $CCCL_VER   = "11.8.89"
-#          cmake --build . --config ${{ matrix.build }}
+
-#
+          # Create the directory where the CUDA Toolkit will be installed
-#      - name: Copy CUDA DLLs
+          mkdir -p $CUDA_TOOLKIT_DIR
-#        run: >
+
-#          Copy-Item -PassThru
+          # Install unzip to extract the downloaded files
-#          -Path "${{ steps.cuda-toolkit.outputs.CUDA_PATH }}/bin/*.dll"
+          choco install unzip -y
-#          -Include cudart64_*,cublas64_*,cublasLt64_*
+
-#          -Destination build/bin/${{ matrix.build }}
+          # Download all the required components
-#
+          curl -O "$CUDA_DOWNLOAD/cuda_cudart/windows-x86_64/cuda_cudart-windows-x86_64-${CUDART_VER}-archive.zip"
-#      - name: Copy SDL2.dll
+          curl -O "$CUDA_DOWNLOAD/cuda_nvcc/windows-x86_64/cuda_nvcc-windows-x86_64-${NVCC_VER}-archive.zip"
-#        if: matrix.sdl2 == 'ON'
+          curl -O "$CUDA_DOWNLOAD/cuda_nvrtc/windows-x86_64/cuda_nvrtc-windows-x86_64-${NVRTC_VER}-archive.zip"
-#        run: copy "$env:SDL2_DIR/../lib/${{ matrix.s2arc }}/SDL2.dll" build/bin/${{ matrix.build }}
+          curl -O "$CUDA_DOWNLOAD/libcublas/windows-x86_64/libcublas-windows-x86_64-${CUBLAS_VER}-archive.zip"
-#
+          curl -O "$CUDA_DOWNLOAD/cuda_nvtx/windows-x86_64/cuda_nvtx-windows-x86_64-${NVTX_VER}-archive.zip"
-#      - name: Upload binaries
+          curl -O "$CUDA_DOWNLOAD/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-${VS_VER}-archive.zip"
-#        if: matrix.sdl2 == 'ON'
+          curl -O "$CUDA_DOWNLOAD/cuda_nvprof/windows-x86_64/cuda_nvprof-windows-x86_64-${NVPROF_VER}-archive.zip"
-#        uses: actions/upload-artifact@v4
+          curl -O "$CUDA_DOWNLOAD/cuda_cccl/windows-x86_64/cuda_cccl-windows-x86_64-${CCCL_VER}-archive.zip"
-#        with:
+
-#          name: whisper-cublas-${{ matrix.cuda-toolkit }}-bin-${{ matrix.arch }}
+          # Extract all the downloaded files to the CUDA Toolkit directory
-#          path: build/bin/${{ matrix.build }}
+          unzip '*.zip' -d $CUDA_TOOLKIT_DIR
+
+          # Copy all the extracted files to the main CUDA Toolkit directory
+          xcopy "$CUDA_TOOLKIT_DIR\cuda_cudart-windows-x86_64-${CUDART_VER}-archive\*" "$CUDA_TOOLKIT_DIR" /E /I /H /Y
+          xcopy "$CUDA_TOOLKIT_DIR\cuda_nvcc-windows-x86_64-${NVCC_VER}-archive\*" "$CUDA_TOOLKIT_DIR" /E /I /H /Y
+          xcopy "$CUDA_TOOLKIT_DIR\cuda_nvrtc-windows-x86_64-${NVRTC_VER}-archive\*" "$CUDA_TOOLKIT_DIR" /E /I /H /Y
+          xcopy "$CUDA_TOOLKIT_DIR\libcublas-windows-x86_64-${CUBLAS_VER}-archive\*" "$CUDA_TOOLKIT_DIR" /E /I /H /Y
+          xcopy "$CUDA_TOOLKIT_DIR\cuda_nvtx-windows-x86_64-${NVTX_VER}-archive\*" "$CUDA_TOOLKIT_DIR" /E /I /H /Y
+          xcopy "$CUDA_TOOLKIT_DIR\cuda_nvprof-windows-x86_64-${NVPROF_VER}-archive\*" "$CUDA_TOOLKIT_DIR" /E /I /H /Y
+          xcopy "$CUDA_TOOLKIT_DIR\cuda_cccl-windows-x86_64-${CCCL_VER}-archive\*" "$CUDA_TOOLKIT_DIR" /E /I /H /Y
+          xcopy "$CUDA_TOOLKIT_DIR\visual_studio_integration-windows-x86_64-${VS_VER}-archive\*" "$CUDA_TOOLKIT_DIR" /E /I /H /Y
+
+          # Visual Studio integration
+          xcopy "$CUDA_TOOLKIT_DIR\visual_studio_integration-windows-x86_64-${VS_VER}-archive\visual_studio_integration\MSBuildExtensions\*" "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Microsoft\VC\v160\BuildCustomizations" /E /I /H /Y
+
+          # Set environment variables
+          echo "$CUDA_TOOLKIT_DIR\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+          echo "$CUDA_TOOLKIT_DIR\libnvvp" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+          echo "CUDA_PATH=$CUDA_TOOLKIT_DIR" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
+          echo "CUDA_PATH_V11_8=$CUDA_TOOLKIT_DIR" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
+        
+      - name: Install Cuda Toolkit 12.2.0
+        if: ${{ matrix.cuda-toolkit == '12.2.0' }}
+        run: |
+          $CUDA_VERSION = ${{ matrix.cuda-toolkit }}
+          $CUDA_TOOLKIT_DIR = "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v$CUDA_VERSION"
+          $CUDA_DOWNLOAD = "https://developer.download.nvidia.com/compute/cuda/redist"
+
+          # Components versions
+          $CUDART_VER   = "12.2.140"
+          $NVCC_VER     = "12.2.140"
+          $NVRTC_VER    = "12.2.140"
+          $CUBLAS_VER   = "12.2.5.6"
+          $NVTX_VER     = "12.2.140"
+          $PROFILER_VER = "12.2.140"
+          $VS_VER       = "12.2.140"
+          $NVPROF_VER   = "12.2.142"
+          $CCCL_VER     = "12.2.140"
+
+          # Create the directory where the CUDA Toolkit will be installed
+          mkdir -p $CUDA_TOOLKIT_DIR
+
+          # Install unzip to extract the downloaded files
+          choco install unzip -y
+
+          # Download all the required components
+          curl -O "$CUDA_DOWNLOAD/cuda_cudart/windows-x86_64/cuda_cudart-windows-x86_64-${CUDART_VER}-archive.zip"
+          curl -O "$CUDA_DOWNLOAD/cuda_nvcc/windows-x86_64/cuda_nvcc-windows-x86_64-${NVCC_VER}-archive.zip"
+          curl -O "$CUDA_DOWNLOAD/cuda_nvrtc/windows-x86_64/cuda_nvrtc-windows-x86_64-${NVRTC_VER}-archive.zip"
+          curl -O "$CUDA_DOWNLOAD/libcublas/windows-x86_64/libcublas-windows-x86_64-${CUBLAS_VER}-archive.zip"
+          curl -O "$CUDA_DOWNLOAD/cuda_nvtx/windows-x86_64/cuda_nvtx-windows-x86_64-${NVTX_VER}-archive.zip"
+          curl -O "$CUDA_DOWNLOAD/cuda_profiler_api/windows-x86_64/cuda_profiler_api-windows-x86_64-${PROFILER_VER}-archive.zip"
+          curl -O "$CUDA_DOWNLOAD/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-${VS_VER}-archive.zip"
+          curl -O "$CUDA_DOWNLOAD/cuda_nvprof/windows-x86_64/cuda_nvprof-windows-x86_64-${NVPROF_VER}-archive.zip"
+          curl -O "$CUDA_DOWNLOAD/cuda_cccl/windows-x86_64/cuda_cccl-windows-x86_64-${CCCL_VER}-archive.zip"
+
+          # Extract all the downloaded files to the CUDA Toolkit directory
+          unzip -q '*.zip' -d $CUDA_TOOLKIT_DIR
+
+          # Copy all the extracted files to the main CUDA Toolkit directory
+          xcopy "$CUDA_TOOLKIT_DIR\cuda_cudart-windows-x86_64-${CUDART_VER}-archive\*" "$CUDA_TOOLKIT_DIR" /E /I /H /Y
+          xcopy "$CUDA_TOOLKIT_DIR\cuda_nvcc-windows-x86_64-${NVCC_VER}-archive\*"     "$CUDA_TOOLKIT_DIR" /E /I /H /Y
+          xcopy "$CUDA_TOOLKIT_DIR\cuda_nvrtc-windows-x86_64-${NVRTC_VER}-archive\*"   "$CUDA_TOOLKIT_DIR" /E /I /H /Y
+          xcopy "$CUDA_TOOLKIT_DIR\libcublas-windows-x86_64-${CUBLAS_VER}-archive\*"   "$CUDA_TOOLKIT_DIR" /E /I /H /Y
+          xcopy "$CUDA_TOOLKIT_DIR\cuda_nvtx-windows-x86_64-${NVTX_VER}-archive\*"     "$CUDA_TOOLKIT_DIR" /E /I /H /Y
+          xcopy "$CUDA_TOOLKIT_DIR\cuda_nvprof-windows-x86_64-${NVPROF_VER}-archive\*" "$CUDA_TOOLKIT_DIR" /E /I /H /Y
+          xcopy "$CUDA_TOOLKIT_DIR\cuda_cccl-windows-x86_64-${CCCL_VER}-archive\*" "$CUDA_TOOLKIT_DIR" /E /I /H /Y
+          xcopy "$CUDA_TOOLKIT_DIR\cuda_profiler_api-windows-x86_64-${PROFILER_VER}-archive\*" "$CUDA_TOOLKIT_DIR" /E /I /H /Y
+          xcopy "$CUDA_TOOLKIT_DIR\visual_studio_integration-windows-x86_64-${VS_VER}-archive\*" "$CUDA_TOOLKIT_DIR" /E /I /H /Y
+
+          # Visual Studio integration
+          xcopy "$CUDA_TOOLKIT_DIR\visual_studio_integration-windows-x86_64-${VS_VER}-archive\visual_studio_integration\MSBuildExtensions\*" "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Microsoft\VC\v160\BuildCustomizations" /E /I /H /Y
+
+          # Set environment variables
+          echo "$CUDA_TOOLKIT_DIR\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+          echo "$CUDA_TOOLKIT_DIR\libnvvp" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+          echo "CUDA_PATH=$CUDA_TOOLKIT_DIR" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
+          echo "CUDA_PATH_V12_2=$CUDA_TOOLKIT_DIR" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
+
+      - name: Add msbuild to PATH
+        uses: microsoft/setup-msbuild@v2
+
+      - name: Install 7-Zip
+        run: choco install 7zip -y
+
+      - name: Fetch SDL2 and set SDL2_DIR
+        if: matrix.sdl2 == 'ON'
+        run: |
+          Invoke-WebRequest -Uri https://github.com/libsdl-org/SDL/releases/download/release-${{ matrix.sdl2_ver }}/SDL2-devel-${{ matrix.sdl2_ver }}-VC.zip -OutFile sdl2.zip
+          7z x sdl2.zip
+          echo "SDL2_DIR=${{ github.workspace }}\SDL2-${{ matrix.sdl2_ver }}\cmake" | Out-File -FilePath $env:GITHUB_ENV -Append
+          echo "${{ github.workspace }}\SDL2-${{ matrix.sdl2_ver }}\cmake" > SDL2_PATH.txt
+
+      - name: Install cmake
+        run: choco install cmake
+
+      - name: Build Project
+        shell: cmd
+        run: |
+          call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat"
+          cmake --version
+          where cmake
+          cmake -S . -B build -G "Ninja Multi-Config" ^
+            -DCMAKE_BUILD_TYPE=${{ matrix.build }} ^
+            -DGGML_CUDA=${{ matrix.cublas }} ^
+            -DWHISPER_SDL2=${{ matrix.sdl2 }} ^
+            -DSDL2_DIR="%SDL2_DIR%"
+          set /A NINJA_JOBS=%NUMBER_OF_PROCESSORS%-1
+          cmake --build build --config ${{ matrix.build }} -j %NUMBER_OF_PROCESSORS%
+
+      - name: Check sccache status after build
+        run: |
+          sccache --show-stats
+
+      - name: Copy CUDA DLLs
+        run: |
+          Get-ChildItem "$env:CUDA_PATH\bin\" -Filter "*.dll" |
+          Copy-Item -Destination "build/bin/${{ matrix.build }}"
+
+      - name: Copy SDL2.dll
+        if: matrix.sdl2 == 'ON'
+        run: copy "$env:SDL2_DIR/../lib/${{ matrix.arch }}/SDL2.dll" build/bin/${{ matrix.build }}
+
+      - name: Upload binaries
+        uses: actions/upload-artifact@v4
+        with:
+          name: whisper-cublas-${{ matrix.cuda-toolkit }}-bin-${{ matrix.arch }}
+          path: build/bin/${{ matrix.build }}
 
   emscripten:
-    runs-on: ubuntu-latest
+    if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' ||
+            github.event.inputs.run_type == 'full-ci' }}
+    runs-on: ubuntu-22.04
 
     strategy:
       matrix:
@@ -521,6 +880,7 @@ jobs:
 
   ios-xcode-build:
     runs-on: macos-latest
+    needs: determine-tag
 
     strategy:
       matrix:
@@ -551,21 +911,39 @@ jobs:
             -DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \
             -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
           cmake --build . --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
-          sudo cmake --install . --config Release
 
       - name: xcodebuild for swift package
         id: xcodebuild
         run: |
-          xcodebuild -scheme whisper-Package -destination 'generic/platform=iOS'
+          ./build-xcframework.sh
 
-#- name: Build objc example
+      - name: Build objc example
-#  run: xcodebuild -project examples/whisper.objc/whisper.objc.xcodeproj -scheme whisper.objc -configuration ${{ matrix.build }} -sdk iphoneos build
+        run: xcodebuild -project examples/whisper.objc/whisper.objc.xcodeproj -scheme whisper.objc -configuration ${{ matrix.build }} -sdk iphoneos CODE_SIGN_IDENTITY="" CODE_SIGNING_REQUIRED=NO FRAMEWORK_FOLDER_PATH=./build-ios build
 
       - name: Build swiftui example
-        run: xcodebuild -project examples/whisper.swiftui/whisper.swiftui.xcodeproj -scheme WhisperCppDemo -configuration ${{ matrix.build }} -sdk iphoneos CODE_SIGNING_REQUIRED=NO CODE_SIGN_IDENTITY= -destination 'generic/platform=iOS' build
+        run: xcodebuild -project examples/whisper.swiftui/whisper.swiftui.xcodeproj -scheme WhisperCppDemo -configuration ${{ matrix.build }} -sdk iphoneos CODE_SIGNING_REQUIRED=NO CODE_SIGN_IDENTITY= -destination 'generic/platform=iOS' FRAMEWORK_FOLDER_PATH=./build-ios build
+
+      - name: Pack artifacts
+        id: pack_artifacts
+        if: ${{ (github.event_name == 'push' && github.ref == 'refs/heads/master') ||
+                github.event.inputs.create_release == 'true' ||
+                github.event.inputs.pre_release_tag != '' }}
+        run: |
+          zip --symlinks -r whisper-${{ needs.determine-tag.outputs.tag_name }}-xcframework.zip build-apple/whisper.xcframework
+
+      - name: Upload artifacts
+        if: ${{ (github.event_name == 'push' && github.ref == 'refs/heads/master') ||
+                github.event.inputs.create_release == 'true' ||
+                github.event.inputs.pre_release_tag != '' }}
+        uses: actions/upload-artifact@v4
+        with:
+          path: whisper-${{ needs.determine-tag.outputs.tag_name }}-xcframework.zip
+          name: whisper-${{ needs.determine-tag.outputs.tag_name }}-xcframework
 
   android:
-    runs-on: ubuntu-latest
+    if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' ||
+            github.event.inputs.run_type == 'full-ci' }}
+    runs-on: ubuntu-22.04
 
     steps:
       - name: Clone
@@ -593,31 +971,30 @@ jobs:
           cd whisper/examples/whisper.android
           ./gradlew assembleRelease --no-daemon
 
-# TODO: disable because of following fail: https://github.com/ggerganov/whisper.cpp/actions/runs/11019444420/job/30627193602
+  android_java:
-#  android_java:
+    runs-on: ubuntu-22.04
-#    runs-on: ubuntu-latest
+
-#
+    steps:
-#    steps:
+      - name: Clone
-#      - name: Clone
+        uses: actions/checkout@v4
-#        uses: actions/checkout@v4
+
-#
+      - name: set up JDK 11
-#      - name: set up JDK 11
+        uses: actions/setup-java@v4
-#        uses: actions/setup-java@v4
+        with:
-#        with:
+          java-version: '11'
-#          java-version: '11'
+          distribution: 'temurin'
-#          distribution: 'temurin'
+          cache: gradle
-#          cache: gradle
+
-#
+      - name: Setup Android SDK
-#      - name: Setup Android SDK
+        uses: android-actions/setup-android@v3
-#        uses: android-actions/setup-android@v3
+        with:
-#        with:
+          cmdline-tools-version: 9.0
-#          cmdline-tools-version: 9.0
+
-#
+      - name: Build
-#      - name: Build
+        run: |
-#        run: |
+          cd examples/whisper.android.java
-#          cd examples/whisper.android.java
+          chmod +x ./gradlew
-#          chmod +x ./gradlew
+          ./gradlew assembleRelease
-#          ./gradlew assembleRelease
 
 # TODO: disabled because of following fail: https://github.com/ggerganov/whisper.cpp/actions/runs/9686220096/job/26735899598
 #  java:
@@ -664,7 +1041,9 @@ jobs:
 #          PGP_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
 
   quantize:
-    runs-on: ubuntu-latest
+    if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' ||
+            github.event.inputs.run_type == 'full-ci' }}
+    runs-on: ubuntu-22.04
 
     steps:
       - name: Clone
@@ -676,3 +1055,95 @@ jobs:
           cmake -B build
           cmake --build build --config Release
           ./build/bin/quantize models/ggml-tiny.en.bin models/ggml-tiny.en-q4_0.bin q4_0
+
+  release:
+    if: ${{ github.event.inputs.create_release == 'true' || github.event.inputs.pre_release_tag != '' }}
+
+    runs-on: ubuntu-latest
+
+    needs:
+      - determine-tag
+      - ios-xcode-build
+
+    steps:
+      - name: Clone
+        id: checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: ccache
+        uses: hendrikmuhs/ccache-action@v1.2.16
+        with:
+          key: release
+          evict-old-files: 1d
+
+      # Downloads all the artifacts from the previous jobs
+      - name: Download artifacts
+        id: download-artifact
+        uses: actions/download-artifact@v4
+        with:
+          path: ./artifact
+
+      - name: Move artifacts
+        id: move_artifacts
+        run: mkdir -p ./artifact/release && mv ./artifact/*/*.zip ./artifact/release
+
+      - name: Create release
+        id: create_release
+        uses: ggml-org/action-create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ needs.determine-tag.outputs.tag_name }}
+          prerelease: ${{ github.event.inputs.pre_release_tag != '' }}
+
+      - name: Upload release
+        id: upload_release
+        uses: actions/github-script@v3
+        with:
+          github-token: ${{secrets.GITHUB_TOKEN}}
+          script: |
+            const path = require('path');
+            const fs = require('fs');
+            const release_id = '${{ steps.create_release.outputs.id }}';
+            for (let file of await fs.readdirSync('./artifact/release')) {
+              if (path.extname(file) === '.zip') {
+                console.log('uploadReleaseAsset', file);
+                await github.repos.uploadReleaseAsset({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  release_id: release_id,
+                  name: file,
+                  data: await fs.readFileSync(`./artifact/release/${file}`)
+                });
+              }
+            }
+
+  coreml-base-en:
+    if: ${{ (github.event_name == 'push' && github.ref == 'refs/heads/master') ||
+            github.event.inputs.create_release == 'true' ||
+            github.event.inputs.pre_release_tag != '' }}
+    runs-on: macos-latest
+    needs: determine-tag
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set environment variables
+        id: set_vars
+        run: |
+          echo "MODEL_NAME=base.en" >> $GITHUB_ENV
+          echo "GEN_MODEL_NAME=whisper-${{ needs.determine-tag.outputs.tag_name }}-ggml-base.en-encoder.mlmodelc" >> $GITHUB_ENV
+
+      - name: Download model
+        run: |
+          ./models/download-ggml-model.sh ${{ env.MODEL_NAME }}
+
+      - name: Generate CoreML model
+        run: |
+          python3.11 -m venv venv
+          source venv/bin/activate
+          pip install ane_transformers openai-whisper coremltools
+          ./models/generate-coreml-model.sh ${{ env.MODEL_NAME }}

.github/workflows/docker.yml

@@ -11,13 +11,13 @@ jobs:
     name: Push Docker image to Docker Hub
     if: github.event.pull_request.draft == false
 
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     env:
       COMMIT_SHA: ${{ github.sha }}
     strategy:
       matrix:
         config:
-          - { tag: "main", dockerfile: ".devops/main.Dockerfile", platform: "linux/amd64,linux/arm64" }
+          - { tag: "main", dockerfile: ".devops/main.Dockerfile", platform: "linux/amd64" }
           #TODO: the cuda image keeps failing - disable for now
           #      https://github.com/ggerganov/whisper.cpp/actions/runs/11019444428/job/30602020339
           #- { tag: "main-cuda", dockerfile: ".devops/main-cuda.Dockerfile", platform: "linux/amd64" }
@@ -28,6 +28,8 @@ jobs:
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
+        with:
+          image: tonistiigi/binfmt:qemu-v7.0.0-28
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3

.github/workflows/examples-wasm.yml

@@ -0,0 +1,91 @@
+name: Examples WASM
+on:
+  push:
+    branches: ["master"]
+
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+concurrency:
+  group: "pages"
+  cancel-in-progress: false
+
+jobs:
+  deploy-wasm-github-pages:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Pages
+        uses: actions/configure-pages@v4
+
+      - name: Setup emsdk
+        uses: mymindstorm/setup-emsdk@v14
+
+      - name: Build WASM Examples
+        # Enable for real build later in whisper.cpp
+        run: |
+          mkdir -p build-em && cd build-em
+          emcmake cmake .. -DCMAKE_BUILD_TYPE=Release
+          make -j
+
+      - name: Create staging directory
+        run: mkdir -p staging
+
+      - name: Create .nojekyll file in staging directory
+        run: touch staging/.nojekyll
+
+      - name: Copy application files
+        run: |
+          build_dir=build-em/bin
+
+          ls ${build_dir}
+
+          # command.wasm
+          target_dir=staging/command.wasm
+          mkdir -p ${target_dir}
+          cp ${build_dir}/command.wasm/{index.html,command.js,helpers.js} ${target_dir}
+          cp ${build_dir}/libcommand.js ${target_dir}
+
+          # bench.wasm
+          target_dir=staging/bench.wasm
+          mkdir -p ${target_dir}
+          cp ${build_dir}/bench.wasm/{index.html,bench.js,helpers.js} ${target_dir}
+          cp ${build_dir}/libbench.js ${target_dir}
+
+          # stream.wasm
+          target_dir=staging/stream.wasm
+          mkdir -p ${target_dir}
+          cp ${build_dir}/stream.wasm/{index.html,stream.js,helpers.js} ${target_dir}
+          cp ${build_dir}/libstream.js ${target_dir}
+
+          # whisper.wasm (this will be the main example page)
+          target_dir=staging
+          mkdir -p ${target_dir}
+          cp ${build_dir}/whisper.wasm/{index.html,main.js,helpers.js} ${target_dir}
+          cp ${build_dir}/libmain.js ${target_dir}
+
+          # Copy Cross-Origin Isolation service worker
+          cp -v examples/coi-serviceworker.js staging/
+
+      - name: List files in staging directory (for debugging)
+        run: |
+          echo "Files in staging directory:"
+          find staging -type f | sort
+
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: ./staging
+
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

.github/workflows/examples.yml

@@ -10,8 +10,8 @@ on:
       - whisper.h
 
 jobs:
-  addon_node-ubuntu-latest:
+  addon_node-ubuntu-22:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     strategy:
       matrix:
         node-version: [ 16.x, 18.x ]
@@ -22,7 +22,7 @@ jobs:
       - name: Dependencies
         run: |
           sudo apt-get update
-          sudo apt-get install build-essential
+          sudo apt-get install build-essential git
           sudo apt-get install cmake
           sudo apt-get install libsdl2-dev
 

.gitignore

@@ -58,3 +58,5 @@ cmake-build-debug/
 .cxx/
 .gradle/
 local.properties
+.log
+.exe

AUTHORS

@@ -1,34 +1,51 @@
-# date: Tue Apr  9 20:27:03 EEST 2024
+# date: Tue Feb  4 13:03:35 EET 2025
 # this file is auto-generated by scripts/gen-authors.sh
 
 0/0 <zero@imaskeleton.me>
 0cc4m <picard12@live.de>
 0xsourcecode <134374803+0xsourcecode@users.noreply.github.com>
+65a <10104049+65a@users.noreply.github.com>
+AIWintermuteAI <32562299+AIWintermuteAI@users.noreply.github.com>
 AT <manyoso@users.noreply.github.com>
 Aarni Koskela <akx@iki.fi>
 Aaron Pham <29749331+aarnphm@users.noreply.github.com>
 Aaron Taylor <aaron@exphat.com>
 Abhilash Majumder <30946547+abhilash1910@users.noreply.github.com>
 Abitofevrything <54505189+abitofevrything@users.noreply.github.com>
+Adam Jones <domdomegg+git@gmail.com>
+Adrien Gallouët <adrien@gallouet.fr>
+Adrien Gallouët <angt@huggingface.co>
 AfryMask <AfryMask@163.com>
 Ahmad Bilal <ahmad.bilal@empglabs.com>
+Ahmad Tameem <113388789+Tameem-10xE@users.noreply.github.com>
 AidanBeltonS <87009434+AidanBeltonS@users.noreply.github.com>
+AidanBeltonS <aidan.belton@codeplay.com>
+Akarshan Biswas <akarshan.biswas@gmail.com>
+Akarshan Biswas <akarshanbiswas@fedoraproject.org>
 Akash Mahajan <akash7190@gmail.com>
 Akash Mahajan <akashmjn@stanford.edu>
 Al Hoang <3811822-hoanga@users.noreply.gitlab.com>
 Alan <unknown>
+Albert Jin <albert.jin@gmail.com>
+Alberto Cabrera Pérez <alberto.cabrera@codeplay.com>
+Alberto Cabrera Pérez <alberto.cabrera@intel.com>
 Aleksander Andrzejewski <18704749+aleksanderandrzejewski@users.noreply.github.com>
 Alex Azarov <alex@azarov.by>
 Alex Bacart <13940752+alex-bacart@users.noreply.github.com>
 Alex Evgrashin <aevgrashin@yandex.ru>
+Alex O'Connell <35843486+acon96@users.noreply.github.com>
 Alexandr Graschenkov <alexandr.graschenkov91@gmail.com>
 Alexandru Mariuti <alex@mariuti.com>
 Alexey Kharlamov <alexey@kharlamov.biz>
 Alfredo Montesinos <alfredo.montesinos@g.austincc.edu>
 Ali Alameh <ali.alameh@isae.edu.lb>
+Alter <0x7c48@gmail.com>
 Ananta Bastola <anantarajbastola@gmail.com>
+Andreas Kieslinger <47689530+aendk@users.noreply.github.com>
+Andreas Lubbe <git@lubbe.org>
 Andreu Huguet <andreuhuguet@gmail.com>
 Andrew Huynh <a5thuynh@gmail.com>
+Andrew Minh Nguyen <40281306+amqdn@users.noreply.github.com>
 Andrew S <andrews54757@gmail.com>
 Andy Maloney <asmaloney@gmail.com>
 Anton Kostin <masguit42@users.noreply.github.com>
@@ -40,8 +57,11 @@ AustinMroz <austinmroz@utexas.edu>
 Avik Sengupta <avik@sengupta.net>
 Bader-eddine Ouaich <49657842+baderouaich@users.noreply.github.com>
 Baffin Lee <baffinlee@gmail.com>
+Ben Ashbaugh <ben.ashbaugh@intel.com>
 Ben Nortier <bjnortier@gmail.com>
 Benjamin Heiniger <benjamin.heiniger@bluewin.ch>
+Bernhard M. Wiedemann <githubbmwprimary@lsmod.de>
+Binozo <70137898+Binozo@users.noreply.github.com>
 Bo-Yi Wu <appleboy.tw@gmail.com>
 Boris Bliznioukov <blib@mail.com>
 Borislav Stanimirov <b.stanimirov@abv.bg>
@@ -49,47 +69,86 @@ Brad Murray <59848399+bradmurray-dt@users.noreply.github.com>
 Brian Murray <brian@bmurray.ca>
 CRD716 <crd716@gmail.com>
 Canis Lupus <Canis-UK@users.noreply.github.com>
+Carlos Zoido <mrgalleta@gmail.com>
 Carolinabanana <140120812+Carolinabanana@users.noreply.github.com>
+CarterLi999 <664681047@qq.com>
 ChangSeok Oh <shivamidow@users.noreply.github.com>
+Changyeon Kim <cyzero.kim@samsung.com>
 Chaoqun <27287694+OpenWaygate@users.noreply.github.com>
+Charles Xu <63788048+chaxu01@users.noreply.github.com>
+Charles Xu <charles.xu@arm.com>
+Chen Xi <xi2.chen@intel.com>
+Chen Xi <xixichen08@foxmail.com>
+Chenguang Li <87689256+noemotiovon@users.noreply.github.com>
 Chia-Hsiang Cheng <88014292+garychia@users.noreply.github.com>
 Chidi Williams <williamschidi1@gmail.com>
+Chris Elrod <elrodc@gmail.com>
 Christian <12550267+iceychris@users.noreply.github.com>
+Christian Kastner <ckk@kvr.at>
 Clifford Heath <clifford.heath@gmail.com>
+Clint Herron <hanclinto@gmail.com>
 Colin <github@whoisc.cc>
+Conrad Kramer <conrad@conradkramer.com>
+Corey Earwood <iamcgn+github@gmail.com>
+CrispStrobe <154636388+CrispStrobe@users.noreply.github.com>
+DAN™ <dranger003@gmail.com>
 DGdev91 <DGdev91@users.noreply.github.com>
 Damian Czaja <trojan295@protonmail.com>
+Dan Johansson <164997844+eddnjjn@users.noreply.github.com>
+Dan Johansson <dan.johansson@arm.com>
 Daniel Bevenius <daniel.bevenius@gmail.com>
+Daniel Valdivia <18384552+dvaldivia@users.noreply.github.com>
+Daniel Ziegenberg <daniel@ziegenberg.at>
+Daniele <57776841+daniandtheweb@users.noreply.github.com>
+Dave <dave-fl@users.noreply.github.com>
+Dave Airlie <airlied@gmail.com>
+Dave Airlie <airlied@redhat.com>
+Daven Sanassy <daven@vochlea.co.uk>
 David <dnhkng@gmail.com>
 David Thorpe <djt@mutablelogic.com>
+DavidKorczynski <david@adalogics.com>
 Davidson Francis <davidsondfgl@gmail.com>
 Dener Stassun <denerstassun@gmail.com>
+Dibakar Gope <dibakar.gope@arm.com>
 Didzis Gosko <didzis@users.noreply.github.com>
+Diego Devesa <slarengh@gmail.com>
 Digipom <admin@digipom.com>
 Dimo <dimo@ieee.org>
+Djip007 <3705339+Djip007@users.noreply.github.com>
+Djip007 <djip.perois@free.fr>
 Dody Suria Wijaya <dodysw@gmail.com>
+Dou Xinpeng <15529241576@163.com>
+Dou Xinpeng <81913537+Dou-Git@users.noreply.github.com>
 Dr. Tom Murphy VII Ph.D <499244+tom7@users.noreply.github.com>
 Duncan McConnell <ddmcconnell4@gmail.com>
 Egor Egorov <me@egorfine.com>
 Elkana Bardugo <ttv200@gmail.com>
 Emmanuel Schmidbauer <eschmidbauer@gmail.com>
 Engininja2 <139037756+Engininja2@users.noreply.github.com>
+Eric Curtin <ericcurtin17@gmail.com>
 Eric Swanson <eswanson@alloscomp.com>
 Eric Tendian <erictendian@gmail.com>
+Eric Zhang <34133756+EZForever@users.noreply.github.com>
 Erik Scholz <Green-Sky@users.noreply.github.com>
 Evan Jones <evan.q.jones@gmail.com>
 Evan Martin <evan.martin@gmail.com>
 Eve <139727413+netrunnereve@users.noreply.github.com>
 Evgeny Kuznetsov <evgeny@kuznetsov.md>
 F1L1P <78918286+F1L1Pv2@users.noreply.github.com>
+Faisal Zaghloul <quic_fzaghlou@quicinc.com>
 Fangjun Kuang <csukuangfj@gmail.com>
 Felix <stenbackfelix@gmail.com>
 Finn Voorhees <finnvoorhees@gmail.com>
+FirstTimeEZ <179362031+FirstTimeEZ@users.noreply.github.com>
 FlippFuzz <41221030+FlippFuzz@users.noreply.github.com>
+Frankie Robertson <frankier@users.noreply.github.com>
 Gang Chen <goncha@gmail.com>
 Gavin Cai <gavin1818@hotmail.com>
 George Hindle <george@georgehindle.com>
 Georgi Gerganov <ggerganov@gmail.com>
+Gilad S <7817232+giladgd@users.noreply.github.com>
+Gilad S <giladgd@users.noreply.github.com>
+Gilad S. <7817232+giladgd@users.noreply.github.com>
 GitAritron <103900385+GitAritron@users.noreply.github.com>
 GiviMAD <GiviMAD@users.noreply.github.com>
 Gleicon Moraes <gleicon@gmail.com>
@@ -98,41 +157,66 @@ Guillaume Wenzek <gwenzek@users.noreply.github.com>
 HY. Kelvin Lee <34256578+hykelvinlee42@users.noreply.github.com>
 Halalaluyafail3 <55773281+Halalaluyafail3@users.noreply.github.com>
 Hang <bebound@gmail.com>
+Haus1 <haus.xda@gmail.com>
 Herman Semenov <GermanAizek@yandex.ru>
+HimariO <dsfhe49854@gmail.com>
+Hong Bo PENG <penghb@cn.ibm.com>
 Hrishikesh Barman <geekodour@users.noreply.github.com>
+Hugo <hugo@whynothugo.nl>
 Ian Bicking <ian@ianbicking.org>
 Ian Bull <irbull@eclipsesource.com>
+Ihar Hrachyshka <ihrachys@redhat.com>
 Ikko Ashimine <eltociear@gmail.com>
+Ikko Eltociear Ashimine <eltociear@gmail.com>
 InconsolableCellist <23345188+InconsolableCellist@users.noreply.github.com>
 Ismatulla Mansurov <47342870+sapoepsilon@users.noreply.github.com>
+Ivan <nekotekina@gmail.com>
+Ivan Filipov <159561759+vanaka11@users.noreply.github.com>
 Ivan Gorin <ivangorin21@gmail.com>
+Ivo von Putzer Reibegg <ivo.putzer@gmail.com>
 JJ <103335846+computerscienceiscool@users.noreply.github.com>
 Jack Mousseau <jmousseau@users.noreply.github.com>
 JacobLinCool <jacoblincool@gmail.com>
 Jakub Ráček <blizzcz@gmail.com>
 Jared Van Bortel <jared@nomic.ai>
 Jay Binks <jaybinks@gmail.com>
+Jayant <jayantyadav202@gmail.com>
+Jeff Bolz <jbolz@nvidia.com>
+Jeroen Mostert <jeroen.mostert@cm.com>
 Jhen-Jie Hong <developer@jhen.me>
 Jhen-Jie Hong <iainst0409@gmail.com>
 JidongZhang-THU <1119708529@qq.com>
 Jo Liss <joliss42@gmail.com>
+Joe Todd <joe.todd@codeplay.com>
 Johan <jr.raffin@gmail.com>
 Johannes Gäßler <johannesg@5d6.de>
 John Balis <phobossystems@gmail.com>
+JohnnyB <jboero@users.noreply.github.com>
 Jonathan Soo <jcsoo@agora.com>
 Jonno <1160532+razodactyl@users.noreply.github.com>
 Joonas Pihlajamaa <joonas.pihlajamaa@iki.fi>
 Jose <34888496+Jerry-Master@users.noreply.github.com>
 Josh Bleecher Snyder <josharian@gmail.com>
+Josscii <jossciiweiyi@gmail.com>
 Judd <foldl@users.noreply.github.com>
 Jumper775 <78500318+jumpers775@users.noreply.github.com>
+Jun Hee Yoo <contact.jhyoo@gmail.com>
+Junil Kim <logyourself@gmail.com>
+Justina Cho <justcho5@gmail.com>
 Justine Tunney <jtunney@gmail.com>
+Justine Tunney <jtunney@mozilla.com>
+KITAITI Makoto <KitaitiMakoto@gmail.com>
 KP Kaiser <kirk@zothcorp.com>
 Kamilake <exjang0@gmail.com>
+Karol Kontny <82021046+kkontny@users.noreply.github.com>
+Karthick <j.karthic2004@gmail.com>
 Kartik Saranathan <278928+Kartiku@users.noreply.github.com>
 Kasumi <90275229+kasumi-1@users.noreply.github.com>
 Kawrakow <48489457+ikawrakow@users.noreply.github.com>
+Kendrick Taylor <kendrick@circuitsix.com>
 Kevin Brothaler <admin@digipom.com>
+Kevin Gibbons <bakkot@gmail.com>
+Konosuke Sakai <konosuke@konosuke.work>
 Konstantin Zhuravlyov <konstantin.zhuravlyov@amd.com>
 Kreijstal <rainb@tfwno.gf>
 Kylin <56434533+KyL0N@users.noreply.github.com>
@@ -147,56 +231,110 @@ Luis Herrera <herrera-luis@users.noreply.github.com>
 Lukas Rist <glaslos@gmail.com>
 M. A. Ali <73258591+MightyStud@users.noreply.github.com>
 M. Eren Akbiyik <erenakbiyik@gmail.com>
+Ma Mingfei <mingfei.ma@intel.com>
 Maciek <maciek.mab122@gmail.com>
+Mahesh Madhav <67384846+heshpdx@users.noreply.github.com>
 Marcin Mielniczuk <marmistrz.dev@zoho.eu>
+Mark Karpelès <MagicalTux@users.noreply.github.com>
+Mark Zhuang <zhuangqiubin@gmail.com>
+Markus Tavenrath <mtavenrath@users.noreply.github.com>
+Martin Delille <martin@delille.org>
 Martin Warnaar <martinwarnaar@gmail.com>
+Masaya, Kato <62578291+msy-kato@users.noreply.github.com>
 Matheus de Sousa <23645013+keyehzy@users.noreply.github.com>
+Mathieu Baudier <mbaudier@argeo.org>
 Mathijs de Bruin <mathijs@mathijsfietst.nl>
 Matija Pevec <mightymatth@users.noreply.github.com>
+Matt Stephenson <mstephenson6@users.noreply.github.com>
+Max Krasnyansky <max.krasnyansky@gmail.com>
+Max Krasnyansky <quic_maxk@quicinc.com>
 Maximiliano Levi <8160966+maxilevi@users.noreply.github.com>
 Meng, Hengyu <hengyu.meng@intel.com>
+Mengqing Cao <cmq0113@163.com>
 Michael Podvitskiy <podvitskiymichael@gmail.com>
 Michael Rienstra <mrienstra@gmail.com>
 Mikhail Grigorev <sleuthhound@gmail.com>
 Mohammadreza Hendiani <hendiani.mohammadreza@gmail.com>
 Mohit Agarwal <mohit@sdf.org>
+Molly Sophia <mollysophia379@gmail.com>
 Murilo Santana <mvrilo@gmail.com>
+NETZkultur GmbH <mulholland@netzkultur.de>
+Natsu <chino@hotococoa.moe>
 Neil Chudleigh <nchudleigh@users.noreply.github.com>
+Neo Zhang <14088817+arthw@users.noreply.github.com>
 Neo Zhang Jianyu <jianyu.zhang@intel.com>
 Neuman Vong <neuman.vong@gmail.com>
+Nicholai Tukanov <nicholaitukanov@gmail.com>
 Nicholas Albion <nalbion@yahoo.com>
+Nico Bosshard <nico@bosshome.ch>
+Nicolò Scipione <nicolo.scipione@codeplay.com>
 Niels Mayer <Niels.Mayer@gmail.com>
+Nikita Sarychev <42014488+sARY77@users.noreply.github.com>
+Nikolaj Olsson <nikse.dk@gmail.com>
 Okabintaro <103938900+Okabintaro@users.noreply.github.com>
 Oleg Sidorov <me@whitebox.io>
 Oleg Sidorov <oleg@sidorov.nl>
+Olivier Chafik <ochafik@users.noreply.github.com>
 Ondrej Kokes <ondrej.kokes@gmail.com>
 Ouadie EL FAROUKI <ouadie.elfarouki@codeplay.com>
+PAB <pierreantoine.bannier@gmail.com>
 Paul Tsochantaris <ptsochantaris@icloud.com>
+Pedro Probst <pprobst@insiberia.net>
+Peng <hzp1024@qq.com>
+Peter <peter277@users.noreply.github.com>
 Philipp Zabel <philipp.zabel@gmail.com>
 Philippe Normand <phil@base-art.net>
+Philippe Normand <philn@igalia.com>
+Plamen Minev <pacominev@gmail.com>
+Prashant Vithule <119530321+Vithulep@users.noreply.github.com>
 Przemysław Pawełczyk <przemoc@gmail.com>
 Qianhe Chen <54462604+chenqianhe@users.noreply.github.com>
+R0CKSTAR <xiaodong.ye@mthreads.com>
+R0CKSTAR <yeahdongcn@gmail.com>
+Radoslav Gerganov <rgerganov@gmail.com>
 Radosław Gryta <radek.gryta@gmail.com>
+Rahul Vadhyar <107788610+RahulVadhyar@users.noreply.github.com>
+Raiya Araki <83504221+rai62@users.noreply.github.com>
 Reinforce-II <fate@eastal.com>
 Reinis Muiznieks <muiznieks.reinis@gmail.com>
 RelatedTitle <r3latedtitle@gmail.com>
+Rémy Oudompheng <oudomphe@phare.normalesup.org>
 RhinoDevel <RhinoDevel@users.noreply.github.com>
 Rich Jones <miserlou@gmail.com>
+Robert Ormandi <52251610+ormandi@users.noreply.github.com>
 Robin <robin.xw@hotmail.com>
 Roddur Dasgupta <roddurd@gmail.com>
 Roland Rabien <figbug@gmail.com>
+Romain Biessy <romain.biessy@codeplay.com>
+Ronsor <ronsor@ronsor.pw>
 Rotem Dan <rotemdan@gmail.com>
 Ryan Hitchman <hitchmanr@gmail.com>
 Ryan Metcalfe <107415876+RyanMetcalfeInt8@users.noreply.github.com>
 RyanChang <ftes90015@gmail.com>
+SRHMorris <69468379+SRHMorris@users.noreply.github.com>
+SXX <sxx1136965276@gmail.com>
+Sacha Arbonel <sacha.arbonel@hotmail.fr>
+Salman Faroz <stsfaroz@gmail.com>
+Salvatore Mesoraca <s.mesoraca16@gmail.com>
 Sam <49637763+Onlyartist9@users.noreply.github.com>
 Sam Pullara <spullara@gmail.com>
+Samuel Durante <44513615+samueldurantes@users.noreply.github.com>
 Sanchit Gandhi <93869735+sanchit-gandhi@users.noreply.github.com>
+Sandro Hanea <40202887+sandrohanea@users.noreply.github.com>
+Sergio López <slp@redhat.com>
 Sergio López <slp@sinrega.org>
+Shanshan Shen <467638484@qq.com>
+Shijie <821898965@qq.com>
+Shupei Fan <dymarkfan@outlook.com>
 Siddharth Ramakrishnan <srr2141@columbia.edu>
+Sigbjørn Skjæret <sigbjorn.skjaeret@scala.com>
 Simon Moisselin <simon.moisstoll@gmail.com>
 Sindre Sorhus <sindresorhus@gmail.com>
 Slava Primenko <primenko.s@gmail.com>
+Srihari-mcw <96763064+Srihari-mcw@users.noreply.github.com>
+Stavros Panakakis <53979866+Stavrospanakakis@users.noreply.github.com>
+Stefan Sydow <s.sydow@heinlein-video.de>
+Stefan Sydow <stefan@sydow.email>
 Syahmi Azhar <prsyahmi@gmail.com>
 Syed Jafri <syedjafri97@gmail.com>
 Sơn Phan Trung <phantrungson17@gmail.com>
@@ -205,37 +343,63 @@ Takeshi Inoue <inoue.takeshi@gmail.com>
 Tamotsu Takahashi <ttakah+github@gmail.com>
 Taras Glek <taras@thegp.com>
 Tauseef Mohiuddin <35351464+tauseefmohammed2@users.noreply.github.com>
+Thamster <Thamster@users.noreply.github.com>
 Thijs Raymakers <thijs@raymakers.nl>
 Thomas Fitzsimmons <fitzsim@fitzsim.org>
 Tiago Fassoni <tiagofassoni@users.noreply.github.com>
 Tienshiao Ma <tienshiao@tienshiao.org>
+Tim Miller <drasticactions@users.noreply.github.com>
 Timothy Cronin <40186632+4imothy@users.noreply.github.com>
 Tobrun <tobrun.van.nuland@gmail.com>
 Todd <taf2@users.noreply.github.com>
+Toliver <teejae@gmail.com>
 Tong Li <31761981+litongjava@users.noreply.github.com>
+Tony Wasserka <4840017+neobrain@users.noreply.github.com>
 Topping1 <78745143+Topping1@users.noreply.github.com>
 Travis Cline <travis.cline@gmail.com>
 UEXTM.com <84163508+uextm@users.noreply.github.com>
+UsernamesLame <156965854+UsernamesLame@users.noreply.github.com>
 Vadim Peretokin <vperetokin@hey.com>
 Valentin Gosu <1454649+valenting@users.noreply.github.com>
+Vin Misra <vinith@alum.mit.edu>
 Vulcan <93451215+trholding@users.noreply.github.com>
 WhiteOlivierus <36532695+WhiteOlivierus@users.noreply.github.com>
+William Tambellini <william.tambellini@gmail.com>
+William Tambellini <wtambellini@sdl.com>
+Wilson Silva <wilson.dsigns@gmail.com>
 Xiang (Kevin) Li <kevinli020508@gmail.com>
 Xiao-Yong Jin <jinxiaoyong@gmail.com>
 XiaotaoChen <chenxiaotao1234@gmail.com>
+Xingchen Song(宋星辰) <xingchensong1996@163.com>
+Xinpeng Dou <81913537+Dou-Git@users.noreply.github.com>
+Xuan Son Nguyen <thichthat@gmail.com>
 Yajing Tang <phillis@google.com>
 Yang Shen <aplshenyang@gmail.com>
 Yunès <jean.baptiste.yunes@free.fr>
+Yuri Khrustalev <ykhrustalev@users.noreply.github.com>
+Yusuf Redžić <48274562+redzic@users.noreply.github.com>
 ZaBlazzingZephyrus <119159668+blazingzephyr@users.noreply.github.com>
+Zhenwei Jin <109658203+kylo5aby@users.noreply.github.com>
+Zhiyuan Li <lizhiyuan@uniartisan.com>
+Zhiyuan Li <uniartisan2017@gmail.com>
 Zigfrid Zvezdin <ziggerZZ@gmail.com>
 Zollner <24618122+Zolliner@users.noreply.github.com>
+a3sh <38979186+A3shTnT@users.noreply.github.com>
+ag2s20150909 <19373730+ag2s20150909@users.noreply.github.com>
+agray3 <agray3@users.noreply.github.com>
 ai-at-home <149282006+ai-at-home@users.noreply.github.com>
+aldorof <aldorof@users.noreply.github.com>
 alonfaraj <alonfaraj@gmail.com>
+amd-dwang <dong.wang@amd.com>
+amritahs-ibm <amritahs@linux.vnet.ibm.com>
 andypayne <apayne@gmail.com>
 ardfork <134447697+ardfork@users.noreply.github.com>
+arizhih <40765267+arizhih@users.noreply.github.com>
 automaticcat <daogiatuank54@gmail.com>
+bandoti <141645996+bandoti@users.noreply.github.com>
 be-next <jerome.ramette@gmail.com>
 bert hubert <bert@hubertnet.nl>
+billyct <billy_allen@126.com>
 bmwl <brian.marshall@tolko.com>
 bobqianic <129547291+bobqianic@users.noreply.github.com>
 bocytko <bocytko+github@gmail.com>
@@ -248,7 +412,9 @@ byte-6174 <88070277+byte-6174@users.noreply.github.com>
 cdosoftei <ciprian.dosoftei@gmail.com>
 clach04 <Chris.Clark@actian.com>
 compilade <113953597+compilade@users.noreply.github.com>
+compilade <git@compilade.net>
 conradg <conradjgodfrey@gmail.com>
+crummyh <elijah@crums.us>
 ddpasa <112642920+ddpasa@users.noreply.github.com>
 denersc <denerstassun@gmail.com>
 dscripka <dscripka@users.noreply.github.com>
@@ -256,28 +422,55 @@ duthils <duthils@duthils.net>
 ecneladis <ecneladis@users.noreply.github.com>
 faker <nspyia2002@gmail.com>
 fitzsim <fitzsim@fitzsim.org>
+fj-y-saito <85871716+fj-y-saito@users.noreply.github.com>
 fraxy-v <65565042+fraxy-v@users.noreply.github.com>
 genevera (she/her) <genevera@users.noreply.github.com>
 geniusnut <geniusnut@gmail.com>
+gilbertgong <gilbert.gong@gmail.com>
+gn64 <yukikaze.jp@gmail.com>
+goldwaving <77494627+goldwaving@users.noreply.github.com>
 greeshmay <greeshmay@gmail.com>
+haopeng <657407891@qq.com>
+hipudding <huafengchun@gmail.com>
+hsinhoyeh <yhh92u@gmail.com>
 hydai <z54981220@gmail.com>
 iamthad <thadeus.j.fleming@gmail.com>
+issixx <46835150+issixx@users.noreply.github.com>
 james wolf <contractorwolf@hotmail.com>
+jdomke <28772296+jdomke@users.noreply.github.com>
+jettoblack <jettoblack@gmail.com>
+jiez <373447296@qq.com>
 joecryptotoo <80373433+joecryptotoo@users.noreply.github.com>
 jorismertz <35079666+jorismertz@users.noreply.github.com>
+junchao-loongson <68935141+junchao-loongson@users.noreply.github.com>
 junkfood <69683722+JunkFood02@users.noreply.github.com>
 jwijffels <jwijffels@bnosac.be>
+k.h.lai <adrian.k.h.lai@outlook.com>
 kamranjon <kamranjon@gmail.com>
 katsu560 <katsu560oo-@docomo.ne.jp>
 kennethge <57784063+kenneth-ge@users.noreply.github.com>
 keyehzy <msamuel@aluno.puc-rio.br>
+kunnis <kunnis@users.noreply.github.com>
+l3utterfly <gc.pthzfoldr@gmail.com>
 leejet <leejet714@gmail.com>
+leo-pony <nengjunma@outlook.com>
+lhez <quic_lih@quicinc.com>
 litong <31761981+litongjava@users.noreply.github.com>
+liuwei-git <14815172+liuwei-git@users.noreply.github.com>
 lnyan <lkwq007@gmail.com>
+luoyu-intel <yu.luo@intel.com>
 m.bell <m.bell@techsmith.com>
+mahorozte <41834471+mahorozte@users.noreply.github.com>
+mashizora <30516315+mashizora@users.noreply.github.com>
+matt23654 <matthew.webber@protonmail.com>
+matteo <matteogeniaccio@yahoo.it>
+mgrachten <maarten@grachten.eu>
 mkiol <mkiol@users.noreply.github.com>
+mky_coder <47767389+mkycoder@users.noreply.github.com>
 novag <7754358+novag@users.noreply.github.com>
 pajowu <pajowu@pajowu.de>
+pengxin99 <pengxin.yuan@intel.com>
+petterreinholdtsen <pere-github@hungry.com>
 polarmoon <90010972+polarmoon@users.noreply.github.com>
 rlapray <lapray.romain@gmail.com>
 sandrohanea <40202887+sandrohanea@users.noreply.github.com>
@@ -287,15 +480,31 @@ shikokuchuo <53399081+shikokuchuo@users.noreply.github.com>
 slaren <slarengh@gmail.com>
 slashlib <slashlib@users.noreply.github.com>
 snadampal <87143774+snadampal@users.noreply.github.com>
+someone13574 <81528246+someone13574@users.noreply.github.com>
 st-gr <38470677+st-gr@users.noreply.github.com>
+stduhpf <stephduh@live.fr>
+stormofice <58337328+stormofice@users.noreply.github.com>
 texmex76 <40733439+texmex76@users.noreply.github.com>
 thefinaldegree <thefinaldegree@gmail.com>
+thewh1teagle <61390950+thewh1teagle@users.noreply.github.com>
+toboil-features <160222185+toboil-features@users.noreply.github.com>
 trixirt <trix@redhat.com>
 ulatekh <ulatekh@yahoo.com>
 undef <undefdev@gmail.com>
+uvos <devnull@uvos.xyz>
+uvos <philipp@uvos.xyz>
+valVk <valVk@users.noreply.github.com>
 venkr <venkateshrameshkumar+1@gmail.com>
 vicalloy <zbirder@gmail.com>
+wangshuai09 <391746016@qq.com>
+woachk <24752637+woachk@users.noreply.github.com>
+xctan <axunlei@gmail.com>
 xdrudis <xavierdrudis@yahoo.es>
+yuri@FreeBSD <yuri@FreeBSD>
+zhangjixiong <code.zjx@gmail.com>
+zhentaoyu <zhentao.yu@intel.com>
 zhouwg <6889919+zhouwg@users.noreply.github.com>
+zhouwg <zhouwg2000@gmail.com>
+谢乃闻 <sienaiwun@users.noreply.github.com>
 布客飞龙 <562826179@qq.com>
 Артём Земляк <azemlyak@smart-consulting.ru>

CMakeLists.txt

@@ -1,6 +1,6 @@
 cmake_minimum_required(VERSION 3.5) # for add_link_options and implicit target directories.
 project("whisper.cpp" C CXX)
-project("whisper.cpp" VERSION 1.7.3)
+project("whisper.cpp" VERSION 1.7.5)
 include(CheckIncludeFileCXX)
 
 set(SOVERSION 1)
@@ -38,8 +38,13 @@ if (EMSCRIPTEN)
 
     # TODO: without these, we get the following error:
     #       wasm-ld: error: --shared-memory is disallowed by whisper.cpp.o because it was not compiled with 'atomics' or 'bulk-memory' features.
-    set(CMAKE_C_FLAGS   "${CMAKE_C_FLAGS}   -pthread -s TOTAL_STACK=5242880")
+    set(CMAKE_C_FLAGS   "${CMAKE_C_FLAGS}   -pthread")
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread -s TOTAL_STACK=5242880")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread")
+
+    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -s TOTAL_STACK=5242880")
+    set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -s TOTAL_STACK=5242880")
+
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-deprecated")
 else()
     if (MINGW)
         set(BUILD_SHARED_LIBS_DEFAULT OFF)
@@ -62,7 +67,8 @@ option(WHISPER_ALL_WARNINGS           "whisper: enable all compiler warnings"
 option(WHISPER_ALL_WARNINGS_3RD_PARTY "whisper: enable all compiler warnings in 3rd party libs" OFF)
 
 # build
-option(WHISPER_FATAL_WARNINGS "whisper: enable -Werror flag" OFF)
+option(WHISPER_FATAL_WARNINGS  "whisper: enable -Werror flag"               OFF)
+option(WHISPER_USE_SYSTEM_GGML "whisper: use system-installed GGML library" OFF)
 
 # sanitizers
 option(WHISPER_SANITIZE_THREAD    "whisper: enable thread sanitizer"    OFF)
@@ -121,7 +127,15 @@ whisper_option_depr(WARNING     WHISPER_SYCL_F16            GGML_SYCL_F16)
 #
 
 if (NOT TARGET ggml)
-    add_subdirectory(ggml)
+    if (WHISPER_USE_SYSTEM_GGML)
+        find_package(ggml REQUIRED)
+        if (NOT ggml_FOUND)
+            message(FATAL_ERROR "System-installed GGML library not found.")
+        endif()
+        add_library(ggml ALIAS ggml::ggml)
+    else()
+        add_subdirectory(ggml)
+    endif()
     # ... otherwise assume ggml is added by a parent CMakeLists.txt
 endif()
 add_subdirectory(src)
@@ -176,8 +190,8 @@ install(FILES "${CMAKE_CURRENT_BINARY_DIR}/whisper.pc"
 #
 
 if (WHISPER_BUILD_TESTS AND NOT CMAKE_JS_VERSION)
-    #include(CTest)
+    include(CTest)
-    #add_subdirectory(tests)
+    add_subdirectory(tests)
 endif ()
 
 if (WHISPER_BUILD_EXAMPLES)

Makefile

@@ -18,17 +18,6 @@ samples:
 	@wget --quiet --show-progress -O samples/mm1.wav https://cdn.openai.com/whisper/draft-20220913a/micro-machines.wav
 	@wget --quiet --show-progress -O samples/a13.mp3 https://upload.wikimedia.org/wikipedia/commons/transcoded/6/6f/Apollo13-wehaveaproblem.ogg/Apollo13-wehaveaproblem.ogg.mp3
 	@wget --quiet --show-progress -O samples/diffusion2023-07-03.flac https://archive.org/download/diffusion2023-07-03/diffusion2023-07-03.flac
-	@echo "Converting to 16-bit WAV ..."
-	@ffmpeg -loglevel -0 -y -i samples/gb0.ogg -ar 16000 -ac 1 -c:a pcm_s16le samples/gb0.wav
-	@ffmpeg -loglevel -0 -y -i samples/gb1.ogg -ar 16000 -ac 1 -c:a pcm_s16le samples/gb1.wav
-	@ffmpeg -loglevel -0 -y -i samples/hp0.ogg -ar 16000 -ac 1 -c:a pcm_s16le samples/hp0.wav
-	@rm samples/*.ogg
-	@ffmpeg -loglevel -0 -y -i samples/mm1.wav -ar 16000 -ac 1 -c:a pcm_s16le samples/mm0.wav
-	@rm samples/mm1.wav
-	@ffmpeg -loglevel -0 -y -i samples/a13.mp3 -ar 16000 -ac 1 -c:a pcm_s16le -ss 00:00:00 -to 00:00:30 samples/a13.wav
-	@rm samples/a13.mp3
-	@ffmpeg -loglevel -0 -y -i samples/diffusion2023-07-03.flac -ar 16000 -ac 1 -c:a pcm_s16le samples/diffusion2023-07-03.wav
-	@rm samples/diffusion2023-07-03.flac
 
 #
 # Models
@@ -59,11 +48,11 @@ tiny.en tiny base.en base small.en small medium.en medium large-v1 large-v2 larg
 	@echo "Running $@ on all samples in ./samples ..."
 	@echo "==============================================="
 	@echo ""
-	@for f in samples/*.wav; do \
+	@for f in samples/*$(.flac .mp3 .ogg .wav); do \
 		echo "----------------------------------------------" ; \
 		echo "[+] Running $@ on $$f ... (run 'ffplay $$f' to listen)" ; \
 	    echo "----------------------------------------------" ; \
 		echo "" ; \
-		./build/bin/main -m models/ggml-$@.bin -f $$f ; \
+		./build/bin/whisper-cli -m models/ggml-$@.bin -f $$f ; \
 		echo "" ; \
 	done

Package.swift

@@ -1,19 +0,0 @@
-// swift-tools-version:5.5
-
-import PackageDescription
-
-let package = Package(
-    name: "whisper",
-    platforms: [
-        .macOS(.v12),
-        .iOS(.v14),
-        .watchOS(.v4),
-        .tvOS(.v14)
-    ],
-    products: [
-        .library(name: "whisper", targets: ["whisper"]),
-    ],
-    targets: [
-        .systemLibrary(name: "whisper", pkgConfig: "whisper"),
-    ]
-)

README.md

@@ -2,19 +2,19 @@
 
 ![whisper.cpp](https://user-images.githubusercontent.com/1991296/235238348-05d0f6a4-da44-4900-a1de-d0707e75b763.jpeg)
 
-[![Actions Status](https://github.com/ggerganov/whisper.cpp/workflows/CI/badge.svg)](https://github.com/ggerganov/whisper.cpp/actions)
+[![Actions Status](https://github.com/ggml-org/whisper.cpp/workflows/CI/badge.svg)](https://github.com/ggml-org/whisper.cpp/actions)
 [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](https://opensource.org/licenses/MIT)
 [![Conan Center](https://shields.io/conan/v/whisper-cpp)](https://conan.io/center/whisper-cpp)
 [![npm](https://img.shields.io/npm/v/whisper.cpp.svg)](https://www.npmjs.com/package/whisper.cpp/)
 
-Stable: [v1.7.3](https://github.com/ggerganov/whisper.cpp/releases/tag/v1.7.3) / [Roadmap | F.A.Q.](https://github.com/ggerganov/whisper.cpp/discussions/126)
+Stable: [v1.7.5](https://github.com/ggml-org/whisper.cpp/releases/tag/v1.7.5) / [Roadmap](https://github.com/orgs/ggml-org/projects/4/)
 
 High-performance inference of [OpenAI's Whisper](https://github.com/openai/whisper) automatic speech recognition (ASR) model:
 
 - Plain C/C++ implementation without dependencies
 - Apple Silicon first-class citizen - optimized via ARM NEON, Accelerate framework, Metal and [Core ML](#core-ml-support)
 - AVX intrinsics support for x86 architectures
-- VSX intrinsics support for POWER architectures
+- [VSX intrinsics support for POWER architectures](#power-vsx-intrinsics)
 - Mixed F16 / F32 precision
 - [Integer quantization support](#quantization)
 - Zero memory allocations at runtime
@@ -23,7 +23,7 @@ High-performance inference of [OpenAI's Whisper](https://github.com/openai/whisp
 - [Efficient GPU support for NVIDIA](#nvidia-gpu-support)
 - [OpenVINO Support](#openvino-support)
 - [Ascend NPU Support](#ascend-npu-support)
-- [C-style API](https://github.com/ggerganov/whisper.cpp/blob/master/include/whisper.h)
+- [C-style API](https://github.com/ggml-org/whisper.cpp/blob/master/include/whisper.h)
 
 Supported platforms:
 
@@ -31,14 +31,14 @@ Supported platforms:
 - [x] [iOS](examples/whisper.objc)
 - [x] [Android](examples/whisper.android)
 - [x] [Java](bindings/java/README.md)
-- [x] Linux / [FreeBSD](https://github.com/ggerganov/whisper.cpp/issues/56#issuecomment-1350920264)
+- [x] Linux / [FreeBSD](https://github.com/ggml-org/whisper.cpp/issues/56#issuecomment-1350920264)
 - [x] [WebAssembly](examples/whisper.wasm)
-- [x] Windows ([MSVC](https://github.com/ggerganov/whisper.cpp/blob/master/.github/workflows/build.yml#L117-L144) and [MinGW](https://github.com/ggerganov/whisper.cpp/issues/168)]
+- [x] Windows ([MSVC](https://github.com/ggml-org/whisper.cpp/blob/master/.github/workflows/build.yml#L117-L144) and [MinGW](https://github.com/ggml-org/whisper.cpp/issues/168)]
-- [x] [Raspberry Pi](https://github.com/ggerganov/whisper.cpp/discussions/166)
+- [x] [Raspberry Pi](https://github.com/ggml-org/whisper.cpp/discussions/166)
-- [x] [Docker](https://github.com/ggerganov/whisper.cpp/pkgs/container/whisper.cpp)
+- [x] [Docker](https://github.com/ggml-org/whisper.cpp/pkgs/container/whisper.cpp)
 
 The entire high-level implementation of the model is contained in [whisper.h](include/whisper.h) and [whisper.cpp](src/whisper.cpp).
-The rest of the code is part of the [`ggml`](https://github.com/ggerganov/ggml) machine learning library.
+The rest of the code is part of the [`ggml`](https://github.com/ggml-org/ggml) machine learning library.
 
 Having such a lightweight implementation of the model allows to easily integrate it in different platforms and applications.
 As an example, here is a video of running the model on an iPhone 13 device - fully offline, on-device: [whisper.objc](examples/whisper.objc)
@@ -51,26 +51,14 @@ https://user-images.githubusercontent.com/1991296/204038393-2f846eae-c255-4099-a
 
 On Apple Silicon, the inference runs fully on the GPU via Metal:
 
-https://github.com/ggerganov/whisper.cpp/assets/1991296/c82e8f86-60dc-49f2-b048-d2fdbd6b5225
+https://github.com/ggml-org/whisper.cpp/assets/1991296/c82e8f86-60dc-49f2-b048-d2fdbd6b5225
-
-Or you can even run it straight in the browser: [talk.wasm](examples/talk.wasm)
-
-## Implementation details
-
-- The core tensor operations are implemented in C ([ggml.h](ggml/include/ggml.h) / [ggml.c](ggml/src/ggml.c))
-- The transformer model and the high-level C-style API are implemented in C++ ([whisper.h](include/whisper.h) / [whisper.cpp](src/whisper.cpp))
-- Sample usage is demonstrated in [main.cpp](examples/main)
-- Sample real-time audio transcription from the microphone is demonstrated in [stream.cpp](examples/stream)
-- Various other examples are available in the [examples](examples) folder
-
-The tensor operators are optimized heavily for Apple silicon CPUs. Depending on the computation size, Arm Neon SIMD intrinsics or CBLAS Accelerate framework routines are used. The latter are especially effective for bigger sizes since the Accelerate framework utilizes the special-purpose AMX coprocessor available in modern Apple products.
 
 ## Quick start
 
 First clone the repository:
 
 ```bash
-git clone https://github.com/ggerganov/whisper.cpp.git
+git clone https://github.com/ggml-org/whisper.cpp.git
 ```
 
 Navigate into the directory:
@@ -85,135 +73,26 @@ Then, download one of the Whisper [models](models/README.md) converted in [`ggml
 sh ./models/download-ggml-model.sh base.en
 ```
 
-Now build the [main](examples/main) example and transcribe an audio file like this:
+Now build the [whisper-cli](examples/cli) example and transcribe an audio file like this:
 
 ```bash
-# build the main example
+# build the project
 cmake -B build
 cmake --build build --config Release
 
 # transcribe an audio file
-./build/bin/main -f samples/jfk.wav
+./build/bin/whisper-cli -f samples/jfk.wav
 ```
 
 ---
 
-For a quick demo, simply run `make base.en`:
+For a quick demo, simply run `make base.en`.
-
-```text
-$ make -j base.en
-
-cc  -I.              -O3 -std=c11   -pthread -DGGML_USE_ACCELERATE   -c ggml.c -o ggml.o
-c++ -I. -I./examples -O3 -std=c++11 -pthread -c whisper.cpp -o whisper.o
-c++ -I. -I./examples -O3 -std=c++11 -pthread examples/main/main.cpp whisper.o ggml.o -o main  -framework Accelerate
-./main -h
-
-usage: ./main [options] file0.wav file1.wav ...
-
-options:
-  -h,        --help              [default] show this help message and exit
-  -t N,      --threads N         [4      ] number of threads to use during computation
-  -p N,      --processors N      [1      ] number of processors to use during computation
-  -ot N,     --offset-t N        [0      ] time offset in milliseconds
-  -on N,     --offset-n N        [0      ] segment index offset
-  -d  N,     --duration N        [0      ] duration of audio to process in milliseconds
-  -mc N,     --max-context N     [-1     ] maximum number of text context tokens to store
-  -ml N,     --max-len N         [0      ] maximum segment length in characters
-  -sow,      --split-on-word     [false  ] split on word rather than on token
-  -bo N,     --best-of N         [5      ] number of best candidates to keep
-  -bs N,     --beam-size N       [5      ] beam size for beam search
-  -wt N,     --word-thold N      [0.01   ] word timestamp probability threshold
-  -et N,     --entropy-thold N   [2.40   ] entropy threshold for decoder fail
-  -lpt N,    --logprob-thold N   [-1.00  ] log probability threshold for decoder fail
-  -debug,    --debug-mode        [false  ] enable debug mode (eg. dump log_mel)
-  -tr,       --translate         [false  ] translate from source language to english
-  -di,       --diarize           [false  ] stereo audio diarization
-  -tdrz,     --tinydiarize       [false  ] enable tinydiarize (requires a tdrz model)
-  -nf,       --no-fallback       [false  ] do not use temperature fallback while decoding
-  -otxt,     --output-txt        [false  ] output result in a text file
-  -ovtt,     --output-vtt        [false  ] output result in a vtt file
-  -osrt,     --output-srt        [false  ] output result in a srt file
-  -olrc,     --output-lrc        [false  ] output result in a lrc file
-  -owts,     --output-words      [false  ] output script for generating karaoke video
-  -fp,       --font-path         [/System/Library/Fonts/Supplemental/Courier New Bold.ttf] path to a monospace font for karaoke video
-  -ocsv,     --output-csv        [false  ] output result in a CSV file
-  -oj,       --output-json       [false  ] output result in a JSON file
-  -ojf,      --output-json-full  [false  ] include more information in the JSON file
-  -of FNAME, --output-file FNAME [       ] output file path (without file extension)
-  -ps,       --print-special     [false  ] print special tokens
-  -pc,       --print-colors      [false  ] print colors
-  -pp,       --print-progress    [false  ] print progress
-  -nt,       --no-timestamps     [false  ] do not print timestamps
-  -l LANG,   --language LANG     [en     ] spoken language ('auto' for auto-detect)
-  -dl,       --detect-language   [false  ] exit after automatically detecting language
-             --prompt PROMPT     [       ] initial prompt
-  -m FNAME,  --model FNAME       [models/ggml-base.en.bin] model path
-  -f FNAME,  --file FNAME        [       ] input WAV file path
-  -oved D,   --ov-e-device DNAME [CPU    ] the OpenVINO device used for encode inference
-  -ls,       --log-score         [false  ] log best decoder scores of tokens
-  -ng,       --no-gpu            [false  ] disable GPU
-
-
-sh ./models/download-ggml-model.sh base.en
-Downloading ggml model base.en ...
-ggml-base.en.bin               100%[========================>] 141.11M  6.34MB/s    in 24s
-Done! Model 'base.en' saved in 'models/ggml-base.en.bin'
-You can now use it like this:
-
-  $ ./main -m models/ggml-base.en.bin -f samples/jfk.wav
-
-
-===============================================
-Running base.en on all samples in ./samples ...
-===============================================
-
-----------------------------------------------
-[+] Running base.en on samples/jfk.wav ... (run 'ffplay samples/jfk.wav' to listen)
-----------------------------------------------
-
-whisper_init_from_file: loading model from 'models/ggml-base.en.bin'
-whisper_model_load: loading model
-whisper_model_load: n_vocab       = 51864
-whisper_model_load: n_audio_ctx   = 1500
-whisper_model_load: n_audio_state = 512
-whisper_model_load: n_audio_head  = 8
-whisper_model_load: n_audio_layer = 6
-whisper_model_load: n_text_ctx    = 448
-whisper_model_load: n_text_state  = 512
-whisper_model_load: n_text_head   = 8
-whisper_model_load: n_text_layer  = 6
-whisper_model_load: n_mels        = 80
-whisper_model_load: f16           = 1
-whisper_model_load: type          = 2
-whisper_model_load: mem required  =  215.00 MB (+    6.00 MB per decoder)
-whisper_model_load: kv self size  =    5.25 MB
-whisper_model_load: kv cross size =   17.58 MB
-whisper_model_load: adding 1607 extra tokens
-whisper_model_load: model ctx     =  140.60 MB
-whisper_model_load: model size    =  140.54 MB
-
-system_info: n_threads = 4 / 10 | AVX = 0 | AVX2 = 0 | AVX512 = 0 | FMA = 0 | NEON = 1 | ARM_FMA = 1 | F16C = 0 | FP16_VA = 1 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 0 | VSX = 0 |
-
-main: processing 'samples/jfk.wav' (176000 samples, 11.0 sec), 4 threads, 1 processors, lang = en, task = transcribe, timestamps = 1 ...
-
-
-[00:00:00.000 --> 00:00:11.000]   And so my fellow Americans, ask not what your country can do for you, ask what you can do for your country.
-
-
-whisper_print_timings:     fallbacks =   0 p /   0 h
-whisper_print_timings:     load time =   113.81 ms
-whisper_print_timings:      mel time =    15.40 ms
-whisper_print_timings:   sample time =    11.58 ms /    27 runs (    0.43 ms per run)
-whisper_print_timings:   encode time =   266.60 ms /     1 runs (  266.60 ms per run)
-whisper_print_timings:   decode time =    66.11 ms /    27 runs (    2.45 ms per run)
-whisper_print_timings:    total time =   476.31 ms
-```
 
 The command downloads the `base.en` model converted to custom `ggml` format and runs the inference on all `.wav` samples in the folder `samples`.
 
-For detailed usage instructions, run: `./main -h`
+For detailed usage instructions, run: `./build/bin/whisper-cli -h`
 
-Note that the [main](examples/main) example currently runs only with 16-bit WAV files, so make sure to convert your input before running the tool.
+Note that the [whisper-cli](examples/cli) example currently runs only with 16-bit WAV files, so make sure to convert your input before running the tool.
 For example, you can use `ffmpeg` like this:
 
 ```bash
@@ -257,6 +136,21 @@ make -j large-v3-turbo
 | medium | 1.5 GiB | ~2.1 GB |
 | large  | 2.9 GiB | ~3.9 GB |
 
+## POWER VSX Intrinsics
+
+`whisper.cpp` supports POWER architectures and includes code which
+significantly speeds operation on Linux running on POWER9/10, making it
+capable of faster-than-realtime transcription on underclocked Raptor
+Talos II. Ensure you have a BLAS package installed, and replace the
+standard cmake setup with:
+
+```bash
+# build with GGML_BLAS defined
+cmake -B build -DGGML_BLAS=1
+cmake --build build --config Release
+./build/bin/whisper-cli [ .. etc .. ]
+```
+
 ## Quantization
 
 `whisper.cpp` supports integer quantization of the Whisper `ggml` models.
@@ -271,7 +165,7 @@ cmake --build build --config Release
 ./build/bin/quantize models/ggml-base.en.bin models/ggml-base.en-q5_0.bin q5_0
 
 # run the examples as usual, specifying the quantized model file
-./build/bin/main -m models/ggml-base.en-q5_0.bin ./samples/gb0.wav
+./build/bin/whisper-cli -m models/ggml-base.en-q5_0.bin ./samples/gb0.wav
 ```
 
 ## Core ML support
@@ -288,11 +182,11 @@ speed-up - more than x3 faster compared with CPU-only execution. Here are the in
   ```
 
   - To ensure `coremltools` operates correctly, please confirm that [Xcode](https://developer.apple.com/xcode/) is installed and execute `xcode-select --install` to install the command-line tools.
-  - Python 3.10 is recommended.
+  - Python 3.11 is recommended.
   - MacOS Sonoma (version 14) or newer is recommended, as older versions of MacOS might experience issues with transcription hallucination.
   - [OPTIONAL] It is recommended to utilize a Python version management system, such as [Miniconda](https://docs.conda.io/en/latest/miniconda.html) for this step:
-    - To create an environment, use: `conda create -n py310-whisper python=3.10 -y`
+    - To create an environment, use: `conda create -n py311-whisper python=3.11 -y`
-    - To activate the environment, use: `conda activate py310-whisper`
+    - To activate the environment, use: `conda activate py311-whisper`
 
 - Generate a Core ML model. For example, to generate a `base.en` model, use:
 
@@ -313,7 +207,7 @@ speed-up - more than x3 faster compared with CPU-only execution. Here are the in
 - Run the examples as usual. For example:
 
   ```text
-  $ ./main -m models/ggml-base.en.bin -f samples/jfk.wav
+  $ ./build/bin/whisper-cli -m models/ggml-base.en.bin -f samples/jfk.wav
 
   ...
 
@@ -329,7 +223,7 @@ speed-up - more than x3 faster compared with CPU-only execution. Here are the in
   The first run on a device is slow, since the ANE service compiles the Core ML model to some device-specific format.
   Next runs are faster.
 
-For more information about the Core ML implementation please refer to PR [#566](https://github.com/ggerganov/whisper.cpp/pull/566).
+For more information about the Core ML implementation please refer to PR [#566](https://github.com/ggml-org/whisper.cpp/pull/566).
 
 ## OpenVINO support
 
@@ -397,7 +291,7 @@ This can result in significant speedup in encoder performance. Here are the inst
 - Run the examples as usual. For example:
 
   ```text
-  $ ./main -m models/ggml-base.en.bin -f samples/jfk.wav
+  $ ./build/bin/whisper-cli -m models/ggml-base.en.bin -f samples/jfk.wav
 
   ...
 
@@ -414,7 +308,7 @@ This can result in significant speedup in encoder performance. Here are the inst
   The first time run on an OpenVINO device is slow, since the OpenVINO framework will compile the IR (Intermediate Representation) model to a device-specific 'blob'. This device-specific blob will get
   cached for the next run.
 
-For more information about the Core ML implementation please refer to PR [#1037](https://github.com/ggerganov/whisper.cpp/pull/1037).
+For more information about the OpenVINO implementation please refer to PR [#1037](https://github.com/ggml-org/whisper.cpp/pull/1037).
 
 ## NVIDIA GPU support
 
@@ -428,6 +322,12 @@ cmake -B build -DGGML_CUDA=1
 cmake --build build -j --config Release
 ```
 
+or for newer NVIDIA GPU's (RTX 5000 series):
+```
+cmake -B build -DGGML_CUDA=1 -DCMAKE_CUDA_ARCHITECTURES="86"
+cmake --build build -j --config Release
+```
+
 ## Vulkan GPU support
 Cross-vendor solution which allows you to accelerate workload on your GPU.
 First, make sure your graphics card driver provides support for Vulkan API.
@@ -473,7 +373,7 @@ cmake --build build -j --config Release
 Run the inference examples as usual, for example:
 
 ```
-./build/bin/main -f samples/jfk.wav -m models/ggml-base.en.bin -t 8
+./build/bin/whisper-cli -f samples/jfk.wav -m models/ggml-base.en.bin -t 8
 ```
 
 *Notes:*
@@ -481,6 +381,37 @@ Run the inference examples as usual, for example:
 - If you have trouble with Ascend NPU device, please create a issue with **[CANN]** prefix/tag.
 - If you run successfully with your Ascend NPU device, please help update the table `Verified devices`.
 
+## FFmpeg support (Linux only)
+
+If you want to support more audio formats (such as Opus and AAC), you can turn on the `WHISPER_FFMPEG` build flag to enable FFmpeg integration.
+
+First, you need to install required libraries:
+
+```bash
+# Debian/Ubuntu
+sudo apt install libavcodec-dev libavformat-dev libavutil-dev
+
+# RHEL/Fedora
+sudo dnf install libavcodec-free-devel libavformat-free-devel libavutil-free-devel
+```
+
+Then you can build the project as follows:
+
+```bash
+cmake -B build -D WHISPER_FFMPEG=yes
+cmake --build build
+```
+
+Run the following example to confirm it's working:
+
+```bash
+# Convert an audio file to Opus format
+ffmpeg -i samples/jfk.wav jfk.opus
+
+# Transcribe the audio file
+./build/bin/whisper-cli --model models/ggml-base.en.bin --file jfk.opus
+```
+
 ## Docker
 
 ### Prerequisites
@@ -492,8 +423,8 @@ Run the inference examples as usual, for example:
 
 We have two Docker images available for this project:
 
-1. `ghcr.io/ggerganov/whisper.cpp:main`: This image includes the main executable file as well as `curl` and `ffmpeg`. (platforms: `linux/amd64`, `linux/arm64`)
+1. `ghcr.io/ggml-org/whisper.cpp:main`: This image includes the main executable file as well as `curl` and `ffmpeg`. (platforms: `linux/amd64`, `linux/arm64`)
-2. `ghcr.io/ggerganov/whisper.cpp:main-cuda`: Same as `main` but compiled with CUDA support. (platforms: `linux/amd64`)
+2. `ghcr.io/ggml-org/whisper.cpp:main-cuda`: Same as `main` but compiled with CUDA support. (platforms: `linux/amd64`)
 
 ### Usage
 
@@ -527,99 +458,17 @@ For detailed instructions on how to use Conan, please refer to the [Conan docume
 
 - Inference only
 
-## Another example
-
-Here is another example of transcribing a [3:24 min speech](https://upload.wikimedia.org/wikipedia/commons/1/1f/George_W_Bush_Columbia_FINAL.ogg)
-in about half a minute on a MacBook M1 Pro, using `medium.en` model:
-
-<details>
-  <summary>Expand to see the result</summary>
-
-```text
-$ ./main -m models/ggml-medium.en.bin -f samples/gb1.wav -t 8
-
-whisper_init_from_file: loading model from 'models/ggml-medium.en.bin'
-whisper_model_load: loading model
-whisper_model_load: n_vocab       = 51864
-whisper_model_load: n_audio_ctx   = 1500
-whisper_model_load: n_audio_state = 1024
-whisper_model_load: n_audio_head  = 16
-whisper_model_load: n_audio_layer = 24
-whisper_model_load: n_text_ctx    = 448
-whisper_model_load: n_text_state  = 1024
-whisper_model_load: n_text_head   = 16
-whisper_model_load: n_text_layer  = 24
-whisper_model_load: n_mels        = 80
-whisper_model_load: f16           = 1
-whisper_model_load: type          = 4
-whisper_model_load: mem required  = 1720.00 MB (+   43.00 MB per decoder)
-whisper_model_load: kv self size  =   42.00 MB
-whisper_model_load: kv cross size =  140.62 MB
-whisper_model_load: adding 1607 extra tokens
-whisper_model_load: model ctx     = 1462.35 MB
-whisper_model_load: model size    = 1462.12 MB
-
-system_info: n_threads = 8 / 10 | AVX = 0 | AVX2 = 0 | AVX512 = 0 | FMA = 0 | NEON = 1 | ARM_FMA = 1 | F16C = 0 | FP16_VA = 1 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 0 | VSX = 0 |
-
-main: processing 'samples/gb1.wav' (3179750 samples, 198.7 sec), 8 threads, 1 processors, lang = en, task = transcribe, timestamps = 1 ...
-
-
-[00:00:00.000 --> 00:00:08.000]   My fellow Americans, this day has brought terrible news and great sadness to our country.
-[00:00:08.000 --> 00:00:17.000]   At nine o'clock this morning, Mission Control in Houston lost contact with our Space Shuttle Columbia.
-[00:00:17.000 --> 00:00:23.000]   A short time later, debris was seen falling from the skies above Texas.
-[00:00:23.000 --> 00:00:29.000]   The Columbia's lost. There are no survivors.
-[00:00:29.000 --> 00:00:32.000]   On board was a crew of seven.
-[00:00:32.000 --> 00:00:39.000]   Colonel Rick Husband, Lieutenant Colonel Michael Anderson, Commander Laurel Clark,
-[00:00:39.000 --> 00:00:48.000]   Captain David Brown, Commander William McCool, Dr. Kultna Shavla, and Ilan Ramon,
-[00:00:48.000 --> 00:00:52.000]   a colonel in the Israeli Air Force.
-[00:00:52.000 --> 00:00:58.000]   These men and women assumed great risk in the service to all humanity.
-[00:00:58.000 --> 00:01:03.000]   In an age when space flight has come to seem almost routine,
-[00:01:03.000 --> 00:01:07.000]   it is easy to overlook the dangers of travel by rocket
-[00:01:07.000 --> 00:01:12.000]   and the difficulties of navigating the fierce outer atmosphere of the Earth.
-[00:01:12.000 --> 00:01:18.000]   These astronauts knew the dangers, and they faced them willingly,
-[00:01:18.000 --> 00:01:23.000]   knowing they had a high and noble purpose in life.
-[00:01:23.000 --> 00:01:31.000]   Because of their courage and daring and idealism, we will miss them all the more.
-[00:01:31.000 --> 00:01:36.000]   All Americans today are thinking as well of the families of these men and women
-[00:01:36.000 --> 00:01:40.000]   who have been given this sudden shock and grief.
-[00:01:40.000 --> 00:01:45.000]   You're not alone. Our entire nation grieves with you,
-[00:01:45.000 --> 00:01:52.000]   and those you love will always have the respect and gratitude of this country.
-[00:01:52.000 --> 00:01:56.000]   The cause in which they died will continue.
-[00:01:56.000 --> 00:02:04.000]   Mankind is led into the darkness beyond our world by the inspiration of discovery
-[00:02:04.000 --> 00:02:11.000]   and the longing to understand. Our journey into space will go on.
-[00:02:11.000 --> 00:02:16.000]   In the skies today, we saw destruction and tragedy.
-[00:02:16.000 --> 00:02:22.000]   Yet farther than we can see, there is comfort and hope.
-[00:02:22.000 --> 00:02:29.000]   In the words of the prophet Isaiah, "Lift your eyes and look to the heavens
-[00:02:29.000 --> 00:02:35.000]   who created all these. He who brings out the starry hosts one by one
-[00:02:35.000 --> 00:02:39.000]   and calls them each by name."
-[00:02:39.000 --> 00:02:46.000]   Because of His great power and mighty strength, not one of them is missing.
-[00:02:46.000 --> 00:02:55.000]   The same Creator who names the stars also knows the names of the seven souls we mourn today.
-[00:02:55.000 --> 00:03:01.000]   The crew of the shuttle Columbia did not return safely to earth,
-[00:03:01.000 --> 00:03:05.000]   yet we can pray that all are safely home.
-[00:03:05.000 --> 00:03:13.000]   May God bless the grieving families, and may God continue to bless America.
-[00:03:13.000 --> 00:03:19.000]   [Silence]
-
-
-whisper_print_timings:     fallbacks =   1 p /   0 h
-whisper_print_timings:     load time =   569.03 ms
-whisper_print_timings:      mel time =   146.85 ms
-whisper_print_timings:   sample time =   238.66 ms /   553 runs (    0.43 ms per run)
-whisper_print_timings:   encode time = 18665.10 ms /     9 runs ( 2073.90 ms per run)
-whisper_print_timings:   decode time = 13090.93 ms /   549 runs (   23.85 ms per run)
-whisper_print_timings:    total time = 32733.52 ms
-```
-
-</details>
-
 ## Real-time audio input example
 
 This is a naive example of performing real-time inference on audio from your microphone.
 The [stream](examples/stream) tool samples the audio every half a second and runs the transcription continuously.
-More info is available in [issue #10](https://github.com/ggerganov/whisper.cpp/issues/10).
+More info is available in [issue #10](https://github.com/ggml-org/whisper.cpp/issues/10).
+You will need to have [sdl2](https://wiki.libsdl.org/SDL2/Installation) installed for it to work properly.
 
 ```bash
-cmake -B build
+cmake -B build -DWHISPER_SDL2=ON
 cmake --build build --config Release
-./build/bin/stream -m ./models/ggml-base.en.bin -t 8 --step 500 --length 5000
+./build/bin/whisper-stream -m ./models/ggml-base.en.bin -t 8 --step 500 --length 5000
 ```
 
 https://user-images.githubusercontent.com/1991296/194935793-76afede7-cfa8-48d8-a80f-28ba83be7d09.mp4
@@ -630,7 +479,7 @@ Adding the `--print-colors` argument will print the transcribed text using an ex
 to highlight words with high or low confidence:
 
 ```bash
-./main -m models/ggml-base.en.bin -f samples/gb0.wav --print-colors
+./build/bin/whisper-cli -m models/ggml-base.en.bin -f samples/gb0.wav --print-colors
 ```
 
 <img width="965" alt="image" src="https://user-images.githubusercontent.com/1991296/197356445-311c8643-9397-4e5e-b46e-0b4b4daa2530.png">
@@ -640,7 +489,7 @@ to highlight words with high or low confidence:
 For example, to limit the line length to a maximum of 16 characters, simply add `-ml 16`:
 
 ```text
-$ ./main -m ./models/ggml-base.en.bin -f ./samples/jfk.wav -ml 16
+$ ./build/bin/whisper-cli -m ./models/ggml-base.en.bin -f ./samples/jfk.wav -ml 16
 
 whisper_model_load: loading model from './models/ggml-base.en.bin'
 ...
@@ -664,7 +513,7 @@ main: processing './samples/jfk.wav' (176000 samples, 11.0 sec), 4 threads, 1 pr
 The `--max-len` argument can be used to obtain word-level timestamps. Simply use `-ml 1`:
 
 ```text
-$ ./main -m ./models/ggml-base.en.bin -f ./samples/jfk.wav -ml 1
+$ ./build/bin/whisper-cli -m ./models/ggml-base.en.bin -f ./samples/jfk.wav -ml 1
 
 whisper_model_load: loading model from './models/ggml-base.en.bin'
 ...
@@ -702,7 +551,7 @@ main: processing './samples/jfk.wav' (176000 samples, 11.0 sec), 4 threads, 1 pr
 
 ## Speaker segmentation via tinydiarize (experimental)
 
-More information about this approach is available here: https://github.com/ggerganov/whisper.cpp/pull/1058
+More information about this approach is available here: https://github.com/ggml-org/whisper.cpp/pull/1058
 
 Sample usage:
 
@@ -711,7 +560,7 @@ Sample usage:
 ./models/download-ggml-model.sh small.en-tdrz
 
 # run as usual, adding the "-tdrz" command-line argument
-./main -f ./samples/a13.wav -m ./models/ggml-small.en-tdrz.bin -tdrz
+./build/bin/whisper-cli -f ./samples/a13.wav -m ./models/ggml-small.en-tdrz.bin -tdrz
 ...
 main: processing './samples/a13.wav' (480000 samples, 30.0 sec), 4 threads, 1 processors, lang = en, task = transcribe, tdrz = 1, timestamps = 1 ...
 ...
@@ -728,14 +577,14 @@ main: processing './samples/a13.wav' (480000 samples, 30.0 sec), 4 threads, 1 pr
 
 ## Karaoke-style movie generation (experimental)
 
-The [main](examples/main) example provides support for output of karaoke-style movies, where the
+The [whisper-cli](examples/cli) example provides support for output of karaoke-style movies, where the
 currently pronounced word is highlighted. Use the `-wts` argument and run the generated bash script.
 This requires to have `ffmpeg` installed.
 
 Here are a few _"typical"_ examples:
 
 ```bash
-./main -m ./models/ggml-base.en.bin -f ./samples/jfk.wav -owts
+./build/bin/whisper-cli -m ./models/ggml-base.en.bin -f ./samples/jfk.wav -owts
 source ./samples/jfk.wav.wts
 ffplay ./samples/jfk.wav.mp4
 ```
@@ -745,7 +594,7 @@ https://user-images.githubusercontent.com/1991296/199337465-dbee4b5e-9aeb-48a3-b
 ---
 
 ```bash
-./main -m ./models/ggml-base.en.bin -f ./samples/mm0.wav -owts
+./build/bin/whisper-cli -m ./models/ggml-base.en.bin -f ./samples/mm0.wav -owts
 source ./samples/mm0.wav.wts
 ffplay ./samples/mm0.wav.mp4
 ```
@@ -755,7 +604,7 @@ https://user-images.githubusercontent.com/1991296/199337504-cc8fd233-0cb7-4920-9
 ---
 
 ```bash
-./main -m ./models/ggml-base.en.bin -f ./samples/gb0.wav -owts
+./build/bin/whisper-cli -m ./models/ggml-base.en.bin -f ./samples/gb0.wav -owts
 source ./samples/gb0.wav.wts
 ffplay ./samples/gb0.wav.mp4
 ```
@@ -766,7 +615,7 @@ https://user-images.githubusercontent.com/1991296/199337538-b7b0c7a3-2753-4a88-a
 
 ## Video comparison of different models
 
-Use the [scripts/bench-wts.sh](https://github.com/ggerganov/whisper.cpp/blob/master/scripts/bench-wts.sh) script to generate a video in the following format:
+Use the [scripts/bench-wts.sh](https://github.com/ggml-org/whisper.cpp/blob/master/scripts/bench-wts.sh) script to generate a video in the following format:
 
 ```bash
 ./scripts/bench-wts.sh samples/jfk.wav
@@ -780,10 +629,10 @@ https://user-images.githubusercontent.com/1991296/223206245-2d36d903-cf8e-4f09-8
 ## Benchmarks
 
 In order to have an objective comparison of the performance of the inference across different system configurations,
-use the [bench](examples/bench) tool. The tool simply runs the Encoder part of the model and prints how much time it
+use the [whisper-bench](examples/bench) tool. The tool simply runs the Encoder part of the model and prints how much time it
 took to execute it. The results are summarized in the following Github issue:
 
-[Benchmark results](https://github.com/ggerganov/whisper.cpp/issues/89)
+[Benchmark results](https://github.com/ggml-org/whisper.cpp/issues/89)
 
 Additionally a script to run whisper.cpp with different models and audio files is provided [bench.py](scripts/bench.py).
 
@@ -810,25 +659,24 @@ You can download the converted models using the [models/download-ggml-model.sh](
 or manually from here:
 
 - https://huggingface.co/ggerganov/whisper.cpp
-- https://ggml.ggerganov.com
 
 For more details, see the conversion script [models/convert-pt-to-ggml.py](models/convert-pt-to-ggml.py) or [models/README.md](models/README.md).
 
-## [Bindings](https://github.com/ggerganov/whisper.cpp/discussions/categories/bindings)
+## [Bindings](https://github.com/ggml-org/whisper.cpp/discussions/categories/bindings)
 
-- [x] Rust: [tazz4843/whisper-rs](https://github.com/tazz4843/whisper-rs) | [#310](https://github.com/ggerganov/whisper.cpp/discussions/310)
+- [x] Rust: [tazz4843/whisper-rs](https://github.com/tazz4843/whisper-rs) | [#310](https://github.com/ggml-org/whisper.cpp/discussions/310)
-- [x] JavaScript: [bindings/javascript](bindings/javascript) | [#309](https://github.com/ggerganov/whisper.cpp/discussions/309)
+- [x] JavaScript: [bindings/javascript](bindings/javascript) | [#309](https://github.com/ggml-org/whisper.cpp/discussions/309)
   - React Native (iOS / Android): [whisper.rn](https://github.com/mybigday/whisper.rn)
-- [x] Go: [bindings/go](bindings/go) | [#312](https://github.com/ggerganov/whisper.cpp/discussions/312)
+- [x] Go: [bindings/go](bindings/go) | [#312](https://github.com/ggml-org/whisper.cpp/discussions/312)
 - [x] Java:
   - [GiviMAD/whisper-jni](https://github.com/GiviMAD/whisper-jni)
-- [x] Ruby: [bindings/ruby](bindings/ruby) | [#507](https://github.com/ggerganov/whisper.cpp/discussions/507)
+- [x] Ruby: [bindings/ruby](bindings/ruby) | [#507](https://github.com/ggml-org/whisper.cpp/discussions/507)
-- [x] Objective-C / Swift: [ggerganov/whisper.spm](https://github.com/ggerganov/whisper.spm) | [#313](https://github.com/ggerganov/whisper.cpp/discussions/313)
+- [x] Objective-C / Swift: [ggml-org/whisper.spm](https://github.com/ggml-org/whisper.spm) | [#313](https://github.com/ggml-org/whisper.cpp/discussions/313)
   - [exPHAT/SwiftWhisper](https://github.com/exPHAT/SwiftWhisper)
-- [x] .NET: | [#422](https://github.com/ggerganov/whisper.cpp/discussions/422)
+- [x] .NET: | [#422](https://github.com/ggml-org/whisper.cpp/discussions/422)
   - [sandrohanea/whisper.net](https://github.com/sandrohanea/whisper.net)
   - [NickDarvey/whisper](https://github.com/NickDarvey/whisper)
-- [x] Python: | [#9](https://github.com/ggerganov/whisper.cpp/issues/9)
+- [x] Python: | [#9](https://github.com/ggml-org/whisper.cpp/issues/9)
   - [stlukey/whispercpp.py](https://github.com/stlukey/whispercpp.py) (Cython)
   - [AIWintermuteAI/whispercpp](https://github.com/AIWintermuteAI/whispercpp) (Updated fork of aarnphm/whispercpp)
   - [aarnphm/whispercpp](https://github.com/aarnphm/whispercpp) (Pybind11)
@@ -836,6 +684,33 @@ For more details, see the conversion script [models/convert-pt-to-ggml.py](model
 - [x] R: [bnosac/audio.whisper](https://github.com/bnosac/audio.whisper)
 - [x] Unity: [macoron/whisper.unity](https://github.com/Macoron/whisper.unity)
 
+## XCFramework
+The XCFramework is a precompiled version of the library for iOS, visionOS, tvOS,
+and macOS. It can be used in Swift projects without the need to compile the
+library from source. For examples:
+```swift
+// swift-tools-version: 5.10
+// The swift-tools-version declares the minimum version of Swift required to build this package.
+
+import PackageDescription
+
+let package = Package(
+    name: "Whisper",
+    targets: [
+        .executableTarget(
+            name: "Whisper",
+            dependencies: [
+                "WhisperFramework"
+            ]),
+        .binaryTarget(
+            name: "WhisperFramework",
+            url: "https://github.com/ggml-org/whisper.cpp/releases/download/v1.7.5/whisper-v1.7.5-xcframework.zip",
+            checksum: "c7faeb328620d6012e130f3d705c51a6ea6c995605f2df50f6e1ad68c59c6c4a"
+        )
+    ]
+)
+```
+
 ## Examples
 
 There are various examples of using the library for different projects in the [examples](examples) folder.
@@ -843,25 +718,24 @@ Some of the examples are even ported to run in the browser using WebAssembly. Ch
 
 | Example                                             | Web                                   | Description                                                                                                                     |
 | --------------------------------------------------- | ------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------- |
-| [main](examples/main)                               | [whisper.wasm](examples/whisper.wasm) | Tool for translating and transcribing audio using Whisper                                                                       |
+| [whisper-cli](examples/cli)                         | [whisper.wasm](examples/whisper.wasm) | Tool for translating and transcribing audio using Whisper                                                                       |
-| [bench](examples/bench)                             | [bench.wasm](examples/bench.wasm)     | Benchmark the performance of Whisper on your machine                                                                            |
+| [whisper-bench](examples/bench)                     | [bench.wasm](examples/bench.wasm)     | Benchmark the performance of Whisper on your machine                                                                            |
-| [stream](examples/stream)                           | [stream.wasm](examples/stream.wasm)   | Real-time transcription of raw microphone capture                                                                               |
+| [whisper-stream](examples/stream)                   | [stream.wasm](examples/stream.wasm)   | Real-time transcription of raw microphone capture                                                                               |
-| [command](examples/command)                         | [command.wasm](examples/command.wasm) | Basic voice assistant example for receiving voice commands from the mic                                                         |
+| [whisper-command](examples/command)                 | [command.wasm](examples/command.wasm) | Basic voice assistant example for receiving voice commands from the mic                                                         |
-| [wchess](examples/wchess)                           | [wchess.wasm](examples/wchess)        | Voice-controlled chess                                                                                                          |
+| [whisper-server](examples/server)                   |                                       | HTTP transcription server with OAI-like API                                                                                     |
-| [talk](examples/talk)                               | [talk.wasm](examples/talk.wasm)       | Talk with a GPT-2 bot                                                                                                           |
+| [whisper-talk-llama](examples/talk-llama)           |                                       | Talk with a LLaMA bot                                                                                                           |
-| [talk-llama](examples/talk-llama)                   |                                       | Talk with a LLaMA bot                                                                                                           |
 | [whisper.objc](examples/whisper.objc)               |                                       | iOS mobile application using whisper.cpp                                                                                        |
 | [whisper.swiftui](examples/whisper.swiftui)         |                                       | SwiftUI iOS / macOS application using whisper.cpp                                                                               |
 | [whisper.android](examples/whisper.android)         |                                       | Android mobile application using whisper.cpp                                                                                    |
 | [whisper.nvim](examples/whisper.nvim)               |                                       | Speech-to-text plugin for Neovim                                                                                                |
 | [generate-karaoke.sh](examples/generate-karaoke.sh) |                                       | Helper script to easily [generate a karaoke video](https://youtu.be/uj7hVta4blM) of raw audio capture                           |
-| [livestream.sh](examples/livestream.sh)             |                                       | [Livestream audio transcription](https://github.com/ggerganov/whisper.cpp/issues/185)                                           |
+| [livestream.sh](examples/livestream.sh)             |                                       | [Livestream audio transcription](https://github.com/ggml-org/whisper.cpp/issues/185)                                            |
 | [yt-wsp.sh](examples/yt-wsp.sh)                     |                                       | Download + transcribe and/or translate any VOD [(original)](https://gist.github.com/DaniruKun/96f763ec1a037cc92fe1a059b643b818) |
-| [server](examples/server)                           |                                       | HTTP transcription server with OAI-like API                                                                                     |
+| [wchess](examples/wchess)                           | [wchess.wasm](examples/wchess)        | Voice-controlled chess                                                                                                          |
 
-## [Discussions](https://github.com/ggerganov/whisper.cpp/discussions)
+## [Discussions](https://github.com/ggml-org/whisper.cpp/discussions)
 
 If you have any kind of feedback about this project feel free to use the Discussions section and open a new topic.
-You can use the [Show and tell](https://github.com/ggerganov/whisper.cpp/discussions/categories/show-and-tell) category
+You can use the [Show and tell](https://github.com/ggml-org/whisper.cpp/discussions/categories/show-and-tell) category
 to share your own projects that use `whisper.cpp`. If you have a question, make sure to check the
-[Frequently asked questions (#126)](https://github.com/ggerganov/whisper.cpp/discussions/126) discussion.
+[Frequently asked questions (#126)](https://github.com/ggml-org/whisper.cpp/discussions/126) discussion.

Sources/whisper/module.modulemap

@@ -1,5 +0,0 @@
-module whisper [system] {
-    header "whisper.h"
-    link "whisper"
-    export *
-}

Sources/whisper/whisper.h

@@ -1,4 +0,0 @@
-#pragma once
-
-#include <whisper.h>
-

bindings/go/Makefile

@@ -11,11 +11,11 @@ UNAME_M := $(shell uname -m)
 endif
 
 GGML_METAL_PATH_RESOURCES := $(abspath ../..)
-BUILD_DIR := build
+BUILD_DIR := build_go
 MODELS_DIR := models
 EXAMPLES_DIR := $(wildcard examples/*)
 INCLUDE_PATH := $(abspath ../../include):$(abspath ../../ggml/include)
-LIBRARY_PATH := $(abspath ../..)
+LIBRARY_PATH := $(abspath ../../${BUILD_DIR}/src:$(abspath ../../${BUILD_DIR}/ggml/src))
 
 ifeq ($(GGML_CUDA),1)
 	LIBRARY_PATH := $(LIBRARY_PATH):$(CUDA_PATH)/targets/$(UNAME_M)-linux/lib/
@@ -29,8 +29,10 @@ endif
 all: clean whisper examples
 
 whisper: mkdir
-	@echo Build whisper
+	cmake -S ../.. -B ../../${BUILD_DIR} \
-	@${MAKE} -C ../.. libwhisper.a
+		-DCMAKE_BUILD_TYPE=Release \
+		-DBUILD_SHARED_LIBS=OFF
+	cmake --build ../../${BUILD_DIR} --target whisper
 
 test: model-small whisper modtidy
 ifeq ($(UNAME_S),Darwin)

bindings/go/README.md

@@ -31,7 +31,7 @@ func main() {
 	if err != nil {
 		panic(err)
 	}
-	if err := context.Process(samples, nil, nil); err != nil {
+	if err := context.Process(samples, nil, nil, nil); err != nil {
 		return err
 	}
 
@@ -51,7 +51,7 @@ func main() {
 In order to build, you need to have the Go compiler installed. You can get it from [here](https://golang.org/dl/). Run the tests with:
 
 ```bash
-git clone https://github.com/ggerganov/whisper.cpp.git
+git clone https://github.com/ggml-org/whisper.cpp.git
 cd whisper.cpp/bindings/go
 make test
 ```
@@ -98,7 +98,7 @@ The API Documentation:
 
 Getting help:
 
-  * Follow the discussion for the go bindings [here](https://github.com/ggerganov/whisper.cpp/discussions/312)
+  * Follow the discussion for the go bindings [here](https://github.com/ggml-org/whisper.cpp/discussions/312)
 
 ## License
 

bindings/go/doc.go

@@ -1,5 +1,5 @@
 /*
-github.com/ggerganov/whisper.cpp/bindings/go
+github.com/ggml-org/whisper.cpp/bindings/go
 provides a speech-to-text service bindings for the Go programming language.
 */
 package whisper

bindings/go/examples/go-model-download/context.go

@@ -9,22 +9,23 @@ import (
 // ContextForSignal returns a context object which is cancelled when a signal
 // is received. It returns nil if no signal parameter is provided
 func ContextForSignal(signals ...os.Signal) context.Context {
-	if len(signals) == 0 {
+    if len(signals) == 0 {
-		return nil
+        return nil
-	}
+    }
 
-	ch := make(chan os.Signal)
+    ch := make(chan os.Signal, 1) // Buffered channel with space for 1 signal
-	ctx, cancel := context.WithCancel(context.Background())
+    ctx, cancel := context.WithCancel(context.Background())
 
-	// Send message on channel when signal received
+    // Send message on channel when signal received
-	signal.Notify(ch, signals...)
+    signal.Notify(ch, signals...)
 
-	// When any signal received, call cancel
+    // When any signal is received, call cancel
-	go func() {
+    go func() {
-		<-ch
+        <-ch
-		cancel()
+        cancel()
-	}()
+    }()
 
-	// Return success
+    // Return success
-	return ctx
+    return ctx
 }
+

bindings/go/examples/go-model-download/main.go

@@ -9,6 +9,7 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"strings"
 	"syscall"
 	"time"
 )
@@ -17,14 +18,27 @@ import (
 // CONSTANTS
 
 const (
-	srcUrl  = "https://huggingface.co/ggerganov/whisper.cpp/resolve/main" // The location of the models
+	srcUrl  = "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/" // The location of the models
-	srcExt  = ".bin"                                                      // Filename extension
+	srcExt  = ".bin"                                                       // Filename extension
-	bufSize = 1024 * 64                                                   // Size of the buffer used for downloading the model
+	bufSize = 1024 * 64                                                    // Size of the buffer used for downloading the model
 )
 
 var (
 	// The models which will be downloaded, if no model is specified as an argument
-	modelNames = []string{"ggml-tiny.en", "ggml-tiny", "ggml-base.en", "ggml-base", "ggml-small.en", "ggml-small", "ggml-medium.en", "ggml-medium", "ggml-large-v1", "ggml-large-v2", "ggml-large-v3", "large-v3-turbo"}
+	modelNames = []string{
+		"tiny", "tiny-q5_1", "tiny-q8_0",
+		"tiny.en", "tiny.en-q5_1", "tiny.en-q8_0",
+		"base", "base-q5_1", "base-q8_0",
+		"base.en", "base.en-q5_1", "base.en-q8_0",
+		"small", "small-q5_1", "small-q8_0",
+		"small.en", "small.en-q5_1", "small.en-q8_0",
+		"medium", "medium-q5_0", "medium-q8_0",
+		"medium.en", "medium.en-q5_0", "medium.en-q8_0",
+		"large-v1",
+		"large-v2", "large-v2-q5_0", "large-v2-q8_0",
+		"large-v3", "large-v3-q5_0",
+		"large-v3-turbo", "large-v3-turbo-q5_0", "large-v3-turbo-q8_0",
+	}
 )
 
 var (
@@ -44,7 +58,25 @@ var (
 func main() {
 	flag.Usage = func() {
 		name := filepath.Base(flag.CommandLine.Name())
-		fmt.Fprintf(flag.CommandLine.Output(), "Usage: %s [options] <model>\n\n", name)
+		fmt.Fprintf(flag.CommandLine.Output(), `
+			Usage: %s [options] [<model>...]
+
+			Options:
+  			-out string     Specify the output folder where models will be saved.
+                  			Default: Current working directory.
+  			-timeout duration Set the maximum duration for downloading a model.
+            			      Example: 10m, 1h (default: 30m0s).
+  			-quiet           Suppress all output except errors.
+
+			Examples:
+  			1. Download a specific model:
+     			%s -out ./models tiny-q8_0
+
+			  2. Download all models:
+     			%s -out ./models
+
+			`, name, name, name)
+
 		flag.PrintDefaults()
 	}
 	flag.Parse()
@@ -114,23 +146,87 @@ func GetOut() (string, error) {
 // GetModels returns the list of models to download
 func GetModels() []string {
 	if flag.NArg() == 0 {
-		return modelNames
+		fmt.Println("No model specified.")
-	} else {
+		fmt.Println("Preparing to download all models...")
-		return flag.Args()
+
+		// Calculate total download size
+		fmt.Println("Calculating total download size...")
+		totalSize, err := CalculateTotalDownloadSize(modelNames)
+		if err != nil {
+			fmt.Println("Error calculating download sizes:", err)
+			os.Exit(1)
+		}
+
+		fmt.Println("View available models: https://huggingface.co/ggerganov/whisper.cpp/tree/main")
+		fmt.Printf("Total download size: %.2f GB\n", float64(totalSize)/(1024*1024*1024))
+		fmt.Println("Would you like to download all models? (y/N)")
+
+		// Prompt for user input
+		var response string
+		fmt.Scanln(&response)
+		if response != "y" && response != "Y" {
+			fmt.Println("Aborting. Specify a model to download.")
+			os.Exit(0)
+		}
+
+		return modelNames // Return all models if confirmed
 	}
+	return flag.Args() // Return specific models if arguments are provided
+}
+
+func CalculateTotalDownloadSize(models []string) (int64, error) {
+	var totalSize int64
+	client := http.Client{}
+
+	for _, model := range models {
+		modelURL, err := URLForModel(model)
+		if err != nil {
+			return 0, err
+		}
+
+		// Issue a HEAD request to get the file size
+		req, err := http.NewRequest("HEAD", modelURL, nil)
+		if err != nil {
+			return 0, err
+		}
+
+		resp, err := client.Do(req)
+		if err != nil {
+			return 0, err
+		}
+		resp.Body.Close()
+
+		if resp.StatusCode != http.StatusOK {
+			fmt.Printf("Warning: Unable to fetch size for %s (HTTP %d)\n", model, resp.StatusCode)
+			continue
+		}
+
+		size := resp.ContentLength
+		totalSize += size
+	}
+	return totalSize, nil
 }
 
 // URLForModel returns the URL for the given model on huggingface.co
 func URLForModel(model string) (string, error) {
+	// Ensure "ggml-" prefix is added only once
+	if !strings.HasPrefix(model, "ggml-") {
+		model = "ggml-" + model
+	}
+
+	// Ensure ".bin" extension is added only once
 	if filepath.Ext(model) != srcExt {
 		model += srcExt
 	}
+
+	// Parse the base URL
 	url, err := url.Parse(srcUrl)
 	if err != nil {
 		return "", err
-	} else {
-		url.Path = filepath.Join(url.Path, model)
 	}
+
+	// Ensure no trailing slash in the base URL
+	url.Path = fmt.Sprintf("%s/%s", strings.TrimSuffix(url.Path, "/"), model)
 	return url.String(), nil
 }
 

bindings/go/examples/go-whisper/process.go

@@ -67,7 +67,7 @@ func Process(model whisper.Model, path string, flags *Flags) error {
 	// Process the data
 	fmt.Fprintf(flags.Output(), "  ...processing %q\n", path)
 	context.ResetTimings()
-	if err := context.Process(data, cb, nil); err != nil {
+	if err := context.Process(data, nil, cb, nil); err != nil {
 		return err
 	}
 

bindings/go/pkg/whisper/context.go

@@ -71,6 +71,10 @@ func (context *context) Language() string {
 	return whisper.Whisper_lang_str(context.params.Language())
 }
 
+func (context *context) DetectedLanguage() string {
+	return whisper.Whisper_lang_str(context.model.ctx.Whisper_full_lang_id())
+}
+
 // Set translate flag
 func (context *context) SetTranslate(v bool) {
 	context.params.SetTranslate(v)
@@ -189,6 +193,7 @@ func (context *context) WhisperLangAutoDetect(offset_ms int, n_threads int) ([]f
 // Process new sample data and return any errors
 func (context *context) Process(
 	data []float32,
+	callEncoderBegin EncoderBeginCallback,
 	callNewSegment SegmentCallback,
 	callProgress ProgressCallback,
 ) error {
@@ -203,7 +208,20 @@ func (context *context) Process(
 	// We don't do parallel processing at the moment
 	processors := 0
 	if processors > 1 {
-		if err := context.model.ctx.Whisper_full_parallel(context.params, data, processors, nil, func(new int) {
+		if err := context.model.ctx.Whisper_full_parallel(context.params, data, processors, callEncoderBegin,
+			func(new int) {
+				if callNewSegment != nil {
+					num_segments := context.model.ctx.Whisper_full_n_segments()
+					s0 := num_segments - new
+					for i := s0; i < num_segments; i++ {
+						callNewSegment(toSegment(context.model.ctx, i))
+					}
+				}
+			}); err != nil {
+			return err
+		}
+	} else if err := context.model.ctx.Whisper_full(context.params, data, callEncoderBegin,
+		func(new int) {
 			if callNewSegment != nil {
 				num_segments := context.model.ctx.Whisper_full_n_segments()
 				s0 := num_segments - new
@@ -211,22 +229,11 @@ func (context *context) Process(
 					callNewSegment(toSegment(context.model.ctx, i))
 				}
 			}
-		}); err != nil {
+		}, func(progress int) {
-			return err
+			if callProgress != nil {
-		}
+				callProgress(progress)
-	} else if err := context.model.ctx.Whisper_full(context.params, data, nil, func(new int) {
-		if callNewSegment != nil {
-			num_segments := context.model.ctx.Whisper_full_n_segments()
-			s0 := num_segments - new
-			for i := s0; i < num_segments; i++ {
-				callNewSegment(toSegment(context.model.ctx, i))
 			}
-		}
+		}); err != nil {
-	}, func(progress int) {
-		if callProgress != nil {
-			callProgress(progress)
-		}
-	}); err != nil {
 		return err
 	}
 

bindings/go/pkg/whisper/context_test.go

@@ -88,6 +88,37 @@ func TestProcess(t *testing.T) {
 	context, err := model.NewContext()
 	assert.NoError(err)
 
-	err = context.Process(data, nil, nil)
+	err = context.Process(data, nil, nil, nil)
 	assert.NoError(err)
 }
+
+func TestDetectedLanguage(t *testing.T) {
+	assert := assert.New(t)
+
+	fh, err := os.Open(SamplePath)
+	assert.NoError(err)
+	defer fh.Close()
+
+	// Decode the WAV file - load the full buffer
+	dec := wav.NewDecoder(fh)
+	buf, err := dec.FullPCMBuffer()
+	assert.NoError(err)
+	assert.Equal(uint16(1), dec.NumChans)
+
+	data := buf.AsFloat32Buffer().Data
+
+	model, err := whisper.New(ModelPath)
+	assert.NoError(err)
+	assert.NotNil(model)
+	defer model.Close()
+
+	context, err := model.NewContext()
+	assert.NoError(err)
+
+	err = context.Process(data, nil, nil, nil)
+	assert.NoError(err)
+
+	expectedLanguage := "en"
+	actualLanguage := context.DetectedLanguage()
+	assert.Equal(expectedLanguage, actualLanguage)
+}

bindings/go/pkg/whisper/interface.go

@@ -16,6 +16,10 @@ type SegmentCallback func(Segment)
 // processing. It is called during the Process function
 type ProgressCallback func(int)
 
+// EncoderBeginCallback is the callback function for checking if we want to
+// continue processing. It is called during the Process function
+type EncoderBeginCallback func() bool
+
 // Model is the interface to a whisper model. Create a new model with the
 // function whisper.New(string)
 type Model interface {
@@ -31,12 +35,13 @@ type Model interface {
 	Languages() []string
 }
 
-// Context is the speach recognition context.
+// Context is the speech recognition context.
 type Context interface {
 	SetLanguage(string) error // Set the language to use for speech recognition, use "auto" for auto detect language.
 	SetTranslate(bool)        // Set translate flag
 	IsMultilingual() bool     // Return true if the model is multilingual.
 	Language() string         // Get language
+	DetectedLanguage() string // Get detected language
 
 	SetOffset(time.Duration)          // Set offset
 	SetDuration(time.Duration)        // Set duration
@@ -58,7 +63,7 @@ type Context interface {
 	// Process mono audio data and return any errors.
 	// If defined, newly generated segments are passed to the
 	// callback function during processing.
-	Process([]float32, SegmentCallback, ProgressCallback) error
+	Process([]float32, EncoderBeginCallback, SegmentCallback, ProgressCallback) error
 
 	// After process is called, return segments until the end of the stream
 	// is reached, when io.EOF is returned.

bindings/go/whisper.go

@@ -9,7 +9,7 @@ import (
 // CGO
 
 /*
-#cgo LDFLAGS: -lwhisper -lm -lstdc++ -fopenmp
+#cgo LDFLAGS: -lwhisper -lggml -lggml-base -lggml-cpu  -lm -lstdc++ -fopenmp
 #cgo darwin LDFLAGS: -framework Accelerate -framework Metal -framework Foundation -framework CoreGraphics
 #include <whisper.h>
 #include <stdlib.h>

bindings/java/README.md

@@ -52,7 +52,7 @@ public class Example {
 In order to build, you need to have the JDK 8 or higher installed. Run the tests with:
 
 ```bash
-git clone https://github.com/ggerganov/whisper.cpp.git
+git clone https://github.com/ggml-org/whisper.cpp.git
 cd whisper.cpp/bindings/java
 
 ./gradlew build

bindings/java/build.gradle

@@ -25,13 +25,13 @@ sourceSets {
 }
 
 tasks.register('copyLibwhisperDynlib', Copy) {
-    from '../../build'
+    from '../../build/src'
-    include 'libwhisper.dynlib'
+    include 'libwhisper.dylib'
     into 'build/generated/resources/main/darwin'
 }
 
 tasks.register('copyLibwhisperSo', Copy) {
-    from '../../build'
+    from '../../build/src'
     include 'libwhisper.so'
     into 'build/generated/resources/main/linux-x86-64'
 }
@@ -55,7 +55,12 @@ java {
     withJavadocJar()
 }
 
+sourcesJar() {
+    dependsOn copyLibs
+}
+
 jar {
+    dependsOn copyLibs
     exclude '**/whisper_java.exp', '**/whisper_java.lib'
 }
 
@@ -67,6 +72,9 @@ tasks.withType(Test) {
     useJUnitPlatform()
 }
 
+test.dependsOn copyLibs
+processResources.dependsOn copyLibs
+
 dependencies {
     implementation "net.java.dev.jna:jna:5.13.0"
     testImplementation "org.junit.jupiter:junit-jupiter:5.9.2"

bindings/java/src/main/java/io/github/ggerganov/whispercpp/WhisperConstants.java

@@ -0,0 +1,24 @@
+package io.github.ggerganov.whispercpp;
+
+/**
+ * Presets for alignment heads in DTW token timestamps
+ */
+public class WhisperConstants {
+    // Alignment heads presets
+    public static final int WHISPER_AHEADS_NONE = 0;
+    public static final int WHISPER_AHEADS_TINY_EN = 1;
+    public static final int WHISPER_AHEADS_TINY = 2;
+    public static final int WHISPER_AHEADS_BASE_EN = 3;
+    public static final int WHISPER_AHEADS_BASE = 4;
+    public static final int WHISPER_AHEADS_SMALL_EN = 5;
+    public static final int WHISPER_AHEADS_SMALL = 6;
+    public static final int WHISPER_AHEADS_MEDIUM_EN = 7;
+    public static final int WHISPER_AHEADS_MEDIUM = 8;
+    public static final int WHISPER_AHEADS_LARGE_V1 = 9;
+    public static final int WHISPER_AHEADS_LARGE_V2 = 10;
+    public static final int WHISPER_AHEADS_LARGE_V3 = 11;
+    public static final int WHISPER_AHEADS_LARGE_V3_TURBO = 12;
+    public static final int WHISPER_AHEADS_CUSTOM = 13;
+    public static final int WHISPER_AHEADS_N_TOP_MOST = 14;
+    public static final int WHISPER_AHEADS_COUNT = 15;
+}

bindings/java/src/main/java/io/github/ggerganov/whispercpp/WhisperContext.java

@@ -1,7 +1,9 @@
 package io.github.ggerganov.whispercpp;
 
+import com.sun.jna.NativeLong;
 import com.sun.jna.Structure;
 import com.sun.jna.ptr.PointerByReference;
+import com.sun.jna.Pointer;
 import io.github.ggerganov.whispercpp.ggml.GgmlType;
 import io.github.ggerganov.whispercpp.WhisperModel;
 import io.github.ggerganov.whispercpp.params.WhisperContextParams;
@@ -9,33 +11,26 @@ import io.github.ggerganov.whispercpp.params.WhisperContextParams;
 import java.util.List;
 
 public class WhisperContext extends Structure {
-    int t_load_us = 0;
+    public NativeLong t_load_us;
-    int t_start_us = 0;
+    public NativeLong t_start_us;
 
     /** weight type (FP32 / FP16 / QX) */
-    GgmlType wtype = GgmlType.GGML_TYPE_F16;
+    public GgmlType wtype = GgmlType.GGML_TYPE_F16;
     /** intermediate type (FP32 or FP16) */
-    GgmlType itype = GgmlType.GGML_TYPE_F16;
+    public GgmlType itype = GgmlType.GGML_TYPE_F16;
 
-//    WhisperModel model;
+    public WhisperContextParams.ByValue params;
-    public PointerByReference model;
+
-//    whisper_vocab vocab;
+    public Pointer model;
-//    whisper_state * state = nullptr;
+    public Pointer vocab;
-    public PointerByReference vocab;
+    public Pointer state;
-    public PointerByReference state;
 
     /** populated by whisper_init_from_file_with_params() */
-    String path_model;
+    public Pointer path_model;
-    WhisperContextParams params;
 
-//    public static class ByReference extends WhisperContext implements Structure.ByReference {
+    @Override
-//    }
+    protected List<String> getFieldOrder() {
-//
+        return List.of("t_load_us", "t_start_us", "wtype", "itype",
-//    public static class ByValue extends WhisperContext implements Structure.ByValue {
+                "params", "model", "vocab", "state", "path_model");
-//    }
+    }
-//
-//    @Override
-//    protected List<String> getFieldOrder() {
-//        return List.of("t_load_us", "t_start_us", "wtype", "itype", "model", "vocab", "state", "path_model");
-//    }
 }

bindings/java/src/main/java/io/github/ggerganov/whispercpp/WhisperCpp.java

@@ -43,11 +43,11 @@ public class WhisperCpp implements AutoCloseable {
      * @param modelPath - absolute path, or just the name (eg: "base", "base-en" or "base.en")
      * @param params - params to use when initialising the context
      */
-    public void initContext(String modelPath, WhisperContextParams params) throws FileNotFoundException {
+    public void initContext(String modelPath, WhisperContextParams.ByValue params) throws FileNotFoundException {
         initContextImpl(modelPath, params);
     }
 
-    private void initContextImpl(String modelPath, WhisperContextParams params) throws FileNotFoundException {
+    private void initContextImpl(String modelPath, WhisperContextParams.ByValue params) throws FileNotFoundException {
         if (ctx != null) {
             lib.whisper_free(ctx);
         }
@@ -69,15 +69,13 @@ public class WhisperCpp implements AutoCloseable {
 
     /**
      * Provides default params which can be used with `whisper_init_from_file_with_params()` etc.
-     * Because this function allocates memory for the params, the caller must call either:
+     * Returns a ByValue instance to ensure proper parameter passing to native code.
-     * - call `whisper_free_context_params()`
-     * - `Native.free(Pointer.nativeValue(pointer));`
      */
-    public WhisperContextParams getContextDefaultParams() {
+    public WhisperContextParams.ByValue getContextDefaultParams() {
-        paramsPointer = lib.whisper_context_default_params_by_ref();
+        WhisperContextParams.ByValue valueParams = new WhisperContextParams.ByValue(
-        WhisperContextParams params = new WhisperContextParams(paramsPointer);
+            lib.whisper_context_default_params_by_ref());
-        params.read();
+        valueParams.read();
-        return params;
+        return valueParams;
     }
     
     /**
@@ -88,7 +86,7 @@ public class WhisperCpp implements AutoCloseable {
      *
      * @param strategy - GREEDY
      */
-    public WhisperFullParams getFullDefaultParams(WhisperSamplingStrategy strategy) {
+    public WhisperFullParams.ByValue getFullDefaultParams(WhisperSamplingStrategy strategy) {
         Pointer pointer;
 
         // whisper_full_default_params_by_ref allocates memory which we need to delete, so only create max 1 pointer for each strategy.
@@ -104,7 +102,7 @@ public class WhisperCpp implements AutoCloseable {
             pointer = beamParamsPointer;
         }
 
-        WhisperFullParams params = new WhisperFullParams(pointer);
+        WhisperFullParams.ByValue params = new WhisperFullParams.ByValue(pointer);
         params.read();
         return params;
     }
@@ -138,15 +136,21 @@ public class WhisperCpp implements AutoCloseable {
     }
 
     /**
-     * Run the entire model: PCM -> log mel spectrogram -> encoder -> decoder -> text.
+     * Run the entire model: PCM -> log mel spectrogram -> encoder -> decoder -> text.
      * Not thread safe for same context
      * Uses the specified decoding strategy to obtain the text.
      */
-    public String fullTranscribe(WhisperFullParams whisperParams, float[] audioData) throws IOException {
+    public String fullTranscribe(WhisperFullParams.ByValue whisperParams, float[] audioData) throws IOException {
         if (ctx == null) {
             throw new IllegalStateException("Model not initialised");
         }
 
+        /*
+        WhisperFullParams.ByValue valueParams = new WhisperFullParams.ByValue(
+            lib.whisper_full_default_params_by_ref(WhisperSamplingStrategy.WHISPER_SAMPLING_BEAM_SEARCH.ordinal()));
+        valueParams.read();
+        */
+
         if (lib.whisper_full(ctx, whisperParams, audioData, audioData.length) != 0) {
             throw new IOException("Failed to process audio");
         }
@@ -163,12 +167,17 @@ public class WhisperCpp implements AutoCloseable {
 
         return str.toString().trim();
     }
+
     public List<WhisperSegment> fullTranscribeWithTime(WhisperFullParams whisperParams, float[] audioData) throws IOException {
         if (ctx == null) {
             throw new IllegalStateException("Model not initialised");
         }
 
-        if (lib.whisper_full(ctx, whisperParams, audioData, audioData.length) != 0) {
+        WhisperFullParams.ByValue valueParams = new WhisperFullParams.ByValue(
+            lib.whisper_full_default_params_by_ref(WhisperSamplingStrategy.WHISPER_SAMPLING_BEAM_SEARCH.ordinal()));
+        valueParams.read();
+
+        if (lib.whisper_full(ctx, valueParams, audioData, audioData.length) != 0) {
             throw new IOException("Failed to process audio");
         }
 

bindings/java/src/main/java/io/github/ggerganov/whispercpp/WhisperCppJnaLibrary.java

@@ -38,7 +38,7 @@ public interface WhisperCppJnaLibrary extends Library {
      * @param params     Pointer to whisper_context_params
      * @return Whisper context on success, null on failure
      */
-    Pointer whisper_init_from_file_with_params(String path_model, WhisperContextParams params);
+    Pointer whisper_init_from_file_with_params(String path_model, WhisperContextParams.ByValue params);
 
     /**
      * Allocate (almost) all memory needed for the model by loading from a buffer.
@@ -180,12 +180,12 @@ public interface WhisperCppJnaLibrary extends Library {
     /**
      * @return the id of the specified language, returns -1 if not found.
      * Examples:
-     *   "de" -> 2
+     *   "de" -> 2
-     *   "german" -> 2
+     *   "german" -> 2
      */
     int whisper_lang_id(String lang);
 
-    /** @return the short string of the specified language id (e.g. 2 -> "de"), returns nullptr if not found */
+    /** @return the short string of the specified language id (e.g. 2 -> "de"), returns nullptr if not found */
     String whisper_lang_str(int id);
 
     /**
@@ -268,20 +268,21 @@ public interface WhisperCppJnaLibrary extends Library {
     void whisper_free_params(Pointer params);
 
     /**
-     * Run the entire model: PCM -> log mel spectrogram -> encoder -> decoder -> text
+     * Run the entire model: PCM -> log mel spectrogram -> encoder -> decoder -> text
      * Not thread safe for same context
      * Uses the specified decoding strategy to obtain the text.
      */
-    int whisper_full(Pointer ctx, WhisperFullParams params, final float[] samples, int n_samples);
+    int whisper_full(Pointer ctx, WhisperFullParams.ByValue params, final float[] samples, int n_samples);
 
-    int whisper_full_with_state(Pointer ctx, Pointer state, WhisperFullParams params, final float[] samples, int n_samples);
+    public int whisper_full_with_state(Pointer ctx, Pointer state, WhisperFullParams.ByValue params, float[] samples, int n_samples);
+    //int whisper_full_with_state(Pointer ctx, Pointer state, WhisperFullParams params, final float[] samples, int n_samples);
 
     // Split the input audio in chunks and process each chunk separately using whisper_full_with_state()
     // Result is stored in the default state of the context
     // Not thread safe if executed in parallel on the same context.
     // It seems this approach can offer some speedup in some cases.
     // However, the transcription accuracy can be worse at the beginning and end of each chunk.
-    int whisper_full_parallel(Pointer ctx, WhisperFullParams params, final float[] samples, int n_samples, int n_processors);
+    int whisper_full_parallel(Pointer ctx, WhisperFullParams.ByValue params, final float[] samples, int n_samples, int n_processors);
 
     /**
      * Number of generated text segments.

bindings/java/src/main/java/io/github/ggerganov/whispercpp/callbacks/GgmlAbortCallback.java

@@ -0,0 +1,17 @@
+package io.github.ggerganov.whispercpp.callbacks;
+
+import com.sun.jna.Callback;
+
+/**
+ * Callback for aborting GGML computation
+ * Maps to the C typedef: bool (*ggml_abort_callback)(void * data)
+ */
+public interface GgmlAbortCallback extends Callback {
+    /**
+     * Return true to abort the computation, false to continue
+     *
+     * @param data User data passed to the callback
+     * @return true to abort, false to continue
+     */
+    boolean invoke(com.sun.jna.Pointer data);
+}

bindings/java/src/main/java/io/github/ggerganov/whispercpp/params/WhisperAhead.java

@@ -0,0 +1,30 @@
+package io.github.ggerganov.whispercpp.params;
+import com.sun.jna.*;
+import java.util.Arrays;
+import java.util.List;
+
+public class WhisperAhead extends Structure {
+
+    public int n_text_layer;
+
+    public int n_head;
+
+    public WhisperAhead() {
+        super();
+    }
+
+    public WhisperAhead(int textLayer, int head) {
+        super();
+        this.n_text_layer = textLayer;
+        this.n_head = head;
+    }
+
+    @Override
+    protected List<String> getFieldOrder() {
+        return Arrays.asList("n_text_layer", "n_head");
+    }
+
+    public static class ByReference extends WhisperAhead implements Structure.ByReference {}
+
+    public static class ByValue extends WhisperAhead implements Structure.ByValue {}
+}

bindings/java/src/main/java/io/github/ggerganov/whispercpp/params/WhisperAheads.java

@@ -0,0 +1,41 @@
+package io.github.ggerganov.whispercpp.params;
+import com.sun.jna.*;
+import java.util.Arrays;
+import java.util.List;
+
+public class WhisperAheads extends Structure {
+    public NativeLong n_heads;
+
+    public Pointer heads;
+
+    public WhisperAheads() {
+        super();
+    }
+
+    /**
+     * Create alignment heads from an array of WhisperAhead objects
+     */
+    public void setHeads(WhisperAhead[] aheadsArray) {
+        this.n_heads = new NativeLong(aheadsArray.length);
+
+        int structSize = aheadsArray[0].size();
+        Memory mem = new Memory(structSize * aheadsArray.length);
+
+        for (int i = 0; i < aheadsArray.length; i++) {
+            aheadsArray[i].write();
+            byte[] buffer = aheadsArray[i].getPointer().getByteArray(0, structSize);
+            mem.write(i * structSize, buffer, 0, buffer.length);
+        }
+
+        this.heads = mem;
+    }
+
+    @Override
+    protected List<String> getFieldOrder() {
+        return Arrays.asList("n_heads", "heads");
+    }
+
+    public static class ByReference extends WhisperAheads implements Structure.ByReference {}
+
+    public static class ByValue extends WhisperAheads implements Structure.ByValue {}
+}

bindings/java/src/main/java/io/github/ggerganov/whispercpp/params/WhisperContextParams.java

@@ -1,7 +1,5 @@
 package io.github.ggerganov.whispercpp.params;
-
 import com.sun.jna.*;
-
 import java.util.Arrays;
 import java.util.List;
 
@@ -11,21 +9,73 @@ import java.util.List;
  * whisper_context_default_params()
  */
 public class WhisperContextParams extends Structure {
-
     public WhisperContextParams(Pointer p) {
         super(p);
     }
 
-    /** Use GPU for inference Number (default = true) */
+    public WhisperContextParams() {
+        super();
+    }
+
+    /** Use GPU for inference (default = true) */
     public CBool use_gpu;
 
-    /** Use GPU for inference Number (default = true) */
+    /** Use flash attention (default = false) */
+    public CBool flash_attn;
+
+    /** CUDA device to use (default = 0) */
+    public int gpu_device;
+
+    /** [EXPERIMENTAL] Enable token-level timestamps with DTW (default = false) */
+    public CBool dtw_token_timestamps;
+
+    /** [EXPERIMENTAL] Alignment heads preset for DTW */
+    public int dtw_aheads_preset;
+
+    /** Number of top layers to use for DTW when using WHISPER_AHEADS_N_TOP_MOST preset */
+    public int dtw_n_top;
+
+    public WhisperAheads.ByValue dtw_aheads;
+
+    /** DTW memory size (internal use) */
+    public NativeLong dtw_mem_size;
+
+    /** Use GPU for inference */
     public void useGpu(boolean enable) {
         use_gpu = enable ? CBool.TRUE : CBool.FALSE;
     }
 
+    /** Use flash attention */
+    public void useFlashAttn(boolean enable) {
+        flash_attn = enable ? CBool.TRUE : CBool.FALSE;
+    }
+
+    /** Enable DTW token-level timestamps */
+    public void enableDtwTokenTimestamps(boolean enable) {
+        dtw_token_timestamps = enable ? CBool.TRUE : CBool.FALSE;
+    }
+
+    /** Set DTW alignment heads preset */
+    public void setDtwAheadsPreset(int preset) {
+        dtw_aheads_preset = preset;
+    }
+
     @Override
     protected List<String> getFieldOrder() {
-        return Arrays.asList("use_gpu");
+        return Arrays.asList(
+            "use_gpu",
+            "flash_attn",
+            "gpu_device",
+            "dtw_token_timestamps",
+            "dtw_aheads_preset",
+            "dtw_n_top",
+            "dtw_aheads",
+            "dtw_mem_size"
+        );
+    }
+
+    public static class ByValue extends WhisperContextParams implements Structure.ByValue {
+        public ByValue() { super(); }
+        public ByValue(Pointer p) { super(p); }
     }
 }

bindings/java/src/main/java/io/github/ggerganov/whispercpp/params/WhisperFullParams.java

@@ -5,6 +5,7 @@ import io.github.ggerganov.whispercpp.callbacks.WhisperEncoderBeginCallback;
 import io.github.ggerganov.whispercpp.callbacks.WhisperLogitsFilterCallback;
 import io.github.ggerganov.whispercpp.callbacks.WhisperNewSegmentCallback;
 import io.github.ggerganov.whispercpp.callbacks.WhisperProgressCallback;
+import io.github.ggerganov.whispercpp.callbacks.GgmlAbortCallback;
 
 import java.util.Arrays;
 import java.util.List;
@@ -16,10 +17,12 @@ import java.util.List;
  */
 public class WhisperFullParams extends Structure {
 
+    public WhisperFullParams() {
+        super();
+    }
+
     public WhisperFullParams(Pointer p) {
         super(p);
-//        super(p, ALIGN_MSVC);
-//        super(p, ALIGN_GNUC);
     }
 
     /** Sampling strategy for whisper_full() function. */
@@ -69,10 +72,10 @@ public class WhisperFullParams extends Structure {
         single_segment = single ? CBool.TRUE : CBool.FALSE;
     }
 
-    /** Flag to print special tokens (e.g., <SOT>, <EOT>, <BEG>, etc.). (default = false) */
+    /** Flag to print special tokens (e.g., <SOT>, <EOT>, <BEG>, etc.). (default = false) */
     public CBool print_special;
 
-    /** Flag to print special tokens (e.g., <SOT>, <EOT>, <BEG>, etc.). (default = false) */
+    /** Flag to print special tokens (e.g., <SOT>, <EOT>, <BEG>, etc.). (default = false) */
     public void printSpecial(boolean enable) {
         print_special = enable ? CBool.TRUE : CBool.FALSE;
     }
@@ -129,6 +132,14 @@ public class WhisperFullParams extends Structure {
     /** Maximum tokens per segment (0, default = no limit) */
     public int max_tokens;
 
+    /** [EXPERIMENTAL] Enable debug mode for extra info */
+    public CBool debug_mode;
+
+    /** Enable debug mode */
+    public void enableDebugMode(boolean enable) {
+        debug_mode = enable ? CBool.TRUE : CBool.FALSE;
+    }
+
     /** Overwrite the audio context size (0 = use default). */
     public int audio_ctx;
 
@@ -181,11 +192,11 @@ public class WhisperFullParams extends Structure {
     }
 
     /** Flag to suppress non-speech tokens. */
-    public CBool suppress_non_speech_tokens;
+    public CBool suppress_nst;
 
     /** Flag to suppress non-speech tokens. */
     public void suppressNonSpeechTokens(boolean enable) {
-        suppress_non_speech_tokens = enable ? CBool.TRUE : CBool.FALSE;
+        suppress_nst = enable ? CBool.TRUE : CBool.FALSE;
     }
 
     /** Initial decoding temperature. */
@@ -274,6 +285,16 @@ public class WhisperFullParams extends Structure {
      */
     public Pointer encoder_begin_callback_user_data;
 
+    /** Callback used to abort GGML computation */
+    public Pointer abort_callback;
+
+    /** User data for the abort_callback */
+    public Pointer abort_callback_user_data;
+
+    public void setAbortCallback(GgmlAbortCallback callback) {
+        abort_callback = CallbackReference.getFunctionPointer(callback);
+    }
+
     /**
      * Callback by each decoder to filter obtained logits.
      * WhisperLogitsFilterCallback
@@ -310,17 +331,28 @@ public class WhisperFullParams extends Structure {
 
     @Override
     protected List<String> getFieldOrder() {
-        return Arrays.asList("strategy", "n_threads", "n_max_text_ctx", "offset_ms", "duration_ms", "translate",
+        return Arrays.asList("strategy", "n_threads", "n_max_text_ctx",
-                "no_context", "single_segment", "no_timestamps",
+                "offset_ms", "duration_ms", "translate", "no_context",
-                "print_special", "print_progress", "print_realtime", "print_timestamps",  "token_timestamps",
+                "no_timestamps", "single_segment", "print_special",
-                "thold_pt", "thold_ptsum", "max_len", "split_on_word", "max_tokens", "audio_ctx",
+                "print_progress", "print_realtime", "print_timestamps",
-                "tdrz_enable", "suppress_regex", "initial_prompt", "prompt_tokens", "prompt_n_tokens", "language", "detect_language",
+                "token_timestamps", "thold_pt", "thold_ptsum", "max_len",
-                "suppress_blank", "suppress_non_speech_tokens", "temperature", "max_initial_ts", "length_penalty",
+                "split_on_word", "max_tokens", "debug_mode", "audio_ctx", 
-                "temperature_inc", "entropy_thold", "logprob_thold", "no_speech_thold", "greedy", "beam_search",
+                "tdrz_enable", "suppress_regex", "initial_prompt",
-                "new_segment_callback", "new_segment_callback_user_data",
+                "prompt_tokens", "prompt_n_tokens", "language", "detect_language",
+                "suppress_blank", "suppress_nst", "temperature",
+                "max_initial_ts", "length_penalty", "temperature_inc",
+                "entropy_thold", "logprob_thold", "no_speech_thold", "greedy",
+                "beam_search", "new_segment_callback", "new_segment_callback_user_data",
                 "progress_callback", "progress_callback_user_data",
                 "encoder_begin_callback", "encoder_begin_callback_user_data",
+                "abort_callback", "abort_callback_user_data",
                 "logits_filter_callback", "logits_filter_callback_user_data",
                 "grammar_rules", "n_grammar_rules", "i_start_rule", "grammar_penalty");
     }
+
+    public static class ByValue extends WhisperFullParams implements Structure.ByValue {
+        public ByValue() { super(); }
+        public ByValue(Pointer p) { super(p); }
+    }
+
 }

bindings/java/src/test/java/io/github/ggerganov/whispercpp/WhisperCppTest.java

@@ -76,7 +76,7 @@ class WhisperCppTest {
         float[] floats = new float[b.length / 2];
 
         //WhisperFullParams params = whisper.getFullDefaultParams(WhisperSamplingStrategy.WHISPER_SAMPLING_GREEDY);
-        WhisperFullParams params = whisper.getFullDefaultParams(WhisperSamplingStrategy.WHISPER_SAMPLING_BEAM_SEARCH);
+        WhisperFullParams.ByValue params = whisper.getFullDefaultParams(WhisperSamplingStrategy.WHISPER_SAMPLING_BEAM_SEARCH);
         params.setProgressCallback((ctx, state, progress, user_data) -> System.out.println("progress: " + progress));
         params.print_progress = CBool.FALSE;
         //params.initial_prompt = "and so my fellow Americans um, like";

bindings/javascript/README.md

@@ -33,6 +33,9 @@ mkdir build-em && cd build-em
 emcmake cmake .. && make -j
 
 # run test
+node ../tests/test-whisper.js
+
+# For Node.js versions prior to v16.4.0, experimental features need to be enabled:
 node --experimental-wasm-threads --experimental-wasm-simd ../tests/test-whisper.js
 
 # publish npm package

bindings/javascript/package.json

@@ -1,6 +1,6 @@
 {
   "name": "whisper.cpp",
-  "version": "1.7.3",
+  "version": "1.7.5",
   "description": "Whisper speech recognition",
   "main": "whisper.js",
   "scripts": {

bindings/ruby/.gitignore

@@ -1,5 +1,6 @@
 LICENSE
 pkg/
-lib/whisper.so
+lib/whisper.*
-lib/whisper.bundle
+ext/sources/*
-lib/whisper.dll
+!ext/sources/CMakeGraphVizOptions.cmake
+ext/mkmf.log

bindings/ruby/README.md

@@ -16,6 +16,18 @@ If bundler is not being used to manage dependencies, install the gem by executin
 
     $ gem install whispercpp
 
+You can pass build options for whisper.cpp, for instance:
+
+    $ bundle config build.whispercpp --enable-ggml-cuda
+
+or,
+
+    $ gem install whispercpp -- --enable-ggml-cuda
+
+See whisper.cpp's [README](https://github.com/ggml-org/whisper.cpp/blob/master/README.md) for available options. You need convert options present the README to Ruby-style options.  
+For boolean options like `GGML_CUDA`, the README says `-DGGML_CUDA=1`. You need strip `-D`, prepend `--enable-` for `1` or `ON` (`--disable-` for `0` or `OFF`) and make it kebab-case: `--enable-ggml-cuda`.  
+For options which require arguments like `CMAKE_CUDA_ARCHITECTURES`, the README says `-DCMAKE_CUDA_ARCHITECTURES="86"`. You need strip `-D`, prepend `--`, make it kebab-case, append `=` and append argument: `--cmake-cuda-architectures="86"`.
+
 Usage
 -----
 
@@ -24,14 +36,15 @@ require "whisper"
 
 whisper = Whisper::Context.new("base")
 
-params = Whisper::Params.new
+params = Whisper::Params.new(
-params.language = "en"
+  language: "en",
-params.offset = 10_000
+  offset: 10_000,
-params.duration = 60_000
+  duration: 60_000,
-params.max_text_tokens = 300
+  max_text_tokens: 300,
-params.translate = true
+  translate: true,
-params.print_timestamps = false
+  print_timestamps: false,
-params.initial_prompt = "Initial prompt here."
+  initial_prompt: "Initial prompt here."
+)
 
 whisper.transcribe("path/to/audio.wav", params) do |whole_text|
   puts whole_text
@@ -60,10 +73,10 @@ You also can use shorthand for pre-converted models:
 whisper = Whisper::Context.new("base.en")
 ```
 
-You can see the list of prepared model names by `Whisper::Model.preconverted_models.keys`:
+You can see the list of prepared model names by `Whisper::Model.pre_converted_models.keys`:
 
 ```ruby
-puts Whisper::Model.preconverted_model_names
+puts Whisper::Model.pre_converted_models.keys
 # tiny
 # tiny.en
 # tiny-q5_1
@@ -87,8 +100,9 @@ whisper = Whisper::Context.new("path/to/your/model.bin")
 Or, you can download model files:
 
 ```ruby
-model_uri = Whisper::Model::URI.new("http://example.net/uri/of/your/model.bin")
+whisper = Whisper::Context.new("https://example.net/uri/of/your/model.bin")
-whisper = Whisper::Context.new(model_uri)
+# Or
+whisper = Whisper::Context.new(URI("https://example.net/uri/of/your/model.bin"))
 ```
 
 See [models][] page for details.
@@ -112,18 +126,18 @@ def format_time(time_ms)
   "%02d:%02d:%02d.%03d" % [hour, min, sec, decimal_part]
 end
 
-whisper.transcribe("path/to/audio.wav", params)
+whisper
-
+  .transcribe("path/to/audio.wav", params)
-whisper.each_segment.with_index do |segment, index|
+  .each_segment.with_index do |segment, index|
-  line = "[%{nth}: %{st} --> %{ed}] %{text}" % {
+    line = "[%{nth}: %{st} --> %{ed}] %{text}" % {
-    nth: index + 1,
+      nth: index + 1,
-    st: format_time(segment.start_time),
+      st: format_time(segment.start_time),
-    ed: format_time(segment.end_time),
+      ed: format_time(segment.end_time),
-    text: segment.text
+      text: segment.text
-  }
+    }
-  line << " (speaker turned)" if segment.speaker_next_turn?
+    line << " (speaker turned)" if segment.speaker_next_turn?
-  puts line
+    puts line
-end
+  end
 
 ```
 
@@ -214,18 +228,30 @@ reader = WaveFile::Reader.new("path/to/audio.wav", WaveFile::Format.new(:mono, :
 samples = reader.enum_for(:each_buffer).map(&:samples).flatten
 
 whisper = Whisper::Context.new("base")
-whisper.full(Whisper::Params.new, samples)
+whisper
-whisper.each_segment do |segment|
+  .full(Whisper::Params.new, samples)
-  puts segment.text
+  .each_segment do |segment|
-end
+    puts segment.text
+  end
 ```
 
-The second argument `samples` may be an array, an object with `length` method, or a MemoryView. If you can prepare audio data as C array and export it as a MemoryView, whispercpp accepts and works with it with zero copy.
+The second argument `samples` may be an array, an object with `length` and `each` method, or a MemoryView. If you can prepare audio data as C array and export it as a MemoryView, whispercpp accepts and works with it with zero copy.
+
+Development
+-----------
+
+    % git clone https://github.com/ggml-org/whisper.cpp.git
+    % cd whisper.cpp/bindings/ruby
+    % rake test
+
+First call of `rake test` builds an extension and downloads a model for testing. After that, you add tests in `tests` directory and modify `ext/ruby_whisper.cpp`.
+
+If something seems wrong on build, running `rake clean` solves some cases.
 
 License
 -------
 
 The same to [whisper.cpp][].
 
-[whisper.cpp]: https://github.com/ggerganov/whisper.cpp
+[whisper.cpp]: https://github.com/ggml-org/whisper.cpp
-[models]: https://github.com/ggerganov/whisper.cpp/tree/master/models
+[models]: https://github.com/ggml-org/whisper.cpp/tree/master/models

bindings/ruby/Rakefile

@@ -3,11 +3,15 @@ require "bundler/gem_tasks"
 require "rake/testtask"
 require_relative "extsources"
 
+SOURCES_DIR = "ext/sources"
+
 SOURCES = FileList[]
 
 EXTSOURCES.each do |src|
   basename = src.pathmap("%f")
-  dest = basename == "LICENSE" ? basename : src.pathmap("%{../..,ext}p")
+  dest = basename == "LICENSE" ? basename
+                               : src.pathmap("%{\\.\\./\\.\\.,#{SOURCES_DIR}}p")
+                                    .pathmap("%{\\.\\./javascript,#{SOURCES_DIR}/bindings/javascript}p")
   dir = dest.pathmap("%d")
   file src
   directory dir
@@ -18,9 +22,10 @@ EXTSOURCES.each do |src|
 end
 
 CLEAN.include SOURCES
-CLEAN.include FileList["ext/*.o", "ext/*.metal", "ext/whisper.{so,bundle,dll}"]
 
-task build: ["ext/Makefile", "ext/ruby_whisper.h", "ext/ruby_whisper.cpp", "whispercpp.gemspec"]
+SRC = FileList["ext/*.{c,cpp,h}"]
+
+task build: SOURCES
 
 directory "pkg"
 CLOBBER.include "pkg"
@@ -29,14 +34,28 @@ LIB_NAME = "whisper".ext(RbConfig::CONFIG["DLEXT"])
 SO_FILE = File.join("ext", LIB_NAME)
 LIB_FILE = File.join("lib", LIB_NAME)
 
-file "ext/Makefile" => ["ext/extconf.rb", "ext/ruby_whisper.h", "ext/ruby_whisper.cpp"] + SOURCES do |t|
+file "ext/Makefile" => SRC + ["ext/extconf.rb"] + SOURCES do |t|
-  Dir.chdir "ext" do
+  chdir "ext" do
     ruby "extconf.rb"
   end
 end
+if File.exist? "ext/Makefile"
+  task :make_clean do
+    cd "ext" do
+      sh "make", "clean"
+    end
+  end
+  task clean: :make_clean
+  task :make_distclean do
+    cd "ext" do
+      sh "make", "distclean"
+    end
+  end
+  task clobber: :make_distclean
+end
 
 file SO_FILE => "ext/Makefile" do |t|
-  Dir.chdir "ext" do
+  chdir "ext" do
     sh "make"
   end
 end
@@ -54,7 +73,7 @@ end
 
 TEST_MEMORY_VIEW = "tests/jfk_reader/jfk_reader.#{RbConfig::CONFIG['DLEXT']}"
 file TEST_MEMORY_VIEW => "tests/jfk_reader/jfk_reader.c" do |t|
-  Dir.chdir "tests/jfk_reader" do
+  chdir "tests/jfk_reader" do
     ruby "extconf.rb"
     sh "make"
   end

bindings/ruby/ext/.gitignore

@@ -4,10 +4,8 @@ whisper.bundle
 whisper.dll
 scripts/get-flags.mk
 *.o
-*.c
+/*/**/*.c
-*.cpp
+/*/**/*.cpp
-*.h
+/*/**/*.h
-*.m
+/*/**/*.m
-*.metal
+/*/**/*.metal
-!ruby_whisper.cpp
-!ruby_whisper.h

bindings/ruby/ext/cpu.mk

@@ -1,9 +0,0 @@
-ggml/src/ggml-cpu/ggml-cpu-cpp.o: \
-	ggml/src/ggml-cpu/ggml-cpu.cpp \
-	ggml/include/ggml-backend.h \
-	ggml/include/ggml.h \
-	ggml/include/ggml-alloc.h \
-	ggml/src/ggml-backend-impl.h \
-	ggml/include/ggml-cpu.h \
-	ggml/src/ggml-impl.h
-	$(CXX) $(CXXFLAGS)   -c $< -o $@

bindings/ruby/ext/dependencies.rb

@@ -0,0 +1,61 @@
+require "tsort"
+
+class Dependencies
+  def initialize(cmake, options)
+    @cmake = cmake
+    @options = options
+
+    generate_dot
+    @libs = parse_dot
+  end
+
+  def to_s
+    @libs.join(" ")
+  end
+
+  private
+
+  def dot_path
+    File.join(__dir__, "build", "whisper.cpp.dot")
+  end
+
+  def generate_dot
+    system @cmake, "-S", "sources", "-B", "build", "--graphviz", dot_path, "-D", "BUILD_SHARED_LIBS=OFF", @options.to_s, exception: true
+  end
+
+  def parse_dot
+    static_lib_shape = nil
+    nodes = {}
+    depends = Hash.new {|h, k| h[k] = []}
+
+    class << depends
+      include TSort
+      alias tsort_each_node each_key
+      def tsort_each_child(node, &block)
+        fetch(node, []).each(&block)
+      end
+    end
+
+    File.open(dot_path).each_line do |line|
+      case line
+      when /\[\s*label\s*=\s*"Static Library"\s*,\s*shape\s*=\s*(?<shape>\w+)\s*\]/
+        static_lib_shape = $~[:shape]
+      when /\A\s*"(?<node>\w+)"\s*\[\s*label\s*=\s*"(?<label>\S+)"\s*,\s*shape\s*=\s*(?<shape>\w+)\s*\]\s*;\s*\z/
+        node = $~[:node]
+        label = $~[:label]
+        shape = $~[:shape]
+        nodes[node] = [label, shape]
+      when /\A\s*"(?<depender>\w+)"\s*->\s*"(?<dependee>\w+)"/
+        depender = $~[:depender]
+        dependee = $~[:dependee]
+        depends[depender] ||= []
+        depends[depender] << dependee
+      end
+    end
+    depends.tsort.filter_map {|node|
+      label, shape = nodes[node]
+      shape == static_lib_shape ? label : nil
+    }.collect {|lib| "lib#{lib}.a"}
+      .reverse
+  end
+end

bindings/ruby/ext/extconf.rb

@@ -1,199 +1,22 @@
-require 'mkmf'
+require "mkmf"
+require_relative "options"
+require_relative "dependencies"
 
-# need to use c++ compiler flags
+cmake = find_executable("cmake") || abort
-$CXXFLAGS << ' -std=c++17'
+options = Options.new
+have_library("gomp") rescue nil
+libs = Dependencies.new(cmake, options)
 
-$LDFLAGS << ' -lstdc++'
+$INCFLAGS << " -Isources/include -Isources/ggml/include -Isources/examples"
+$LOCAL_LIBS << " #{libs}"
+$cleanfiles << " build #{libs}"
 
-# Set to true when building binary gems
+create_makefile "whisper" do |conf|
-if enable_config('static-stdlib', false)
+  conf << <<~EOF
-  $LDFLAGS << ' -static-libgcc -static-libstdc++'
+    $(TARGET_SO): #{libs}
-end
+    #{libs}: cmake-targets
-
+    cmake-targets:
-if enable_config('march-tune-native', false)
+    #{"\t"}#{cmake} -S sources -B build -D BUILD_SHARED_LIBS=OFF -D CMAKE_ARCHIVE_OUTPUT_DIRECTORY=#{__dir__} -D CMAKE_POSITION_INDEPENDENT_CODE=ON #{options}
-  $CFLAGS << ' -march=native -mtune=native'
+    #{"\t"}#{cmake} --build build --config Release --target common whisper
-  $CXXFLAGS << ' -march=native -mtune=native'
+  EOF
-end
-
-if ENV['WHISPER_METAL']
-  $GGML_METAL ||= true
-  $DEPRECATE_WARNING ||= true
-end
-
-$UNAME_S = `uname -s`.chomp
-$UNAME_P = `uname -p`.chomp
-$UNAME_M = `uname -m`.chomp
-
-if $UNAME_S == 'Darwin'
-  unless ENV['GGML_NO_METAL']
-    $GGML_METAL ||= true
-  end
-  $GGML_NO_OPENMP ||= true
-end
-
-if $GGML_METAL
-  $GGML_METAL_EMBED_LIBRARY = true
-end
-
-$MK_CPPFLAGS = '-Iggml/include -Iggml/src -Iggml/src/ggml-cpu -Iinclude -Isrc -Iexamples'
-$MK_CFLAGS   = '-std=c11   -fPIC'
-$MK_CXXFLAGS = '-std=c++17 -fPIC'
-$MK_NVCCFLAGS = '-std=c++17'
-$MK_LDFLAGS = ''
-
-$OBJ_GGML = []
-$OBJ_WHISPER = []
-$OBJ_COMMON = []
-$OBJ_SDL = []
-
-$MK_CPPFLAGS << ' -D_XOPEN_SOURCE=600'
-
-if $UNAME_S == 'Linux'
-  $MK_CPPFLAGS << ' -D_GNU_SOURCE'
-end
-
-if $UNAME_S == 'Darwin'
-  $MK_CPPFLAGS << ' -D_DARWIN_C_SOURCE'
-end
-
-if ENV['WHISPER_DEBUG']
-  $MK_CFLAGS    << ' -O0 -g'
-  $MK_CXXFLAGS  << ' -O0 -g'
-  $MK_LDFLAGS   << ' -g'
-  $MK_NVCCFLAGS << ' -O0 -g'
-else
-  $MK_CPPFLAGS   << ' -DNDEBUG'
-  $MK_CFLAGS     << ' -O3'
-  $MK_CXXFLAGS   << ' -O3'
-  $MK_NVCCFLAGS  << ' -O3'
-end
-
-$WARN_FLAGS =
-  ' -Wall' <<
-  ' -Wextra' <<
-  ' -Wpedantic' <<
-  ' -Wcast-qual' <<
-  ' -Wno-unused-function'
-
-$MK_CFLAGS <<
-  $WARN_FLAGS <<
-  ' -Wshadow' <<
-  ' -Wstrict-prototypes' <<
-  ' -Wpointer-arith' <<
-  ' -Wmissing-prototypes' <<
-  ' -Werror=implicit-int' <<
-  ' -Werror=implicit-function-declaration'
-
-$MK_CXXFLAGS <<
-  $WARN_FLAGS <<
-  ' -Wmissing-declarations' <<
-  ' -Wmissing-noreturn'
-
-unless `#{cc_command} #{$LDFLAGS} -Wl,-v 2>&1`.chomp.include? 'dyld-1015.7'
-  $MK_CPPFLAGS << ' -DHAVE_BUGGY_APPLE_LINKER'
-end
-
-if %w[Linux Darwin FreeBSD NetBSD OpenBSD Haiku].include? $UNAME_S
-  $MK_CFLAGS   << ' -pthread'
-  $MK_CXXFLAGS << ' -pthread'
-end
-
-unless $_WIN32
-  $DSO_EXT = '.so'
-else
-  $DSO_EXT = '.dll'
-end
-
-unless ENV['RISCV']
-  if %w[x86_64 i686 amd64].include? $UNAME_M
-    $HOST_CXXFLAGS ||= ''
-
-    $MK_CFLAGS     << ' -march=native -mtune=native'
-    $HOST_CXXFLAGS << ' -march=native -mtune=native'
-  end
-else
-  $MK_CFLAGS   << ' -march=rv64gcv -mabi=lp64d'
-  $MK_CXXFLAGS << ' -march=rv64gcv -mabi=lp64d'
-end
-
-unless ENV['GGML_NO_ACCELERATE']
-  if $UNAME_S == 'Darwin'
-    $MK_CPPFLAGS << ' -DGGML_USE_ACCELERATE -DGGML_USE_BLAS -DGGML_BLAS_USE_ACCELERATE'
-    $MK_CPPFLAGS << ' -DACCELERATE_NEW_LAPACK'
-    $MK_CPPFLAGS << ' -DACCELERATE_LAPACK_ILP64'
-    $MK_LDFLAGS  << ' -framework Accelerate'
-    $OBJ_GGML    << 'ggml/src/ggml-blas/ggml-blas.o'
-  end
-end
-
-if ENV['GGML_OPENBLAS']
-  $MK_CPPFLAGS << " -DGGML_USE_BLAS #{`pkg-config --cflags-only-I openblas`.chomp}"
-  $MK_CFLAGS   << " #{`pkg-config --cflags-only-other openblas)`.chomp}"
-  $MK_LDFLAGS  << " #{`pkg-config --libs openblas`}"
-  $OBJ_GGML    << 'ggml/src/ggml-blas/ggml-blas.o'
-end
-
-if ENV['GGML_OPENBLAS64']
-  $MK_CPPFLAGS << " -DGGML_USE_BLAS #{`pkg-config --cflags-only-I openblas64`.chomp}"
-  $MK_CFLAGS   << " #{`pkg-config --cflags-only-other openblas64)`.chomp}"
-  $MK_LDFLAGS  << " #{`pkg-config --libs openblas64`}"
-  $OBJ_GGML    << 'ggml/src/ggml-blas/ggml-blas.o'
-end
-
-if $GGML_METAL
-  $MK_CPPFLAGS << ' -DGGML_USE_METAL'
-  $MK_LDFLAGS  << ' -framework Foundation -framework Metal -framework MetalKit'
-  $OBJ_GGML    << 'ggml/src/ggml-metal/ggml-metal.o'
-
-  if ENV['GGML_METAL_NDEBUG']
-    $MK_CPPFLAGS << ' -DGGML_METAL_NDEBUG'
-  end
-
-  if $GGML_METAL_EMBED_LIBRARY
-    $MK_CPPFLAGS << ' -DGGML_METAL_EMBED_LIBRARY'
-    $OBJ_GGML    << 'ggml/src/ggml-metal/ggml-metal-embed.o'
-  end
-end
-
-$OBJ_GGML <<
-  'ggml/src/ggml.o' <<
-  'ggml/src/ggml-alloc.o' <<
-  'ggml/src/ggml-backend.o' <<
-  'ggml/src/ggml-backend-reg.o' <<
-  'ggml/src/ggml-opt.o' <<
-  'ggml/src/ggml-quants.o' <<
-  'ggml/src/ggml-threading.o' <<
-  'ggml/src/ggml-cpu/ggml-cpu.o' <<
-  'ggml/src/ggml-cpu/ggml-cpu-cpp.o' <<
-  'ggml/src/ggml-cpu/ggml-cpu-aarch64.o' <<
-  'ggml/src/ggml-cpu/ggml-cpu-hbm.o' <<
-  'ggml/src/ggml-cpu/ggml-cpu-quants.o' <<
-  'ggml/src/ggml-cpu/ggml-cpu-traits.o'
-
-$OBJ_WHISPER <<
-  'src/whisper.o'
-
-$objs = $OBJ_GGML + $OBJ_WHISPER + $OBJ_COMMON + $OBJ_SDL
-$objs << "ruby_whisper.o"
-
-$CPPFLAGS  = "#{$MK_CPPFLAGS} #{$CPPFLAGS}"
-$CFLAGS    = "#{$CPPFLAGS} #{$MK_CFLAGS} #{$GF_CFLAGS} #{$CFLAGS}"
-$BASE_CXXFLAGS = "#{$MK_CXXFLAGS} #{$CXXFLAGS}"
-$CXXFLAGS  = "#{$BASE_CXXFLAGS} #{$HOST_CXXFLAGS} #{$GF_CXXFLAGS} #{$CPPFLAGS}"
-$NVCCFLAGS = "#{$MK_NVCCFLAGS} #{$NVCCFLAGS}"
-$LDFLAGS   = "#{$MK_LDFLAGS} #{$LDFLAGS}"
-
-create_makefile('whisper')
-
-File.open 'Makefile', 'a' do |file|
-  file.puts 'include scripts/get-flags.mk'
-  file.puts 'include cpu.mk'
-
-  if $GGML_METAL
-    file.puts 'include metal.mk'
-
-    if $GGML_METAL_EMBED_LIBRARY
-      file.puts 'include metal-embed.mk'
-    end
-  end
 end

bindings/ruby/ext/metal-embed.mk

@@ -1,17 +0,0 @@
-ggml/src/ggml-metal/ggml-metal-embed.o: \
-	ggml/src/ggml-metal/ggml-metal.metal \
-	ggml/src/ggml-metal/ggml-metal-impl.h \
-	ggml/src/ggml-common.h
-	@echo "Embedding Metal library"
-	@sed -e '/__embed_ggml-common.h__/r      ggml/src/ggml-common.h'                -e '/__embed_ggml-common.h__/d'      < ggml/src/ggml-metal/ggml-metal.metal           > ggml/src/ggml-metal/ggml-metal-embed.metal.tmp
-	@sed -e '/#include "ggml-metal-impl.h"/r ggml/src/ggml-metal/ggml-metal-impl.h' -e '/#include "ggml-metal-impl.h"/d' < ggml/src/ggml-metal/ggml-metal-embed.metal.tmp > ggml/src/ggml-metal/ggml-metal-embed.metal
-	$(eval TEMP_ASSEMBLY=$(shell mktemp -d))
-	@echo ".section __DATA, __ggml_metallib"                       >  $(TEMP_ASSEMBLY)/ggml-metal-embed.s
-	@echo ".globl _ggml_metallib_start"                            >> $(TEMP_ASSEMBLY)/ggml-metal-embed.s
-	@echo "_ggml_metallib_start:"                                  >> $(TEMP_ASSEMBLY)/ggml-metal-embed.s
-	@echo ".incbin \"ggml/src/ggml-metal/ggml-metal-embed.metal\"" >> $(TEMP_ASSEMBLY)/ggml-metal-embed.s
-	@echo ".globl _ggml_metallib_end"                              >> $(TEMP_ASSEMBLY)/ggml-metal-embed.s
-	@echo "_ggml_metallib_end:"                                    >> $(TEMP_ASSEMBLY)/ggml-metal-embed.s
-	$(CC) $(CFLAGS) -c $(TEMP_ASSEMBLY)/ggml-metal-embed.s -o $@
-	@rm -f ${TEMP_ASSEMBLY}/ggml-metal-embed.s
-	@rmdir ${TEMP_ASSEMBLY}

bindings/ruby/ext/metal.mk

@@ -1,6 +0,0 @@
-ggml/src/ggml-metal/ggml-metal.o: \
-	ggml/src/ggml-metal/ggml-metal.m \
-	ggml/src/ggml-metal/ggml-metal-impl.h \
-	ggml/include/ggml-metal.h \
-	ggml/include/ggml.h
-	$(CC) $(CFLAGS) -c $< -o $@

bindings/ruby/ext/options.rb

@@ -0,0 +1,219 @@
+class Options
+  def initialize
+    @options = {}
+    @pending_options = []
+    @ignored_options = []
+
+    configure
+  end
+
+  def help
+    @options
+      .collect_concat {|name, (type, value)|
+        option = option_name(name)
+        if type == :bool
+          ["--enable-#{option}", "--disable-#{option}"]
+        else
+          "--#{option}=#{type.upcase}"
+        end
+      }
+      .join($/)
+  end
+
+  def to_s
+    @options
+      .reject {|name, (type, value)| value.nil?}
+      .collect {|name, (type, value)| "-D #{name}=#{value == true ? "ON" : value == false ? "OFF" : value.shellescape}"}
+      .join(" ")
+  end
+
+  def cmake_options
+    return @cmake_options if @cmake_options
+
+    output = nil
+    Dir.chdir __dir__ do
+      output = `cmake -S sources -B build -L`
+    end
+    started = false
+    @cmake_options = output.lines.filter_map {|line|
+      if line.chomp == "-- Cache values"
+        started = true
+        next
+      end
+      next unless started
+      option, value = line.chomp.split("=", 2)
+      name, type = option.split(":", 2)
+      [name, type, value]
+    }
+  end
+
+  def missing_options
+    cmake_options.collect {|name, type, value| name} -
+      @options.keys - @pending_options - @ignored_options
+  end
+
+  def extra_options
+    @options.keys + @pending_options - @ignored_options -
+      cmake_options.collect {|name, type, value| name}
+  end
+
+  private
+
+  def configure
+    filepath "ACCELERATE_FRAMEWORK"
+    ignored "BUILD_SHARED_LIBS"
+    ignored "BUILD_TESTING"
+    ignored "CMAKE_BUILD_TYPE"
+    ignored "CMAKE_INSTALL_PREFIX"
+    string "CMAKE_OSX_ARCHITECTURES"
+    ignored "CMAKE_OSX_DEPLOYMENT_TARGET"
+    string "CMAKE_OSX_SYSROOT"
+    filepath "FOUNDATION_LIBRARY"
+    bool "GGML_ACCELERATE"
+    bool "GGML_ALL_WARNINGS_3RD_PARTY"
+    bool "GGML_AMX_BF16"
+    bool "GGML_AMX_INT8"
+    bool "GGML_AMX_TILE"
+    bool "GGML_AVX"
+    bool "GGML_AVX2"
+    bool "GGML_AVX512"
+    bool "GGML_AVX512_BF16"
+    bool "GGML_AVX512_VBMI"
+    bool "GGML_AVX512_VNNI"
+    bool "GGML_AVX_VNNI"
+    ignored "GGML_BACKEND_DL"
+    ignored "GGML_BIN_INSTALL_DIR"
+    bool "GGML_BLAS"
+    string "GGML_BLAS_VENDOR"
+    bool "GGML_BMI2"
+    ignored "GGML_BUILD_EXAMPLES"
+    ignored "GGML_BUILD_TESTS"
+    filepath "GGML_CCACHE_FOUND"
+    bool "GGML_CPU"
+    bool "GGML_CPU_AARCH64"
+    ignored "GGML_CPU_ALL_VARIANTS"
+    string "GGML_CPU_ARM_ARCH"
+    bool "GGML_CPU_HBM"
+    bool "GGML_CPU_KLEIDIAI"
+    string "GGML_CPU_POWERPC_CPUTYPE"
+    bool "GGML_CUDA"
+    string "GGML_CUDA_COMPRESSION_MODE"
+    bool "GGML_CUDA_F16"
+    bool "GGML_CUDA_FA"
+    bool "GGML_CUDA_FA_ALL_QUANTS"
+    bool "GGML_CUDA_FORCE_CUBLAS"
+    bool "GGML_CUDA_FORCE_MMQ"
+    ignored "GGML_CUDA_GRAPHS"
+    bool "GGML_CUDA_NO_PEER_COPY"
+    bool "GGML_CUDA_NO_VMM"
+    string "GGML_CUDA_PEER_MAX_BATCH_SIZE"
+    bool "GGML_F16C"
+    bool "GGML_FMA"
+    bool "GGML_GPROF"
+    bool "GGML_HIP"
+    bool "GGML_HIP_GRAPHS"
+    bool "GGML_HIP_NO_VMM"
+    bool "GGML_HIP_ROCWMMA_FATTN"
+    bool "GGML_HIP_UMA"
+    ignored "GGML_INCLUDE_INSTALL_DIR"
+    bool "GGML_KOMPUTE"
+    bool "GGML_LASX"
+    ignored "GGML_LIB_INSTALL_DIR"
+    ignored "GGML_LLAMAFILE"
+    bool "GGML_LSX"
+    bool "GGML_LTO"
+    bool "GGML_METAL"
+    bool "GGML_METAL_EMBED_LIBRARY"
+    string "GGML_METAL_MACOSX_VERSION_MIN"
+    bool "GGML_METAL_NDEBUG"
+    bool "GGML_METAL_SHADER_DEBUG"
+    string "GGML_METAL_STD"
+    bool "GGML_METAL_USE_BF16"
+    bool "GGML_MUSA"
+    bool "GGML_NATIVE"
+    bool "GGML_OPENCL"
+    bool "GGML_OPENCL_EMBED_KERNELS"
+    bool "GGML_OPENCL_PROFILING"
+    string "GGML_OPENCL_TARGET_VERSION"
+    bool "GGML_OPENCL_USE_ADRENO_KERNELS"
+    bool "GGML_OPENMP"
+    bool "GGML_RPC"
+    bool "GGML_RVV"
+    bool "GGML_RV_ZFH"
+    pending "GGML_SCCACHE_FOUND"
+    string "GGML_SCHED_MAX_COPIES"
+    ignored "GGML_STATIC"
+    bool "GGML_SYCL"
+    string "GGML_SYCL_DEVICE_ARCH"
+    bool "GGML_SYCL_F16"
+    bool "GGML_SYCL_GRAPH"
+    string "GGML_SYCL_TARGET"
+    bool "GGML_VULKAN"
+    bool "GGML_VULKAN_CHECK_RESULTS"
+    bool "GGML_VULKAN_DEBUG"
+    bool "GGML_VULKAN_MEMORY_DEBUG"
+    bool "GGML_VULKAN_PERF"
+    ignored "GGML_VULKAN_RUN_TESTS"
+    filepath "GGML_VULKAN_SHADERS_GEN_TOOLCHAIN"
+    bool "GGML_VULKAN_SHADER_DEBUG_INFO"
+    pending "GGML_VULKAN_VALIDATE"
+    bool "GGML_VXE"
+    filepath "GIT_EXE"
+    filepath "MATH_LIBRARY"
+    filepath "METALKIT_FRAMEWORK"
+    filepath "METAL_FRAMEWORK"
+    bool "WHISPER_ALL_WARNINGS"
+    bool "WHISPER_ALL_WARNINGS_3RD_PARTY"
+    ignored "WHISPER_BIN_INSTALL_DIR"
+    ignored "WHISPER_BUILD_EXAMPLES"
+    ignored "WHISPER_BUILD_SERVER"
+    ignored"WHISPER_BUILD_TESTS"
+    bool "WHISPER_CCACHE"
+    bool "WHISPER_COREML"
+    bool "WHISPER_COREML_ALLOW_FALLBACK"
+    ignored "WHISPER_CURL"
+    bool "WHISPER_FATAL_WARNINGS"
+    ignored "WHISPER_FFMPEG"
+    ignored "WHISPER_INCLUDE_INSTALL_DIR"
+    ignored "WHISPER_LIB_INSTALL_DIR"
+    bool "WHISPER_OPENVINO"
+    bool "WHISPER_SANITIZE_ADDRESS"
+    bool "WHISPER_SANITIZE_THREAD"
+    bool "WHISPER_SANITIZE_UNDEFINED"
+    ignored "WHISPER_SDL2"
+    pending "WHISPER_USE_SYSTEM_GGML"
+  end
+
+  def option_name(name)
+    name.downcase.gsub("_", "-")
+  end
+
+  def bool(name)
+    option = option_name(name)
+    value = enable_config(option)
+    @options[name] = [:bool, value]
+  end
+
+  def string(name, type=:string)
+    option = "--#{option_name(name)}"
+    value = arg_config(option)
+    raise "String expected for #{option}" if value == true || value&.empty?
+    @options[name] = [type, value]
+  end
+
+  def path(name)
+    string(name, :path)
+  end
+
+  def filepath(name)
+    string(name, :filepath)
+  end
+
+  def pending(name)
+    @pending_options << name
+  end
+
+  def ignored(name)
+    @ignored_options << name
+  end
+end

bindings/ruby/ext/ruby_whisper.c

@@ -0,0 +1,164 @@
+#include <ruby.h>
+#include <ruby/memory_view.h>
+#include "ruby_whisper.h"
+
+VALUE mWhisper;
+VALUE cContext;
+VALUE cParams;
+VALUE eError;
+
+VALUE cSegment;
+VALUE cModel;
+
+ID id_to_s;
+ID id_call;
+ID id___method__;
+ID id_to_enum;
+ID id_length;
+ID id_next;
+ID id_new;
+ID id_to_path;
+ID id_URI;
+ID id_pre_converted_models;
+
+static bool is_log_callback_finalized = false;
+
+// High level API
+extern VALUE ruby_whisper_segment_allocate(VALUE klass);
+
+extern void init_ruby_whisper_context(VALUE *mWhisper);
+extern void init_ruby_whisper_params(VALUE *mWhisper);
+extern void init_ruby_whisper_error(VALUE *mWhisper);
+extern void init_ruby_whisper_segment(VALUE *mWhisper, VALUE *cSegment);
+extern void init_ruby_whisper_model(VALUE *mWhisper);
+extern void register_callbacks(ruby_whisper_params *rwp, VALUE *context);
+
+/*
+ * call-seq:
+ *   lang_max_id -> Integer
+ */
+static VALUE ruby_whisper_s_lang_max_id(VALUE self) {
+  return INT2NUM(whisper_lang_max_id());
+}
+
+/*
+ * call-seq:
+ *   lang_id(lang_name) -> Integer
+ */
+static VALUE ruby_whisper_s_lang_id(VALUE self, VALUE lang) {
+  const char * lang_str = StringValueCStr(lang);
+  const int id = whisper_lang_id(lang_str);
+  if (-1 == id) {
+    rb_raise(rb_eArgError, "language not found: %s", lang_str);
+  }
+  return INT2NUM(id);
+}
+
+/*
+ * call-seq:
+ *   lang_str(lang_id) -> String
+ */
+static VALUE ruby_whisper_s_lang_str(VALUE self, VALUE id) {
+  const int lang_id = NUM2INT(id);
+  const char * str = whisper_lang_str(lang_id);
+  if (NULL == str) {
+    rb_raise(rb_eIndexError, "id %d outside of language id", lang_id);
+  }
+  return rb_str_new2(str);
+}
+
+/*
+ * call-seq:
+ *   lang_str(lang_id) -> String
+ */
+static VALUE ruby_whisper_s_lang_str_full(VALUE self, VALUE id) {
+  const int lang_id = NUM2INT(id);
+  const char * str_full = whisper_lang_str_full(lang_id);
+  if (NULL == str_full) {
+    rb_raise(rb_eIndexError, "id %d outside of language id", lang_id);
+  }
+  return rb_str_new2(str_full);
+}
+
+static VALUE ruby_whisper_s_finalize_log_callback(VALUE self, VALUE id) {
+  is_log_callback_finalized = true;
+  return Qnil;
+}
+
+static void
+ruby_whisper_log_callback(enum ggml_log_level level, const char * buffer, void * user_data) {
+  if (is_log_callback_finalized) {
+    return;
+  }
+  VALUE log_callback = rb_iv_get(mWhisper, "log_callback");
+  VALUE udata = rb_iv_get(mWhisper, "user_data");
+  rb_funcall(log_callback, id_call, 3, INT2NUM(level), rb_str_new2(buffer), udata);
+}
+
+/*
+ * call-seq:
+ *   log_set ->(level, buffer, user_data) { ... }, user_data -> nil
+ */
+static VALUE ruby_whisper_s_log_set(VALUE self, VALUE log_callback, VALUE user_data) {
+  VALUE old_callback = rb_iv_get(self, "log_callback");
+  if (!NIL_P(old_callback)) {
+    rb_undefine_finalizer(old_callback);
+  }
+
+  rb_iv_set(self, "log_callback", log_callback);
+  rb_iv_set(self, "user_data", user_data);
+
+  VALUE finalize_log_callback = rb_funcall(mWhisper, rb_intern("method"), 1, rb_str_new2("finalize_log_callback"));
+  rb_define_finalizer(log_callback, finalize_log_callback);
+
+  whisper_log_set(ruby_whisper_log_callback, NULL);
+
+  return Qnil;
+}
+
+static void rb_whisper_model_mark(ruby_whisper_model *rwm) {
+  rb_gc_mark(rwm->context);
+}
+
+static VALUE ruby_whisper_model_allocate(VALUE klass) {
+  ruby_whisper_model *rwm;
+  rwm = ALLOC(ruby_whisper_model);
+  return Data_Wrap_Struct(klass, rb_whisper_model_mark, RUBY_DEFAULT_FREE, rwm);
+}
+
+void Init_whisper() {
+  id_to_s = rb_intern("to_s");
+  id_call = rb_intern("call");
+  id___method__ = rb_intern("__method__");
+  id_to_enum = rb_intern("to_enum");
+  id_length = rb_intern("length");
+  id_next = rb_intern("next");
+  id_new = rb_intern("new");
+  id_to_path = rb_intern("to_path");
+  id_URI = rb_intern("URI");
+  id_pre_converted_models = rb_intern("pre_converted_models");
+
+  mWhisper = rb_define_module("Whisper");
+
+  rb_define_const(mWhisper, "LOG_LEVEL_NONE", INT2NUM(GGML_LOG_LEVEL_NONE));
+  rb_define_const(mWhisper, "LOG_LEVEL_INFO", INT2NUM(GGML_LOG_LEVEL_INFO));
+  rb_define_const(mWhisper, "LOG_LEVEL_WARN", INT2NUM(GGML_LOG_LEVEL_WARN));
+  rb_define_const(mWhisper, "LOG_LEVEL_ERROR", INT2NUM(GGML_LOG_LEVEL_ERROR));
+  rb_define_const(mWhisper, "LOG_LEVEL_DEBUG", INT2NUM(GGML_LOG_LEVEL_DEBUG));
+  rb_define_const(mWhisper, "LOG_LEVEL_CONT", INT2NUM(GGML_LOG_LEVEL_CONT));
+
+  rb_define_singleton_method(mWhisper, "lang_max_id", ruby_whisper_s_lang_max_id, 0);
+  rb_define_singleton_method(mWhisper, "lang_id", ruby_whisper_s_lang_id, 1);
+  rb_define_singleton_method(mWhisper, "lang_str", ruby_whisper_s_lang_str, 1);
+  rb_define_singleton_method(mWhisper, "lang_str_full", ruby_whisper_s_lang_str_full, 1);
+  rb_define_singleton_method(mWhisper, "log_set", ruby_whisper_s_log_set, 2);
+  rb_define_private_method(rb_singleton_class(mWhisper), "finalize_log_callback", ruby_whisper_s_finalize_log_callback, 1);
+
+  init_ruby_whisper_context(&mWhisper);
+  init_ruby_whisper_params(&mWhisper);
+  init_ruby_whisper_error(&mWhisper);
+  init_ruby_whisper_segment(&mWhisper, &cContext);
+  init_ruby_whisper_model(&mWhisper);
+
+  rb_require("whisper/model/uri");
+}

bindings/ruby/ext/ruby_whisper.h

@@ -1,5 +1,5 @@
-#ifndef __RUBY_WHISPER_H
+#ifndef RUBY_WHISPER_H
-#define __RUBY_WHISPER_H
+#define RUBY_WHISPER_H
 
 #include "whisper.h"
 
@@ -22,4 +22,13 @@ typedef struct {
   ruby_whisper_callback_container *abort_callback_container;
 } ruby_whisper_params;
 
+typedef struct {
+  VALUE context;
+  int index;
+} ruby_whisper_segment;
+
+typedef struct {
+  VALUE context;
+} ruby_whisper_model;
+
 #endif

bindings/ruby/ext/ruby_whisper_context.c

@@ -0,0 +1,613 @@
+#include <ruby.h>
+#include <ruby/memory_view.h>
+#include "ruby_whisper.h"
+
+extern ID id_to_s;
+extern ID id___method__;
+extern ID id_to_enum;
+extern ID id_length;
+extern ID id_next;
+extern ID id_new;
+extern ID id_to_path;
+extern ID id_URI;
+extern ID id_pre_converted_models;
+
+extern VALUE cContext;
+extern VALUE eError;
+extern VALUE cModel;
+
+extern VALUE ruby_whisper_transcribe(int argc, VALUE *argv, VALUE self);
+extern VALUE rb_whisper_model_initialize(VALUE context);
+extern VALUE rb_whisper_segment_initialize(VALUE context, int index);
+extern void register_callbacks(ruby_whisper_params *rwp, VALUE *context);
+
+static void
+ruby_whisper_free(ruby_whisper *rw)
+{
+  if (rw->context) {
+    whisper_free(rw->context);
+    rw->context = NULL;
+  }
+}
+
+void
+rb_whisper_mark(ruby_whisper *rw)
+{
+  // call rb_gc_mark on any ruby references in rw
+}
+
+void
+rb_whisper_free(ruby_whisper *rw)
+{
+  ruby_whisper_free(rw);
+  free(rw);
+}
+
+static VALUE
+ruby_whisper_allocate(VALUE klass)
+{
+  ruby_whisper *rw;
+  rw = ALLOC(ruby_whisper);
+  rw->context = NULL;
+  return Data_Wrap_Struct(klass, rb_whisper_mark, rb_whisper_free, rw);
+}
+
+/*
+ * call-seq:
+ *   new("base.en") -> Whisper::Context
+ *   new("path/to/model.bin") -> Whisper::Context
+ *   new(Whisper::Model::URI.new("https://example.net/uri/of/model.bin")) -> Whisper::Context
+ */
+static VALUE
+ruby_whisper_initialize(int argc, VALUE *argv, VALUE self)
+{
+  ruby_whisper *rw;
+  VALUE whisper_model_file_path;
+
+  // TODO: we can support init from buffer here too maybe another ruby object to expose
+  rb_scan_args(argc, argv, "01", &whisper_model_file_path);
+  Data_Get_Struct(self, ruby_whisper, rw);
+
+  VALUE pre_converted_models = rb_funcall(cModel, id_pre_converted_models, 0);
+  VALUE pre_converted_model = rb_hash_aref(pre_converted_models, whisper_model_file_path);
+  if (!NIL_P(pre_converted_model)) {
+    whisper_model_file_path = pre_converted_model;
+  }
+  if (TYPE(whisper_model_file_path) == T_STRING) {
+    const char * whisper_model_file_path_str = StringValueCStr(whisper_model_file_path);
+    if (strncmp("http://", whisper_model_file_path_str, 7) == 0 || strncmp("https://", whisper_model_file_path_str, 8) == 0) {
+      VALUE uri_class = rb_const_get(cModel, id_URI);
+      whisper_model_file_path = rb_class_new_instance(1, &whisper_model_file_path, uri_class);
+    }
+  }
+  if (rb_obj_is_kind_of(whisper_model_file_path, rb_path2class("URI::HTTP"))) {
+    VALUE uri_class = rb_const_get(cModel, id_URI);
+    whisper_model_file_path = rb_class_new_instance(1, &whisper_model_file_path, uri_class);
+  }
+  if (rb_respond_to(whisper_model_file_path, id_to_path)) {
+    whisper_model_file_path = rb_funcall(whisper_model_file_path, id_to_path, 0);
+  }
+  if (!rb_respond_to(whisper_model_file_path, id_to_s)) {
+    rb_raise(rb_eRuntimeError, "Expected file path to model to initialize Whisper::Context");
+  }
+  rw->context = whisper_init_from_file_with_params(StringValueCStr(whisper_model_file_path), whisper_context_default_params());
+  if (rw->context == NULL) {
+    rb_raise(rb_eRuntimeError, "error: failed to initialize whisper context");
+  }
+  return self;
+}
+
+/*
+ * call-seq:
+ *   model_n_vocab -> Integer
+ */
+VALUE ruby_whisper_model_n_vocab(VALUE self)
+{
+  ruby_whisper *rw;
+  Data_Get_Struct(self, ruby_whisper, rw);
+  return INT2NUM(whisper_model_n_vocab(rw->context));
+}
+
+/*
+ * call-seq:
+ *   model_n_audio_ctx -> Integer
+ */
+VALUE ruby_whisper_model_n_audio_ctx(VALUE self)
+{
+  ruby_whisper *rw;
+  Data_Get_Struct(self, ruby_whisper, rw);
+  return INT2NUM(whisper_model_n_audio_ctx(rw->context));
+}
+
+/*
+ * call-seq:
+ *   model_n_audio_state -> Integer
+ */
+VALUE ruby_whisper_model_n_audio_state(VALUE self)
+{
+  ruby_whisper *rw;
+  Data_Get_Struct(self, ruby_whisper, rw);
+  return INT2NUM(whisper_model_n_audio_state(rw->context));
+}
+
+/*
+ * call-seq:
+ *   model_n_audio_head -> Integer
+ */
+VALUE ruby_whisper_model_n_audio_head(VALUE self)
+{
+  ruby_whisper *rw;
+  Data_Get_Struct(self, ruby_whisper, rw);
+  return INT2NUM(whisper_model_n_audio_head(rw->context));
+}
+
+/*
+ * call-seq:
+ *   model_n_audio_layer -> Integer
+ */
+VALUE ruby_whisper_model_n_audio_layer(VALUE self)
+{
+  ruby_whisper *rw;
+  Data_Get_Struct(self, ruby_whisper, rw);
+  return INT2NUM(whisper_model_n_audio_layer(rw->context));
+}
+
+/*
+ * call-seq:
+ *   model_n_text_ctx -> Integer
+ */
+VALUE ruby_whisper_model_n_text_ctx(VALUE self)
+{
+  ruby_whisper *rw;
+  Data_Get_Struct(self, ruby_whisper, rw);
+  return INT2NUM(whisper_model_n_text_ctx(rw->context));
+}
+
+/*
+ * call-seq:
+ *   model_n_text_state -> Integer
+ */
+VALUE ruby_whisper_model_n_text_state(VALUE self)
+{
+  ruby_whisper *rw;
+  Data_Get_Struct(self, ruby_whisper, rw);
+  return INT2NUM(whisper_model_n_text_state(rw->context));
+}
+
+/*
+ * call-seq:
+ *   model_n_text_head -> Integer
+ */
+VALUE ruby_whisper_model_n_text_head(VALUE self)
+{
+  ruby_whisper *rw;
+  Data_Get_Struct(self, ruby_whisper, rw);
+  return INT2NUM(whisper_model_n_text_head(rw->context));
+}
+
+/*
+ * call-seq:
+ *   model_n_text_layer -> Integer
+ */
+VALUE ruby_whisper_model_n_text_layer(VALUE self)
+{
+  ruby_whisper *rw;
+  Data_Get_Struct(self, ruby_whisper, rw);
+  return INT2NUM(whisper_model_n_text_layer(rw->context));
+}
+
+/*
+ * call-seq:
+ *   model_n_mels -> Integer
+ */
+VALUE ruby_whisper_model_n_mels(VALUE self)
+{
+  ruby_whisper *rw;
+  Data_Get_Struct(self, ruby_whisper, rw);
+  return INT2NUM(whisper_model_n_mels(rw->context));
+}
+
+/*
+ * call-seq:
+ *   model_ftype -> Integer
+ */
+VALUE ruby_whisper_model_ftype(VALUE self)
+{
+  ruby_whisper *rw;
+  Data_Get_Struct(self, ruby_whisper, rw);
+  return INT2NUM(whisper_model_ftype(rw->context));
+}
+
+/*
+ * call-seq:
+ *   model_type -> String
+ */
+VALUE ruby_whisper_model_type(VALUE self)
+{
+  ruby_whisper *rw;
+  Data_Get_Struct(self, ruby_whisper, rw);
+  return rb_str_new2(whisper_model_type_readable(rw->context));
+}
+
+/*
+ * Run the entire model: PCM -> log mel spectrogram -> encoder -> decoder -> text
+ * Not thread safe for same context
+ * Uses the specified decoding strategy to obtain the text.
+ *
+ * call-seq:
+ *   full(params, samples, n_samples) -> nil
+ *   full(params, samples) -> nil
+ *
+ * The second argument +samples+ must be an array of samples, respond to :length, or be a MemoryView of an array of float. It must be 32 bit float PCM audio data.
+ */
+VALUE ruby_whisper_full(int argc, VALUE *argv, VALUE self)
+{
+  if (argc < 2 || argc > 3) {
+    rb_raise(rb_eArgError, "wrong number of arguments (given %d, expected 2..3)", argc);
+  }
+
+  ruby_whisper *rw;
+  ruby_whisper_params *rwp;
+  Data_Get_Struct(self, ruby_whisper, rw);
+  VALUE params = argv[0];
+  Data_Get_Struct(params, ruby_whisper_params, rwp);
+  VALUE samples = argv[1];
+  int n_samples;
+  rb_memory_view_t view;
+  const bool memory_view_available_p = rb_memory_view_available_p(samples);
+  if (argc == 3) {
+    n_samples = NUM2INT(argv[2]);
+    if (TYPE(samples) == T_ARRAY) {
+      if (RARRAY_LEN(samples) < n_samples) {
+        rb_raise(rb_eArgError, "samples length %ld is less than n_samples %d", RARRAY_LEN(samples), n_samples);
+      }
+    }
+    // Should check when samples.respond_to?(:length)?
+  } else {
+    if (TYPE(samples) == T_ARRAY) {
+      n_samples = RARRAY_LEN(samples);
+    } else if (memory_view_available_p) {
+      if (!rb_memory_view_get(samples, &view, RUBY_MEMORY_VIEW_SIMPLE)) {
+        view.obj = Qnil;
+        rb_raise(rb_eArgError, "unable to get a memory view");
+      }
+      n_samples = view.byte_size / view.item_size;
+    } else if (rb_respond_to(samples, id_length)) {
+      n_samples = NUM2INT(rb_funcall(samples, id_length, 0));
+    } else {
+      rb_raise(rb_eArgError, "samples must respond to :length or be a MemoryView of an array of flaot when n_samples is not given");
+    }
+  }
+  float * c_samples = (float *)malloc(n_samples * sizeof(float));
+  if (memory_view_available_p)  {
+    c_samples = (float *)view.data;
+  } else {
+    if (TYPE(samples) == T_ARRAY) {
+      for (int i = 0; i < n_samples; i++) {
+        c_samples[i] = RFLOAT_VALUE(rb_ary_entry(samples, i));
+      }
+    } else {
+      // TODO: use rb_block_call
+      VALUE iter = rb_funcall(samples, id_to_enum, 1, rb_str_new2("each"));
+      for (int i = 0; i < n_samples; i++) {
+        // TODO: check if iter is exhausted and raise ArgumentError appropriately
+        VALUE sample = rb_funcall(iter, id_next, 0);
+        c_samples[i] = RFLOAT_VALUE(sample);
+      }
+    }
+  }
+  register_callbacks(rwp, &self);
+  const int result = whisper_full(rw->context, rwp->params, c_samples, n_samples);
+  if (0 == result) {
+    return self;
+  } else {
+    rb_exc_raise(rb_funcall(eError, id_new, 1, result));
+  }
+}
+
+/*
+ * Split the input audio in chunks and process each chunk separately using whisper_full_with_state()
+ * Result is stored in the default state of the context
+ * Not thread safe if executed in parallel on the same context.
+ * It seems this approach can offer some speedup in some cases.
+ * However, the transcription accuracy can be worse at the beginning and end of each chunk.
+ *
+ * call-seq:
+ *   full_parallel(params, samples) -> nil
+ *   full_parallel(params, samples, n_samples) -> nil
+ *   full_parallel(params, samples, n_samples, n_processors) -> nil
+ *   full_parallel(params, samples, nil, n_processors) -> nil
+ */
+static VALUE
+ruby_whisper_full_parallel(int argc, VALUE *argv,VALUE self)
+{
+  if (argc < 2 || argc > 4) {
+    rb_raise(rb_eArgError, "wrong number of arguments (given %d, expected 2..3)", argc);
+  }
+
+  ruby_whisper *rw;
+  ruby_whisper_params *rwp;
+  Data_Get_Struct(self, ruby_whisper, rw);
+  VALUE params = argv[0];
+  Data_Get_Struct(params, ruby_whisper_params, rwp);
+  VALUE samples = argv[1];
+  int n_samples;
+  int n_processors;
+  rb_memory_view_t view;
+  const bool memory_view_available_p = rb_memory_view_available_p(samples);
+  switch (argc) {
+  case 2:
+    n_processors = 1;
+    break;
+  case 3:
+    n_processors = 1;
+    break;
+  case 4:
+    n_processors = NUM2INT(argv[3]);
+    break;
+  }
+  if (argc >= 3 && !NIL_P(argv[2])) {
+    n_samples = NUM2INT(argv[2]);
+    if (TYPE(samples) == T_ARRAY) {
+      if (RARRAY_LEN(samples) < n_samples) {
+        rb_raise(rb_eArgError, "samples length %ld is less than n_samples %d", RARRAY_LEN(samples), n_samples);
+      }
+    }
+    // Should check when samples.respond_to?(:length)?
+  } else if (memory_view_available_p) {
+    if (!rb_memory_view_get(samples, &view, RUBY_MEMORY_VIEW_SIMPLE)) {
+      view.obj = Qnil;
+      rb_raise(rb_eArgError, "unable to get a memory view");
+    }
+    n_samples = view.byte_size / view.item_size;
+  } else {
+    if (TYPE(samples) == T_ARRAY) {
+      n_samples = RARRAY_LEN(samples);
+    } else if (rb_respond_to(samples, id_length)) {
+      n_samples = NUM2INT(rb_funcall(samples, id_length, 0));
+    } else {
+      rb_raise(rb_eArgError, "samples must respond to :length or be a MemoryView of an array of flaot when n_samples is not given");
+    }
+  }
+  float * c_samples = (float *)malloc(n_samples * sizeof(float));
+  if (memory_view_available_p) {
+    c_samples = (float *)view.data;
+  } else {
+    if (TYPE(samples) == T_ARRAY) {
+      for (int i = 0; i < n_samples; i++) {
+        c_samples[i] = RFLOAT_VALUE(rb_ary_entry(samples, i));
+      }
+    } else {
+      // FIXME: use rb_block_call
+      VALUE iter = rb_funcall(samples, id_to_enum, 1, rb_str_new2("each"));
+      for (int i = 0; i < n_samples; i++) {
+        // TODO: check if iter is exhausted and raise ArgumentError
+        VALUE sample = rb_funcall(iter, id_next, 0);
+        c_samples[i] = RFLOAT_VALUE(sample);
+      }
+    }
+  }
+  register_callbacks(rwp, &self);
+  const int result = whisper_full_parallel(rw->context, rwp->params, c_samples, n_samples, n_processors);
+  if (0 == result) {
+    return self;
+  } else {
+    rb_exc_raise(rb_funcall(eError, id_new, 1, result));
+  }
+}
+
+/*
+ * Number of segments.
+ *
+ * call-seq:
+ *   full_n_segments -> Integer
+ */
+static VALUE
+ruby_whisper_full_n_segments(VALUE self)
+{
+  ruby_whisper *rw;
+  Data_Get_Struct(self, ruby_whisper, rw);
+  return INT2NUM(whisper_full_n_segments(rw->context));
+}
+
+/*
+ * Language ID, which can be converted to string by Whisper.lang_str and Whisper.lang_str_full.
+ *
+ * call-seq:
+ *   full_lang_id -> Integer
+ */
+static VALUE
+ruby_whisper_full_lang_id(VALUE self)
+{
+  ruby_whisper *rw;
+  Data_Get_Struct(self, ruby_whisper, rw);
+  return INT2NUM(whisper_full_lang_id(rw->context));
+}
+
+static int ruby_whisper_full_check_segment_index(const ruby_whisper * rw, const VALUE i_segment)
+{
+  const int c_i_segment = NUM2INT(i_segment);
+  if (c_i_segment < 0 || c_i_segment >= whisper_full_n_segments(rw->context)) {
+    rb_raise(rb_eIndexError, "segment index %d out of range", c_i_segment);
+  }
+  return c_i_segment;
+}
+
+/*
+ * Start time of a segment indexed by +segment_index+ in centiseconds (10 times milliseconds).
+ *
+ *   full_get_segment_t0(3) # => 1668 (16680 ms)
+ *
+ * call-seq:
+ *   full_get_segment_t0(segment_index) -> Integer
+ */
+static VALUE
+ruby_whisper_full_get_segment_t0(VALUE self, VALUE i_segment)
+{
+  ruby_whisper *rw;
+  Data_Get_Struct(self, ruby_whisper, rw);
+  const int c_i_segment = ruby_whisper_full_check_segment_index(rw, i_segment);
+  const int64_t t0 = whisper_full_get_segment_t0(rw->context, c_i_segment);
+  return INT2NUM(t0);
+}
+
+/*
+ * End time of a segment indexed by +segment_index+ in centiseconds (10 times milliseconds).
+ *
+ *   full_get_segment_t1(3) # => 1668 (16680 ms)
+ *
+ * call-seq:
+ *   full_get_segment_t1(segment_index) -> Integer
+ */
+static VALUE
+ruby_whisper_full_get_segment_t1(VALUE self, VALUE i_segment)
+{
+  ruby_whisper *rw;
+  Data_Get_Struct(self, ruby_whisper, rw);
+  const int c_i_segment = ruby_whisper_full_check_segment_index(rw, i_segment);
+  const int64_t t1 = whisper_full_get_segment_t1(rw->context, c_i_segment);
+  return INT2NUM(t1);
+}
+
+/*
+ * Whether the next segment indexed by +segment_index+ is predicated as a speaker turn.
+ *
+ *   full_get_segment_speacker_turn_next(3) # => true
+ *
+ * call-seq:
+ *   full_get_segment_speacker_turn_next(segment_index) -> bool
+ */
+static VALUE
+ruby_whisper_full_get_segment_speaker_turn_next(VALUE self, VALUE i_segment)
+{
+  ruby_whisper *rw;
+  Data_Get_Struct(self, ruby_whisper, rw);
+  const int c_i_segment = ruby_whisper_full_check_segment_index(rw, i_segment);
+  const bool speaker_turn_next = whisper_full_get_segment_speaker_turn_next(rw->context, c_i_segment);
+  return speaker_turn_next ? Qtrue : Qfalse;
+}
+
+/*
+ * Text of a segment indexed by +segment_index+.
+ *
+ *   full_get_segment_text(3) # => "ask not what your country can do for you, ..."
+ *
+ * call-seq:
+ *   full_get_segment_text(segment_index) -> String
+ */
+static VALUE
+ruby_whisper_full_get_segment_text(VALUE self, VALUE i_segment)
+{
+  ruby_whisper *rw;
+  Data_Get_Struct(self, ruby_whisper, rw);
+  const int c_i_segment = ruby_whisper_full_check_segment_index(rw, i_segment);
+  const char * text = whisper_full_get_segment_text(rw->context, c_i_segment);
+  return rb_str_new2(text);
+}
+
+/*
+ * call-seq:
+ *   full_get_segment_no_speech_prob(segment_index) -> Float
+ */
+static VALUE
+ruby_whisper_full_get_segment_no_speech_prob(VALUE self, VALUE i_segment)
+{
+  ruby_whisper *rw;
+  Data_Get_Struct(self, ruby_whisper, rw);
+  const int c_i_segment = ruby_whisper_full_check_segment_index(rw, i_segment);
+  const float no_speech_prob = whisper_full_get_segment_no_speech_prob(rw->context, c_i_segment);
+  return DBL2NUM(no_speech_prob);
+}
+
+// High level API
+
+static VALUE
+ruby_whisper_full_get_segment(VALUE self, VALUE i_segment)
+{
+  return rb_whisper_segment_initialize(self, NUM2INT(i_segment));
+}
+
+/*
+ * Yields each Whisper::Segment:
+ *
+ *   whisper.transcribe("path/to/audio.wav", params)
+ *   whisper.each_segment do |segment|
+ *     puts segment.text
+ *   end
+ *
+ * Returns an Enumerator if no block given:
+ *
+ *   whisper.transcribe("path/to/audio.wav", params)
+ *   enum = whisper.each_segment
+ *   enum.to_a # => [#<Whisper::Segment>, ...]
+ *
+ * call-seq:
+ *   each_segment {|segment| ... }
+ *   each_segment -> Enumerator
+ */
+static VALUE
+ruby_whisper_each_segment(VALUE self)
+{
+  if (!rb_block_given_p()) {
+    const VALUE method_name = rb_funcall(self, id___method__, 0);
+    return rb_funcall(self, id_to_enum, 1, method_name);
+  }
+
+  ruby_whisper *rw;
+  Data_Get_Struct(self, ruby_whisper, rw);
+
+  const int n_segments = whisper_full_n_segments(rw->context);
+  for (int i = 0; i < n_segments; ++i) {
+    rb_yield(rb_whisper_segment_initialize(self, i));
+  }
+
+  return self;
+}
+
+/*
+ * call-seq:
+ *   model -> Whisper::Model
+ */
+static VALUE
+ruby_whisper_get_model(VALUE self)
+{
+  return rb_whisper_model_initialize(self);
+}
+
+void
+init_ruby_whisper_context(VALUE *mWhisper)
+{
+  cContext = rb_define_class_under(*mWhisper, "Context", rb_cObject);
+
+  rb_define_alloc_func(cContext, ruby_whisper_allocate);
+  rb_define_method(cContext, "initialize", ruby_whisper_initialize, -1);
+
+  rb_define_method(cContext, "transcribe", ruby_whisper_transcribe, -1);
+  rb_define_method(cContext, "model_n_vocab", ruby_whisper_model_n_vocab, 0);
+  rb_define_method(cContext, "model_n_audio_ctx", ruby_whisper_model_n_audio_ctx, 0);
+  rb_define_method(cContext, "model_n_audio_state", ruby_whisper_model_n_audio_state, 0);
+  rb_define_method(cContext, "model_n_audio_head", ruby_whisper_model_n_audio_head, 0);
+  rb_define_method(cContext, "model_n_audio_layer", ruby_whisper_model_n_audio_layer, 0);
+  rb_define_method(cContext, "model_n_text_ctx", ruby_whisper_model_n_text_ctx, 0);
+  rb_define_method(cContext, "model_n_text_state", ruby_whisper_model_n_text_state, 0);
+  rb_define_method(cContext, "model_n_text_head", ruby_whisper_model_n_text_head, 0);
+  rb_define_method(cContext, "model_n_text_layer", ruby_whisper_model_n_text_layer, 0);
+  rb_define_method(cContext, "model_n_mels", ruby_whisper_model_n_mels, 0);
+  rb_define_method(cContext, "model_ftype", ruby_whisper_model_ftype, 0);
+  rb_define_method(cContext, "model_type", ruby_whisper_model_type, 0);
+  rb_define_method(cContext, "full_n_segments", ruby_whisper_full_n_segments, 0);
+  rb_define_method(cContext, "full_lang_id", ruby_whisper_full_lang_id, 0);
+  rb_define_method(cContext, "full_get_segment_t0", ruby_whisper_full_get_segment_t0, 1);
+  rb_define_method(cContext, "full_get_segment_t1", ruby_whisper_full_get_segment_t1, 1);
+  rb_define_method(cContext, "full_get_segment_speaker_turn_next", ruby_whisper_full_get_segment_speaker_turn_next, 1);
+  rb_define_method(cContext, "full_get_segment_text", ruby_whisper_full_get_segment_text, 1);
+  rb_define_method(cContext, "full_get_segment_no_speech_prob", ruby_whisper_full_get_segment_no_speech_prob, 1);
+  rb_define_method(cContext, "full", ruby_whisper_full, -1);
+  rb_define_method(cContext, "full_parallel", ruby_whisper_full_parallel, -1);
+
+  // High leve
+  rb_define_method(cContext, "full_get_segment", ruby_whisper_full_get_segment, 1);
+  rb_define_method(cContext, "each_segment", ruby_whisper_each_segment, 0);
+
+  rb_define_method(cContext, "model", ruby_whisper_get_model, 0);
+}

bindings/ruby/ext/ruby_whisper_error.c

@@ -0,0 +1,52 @@
+#include <ruby.h>
+
+extern VALUE eError;
+
+VALUE ruby_whisper_error_initialize(VALUE self, VALUE code)
+{
+  const int c_code = NUM2INT(code);
+  const char *raw_message;
+  switch (c_code) {
+  case -2:
+    raw_message = "failed to compute log mel spectrogram";
+    break;
+  case -3:
+    raw_message = "failed to auto-detect language";
+    break;
+  case -4:
+    raw_message = "too many decoders requested";
+    break;
+  case -5:
+    raw_message = "audio_ctx is larger than the maximum allowed";
+    break;
+  case -6:
+    raw_message = "failed to encode";
+    break;
+  case -7:
+    raw_message = "whisper_kv_cache_init() failed for self-attention cache";
+    break;
+  case -8:
+    raw_message = "failed to decode";
+    break;
+  case -9:
+    raw_message = "failed to decode";
+    break;
+  default:
+    raw_message = "unknown error";
+    break;
+  }
+  const VALUE message = rb_str_new2(raw_message);
+  rb_call_super(1, &message);
+  rb_iv_set(self, "@code", code);
+
+  return self;
+}
+
+void
+init_ruby_whisper_error(VALUE *mWhisper)
+{
+  eError = rb_define_class_under(*mWhisper, "Error", rb_eStandardError);
+
+  rb_define_attr(eError, "code", true, false);
+  rb_define_method(eError, "initialize", ruby_whisper_error_initialize, 1);
+}

bindings/ruby/ext/ruby_whisper_model.c

@@ -0,0 +1,210 @@
+#include <ruby.h>
+#include "ruby_whisper.h"
+
+extern VALUE cModel;
+
+static void rb_whisper_model_mark(ruby_whisper_model *rwm) {
+  rb_gc_mark(rwm->context);
+}
+
+static VALUE ruby_whisper_model_allocate(VALUE klass) {
+  ruby_whisper_model *rwm;
+  rwm = ALLOC(ruby_whisper_model);
+  return Data_Wrap_Struct(klass, rb_whisper_model_mark, RUBY_DEFAULT_FREE, rwm);
+}
+
+VALUE rb_whisper_model_initialize(VALUE context) {
+  ruby_whisper_model *rwm;
+  const VALUE model = ruby_whisper_model_allocate(cModel);
+  Data_Get_Struct(model, ruby_whisper_model, rwm);
+  rwm->context = context;
+  return model;
+};
+
+/*
+ * call-seq:
+ *   n_vocab -> Integer
+ */
+static VALUE
+ruby_whisper_model_n_vocab(VALUE self)
+{
+  ruby_whisper_model *rwm;
+  Data_Get_Struct(self, ruby_whisper_model, rwm);
+  ruby_whisper *rw;
+  Data_Get_Struct(rwm->context, ruby_whisper, rw);
+  return INT2NUM(whisper_model_n_vocab(rw->context));
+}
+
+/*
+ * call-seq:
+ *   n_audio_ctx -> Integer
+ */
+static VALUE
+ruby_whisper_model_n_audio_ctx(VALUE self)
+{
+  ruby_whisper_model *rwm;
+  Data_Get_Struct(self, ruby_whisper_model, rwm);
+  ruby_whisper *rw;
+  Data_Get_Struct(rwm->context, ruby_whisper, rw);
+  return INT2NUM(whisper_model_n_audio_ctx(rw->context));
+}
+
+/*
+ * call-seq:
+ *   n_audio_state -> Integer
+ */
+static VALUE
+ruby_whisper_model_n_audio_state(VALUE self)
+{
+  ruby_whisper_model *rwm;
+  Data_Get_Struct(self, ruby_whisper_model, rwm);
+  ruby_whisper *rw;
+  Data_Get_Struct(rwm->context, ruby_whisper, rw);
+  return INT2NUM(whisper_model_n_audio_state(rw->context));
+}
+
+/*
+ * call-seq:
+ *   n_audio_head -> Integer
+ */
+static VALUE
+ruby_whisper_model_n_audio_head(VALUE self)
+{
+  ruby_whisper_model *rwm;
+  Data_Get_Struct(self, ruby_whisper_model, rwm);
+  ruby_whisper *rw;
+  Data_Get_Struct(rwm->context, ruby_whisper, rw);
+  return INT2NUM(whisper_model_n_audio_head(rw->context));
+}
+
+/*
+ * call-seq:
+ *   n_audio_layer -> Integer
+ */
+static VALUE
+ruby_whisper_model_n_audio_layer(VALUE self)
+{
+  ruby_whisper_model *rwm;
+  Data_Get_Struct(self, ruby_whisper_model, rwm);
+  ruby_whisper *rw;
+  Data_Get_Struct(rwm->context, ruby_whisper, rw);
+  return INT2NUM(whisper_model_n_audio_layer(rw->context));
+}
+
+/*
+ * call-seq:
+ *   n_text_ctx -> Integer
+ */
+static VALUE
+ruby_whisper_model_n_text_ctx(VALUE self)
+{
+  ruby_whisper_model *rwm;
+  Data_Get_Struct(self, ruby_whisper_model, rwm);
+  ruby_whisper *rw;
+  Data_Get_Struct(rwm->context, ruby_whisper, rw);
+  return INT2NUM(whisper_model_n_text_ctx(rw->context));
+}
+
+/*
+ * call-seq:
+ *   n_text_state -> Integer
+ */
+static VALUE
+ruby_whisper_model_n_text_state(VALUE self)
+{
+  ruby_whisper_model *rwm;
+  Data_Get_Struct(self, ruby_whisper_model, rwm);
+  ruby_whisper *rw;
+  Data_Get_Struct(rwm->context, ruby_whisper, rw);
+  return INT2NUM(whisper_model_n_text_state(rw->context));
+}
+
+/*
+ * call-seq:
+ *   n_text_head -> Integer
+ */
+static VALUE
+ruby_whisper_model_n_text_head(VALUE self)
+{
+  ruby_whisper_model *rwm;
+  Data_Get_Struct(self, ruby_whisper_model, rwm);
+  ruby_whisper *rw;
+  Data_Get_Struct(rwm->context, ruby_whisper, rw);
+  return INT2NUM(whisper_model_n_text_head(rw->context));
+}
+
+/*
+ * call-seq:
+ *   n_text_layer -> Integer
+ */
+static VALUE
+ruby_whisper_model_n_text_layer(VALUE self)
+{
+  ruby_whisper_model *rwm;
+  Data_Get_Struct(self, ruby_whisper_model, rwm);
+  ruby_whisper *rw;
+  Data_Get_Struct(rwm->context, ruby_whisper, rw);
+  return INT2NUM(whisper_model_n_text_layer(rw->context));
+}
+
+/*
+ * call-seq:
+ *   n_mels -> Integer
+ */
+static VALUE
+ruby_whisper_model_n_mels(VALUE self)
+{
+  ruby_whisper_model *rwm;
+  Data_Get_Struct(self, ruby_whisper_model, rwm);
+  ruby_whisper *rw;
+  Data_Get_Struct(rwm->context, ruby_whisper, rw);
+  return INT2NUM(whisper_model_n_mels(rw->context));
+}
+
+/*
+ * call-seq:
+ *   ftype -> Integer
+ */
+static VALUE
+ruby_whisper_model_ftype(VALUE self)
+{
+  ruby_whisper_model *rwm;
+  Data_Get_Struct(self, ruby_whisper_model, rwm);
+  ruby_whisper *rw;
+  Data_Get_Struct(rwm->context, ruby_whisper, rw);
+  return INT2NUM(whisper_model_ftype(rw->context));
+}
+
+/*
+ * call-seq:
+ *   type -> String
+ */
+static VALUE
+ruby_whisper_model_type(VALUE self)
+{
+  ruby_whisper_model *rwm;
+  Data_Get_Struct(self, ruby_whisper_model, rwm);
+  ruby_whisper *rw;
+  Data_Get_Struct(rwm->context, ruby_whisper, rw);
+  return rb_str_new2(whisper_model_type_readable(rw->context));
+}
+
+void
+init_ruby_whisper_model(VALUE *mWhisper)
+{
+  cModel = rb_define_class_under(*mWhisper, "Model", rb_cObject);
+
+  rb_define_alloc_func(cModel, ruby_whisper_model_allocate);
+  rb_define_method(cModel, "n_vocab", ruby_whisper_model_n_vocab, 0);
+  rb_define_method(cModel, "n_audio_ctx", ruby_whisper_model_n_audio_ctx, 0);
+  rb_define_method(cModel, "n_audio_state", ruby_whisper_model_n_audio_state, 0);
+  rb_define_method(cModel, "n_audio_head", ruby_whisper_model_n_audio_head, 0);
+  rb_define_method(cModel, "n_audio_layer", ruby_whisper_model_n_audio_layer, 0);
+  rb_define_method(cModel, "n_text_ctx", ruby_whisper_model_n_text_ctx, 0);
+  rb_define_method(cModel, "n_text_state", ruby_whisper_model_n_text_state, 0);
+  rb_define_method(cModel, "n_text_head", ruby_whisper_model_n_text_head, 0);
+  rb_define_method(cModel, "n_text_layer", ruby_whisper_model_n_text_layer, 0);
+  rb_define_method(cModel, "n_mels", ruby_whisper_model_n_mels, 0);
+  rb_define_method(cModel, "ftype", ruby_whisper_model_ftype, 0);
+  rb_define_method(cModel, "type", ruby_whisper_model_type, 0);
+}

bindings/ruby/ext/ruby_whisper_segment.c

@@ -0,0 +1,123 @@
+#include <ruby.h>
+#include "ruby_whisper.h"
+
+extern VALUE cSegment;
+
+static void
+rb_whisper_segment_mark(ruby_whisper_segment *rws)
+{
+  rb_gc_mark(rws->context);
+}
+
+VALUE
+ruby_whisper_segment_allocate(VALUE klass)
+{
+  ruby_whisper_segment *rws;
+  rws = ALLOC(ruby_whisper_segment);
+  return Data_Wrap_Struct(klass, rb_whisper_segment_mark, RUBY_DEFAULT_FREE, rws);
+}
+
+VALUE
+rb_whisper_segment_initialize(VALUE context, int index)
+{
+  ruby_whisper_segment *rws;
+  const VALUE segment = ruby_whisper_segment_allocate(cSegment);
+  Data_Get_Struct(segment, ruby_whisper_segment, rws);
+  rws->context = context;
+  rws->index = index;
+  return segment;
+};
+
+/*
+ * Start time in milliseconds.
+ *
+ * call-seq:
+ *   start_time -> Integer
+ */
+static VALUE
+ruby_whisper_segment_get_start_time(VALUE self)
+{
+  ruby_whisper_segment *rws;
+  Data_Get_Struct(self, ruby_whisper_segment, rws);
+  ruby_whisper *rw;
+  Data_Get_Struct(rws->context, ruby_whisper, rw);
+  const int64_t t0 = whisper_full_get_segment_t0(rw->context, rws->index);
+  // able to multiply 10 without overflow because to_timestamp() in whisper.cpp does it
+  return INT2NUM(t0 * 10);
+}
+
+/*
+ * End time in milliseconds.
+ *
+ * call-seq:
+ *   end_time -> Integer
+ */
+static VALUE
+ruby_whisper_segment_get_end_time(VALUE self)
+{
+  ruby_whisper_segment *rws;
+  Data_Get_Struct(self, ruby_whisper_segment, rws);
+  ruby_whisper *rw;
+  Data_Get_Struct(rws->context, ruby_whisper, rw);
+  const int64_t t1 = whisper_full_get_segment_t1(rw->context, rws->index);
+  // able to multiply 10 without overflow because to_timestamp() in whisper.cpp does it
+  return INT2NUM(t1 * 10);
+}
+
+/*
+ * Whether the next segment is predicted as a speaker turn.
+ *
+ * call-seq:
+ *   speaker_turn_next? -> bool
+ */
+static VALUE
+ruby_whisper_segment_get_speaker_turn_next(VALUE self)
+{
+  ruby_whisper_segment *rws;
+  Data_Get_Struct(self, ruby_whisper_segment, rws);
+  ruby_whisper *rw;
+  Data_Get_Struct(rws->context, ruby_whisper, rw);
+  return whisper_full_get_segment_speaker_turn_next(rw->context, rws->index) ? Qtrue : Qfalse;
+}
+
+/*
+ * call-seq:
+ *   text -> String
+ */
+static VALUE
+ruby_whisper_segment_get_text(VALUE self)
+{
+  ruby_whisper_segment *rws;
+  Data_Get_Struct(self, ruby_whisper_segment, rws);
+  ruby_whisper *rw;
+  Data_Get_Struct(rws->context, ruby_whisper, rw);
+  const char * text = whisper_full_get_segment_text(rw->context, rws->index);
+  return rb_str_new2(text);
+}
+
+/*
+ * call-seq:
+ *   no_speech_prob -> Float
+ */
+static VALUE
+ruby_whisper_segment_get_no_speech_prob(VALUE self)
+{
+  ruby_whisper_segment *rws;
+  Data_Get_Struct(self, ruby_whisper_segment, rws);
+  ruby_whisper *rw;
+  Data_Get_Struct(rws->context, ruby_whisper, rw);
+  return DBL2NUM(whisper_full_get_segment_no_speech_prob(rw->context, rws->index));
+}
+
+void
+init_ruby_whisper_segment(VALUE *mWhisper, VALUE *cContext)
+{
+  cSegment  = rb_define_class_under(*mWhisper, "Segment", rb_cObject);
+
+  rb_define_alloc_func(cSegment, ruby_whisper_segment_allocate);
+  rb_define_method(cSegment, "start_time", ruby_whisper_segment_get_start_time, 0);
+  rb_define_method(cSegment, "end_time", ruby_whisper_segment_get_end_time, 0);
+  rb_define_method(cSegment, "speaker_next_turn?", ruby_whisper_segment_get_speaker_turn_next, 0);
+  rb_define_method(cSegment, "text", ruby_whisper_segment_get_text, 0);
+  rb_define_method(cSegment, "no_speech_prob", ruby_whisper_segment_get_no_speech_prob, 0);
+}

bindings/ruby/ext/ruby_whisper_transcribe.cpp

@@ -0,0 +1,83 @@
+#include <ruby.h>
+#include "ruby_whisper.h"
+#include "common-whisper.h"
+#include <string>
+#include <vector>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern ID id_to_s;
+extern ID id_call;
+
+extern void
+register_callbacks(ruby_whisper_params * rwp, VALUE * self);
+
+/*
+ * transcribe a single file
+ * can emit to a block results
+ *
+ *   params = Whisper::Params.new
+ *   params.duration = 60_000
+ *   whisper.transcribe "path/to/audio.wav", params do |text|
+ *     puts text
+ *   end
+ *
+ * call-seq:
+ *   transcribe(path_to_audio, params) {|text| ...}
+ **/
+VALUE
+ruby_whisper_transcribe(int argc, VALUE *argv, VALUE self) {
+  ruby_whisper *rw;
+  ruby_whisper_params *rwp;
+  VALUE wave_file_path, blk, params;
+
+  rb_scan_args(argc, argv, "02&", &wave_file_path, &params, &blk);
+  Data_Get_Struct(self, ruby_whisper, rw);
+  Data_Get_Struct(params, ruby_whisper_params, rwp);
+
+  if (!rb_respond_to(wave_file_path, id_to_s)) {
+    rb_raise(rb_eRuntimeError, "Expected file path to wave file");
+  }
+
+  std::string fname_inp = StringValueCStr(wave_file_path);
+
+  std::vector<float> pcmf32; // mono-channel F32 PCM
+  std::vector<std::vector<float>> pcmf32s; // stereo-channel F32 PCM
+
+  if (!read_audio_data(fname_inp, pcmf32, pcmf32s, rwp->diarize)) {
+    fprintf(stderr, "error: failed to open '%s' as WAV file\n", fname_inp.c_str());
+    return self;
+  }
+  {
+    static bool is_aborted = false; // NOTE: this should be atomic to avoid data race
+
+    rwp->params.encoder_begin_callback = [](struct whisper_context * /*ctx*/, struct whisper_state * /*state*/, void * user_data) {
+      bool is_aborted = *(bool*)user_data;
+      return !is_aborted;
+    };
+    rwp->params.encoder_begin_callback_user_data = &is_aborted;
+  }
+
+  register_callbacks(rwp, &self);
+
+  if (whisper_full_parallel(rw->context, rwp->params, pcmf32.data(), pcmf32.size(), 1) != 0) {
+    fprintf(stderr, "failed to process audio\n");
+    return self;
+  }
+  const int n_segments = whisper_full_n_segments(rw->context);
+  VALUE output = rb_str_new2("");
+  for (int i = 0; i < n_segments; ++i) {
+    const char * text = whisper_full_get_segment_text(rw->context, i);
+    output = rb_str_concat(output, rb_str_new2(text));
+  }
+  VALUE idCall = id_call;
+  if (blk != Qnil) {
+    rb_funcall(blk, idCall, 1, output);
+  }
+  return self;
+}
+#ifdef __cplusplus
+}
+#endif

bindings/ruby/ext/sources/CMakeGraphVizOptions.cmake

@@ -0,0 +1,8 @@
+set(GRAPHVIZ_EXECUTABLES FALSE)
+set(GRAPHVIZ_STATIC_LIBS TRUE)
+set(GRAPHVIZ_SHARED_LIBS FALSE)
+set(GRAPHVIZ_MODULE_LIBS FALSE)
+set(GRAPHVIZ_INTERFACE_LIBS FALSE)
+set(GRAPHVIZ_OBJECT_LIBS FALSE)
+set(GRAPHVIZ_UNKNOWN_LIBS FALSE)
+set(GRAPHVIZ_GENERATE_DEPENDERS FALSE)

bindings/ruby/extsources.rb

@@ -1,6 +1,34 @@
-require "yaml"
+ignored_dirs = %w[
+  .devops
+  examples/wchess/wchess.wasm
+  examples/whisper.android
+  examples/whisper.android.java
+  examples/whisper.objc
+  examples/whisper.swiftui
+  grammars
+  models
+  samples
+  scripts
+]
+ignored_files = %w[
+  AUTHORS
+  Makefile
+  README.md
+  README_sycl.md
+  .gitignore
+  .gitmodules
+  whisper.nvim
+  twitch.sh
+  yt-wsp.sh
+]
 
-sources = `git ls-files -z ../..`.split("\x0")
+EXTSOURCES =
-paths = YAML.load_file("../../.github/workflows/bindings-ruby.yml")[true]["push"]["paths"]
+  `git ls-files -z ../..`.split("\x0")
-paths.delete "bindings/ruby/**"
+    .select {|file|
-EXTSOURCES = (Dir.glob(paths, base: "../..").collect {|path| "../../#{path}"} << "../../LICENSE") & sources
+      basename = File.basename(file)
+
+      ignored_dirs.all? {|dir| !file.start_with?("../../#{dir}")} &&
+        !ignored_files.include?(basename) &&
+        (file.start_with?("../..") || file.start_with?("../javascript")) &&
+        (!file.start_with?("../../.github/") || basename == "bindings-ruby.yml")
+    }

bindings/ruby/lib/whisper.rb

@@ -1,2 +0,0 @@
-require "whisper.so"
-require "whisper/model/uri"

bindings/ruby/lib/whisper/model/uri.rb

@@ -1,157 +1,170 @@
-require "whisper.so"
 require "uri"
 require "net/http"
 require "time"
 require "pathname"
 require "io/console/size"
 
-class Whisper::Model
+module Whisper
-  class URI
+  class Model
-    def initialize(uri)
+    class URI
-      @uri = URI(uri)
+      def initialize(uri)
-    end
+        @uri = URI(uri)
+      end
 
-    def to_path
+      def to_path
-      cache
+        cache
-      cache_path.to_path
+        cache_path.to_path
-    end
+      end
 
-    def clear_cache
+      def clear_cache
-      path = cache_path
+        path = cache_path
-      path.delete if path.exist?
+        path.delete if path.exist?
-    end
+      end
 
-    private
+      private
 
-    def cache_path
+      def cache_path
-      base_cache_dir/@uri.host/@uri.path[1..]
+        base_cache_dir/@uri.host/@uri.path[1..]
-    end
+      end
 
-    def base_cache_dir
+      def base_cache_dir
-      base = case RUBY_PLATFORM
+        base = case RUBY_PLATFORM
-             when /mswin|mingw/
+               when /mswin|mingw/
-               ENV.key?("LOCALAPPDATA") ? Pathname(ENV["LOCALAPPDATA"]) : Pathname(Dir.home)/"AppData/Local"
+                 ENV.key?("LOCALAPPDATA") ? Pathname(ENV["LOCALAPPDATA"]) : Pathname(Dir.home)/"AppData/Local"
-             when /darwin/
+               when /darwin/
-               Pathname(Dir.home)/"Library/Caches"
+                 Pathname(Dir.home)/"Library/Caches"
-             else
+               else
-               ENV.key?("XDG_CACHE_HOME") ? ENV["XDG_CACHE_HOME"] : Pathname(Dir.home)/".cache"
+                 ENV.key?("XDG_CACHE_HOME") ? Pathname(ENV["XDG_CACHE_HOME"]) : Pathname(Dir.home)/".cache"
-             end
+               end
-      base/"whisper.cpp"
+        base/"whisper.cpp"
-    end
+      end
 
-    def cache
+      def cache
-      path = cache_path
+        path = cache_path
-      headers = {}
+        headers = {}
-      headers["if-modified-since"] = path.mtime.httpdate if path.exist?
+        headers["if-modified-since"] = path.mtime.httpdate if path.exist?
-      request @uri, headers
+        request @uri, headers
-      path
+        path
-    end
+      end
 
-    def request(uri, headers)
+      def request(uri, headers)
-      Net::HTTP.start uri.host, uri.port, use_ssl: uri.scheme == "https" do |http|
+        Net::HTTP.start uri.host, uri.port, use_ssl: uri.scheme == "https" do |http|
-        request = Net::HTTP::Get.new(uri, headers)
+          request = Net::HTTP::Get.new(uri, headers)
-        http.request request do |response|
+          http.request request do |response|
-          case response
+            case response
-          when Net::HTTPNotModified
+            when Net::HTTPNotModified
             # noop
-          when Net::HTTPOK
+            when Net::HTTPOK
-            download response
+              download response
-          when Net::HTTPRedirection
+            when Net::HTTPRedirection
-            request URI(response["location"]), headers
+              request URI(response["location"]), headers
-          else
+            else
-            return if headers.key?("if-modified-since") # Use cache file
+              return if headers.key?("if-modified-since") # Use cache file
 
-            raise "#{response.code} #{response.message}\n#{response.body}"
+              raise "#{response.code} #{response.message}\n#{response.body}"
+            end
           end
         end
-      end
+      rescue => err
-    end
+        if cache_path.exist?
-
+          warn err
-    def download(response)
+        # Use cache file
-      path = cache_path
+        else
-      path.dirname.mkpath unless path.dirname.exist?
+          raise
-      downloading_path = Pathname("#{path}.downloading")
-      size = response.content_length
-      downloading_path.open "wb" do |file|
-        downloaded = 0
-        response.read_body do |chunk|
-          file << chunk
-          downloaded += chunk.bytesize
-          show_progress downloaded, size
         end
       end
-      downloading_path.rename path
-    end
 
-    def show_progress(current, size)
+      def download(response)
-      return unless $stderr.tty?
+        path = cache_path
-      return unless size
+        path.dirname.mkpath unless path.dirname.exist?
-
+        downloading_path = Pathname("#{path}.downloading")
-      unless @prev
+        size = response.content_length
-        @prev = Time.now
+        downloading_path.open "wb" do |file|
-        $stderr.puts "Downloading #{@uri}"
+          downloaded = 0
+          response.read_body do |chunk|
+            file << chunk
+            downloaded += chunk.bytesize
+            show_progress downloaded, size
+          end
+          $stderr.puts
+        end
+        downloading_path.rename path
       end
 
-      now = Time.now
+      def show_progress(current, size)
-      return if now - @prev < 1 && current < size
+        progress_rate_available = size && $stderr.tty?
 
-      progress_width = 20
+        unless @prev
-      progress = current.to_f / size
+          @prev = Time.now
-      arrow_length = progress * progress_width
+          $stderr.puts "Downloading #{@uri} to #{cache_path}"
-      arrow = "=" * (arrow_length - 1) + ">" + " " * (progress_width - arrow_length)
+        end
-      line = "[#{arrow}] (#{format_bytesize(current)} / #{format_bytesize(size)})"
+
-      padding = ' ' * ($stderr.winsize[1] - line.size)
+        now = Time.now
-      $stderr.print "\r#{line}#{padding}"
+
-      $stderr.puts if current >= size
+        if progress_rate_available
-      @prev = now
+          return if now - @prev < 1 && current < size
+
+          progress_width = 20
+          progress = current.to_f / size
+          arrow_length = progress * progress_width
+          arrow = "=" * (arrow_length - 1) + ">" + " " * (progress_width - arrow_length)
+          line = "[#{arrow}] (#{format_bytesize(current)} / #{format_bytesize(size)})"
+          padding = ' ' * ($stderr.winsize[1] - line.size)
+          $stderr.print "\r#{line}#{padding}"
+        else
+          return if now - @prev < 1
+
+          $stderr.print "."
+        end
+        @prev = now
+      end
+
+      def format_bytesize(bytesize)
+        return "0.0 B" if bytesize.zero?
+
+        units = %w[B KiB MiB GiB TiB]
+        exp = (Math.log(bytesize) / Math.log(1024)).to_i
+        format("%.1f %s", bytesize.to_f / 1024 ** exp, units[exp])
+      end
     end
 
-    def format_bytesize(bytesize)
+    @pre_converted_models = %w[
-      return "0.0 B" if bytesize.zero?
+      tiny
+      tiny.en
+      tiny-q5_1
+      tiny.en-q5_1
+      tiny-q8_0
+      base
+      base.en
+      base-q5_1
+      base.en-q5_1
+      base-q8_0
+      small
+      small.en
+      small.en-tdrz
+      small-q5_1
+      small.en-q5_1
+      small-q8_0
+      medium
+      medium.en
+      medium-q5_0
+      medium.en-q5_0
+      medium-q8_0
+      large-v1
+      large-v2
+      large-v2-q5_0
+      large-v2-q8_0
+      large-v3
+      large-v3-q5_0
+      large-v3-turbo
+      large-v3-turbo-q5_0
+      large-v3-turbo-q8_0
+    ].each_with_object({}) {|name, models|
+      models[name] = URI.new("https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-#{name}.bin")
+    }
 
-      units = %w[B KiB MiB GiB TiB]
+    class << self
-      exp = (Math.log(bytesize) / Math.log(1024)).to_i
+      attr_reader :pre_converted_models
-      format("%.1f %s", bytesize.to_f / 1024 ** exp, units[exp])
     end
   end
-
-  @pre_converted_models = {}
-  %w[
-    tiny
-    tiny.en
-    tiny-q5_1
-    tiny.en-q5_1
-    tiny-q8_0
-    base
-    base.en
-    base-q5_1
-    base.en-q5_1
-    base-q8_0
-    small
-    small.en
-    small.en-tdrz
-    small-q5_1
-    small.en-q5_1
-    small-q8_0
-    medium
-    medium.en
-    medium-q5_0
-    medium.en-q5_0
-    medium-q8_0
-    large-v1
-    large-v2
-    large-v2-q5_0
-    large-v2-q8_0
-    large-v3
-    large-v3-q5_0
-    large-v3-turbo
-    large-v3-turbo-q5_0
-    large-v3-turbo-q8_0
-  ].each do |name|
-    @pre_converted_models[name] = URI.new("https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-#{name}.bin")
-  end
-
-  class << self
-    attr_reader :pre_converted_models
-  end
 end

bindings/ruby/sig/whisper.rbs

@@ -0,0 +1,395 @@
+module Whisper
+  interface _Samples
+    def length: () -> Integer
+    def each: { (Float) -> void } -> void
+  end
+
+  type log_callback = ^(Integer level, String message, Object user_data) -> void
+  type new_segment_callback = ^(Whisper::Context, void, Integer n_new, Object user_data) -> void
+  type progress_callback = ^(Whisper::Context, void, Integer progress, Object user_data) -> void
+  type abort_callback = ^(Whisper::Context, void, Object user_data) -> boolish
+
+  LOG_LEVEL_NONE: Integer
+  LOG_LEVEL_INFO: Integer
+  LOG_LEVEL_WARN: Integer
+  LOG_LEVEL_ERROR: Integer
+  LOG_LEVEL_DEBUG: Integer
+  LOG_LEVEL_CONT: Integer
+
+  def self.lang_max_id: () -> Integer
+  def self.lang_id: (string name) -> Integer
+  def self.lang_str: (Integer id) -> String
+  def self.lang_str_full: (Integer id) -> String
+  def self.log_set: (log_callback, Object? user_data) -> log_callback
+
+  class Context
+    def self.new: (path | ::URI::HTTP) -> instance
+
+    # transcribe a single file
+    # can emit to a block results
+    #
+    #   params = Whisper::Params.new
+    #   params.duration = 60_000
+    #   whisper.transcribe "path/to/audio.wav", params do |text|
+    #     puts text
+    #   end
+    #
+    def transcribe: (string, Params) -> self
+                  | (string, Params) { (String) -> void } -> self
+
+    def model_n_vocab: () -> Integer
+    def model_n_audio_ctx: () -> Integer
+    def model_n_audio_state: () -> Integer
+    def model_n_text_head: () -> Integer
+    def model_n_text_layer: () -> Integer
+    def model_n_mels: () -> Integer
+    def model_ftype: () -> Integer
+    def model_type: () -> String
+
+    # Yields each Whisper::Segment:
+    #
+    #   whisper.transcribe("path/to/audio.wav", params)
+    #   whisper.each_segment do |segment|
+    #     puts segment.text
+    #   end
+    #
+    # Returns an Enumerator if no block given:
+    #
+    #   whisper.transcribe("path/to/audio.wav", params)
+    #   enum = whisper.each_segment
+    #   enum.to_a # => [#<Whisper::Segment>, ...]
+    #
+    def each_segment: { (Segment) -> void } -> void
+                    | () -> Enumerator[Segment]
+
+    def model: () -> Model
+    def full_get_segment: (Integer nth) -> Segment
+    def full_n_segments: () -> Integer
+
+    # Language ID, which can be converted to string by Whisper.lang_str and Whisper.lang_str_full.
+    #
+    def full_lang_id: () -> Integer
+
+    # Start time of a segment indexed by +segment_index+ in centiseconds (10 times milliseconds).
+    #
+    #   full_get_segment_t0(3) # => 1668 (16680 ms)
+    #
+    def full_get_segment_t0: (Integer) -> Integer
+
+    # End time of a segment indexed by +segment_index+ in centiseconds (10 times milliseconds).
+    #
+    #   full_get_segment_t1(3) # => 1668 (16680 ms)
+    #
+    def full_get_segment_t1: (Integer) -> Integer
+
+    # Whether the next segment indexed by +segment_index+ is predicated as a speaker turn.
+    #
+    #   full_get_segment_speacker_turn_next(3) # => true
+    #
+    def full_get_segment_speaker_turn_next: (Integer) -> (true | false)
+
+    # Text of a segment indexed by +segment_index+.
+    #
+    #   full_get_segment_text(3) # => "ask not what your country can do for you, ..."
+    #
+    def full_get_segment_text: (Integer) -> String
+
+    def full_get_segment_no_speech_prob: (Integer) -> Float
+
+    # Run the entire model: PCM -> log mel spectrogram -> encoder -> decoder -> text
+    # Not thread safe for same context
+    # Uses the specified decoding strategy to obtain the text.
+    #
+    # The second argument +samples+ must be an array of samples, respond to :length, or be a MemoryView of an array of float. It must be 32 bit float PCM audio data.
+    #
+    def full: (Params, Array[Float] samples, ?Integer n_samples) -> self
+            | (Params, _Samples, ?Integer n_samples) -> self
+
+    # Split the input audio in chunks and process each chunk separately using whisper_full_with_state()
+    # Result is stored in the default state of the context
+    # Not thread safe if executed in parallel on the same context.
+    # It seems this approach can offer some speedup in some cases.
+    # However, the transcription accuracy can be worse at the beginning and end of each chunk.
+    #
+    def full_parallel: (Params, Array[Float], ?Integer n_samples) -> self
+                     | (Params, _Samples, ?Integer n_samples) -> self
+                     | (Params, _Samples, ?Integer? n_samples, Integer n_processors) -> self
+  end
+
+  class Params
+    def self.new: (
+      ?language: string,
+      ?translate: boolish,
+      ?no_context: boolish,
+      ?single_segment: boolish,
+      ?print_special: boolish,
+      ?print_progress: boolish,
+      ?print_realtime: boolish,
+      ?print_timestamps: boolish,
+      ?suppress_blank: boolish,
+      ?suppress_nst: boolish,
+      ?token_timestamps: boolish,
+      ?split_on_word: boolish,
+      ?initial_prompt: string | nil,
+      ?diarize: boolish,
+      ?offset: Integer,
+      ?duration: Integer,
+      ?max_text_tokens: Integer,
+      ?temperature: Float,
+      ?max_initial_ts: Float,
+      ?length_penalty: Float,
+      ?temperature_inc: Float,
+      ?entropy_thold: Float,
+      ?logprob_thold: Float,
+      ?no_speech_thold: Float,
+      ?new_segment_callback: new_segment_callback,
+      ?new_segment_callback_user_data: Object,
+      ?progress_callback: progress_callback,
+      ?progress_callback_user_data: Object,
+      ?abort_callback: abort_callback,
+      ?abort_callback_user_data: Object
+    ) -> instance
+
+    # params.language = "auto" | "en", etc...
+    #
+    def language=: (String) -> String # TODO: Enumerate lang names
+
+    def language: () -> String
+    def translate=: (boolish) -> boolish
+    def translate: () -> (true | false)
+    def no_context=: (boolish) -> boolish
+
+    # If true, does not use past transcription (if any) as initial prompt for the decoder.
+    #
+    def no_context: () -> (true | false)
+
+    def single_segment=: (boolish) -> boolish
+
+    # If true, forces single segment output (useful for streaming).
+    #
+    def single_segment: () -> (true | false)
+
+    def print_special=: (boolish) -> boolish
+
+    # If true, prints special tokens (e.g. <SOT>, <EOT>, <BEG>, etc.).
+    #
+    def print_special: () -> (true | false)
+
+    def print_progress=: (boolish) -> boolish
+
+    # If true, prints progress information.
+    #
+    def print_progress: () -> (true | false)
+
+    def print_realtime=: (boolish) -> boolish
+
+    # If true, prints results from within whisper.cpp. (avoid it, use callback instead)
+    #
+    def print_realtime: () -> (true | false)
+
+    # If true, prints timestamps for each text segment when printing realtime.
+    #
+    def print_timestamps=: (boolish) -> boolish
+
+    def print_timestamps: () -> (true | false)
+
+    def suppress_blank=: (boolish) -> boolish
+
+    # If true, suppresses blank outputs.
+    #
+    def suppress_blank: () -> (true | false)
+
+    def suppress_nst=: (boolish) -> boolish
+
+    # If true, suppresses non-speech-tokens.
+    #
+    def suppress_nst: () -> (true | false)
+
+    def token_timestamps=: (boolish) -> boolish
+
+    # If true, enables token-level timestamps.
+    #
+    def token_timestamps: () -> (true | false)
+
+    def split_on_word=: (boolish) -> boolish
+
+    # If true, split on word rather than on token (when used with max_len).
+    #
+    def split_on_word: () -> (true | false)
+
+    def initial_prompt=: (_ToS) -> _ToS
+
+    # Tokens to provide to the whisper decoder as initial prompt
+    # these are prepended to any existing text context from a previous call
+    # use whisper_tokenize() to convert text to tokens.
+    # Maximum of whisper_n_text_ctx()/2 tokens are used (typically 224).
+    #
+    def initial_prompt: () -> (String | nil)
+
+    def diarize=: (boolish) -> boolish
+
+    # If true, enables diarization.
+    #
+    def diarize: () -> (true | false)
+
+    def offset=: (Integer) -> Integer
+
+    # Start offset in ms.
+    #
+    def offset: () -> Integer
+
+    def duration=: (Integer) -> Integer
+
+    # Audio duration to process in ms.
+    #
+    def duration: () -> Integer
+
+    def max_text_tokens=: (Integer) -> Integer
+
+    # Max tokens to use from past text as prompt for the decoder.
+    #
+    def max_text_tokens: () -> Integer
+
+    def temperature=: (Float) -> Float
+    def temperature: () -> Float
+    def max_initial_ts=: (Float) -> Float
+
+    # See https://github.com/openai/whisper/blob/f82bc59f5ea234d4b97fb2860842ed38519f7e65/whisper/decoding.py#L97
+    #
+    def max_initial_ts: () -> Float
+
+    def length_penalty=: (Float) -> Float
+    def length_penalty: () -> Float
+    def temperature_inc=: (Float) -> Float
+    def temperature_inc: () -> Float
+    def entropy_thold=: (Float) -> Float
+
+    # Similar to OpenAI's "compression_ratio_threshold"
+    #
+    def entropy_thold: () -> Float
+
+    def logprob_thold=: (Float) -> Float
+    def logprob_thold: () -> Float
+    def no_speech_thold=: (Float) -> Float
+    def no_speech_thold: () -> Float
+
+    # Sets new segment callback, called for every newly generated text segment.
+    #
+    #   params.new_segment_callback = ->(context, _, n_new, user_data) {
+    #     # ...
+    #   }
+    #
+    def new_segment_callback=: (new_segment_callback) -> new_segment_callback
+    def new_segment_callback: () -> (new_segment_callback | nil)
+
+    # Sets user data passed to the last argument of new segment callback.
+    #
+    def new_segment_callback_user_data=: (Object) -> Object
+
+    def new_segment_callback_user_data: () -> Object
+
+    # Sets progress callback, called on each progress update.
+    #
+    #   params.new_segment_callback = ->(context, _, progress, user_data) {
+    #     # ...
+    #   }
+    #
+    # +progress+ is an Integer between 0 and 100.
+    #
+    def progress_callback=: (progress_callback) -> progress_callback
+
+    def progress_callback: () -> (progress_callback | nil)
+
+    # Sets user data passed to the last argument of progress callback.
+    #
+    def progress_callback_user_data=: (Object) -> Object
+
+    def progress_callback_user_data: () -> Object
+
+    # Sets abort callback, called to check if the process should be aborted.
+    #
+    #   params.abort_callback = ->(user_data) {
+    #     # ...
+    #   }
+    #
+    #
+    def abort_callback=: (abort_callback) -> abort_callback
+
+    def abort_callback: () -> (abort_callback | nil)
+
+    # Sets user data passed to the last argument of abort callback.
+    #
+    def abort_callback_user_data=: (Object) -> Object
+
+    def abort_callback_user_data: () -> Object
+
+    # Hook called on new segment. Yields each Whisper::Segment.
+    #
+    #   whisper.on_new_segment do |segment|
+    #     # ...
+    #   end
+    #
+    def on_new_segment: { (Segment) -> void } -> void
+
+    # Hook called on progress update. Yields each progress Integer between 0 and 100.
+    #
+    def on_progress: { (Integer progress) -> void } -> void
+
+    # Call block to determine whether abort or not. Return +true+ when you want to abort.
+    #
+    #   params.abort_on do
+    #     if some_condition
+    #       true # abort
+    #     else
+    #       false # continue
+    #     end
+    #   end
+    #
+    def abort_on: { (Object user_data) -> boolish } -> void
+  end
+
+  class Model
+    def self.pre_converted_models: () -> Hash[String, Model::URI]
+    def self.new: () -> instance
+    def n_vocab: () -> Integer
+    def n_audio_ctx: () -> Integer
+    def n_audio_state: () -> Integer
+    def n_audio_head: () -> Integer
+    def n_audio_layer: () -> Integer
+    def n_text_ctx: () -> Integer
+    def n_text_state: () -> Integer
+    def n_text_head: () -> Integer
+    def n_text_layer: () -> Integer
+    def n_mels: () -> Integer
+    def ftype: () -> Integer
+    def type: () -> String
+
+    class URI
+      def self.new: (string | ::URI::HTTP) -> instance
+      def to_path: -> String
+      def clear_cache: -> void
+    end
+  end
+
+  class Segment
+    # Start time in milliseconds.
+    #
+    def start_time: () -> Integer
+
+    # End time in milliseconds.
+    #
+    def end_time: () -> Integer
+
+    # Whether the next segment is predicted as a speaker turn.
+    def speaker_next_turn?: () -> (true | false)
+
+    def text: () -> String
+    def no_speech_prob: () -> Float
+  end
+
+  class Error < StandardError
+    attr_reader code: Integer
+
+    def self.new: (Integer code) -> instance
+  end
+end

bindings/ruby/tests/helper.rb

@@ -4,4 +4,32 @@ require_relative "jfk_reader/jfk_reader"
 
 class TestBase < Test::Unit::TestCase
   AUDIO = File.join(__dir__, "..", "..", "..", "samples", "jfk.wav")
+
+  class << self
+    attr_reader :whisper
+
+    def startup
+      @whisper = Whisper::Context.new("base.en")
+      params = Whisper::Params.new
+      params.print_timestamps = false
+      @whisper.transcribe(TestBase::AUDIO, params)
+    end
+  end
+
+  private
+
+  def whisper
+    self.class.whisper
+  end
+
+  module BuildOptions
+    load "ext/options.rb", self
+    Options.include self
+
+    def enable_config(name)
+    end
+
+    def arg_config(name)
+    end
+  end
 end

bindings/ruby/tests/test_callback.rb

@@ -25,7 +25,7 @@ class TestCallback < TestBase
         assert start_time >= 0
         assert_kind_of Integer, end_time
         assert end_time > 0
-        assert_match /ask not what your country can do for you, ask what you can do for your country/, text if i_segment == 0
+        assert_match(/ask not what your country can do for you, ask what you can do for your country/, text) if i_segment == 0
       end
     }
 
@@ -145,9 +145,9 @@ class TestCallback < TestBase
 
   def test_abort_on
     do_abort = false
-    aborted_from_callback = false
+    _aborted_from_callback = false
     @params.on_new_segment do |segment|
-      do_abort = true if segment.text.match? /ask/
+      do_abort = true if segment.text.match?(/ask/)
     end
     i = 0
     @params.abort_on do

bindings/ruby/tests/test_error.rb

@@ -4,7 +4,7 @@ class TestError < TestBase
   def test_error
     error = Whisper::Error.new(-2)
     assert_equal "failed to compute log mel spectrogram", error.message
-    assert_equal -2, error.code
+    assert_equal(-2, error.code)
   end
 
   def test_unknown_error
@@ -14,7 +14,7 @@ class TestError < TestBase
 
   def test_non_int_code
     assert_raise TypeError do
-      error = Whisper::Error.new("non int")
+      _error = Whisper::Error.new("non int")
     end
   end
 end

bindings/ruby/tests/test_model.rb

@@ -68,4 +68,42 @@ class TestModel < TestBase
     assert_path_exist path
     assert_equal 147964211, File.size(path)
   end
+
+  def test_uri_string
+    path = "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin"
+    whisper = Whisper::Context.new(path)
+    model = whisper.model
+
+    assert_equal 51864, model.n_vocab
+    assert_equal 1500, model.n_audio_ctx
+    assert_equal 512, model.n_audio_state
+    assert_equal 8, model.n_audio_head
+    assert_equal 6, model.n_audio_layer
+    assert_equal 448, model.n_text_ctx
+    assert_equal 512, model.n_text_state
+    assert_equal 8, model.n_text_head
+    assert_equal 6, model.n_text_layer
+    assert_equal 80, model.n_mels
+    assert_equal 1, model.ftype
+    assert_equal "base", model.type
+  end
+
+  def test_uri
+    path = URI("https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin")
+    whisper = Whisper::Context.new(path)
+    model = whisper.model
+
+    assert_equal 51864, model.n_vocab
+    assert_equal 1500, model.n_audio_ctx
+    assert_equal 512, model.n_audio_state
+    assert_equal 8, model.n_audio_head
+    assert_equal 6, model.n_audio_layer
+    assert_equal 448, model.n_text_ctx
+    assert_equal 512, model.n_text_state
+    assert_equal 8, model.n_text_head
+    assert_equal 6, model.n_text_layer
+    assert_equal 80, model.n_mels
+    assert_equal 1, model.ftype
+    assert_equal "base", model.type
+  end
 end

bindings/ruby/tests/test_package.rb

@@ -21,11 +21,26 @@ class TestPackage < TestBase
       match_data = `rake -Tbuild`.match(/(whispercpp-(.+)\.gem)/)
       filename = match_data[1]
       version = match_data[2]
-      basename = "whisper.#{RbConfig::CONFIG["DLEXT"]}"
       Dir.mktmpdir do |dir|
-        system "gem", "install", "--install-dir", dir.shellescape, "pkg/#{filename.shellescape}", exception: true
+        system "gem", "install", "--install-dir", dir.shellescape, "--no-document", "pkg/#{filename.shellescape}", exception: true
-        assert_path_exist File.join(dir, "gems/whispercpp-#{version}/lib", basename)
+        assert_installed dir, version
       end
     end
+
+    private
+
+    def assert_installed(dir, version)
+      assert_path_exist File.join(dir, "gems/whispercpp-#{version}/lib", "whisper.#{RbConfig::CONFIG["DLEXT"]}")
+      assert_path_exist File.join(dir, "gems/whispercpp-#{version}/LICENSE")
+      assert_path_not_exist File.join(dir, "gems/whispercpp-#{version}/ext/build")
+    end
+  end
+
+  def test_build_options
+    options = BuildOptions::Options.new
+    assert_empty options.missing_options
+    unless ENV["CI"]
+      assert_empty options.extra_options
+    end
   end
 end

bindings/ruby/tests/test_params.rb

@@ -1,6 +1,39 @@
 require_relative "helper"
 
 class TestParams < TestBase
+  PARAM_NAMES = [
+    :language,
+    :translate,
+    :no_context,
+    :single_segment,
+    :print_special,
+    :print_progress,
+    :print_realtime,
+    :print_timestamps,
+    :suppress_blank,
+    :suppress_nst,
+    :token_timestamps,
+    :split_on_word,
+    :initial_prompt,
+    :diarize,
+    :offset,
+    :duration,
+    :max_text_tokens,
+    :temperature,
+    :max_initial_ts,
+    :length_penalty,
+    :temperature_inc,
+    :entropy_thold,
+    :logprob_thold,
+    :no_speech_thold,
+    :new_segment_callback,
+    :new_segment_callback_user_data,
+    :progress_callback,
+    :progress_callback_user_data,
+    :abort_callback,
+    :abort_callback_user_data,
+  ]
+
   def setup
     @params  = Whisper::Params.new
   end
@@ -89,11 +122,11 @@ class TestParams < TestBase
     assert !@params.suppress_blank
   end
 
-  def test_suppress_non_speech_tokens
+  def test_suppress_nst
-    @params.suppress_non_speech_tokens = true
+    @params.suppress_nst = true
-    assert @params.suppress_non_speech_tokens
+    assert @params.suppress_nst
-    @params.suppress_non_speech_tokens = false
+    @params.suppress_nst = false
-    assert !@params.suppress_non_speech_tokens
+    assert !@params.suppress_nst
   end
 
   def test_token_timestamps
@@ -129,7 +162,7 @@ class TestParams < TestBase
   end
 
   def test_length_penalty
-    assert_equal -1.0, @params.length_penalty
+    assert_equal(-1.0, @params.length_penalty)
     @params.length_penalty = 0.5
     assert_equal 0.5, @params.length_penalty
   end
@@ -147,9 +180,9 @@ class TestParams < TestBase
   end
 
   def test_logprob_thold
-    assert_in_delta -1.0, @params.logprob_thold
+    assert_in_delta(-1.0, @params.logprob_thold)
     @params.logprob_thold = -0.5
-    assert_in_delta -0.5, @params.logprob_thold
+    assert_in_delta(-0.5, @params.logprob_thold)
   end
 
   def test_no_speech_thold
@@ -157,4 +190,57 @@ class TestParams < TestBase
     @params.no_speech_thold = 0.2
     assert_in_delta 0.2, @params.no_speech_thold
   end
+
+  def test_new_with_kw_args
+    params = Whisper::Params.new(language: "es")
+    assert_equal "es", params.language
+    assert_equal 1.0, params.max_initial_ts
+  end
+
+  def test_new_with_kw_args_non_existent
+    assert_raise ArgumentError do
+      Whisper::Params.new(non_existent: "value")
+    end
+  end
+
+  def test_new_with_kw_args_wrong_type
+    assert_raise TypeError do
+      Whisper::Params.new(language: 3)
+    end
+  end
+
+  data(PARAM_NAMES.collect {|param| [param, param]}.to_h)
+  def test_new_with_kw_args_default_values(param)
+    default_value = @params.send(param)
+    value = case [param, default_value]
+            in [*, true | false]
+              !default_value
+            in [*, Integer | Float]
+              default_value + 1
+            in [:language, *]
+              "es"
+            in [:initial_prompt, *]
+              "Initial prompt"
+            in [/_callback\Z/, *]
+              proc {}
+            in [/_user_data\Z/, *]
+              Object.new
+            end
+    params = Whisper::Params.new(param => value)
+    if Float === value
+      assert_in_delta value, params.send(param)
+    else
+      assert_equal value, params.send(param)
+    end
+
+    PARAM_NAMES.reject {|name| name == param}.each do |name|
+      expected = @params.send(name)
+      actual = params.send(name)
+      if Float === expected
+        assert_in_delta expected, actual
+      else
+        assert_equal expected, actual
+      end
+    end
+  end
 end

bindings/ruby/tests/test_segment.rb

@@ -1,17 +1,6 @@
 require_relative "helper"
 
 class TestSegment < TestBase
-  class << self
-    attr_reader :whisper
-
-    def startup
-      @whisper = Whisper::Context.new("base.en")
-      params = Whisper::Params.new
-      params.print_timestamps = false
-      @whisper.transcribe(TestBase::AUDIO, params)
-    end
-  end
-
   def test_iteration
     whisper.each_segment do |segment|
       assert_instance_of Whisper::Segment, segment
@@ -43,6 +32,14 @@ class TestSegment < TestBase
     end
   end
 
+  def test_no_speech_prob
+    no_speech_prob = nil
+    whisper.each_segment do |segment|
+      no_speech_prob = segment.no_speech_prob
+    end
+    assert no_speech_prob > 0.0
+  end
+
   def test_on_new_segment
     params = Whisper::Params.new
     seg = nil
@@ -52,13 +49,13 @@ class TestSegment < TestBase
       if index == 0
         seg = segment
         assert_equal 0, segment.start_time
-        assert_match /ask not what your country can do for you, ask what you can do for your country/, segment.text
+        assert_match(/ask not what your country can do for you, ask what you can do for your country/, segment.text)
       end
       index += 1
     end
     whisper.transcribe(AUDIO, params)
     assert_equal 0, seg.start_time
-    assert_match /ask not what your country can do for you, ask what you can do for your country/, seg.text
+    assert_match(/ask not what your country can do for you, ask what you can do for your country/, seg.text)
   end
 
   def test_on_new_segment_twice
@@ -74,10 +71,4 @@ class TestSegment < TestBase
     end
     whisper.transcribe(AUDIO, params)
   end
-
-  private
-
-  def whisper
-    self.class.whisper
-  end
 end

bindings/ruby/tests/test_whisper.rb

@@ -16,26 +16,11 @@ class TestWhisper < TestBase
     params.print_timestamps = false
 
     @whisper.transcribe(AUDIO, params) {|text|
-      assert_match /ask not what your country can do for you, ask what you can do for your country/, text
+      assert_match(/ask not what your country can do for you, ask what you can do for your country/, text)
     }
   end
 
   sub_test_case "After transcription" do
-    class << self
-      attr_reader :whisper
-
-      def startup
-        @whisper = Whisper::Context.new("base.en")
-        params = Whisper::Params.new
-        params.print_timestamps = false
-        @whisper.transcribe(TestBase::AUDIO, params)
-      end
-    end
-
-    def whisper
-      self.class.whisper
-    end
-
     def test_full_n_segments
       assert_equal 1, whisper.full_n_segments
     end
@@ -44,6 +29,12 @@ class TestWhisper < TestBase
       assert_equal 0, whisper.full_lang_id
     end
 
+    def test_full_get_segment
+      segment = whisper.full_get_segment(0)
+      assert_equal 0, segment.start_time
+      assert_match(/ask not what your country can do for you, ask what you can do for your country/, segment.text)
+    end
+
     def test_full_get_segment_t0
       assert_equal 0, whisper.full_get_segment_t0(0)
       assert_raise IndexError do
@@ -68,7 +59,13 @@ class TestWhisper < TestBase
     end
 
     def test_full_get_segment_text
-      assert_match /ask not what your country can do for you, ask what you can do for your country/, whisper.full_get_segment_text(0)
+      assert_match(/ask not what your country can do for you, ask what you can do for your country/, whisper.full_get_segment_text(0))
+    end
+
+    def test_full_get_segment_no_speech_prob
+      prob = whisper.full_get_segment_no_speech_prob(0)
+      assert prob > 0.0
+      assert prob < 1.0
     end
   end
 
@@ -137,14 +134,14 @@ class TestWhisper < TestBase
       @whisper.full(@params, @samples, @samples.length)
 
       assert_equal 1, @whisper.full_n_segments
-      assert_match /ask not what your country can do for you, ask what you can do for your country/, @whisper.each_segment.first.text
+      assert_match(/ask not what your country can do for you, ask what you can do for your country/, @whisper.each_segment.first.text)
     end
 
     def test_full_without_length
       @whisper.full(@params, @samples)
 
       assert_equal 1, @whisper.full_n_segments
-      assert_match /ask not what your country can do for you, ask what you can do for your country/, @whisper.each_segment.first.text
+      assert_match(/ask not what your country can do for you, ask what you can do for your country/, @whisper.each_segment.first.text)
     end
 
     def test_full_enumerator
@@ -152,7 +149,7 @@ class TestWhisper < TestBase
       @whisper.full(@params, samples, @samples.length)
 
       assert_equal 1, @whisper.full_n_segments
-      assert_match /ask not what your country can do for you, ask what you can do for your country/, @whisper.each_segment.first.text
+      assert_match(/ask not what your country can do for you, ask what you can do for your country/, @whisper.each_segment.first.text)
     end
 
     def test_full_enumerator_without_length
@@ -174,26 +171,28 @@ class TestWhisper < TestBase
       @whisper.full(@params, samples)
 
       assert_equal 1, @whisper.full_n_segments
-      assert_match /ask not what your country can do for you, ask what you can do for your country/, @whisper.each_segment.first.text
+      assert_match(/ask not what your country can do for you, ask what you can do for your country/, @whisper.each_segment.first.text)
     end
 
     def test_full_parallel
-      @whisper.full_parallel(@params, @samples, @samples.length, Etc.nprocessors)
+      nprocessors = 2
+      @whisper.full_parallel(@params, @samples, @samples.length, nprocessors)
 
-      assert_equal Etc.nprocessors, @whisper.full_n_segments
+      assert_equal nprocessors, @whisper.full_n_segments
       text = @whisper.each_segment.collect(&:text).join
-      assert_match /ask what you can do/i, text
+      assert_match(/ask what you can do/i, text)
-      assert_match /for your country/i, text
+      assert_match(/for your country/i, text)
     end
 
     def test_full_parallel_with_memory_view
+      nprocessors = 2
       samples = JFKReader.new(AUDIO)
-      @whisper.full_parallel(@params, samples, nil, Etc.nprocessors)
+      @whisper.full_parallel(@params, samples, nil, nprocessors)
 
-      assert_equal Etc.nprocessors, @whisper.full_n_segments
+      assert_equal nprocessors, @whisper.full_n_segments
       text = @whisper.each_segment.collect(&:text).join
-      assert_match /ask what you can do/i, text
+      assert_match(/ask what you can do/i, text)
-      assert_match /for your country/i, text
+      assert_match(/for your country/i, text)
     end
 
     def test_full_parallel_without_length_and_n_processors
@@ -201,17 +200,18 @@ class TestWhisper < TestBase
 
       assert_equal 1, @whisper.full_n_segments
       text = @whisper.each_segment.collect(&:text).join
-      assert_match /ask what you can do/i, text
+      assert_match(/ask what you can do/i, text)
-      assert_match /for your country/i, text
+      assert_match(/for your country/i, text)
     end
 
     def test_full_parallel_without_length
-      @whisper.full_parallel(@params, @samples, nil, Etc.nprocessors)
+      nprocessors = 2
+      @whisper.full_parallel(@params, @samples, nil, nprocessors)
 
-      assert_equal Etc.nprocessors, @whisper.full_n_segments
+      assert_equal nprocessors, @whisper.full_n_segments
       text = @whisper.each_segment.collect(&:text).join
-      assert_match /ask what you can do/i, text
+      assert_match(/ask what you can do/i, text)
-      assert_match /for your country/i, text
+      assert_match(/for your country/i, text)
     end
 
     def test_full_parallel_without_n_processors
@@ -219,8 +219,8 @@ class TestWhisper < TestBase
 
       assert_equal 1, @whisper.full_n_segments
       text = @whisper.each_segment.collect(&:text).join
-      assert_match /ask what you can do/i, text
+      assert_match(/ask what you can do/i, text)
-      assert_match /for your country/i, text
+      assert_match(/for your country/i, text)
     end
   end
 end

bindings/ruby/whispercpp.gemspec

@@ -3,8 +3,8 @@ require_relative "extsources"
 Gem::Specification.new do |s|
   s.name    = "whispercpp"
   s.authors = ["Georgi Gerganov", "Todd A. Fisher"]
-  s.version = '1.3.0'
+  s.version = '1.3.2'
-  s.date    = '2024-05-14'
+  s.date    = '2025-04-17'
   s.description = %q{High-performance inference of OpenAI's Whisper automatic speech recognition (ASR) model via Ruby}
   s.email   = 'todd.fisher@gmail.com'
   s.extra_rdoc_files = ['LICENSE', 'README.md']
@@ -15,7 +15,8 @@ Gem::Specification.new do |s|
                 if s.extra_rdoc_files.include?(basename)
                   basename
                 else
-                  file.sub("../..", "ext")
+                  file.sub("../..", "ext/sources")
+                      .sub("../javascript", "ext/sources/bindings/javascript")
                 end
               }
 
@@ -26,7 +27,7 @@ Gem::Specification.new do |s|
   s.required_ruby_version = '>= 3.1.0'
 
   #### Documentation and testing.
-  s.homepage = 'https://github.com/ggerganov/whisper.cpp'
+  s.homepage = 'https://github.com/ggml-org/whisper.cpp'
   s.rdoc_options = ['--main', 'README.md']
 
 

build-xcframework.sh

@@ -0,0 +1,547 @@
+#!/bin/bash
+#
+# Options
+IOS_MIN_OS_VERSION=16.4
+MACOS_MIN_OS_VERSION=13.3
+VISIONOS_MIN_OS_VERSION=1.0
+TVOS_MIN_OS_VERSION=16.4
+
+BUILD_SHARED_LIBS=OFF
+WHISPER_BUILD_EXAMPLES=OFF
+WHISPER_BUILD_TESTS=OFF
+WHISPER_BUILD_SERVER=OFF
+GGML_METAL=ON
+GGML_METAL_EMBED_LIBRARY=ON
+GGML_BLAS_DEFAULT=ON
+GGML_METAL_USE_BF16=ON
+GGML_OPENMP=OFF
+
+COMMON_C_FLAGS="-Wno-macro-redefined -Wno-shorten-64-to-32 -Wno-unused-command-line-argument -g"
+COMMON_CXX_FLAGS="-Wno-macro-redefined -Wno-shorten-64-to-32 -Wno-unused-command-line-argument -g"
+
+# Common options for all builds
+COMMON_CMAKE_ARGS=(
+    -DCMAKE_XCODE_ATTRIBUTE_CODE_SIGNING_REQUIRED=NO
+    -DCMAKE_XCODE_ATTRIBUTE_CODE_SIGN_IDENTITY=""
+    -DCMAKE_XCODE_ATTRIBUTE_CODE_SIGNING_ALLOWED=NO
+    -DCMAKE_XCODE_ATTRIBUTE_DEBUG_INFORMATION_FORMAT="dwarf-with-dsym"
+    -DCMAKE_XCODE_ATTRIBUTE_GCC_GENERATE_DEBUGGING_SYMBOLS=YES
+    -DCMAKE_XCODE_ATTRIBUTE_COPY_PHASE_STRIP=NO
+    -DCMAKE_XCODE_ATTRIBUTE_STRIP_INSTALLED_PRODUCT=NO
+    -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
+    -DBUILD_SHARED_LIBS=${BUILD_SHARED_LIBS}
+    -DWHISPER_BUILD_EXAMPLES=${WHISPER_BUILD_EXAMPLES}
+    -DWHISPER_BUILD_TESTS=${WHISPER_BUILD_TESTS}
+    -DWHISPER_BUILD_SERVER=${WHISPER_BUILD_SERVER}
+    -DGGML_METAL_EMBED_LIBRARY=${GGML_METAL_EMBED_LIBRARY}
+    -DGGML_BLAS_DEFAULT=${GGML_BLAS_DEFAULT}
+    -DGGML_METAL=${GGML_METAL}
+    -DGGML_METAL_USE_BF16=${GGML_METAL_USE_BF16}
+    -DGGML_NATIVE=OFF
+    -DGGML_OPENMP=${GGML_OPENMP}
+)
+
+XCODE_VERSION=$(xcodebuild -version 2>/dev/null | head -n1 | awk '{ print $2 }')
+MAJOR_VERSION=$(echo $XCODE_VERSION | cut -d. -f1)
+MINOR_VERSION=$(echo $XCODE_VERSION | cut -d. -f2)
+echo "Detected Xcode version: $XCODE_VERSION"
+
+check_required_tool() {
+    local tool=$1
+    local install_message=$2
+
+    if ! command -v $tool &> /dev/null; then
+        echo "Error: $tool is required but not found."
+        echo "$install_message"
+        exit 1
+    fi
+}
+echo "Checking for required tools..."
+check_required_tool "cmake" "Please install CMake 3.28.0 or later (brew install cmake)"
+check_required_tool "xcodebuild" "Please install Xcode and Xcode Command Line Tools (xcode-select --install)"
+check_required_tool "libtool" "Please install libtool which should be available with Xcode Command Line Tools (CLT). Make sure Xcode CLT is installed (xcode-select --install)"
+check_required_tool "dsymutil" "Please install Xcode and Xcode Command Line Tools (xcode-select --install)"
+
+set -e
+
+## Clean up previous builds
+rm -rf build-apple
+rm -rf build-ios-sim
+rm -rf build-ios-device
+rm -rf build-macos
+rm -rf build-visionos
+rm -rf build-visionos-sim
+rm -rf build-tvos-sim
+rm -rf build-tvos-device
+
+# Setup the xcframework build directory structure
+setup_framework_structure() {
+    local build_dir=$1
+    local min_os_version=$2
+    local platform=$3  # "ios", "macos", "visionos", or "tvos"
+    local framework_name="whisper"
+
+    echo "Creating ${platform}-style framework structure for ${build_dir}"
+
+    if [[ "$platform" == "macos" ]]; then
+        # macOS versioned structure uses versioned directories
+        mkdir -p ${build_dir}/framework/${framework_name}.framework/Versions/A/Headers
+        mkdir -p ${build_dir}/framework/${framework_name}.framework/Versions/A/Modules
+        mkdir -p ${build_dir}/framework/${framework_name}.framework/Versions/A/Resources
+
+        # Create symbolic links
+        ln -sf A ${build_dir}/framework/${framework_name}.framework/Versions/Current
+        ln -sf Versions/Current/Headers ${build_dir}/framework/${framework_name}.framework/Headers
+        ln -sf Versions/Current/Modules ${build_dir}/framework/${framework_name}.framework/Modules
+        ln -sf Versions/Current/Resources ${build_dir}/framework/${framework_name}.framework/Resources
+        ln -sf Versions/Current/${framework_name} ${build_dir}/framework/${framework_name}.framework/${framework_name}
+
+        # Set header and module paths
+        local header_path=${build_dir}/framework/${framework_name}.framework/Versions/A/Headers/
+        local module_path=${build_dir}/framework/${framework_name}.framework/Versions/A/Modules/
+    else
+        # iOS/VisionOS/tvOS use a flat structure
+        mkdir -p ${build_dir}/framework/${framework_name}.framework/Headers
+        mkdir -p ${build_dir}/framework/${framework_name}.framework/Modules
+
+        # Remove any existing structure to ensure clean build
+        rm -rf ${build_dir}/framework/${framework_name}.framework/Versions
+
+        # Set header and module paths
+        local header_path=${build_dir}/framework/${framework_name}.framework/Headers/
+        local module_path=${build_dir}/framework/${framework_name}.framework/Modules/
+    fi
+
+    # Copy all required headers (common for all platforms)
+    cp include/whisper.h           ${header_path}
+    cp ggml/include/ggml.h         ${header_path}
+    cp ggml/include/ggml-alloc.h   ${header_path}
+    cp ggml/include/ggml-backend.h ${header_path}
+    cp ggml/include/ggml-metal.h   ${header_path}
+    cp ggml/include/ggml-cpu.h     ${header_path}
+    cp ggml/include/ggml-blas.h    ${header_path}
+    cp ggml/include/gguf.h         ${header_path}
+
+    # Create module map (common for all platforms)
+    cat > ${module_path}module.modulemap << EOF
+framework module whisper {
+    header "whisper.h"
+    header "ggml.h"
+    header "ggml-alloc.h"
+    header "ggml-backend.h"
+    header "ggml-metal.h"
+    header "ggml-cpu.h"
+    header "ggml-blas.h"
+    header "gguf.h"
+
+    link "c++"
+    link framework "Accelerate"
+    link framework "Metal"
+    link framework "Foundation"
+
+    export *
+}
+EOF
+
+    # Platform-specific settings for Info.plist
+    local platform_name=""
+    local sdk_name=""
+    local supported_platform=""
+
+    case "$platform" in
+        "ios")
+            platform_name="iphoneos"
+            sdk_name="iphoneos${min_os_version}"
+            supported_platform="iPhoneOS"
+            local plist_path="${build_dir}/framework/${framework_name}.framework/Info.plist"
+            local device_family='    <key>UIDeviceFamily</key>
+    <array>
+        <integer>1</integer>
+        <integer>2</integer>
+    </array>'
+            ;;
+        "macos")
+            platform_name="macosx"
+            sdk_name="macosx${min_os_version}"
+            supported_platform="MacOSX"
+            local plist_path="${build_dir}/framework/${framework_name}.framework/Versions/A/Resources/Info.plist"
+            local device_family=""
+            ;;
+        "visionos")
+            platform_name="xros"
+            sdk_name="xros${min_os_version}"
+            supported_platform="XRPlatform"
+            local plist_path="${build_dir}/framework/${framework_name}.framework/Info.plist"
+            local device_family=""
+            ;;
+        "tvos")
+            platform_name="appletvos"
+            sdk_name="appletvos${min_os_version}"
+            supported_platform="AppleTVOS"
+            local plist_path="${build_dir}/framework/${framework_name}.framework/Info.plist"
+            local device_family='    <key>UIDeviceFamily</key>
+    <array>
+        <integer>3</integer>
+    </array>'
+            ;;
+    esac
+
+    # Create Info.plist
+    cat > ${plist_path} << EOF
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+    <key>CFBundleDevelopmentRegion</key>
+    <string>en</string>
+    <key>CFBundleExecutable</key>
+    <string>whisper</string>
+    <key>CFBundleIdentifier</key>
+    <string>org.ggml.whisper</string>
+    <key>CFBundleInfoDictionaryVersion</key>
+    <string>6.0</string>
+    <key>CFBundleName</key>
+    <string>whisper</string>
+    <key>CFBundlePackageType</key>
+    <string>FMWK</string>
+    <key>CFBundleShortVersionString</key>
+    <string>1.0</string>
+    <key>CFBundleVersion</key>
+    <string>1</string>
+    <key>MinimumOSVersion</key>
+    <string>${min_os_version}</string>
+    <key>CFBundleSupportedPlatforms</key>
+    <array>
+        <string>${supported_platform}</string>
+    </array>${device_family}
+    <key>DTPlatformName</key>
+    <string>${platform_name}</string>
+    <key>DTSDKName</key>
+    <string>${sdk_name}</string>
+</dict>
+</plist>
+EOF
+}
+
+# Create dynamic libraries from static libraries.
+combine_static_libraries() {
+    local build_dir="$1"
+    local release_dir="$2"
+    local platform="$3"  # "ios", "macos", "visionos", or "tvos"
+    local is_simulator="$4"
+    local base_dir="$(pwd)"
+    local framework_name="whisper"
+
+    # Determine output path based on platform
+    local output_lib=""
+    if [[ "$platform" == "macos" ]]; then
+        # macOS uses versioned structure
+        output_lib="${build_dir}/framework/${framework_name}.framework/Versions/A/${framework_name}"
+    else
+        # iOS, visionOS, and tvOS use a directory flat structure
+        output_lib="${build_dir}/framework/${framework_name}.framework/${framework_name}"
+    fi
+
+    local libs=(
+        "${base_dir}/${build_dir}/src/${release_dir}/libwhisper.a"
+        "${base_dir}/${build_dir}/ggml/src/${release_dir}/libggml.a"
+        "${base_dir}/${build_dir}/ggml/src/${release_dir}/libggml-base.a"
+        "${base_dir}/${build_dir}/ggml/src/${release_dir}/libggml-cpu.a"
+        "${base_dir}/${build_dir}/ggml/src/ggml-metal/${release_dir}/libggml-metal.a"
+        "${base_dir}/${build_dir}/ggml/src/ggml-blas/${release_dir}/libggml-blas.a"
+    )
+    if [[ "$platform" == "macos" || "$platform" == "ios" ]]; then
+        echo "Adding libwhisper.coreml library to the build."
+        libs+=(
+            "${base_dir}/${build_dir}/src/${release_dir}/libwhisper.coreml.a"
+        )
+    fi
+
+    # Create temporary directory for processing
+    local temp_dir="${base_dir}/${build_dir}/temp"
+    echo "Creating temporary directory: ${temp_dir}"
+    mkdir -p "${temp_dir}"
+
+    # Since we have multiple architectures libtool will find object files that do not
+    # match the target architecture. We suppress these warnings.
+    libtool -static -o "${temp_dir}/combined.a" "${libs[@]}" 2> /dev/null
+
+    # Determine SDK, architectures, and install_name based on platform and simulator flag.
+    local sdk=""
+    local archs=""
+    local min_version_flag=""
+    local install_name=""
+    local frameworks="-framework Foundation -framework Metal -framework Accelerate"
+
+    case "$platform" in
+        "ios")
+            if [[ "$is_simulator" == "true" ]]; then
+                sdk="iphonesimulator"
+                archs="arm64 x86_64"
+                min_version_flag="-mios-simulator-version-min=${IOS_MIN_OS_VERSION}"
+            else
+                sdk="iphoneos"
+                archs="arm64"
+                min_version_flag="-mios-version-min=${IOS_MIN_OS_VERSION}"
+            fi
+            install_name="@rpath/whisper.framework/whisper"
+            frameworks+=" -framework CoreML"
+            ;;
+        "macos")
+            sdk="macosx"
+            archs="arm64 x86_64"
+            min_version_flag="-mmacosx-version-min=${MACOS_MIN_OS_VERSION}"
+            install_name="@rpath/whisper.framework/Versions/Current/whisper"
+            frameworks+=" -framework CoreML"
+            ;;
+        "visionos")
+            if [[ "$is_simulator" == "true" ]]; then
+                sdk="xrsimulator"
+                archs="arm64 x86_64"
+                min_version_flag="-mtargetos=xros${VISIONOS_MIN_OS_VERSION}-simulator"
+            else
+                sdk="xros"
+                archs="arm64"
+                min_version_flag="-mtargetos=xros${VISIONOS_MIN_OS_VERSION}"
+            fi
+            # Use flat structure for visionOS, same as iOS
+            install_name="@rpath/whisper.framework/whisper"
+            ;;
+        "tvos")
+            if [[ "$is_simulator" == "true" ]]; then
+                sdk="appletvsimulator"
+                archs="arm64 x86_64"
+                min_version_flag="-mtvos-simulator-version-min=${TVOS_MIN_OS_VERSION}"
+            else
+                sdk="appletvos"
+                archs="arm64"
+                min_version_flag="-mtvos-version-min=${TVOS_MIN_OS_VERSION}"
+            fi
+            install_name="@rpath/whisper.framework/whisper"
+            ;;
+    esac
+
+    # Build architecture flags
+    local arch_flags=""
+    for arch in $archs; do
+        arch_flags+=" -arch $arch"
+    done
+
+    # Create dynamic library
+    echo "Creating dynamic library for ${platform}."
+    xcrun -sdk $sdk clang++ -dynamiclib \
+        -isysroot $(xcrun --sdk $sdk --show-sdk-path) \
+        $arch_flags \
+        $min_version_flag \
+        -Wl,-force_load,"${temp_dir}/combined.a" \
+        $frameworks \
+        -install_name "$install_name" \
+        -o "${base_dir}/${output_lib}"
+
+    # Platform-specific post-processing for device builds
+    if [[ "$is_simulator" == "false" ]]; then
+        if command -v xcrun vtool &>/dev/null; then
+            case "$platform" in
+                "ios")
+                    echo "Marking binary as a framework binary for iOS..."
+                    xcrun vtool -set-build-version ios ${IOS_MIN_OS_VERSION} ${IOS_MIN_OS_VERSION} -replace \
+                        -output "${base_dir}/${output_lib}" "${base_dir}/${output_lib}"
+                    ;;
+                "visionos")
+                    echo "Marking binary as a framework binary for visionOS..."
+                    if [[ "$MAJOR_VERSION" -gt 16 ]] || [[ "$MAJOR_VERSION" -eq 16 && "$MINOR_VERSION" -gt 2 ]]; then
+                        echo "Xcode version greater than 16.2, using visionOS."
+                        VISION_OS_BUILD_VERSION="visionos"
+                    else
+                        echo "Xcode version less than or equal to 16.2, using xros."
+                        VISION_OS_BUILD_VERSION="xros"
+                    fi
+                    xcrun vtool -set-build-version ${VISION_OS_BUILD_VERSION} ${VISIONOS_MIN_OS_VERSION} ${VISIONOS_MIN_OS_VERSION} -replace \
+                        -output "${base_dir}/${output_lib}" "${base_dir}/${output_lib}"
+                    ;;
+                "tvos")
+                    echo "Marking binary as a framework binary for tvOS..."
+                    xcrun vtool -set-build-version tvos ${TVOS_MIN_OS_VERSION} ${TVOS_MIN_OS_VERSION} -replace \
+                        -output "${base_dir}/${output_lib}" "${base_dir}/${output_lib}"
+                    ;;
+            esac
+        else
+            echo "Warning: vtool not found. Binary may not pass App Store validation."
+        fi
+    fi
+
+    echo "Creating properly formatted dSYM..."
+    # Create a separate directory for dSYMs for all platforms
+    mkdir -p "${base_dir}/${build_dir}/dSYMs"
+
+    # iOS and visionOS style dSYM (flat structure)
+    if [[ "$platform" == "ios" || "$platform" == "visionos" || "$platform" == "tvos" ]]; then
+        # Generate dSYM in the dSYMs directory
+        xcrun dsymutil "${base_dir}/${output_lib}" -o "${base_dir}/${build_dir}/dSYMs/whisper.dSYM"
+
+        # Create a copy of the binary that will be stripped
+        cp "${base_dir}/${output_lib}" "${temp_dir}/binary_to_strip"
+
+        # Strip debug symbols from the copy
+        xcrun strip -S "${temp_dir}/binary_to_strip" -o "${temp_dir}/stripped_lib"
+
+        # Replace the original with the stripped version
+        mv "${temp_dir}/stripped_lib" "${base_dir}/${output_lib}"
+    else
+        # macOS style dSYM
+        # First strip debug info to a separate file
+        xcrun strip -S "${base_dir}/${output_lib}" -o "${temp_dir}/stripped_lib"
+
+        # Generate dSYM in the dSYMs directory
+        xcrun dsymutil "${base_dir}/${output_lib}" -o "${base_dir}/${build_dir}/dSYMs/whisper.dSYM"
+
+        # Replace original binary with stripped version
+        mv "${temp_dir}/stripped_lib" "${base_dir}/${output_lib}"
+    fi
+
+    # Remove any automatically generated dSYM files in the framework structure as they will
+    # otherwise case Invalid Bundle Structure validation errors.
+    if [ -d "${base_dir}/${output_lib}.dSYM" ]; then
+        echo "Removing generated dSYM file in framework structure: ${base_dir}/${output_lib}.dSYM"
+        rm -rf "${base_dir}/${output_lib}.dSYM"
+    fi
+
+    # Clean up
+    rm -rf "${temp_dir}"
+}
+
+echo "Building for iOS simulator..."
+cmake -B build-ios-sim -G Xcode \
+    "${COMMON_CMAKE_ARGS[@]}" \
+    -DCMAKE_OSX_DEPLOYMENT_TARGET=${IOS_MIN_OS_VERSION} \
+    -DIOS=ON \
+    -DCMAKE_SYSTEM_NAME=iOS \
+    -DCMAKE_OSX_SYSROOT=iphonesimulator \
+    -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \
+    -DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=iphonesimulator \
+    -DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \
+    -DCMAKE_CXX_FLAGS="${COMMON_CXX_FLAGS}" \
+    -DWHISPER_COREML="ON" \
+    -DWHISPER_COREML_ALLOW_FALLBACK="ON" \
+    -S .
+cmake --build build-ios-sim --config Release -- -quiet
+
+echo "Building for iOS devices..."
+cmake -B build-ios-device -G Xcode \
+    "${COMMON_CMAKE_ARGS[@]}" \
+    -DCMAKE_OSX_DEPLOYMENT_TARGET=${IOS_MIN_OS_VERSION} \
+    -DCMAKE_OSX_SYSROOT=iphoneos \
+    -DCMAKE_OSX_ARCHITECTURES="arm64" \
+    -DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=iphoneos \
+    -DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \
+    -DCMAKE_CXX_FLAGS="${COMMON_CXX_FLAGS}" \
+    -DWHISPER_COREML="ON" \
+    -DWHISPER_COREML_ALLOW_FALLBACK="ON" \
+    -S .
+cmake --build build-ios-device --config Release -- -quiet
+
+echo "Building for macOS..."
+cmake -B build-macos -G Xcode \
+    "${COMMON_CMAKE_ARGS[@]}" \
+    -DCMAKE_OSX_DEPLOYMENT_TARGET=${MACOS_MIN_OS_VERSION} \
+    -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \
+    -DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \
+    -DCMAKE_CXX_FLAGS="${COMMON_CXX_FLAGS}" \
+    -DWHISPER_COREML="ON" \
+    -DWHISPER_COREML_ALLOW_FALLBACK="ON" \
+    -S .
+cmake --build build-macos --config Release -- -quiet
+
+echo "Building for visionOS..."
+cmake -B build-visionos -G Xcode \
+    "${COMMON_CMAKE_ARGS[@]}" \
+    -DCMAKE_OSX_DEPLOYMENT_TARGET=${VISIONOS_MIN_OS_VERSION} \
+    -DCMAKE_OSX_ARCHITECTURES="arm64" \
+    -DCMAKE_SYSTEM_NAME=visionOS \
+    -DCMAKE_OSX_SYSROOT=xros \
+    -DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=xros \
+    -DCMAKE_C_FLAGS="-D_XOPEN_SOURCE=700 ${COMMON_C_FLAGS}" \
+    -DCMAKE_CXX_FLAGS="-D_XOPEN_SOURCE=700 ${COMMON_CXX_FLAGS}" \
+    -S .
+cmake --build build-visionos --config Release -- -quiet
+
+echo "Building for visionOS simulator..."
+cmake -B build-visionos-sim -G Xcode \
+    "${COMMON_CMAKE_ARGS[@]}" \
+    -DCMAKE_OSX_DEPLOYMENT_TARGET=${VISIONOS_MIN_OS_VERSION} \
+    -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \
+    -DCMAKE_SYSTEM_NAME=visionOS \
+    -DCMAKE_OSX_SYSROOT=xrsimulator \
+    -DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=xrsimulator \
+    -DCMAKE_C_FLAGS="-D_XOPEN_SOURCE=700 ${COMMON_C_FLAGS}" \
+    -DCMAKE_CXX_FLAGS="-D_XOPEN_SOURCE=700 ${COMMON_CXX_FLAGS}" \
+    -S .
+cmake --build build-visionos-sim --config Release -- -quiet
+
+# Add tvOS builds (might need the same u_int definitions as watchOS and visionOS)
+echo "Building for tvOS simulator..."
+cmake -B build-tvos-sim -G Xcode \
+    "${COMMON_CMAKE_ARGS[@]}" \
+    -DCMAKE_OSX_DEPLOYMENT_TARGET=${TVOS_MIN_OS_VERSION} \
+    -DCMAKE_SYSTEM_NAME=tvOS \
+    -DCMAKE_OSX_SYSROOT=appletvsimulator \
+    -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \
+    -DGGML_METAL=ON \
+    -DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=appletvsimulator \
+    -DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \
+    -DCMAKE_CXX_FLAGS="${COMMON_CXX_FLAGS}" \
+    -S .
+cmake --build build-tvos-sim --config Release -- -quiet
+
+echo "Building for tvOS devices..."
+cmake -B build-tvos-device -G Xcode \
+    "${COMMON_CMAKE_ARGS[@]}" \
+    -DCMAKE_OSX_DEPLOYMENT_TARGET=${TVOS_MIN_OS_VERSION} \
+    -DCMAKE_SYSTEM_NAME=tvOS \
+    -DCMAKE_OSX_SYSROOT=appletvos \
+    -DCMAKE_OSX_ARCHITECTURES="arm64" \
+    -DGGML_METAL=ON \
+    -DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=appletvos \
+    -DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \
+    -DCMAKE_CXX_FLAGS="${COMMON_CXX_FLAGS}" \
+    -S .
+cmake --build build-tvos-device --config Release -- -quiet
+
+# Setup frameworks and copy binaries and headers
+echo "Setting up framework structures..."
+setup_framework_structure "build-ios-sim" ${IOS_MIN_OS_VERSION} "ios"
+setup_framework_structure "build-ios-device" ${IOS_MIN_OS_VERSION} "ios"
+setup_framework_structure "build-macos" ${MACOS_MIN_OS_VERSION} "macos"
+setup_framework_structure "build-visionos" ${VISIONOS_MIN_OS_VERSION} "visionos"
+setup_framework_structure "build-visionos-sim" ${VISIONOS_MIN_OS_VERSION} "visionos"
+setup_framework_structure "build-tvos-sim" ${TVOS_MIN_OS_VERSION} "tvos"
+setup_framework_structure "build-tvos-device" ${TVOS_MIN_OS_VERSION} "tvos"
+
+# Create dynamic libraries from static libraries
+echo "Creating dynamic libraries from static libraries..."
+combine_static_libraries "build-ios-sim" "Release-iphonesimulator" "ios" "true"
+combine_static_libraries "build-ios-device" "Release-iphoneos" "ios" "false"
+combine_static_libraries "build-macos" "Release" "macos" "false"
+combine_static_libraries "build-visionos" "Release-xros" "visionos" "false"
+combine_static_libraries "build-visionos-sim" "Release-xrsimulator" "visionos" "true"
+combine_static_libraries "build-tvos-sim" "Release-appletvsimulator" "tvos" "true"
+combine_static_libraries "build-tvos-device" "Release-appletvos" "tvos" "false"
+
+# Create XCFramework with correct debug symbols paths
+echo "Creating XCFramework..."
+xcodebuild -create-xcframework \
+    -framework $(pwd)/build-ios-sim/framework/whisper.framework \
+    -debug-symbols $(pwd)/build-ios-sim/dSYMs/whisper.dSYM \
+    -framework $(pwd)/build-ios-device/framework/whisper.framework \
+    -debug-symbols $(pwd)/build-ios-device/dSYMs/whisper.dSYM \
+    -framework $(pwd)/build-macos/framework/whisper.framework \
+    -debug-symbols $(pwd)/build-macos/dSYMS/whisper.dSYM \
+    -framework $(pwd)/build-visionos/framework/whisper.framework \
+    -debug-symbols $(pwd)/build-visionos/dSYMs/whisper.dSYM \
+    -framework $(pwd)/build-visionos-sim/framework/whisper.framework \
+    -debug-symbols $(pwd)/build-visionos-sim/dSYMs/whisper.dSYM \
+    -framework $(pwd)/build-tvos-device/framework/whisper.framework \
+    -debug-symbols $(pwd)/build-tvos-device/dSYMs/whisper.dSYM \
+    -framework $(pwd)/build-tvos-sim/framework/whisper.framework \
+    -debug-symbols $(pwd)/build-tvos-sim/dSYMs/whisper.dSYM \
+    -output $(pwd)/build-apple/whisper.xcframework

ci/README.md

@@ -0,0 +1,41 @@
+# CI
+
+In addition to [Github Actions](https://github.com/ggerganov/whisper.cpp/actions) `whisper.cpp` uses a custom CI framework:
+
+https://github.com/ggml-org/ci
+
+It monitors the `master` branch for new commits and runs the
+[ci/run.sh](https://github.com/ggerganov/whisper.cpp/blob/master/ci/run.sh) script on dedicated cloud instances. This allows us
+to execute heavier workloads compared to just using Github Actions. Also with time, the cloud instances will be scaled
+to cover various hardware architectures, including GPU and Apple Silicon instances.
+
+Collaborators can optionally trigger the CI run by adding the `ggml-ci` keyword to their commit message.
+Only the branches of this repo are monitored for this keyword.
+
+It is a good practice, before publishing changes to execute the full CI locally on your machine:
+
+```bash
+mkdir tmp
+
+# CPU-only build
+bash ./ci/run.sh ./tmp/results ./tmp/mnt
+
+# with CUDA support
+GG_BUILD_CUDA=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
+```
+
+## Environment Variables
+
+The CI script supports several environment variables to control the build:
+
+| Variable | Description |
+|----------|-------------|
+| `GG_BUILD_CUDA` | Enable NVIDIA CUDA GPU acceleration |
+| `GG_BUILD_SYCL` | Enable Intel SYCL acceleration |
+| `GG_BUILD_VULKAN` | Enable Vulkan GPU acceleration |
+| `GG_BUILD_METAL` | Enable Metal acceleration on Apple Silicon |
+| `GG_BUILD_BLAS` | Enable BLAS CPU acceleration |
+| `GG_BUILD_OPENVINO` | Enable OpenVINO support |
+| `GG_BUILD_COREML` | Enable Core ML support for Apple Neural Engine |
+| `GG_BUILD_LOW_PERF` | Limit tests for low-performance hardware |
+| `GG_BUILD_TEST_MODELS` | Comma-separated list of models to test (e.g. "tiny.en,tiny,base,medium", defaults to all models unless `GG_BUILD_LOW_PERF` is set) |

ci/run.sh

@@ -0,0 +1,336 @@
+#!/bin/bash
+#
+# sample usage:
+#
+# mkdir tmp
+#
+# # CPU-only build
+# bash ./ci/run.sh ./tmp/results ./tmp/mnt
+#
+# # with CUDA support
+# GG_BUILD_CUDA=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
+#
+# # with SYCL support
+# GG_BUILD_SYCL=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
+
+if [ -z "$2" ]; then
+    echo "usage: $0 <output-dir> <mnt-dir>"
+    exit 1
+fi
+
+mkdir -p "$1"
+mkdir -p "$2"
+
+OUT=$(realpath "$1")
+MNT=$(realpath "$2")
+
+rm -f "$OUT/*.log"
+rm -f "$OUT/*.exit"
+rm -f "$OUT/*.md"
+
+sd=`dirname $0`
+cd $sd/../
+SRC=`pwd`
+
+ALL_MODELS=( "tiny.en" "tiny" "base.en" "base" "small.en" "small" "medium.en" "medium" "large-v1" "large-v2" "large-v3" "large-v3-turbo" )
+BENCH_N_THREADS=4
+BENCH_ENCODER_ONLY=0
+BENCH_FLASH_ATTN=0
+
+# check for user-specified models first. if not specified, use fast models
+if [ ! -z ${GG_BUILD_TEST_MODELS} ]; then
+    IFS=',' read -r -a MODELS <<< "${GG_BUILD_TEST_MODELS}"
+else
+    if [ ! -z ${GG_BUILD_LOW_PERF} ]; then
+        MODELS=( "tiny" "base" "small" )
+    else
+        MODELS=("${ALL_MODELS[@]}")
+    fi
+fi
+
+CMAKE_EXTRA="-DWHISPER_FATAL_WARNINGS=ON"
+
+if [ ! -z ${GG_BUILD_CUDA} ]; then
+    CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=native"
+fi
+
+if [ ! -z ${GG_BUILD_SYCL} ]; then
+    if [ -z ${ONEAPI_ROOT} ]; then
+        echo "Not detected ONEAPI_ROOT, please install oneAPI base toolkit and enable it by:"
+        echo "source /opt/intel/oneapi/setvars.sh"
+        exit 1
+    fi
+
+    CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DGGML_SYCL_F16=ON"
+fi
+
+if [ ! -z ${GG_BUILD_OPENVINO} ]; then
+    CMAKE_EXTRA="${CMAKE_EXTRA} -DWHISPER_OPENVINO=ON"
+fi
+
+if [ ! -z ${GG_BUILD_METAL} ]; then
+    CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_METAL=ON"
+fi
+
+if [ ! -z ${GG_BUILD_VULKAN} ]; then
+    CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_VULKAN=ON"
+fi
+
+if [ ! -z ${GG_BUILD_BLAS} ]; then
+    CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_BLAS=ON"
+fi
+
+if [ ! -z ${GG_BUILD_COREML} ]; then
+    CMAKE_EXTRA="${CMAKE_EXTRA} -DWHISPER_COREML=ON"
+fi
+
+## helpers
+
+# download a file if it does not exist or if it is outdated
+function gg_wget {
+    local out=$1
+    local url=$2
+
+    local cwd=`pwd`
+
+    mkdir -p $out
+    cd $out
+
+    # should not re-download if file is the same
+    wget -nv -N $url
+
+    cd $cwd
+}
+
+function gg_download_model {
+    local model_name=$1
+    local model_file="$MNT/models/ggml-${model_name}.bin"
+
+    if [ ! -f ${model_file} ]; then
+        local cwd=`pwd`
+        mkdir -p "$MNT/models"
+        cd "$MNT/models"
+        bash "$cwd/models/download-ggml-model.sh" ${model_name} .
+        cd "$cwd"
+    fi
+}
+
+function gg_printf {
+    printf -- "$@" >> $OUT/README.md
+}
+
+# Helper function to check command exit status
+function gg_check_last_command_status {
+    local exit_file=$1
+    local command_name=$2
+
+    local exit_status=$?
+    echo "$exit_status" > "$exit_file"
+
+    if [ $exit_status -ne 0 ]; then
+        echo "Error: Command $command_name failed with exit status $exit_status"
+        return 1
+    fi
+
+    return 0
+}
+
+# Usage: gg_run <test_name> [additional_args...]
+#
+# Parameters:
+#   test_name       - Name of the test to run (calls gg_run_<test_name>)
+#   additional_args - Any additional arguments to pass to the test function (first argument is appended to the log filename)
+function gg_run {
+    ci=$1
+
+    if [ $# -gt 1 ]; then
+        ci="${ci}_${2}"
+    fi
+
+    set -o pipefail
+    set -x
+
+    gg_run_$1 "$@" | tee $OUT/$ci.log
+    cur=$?
+    echo "$cur" > $OUT/$ci.exit
+
+    set +x
+    set +o pipefail
+
+    gg_sum_$1 "$@"
+
+    ret=$((ret | cur))
+}
+
+function gg_check_build_requirements {
+    if ! command -v cmake &> /dev/null; then
+        gg_printf 'cmake not found, please install'
+    fi
+
+    if ! command -v make &> /dev/null; then
+        gg_printf 'make not found, please install'
+    fi
+}
+
+## ci
+
+function gg_run_ctest {
+    mode=$2
+
+    cd ${SRC}
+    
+    rm -rf build-ci-${mode} && mkdir build-ci-${mode} && cd build-ci-${mode}
+
+    set -e
+
+    gg_check_build_requirements
+
+    (time cmake -DCMAKE_BUILD_TYPE=${mode} ${CMAKE_EXTRA} .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log
+    (time make -j$(nproc)                                    ) 2>&1 | tee -a $OUT/${ci}-make.log
+
+    (time ctest --output-on-failure -L main -E test-opt ) 2>&1 | tee -a $OUT/${ci}-ctest.log
+
+    set +e
+}
+
+function gg_sum_ctest {
+    mode=$2
+
+    gg_printf '### %s\n\n' "${ci}"
+
+    gg_printf 'Runs ctest in '${mode}' mode\n'
+    gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
+    gg_printf '```\n'
+    gg_printf '%s\n' "$(cat $OUT/${ci}-ctest.log)"
+    gg_printf '```\n'
+}
+
+function gg_run_bench {
+    cd ${SRC}
+
+    # set flash attention flag if enabled
+    fattn=""
+    if [ "$BENCH_FLASH_ATTN" -eq 1 ]; then
+        fattn="-fa"
+    fi
+
+    # run memcpy benchmark if not encoder-only mode
+    if [ "$BENCH_ENCODER_ONLY" -eq 0 ]; then
+        echo "Running memcpy benchmark"
+        (time ./build-ci-release/bin/whisper-bench -w 1 -t $BENCH_N_THREADS 2>&1) | tee -a $OUT/${ci}-memcpy.log
+        gg_check_last_command_status "$OUT/${ci}-memcpy.exit" "memcpy benchmark"
+        
+        echo "Running ggml_mul_mat benchmark with $BENCH_N_THREADS threads"
+        (time ./build-ci-release/bin/whisper-bench -w 2 -t $BENCH_N_THREADS 2>&1) | tee -a $OUT/${ci}-mul_mat.log
+        gg_check_last_command_status "$OUT/${ci}-mul_mat.exit" "ggml_mul_mat benchmark"
+    fi
+
+    echo "Running benchmark for all models"
+
+    # generate header for the benchmark table
+    {
+        printf "| %16s | %13s | %3s | %3s | %7s | %7s | %7s | %7s | %7s |\n" "Config" "Model" "Th" "FA" "Enc." "Dec." "Bch5" "PP" "Commit"
+        printf "| %16s | %13s | %3s | %3s | %7s | %7s | %7s | %7s | %7s |\n" "---" "---" "---" "---" "---" "---" "---" "---" "---"
+    } | tee -a $OUT/${ci}-models-table.log
+
+    # run benchmark for each model
+    for model in "${MODELS[@]}"; do
+        echo "Benchmarking model: $model"
+
+        # run the benchmark and capture output
+        output=$(./build-ci-release/bin/whisper-bench -m $MNT/models/ggml-$model.bin -t $BENCH_N_THREADS $fattn 2>&1)
+        ret=$?
+
+        # save the raw output
+        echo "$output" > $OUT/${ci}-bench-$model.log
+
+        if [ $ret -eq 0 ]; then
+            # parse the benchmark results
+            encode_time=$(echo "$output" | grep "encode time" | awk '{print $11}')
+            decode_time=$(echo "$output" | grep "decode time" | awk '{print $11}')
+            batchd_time=$(echo "$output" | grep "batchd time" | awk '{print $11}')
+            prompt_time=$(echo "$output" | grep "prompt time" | awk '{print $11}')
+            system_info=$(echo "$output" | grep "system_info")
+            actual_threads=$(echo "$output" | grep "system_info" | awk '{print $4}')
+
+            # determine configuration
+            config=""
+            if [[ $system_info == *"AVX2 = 1"* ]]; then
+                config="$config AVX2"
+            fi
+            if [[ $system_info == *"NEON = 1"* ]]; then
+                config="$config NEON"
+            fi
+            if [[ $system_info == *"BLAS = 1"* ]]; then
+                config="$config BLAS"
+            fi
+            if [[ $system_info == *"COREML = 1"* ]]; then
+                config="$config COREML"
+            fi
+            if [[ $system_info == *"CUDA = 1"* ]]; then
+                config="$config CUDA"
+            fi
+            if [[ $system_info == *"METAL = 1"* ]]; then
+                config="$config METAL"
+            fi
+
+            # get commit hash
+            commit=$(git rev-parse --short HEAD)
+
+            # add row to benchmark table
+            printf "| %16s | %13s | %3s | %3s | %7s | %7s | %7s | %7s | %7s |\n" \
+                "$config" "$model" "$actual_threads" "$BENCH_FLASH_ATTN" "$encode_time" "$decode_time" "$batchd_time" "$prompt_time" "$commit" \
+                | tee -a $OUT/${ci}-models-table.log
+        else
+            echo "Benchmark failed for model: $model" | tee -a $OUT/${ci}-bench-errors.log
+        fi
+    done
+}
+
+function gg_sum_bench {
+    gg_printf '### %s\n\n' "${ci}"
+
+    gg_printf 'Whisper Benchmark Results\n'
+    gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
+
+    # show memcpy and ggml_mul_mat benchmark results if available
+    if [ "$BENCH_ENCODER_ONLY" -eq 0 ]; then
+        if [ -f "$OUT/${ci}-memcpy.log" ]; then
+            gg_printf '#### memcpy Benchmark\n\n'
+            gg_printf '```\n%s\n```\n\n' "$(cat $OUT/${ci}-memcpy.log)"
+        fi
+
+        if [ -f "$OUT/${ci}-mul_mat.log" ]; then
+            gg_printf '#### ggml_mul_mat Benchmark\n\n'
+            gg_printf '```\n%s\n```\n\n' "$(cat $OUT/${ci}-mul_mat.log)"
+        fi
+    fi
+
+    # show model benchmark results
+    gg_printf '#### Model Benchmarks\n\n'
+    if [ -f "$OUT/${ci}-models-table.log" ]; then
+        gg_printf '%s\n\n' "$(cat $OUT/${ci}-models-table.log)"
+    else
+        gg_printf 'No model benchmark results available.\n\n'
+    fi
+
+    # show any errors that occurred
+    if [ -f "$OUT/${ci}-bench-errors.log" ]; then
+        gg_printf '#### Benchmark Errors\n\n'
+        gg_printf '```\n%s\n```\n\n' "$(cat $OUT/${ci}-bench-errors.log)"
+    fi
+}
+
+ret=0
+
+for model in "${MODELS[@]}"; do
+    test $ret -eq 0 && gg_download_model ${model}
+done
+if [ -z ${GG_BUILD_SYCL}]; then
+    test $ret -eq 0 && gg_run ctest debug
+fi
+test $ret -eq 0 && gg_run ctest release
+
+test $ret -eq 0 && gg_run bench
+
+exit $ret

close-issue.yml

@@ -0,0 +1,28 @@
+name: Close inactive issues
+on:
+  schedule:
+    - cron: "42 0 * * *"
+
+# Fine-grant permission
+# https://docs.github.com/en/actions/security-for-github-actions/security-guides/automatic-token-authentication#modifying-the-permissions-for-the-github_token
+permissions:
+  issues: write
+
+jobs:
+  close-issues:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+    steps:
+      - uses: actions/stale@v5
+        with:
+          exempt-issue-labels: "refactor,help wanted,good first issue,research,bug,roadmap"
+          days-before-issue-stale: 30
+          days-before-issue-close: 14
+          stale-issue-label: "stale"
+          close-issue-message: "This issue was closed because it has been inactive for 14 days since being marked as stale."
+          days-before-pr-stale: -1
+          days-before-pr-close: -1
+          operations-per-run: 10000
+          repo-token: ${{ secrets.GITHUB_TOKEN }}

cmake/DefaultTargetOptions.cmake

@@ -13,5 +13,4 @@ set_target_properties(${TARGET}
     PROPERTIES
         EXPORT_COMPILE_COMMANDS ON
         RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin"
-        INSTALL_RPATH            "${CMAKE_INSTALL_PREFIX}/lib"
 )

cmake/build-info.cmake

@@ -42,6 +42,8 @@ endif()
 if(MSVC)
     set(BUILD_COMPILER "${CMAKE_C_COMPILER_ID} ${CMAKE_C_COMPILER_VERSION}")
     set(BUILD_TARGET ${CMAKE_VS_PLATFORM_NAME})
+    add_compile_options("$<$<COMPILE_LANGUAGE:C>:/utf-8>")
+    add_compile_options("$<$<COMPILE_LANGUAGE:CXX>:/utf-8>")
 else()
     execute_process(
         COMMAND sh -c "$@ --version | head -1" _ ${CMAKE_C_COMPILER}

examples/CMakeLists.txt

@@ -14,10 +14,6 @@ if (WHISPER_SDL2)
     message(STATUS "SDL2_LIBRARIES    = ${SDL2_LIBRARIES}")
 endif()
 
-if (WHISPER_CLBLAST)
-    find_package(CLBlast REQUIRED)
-endif()
-
 # common
 
 set(TARGET common)
@@ -56,6 +52,8 @@ add_library(${TARGET} STATIC
     common.cpp
     common-ggml.h
     common-ggml.cpp
+    common-whisper.h
+    common-whisper.cpp
     grammar-parser.h
     grammar-parser.cpp
     ${COMMON_SOURCES_FFMPEG}
@@ -63,7 +61,7 @@ add_library(${TARGET} STATIC
 
 include(DefaultTargetOptions)
 
-target_link_libraries(${TARGET} PRIVATE whisper ${COMMON_EXTRA_LIBS})
+target_link_libraries(${TARGET} PRIVATE whisper ${COMMON_EXTRA_LIBS} ${CMAKE_DL_LIBS})
 
 set_target_properties(${TARGET} PROPERTIES POSITION_INDEPENDENT_CODE ON)
 set_target_properties(${TARGET} PROPERTIES FOLDER "libs")
@@ -97,52 +95,29 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR})
 
 if (EMSCRIPTEN)
     add_subdirectory(whisper.wasm)
-    set_target_properties(libmain PROPERTIES FOLDER "libs")
     add_subdirectory(stream.wasm)
-    set_target_properties(libstream PROPERTIES FOLDER "libs")
     add_subdirectory(command.wasm)
-    set_target_properties(libcommand PROPERTIES FOLDER "libs")
-    #add_subdirectory(talk.wasm)
-    #set_target_properties(libtalk PROPERTIES FOLDER "libs")
     add_subdirectory(bench.wasm)
-    set_target_properties(libbench PROPERTIES FOLDER "libs")
 elseif(CMAKE_JS_VERSION)
     add_subdirectory(addon.node)
-    set_target_properties(addon.node PROPERTIES FOLDER "examples")
 else()
-    add_subdirectory(main)
+    add_subdirectory(cli)
-    set_target_properties(main PROPERTIES FOLDER "examples")
-if (WHISPER_SDL2)
-    add_subdirectory(stream)
-    set_target_properties(stream PROPERTIES FOLDER "examples")
-endif (WHISPER_SDL2)
-    add_subdirectory(server)
-    set_target_properties(server PROPERTIES FOLDER "examples")
-if (WHISPER_SDL2)
-    add_subdirectory(command)
-    set_target_properties(command PROPERTIES FOLDER "examples")
-endif (WHISPER_SDL2)
     add_subdirectory(bench)
-    set_target_properties(bench PROPERTIES FOLDER "examples")
+    add_subdirectory(server)
     add_subdirectory(quantize)
-    set_target_properties(quantize PROPERTIES FOLDER "examples")
+    if (WHISPER_SDL2)
-if (WHISPER_SDL2)
+        add_subdirectory(stream)
-    # TODO: disabled until update
+        add_subdirectory(command)
-    #       https://github.com/ggerganov/whisper.cpp/issues/1818
+        add_subdirectory(talk-llama)
-    #add_subdirectory(talk)
+        add_subdirectory(lsp)
-    #set_target_properties(talk PROPERTIES FOLDER "examples")
+        if (GGML_SYCL)
-    add_subdirectory(talk-llama)
+            add_subdirectory(sycl)
-    set_target_properties(talk-llama PROPERTIES FOLDER "examples")
+        endif()
-    add_subdirectory(lsp)
+    endif (WHISPER_SDL2)
-    set_target_properties(lsp PROPERTIES FOLDER "examples")
+
-    if (GGML_SYCL)
+    add_subdirectory(deprecation-warning)
-        add_subdirectory(sycl)
-        set_target_properties(ls-sycl-device PROPERTIES FOLDER "examples")
-    endif()
-endif (WHISPER_SDL2)
 endif()
 
 if (WHISPER_SDL2)
     add_subdirectory(wchess)
-    set_target_properties(wchess PROPERTIES FOLDER "examples")
 endif (WHISPER_SDL2)

examples/addon.node/__test__/whisper.spec.js

@@ -18,6 +18,13 @@ const whisperParamsMock = {
   translate: true,
   no_timestamps: false,
   audio_ctx: 0,
+  max_len: 0,
+  prompt: "",
+  print_progress: false,
+  progress_callback: (progress) => {
+    console.log(`Progress: ${progress}`);
+  },
+  max_context: -1
 };
 
 describe("Run whisper.node", () => {

examples/addon.node/addon.cpp

@@ -1,5 +1,6 @@
 #include "napi.h"
 #include "common.h"
+#include "common-whisper.h"
 
 #include "whisper.h"
 
@@ -127,192 +128,227 @@ void whisper_print_segment_callback(struct whisper_context * ctx, struct whisper
 
 void cb_log_disable(enum ggml_log_level, const char *, void *) {}
 
-int run(whisper_params &params, std::vector<std::vector<std::string>> &result) {
+class ProgressWorker : public Napi::AsyncWorker {
-    if (params.no_prints) {
-        whisper_log_set(cb_log_disable, NULL);
-    }
-
-    if (params.fname_inp.empty() && params.pcmf32.empty()) {
-        fprintf(stderr, "error: no input files or audio buffer specified\n");
-        return 2;
-    }
-
-    if (params.language != "auto" && whisper_lang_id(params.language.c_str()) == -1) {
-        fprintf(stderr, "error: unknown language '%s'\n", params.language.c_str());
-        exit(0);
-    }
-
-    // whisper init
-
-    struct whisper_context_params cparams = whisper_context_default_params();
-    cparams.use_gpu = params.use_gpu;
-    cparams.flash_attn = params.flash_attn;
-    struct whisper_context * ctx = whisper_init_from_file_with_params(params.model.c_str(), cparams);
-
-    if (ctx == nullptr) {
-        fprintf(stderr, "error: failed to initialize whisper context\n");
-        return 3;
-    }
-
-    // if params.pcmf32 is provided, set params.fname_inp to "buffer"
-    // this is simpler than further modifications in the code
-    if (!params.pcmf32.empty()) {
-        fprintf(stderr, "info: using audio buffer as input\n");
-        params.fname_inp.clear();
-        params.fname_inp.emplace_back("buffer");
-    }
-
-    for (int f = 0; f < (int) params.fname_inp.size(); ++f) {
-        const auto fname_inp = params.fname_inp[f];
-        const auto fname_out = f < (int)params.fname_out.size() && !params.fname_out[f].empty() ? params.fname_out[f] : params.fname_inp[f];
-
-        std::vector<float> pcmf32; // mono-channel F32 PCM
-        std::vector<std::vector<float>> pcmf32s; // stereo-channel F32 PCM
-
-        // read the input audio file if params.pcmf32 is not provided
-        if (params.pcmf32.empty()) {
-            if (!::read_wav(fname_inp, pcmf32, pcmf32s, params.diarize)) {
-                fprintf(stderr, "error: failed to read WAV file '%s'\n", fname_inp.c_str());
-                continue;
-            }
-        } else {
-            pcmf32 = params.pcmf32;
-        }
-
-        // print system information
-        if (!params.no_prints) {
-            fprintf(stderr, "\n");
-            fprintf(stderr, "system_info: n_threads = %d / %d | %s\n",
-                    params.n_threads*params.n_processors, std::thread::hardware_concurrency(), whisper_print_system_info());
-        }
-
-        // print some info about the processing
-        if (!params.no_prints) {
-            fprintf(stderr, "\n");
-            if (!whisper_is_multilingual(ctx)) {
-                if (params.language != "en" || params.translate) {
-                    params.language = "en";
-                    params.translate = false;
-                    fprintf(stderr, "%s: WARNING: model is not multilingual, ignoring language and translation options\n", __func__);
-                }
-            }
-            fprintf(stderr, "%s: processing '%s' (%d samples, %.1f sec), %d threads, %d processors, lang = %s, task = %s, timestamps = %d, audio_ctx = %d ...\n",
-                    __func__, fname_inp.c_str(), int(pcmf32.size()), float(pcmf32.size())/WHISPER_SAMPLE_RATE,
-                    params.n_threads, params.n_processors,
-                    params.language.c_str(),
-                    params.translate ? "translate" : "transcribe",
-                    params.no_timestamps ? 0 : 1,
-                    params.audio_ctx);
-
-            fprintf(stderr, "\n");
-        }
-
-        // run the inference
-        {
-            whisper_full_params wparams = whisper_full_default_params(WHISPER_SAMPLING_GREEDY);
-
-            wparams.strategy = params.beam_size > 1 ? WHISPER_SAMPLING_BEAM_SEARCH : WHISPER_SAMPLING_GREEDY;
-
-            wparams.print_realtime   = false;
-            wparams.print_progress   = params.print_progress;
-            wparams.print_timestamps = !params.no_timestamps;
-            wparams.print_special    = params.print_special;
-            wparams.translate        = params.translate;
-            wparams.language         = params.language.c_str();
-            wparams.n_threads        = params.n_threads;
-            wparams.n_max_text_ctx   = params.max_context >= 0 ? params.max_context : wparams.n_max_text_ctx;
-            wparams.offset_ms        = params.offset_t_ms;
-            wparams.duration_ms      = params.duration_ms;
-
-            wparams.token_timestamps = params.output_wts || params.max_len > 0;
-            wparams.thold_pt         = params.word_thold;
-            wparams.entropy_thold    = params.entropy_thold;
-            wparams.logprob_thold    = params.logprob_thold;
-            wparams.max_len          = params.output_wts && params.max_len == 0 ? 60 : params.max_len;
-            wparams.audio_ctx        = params.audio_ctx;
-
-            wparams.greedy.best_of        = params.best_of;
-            wparams.beam_search.beam_size = params.beam_size;
-
-            wparams.initial_prompt   = params.prompt.c_str();
-
-            wparams.no_timestamps    = params.no_timestamps;
-
-            whisper_print_user_data user_data = { &params, &pcmf32s };
-
-            // this callback is called on each new segment
-            if (!wparams.print_realtime) {
-                wparams.new_segment_callback           = whisper_print_segment_callback;
-                wparams.new_segment_callback_user_data = &user_data;
-            }
-
-            // example for abort mechanism
-            // in this example, we do not abort the processing, but we could if the flag is set to true
-            // the callback is called before every encoder run - if it returns false, the processing is aborted
-            {
-                static bool is_aborted = false; // NOTE: this should be atomic to avoid data race
-
-                wparams.encoder_begin_callback = [](struct whisper_context * /*ctx*/, struct whisper_state * /*state*/, void * user_data) {
-                    bool is_aborted = *(bool*)user_data;
-                    return !is_aborted;
-                };
-                wparams.encoder_begin_callback_user_data = &is_aborted;
-            }
-
-            if (whisper_full_parallel(ctx, wparams, pcmf32.data(), pcmf32.size(), params.n_processors) != 0) {
-                fprintf(stderr, "failed to process audio\n");
-                return 10;
-            }
-        }
-    }
-
-    const int n_segments = whisper_full_n_segments(ctx);
-    result.resize(n_segments);
-    for (int i = 0; i < n_segments; ++i) {
-        const char * text = whisper_full_get_segment_text(ctx, i);
-        const int64_t t0 = whisper_full_get_segment_t0(ctx, i);
-        const int64_t t1 = whisper_full_get_segment_t1(ctx, i);
-
-        result[i].emplace_back(to_timestamp(t0, params.comma_in_time));
-        result[i].emplace_back(to_timestamp(t1, params.comma_in_time));
-        result[i].emplace_back(text);
-    }
-
-    whisper_print_timings(ctx);
-    whisper_free(ctx);
-
-    return 0;
-}
-
-class Worker : public Napi::AsyncWorker {
  public:
-  Worker(Napi::Function& callback, whisper_params params)
+    ProgressWorker(Napi::Function& callback, whisper_params params, Napi::Function progress_callback, Napi::Env env)
-      : Napi::AsyncWorker(callback), params(params) {}
+        : Napi::AsyncWorker(callback), params(params), env(env) {
-
+        // Create thread-safe function
-  void Execute() override {
+        if (!progress_callback.IsEmpty()) {
-    run(params, result);
+            tsfn = Napi::ThreadSafeFunction::New(
-  }
+                env,
-
+                progress_callback,
-  void OnOK() override {
+                "Progress Callback",
-    Napi::HandleScope scope(Env());
+                0,
-    Napi::Object res = Napi::Array::New(Env(), result.size());
+                1
-    for (uint64_t i = 0; i < result.size(); ++i) {
+            );
-      Napi::Object tmp = Napi::Array::New(Env(), 3);
+        }
-      for (uint64_t j = 0; j < 3; ++j) {
+    }
-        tmp[j] = Napi::String::New(Env(), result[i][j]);
+
-      }
+    ~ProgressWorker() {
-      res[i] = tmp;
+        if (tsfn) {
+            // Make sure to release the thread-safe function on destruction
+            tsfn.Release();
+        }
+    }
+
+    void Execute() override {
+        // Use custom run function with progress callback support
+        run_with_progress(params, result);
+    }
+
+    void OnOK() override {
+        Napi::HandleScope scope(Env());
+        Napi::Object res = Napi::Array::New(Env(), result.size());
+        for (uint64_t i = 0; i < result.size(); ++i) {
+            Napi::Object tmp = Napi::Array::New(Env(), 3);
+            for (uint64_t j = 0; j < 3; ++j) {
+                tmp[j] = Napi::String::New(Env(), result[i][j]);
+            }
+            res[i] = tmp;
+        }
+        Callback().Call({Env().Null(), res});
+    }
+
+    // Progress callback function - using thread-safe function
+    void OnProgress(int progress) {
+        if (tsfn) {
+            // Use thread-safe function to call JavaScript callback
+            auto callback = [progress](Napi::Env env, Napi::Function jsCallback) {
+                jsCallback.Call({Napi::Number::New(env, progress)});
+            };
+            
+            tsfn.BlockingCall(callback);
+        }
     }
-    Callback().Call({Env().Null(), res});
-  }
 
  private:
-  whisper_params params;
+    whisper_params params;
-  std::vector<std::vector<std::string>> result;
+    std::vector<std::vector<std::string>> result;
+    Napi::Env env;
+    Napi::ThreadSafeFunction tsfn;
+
+    // Custom run function with progress callback support
+    int run_with_progress(whisper_params &params, std::vector<std::vector<std::string>> &result) {
+        if (params.no_prints) {
+            whisper_log_set(cb_log_disable, NULL);
+        }
+
+        if (params.fname_inp.empty() && params.pcmf32.empty()) {
+            fprintf(stderr, "error: no input files or audio buffer specified\n");
+            return 2;
+        }
+
+        if (params.language != "auto" && whisper_lang_id(params.language.c_str()) == -1) {
+            fprintf(stderr, "error: unknown language '%s'\n", params.language.c_str());
+            exit(0);
+        }
+
+        // whisper init
+        struct whisper_context_params cparams = whisper_context_default_params();
+        cparams.use_gpu = params.use_gpu;
+        cparams.flash_attn = params.flash_attn;
+        struct whisper_context * ctx = whisper_init_from_file_with_params(params.model.c_str(), cparams);
+
+        if (ctx == nullptr) {
+            fprintf(stderr, "error: failed to initialize whisper context\n");
+            return 3;
+        }
+
+        // If params.pcmf32 provides, set params.fname_inp as "buffer"
+        if (!params.pcmf32.empty()) {
+            fprintf(stderr, "info: using audio buffer as input\n");
+            params.fname_inp.clear();
+            params.fname_inp.emplace_back("buffer");
+        }
+
+        for (int f = 0; f < (int) params.fname_inp.size(); ++f) {
+            const auto fname_inp = params.fname_inp[f];
+            const auto fname_out = f < (int)params.fname_out.size() && !params.fname_out[f].empty() ? params.fname_out[f] : params.fname_inp[f];
+
+            std::vector<float> pcmf32; // mono-channel F32 PCM
+            std::vector<std::vector<float>> pcmf32s; // stereo-channel F32 PCM
+
+            // If params.pcmf32 is empty, read input audio file
+            if (params.pcmf32.empty()) {
+                if (!::read_audio_data(fname_inp, pcmf32, pcmf32s, params.diarize)) {
+                    fprintf(stderr, "error: failed to read audio file '%s'\n", fname_inp.c_str());
+                    continue;
+                }
+            } else {
+                pcmf32 = params.pcmf32;
+            }
+
+            // Print system info
+            if (!params.no_prints) {
+                fprintf(stderr, "\n");
+                fprintf(stderr, "system_info: n_threads = %d / %d | %s\n",
+                        params.n_threads*params.n_processors, std::thread::hardware_concurrency(), whisper_print_system_info());
+            }
+
+            // Print processing info
+            if (!params.no_prints) {
+                fprintf(stderr, "\n");
+                if (!whisper_is_multilingual(ctx)) {
+                    if (params.language != "en" || params.translate) {
+                        params.language = "en";
+                        params.translate = false;
+                        fprintf(stderr, "%s: WARNING: model is not multilingual, ignoring language and translation options\n", __func__);
+                    }
+                }
+                fprintf(stderr, "%s: processing '%s' (%d samples, %.1f sec), %d threads, %d processors, lang = %s, task = %s, timestamps = %d, audio_ctx = %d ...\n",
+                        __func__, fname_inp.c_str(), int(pcmf32.size()), float(pcmf32.size())/WHISPER_SAMPLE_RATE,
+                        params.n_threads, params.n_processors,
+                        params.language.c_str(),
+                        params.translate ? "translate" : "transcribe",
+                        params.no_timestamps ? 0 : 1,
+                        params.audio_ctx);
+
+                fprintf(stderr, "\n");
+            }
+
+            // Run inference
+            {
+                whisper_full_params wparams = whisper_full_default_params(WHISPER_SAMPLING_GREEDY);
+
+                wparams.strategy = params.beam_size > 1 ? WHISPER_SAMPLING_BEAM_SEARCH : WHISPER_SAMPLING_GREEDY;
+
+                wparams.print_realtime   = false;
+                wparams.print_progress   = params.print_progress;
+                wparams.print_timestamps = !params.no_timestamps;
+                wparams.print_special    = params.print_special;
+                wparams.translate        = params.translate;
+                wparams.language         = params.language.c_str();
+                wparams.n_threads        = params.n_threads;
+                wparams.n_max_text_ctx   = params.max_context >= 0 ? params.max_context : wparams.n_max_text_ctx;
+                wparams.offset_ms        = params.offset_t_ms;
+                wparams.duration_ms      = params.duration_ms;
+
+                wparams.token_timestamps = params.output_wts || params.max_len > 0;
+                wparams.thold_pt         = params.word_thold;
+                wparams.entropy_thold    = params.entropy_thold;
+                wparams.logprob_thold    = params.logprob_thold;
+                wparams.max_len          = params.output_wts && params.max_len == 0 ? 60 : params.max_len;
+                wparams.audio_ctx        = params.audio_ctx;
+
+                wparams.greedy.best_of        = params.best_of;
+                wparams.beam_search.beam_size = params.beam_size;
+
+                wparams.initial_prompt   = params.prompt.c_str();
+
+                wparams.no_timestamps    = params.no_timestamps;
+
+                whisper_print_user_data user_data = { &params, &pcmf32s };
+
+                // This callback is called for each new segment
+                if (!wparams.print_realtime) {
+                    wparams.new_segment_callback           = whisper_print_segment_callback;
+                    wparams.new_segment_callback_user_data = &user_data;
+                }
+
+                // Set progress callback
+                wparams.progress_callback = [](struct whisper_context * /*ctx*/, struct whisper_state * /*state*/, int progress, void * user_data) {
+                    ProgressWorker* worker = static_cast<ProgressWorker*>(user_data);
+                    worker->OnProgress(progress);
+                };
+                wparams.progress_callback_user_data = this;
+
+                // Abort mechanism example
+                {
+                    static bool is_aborted = false; // Note: this should be atomic to avoid data races
+
+                    wparams.encoder_begin_callback = [](struct whisper_context * /*ctx*/, struct whisper_state * /*state*/, void * user_data) {
+                        bool is_aborted = *(bool*)user_data;
+                        return !is_aborted;
+                    };
+                    wparams.encoder_begin_callback_user_data = &is_aborted;
+                }
+
+                if (whisper_full_parallel(ctx, wparams, pcmf32.data(), pcmf32.size(), params.n_processors) != 0) {
+                    fprintf(stderr, "failed to process audio\n");
+                    return 10;
+                }
+            }
+    }
+
+        const int n_segments = whisper_full_n_segments(ctx);
+        result.resize(n_segments);
+        for (int i = 0; i < n_segments; ++i) {
+            const char * text = whisper_full_get_segment_text(ctx, i);
+            const int64_t t0 = whisper_full_get_segment_t0(ctx, i);
+            const int64_t t1 = whisper_full_get_segment_t1(ctx, i);
+
+            result[i].emplace_back(to_timestamp(t0, params.comma_in_time));
+            result[i].emplace_back(to_timestamp(t1, params.comma_in_time));
+            result[i].emplace_back(text);
+        }
+
+        whisper_print_timings(ctx);
+        whisper_free(ctx);
+
+        return 0;
+    }
 };
 
-
-
 Napi::Value whisper(const Napi::CallbackInfo& info) {
   Napi::Env env = info.Env();
   if (info.Length() <= 0 || !info[0].IsObject()) {
@@ -330,6 +366,30 @@ Napi::Value whisper(const Napi::CallbackInfo& info) {
   bool no_timestamps = whisper_params.Get("no_timestamps").As<Napi::Boolean>();
   int32_t audio_ctx = whisper_params.Get("audio_ctx").As<Napi::Number>();
   bool comma_in_time = whisper_params.Get("comma_in_time").As<Napi::Boolean>();
+  int32_t max_len = whisper_params.Get("max_len").As<Napi::Number>();
+  
+  // Add support for max_context
+  int32_t max_context = -1;
+  if (whisper_params.Has("max_context") && whisper_params.Get("max_context").IsNumber()) {
+    max_context = whisper_params.Get("max_context").As<Napi::Number>();
+  }
+  
+  // support prompt
+  std::string prompt = "";
+  if (whisper_params.Has("prompt") && whisper_params.Get("prompt").IsString()) {
+    prompt = whisper_params.Get("prompt").As<Napi::String>();
+  }
+  
+  // Add support for print_progress
+  bool print_progress = false;
+  if (whisper_params.Has("print_progress")) {
+    print_progress = whisper_params.Get("print_progress").As<Napi::Boolean>();
+  }
+  // Add support for progress_callback
+  Napi::Function progress_callback;
+  if (whisper_params.Has("progress_callback") && whisper_params.Get("progress_callback").IsFunction()) {
+    progress_callback = whisper_params.Get("progress_callback").As<Napi::Function>();
+  }
 
   Napi::Value pcmf32Value = whisper_params.Get("pcmf32");
   std::vector<float> pcmf32_vec;
@@ -352,9 +412,14 @@ Napi::Value whisper(const Napi::CallbackInfo& info) {
   params.audio_ctx = audio_ctx;
   params.pcmf32 = pcmf32_vec;
   params.comma_in_time = comma_in_time;
+  params.max_len = max_len;
+  params.max_context = max_context;
+  params.print_progress = print_progress;
+  params.prompt = prompt;
 
   Napi::Function callback = info[1].As<Napi::Function>();
-  Worker* worker = new Worker(callback, params);
+  // Create a new Worker class with progress callback support
+  ProgressWorker* worker = new ProgressWorker(callback, params, progress_callback, env);
   worker->Queue();
   return env.Undefined();
 }

examples/addon.node/index.js

@@ -18,6 +18,10 @@ const whisperParams = {
   translate: true,
   no_timestamps: false,
   audio_ctx: 0,
+  max_len: 0,
+  progress_callback: (progress) => {
+      console.log(`progress: ${progress}%`);
+    }
 };
 
 const arguments = process.argv.slice(2);

examples/bench.wasm/README.md

@@ -2,7 +2,7 @@
 
 Benchmark the performance of whisper.cpp in the browser using WebAssembly
 
-Link: https://whisper.ggerganov.com/bench/
+Link: https://ggerganov.github.io/whisper.cpp/bench.wasm
 
 Terminal version: [examples/bench](/examples/bench)
 
@@ -15,7 +15,17 @@ cd whisper.cpp
 mkdir build-em && cd build-em
 emcmake cmake ..
 make -j
+```
+The example can then be started by running a local HTTP server:
+```console
+python3 examples/server.py
+```
+And then opening a browser to the following URL:
+http://localhost:8000/bench.wasm
 
+To run the example in a different server, you need to copy the following files
+to the server's HTTP path:
+```
 # copy the produced page to your HTTP path
 cp bin/bench.wasm/*       /path/to/html/
 cp bin/libbench.worker.js /path/to/html/

examples/bench.wasm/index-tmpl.html

@@ -24,6 +24,8 @@
                 overflow-x: scroll;
             }
         </style>
+        <script src="../coi-serviceworker.js"></script>
+        <link rel="icon" href="data:,">
     </head>
     <body>
         <div id="main-container">
@@ -36,11 +38,10 @@
             <br><br>
 
             <b>More examples:</b>
-                <a href="https://whisper.ggerganov.com/">main</a> |
+                <a href="../">main</a> |
-                <a href="https://whisper.ggerganov.com/bench">bench</a> |
+                <a href="../bench.wasm/">bench</a> |
-                <a href="https://whisper.ggerganov.com/stream">stream</a> |
+                <a href="../stream.wasm">stream</a> |
-                <a href="https://whisper.ggerganov.com/command">command</a> |
+                <a href="../command.wasm/">command</a> |
-                <a href="https://whisper.ggerganov.com/talk">talk</a> |
 
             <br><br>
 

examples/bench/CMakeLists.txt

@@ -1,6 +1,8 @@
-set(TARGET bench)
+set(TARGET whisper-bench)
 add_executable(${TARGET} bench.cpp)
 
 include(DefaultTargetOptions)
 
 target_link_libraries(${TARGET} PRIVATE whisper ${CMAKE_THREAD_LIBS_INIT})
+
+install(TARGETS ${TARGET} RUNTIME)

examples/bench/README.md

@@ -1,17 +1,14 @@
-# bench
+# whisper.cpp/examples/bench
 
 A very basic tool for benchmarking the inference performance on your device. The tool simply runs the Encoder part of
 the transformer on some random audio data and records the execution time. This way we can have an objective comparison
 of the performance of the model for various setups.
 
-Benchmark results are tracked in the following Github issue: https://github.com/ggerganov/whisper.cpp/issues/89
+Benchmark results are tracked in the following Github issue: https://github.com/ggml-org/whisper.cpp/issues/89
 
 ```bash
-# build the bench tool
+# run the bench too on the small.en model using 4 threads
-$ make bench
+$ ./build/bin/whisper-bench -m ./models/ggml-small.en.bin -t 4
-
-# run it on the small.en model using 4 threads
-$ ./bench -m ./models/ggml-small.en.bin -t 4
 
 whisper_model_load: loading model from './models/ggml-small.en.bin'
 whisper_model_load: n_vocab       = 51864
@@ -43,7 +40,7 @@ system_info: n_threads = 4 | AVX2 = 0 | AVX512 = 0 | NEON = 1 | FP16_VA = 1 | WA
 
 If you wish, you can submit these results here:
 
-  https://github.com/ggerganov/whisper.cpp/issues/89
+  https://github.com/ggml-org/whisper.cpp/issues/89
 
 Please include the following information:
 

examples/bench/bench.cpp

@@ -50,11 +50,11 @@ void whisper_print_usage(int /*argc*/, char ** argv, const whisper_params & para
     fprintf(stderr, "  -t N,     --threads N   [%-7d] number of threads to use during computation\n", params.n_threads);
     fprintf(stderr, "  -m FNAME, --model FNAME [%-7s] model path\n",                                  params.model.c_str());
     fprintf(stderr, "  -w N,     --what N      [%-7d] what to benchmark:\n",                          params.what);
-    fprintf(stderr, "  -ng,      --no-gpu      [%-7s] disable GPU\n",                                 params.use_gpu ? "false" : "true");
-    fprintf(stderr, "  -fa,      --flash-attn  [%-7s] enable flash attention\n",                      params.flash_attn ? "true" : "false");
     fprintf(stderr, "                           %-7s  0 - whisper\n",                                 "");
     fprintf(stderr, "                           %-7s  1 - memcpy\n",                                  "");
     fprintf(stderr, "                           %-7s  2 - ggml_mul_mat\n",                            "");
+    fprintf(stderr, "  -ng,      --no-gpu      [%-7s] disable GPU\n",                                 params.use_gpu ? "false" : "true");
+    fprintf(stderr, "  -fa,      --flash-attn  [%-7s] enable flash attention\n",                      params.flash_attn ? "true" : "false");
     fprintf(stderr, "\n");
 }
 

examples/cli/CMakeLists.txt

@@ -1,6 +1,8 @@
-set(TARGET main)
+set(TARGET whisper-cli)
-add_executable(${TARGET} main.cpp)
+add_executable(${TARGET} cli.cpp)
 
 include(DefaultTargetOptions)
 
 target_link_libraries(${TARGET} PRIVATE common whisper ${FFMPEG_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT})
+
+install(TARGETS ${TARGET} RUNTIME)

examples/cli/README.md

@@ -1,12 +1,12 @@
-# main
+# whisper.cpp/examples/cli
 
 This is the main example demonstrating most of the functionality of the Whisper model.
 It can be used as a reference for using the `whisper.cpp` library in other projects.
 
 ```
-./main -h
+./build/bin/whisper-cli -h
 
-usage: ./main [options] file0.wav file1.wav ...
+usage: ./build-pkg/bin/whisper-cli [options] file0.wav file1.wav ...
 
 options:
   -h,        --help              [default] show this help message and exit
@@ -20,9 +20,12 @@ options:
   -sow,      --split-on-word     [false  ] split on word rather than on token
   -bo N,     --best-of N         [5      ] number of best candidates to keep
   -bs N,     --beam-size N       [5      ] beam size for beam search
+  -ac N,     --audio-ctx N       [0      ] audio context size (0 - all)
   -wt N,     --word-thold N      [0.01   ] word timestamp probability threshold
   -et N,     --entropy-thold N   [2.40   ] entropy threshold for decoder fail
   -lpt N,    --logprob-thold N   [-1.00  ] log probability threshold for decoder fail
+  -tp,       --temperature N     [0.00   ] The sampling temperature, between 0 and 1
+  -tpi,      --temperature-inc N [0.20   ] The increment of temperature, between 0 and 1
   -debug,    --debug-mode        [false  ] enable debug mode (eg. dump log_mel)
   -tr,       --translate         [false  ] translate from source language to english
   -di,       --diarize           [false  ] stereo audio diarization
@@ -38,16 +41,23 @@ options:
   -oj,       --output-json       [false  ] output result in a JSON file
   -ojf,      --output-json-full  [false  ] include more information in the JSON file
   -of FNAME, --output-file FNAME [       ] output file path (without file extension)
+  -np,       --no-prints         [false  ] do not print anything other than the results
   -ps,       --print-special     [false  ] print special tokens
   -pc,       --print-colors      [false  ] print colors
   -pp,       --print-progress    [false  ] print progress
   -nt,       --no-timestamps     [false  ] do not print timestamps
   -l LANG,   --language LANG     [en     ] spoken language ('auto' for auto-detect)
   -dl,       --detect-language   [false  ] exit after automatically detecting language
-             --prompt PROMPT     [       ] initial prompt
+             --prompt PROMPT     [       ] initial prompt (max n_text_ctx/2 tokens)
   -m FNAME,  --model FNAME       [models/ggml-base.en.bin] model path
   -f FNAME,  --file FNAME        [       ] input WAV file path
   -oved D,   --ov-e-device DNAME [CPU    ] the OpenVINO device used for encode inference
+  -dtw MODEL --dtw MODEL         [       ] compute token-level timestamps
   -ls,       --log-score         [false  ] log best decoder scores of tokens
   -ng,       --no-gpu            [false  ] disable GPU
+  -fa,       --flash-attn        [false  ] flash attention
+  --suppress-regex REGEX         [       ] regular expression matching tokens to suppress
+  --grammar GRAMMAR              [       ] GBNF grammar to guide decoding
+  --grammar-rule RULE            [       ] top-level GBNF grammar rule name
+  --grammar-penalty N            [100.0  ] scales down logits of nongrammar tokens
 ```

examples/cli/cli.cpp

@@ -1,4 +1,5 @@
 #include "common.h"
+#include "common-whisper.h"
 
 #include "whisper.h"
 #include "grammar-parser.h"
@@ -6,12 +7,18 @@
 #include <cmath>
 #include <fstream>
 #include <cstdio>
-#include <regex>
 #include <string>
 #include <thread>
 #include <vector>
 #include <cstring>
 
+#if defined(_WIN32)
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#endif
+
 #if defined(_MSC_VER)
 #pragma warning(disable: 4244 4267) // possible loss of data
 #endif
@@ -43,6 +50,7 @@ struct whisper_params {
     float word_thold      =  0.01f;
     float entropy_thold   =  2.40f;
     float logprob_thold   = -1.00f;
+    float no_speech_thold =  0.6f;
     float grammar_penalty = 100.0f;
     float temperature     = 0.0f;
     float temperature_inc = 0.2f;
@@ -70,6 +78,7 @@ struct whisper_params {
     bool log_score       = false;
     bool use_gpu         = true;
     bool flash_attn      = false;
+    bool suppress_nst    = false;
 
     std::string language  = "en";
     std::string prompt;
@@ -104,6 +113,11 @@ static char * whisper_param_turn_lowercase(char * in){
     return in;
 }
 
+static char * requires_value_error(const std::string & arg) {
+    fprintf(stderr, "error: argument %s requires value\n", arg.c_str());
+    exit(0);
+}
+
 static bool whisper_params_parse(int argc, char ** argv, whisper_params & params) {
     for (int i = 1; i < argc; i++) {
         std::string arg = argv[i];
@@ -122,21 +136,23 @@ static bool whisper_params_parse(int argc, char ** argv, whisper_params & params
             whisper_print_usage(argc, argv, params);
             exit(0);
         }
-        else if (arg == "-t"    || arg == "--threads")         { params.n_threads       = std::stoi(argv[++i]); }
+        #define ARGV_NEXT (((i + 1) < argc) ? argv[++i] : requires_value_error(arg))
-        else if (arg == "-p"    || arg == "--processors")      { params.n_processors    = std::stoi(argv[++i]); }
+        else if (arg == "-t"    || arg == "--threads")         { params.n_threads       = std::stoi(ARGV_NEXT); }
-        else if (arg == "-ot"   || arg == "--offset-t")        { params.offset_t_ms     = std::stoi(argv[++i]); }
+        else if (arg == "-p"    || arg == "--processors")      { params.n_processors    = std::stoi(ARGV_NEXT); }
-        else if (arg == "-on"   || arg == "--offset-n")        { params.offset_n        = std::stoi(argv[++i]); }
+        else if (arg == "-ot"   || arg == "--offset-t")        { params.offset_t_ms     = std::stoi(ARGV_NEXT); }
-        else if (arg == "-d"    || arg == "--duration")        { params.duration_ms     = std::stoi(argv[++i]); }
+        else if (arg == "-on"   || arg == "--offset-n")        { params.offset_n        = std::stoi(ARGV_NEXT); }
-        else if (arg == "-mc"   || arg == "--max-context")     { params.max_context     = std::stoi(argv[++i]); }
+        else if (arg == "-d"    || arg == "--duration")        { params.duration_ms     = std::stoi(ARGV_NEXT); }
-        else if (arg == "-ml"   || arg == "--max-len")         { params.max_len         = std::stoi(argv[++i]); }
+        else if (arg == "-mc"   || arg == "--max-context")     { params.max_context     = std::stoi(ARGV_NEXT); }
-        else if (arg == "-bo"   || arg == "--best-of")         { params.best_of         = std::stoi(argv[++i]); }
+        else if (arg == "-ml"   || arg == "--max-len")         { params.max_len         = std::stoi(ARGV_NEXT); }
-        else if (arg == "-bs"   || arg == "--beam-size")       { params.beam_size       = std::stoi(argv[++i]); }
+        else if (arg == "-bo"   || arg == "--best-of")         { params.best_of         = std::stoi(ARGV_NEXT); }
-        else if (arg == "-ac"   || arg == "--audio-ctx")       { params.audio_ctx       = std::stoi(argv[++i]); }
+        else if (arg == "-bs"   || arg == "--beam-size")       { params.beam_size       = std::stoi(ARGV_NEXT); }
-        else if (arg == "-wt"   || arg == "--word-thold")      { params.word_thold      = std::stof(argv[++i]); }
+        else if (arg == "-ac"   || arg == "--audio-ctx")       { params.audio_ctx       = std::stoi(ARGV_NEXT); }
-        else if (arg == "-et"   || arg == "--entropy-thold")   { params.entropy_thold   = std::stof(argv[++i]); }
+        else if (arg == "-wt"   || arg == "--word-thold")      { params.word_thold      = std::stof(ARGV_NEXT); }
-        else if (arg == "-lpt"  || arg == "--logprob-thold")   { params.logprob_thold   = std::stof(argv[++i]); }
+        else if (arg == "-et"   || arg == "--entropy-thold")   { params.entropy_thold   = std::stof(ARGV_NEXT); }
-        else if (arg == "-tp"   || arg == "--temperature")     { params.temperature     = std::stof(argv[++i]); }
+        else if (arg == "-lpt"  || arg == "--logprob-thold")   { params.logprob_thold   = std::stof(ARGV_NEXT); }
-        else if (arg == "-tpi"  || arg == "--temperature-inc") { params.temperature_inc = std::stof(argv[++i]); }
+        else if (arg == "-nth"  || arg == "--no-speech-thold") { params.no_speech_thold = std::stof(ARGV_NEXT); }
+        else if (arg == "-tp"   || arg == "--temperature")     { params.temperature     = std::stof(ARGV_NEXT); }
+        else if (arg == "-tpi"  || arg == "--temperature-inc") { params.temperature_inc = std::stof(ARGV_NEXT); }
         else if (arg == "-debug"|| arg == "--debug-mode")      { params.debug_mode      = true; }
         else if (arg == "-tr"   || arg == "--translate")       { params.translate       = true; }
         else if (arg == "-di"   || arg == "--diarize")         { params.diarize         = true; }
@@ -148,30 +164,31 @@ static bool whisper_params_parse(int argc, char ** argv, whisper_params & params
         else if (arg == "-osrt" || arg == "--output-srt")      { params.output_srt      = true; }
         else if (arg == "-owts" || arg == "--output-words")    { params.output_wts      = true; }
         else if (arg == "-olrc" || arg == "--output-lrc")      { params.output_lrc      = true; }
-        else if (arg == "-fp"   || arg == "--font-path")       { params.font_path       = argv[++i]; }
+        else if (arg == "-fp"   || arg == "--font-path")       { params.font_path       = ARGV_NEXT; }
         else if (arg == "-ocsv" || arg == "--output-csv")      { params.output_csv      = true; }
         else if (arg == "-oj"   || arg == "--output-json")     { params.output_jsn      = true; }
         else if (arg == "-ojf"  || arg == "--output-json-full"){ params.output_jsn_full = params.output_jsn = true; }
-        else if (arg == "-of"   || arg == "--output-file")     { params.fname_out.emplace_back(argv[++i]); }
+        else if (arg == "-of"   || arg == "--output-file")     { params.fname_out.emplace_back(ARGV_NEXT); }
         else if (arg == "-np"   || arg == "--no-prints")       { params.no_prints       = true; }
         else if (arg == "-ps"   || arg == "--print-special")   { params.print_special   = true; }
         else if (arg == "-pc"   || arg == "--print-colors")    { params.print_colors    = true; }
         else if (arg == "-pp"   || arg == "--print-progress")  { params.print_progress  = true; }
         else if (arg == "-nt"   || arg == "--no-timestamps")   { params.no_timestamps   = true; }
-        else if (arg == "-l"    || arg == "--language")        { params.language        = whisper_param_turn_lowercase(argv[++i]); }
+        else if (arg == "-l"    || arg == "--language")        { params.language        = whisper_param_turn_lowercase(ARGV_NEXT); }
         else if (arg == "-dl"   || arg == "--detect-language") { params.detect_language = true; }
-        else if (                  arg == "--prompt")          { params.prompt          = argv[++i]; }
+        else if (                  arg == "--prompt")          { params.prompt          = ARGV_NEXT; }
-        else if (arg == "-m"    || arg == "--model")           { params.model           = argv[++i]; }
+        else if (arg == "-m"    || arg == "--model")           { params.model           = ARGV_NEXT; }
-        else if (arg == "-f"    || arg == "--file")            { params.fname_inp.emplace_back(argv[++i]); }
+        else if (arg == "-f"    || arg == "--file")            { params.fname_inp.emplace_back(ARGV_NEXT); }
-        else if (arg == "-oved" || arg == "--ov-e-device")     { params.openvino_encode_device = argv[++i]; }
+        else if (arg == "-oved" || arg == "--ov-e-device")     { params.openvino_encode_device = ARGV_NEXT; }
-        else if (arg == "-dtw"  || arg == "--dtw")             { params.dtw             = argv[++i]; }
+        else if (arg == "-dtw"  || arg == "--dtw")             { params.dtw             = ARGV_NEXT; }
         else if (arg == "-ls"   || arg == "--log-score")       { params.log_score       = true; }
         else if (arg == "-ng"   || arg == "--no-gpu")          { params.use_gpu         = false; }
         else if (arg == "-fa"   || arg == "--flash-attn")      { params.flash_attn      = true; }
-        else if (                  arg == "--suppress-regex")  { params.suppress_regex  = argv[++i]; }
+        else if (arg == "-sns"  || arg == "--suppress-nst")    { params.suppress_nst    = true; }
-        else if (                  arg == "--grammar")         { params.grammar         = argv[++i]; }
+        else if (                  arg == "--suppress-regex")  { params.suppress_regex  = ARGV_NEXT; }
-        else if (                  arg == "--grammar-rule")    { params.grammar_rule    = argv[++i]; }
+        else if (                  arg == "--grammar")         { params.grammar         = ARGV_NEXT; }
-        else if (                  arg == "--grammar-penalty") { params.grammar_penalty = std::stof(argv[++i]); }
+        else if (                  arg == "--grammar-rule")    { params.grammar_rule    = ARGV_NEXT; }
+        else if (                  arg == "--grammar-penalty") { params.grammar_penalty = std::stof(ARGV_NEXT); }
         else {
             fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());
             whisper_print_usage(argc, argv, params);
@@ -184,7 +201,8 @@ static bool whisper_params_parse(int argc, char ** argv, whisper_params & params
 
 static void whisper_print_usage(int /*argc*/, char ** argv, const whisper_params & params) {
     fprintf(stderr, "\n");
-    fprintf(stderr, "usage: %s [options] file0.wav file1.wav ...\n", argv[0]);
+    fprintf(stderr, "usage: %s [options] file0 file1 ...\n", argv[0]);
+    fprintf(stderr, "supported audio formats: flac, mp3, ogg, wav\n");
     fprintf(stderr, "\n");
     fprintf(stderr, "options:\n");
     fprintf(stderr, "  -h,        --help              [default] show this help message and exit\n");
@@ -202,6 +220,7 @@ static void whisper_print_usage(int /*argc*/, char ** argv, const whisper_params
     fprintf(stderr, "  -wt N,     --word-thold N      [%-7.2f] word timestamp probability threshold\n",         params.word_thold);
     fprintf(stderr, "  -et N,     --entropy-thold N   [%-7.2f] entropy threshold for decoder fail\n",           params.entropy_thold);
     fprintf(stderr, "  -lpt N,    --logprob-thold N   [%-7.2f] log probability threshold for decoder fail\n",   params.logprob_thold);
+    fprintf(stderr, "  -nth N,    --no-speech-thold N [%-7.2f] no speech threshold\n",                          params.no_speech_thold);
     fprintf(stderr, "  -tp,       --temperature N     [%-7.2f] The sampling temperature, between 0 and 1\n",    params.temperature);
     fprintf(stderr, "  -tpi,      --temperature-inc N [%-7.2f] The increment of temperature, between 0 and 1\n",params.temperature_inc);
     fprintf(stderr, "  -debug,    --debug-mode        [%-7s] enable debug mode (eg. dump log_mel)\n",           params.debug_mode ? "true" : "false");
@@ -228,12 +247,13 @@ static void whisper_print_usage(int /*argc*/, char ** argv, const whisper_params
     fprintf(stderr, "  -dl,       --detect-language   [%-7s] exit after automatically detecting language\n",    params.detect_language ? "true" : "false");
     fprintf(stderr, "             --prompt PROMPT     [%-7s] initial prompt (max n_text_ctx/2 tokens)\n",       params.prompt.c_str());
     fprintf(stderr, "  -m FNAME,  --model FNAME       [%-7s] model path\n",                                     params.model.c_str());
-    fprintf(stderr, "  -f FNAME,  --file FNAME        [%-7s] input WAV file path\n",                            "");
+    fprintf(stderr, "  -f FNAME,  --file FNAME        [%-7s] input audio file path\n",                            "");
     fprintf(stderr, "  -oved D,   --ov-e-device DNAME [%-7s] the OpenVINO device used for encode inference\n",  params.openvino_encode_device.c_str());
     fprintf(stderr, "  -dtw MODEL --dtw MODEL         [%-7s] compute token-level timestamps\n",                 params.dtw.c_str());
     fprintf(stderr, "  -ls,       --log-score         [%-7s] log best decoder scores of tokens\n",              params.log_score?"true":"false");
     fprintf(stderr, "  -ng,       --no-gpu            [%-7s] disable GPU\n",                                    params.use_gpu ? "false" : "true");
     fprintf(stderr, "  -fa,       --flash-attn        [%-7s] flash attention\n",                                params.flash_attn ? "true" : "false");
+    fprintf(stderr, "  -sns,      --suppress-nst      [%-7s] suppress non-speech tokens\n",                     params.suppress_nst ? "true" : "false");
     fprintf(stderr, "  --suppress-regex REGEX         [%-7s] regular expression matching tokens to suppress\n", params.suppress_regex.c_str());
     fprintf(stderr, "  --grammar GRAMMAR              [%-7s] GBNF grammar to guide decoding\n",                 params.grammar.c_str());
     fprintf(stderr, "  --grammar-rule RULE            [%-7s] top-level GBNF grammar rule name\n",               params.grammar_rule.c_str());
@@ -904,6 +924,13 @@ static bool output_lrc(struct whisper_context * ctx, const char * fname, const w
 static void cb_log_disable(enum ggml_log_level , const char * , void * ) { }
 
 int main(int argc, char ** argv) {
+#if defined(_WIN32)
+    // Set the console output code page to UTF-8, while command line arguments
+    // are still encoded in the system's code page. In this way, we can print
+    // non-ASCII characters to the console, and access files with non-ASCII paths.
+    SetConsoleOutputCP(CP_UTF8);
+#endif
+
     whisper_params params;
 
     // If the only argument starts with "@", read arguments line-by-line
@@ -1045,8 +1072,8 @@ int main(int argc, char ** argv) {
         std::vector<float> pcmf32;               // mono-channel F32 PCM
         std::vector<std::vector<float>> pcmf32s; // stereo-channel F32 PCM
 
-        if (!::read_wav(fname_inp, pcmf32, pcmf32s, params.diarize)) {
+        if (!::read_audio_data(fname_inp, pcmf32, pcmf32s, params.diarize)) {
-            fprintf(stderr, "error: failed to read WAV file '%s'\n", fname_inp.c_str());
+            fprintf(stderr, "error: failed to read audio file '%s'\n", fname_inp.c_str());
             continue;
         }
 
@@ -1121,9 +1148,12 @@ int main(int argc, char ** argv) {
 
             wparams.entropy_thold    = params.entropy_thold;
             wparams.logprob_thold    = params.logprob_thold;
+            wparams.no_speech_thold  = params.no_speech_thold;
 
             wparams.no_timestamps    = params.no_timestamps;
 
+            wparams.suppress_nst     = params.suppress_nst;
+
             whisper_print_user_data user_data = { &params, &pcmf32s, 0 };
 
             const auto & grammar_parsed = params.grammar_parsed;

examples/coi-serviceworker.js

@@ -0,0 +1,146 @@
+/*! coi-serviceworker v0.1.7 - Guido Zuidhof and contributors, licensed under MIT */
+let coepCredentialless = false;
+if (typeof window === 'undefined') {
+    self.addEventListener("install", () => self.skipWaiting());
+    self.addEventListener("activate", (event) => event.waitUntil(self.clients.claim()));
+
+    self.addEventListener("message", (ev) => {
+        if (!ev.data) {
+            return;
+        } else if (ev.data.type === "deregister") {
+            self.registration
+                .unregister()
+                .then(() => {
+                    return self.clients.matchAll();
+                })
+                .then(clients => {
+                    clients.forEach((client) => client.navigate(client.url));
+                });
+        } else if (ev.data.type === "coepCredentialless") {
+            coepCredentialless = ev.data.value;
+        }
+    });
+
+    self.addEventListener("fetch", function (event) {
+        const r = event.request;
+        if (r.cache === "only-if-cached" && r.mode !== "same-origin") {
+            return;
+        }
+
+        const request = (coepCredentialless && r.mode === "no-cors")
+            ? new Request(r, {
+                credentials: "omit",
+            })
+            : r;
+        event.respondWith(
+            fetch(request)
+                .then((response) => {
+                    if (response.status === 0) {
+                        return response;
+                    }
+
+                    const newHeaders = new Headers(response.headers);
+                    newHeaders.set("Cross-Origin-Embedder-Policy",
+                        coepCredentialless ? "credentialless" : "require-corp"
+                    );
+                    if (!coepCredentialless) {
+                        newHeaders.set("Cross-Origin-Resource-Policy", "cross-origin");
+                    }
+                    newHeaders.set("Cross-Origin-Opener-Policy", "same-origin");
+
+                    return new Response(response.body, {
+                        status: response.status,
+                        statusText: response.statusText,
+                        headers: newHeaders,
+                    });
+                })
+                .catch((e) => console.error(e))
+        );
+    });
+
+} else {
+    (() => {
+        const reloadedBySelf = window.sessionStorage.getItem("coiReloadedBySelf");
+        window.sessionStorage.removeItem("coiReloadedBySelf");
+        const coepDegrading = (reloadedBySelf == "coepdegrade");
+
+        // You can customize the behavior of this script through a global `coi` variable.
+        const coi = {
+            shouldRegister: () => !reloadedBySelf,
+            shouldDeregister: () => false,
+            coepCredentialless: () => true,
+            coepDegrade: () => true,
+            doReload: () => window.location.reload(),
+            quiet: false,
+            ...window.coi
+        };
+
+        const n = navigator;
+        const controlling = n.serviceWorker && n.serviceWorker.controller;
+
+        // Record the failure if the page is served by serviceWorker.
+        if (controlling && !window.crossOriginIsolated) {
+            window.sessionStorage.setItem("coiCoepHasFailed", "true");
+        }
+        const coepHasFailed = window.sessionStorage.getItem("coiCoepHasFailed");
+
+        if (controlling) {
+            // Reload only on the first failure.
+            const reloadToDegrade = coi.coepDegrade() && !(
+                coepDegrading || window.crossOriginIsolated
+            );
+            n.serviceWorker.controller.postMessage({
+                type: "coepCredentialless",
+                value: (reloadToDegrade || coepHasFailed && coi.coepDegrade())
+                    ? false
+                    : coi.coepCredentialless(),
+            });
+            if (reloadToDegrade) {
+                !coi.quiet && console.log("Reloading page to degrade COEP.");
+                window.sessionStorage.setItem("coiReloadedBySelf", "coepdegrade");
+                coi.doReload("coepdegrade");
+            }
+
+            if (coi.shouldDeregister()) {
+                n.serviceWorker.controller.postMessage({ type: "deregister" });
+            }
+        }
+
+        // If we're already coi: do nothing. Perhaps it's due to this script doing its job, or COOP/COEP are
+        // already set from the origin server. Also if the browser has no notion of crossOriginIsolated, just give up here.
+        if (window.crossOriginIsolated !== false || !coi.shouldRegister()) return;
+
+        if (!window.isSecureContext) {
+            !coi.quiet && console.log("COOP/COEP Service Worker not registered, a secure context is required.");
+            return;
+        }
+
+        // In some environments (e.g. Firefox private mode) this won't be available
+        if (!n.serviceWorker) {
+            !coi.quiet && console.error("COOP/COEP Service Worker not registered, perhaps due to private mode.");
+            return;
+        }
+
+        n.serviceWorker.register(window.document.currentScript.src).then(
+            (registration) => {
+                !coi.quiet && console.log("COOP/COEP Service Worker registered", registration.scope);
+
+                registration.addEventListener("updatefound", () => {
+                    !coi.quiet && console.log("Reloading page to make use of updated COOP/COEP Service Worker.");
+                    window.sessionStorage.setItem("coiReloadedBySelf", "updatefound");
+                    coi.doReload();
+                });
+
+                // If the registration is active, but it's not controlling the page
+                if (registration.active && !n.serviceWorker.controller) {
+                    !coi.quiet && console.log("Reloading page to make use of COOP/COEP Service Worker.");
+                    window.sessionStorage.setItem("coiReloadedBySelf", "notcontrolling");
+                    coi.doReload();
+                }
+            },
+            (err) => {
+                !coi.quiet && console.error("COOP/COEP Service Worker failed to register:", err);
+            }
+        );
+    })();
+}

examples/command.wasm/README.md

@@ -3,7 +3,7 @@
 This is a basic Voice Assistant example that accepts voice commands from the microphone.
 It runs in fully in the browser via WebAseembly.
 
-Online demo: https://whisper.ggerganov.com/command/
+Online demo: https://ggerganov.github.io/whisper.cpp/command.wasm
 
 Terminal version: [examples/command](/examples/command)
 
@@ -15,9 +15,18 @@ git clone https://github.com/ggerganov/whisper.cpp
 cd whisper.cpp
 mkdir build-em && cd build-em
 emcmake cmake ..
-make -j
+make -j libcommand
+```
+The example can then be started by running a local HTTP server:
+```console
+python3 examples/server.py
+```
+And then opening a browser to the following URL:
+http://localhost:8000/command.wasm/
 
-# copy the produced page to your HTTP path
+To run the example in a different server, you need to copy the following files
+to the server's HTTP path:
+```
 cp bin/command.wasm/*       /path/to/html/
 cp bin/libcommand.worker.js /path/to/html/
 ```

examples/command.wasm/index-tmpl.html

@@ -24,6 +24,8 @@
                 overflow-x: scroll;
             }
         </style>
+        <script src="../coi-serviceworker.js"></script>
+        <link rel="icon" href="data:,">
     </head>
     <body>
         <div id="main-container">
@@ -36,11 +38,10 @@
             <br><br>
 
             <b>More examples:</b>
-                <a href="https://whisper.ggerganov.com/">main</a> |
+                <a href="../">main</a> |
-                <a href="https://whisper.ggerganov.com/bench">bench</a> |
+                <a href="../bench.wasm/">bench</a> |
-                <a href="https://whisper.ggerganov.com/stream">stream</a> |
+                <a href="../stream.wasm">stream</a> |
-                <a href="https://whisper.ggerganov.com/command">command</a> |
+                <a href="../command.wasm/">command</a> |
-                <a href="https://whisper.ggerganov.com/talk">talk</a> |
 
             <br><br>
 

examples/command/CMakeLists.txt

@@ -1,9 +1,10 @@
 if (WHISPER_SDL2)
-    # command
+    set(TARGET whisper-command)
-    set(TARGET command)
     add_executable(${TARGET} command.cpp)
 
     include(DefaultTargetOptions)
 
     target_link_libraries(${TARGET} PRIVATE common common-sdl whisper ${CMAKE_THREAD_LIBS_INIT})
+
+    install(TARGETS ${TARGET} RUNTIME)
 endif ()

examples/command/README.md

@@ -1,14 +1,14 @@
-# command
+# whisper.cpp/examples/command
 
 This is a basic Voice Assistant example that accepts voice commands from the microphone.
 More info is available in [issue #171](https://github.com/ggerganov/whisper.cpp/issues/171).
 
 ```bash
 # Run with default arguments and small model
-./command -m ./models/ggml-small.en.bin -t 8
+./whisper-command -m ./models/ggml-small.en.bin -t 8
 
 # On Raspberry Pi, use tiny or base models + "-ac 768" for better performance
-./command -m ./models/ggml-tiny.en.bin -ac 768 -t 3 -c 0
+./whisper-command -m ./models/ggml-tiny.en.bin -ac 768 -t 3 -c 0
 ```
 
 https://user-images.githubusercontent.com/1991296/204038393-2f846eae-c255-4099-a76d-5735c25c49da.mp4
@@ -23,10 +23,10 @@ Initial tests show that this approach might be extremely efficient in terms of p
 
 ```bash
 # Run in guided mode, the list of allowed commands is in commands.txt
-./command -m ./models/ggml-base.en.bin -cmd ./examples/command/commands.txt
+./whisper-command -m ./models/ggml-base.en.bin -cmd ./examples/command/commands.txt
 
 # On Raspberry Pi, in guided mode you can use "-ac 128" for extra performance
-./command -m ./models/ggml-tiny.en.bin -cmd ./examples/command/commands.txt -ac 128 -t 3 -c 0
+./whisper-command -m ./models/ggml-tiny.en.bin -cmd ./examples/command/commands.txt -ac 128 -t 3 -c 0
 ```
 
 https://user-images.githubusercontent.com/1991296/207435352-8fc4ed3f-bde5-4555-9b8b-aeeb76bee969.mp4
@@ -34,7 +34,7 @@ https://user-images.githubusercontent.com/1991296/207435352-8fc4ed3f-bde5-4555-9
 
 ## Building
 
-The `command` tool depends on SDL2 library to capture audio from the microphone. You can build it like this:
+The `whisper-command` tool depends on SDL2 library to capture audio from the microphone. You can build it like this:
 
 ```bash
 # Install SDL2
@@ -47,5 +47,6 @@ sudo dnf install SDL2 SDL2-devel
 # Install SDL2 on Mac OS
 brew install sdl2
 
-make command
+cmake -B build -DWHISPER_SDL2=ON
+cmake --build build --config Release
 ```

examples/command/command.cpp

@@ -3,7 +3,7 @@
 // Speak short text commands to the microphone.
 // This program will detect your voice command and convert them to text.
 //
-// ref: https://github.com/ggerganov/whisper.cpp/issues/171
+// ref: https://github.com/ggml-org/whisper.cpp/issues/171
 //
 
 #include "common-sdl.h"
@@ -11,16 +11,15 @@
 #include "whisper.h"
 #include "grammar-parser.h"
 
-#include <sstream>
+#include <algorithm>
-#include <cassert>
+#include <chrono>
 #include <cstdio>
 #include <fstream>
-#include <mutex>
+#include <map>
-#include <regex>
+#include <sstream>
 #include <string>
 #include <thread>
 #include <vector>
-#include <map>
 
 // command-line parameters
 struct whisper_params {

examples/common-sdl.cpp

@@ -159,15 +159,11 @@ void audio_async::callback(uint8_t * stream, int len) {
 
             memcpy(&m_audio[m_audio_pos], stream, n0 * sizeof(float));
             memcpy(&m_audio[0], stream + n0 * sizeof(float), (n_samples - n0) * sizeof(float));
-
-            m_audio_pos = (m_audio_pos + n_samples) % m_audio.size();
-            m_audio_len = m_audio.size();
         } else {
             memcpy(&m_audio[m_audio_pos], stream, n_samples * sizeof(float));
-
-            m_audio_pos = (m_audio_pos + n_samples) % m_audio.size();
-            m_audio_len = std::min(m_audio_len + n_samples, m_audio.size());
         }
+        m_audio_pos = (m_audio_pos + n_samples) % m_audio.size();
+        m_audio_len = std::min(m_audio_len + n_samples, m_audio.size());
     }
 }