2022-09-25 18:23:15 +00:00
|
|
|
#pragma once
|
|
|
|
|
|
2022-11-09 19:41:21 +00:00
|
|
|
//
|
|
|
|
|
// GGML Tensor Library
|
|
|
|
|
//
|
|
|
|
|
// This documentation is still a work in progress.
|
|
|
|
|
// If you wish some specific topics to be covered, feel free to drop a comment:
|
|
|
|
|
//
|
|
|
|
|
// https://github.com/ggerganov/whisper.cpp/issues/40
|
|
|
|
|
//
|
|
|
|
|
// ## Overview
|
|
|
|
|
//
|
|
|
|
|
// This library implements:
|
|
|
|
|
//
|
|
|
|
|
// - a set of tensor operations
|
|
|
|
|
// - automatic differentiation
|
|
|
|
|
// - basic optimization algorithms
|
|
|
|
|
//
|
|
|
|
|
// The aim of this library is to provide a minimalistic approach for various machine learning tasks. This includes,
|
|
|
|
|
// but is not limited to, the following:
|
|
|
|
|
//
|
|
|
|
|
// - linear regression
|
|
|
|
|
// - support vector machines
|
|
|
|
|
// - neural networks
|
|
|
|
|
//
|
|
|
|
|
// The library allows the user to define a certain function using the available tensor operations. This function
|
|
|
|
|
// definition is represented internally via a computation graph. Each tensor operation in the function definition
|
|
|
|
|
// corresponds to a node in the graph. Having the computation graph defined, the user can choose to compute the
|
|
|
|
|
// function's value and/or its gradient with respect to the input variables. Optionally, the function can be optimized
|
|
|
|
|
// using one of the available optimization algorithms.
|
|
|
|
|
//
|
|
|
|
|
// For example, here we define the function: f(x) = a*x^2 + b
|
|
|
|
|
//
|
|
|
|
|
// {
|
|
|
|
|
// struct ggml_init_params params = {
|
|
|
|
|
// .mem_size = 16*1024*1024,
|
|
|
|
|
// .mem_buffer = NULL,
|
|
|
|
|
// };
|
|
|
|
|
//
|
|
|
|
|
// // memory allocation happens here
|
|
|
|
|
// struct ggml_context * ctx = ggml_init(params);
|
|
|
|
|
//
|
|
|
|
|
// struct ggml_tensor * x = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1);
|
|
|
|
|
//
|
|
|
|
|
// ggml_set_param(ctx, x); // x is an input variable
|
|
|
|
|
//
|
|
|
|
|
// struct ggml_tensor * a = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1);
|
|
|
|
|
// struct ggml_tensor * b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1);
|
|
|
|
|
// struct ggml_tensor * x2 = ggml_mul(ctx, x, x);
|
|
|
|
|
// struct ggml_tensor * f = ggml_add(ctx, ggml_mul(ctx, a, x2), b);
|
|
|
|
|
//
|
|
|
|
|
// ...
|
|
|
|
|
// }
|
|
|
|
|
//
|
|
|
|
|
// Notice that the function definition above does not involve any actual computation. The computation is performed only
|
|
|
|
|
// when the user explicitly requests it. For example, to compute the function's value at x = 2.0:
|
|
|
|
|
//
|
|
|
|
|
// {
|
|
|
|
|
// ...
|
|
|
|
|
//
|
2023-11-03 19:35:05 +00:00
|
|
|
// struct ggml_cgraph * gf = ggml_new_graph(ctx);
|
|
|
|
|
// ggml_build_forward_expand(gf, f);
|
2022-11-09 19:41:21 +00:00
|
|
|
//
|
|
|
|
|
// // set the input variable and parameter values
|
|
|
|
|
// ggml_set_f32(x, 2.0f);
|
|
|
|
|
// ggml_set_f32(a, 3.0f);
|
|
|
|
|
// ggml_set_f32(b, 4.0f);
|
|
|
|
|
//
|
2023-09-05 10:54:40 +00:00
|
|
|
// ggml_graph_compute_with_ctx(ctx, &gf, n_threads);
|
2022-11-09 19:41:21 +00:00
|
|
|
//
|
|
|
|
|
// printf("f = %f\n", ggml_get_f32_1d(f, 0));
|
|
|
|
|
//
|
|
|
|
|
// ...
|
|
|
|
|
// }
|
|
|
|
|
//
|
|
|
|
|
// The actual computation is performed in the ggml_graph_compute() function.
|
|
|
|
|
//
|
|
|
|
|
// The ggml_new_tensor_...() functions create new tensors. They are allocated in the memory buffer provided to the
|
|
|
|
|
// ggml_init() function. You have to be careful not to exceed the memory buffer size. Therefore, you have to know
|
|
|
|
|
// in advance how much memory you need for your computation. Alternatively, you can allocate a large enough memory
|
|
|
|
|
// and after defining the computation graph, call the ggml_used_mem() function to find out how much memory was
|
|
|
|
|
// actually needed.
|
|
|
|
|
//
|
|
|
|
|
// The ggml_set_param() function marks a tensor as an input variable. This is used by the automatic
|
|
|
|
|
// differentiation and optimization algorithms.
|
|
|
|
|
//
|
|
|
|
|
// The described approach allows to define the function graph once and then compute its forward or backward graphs
|
|
|
|
|
// multiple times. All computations will use the same memory buffer allocated in the ggml_init() function. This way
|
|
|
|
|
// the user can avoid the memory allocation overhead at runtime.
|
|
|
|
|
//
|
|
|
|
|
// The library supports multi-dimensional tensors - up to 4 dimensions. The FP16 and FP32 data types are first class
|
|
|
|
|
// citizens, but in theory the library can be extended to support FP8 and integer data types.
|
|
|
|
|
//
|
|
|
|
|
// Each tensor operation produces a new tensor. Initially the library was envisioned to support only the use of unary
|
|
|
|
|
// and binary operations. Most of the available operations fall into one of these two categories. With time, it became
|
|
|
|
|
// clear that the library needs to support more complex operations. The way to support these operations is not clear
|
|
|
|
|
// yet, but a few examples are demonstrated in the following operations:
|
|
|
|
|
//
|
|
|
|
|
// - ggml_permute()
|
|
|
|
|
// - ggml_conv_1d_1s()
|
|
|
|
|
// - ggml_conv_1d_2s()
|
|
|
|
|
//
|
|
|
|
|
// For each tensor operator, the library implements a forward and backward computation function. The forward function
|
|
|
|
|
// computes the output tensor value given the input tensor values. The backward function computes the adjoint of the
|
|
|
|
|
// input tensors given the adjoint of the output tensor. For a detailed explanation of what this means, take a
|
|
|
|
|
// calculus class, or watch the following video:
|
|
|
|
|
//
|
|
|
|
|
// What is Automatic Differentiation?
|
|
|
|
|
// https://www.youtube.com/watch?v=wG_nF1awSSY
|
|
|
|
|
//
|
|
|
|
|
//
|
|
|
|
|
// ## Tensor data (struct ggml_tensor)
|
|
|
|
|
//
|
|
|
|
|
// The tensors are stored in memory via the ggml_tensor struct. The structure provides information about the size of
|
|
|
|
|
// the tensor, the data type, and the memory buffer where the tensor data is stored. Additionally, it contains
|
|
|
|
|
// pointers to the "source" tensors - i.e. the tensors that were used to compute the current tensor. For example:
|
|
|
|
|
//
|
|
|
|
|
// {
|
|
|
|
|
// struct ggml_tensor * c = ggml_add(ctx, a, b);
|
|
|
|
|
//
|
|
|
|
|
// assert(c->src[0] == a);
|
|
|
|
|
// assert(c->src[1] == b);
|
|
|
|
|
// }
|
|
|
|
|
//
|
|
|
|
|
// The multi-dimensional tensors are stored in row-major order. The ggml_tensor struct contains fields for the
|
|
|
|
|
// number of elements in each dimension ("ne") as well as the number of bytes ("nb", a.k.a. stride). This allows
|
|
|
|
|
// to store tensors that are not contiguous in memory, which is useful for operations such as transposition and
|
|
|
|
|
// permutation. All tensor operations have to take the stride into account and not assume that the tensor is
|
|
|
|
|
// contiguous in memory.
|
|
|
|
|
//
|
|
|
|
|
// The data of the tensor is accessed via the "data" pointer. For example:
|
|
|
|
|
//
|
|
|
|
|
// {
|
2023-09-05 10:54:40 +00:00
|
|
|
// const int nx = 2;
|
|
|
|
|
// const int ny = 3;
|
2022-11-09 19:41:21 +00:00
|
|
|
//
|
2023-09-05 10:54:40 +00:00
|
|
|
// struct ggml_tensor * a = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, nx, ny);
|
2022-11-09 19:41:21 +00:00
|
|
|
//
|
2023-09-05 10:54:40 +00:00
|
|
|
// for (int y = 0; y < ny; y++) {
|
|
|
|
|
// for (int x = 0; x < nx; x++) {
|
|
|
|
|
// *(float *) ((char *) a->data + y*a->nb[1] + x*a->nb[0]) = x + y;
|
|
|
|
|
// }
|
|
|
|
|
// }
|
2022-11-09 19:41:21 +00:00
|
|
|
//
|
|
|
|
|
// ...
|
|
|
|
|
// }
|
|
|
|
|
//
|
|
|
|
|
// Alternatively, there are helper functions, such as ggml_get_f32_1d() and ggml_set_f32_1d() that can be used.
|
|
|
|
|
//
|
|
|
|
|
// ## The matrix multiplication operator (ggml_mul_mat)
|
|
|
|
|
//
|
|
|
|
|
// TODO
|
|
|
|
|
//
|
|
|
|
|
//
|
|
|
|
|
// ## Multi-threading
|
|
|
|
|
//
|
|
|
|
|
// TODO
|
|
|
|
|
//
|
|
|
|
|
//
|
|
|
|
|
// ## Overview of ggml.c
|
|
|
|
|
//
|
|
|
|
|
// TODO
|
|
|
|
|
//
|
|
|
|
|
//
|
|
|
|
|
// ## SIMD optimizations
|
|
|
|
|
//
|
|
|
|
|
// TODO
|
|
|
|
|
//
|
|
|
|
|
//
|
|
|
|
|
// ## Debugging ggml
|
|
|
|
|
//
|
|
|
|
|
// TODO
|
|
|
|
|
//
|
|
|
|
|
//
|
|
|
|
|
|
2023-04-29 09:31:52 +00:00
|
|
|
#ifdef GGML_SHARED
|
|
|
|
|
# if defined(_WIN32) && !defined(__MINGW32__)
|
|
|
|
|
# ifdef GGML_BUILD
|
2024-11-14 17:04:35 +00:00
|
|
|
# define GGML_API __declspec(dllexport) extern
|
2023-04-29 09:31:52 +00:00
|
|
|
# else
|
2024-11-14 17:04:35 +00:00
|
|
|
# define GGML_API __declspec(dllimport) extern
|
2023-04-29 09:31:52 +00:00
|
|
|
# endif
|
|
|
|
|
# else
|
2024-11-14 17:04:35 +00:00
|
|
|
# define GGML_API __attribute__ ((visibility ("default"))) extern
|
2023-04-29 09:31:52 +00:00
|
|
|
# endif
|
|
|
|
|
#else
|
2024-11-14 17:04:35 +00:00
|
|
|
# define GGML_API extern
|
2022-09-25 18:23:15 +00:00
|
|
|
#endif
|
|
|
|
|
|
2023-09-05 10:54:40 +00:00
|
|
|
// TODO: support for clang
|
|
|
|
|
#ifdef __GNUC__
|
|
|
|
|
# define GGML_DEPRECATED(func, hint) func __attribute__((deprecated(hint)))
|
|
|
|
|
#elif defined(_MSC_VER)
|
|
|
|
|
# define GGML_DEPRECATED(func, hint) __declspec(deprecated(hint)) func
|
|
|
|
|
#else
|
|
|
|
|
# define GGML_DEPRECATED(func, hint) func
|
|
|
|
|
#endif
|
|
|
|
|
|
2023-09-15 11:49:56 +00:00
|
|
|
#ifndef __GNUC__
|
|
|
|
|
# define GGML_ATTRIBUTE_FORMAT(...)
|
|
|
|
|
#elif defined(__MINGW32__)
|
|
|
|
|
# define GGML_ATTRIBUTE_FORMAT(...) __attribute__((format(gnu_printf, __VA_ARGS__)))
|
|
|
|
|
#else
|
|
|
|
|
# define GGML_ATTRIBUTE_FORMAT(...) __attribute__((format(printf, __VA_ARGS__)))
|
|
|
|
|
#endif
|
|
|
|
|
|
2022-09-25 18:23:15 +00:00
|
|
|
#include <stdbool.h>
|
2024-03-27 16:55:10 +00:00
|
|
|
#include <stddef.h>
|
|
|
|
|
#include <stdint.h>
|
|
|
|
|
#include <stdio.h>
|
2022-09-25 18:23:15 +00:00
|
|
|
|
2023-04-29 09:31:52 +00:00
|
|
|
#define GGML_FILE_MAGIC 0x67676d6c // "ggml"
|
2024-07-30 13:56:35 +00:00
|
|
|
#define GGML_FILE_VERSION 2
|
2023-04-29 09:31:52 +00:00
|
|
|
|
2023-05-20 15:56:30 +00:00
|
|
|
#define GGML_QNT_VERSION 2 // bump this on quantization format changes
|
2023-05-14 15:04:23 +00:00
|
|
|
#define GGML_QNT_VERSION_FACTOR 1000 // do not change this
|
|
|
|
|
|
2023-11-03 19:35:05 +00:00
|
|
|
#define GGML_MAX_DIMS 4
|
2023-12-13 19:55:03 +00:00
|
|
|
#define GGML_MAX_PARAMS 2048
|
|
|
|
|
#define GGML_MAX_SRC 10
|
Threadpool: take 2 (llama/8672)
* Introduce ggml_compute_threadpool
- OpenMP functional: check
- Vanilla ggml functional: Check
- ggml w/threadpool functional: Check
- OpenMP no regression: No glaring problems
- Vanilla ggml no regression: No glaring problems
- ggml w/threadpool no regression: No glaring problems
* Minor fixes
* fixed use after release bug
* fixed a harmless race condition
* Fix Android bulid issue
* fix more race conditions
* fix deadlock for cases where cgraph.n_nodes == 1
and fix --poll case
* threadpool: use cpu_get_num_math to set the default number of threadpool threads
This way we avoid using E-Cores and Hyperthreaded siblings.
* bench: create fresh threadpool for each test
For benchmarking it's better to start a fresh pool for each test with the exact number of threads
needed for that test. Having larger pools is suboptimal (causes more load, etc).
* atomics: always use stdatomics with clang and use relaxed memory order when polling in ggml_barrier
This also removes sched_yield() calls from ggml_barrier() to match OpenMP behavior.
* threadpool: make polling the default to match openmp behavior
All command line args now allow for setting poll to 0 (false).
* threadpool: do not wakeup threads in already paused threadpool
* fix potential race condition in check_for_work
* threadpool: do not create two threadpools if their params are identical
* threadpool: reduce pause/resume/wakeup overhead in common cases
We now start threadpool in paused state only if we have two.
The resume is now implicit (ie new work) which allows for reduced locking and context-switch overhead.
* threadpool: add support for hybrid polling
poll params (--poll, ...) now specify "polling level", i.e. how aggresively we poll before waiting on cond.var.
poll=0 means no polling, 1 means poll for 128K rounds then wait, 2 for 256K rounds, ...
The default value of 50 (ie 50x128K rounds) seems like a decent default across modern platforms.
We can tune this further as things evolve.
* threadpool: reduce the number of barrier required
New work is now indicated with an atomic counter that is incremented for
each new graph that needs to be computed.
This removes the need for extra barrier for clearing the "new_work" and
removes the special case for trivial graphs.
* threadpool: remove special-casing for disposable threadpools
With the efficient hybrid polling there is no need to make disposable pools any different.
This simplifies the overall logic and reduces branching.
Include n_threads in debug print for disposable threadpool.
Declare pause and stop flags as atomic_bool
This doesn't actually generate any memory barriers and simply informs
the thread sanitizer that these flags can be written & read by different
threads without locking.
* threadpool: do not clear barrier counters between graphs computes (fixes race with small graphs)
This fixes the race condition with very small graphs where the main thread happens to
start a new graph while the workers are just about to exit from barriers.
* threadpool: use relaxed order for chunk sync
Full memory barrier is an overkill for this since each thread works on different chunk
* threadpool: remove abort_callback from threadpool state
* threadpool: better naming for thread/cpumask releated functions
* threadpool: consistent use of int type for n_threads params
* threadpool: add support for ggml_threadpool_params_default/init
Also removes the need for explicit mask_specified param.
all-zero cpumask means use default (usually inherited) cpu affinity mask.
* threadpool: move typedef into ggml.h
* threadpool: fix apply_priority() function name
* threadpool: fix swift wrapper errors due to n_threads int type cleanup
* threadpool: enable --cpu-mask and other threadpool related options only if threadpool is enabled
* threadpool: replace checks for compute_thread ret code with proper status check
* threadpool: simplify threadpool init logic and fix main thread affinity application
Most of the init code is now exactly the same between threadpool and openmp.
* threadpool: update threadpool resume/pause function names
* threadpool: enable openmp by default for now
* threadpool: don't forget to free workers state when omp is enabled
* threadpool: avoid updating process priority on the platforms that do not require it
On Windows we need to change overall process priority class in order to set thread priorities,
but on Linux, Mac, etc we do not need to touch the overall process settings.
* threadpool: update calling thread prio and affinity only at start/resume
This avoids extra syscalls for each graph_compute()
* llama-bench: turn threadpool params into vectors, add output headers, etc
* llama-bench: add support for cool off between tests --delay
This helps for long running tests on platforms that are thermally limited (phones, laptops, etc).
--delay (disabled by default) introduces the sleep for N seconds before starting each test.
* threadpool: move process priority setting into the apps (bench and cli)
This avoids changing the overall process priority on Windows for the apps
that use ggml/llama.cpp directy.
* threadpool: move all pause/resume logic into ggml
* threadpool: futher api cleanup and prep for future refactoring
All threadpool related functions and structs use ggml_threadpool prefix.
* threadpool: minor indent fixes
* threadpool: improve setprioty error message
* Update examples/llama-bench/llama-bench.cpp
Co-authored-by: slaren <slarengh@gmail.com>
* threadpool: fix indent in set_threadpool call
* use int32_t for n_thread type in public llama.cpp API
* threadpool: use _new and _free instead of _create and _release
* fix two more public APIs to use int32_t for n_threads
* build: set _GNU_SOURCE for Adroid
---------
Co-authored-by: Max Krasnyansky <quic_maxk@quicinc.com>
Co-authored-by: fmz <quic_fzaghlou@quic.com>
Co-authored-by: Max Krasnyansky <max.krasnyansky@gmail.com>
Co-authored-by: slaren <slarengh@gmail.com>
2024-08-29 23:20:53 +00:00
|
|
|
#define GGML_MAX_N_THREADS 512
|
2024-09-24 10:23:59 +00:00
|
|
|
#define GGML_MAX_OP_PARAMS 64
|
Threadpool: take 2 (llama/8672)
* Introduce ggml_compute_threadpool
- OpenMP functional: check
- Vanilla ggml functional: Check
- ggml w/threadpool functional: Check
- OpenMP no regression: No glaring problems
- Vanilla ggml no regression: No glaring problems
- ggml w/threadpool no regression: No glaring problems
* Minor fixes
* fixed use after release bug
* fixed a harmless race condition
* Fix Android bulid issue
* fix more race conditions
* fix deadlock for cases where cgraph.n_nodes == 1
and fix --poll case
* threadpool: use cpu_get_num_math to set the default number of threadpool threads
This way we avoid using E-Cores and Hyperthreaded siblings.
* bench: create fresh threadpool for each test
For benchmarking it's better to start a fresh pool for each test with the exact number of threads
needed for that test. Having larger pools is suboptimal (causes more load, etc).
* atomics: always use stdatomics with clang and use relaxed memory order when polling in ggml_barrier
This also removes sched_yield() calls from ggml_barrier() to match OpenMP behavior.
* threadpool: make polling the default to match openmp behavior
All command line args now allow for setting poll to 0 (false).
* threadpool: do not wakeup threads in already paused threadpool
* fix potential race condition in check_for_work
* threadpool: do not create two threadpools if their params are identical
* threadpool: reduce pause/resume/wakeup overhead in common cases
We now start threadpool in paused state only if we have two.
The resume is now implicit (ie new work) which allows for reduced locking and context-switch overhead.
* threadpool: add support for hybrid polling
poll params (--poll, ...) now specify "polling level", i.e. how aggresively we poll before waiting on cond.var.
poll=0 means no polling, 1 means poll for 128K rounds then wait, 2 for 256K rounds, ...
The default value of 50 (ie 50x128K rounds) seems like a decent default across modern platforms.
We can tune this further as things evolve.
* threadpool: reduce the number of barrier required
New work is now indicated with an atomic counter that is incremented for
each new graph that needs to be computed.
This removes the need for extra barrier for clearing the "new_work" and
removes the special case for trivial graphs.
* threadpool: remove special-casing for disposable threadpools
With the efficient hybrid polling there is no need to make disposable pools any different.
This simplifies the overall logic and reduces branching.
Include n_threads in debug print for disposable threadpool.
Declare pause and stop flags as atomic_bool
This doesn't actually generate any memory barriers and simply informs
the thread sanitizer that these flags can be written & read by different
threads without locking.
* threadpool: do not clear barrier counters between graphs computes (fixes race with small graphs)
This fixes the race condition with very small graphs where the main thread happens to
start a new graph while the workers are just about to exit from barriers.
* threadpool: use relaxed order for chunk sync
Full memory barrier is an overkill for this since each thread works on different chunk
* threadpool: remove abort_callback from threadpool state
* threadpool: better naming for thread/cpumask releated functions
* threadpool: consistent use of int type for n_threads params
* threadpool: add support for ggml_threadpool_params_default/init
Also removes the need for explicit mask_specified param.
all-zero cpumask means use default (usually inherited) cpu affinity mask.
* threadpool: move typedef into ggml.h
* threadpool: fix apply_priority() function name
* threadpool: fix swift wrapper errors due to n_threads int type cleanup
* threadpool: enable --cpu-mask and other threadpool related options only if threadpool is enabled
* threadpool: replace checks for compute_thread ret code with proper status check
* threadpool: simplify threadpool init logic and fix main thread affinity application
Most of the init code is now exactly the same between threadpool and openmp.
* threadpool: update threadpool resume/pause function names
* threadpool: enable openmp by default for now
* threadpool: don't forget to free workers state when omp is enabled
* threadpool: avoid updating process priority on the platforms that do not require it
On Windows we need to change overall process priority class in order to set thread priorities,
but on Linux, Mac, etc we do not need to touch the overall process settings.
* threadpool: update calling thread prio and affinity only at start/resume
This avoids extra syscalls for each graph_compute()
* llama-bench: turn threadpool params into vectors, add output headers, etc
* llama-bench: add support for cool off between tests --delay
This helps for long running tests on platforms that are thermally limited (phones, laptops, etc).
--delay (disabled by default) introduces the sleep for N seconds before starting each test.
* threadpool: move process priority setting into the apps (bench and cli)
This avoids changing the overall process priority on Windows for the apps
that use ggml/llama.cpp directy.
* threadpool: move all pause/resume logic into ggml
* threadpool: futher api cleanup and prep for future refactoring
All threadpool related functions and structs use ggml_threadpool prefix.
* threadpool: minor indent fixes
* threadpool: improve setprioty error message
* Update examples/llama-bench/llama-bench.cpp
Co-authored-by: slaren <slarengh@gmail.com>
* threadpool: fix indent in set_threadpool call
* use int32_t for n_thread type in public llama.cpp API
* threadpool: use _new and _free instead of _create and _release
* fix two more public APIs to use int32_t for n_threads
* build: set _GNU_SOURCE for Adroid
---------
Co-authored-by: Max Krasnyansky <quic_maxk@quicinc.com>
Co-authored-by: fmz <quic_fzaghlou@quic.com>
Co-authored-by: Max Krasnyansky <max.krasnyansky@gmail.com>
Co-authored-by: slaren <slarengh@gmail.com>
2024-08-29 23:20:53 +00:00
|
|
|
|
2024-09-24 10:23:59 +00:00
|
|
|
#ifndef GGML_MAX_NAME
|
|
|
|
|
# define GGML_MAX_NAME 64
|
2024-01-10 13:13:42 +00:00
|
|
|
#endif
|
2024-09-24 10:23:59 +00:00
|
|
|
|
2023-11-03 19:35:05 +00:00
|
|
|
#define GGML_DEFAULT_N_THREADS 4
|
|
|
|
|
#define GGML_DEFAULT_GRAPH_SIZE 2048
|
2024-09-24 10:23:59 +00:00
|
|
|
|
2023-09-05 10:54:40 +00:00
|
|
|
#if UINTPTR_MAX == 0xFFFFFFFF
|
|
|
|
|
#define GGML_MEM_ALIGN 4
|
|
|
|
|
#else
|
|
|
|
|
#define GGML_MEM_ALIGN 16
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
#define GGML_EXIT_SUCCESS 0
|
|
|
|
|
#define GGML_EXIT_ABORTED 1
|
|
|
|
|
|
2024-08-13 19:13:15 +00:00
|
|
|
#define GGML_ROPE_TYPE_NEOX 2
|
|
|
|
|
|
2023-11-03 19:35:05 +00:00
|
|
|
#define GGUF_MAGIC "GGUF"
|
|
|
|
|
|
|
|
|
|
#define GGUF_VERSION 3
|
2023-09-05 10:54:40 +00:00
|
|
|
|
|
|
|
|
#define GGUF_DEFAULT_ALIGNMENT 32
|
|
|
|
|
|
2023-07-02 18:45:27 +00:00
|
|
|
#define GGML_UNUSED(x) (void)(x)
|
|
|
|
|
|
2023-09-05 10:54:40 +00:00
|
|
|
#define GGML_PAD(x, n) (((x) + (n) - 1) & ~((n) - 1))
|
|
|
|
|
|
2023-11-03 19:35:05 +00:00
|
|
|
#ifndef NDEBUG
|
2024-09-24 10:23:59 +00:00
|
|
|
# define GGML_UNREACHABLE() do { fprintf(stderr, "statement should be unreachable\n"); abort(); } while(0)
|
2023-11-03 19:35:05 +00:00
|
|
|
#elif defined(__GNUC__)
|
2024-09-24 10:23:59 +00:00
|
|
|
# define GGML_UNREACHABLE() __builtin_unreachable()
|
2023-12-29 09:30:47 +00:00
|
|
|
#elif defined(_MSC_VER)
|
2024-09-24 10:23:59 +00:00
|
|
|
# define GGML_UNREACHABLE() __assume(0)
|
2023-11-03 19:35:05 +00:00
|
|
|
#else
|
2024-09-24 10:23:59 +00:00
|
|
|
# define GGML_UNREACHABLE() ((void) 0)
|
2023-11-03 19:35:05 +00:00
|
|
|
#endif
|
|
|
|
|
|
2024-07-27 02:41:55 +00:00
|
|
|
#ifdef __cplusplus
|
2024-09-24 10:23:59 +00:00
|
|
|
# define GGML_NORETURN [[noreturn]]
|
2024-07-27 02:41:55 +00:00
|
|
|
#elif defined(_MSC_VER)
|
2024-09-24 10:23:59 +00:00
|
|
|
# define GGML_NORETURN __declspec(noreturn)
|
2024-07-27 02:41:55 +00:00
|
|
|
#else
|
2024-09-24 10:23:59 +00:00
|
|
|
# define GGML_NORETURN _Noreturn
|
2024-07-27 02:41:55 +00:00
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
#define GGML_ABORT(...) ggml_abort(__FILE__, __LINE__, __VA_ARGS__)
|
|
|
|
|
#define GGML_ASSERT(x) if (!(x)) GGML_ABORT("GGML_ASSERT(%s) failed", #x)
|
|
|
|
|
|
2023-07-02 18:45:27 +00:00
|
|
|
// used to copy the number of elements and stride in bytes of tensors into local variables.
|
|
|
|
|
// main purpose is to reduce code duplication and improve readability.
|
|
|
|
|
//
|
|
|
|
|
// example:
|
|
|
|
|
//
|
|
|
|
|
// GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne);
|
|
|
|
|
// GGML_TENSOR_LOCALS(size_t, nb1, src1, nb);
|
|
|
|
|
//
|
|
|
|
|
#define GGML_TENSOR_LOCALS_1(type, prefix, pointer, array) \
|
|
|
|
|
const type prefix##0 = (pointer)->array[0]; \
|
|
|
|
|
GGML_UNUSED(prefix##0);
|
|
|
|
|
#define GGML_TENSOR_LOCALS_2(type, prefix, pointer, array) \
|
|
|
|
|
GGML_TENSOR_LOCALS_1 (type, prefix, pointer, array) \
|
|
|
|
|
const type prefix##1 = (pointer)->array[1]; \
|
|
|
|
|
GGML_UNUSED(prefix##1);
|
|
|
|
|
#define GGML_TENSOR_LOCALS_3(type, prefix, pointer, array) \
|
|
|
|
|
GGML_TENSOR_LOCALS_2 (type, prefix, pointer, array) \
|
|
|
|
|
const type prefix##2 = (pointer)->array[2]; \
|
|
|
|
|
GGML_UNUSED(prefix##2);
|
|
|
|
|
#define GGML_TENSOR_LOCALS(type, prefix, pointer, array) \
|
|
|
|
|
GGML_TENSOR_LOCALS_3 (type, prefix, pointer, array) \
|
|
|
|
|
const type prefix##3 = (pointer)->array[3]; \
|
|
|
|
|
GGML_UNUSED(prefix##3);
|
|
|
|
|
|
2023-12-07 20:27:19 +00:00
|
|
|
#define GGML_TENSOR_UNARY_OP_LOCALS \
|
|
|
|
|
GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \
|
|
|
|
|
GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \
|
|
|
|
|
GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) \
|
|
|
|
|
GGML_TENSOR_LOCALS(size_t, nb, dst, nb)
|
|
|
|
|
|
|
|
|
|
#define GGML_TENSOR_BINARY_OP_LOCALS \
|
|
|
|
|
GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \
|
|
|
|
|
GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \
|
|
|
|
|
GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne) \
|
|
|
|
|
GGML_TENSOR_LOCALS(size_t, nb1, src1, nb) \
|
|
|
|
|
GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) \
|
|
|
|
|
GGML_TENSOR_LOCALS(size_t, nb, dst, nb)
|
|
|
|
|
|
2024-06-26 16:34:09 +00:00
|
|
|
#define GGML_TENSOR_BINARY_OP_LOCALS01 \
|
|
|
|
|
GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \
|
|
|
|
|
GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \
|
|
|
|
|
GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne) \
|
|
|
|
|
GGML_TENSOR_LOCALS(size_t, nb1, src1, nb)
|
|
|
|
|
|
2023-04-29 09:31:52 +00:00
|
|
|
#ifdef __cplusplus
|
|
|
|
|
extern "C" {
|
|
|
|
|
#endif
|
|
|
|
|
|
2024-07-27 02:41:55 +00:00
|
|
|
GGML_NORETURN GGML_ATTRIBUTE_FORMAT(3, 4)
|
|
|
|
|
GGML_API void ggml_abort(const char * file, int line, const char * fmt, ...);
|
|
|
|
|
|
2024-03-04 09:05:42 +00:00
|
|
|
enum ggml_status {
|
|
|
|
|
GGML_STATUS_ALLOC_FAILED = -2,
|
|
|
|
|
GGML_STATUS_FAILED = -1,
|
|
|
|
|
GGML_STATUS_SUCCESS = 0,
|
|
|
|
|
GGML_STATUS_ABORTED = 1,
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
// get ggml_status name string
|
2024-10-02 23:49:47 +00:00
|
|
|
GGML_API const char * ggml_status_to_string(enum ggml_status status);
|
2024-03-04 09:05:42 +00:00
|
|
|
|
2024-05-08 06:30:09 +00:00
|
|
|
// ieee 754-2008 half-precision float16
|
|
|
|
|
// todo: make this not an integral type
|
2023-04-29 09:31:52 +00:00
|
|
|
typedef uint16_t ggml_fp16_t;
|
2024-05-08 06:30:09 +00:00
|
|
|
GGML_API float ggml_fp16_to_fp32(ggml_fp16_t);
|
|
|
|
|
GGML_API ggml_fp16_t ggml_fp32_to_fp16(float);
|
|
|
|
|
GGML_API void ggml_fp16_to_fp32_row(const ggml_fp16_t *, float *, int64_t);
|
|
|
|
|
GGML_API void ggml_fp32_to_fp16_row(const float *, ggml_fp16_t *, int64_t);
|
|
|
|
|
|
|
|
|
|
// google brain half-precision bfloat16
|
|
|
|
|
typedef struct { uint16_t bits; } ggml_bf16_t;
|
|
|
|
|
GGML_API ggml_bf16_t ggml_fp32_to_bf16(float);
|
|
|
|
|
GGML_API float ggml_bf16_to_fp32(ggml_bf16_t); // consider just doing << 16
|
|
|
|
|
GGML_API void ggml_bf16_to_fp32_row(const ggml_bf16_t *, float *, int64_t);
|
2024-08-02 19:11:39 +00:00
|
|
|
GGML_API void ggml_fp32_to_bf16_row_ref(const float *, ggml_bf16_t *, int64_t);
|
2024-05-08 06:30:09 +00:00
|
|
|
GGML_API void ggml_fp32_to_bf16_row(const float *, ggml_bf16_t *, int64_t);
|
2023-05-02 18:23:54 +00:00
|
|
|
|
2023-04-29 09:31:52 +00:00
|
|
|
struct ggml_object;
|
|
|
|
|
struct ggml_context;
|
2024-09-20 18:24:06 +00:00
|
|
|
struct ggml_cgraph;
|
2023-04-29 09:31:52 +00:00
|
|
|
|
2024-03-14 10:38:37 +00:00
|
|
|
// NOTE: always add types at the end of the enum to keep backward compatibility
|
2023-04-29 09:31:52 +00:00
|
|
|
enum ggml_type {
|
2024-03-14 10:38:37 +00:00
|
|
|
GGML_TYPE_F32 = 0,
|
|
|
|
|
GGML_TYPE_F16 = 1,
|
|
|
|
|
GGML_TYPE_Q4_0 = 2,
|
|
|
|
|
GGML_TYPE_Q4_1 = 3,
|
2023-05-14 15:04:23 +00:00
|
|
|
// GGML_TYPE_Q4_2 = 4, support has been removed
|
2024-03-14 10:38:37 +00:00
|
|
|
// GGML_TYPE_Q4_3 = 5, support has been removed
|
|
|
|
|
GGML_TYPE_Q5_0 = 6,
|
|
|
|
|
GGML_TYPE_Q5_1 = 7,
|
|
|
|
|
GGML_TYPE_Q8_0 = 8,
|
|
|
|
|
GGML_TYPE_Q8_1 = 9,
|
|
|
|
|
GGML_TYPE_Q2_K = 10,
|
|
|
|
|
GGML_TYPE_Q3_K = 11,
|
|
|
|
|
GGML_TYPE_Q4_K = 12,
|
|
|
|
|
GGML_TYPE_Q5_K = 13,
|
|
|
|
|
GGML_TYPE_Q6_K = 14,
|
|
|
|
|
GGML_TYPE_Q8_K = 15,
|
2024-01-08 15:02:32 +00:00
|
|
|
GGML_TYPE_IQ2_XXS = 16,
|
2024-01-11 19:39:39 +00:00
|
|
|
GGML_TYPE_IQ2_XS = 17,
|
2024-01-30 13:14:12 +00:00
|
|
|
GGML_TYPE_IQ3_XXS = 18,
|
2024-02-18 16:16:55 +00:00
|
|
|
GGML_TYPE_IQ1_S = 19,
|
2024-02-21 14:19:39 +00:00
|
|
|
GGML_TYPE_IQ4_NL = 20,
|
2024-02-24 14:23:52 +00:00
|
|
|
GGML_TYPE_IQ3_S = 21,
|
2024-02-26 16:28:38 +00:00
|
|
|
GGML_TYPE_IQ2_S = 22,
|
2024-02-27 14:34:24 +00:00
|
|
|
GGML_TYPE_IQ4_XS = 23,
|
2024-03-14 10:38:37 +00:00
|
|
|
GGML_TYPE_I8 = 24,
|
|
|
|
|
GGML_TYPE_I16 = 25,
|
|
|
|
|
GGML_TYPE_I32 = 26,
|
2024-03-27 16:55:10 +00:00
|
|
|
GGML_TYPE_I64 = 27,
|
|
|
|
|
GGML_TYPE_F64 = 28,
|
|
|
|
|
GGML_TYPE_IQ1_M = 29,
|
2024-05-08 06:30:09 +00:00
|
|
|
GGML_TYPE_BF16 = 30,
|
2024-07-10 12:14:51 +00:00
|
|
|
GGML_TYPE_Q4_0_4_4 = 31,
|
|
|
|
|
GGML_TYPE_Q4_0_4_8 = 32,
|
|
|
|
|
GGML_TYPE_Q4_0_8_8 = 33,
|
ggml-quants : ternary packing for TriLMs and BitNet b1.58 (llama/8151)
* ggml-quants : 1.625 bpw ternary packing for BitNet 1.58b
* ggml-quants : faster 1.625 bpw AVX2 vec_dot
Not using a lookup table anymore makes it match q4_0 speed.
* gguf-py : fix formatting
* llama : remove spaces on empty line
* ggml-quants : subtract 1 when back in epi8
This makes the 1.625 bpw type go faster than q4_0. Still not the fastest.
* ggml-quants : Q2_2 now faster than Q4_K on with AVX2
* ggml-quants : cleanup Q1_3 code formatting
* ggml-quants : ARM NEON vec_dot for q2_2 and q1_3
* ggml-quants : use ceiling division when quantizing q1_3
* convert-hf : simplify BitNet pre-quantization
This still results in the exact same tensor weights and scales,
but it reveals some weirdness in the current algorithm.
* convert-hf : allow converting the weird BitNet 1.3B
Its FFN size is 5460 which is not convenient.
The offending tensors are kept in F16,
which makes the final model 5.01 bpw.
* bitnet : replace 1.58b with b1.58, as in the paper
* ggml-quants : fix build failure on Windows
* ggml-quants : attempt to fix Arm 32-bit support
* ggml : add some informative comments in q1_3 vec_dot
* ggml : add TQ1_0 and TQ2_0 ternary quantization types
* ggml : even faster TQ2_0
* ggml : also faster TQ1_0
Same optimization as for TQ2_0 by offsetting the sum instead of the weights.
This makes TQ1_0 almost as fast as Q8_0 on AVX2.
* ggml : fix build issues in certain environments
* ggml : add NEON vec_dot implementation for TQ1_0 and TQ2_0
* ggml : avoid directly using vmlal_high_s8, for 32-bit ARM compat
The compiler seems smart enough to use the same instruction
even when using vget_high_s8 instead.
* ggml : remove q1_3 and q2_2
No more 1.625 bpw and 2.000 bpw,
now instead using 1.6875 bpw and 2.0625 bpw
with TQ1_0 and TQ2_0, respectively.
* llama : remove the separate scale tensors of BitNet b1.58
They won't be needed, since the remaining ternary quant types have
built-in scales.
* ggml-quants : rename fields of TQ1_0 and TQ2_0 structs for consistency
* ggml-quants : allow using vdotq_s32 in TQ2_0 vec_dot
Not yet tested on hardware which supports it,
might not work or might not even compile. But also it might.
It should make the performance better on recent ARM CPUs.
* ggml-quants : remove comment about possible format change of TQ2_0
Making it slightly more convenient for AVX512
but less convenient for everything else is not worth the trouble.
* gguf-py : Numpy (de)quantization for TQ1_0 and TQ2_0
* ggml-quants : use roundf instead of nearest_int for TQ1_0 and TQ2_0
This does not change anything for ternary models,
since their values should never end up being in halfway cases anyway.
* convert : allow direct conversion to TQ1_0 and TQ2_0
The token embeddings and output tensors are kept in F16
to allow quantizing them to Q4_K and Q6_K with llama-quantize.
* llama : handle fallback for TQ1_0 and TQ2_0 with Q4_0
Q4_0 is not completely symmetric (so not lossless for ternary models),
but it should be good enough.
* ggml-quants : allow using ARM dot product instructions for TQ1_0
* ggml-quants : deduplicate TQ1_0 and TQ2_0 __ARM_FEATURE_DOTPROD support
* ggml : remove unused ggml_mul special case
It would otherwise conflict with the more general
optimization coming with Mamba-2.
* ggml : handle TQ1_0 and TQ2_0 in dequantization-based operators
* test-backend-ops : add TQ1_0 and TQ2_0 comments for later
Not yet adding uncommented, because some backends like SYCL and Metal
do not properly handle unknown types in supports_op for GGML_OP_MUL_MAT.
(and Metal also doesn't handle it with GGML_OP_GET_ROWS)
Support for TQ1_0 and TQ2_0 for other backends than CPU
will be added in follow-up pull requests.
2024-09-06 01:48:47 +00:00
|
|
|
GGML_TYPE_TQ1_0 = 34,
|
|
|
|
|
GGML_TYPE_TQ2_0 = 35,
|
2024-11-28 12:52:03 +00:00
|
|
|
GGML_TYPE_IQ4_NL_4_4 = 36,
|
|
|
|
|
// GGML_TYPE_IQ4_NL_4_8 = 37,
|
|
|
|
|
// GGML_TYPE_IQ4_NL_8_8 = 38,
|
2023-04-29 09:31:52 +00:00
|
|
|
GGML_TYPE_COUNT,
|
|
|
|
|
};
|
|
|
|
|
|
2023-12-22 15:53:39 +00:00
|
|
|
// precision
|
|
|
|
|
enum ggml_prec {
|
|
|
|
|
GGML_PREC_DEFAULT,
|
|
|
|
|
GGML_PREC_F32,
|
|
|
|
|
};
|
|
|
|
|
|
2023-11-03 19:35:05 +00:00
|
|
|
enum ggml_backend_type {
|
2024-02-25 10:09:09 +00:00
|
|
|
GGML_BACKEND_TYPE_CPU = 0,
|
|
|
|
|
GGML_BACKEND_TYPE_GPU = 10,
|
|
|
|
|
GGML_BACKEND_TYPE_GPU_SPLIT = 20,
|
2023-05-14 15:04:23 +00:00
|
|
|
};
|
|
|
|
|
|
2023-04-30 15:51:57 +00:00
|
|
|
// model file types
|
|
|
|
|
enum ggml_ftype {
|
2024-03-14 10:38:37 +00:00
|
|
|
GGML_FTYPE_UNKNOWN = -1,
|
|
|
|
|
GGML_FTYPE_ALL_F32 = 0,
|
|
|
|
|
GGML_FTYPE_MOSTLY_F16 = 1, // except 1d tensors
|
|
|
|
|
GGML_FTYPE_MOSTLY_Q4_0 = 2, // except 1d tensors
|
|
|
|
|
GGML_FTYPE_MOSTLY_Q4_1 = 3, // except 1d tensors
|
2023-04-30 15:51:57 +00:00
|
|
|
GGML_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4, // tok_embeddings.weight and output.weight are F16
|
2024-03-14 10:38:37 +00:00
|
|
|
GGML_FTYPE_MOSTLY_Q8_0 = 7, // except 1d tensors
|
|
|
|
|
GGML_FTYPE_MOSTLY_Q5_0 = 8, // except 1d tensors
|
|
|
|
|
GGML_FTYPE_MOSTLY_Q5_1 = 9, // except 1d tensors
|
|
|
|
|
GGML_FTYPE_MOSTLY_Q2_K = 10, // except 1d tensors
|
|
|
|
|
GGML_FTYPE_MOSTLY_Q3_K = 11, // except 1d tensors
|
|
|
|
|
GGML_FTYPE_MOSTLY_Q4_K = 12, // except 1d tensors
|
|
|
|
|
GGML_FTYPE_MOSTLY_Q5_K = 13, // except 1d tensors
|
|
|
|
|
GGML_FTYPE_MOSTLY_Q6_K = 14, // except 1d tensors
|
2024-01-08 15:02:32 +00:00
|
|
|
GGML_FTYPE_MOSTLY_IQ2_XXS = 15, // except 1d tensors
|
2024-01-11 19:39:39 +00:00
|
|
|
GGML_FTYPE_MOSTLY_IQ2_XS = 16, // except 1d tensors
|
2024-01-30 13:14:12 +00:00
|
|
|
GGML_FTYPE_MOSTLY_IQ3_XXS = 17, // except 1d tensors
|
2024-02-18 16:16:55 +00:00
|
|
|
GGML_FTYPE_MOSTLY_IQ1_S = 18, // except 1d tensors
|
2024-02-21 14:19:39 +00:00
|
|
|
GGML_FTYPE_MOSTLY_IQ4_NL = 19, // except 1d tensors
|
2024-02-24 14:23:52 +00:00
|
|
|
GGML_FTYPE_MOSTLY_IQ3_S = 20, // except 1d tensors
|
2024-02-26 16:28:38 +00:00
|
|
|
GGML_FTYPE_MOSTLY_IQ2_S = 21, // except 1d tensors
|
2024-02-27 14:34:24 +00:00
|
|
|
GGML_FTYPE_MOSTLY_IQ4_XS = 22, // except 1d tensors
|
2024-03-27 16:55:10 +00:00
|
|
|
GGML_FTYPE_MOSTLY_IQ1_M = 23, // except 1d tensors
|
2024-05-08 06:30:09 +00:00
|
|
|
GGML_FTYPE_MOSTLY_BF16 = 24, // except 1d tensors
|
2024-07-10 12:14:51 +00:00
|
|
|
GGML_FTYPE_MOSTLY_Q4_0_4_4 = 25, // except 1d tensors
|
|
|
|
|
GGML_FTYPE_MOSTLY_Q4_0_4_8 = 26, // except 1d tensors
|
|
|
|
|
GGML_FTYPE_MOSTLY_Q4_0_8_8 = 27, // except 1d tensors
|
2023-04-30 15:51:57 +00:00
|
|
|
};
|
|
|
|
|
|
2023-04-29 09:31:52 +00:00
|
|
|
// available tensor operations:
|
|
|
|
|
enum ggml_op {
|
|
|
|
|
GGML_OP_NONE = 0,
|
|
|
|
|
|
|
|
|
|
GGML_OP_DUP,
|
|
|
|
|
GGML_OP_ADD,
|
2023-05-14 15:04:23 +00:00
|
|
|
GGML_OP_ADD1,
|
|
|
|
|
GGML_OP_ACC,
|
2023-04-29 09:31:52 +00:00
|
|
|
GGML_OP_SUB,
|
|
|
|
|
GGML_OP_MUL,
|
|
|
|
|
GGML_OP_DIV,
|
|
|
|
|
GGML_OP_SQR,
|
|
|
|
|
GGML_OP_SQRT,
|
2023-05-14 15:04:23 +00:00
|
|
|
GGML_OP_LOG,
|
2024-08-12 13:02:08 +00:00
|
|
|
GGML_OP_SIN,
|
|
|
|
|
GGML_OP_COS,
|
2023-04-29 09:31:52 +00:00
|
|
|
GGML_OP_SUM,
|
2023-05-14 15:04:23 +00:00
|
|
|
GGML_OP_SUM_ROWS,
|
2023-04-29 09:31:52 +00:00
|
|
|
GGML_OP_MEAN,
|
2023-07-02 18:45:27 +00:00
|
|
|
GGML_OP_ARGMAX,
|
2024-10-03 15:29:59 +00:00
|
|
|
GGML_OP_COUNT_EQUAL,
|
2023-04-29 09:31:52 +00:00
|
|
|
GGML_OP_REPEAT,
|
2023-06-25 11:22:21 +00:00
|
|
|
GGML_OP_REPEAT_BACK,
|
2023-09-05 10:54:40 +00:00
|
|
|
GGML_OP_CONCAT,
|
2023-05-14 15:04:23 +00:00
|
|
|
GGML_OP_SILU_BACK,
|
2023-04-29 09:31:52 +00:00
|
|
|
GGML_OP_NORM, // normalize
|
|
|
|
|
GGML_OP_RMS_NORM,
|
2023-05-14 15:04:23 +00:00
|
|
|
GGML_OP_RMS_NORM_BACK,
|
2023-09-05 10:54:40 +00:00
|
|
|
GGML_OP_GROUP_NORM,
|
2023-04-29 09:31:52 +00:00
|
|
|
|
|
|
|
|
GGML_OP_MUL_MAT,
|
2023-12-07 20:27:19 +00:00
|
|
|
GGML_OP_MUL_MAT_ID,
|
2023-06-25 11:22:21 +00:00
|
|
|
GGML_OP_OUT_PROD,
|
2023-04-29 09:31:52 +00:00
|
|
|
|
|
|
|
|
GGML_OP_SCALE,
|
2023-05-14 15:04:23 +00:00
|
|
|
GGML_OP_SET,
|
2023-04-29 09:31:52 +00:00
|
|
|
GGML_OP_CPY,
|
|
|
|
|
GGML_OP_CONT,
|
|
|
|
|
GGML_OP_RESHAPE,
|
|
|
|
|
GGML_OP_VIEW,
|
|
|
|
|
GGML_OP_PERMUTE,
|
|
|
|
|
GGML_OP_TRANSPOSE,
|
|
|
|
|
GGML_OP_GET_ROWS,
|
2023-05-14 15:04:23 +00:00
|
|
|
GGML_OP_GET_ROWS_BACK,
|
|
|
|
|
GGML_OP_DIAG,
|
2023-04-29 09:31:52 +00:00
|
|
|
GGML_OP_DIAG_MASK_INF,
|
2023-05-14 15:04:23 +00:00
|
|
|
GGML_OP_DIAG_MASK_ZERO,
|
2023-04-29 09:31:52 +00:00
|
|
|
GGML_OP_SOFT_MAX,
|
2023-06-25 11:22:21 +00:00
|
|
|
GGML_OP_SOFT_MAX_BACK,
|
2023-04-29 09:31:52 +00:00
|
|
|
GGML_OP_ROPE,
|
2023-05-14 15:04:23 +00:00
|
|
|
GGML_OP_ROPE_BACK,
|
2023-05-20 15:56:30 +00:00
|
|
|
GGML_OP_CLAMP,
|
2023-11-03 19:35:05 +00:00
|
|
|
GGML_OP_CONV_TRANSPOSE_1D,
|
2023-11-12 13:31:08 +00:00
|
|
|
GGML_OP_IM2COL,
|
2024-07-30 13:56:35 +00:00
|
|
|
GGML_OP_IM2COL_BACK,
|
2023-09-05 10:54:40 +00:00
|
|
|
GGML_OP_CONV_TRANSPOSE_2D,
|
|
|
|
|
GGML_OP_POOL_1D,
|
|
|
|
|
GGML_OP_POOL_2D,
|
2024-07-30 13:56:35 +00:00
|
|
|
GGML_OP_POOL_2D_BACK,
|
2023-09-05 10:54:40 +00:00
|
|
|
GGML_OP_UPSCALE, // nearest interpolate
|
2023-12-13 19:55:03 +00:00
|
|
|
GGML_OP_PAD,
|
2024-12-03 19:20:04 +00:00
|
|
|
GGML_OP_PAD_REFLECT_1D,
|
2024-03-03 12:23:52 +00:00
|
|
|
GGML_OP_ARANGE,
|
|
|
|
|
GGML_OP_TIMESTEP_EMBEDDING,
|
2023-12-07 20:27:19 +00:00
|
|
|
GGML_OP_ARGSORT,
|
2023-12-13 19:55:03 +00:00
|
|
|
GGML_OP_LEAKY_RELU,
|
2023-04-29 09:31:52 +00:00
|
|
|
|
ggml : add Flash Attention (llama/5021)
* ggml : add ggml_flash_attn_ext API
* ggml : fix GQA support in ggml_flash_attn_ext
* ggml : online attention (CPU)
* metal : initial implementation
* metal : f16 precision
* metal : reduce branches
* metal : specialize for head size
* wip : 8 rows per simd group
* wip : 4 rows per simd group
* wip : template for rows per warp
* metal : parallelize across KV size
* metal : parallel reduce across heads
* metal : efficient flash_attn_f16 implementation
* metal : avoid redundant loads of the attention
* metal : scale and mask in matrix form
* metal : fix comment
* llama : avoid ggml_cast, use F32 query
* metal : add parallel reduce version (disabled)
* metal : move output into local memory + optimize
- the result from each simdgroup now stays in the registers
- significantly reduced SRAM usage
- more efficient skipping of -INF blocks
- avoid simdgroup barrier in hot loop
- add comments
* metal : add tests, fix scaling, support C > 32
* metal : improve precision
* ggml : fix f16 mad
* metal : minor
* metal : support Q > 8
* tests : add ATTN tests
* metal : disable buffer allocation logs
* tests : more
* metal : faster inner loop for C == 32
* metal : fix array initialization
* tests : ifdef
* ggml : switch to padded F16 mask for ggml_soft_max, ggml_flash_attn_ext
* ggml : fix ggml_soft_max mask requirement
* cuda : fix soft_max to use correct mask size
* cuda : add flash_attn kernel (wip)
* metal : optimize softmax for C > 32
* metal : optimize softmax
* tests : minor fix
* cuda : avoid zeroing fragments
* tests : update dims
* cuda : fix __hisinf() result check
* cuda : avoid warp_reduce for smax
* cuda : use int instead of int64_t
Noticeably improves performance (thanks to Johannes)
* cuda : make loops use the same loop values
Thanks Johannes again for the tip
* cuda : unroll some of the loops
* cuda : avoid __hisinf branches
* cuda : use half2 in softmax
* cuda : switch to 1 warp for bs > 16
* cuda : speed-up reduce part of the kernel
* cuda : unroll Q*K^T loop
* cuda : fix -INF block check
* cuda : simplify softmax
* cuda : fix matrix names
* cuda : minor
* llama : adapt to F16 KQ_pos
* llama : adapt new models to F16 KQ_mask
* ggml : fix F16 store (ARM NEON)
* llama : fix type of KQ_mask and KQ_pos
* ggml : fix CPU soft_max
* tests : add hs=256
* cuda : fix build
* metal : improve perf via smaller int registers
* cuda : adapt soft_max to F16 mask and pos
* CUDA: faster FlashAttention, kernel for bs == 1
* 16 cols for Phi-2
* no vec for hs, no hs==256 ncols==32 for Volta
* adjust kernel selection logic
* 4 warps, 256 stride for all D
* no ncols == 64
* Multiple parallel blocks for batch size 1
* fix compile warnings
* fix excessive KQ_b loads
* fix cmake build
* fix KV cache padding, NaN from INFINITY (llama/6438)
* llama : flash_attn cparam + fix defrag
* server: support flash_attn param
* server: bench: enable flash_attn param
* CUDA: refactor host code, dyn. par. blocks
* fix flash_attn_vec_f16 race condition
* flush softmax exp below threshold to 0
* store temp KQ in registers
* Calculate KQ as FP32 if KQV has GGML_PREC_F32
* Add __hgt2_mask implementation for CUDA 11
* fix KQ FP32 precision fpr parallel_blocks > 1
* llama-bench : add -fa,--flash-attn arg
* metal : add BS=1 kernel for flash attention (llama/6508)
* metal : add BS=1 kernel for flash attention (wip)
* metal : support more than 1 warps
* metal : opts
* metal : opt
* metal : switch to parallel reduce
* metal : reduce registers
* metal : simplify
* metal : initial FA vec kernel
* metal : use F32 attention accumulators
* batched-bench : add fattn arg
* llama : simplify llama_build_kv_store
ggml-ci
* llama : adapt build_olmo to changes
* ggml : fix arm fp16 store on windows
* metal : clean-up
* metal : clean-up kernel code
* metal : minor
* tests : remove benchmarks
ggml-ci
* ggml : fix avx512 const correctness
ggml-ci
* ggml : fix soft_max with bias on CPU
ggml-ci
* common : print --flash-attn in help
* ggml : fix num dimensions in ggml_flash_attn_ext
* llama : force disable flash attention for incompatible models
* ggml : ggml_soft_max support F16/F32 mask/pos
ggml-ci
* cuda : uint -> uint32_t
* cuda : "constexpr dim3" -> "const dim3"
ggml-ci
* cuda : try to fix __hgt2_mask
ggml-ci
* ggml : add TODO's for F16/F32 mask/pos support in other backends
* llama : replace bool need_kq_pos with use_alibi
* llama : prep ALiBi support for BERT models
ggml-ci
* llama : fix n_batch requirements
ggml-ci
* cont
* server : add help for --flash-attn arg
* llama : disable FA for AMD
* tests : remove TMP_ATTN_BENCH
ggml-ci
* llama : support save/load state with FA enabled
ggml-ci
* ci : add CUDA save-load-state tests
ggml-ci
* llama : llama_kv_cache_clear zeroes data + fix save-load seq
ggml-ci
* llama : fix copy-paste errors, add TODO
* llama : disallow incompatible states
* llama : update llama_state_get_size after v_trans field
* metal : remove tmp log
* llama : add static reminder for llama_state_get_size
* metal : fix max nsg
ggml-ci
* ci : fix arg order
ggml-ci
---------
Co-authored-by: Johannes Gäßler <johannesg@5d6.de>
Co-authored-by: Pierrick HYMBERT <pierrick.hymbert@gmail.com>
2024-04-30 09:16:08 +00:00
|
|
|
GGML_OP_FLASH_ATTN_EXT,
|
2023-06-25 11:22:21 +00:00
|
|
|
GGML_OP_FLASH_ATTN_BACK,
|
llama : support Mamba Selective State Space Models (llama/5328)
* mamba : begin working on support for Mamba SSM
* mamba : begin figuring out how to (ab)use the kv cache for Mamba
* mamba : recurrent inference almost works, but incoherent
* mamba : recurrent inference WORKS!!!
* convert : optionally use d_conv and d_state from config.json for Mamba
* mamba : refactor recurrent conv, resulting in 20% perf increase
It's still slower than I'd like, but I did not really optimize `ggml_exp` yet.
I also refactored `ggml_exp` to work with tensors with more than 2 dimensions.
* ggml : parallelize ggml_exp
This results in 8% faster token generation for Mamba-130M.
* mamba : simplify the conv step with a self-overlapping view
Turns out the conv_state can be made smaller by one column.
Note that this breaks existing GGUFs of Mamba,
because the key_value_length field is tied to the conv_state size.
Convolution with a self-overlapping view is cool!
And it's much simpler than what I initially thought would be necessary
to make the convolution step work with more than 1 token at a time.
Next step is to make the SSM step work on batches of tokens too,
and thus I need to figure out a way to make a parallel selective scan
which will keep the ssm_state small and won't make it bigger
by a factor of (n_layer * batch_size).
* llama : fix Mamba KV self size wrongly displaying as f16 instead of f32
Relatedly, I also tried to see if other types than f32 worked for the states,
but they don't, because of the operators used.
It's probably better anyway to keep lots of precision there,
since the states are small anyway.
* mamba : fix self-overlapping view depth stride
* mamba : handle batches of more than 1 token
This means running Mamba no longer crashes when using the default settings!
And probably also slightly faster prompt processing.
Both batched and non-batched processing yield the same output.
Previously, the state was not cleared when starting a sequence.
Next step is to make the KV cache API work as expected for Mamba models.
* ggml: add ggml_ssm_scan to help with parallel selective scan
If the selective scan was implemented without a custom operator,
there would be waaay too many nodes in the graph. For example,
for Mamba-130M, with a batch size of 512 (the default),
a naive selective scan could add at least 24*512=12288 nodes,
which is more than LLAMA_MAX_NODES (8192),
and that's only for the smallest Mamba model.
So it's much cleaner with a custom operator.
Not sure about the name, though.
* ggml : in ggml_ssm_scan, merge multiple rows in the same vec operation
This will help with performance on CPU if ggml_vec_mul_f32
and ggml_vec_add_f32 are ever optimized with SIMD.
* mamba : very basic quantization support
Mostly works, but there is currently no difference
between the variants of a k-quant (e.g. Q4_K_S and Q4_K_M are the same).
Most of the SSM-specific weights can be kept in f32 without affecting
the size that much, since they are relatively small.
(the linear projection weights are responsible for most of Mamba's size)
Too much quantization seems to make the state degrade quite fast, and
the model begins to output gibberish.
It seems to affect bigger models to a lesser extent than small models,
but I'm not sure by how much.
Experimentation will be needed to figure out which weights are more important
for the _M (and _L?) variants of k-quants for Mamba.
* convert : fix wrong name for layer norm weight of offical Mamba models
I was using Q-bert/Mamba-* models before, which have a slighlty different
naming scheme for the weights.
(they start with "model.layers" instead of "backbone.layers")
* mamba : fuse more steps of the SSM scan in the ggml_ssm_scan operator
This increases performance on CPU by around 30% for prompt processing,
and by around 20% for text generation.
However, it also makes the ggml_exp and ggml_soft_plus operators unused.
Whether or not they should be kept will be decided later.
* convert : for Mamba, also consider the "MambaLMHeadModel" arch name
It's the name of the class of the official implementation,
though they don't use it (yet) in the "architectures" field of config.json
* mamba : fix vocab size problems with official models
The perplexity was waaaay to high for models with a non-round vocab size.
Not sure why, but it needed to be fixed in the metadata.
Note that this breaks existing GGUF-converted Mamba models,
but **only if** the vocab size was not already rounded.
* ggml : remove ggml_exp and ggml_soft_plus
They did not exist anyway outside of this branch,
and since ggml_ssm_scan fused operations together, they are unused.
It's always possible to bring them back if needed.
* mamba : remove some useless comments
No code change.
* convert : fix flake8 linter errors
* mamba : apply suggestions from code review
* mamba : remove unecessary branch for row-wise ssm_state and C multiplication
It was previously done to avoid permuting when only one token is processed
at a time (like when generating text), but permuting is cheap,
and dynamically changing the compute graph is not future-proof.
* ggml : in ggml_ssm_scan, use more appropriate asserts
* ggml : rename the destination pointer in ggml_compute_forward_ssm_scan_f32
* mamba : multiple sequences, but one at a time
This is a step towards making this Mamba implementation usable
with the server example (the way the system prompt is kept when clearing
the client slots will need to be changed before this can work, though).
The KV cache size for this kind of model is tied to the maximum number
of sequences kept at any single time.
For now, this number is obtained from n_parallel (plus one,
to have an extra sequence to dedicate to the system prompt),
but there might be a better way to do this which won't also
make the main example use 2 cells even if only 1 is really used.
(for this specific case, --parallel 0 helps)
Simultaneous sequence processing will probably require changes to
ggml_ssm_scan, and possibly a new operator for the conv step.
* mamba : support llama_kv_cache_seq_cp
This (mis)uses the logic around K shifts, because tokens in a state
can't be shifted anyway, and because inp_K_shift has the right shape and type.
Using ggml_get_rows is a nice way to do copies, but copy chains can't work.
Fortunately, copy chains don't really seem to be used in the examples.
Each KV cell is dedicated to the sequence ID corresponding to its own index.
* mamba : use a state mask
It's cleaner than the previous heuristic of
checking for the pos of the first token in the batch.
inp_KQ_mask could not be re-used for this, because it has the wrong shape
and because it seems more suited to the next step of
simultaneous sequence processing (helping with the problem of
remembering which token belongs to which sequence(s)/state(s)).
* llama : replace the usage of n_ctx with kv_self.size in many places
* mamba : use n_tokens directly instead of n_tok
* mamba : in comments, properly refer to KV cells instead of slots
* mamba : reduce memory usage of ggml_ssm_scan
From 290.37 MiB to 140.68 MiB of CPU compute buffer size
with Mamba 3B with a batch size of 512.
The result tensor of ggml_ssm_scan was previously a big part
of the CPU compute buffer size. To make it smaller,
it does not contain the intermediate ssm states anymore.
Both y and the last ssm state are combined in the result tensor,
because it seems only a single tensor can be returned by an operator
with the way the graph is built.
* mamba : simultaneous sequence processing
A batch can now contain tokens from multiple sequences.
This is necessary for at least the parallel example, the server example,
and the HellaSwag test in the perplexity example.
However, for this to be useful, uses of llama_kv_cache_seq_rm/cp
will need to be changed to work on whole sequences.
* ggml : add ggml_ssm_conv as a new operator for the conv step of Mamba
This operator makes it possible to use and update the correct states
for each token of the batch in the same way as ggml_ssm_scan.
Other solutions which use existing operators would need loops which would
add too many nodes to the graph (at least the ones I thought of).
Using this operator further reduces the size of the CPU compute buffer
from 140.68 MiB to 103.20 MiB with Mamba 3B with a batch size of 512.
And (at least on CPU), it's a bit faster than before.
Note that "ggml_ssm_conv" is probably not the most appropriate name,
and it could be changed if a better one is found.
* llama : add inp_s_seq as a new input tensor
The most convenient implementation to select the correct state (for Mamba)
for each token is to directly get the correct index from a tensor.
This is why inp_s_seq is storing int32_t and not floats.
The other, less convenient way to select the correct state would be
to have inp_KQ_mask contain 1.0f for each state used by a token
and 0.0f otherwise. This complicates quickly fetching the first used
state of a token, and is also less efficient because a whole row
of the mask would always need to be read for each token.
Using indexes makes it easy to stop searching when there are
no more sequences for a token, and the first sequence assigned
is always very quickly available (it's the first element of each row).
* mamba : support llama_kv_cache_seq_cp copy chains
* mamba : support shifting and dividing the kv cache pos
* mamba : make the server and parallel examples work with whole sequences
A seq_id is dedicated to the system prompt in both cases.
* llama : make llama_kv_cache_seq_rm return whether it succeeded or not
* mamba : dedicate an input tensor for state copy indices
This is cleaner and makes it easier to adapt when/if token positions
(and by extension, inp_K_shift) are no longer integers.
* mamba : adapt perplexity, batched, and batched-bench examples
* perplexity : limit the max number of sequences
This adapts to what the loaded model can provide.
* llama : add llama_n_max_seq to get the upper limit for seq_ids
Used by the perplexity example.
* batched : pass n_parallel to the model's context params
This should have been there already, but it wasn't.
* batched-bench : reserve sequences to support Mamba
* batched-bench : fix tokens being put in wrong sequences
Generation quality isn't what's measured in there anyway,
but at least using the correct sequences avoids using non-consecutive
token positions.
* mamba : stop abusing attention metadata
This breaks existing converted-to-GGUF Mamba models,
but will allow supporting mixed architectures like MambaFormer
without needing to break Mamba models.
This will also allow changing the size of Mamba's states
without having to reconvert models in the future.
(e.g. using something else than d_conv - 1 columns for the conv_states
will not require breaking existing converted Mamba models again)
* gguf-py : add new KV metadata key-value pairs for Mamba
* llama : add new metadata key-value pairs for Mamba
* llama : guard against divisions by zero when n_head is 0
* mamba : rename "unlimited" KV cache property to "recurrent"
* mamba : more correctly update the "used" field of the KV cache
* ggml : in ggml_ssm_scan, use a threshold for soft_plus
This is how the official Mamba implementation does it,
and it's also what torch.nn.Softplus does.
* convert : for Mamba, fallback to internal NeoX tokenizer
The resulting models are exactly the same
as if the tokenizer.json and tokenizer_config.json of GPT-NeoX were there.
* mamba : support state saving and restoring
* ggml : implicitly pass src tensors through dst for Mamba-related ops
* mamba : clarify some comments
* server : fix cache_tokens not getting correctly resized
Otherwise, when the "we have to evaluate at least 1 token" special case
was triggered, an extra token was kept in cache_tokens even if it was
removed from the KV cache.
For Mamba, this caused useless prompt reprocessing when the previous
request triggered the above case.
* convert-hf : support new metadata keys for Mamba
For the models available at
https://huggingface.co/collections/state-spaces/transformers-compatible-mamba-65e7b40ab87e5297e45ae406
* mamba : rename metadata to be more similar to transformers library
This breaks existing converted-to-GGUF models,
but the metadata names are more "standard".
* mamba : support mamba-*-hf models
These models share their token_embd.weight with their output.weight
* mamba : add missing spaces
This is purely a formatting change.
* convert-hf : omit output.weight when identical with token_embd.weight
Only for Mamba for now, but it might be relevant for other models eventually.
Most Mamba models actually share these two tensors, albeit implicitly.
* readme : add Mamba to supported models, and add recent API changes
* mamba : move state_seq and state_mask views outside layer loop
A few tensors were also missing `struct` in front of `ggml_tensor`.
2024-03-08 22:31:00 +00:00
|
|
|
GGML_OP_SSM_CONV,
|
|
|
|
|
GGML_OP_SSM_SCAN,
|
2023-06-25 11:22:21 +00:00
|
|
|
GGML_OP_WIN_PART,
|
|
|
|
|
GGML_OP_WIN_UNPART,
|
2023-09-05 10:54:40 +00:00
|
|
|
GGML_OP_GET_REL_POS,
|
|
|
|
|
GGML_OP_ADD_REL_POS,
|
2024-11-07 07:19:10 +00:00
|
|
|
GGML_OP_RWKV_WKV6,
|
2023-09-05 10:54:40 +00:00
|
|
|
|
|
|
|
|
GGML_OP_UNARY,
|
2023-04-29 09:31:52 +00:00
|
|
|
|
|
|
|
|
GGML_OP_MAP_UNARY,
|
|
|
|
|
GGML_OP_MAP_BINARY,
|
|
|
|
|
|
2023-09-05 10:54:40 +00:00
|
|
|
GGML_OP_MAP_CUSTOM1_F32,
|
|
|
|
|
GGML_OP_MAP_CUSTOM2_F32,
|
|
|
|
|
GGML_OP_MAP_CUSTOM3_F32,
|
|
|
|
|
|
2023-06-25 11:22:21 +00:00
|
|
|
GGML_OP_MAP_CUSTOM1,
|
|
|
|
|
GGML_OP_MAP_CUSTOM2,
|
|
|
|
|
GGML_OP_MAP_CUSTOM3,
|
|
|
|
|
|
|
|
|
|
GGML_OP_CROSS_ENTROPY_LOSS,
|
|
|
|
|
GGML_OP_CROSS_ENTROPY_LOSS_BACK,
|
2024-09-20 12:36:38 +00:00
|
|
|
GGML_OP_OPT_STEP_ADAMW,
|
2023-06-25 11:22:21 +00:00
|
|
|
|
2023-04-29 09:31:52 +00:00
|
|
|
GGML_OP_COUNT,
|
|
|
|
|
};
|
|
|
|
|
|
2023-09-05 10:54:40 +00:00
|
|
|
enum ggml_unary_op {
|
|
|
|
|
GGML_UNARY_OP_ABS,
|
|
|
|
|
GGML_UNARY_OP_SGN,
|
|
|
|
|
GGML_UNARY_OP_NEG,
|
|
|
|
|
GGML_UNARY_OP_STEP,
|
|
|
|
|
GGML_UNARY_OP_TANH,
|
|
|
|
|
GGML_UNARY_OP_ELU,
|
|
|
|
|
GGML_UNARY_OP_RELU,
|
2024-05-01 21:44:26 +00:00
|
|
|
GGML_UNARY_OP_SIGMOID,
|
2023-09-05 10:54:40 +00:00
|
|
|
GGML_UNARY_OP_GELU,
|
|
|
|
|
GGML_UNARY_OP_GELU_QUICK,
|
|
|
|
|
GGML_UNARY_OP_SILU,
|
2024-01-22 13:09:35 +00:00
|
|
|
GGML_UNARY_OP_HARDSWISH,
|
|
|
|
|
GGML_UNARY_OP_HARDSIGMOID,
|
2024-09-01 14:38:17 +00:00
|
|
|
GGML_UNARY_OP_EXP,
|
2023-12-07 20:27:19 +00:00
|
|
|
|
|
|
|
|
GGML_UNARY_OP_COUNT,
|
2023-09-05 10:54:40 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
|
|
enum ggml_object_type {
|
2024-02-25 10:09:09 +00:00
|
|
|
GGML_OBJECT_TYPE_TENSOR,
|
|
|
|
|
GGML_OBJECT_TYPE_GRAPH,
|
|
|
|
|
GGML_OBJECT_TYPE_WORK_BUFFER
|
2023-09-05 10:54:40 +00:00
|
|
|
};
|
2023-04-29 09:31:52 +00:00
|
|
|
|
2023-11-03 19:35:05 +00:00
|
|
|
enum ggml_log_level {
|
2024-09-15 17:46:12 +00:00
|
|
|
GGML_LOG_LEVEL_NONE = 0,
|
2024-11-01 22:50:59 +00:00
|
|
|
GGML_LOG_LEVEL_DEBUG = 1,
|
|
|
|
|
GGML_LOG_LEVEL_INFO = 2,
|
|
|
|
|
GGML_LOG_LEVEL_WARN = 3,
|
|
|
|
|
GGML_LOG_LEVEL_ERROR = 4,
|
2024-09-24 07:15:35 +00:00
|
|
|
GGML_LOG_LEVEL_CONT = 5, // continue previous log
|
2023-11-03 19:35:05 +00:00
|
|
|
};
|
|
|
|
|
|
2024-09-20 12:36:38 +00:00
|
|
|
// this tensor...
|
2024-02-11 12:37:58 +00:00
|
|
|
enum ggml_tensor_flag {
|
2024-09-29 21:18:02 +00:00
|
|
|
GGML_TENSOR_FLAG_INPUT = 1, // ...is an input for the GGML compute graph
|
|
|
|
|
GGML_TENSOR_FLAG_OUTPUT = 2, // ...is an output for the GGML compute graph
|
|
|
|
|
GGML_TENSOR_FLAG_PARAM = 4, // ...contains trainable parameters
|
|
|
|
|
GGML_TENSOR_FLAG_LOSS = 8, // ...defines loss for numerical optimization (multiple loss tensors add up)
|
2024-02-11 12:37:58 +00:00
|
|
|
};
|
|
|
|
|
|
2024-11-03 18:34:08 +00:00
|
|
|
struct ggml_init_params {
|
|
|
|
|
// memory pool
|
|
|
|
|
size_t mem_size; // bytes
|
|
|
|
|
void * mem_buffer; // if NULL, memory will be allocated internally
|
|
|
|
|
bool no_alloc; // don't allocate memory for the tensor data
|
|
|
|
|
};
|
|
|
|
|
|
2023-04-29 09:31:52 +00:00
|
|
|
// n-dimensional tensor
|
|
|
|
|
struct ggml_tensor {
|
2024-09-20 18:24:06 +00:00
|
|
|
enum ggml_type type;
|
2024-05-15 13:08:48 +00:00
|
|
|
|
|
|
|
|
GGML_DEPRECATED(enum ggml_backend_type backend, "use the buffer type to find the storage location of the tensor");
|
2023-11-03 19:35:05 +00:00
|
|
|
|
|
|
|
|
struct ggml_backend_buffer * buffer;
|
2023-04-29 09:31:52 +00:00
|
|
|
|
|
|
|
|
int64_t ne[GGML_MAX_DIMS]; // number of elements
|
|
|
|
|
size_t nb[GGML_MAX_DIMS]; // stride in bytes:
|
2023-11-03 19:35:05 +00:00
|
|
|
// nb[0] = ggml_type_size(type)
|
|
|
|
|
// nb[1] = nb[0] * (ne[0] / ggml_blck_size(type)) + padding
|
2023-04-29 09:31:52 +00:00
|
|
|
// nb[i] = nb[i-1] * ne[i-1]
|
|
|
|
|
|
|
|
|
|
// compute data
|
|
|
|
|
enum ggml_op op;
|
|
|
|
|
|
2023-09-05 10:54:40 +00:00
|
|
|
// op params - allocated as int32_t for alignment
|
|
|
|
|
int32_t op_params[GGML_MAX_OP_PARAMS / sizeof(int32_t)];
|
|
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
int32_t flags;
|
2023-04-29 09:31:52 +00:00
|
|
|
|
2023-09-05 10:54:40 +00:00
|
|
|
struct ggml_tensor * src[GGML_MAX_SRC];
|
2023-04-29 09:31:52 +00:00
|
|
|
|
2024-06-26 16:34:09 +00:00
|
|
|
// source tensor and offset for views
|
2023-09-05 17:57:27 +00:00
|
|
|
struct ggml_tensor * view_src;
|
|
|
|
|
size_t view_offs;
|
|
|
|
|
|
2023-04-29 09:31:52 +00:00
|
|
|
void * data;
|
2023-05-02 18:23:54 +00:00
|
|
|
|
2023-06-25 11:22:21 +00:00
|
|
|
char name[GGML_MAX_NAME];
|
|
|
|
|
|
|
|
|
|
void * extra; // extra things e.g. for ggml-cuda.cu
|
2023-05-02 18:23:54 +00:00
|
|
|
|
2024-11-16 12:49:35 +00:00
|
|
|
char padding[8];
|
2023-04-29 09:31:52 +00:00
|
|
|
};
|
|
|
|
|
|
2023-06-25 11:22:21 +00:00
|
|
|
static const size_t GGML_TENSOR_SIZE = sizeof(struct ggml_tensor);
|
|
|
|
|
|
2024-02-09 09:42:27 +00:00
|
|
|
// Abort callback
|
|
|
|
|
// If not NULL, called before ggml computation
|
|
|
|
|
// If it returns true, the computation is aborted
|
|
|
|
|
typedef bool (*ggml_abort_callback)(void * data);
|
|
|
|
|
|
2024-02-16 09:31:07 +00:00
|
|
|
|
2024-02-24 16:27:36 +00:00
|
|
|
//
|
|
|
|
|
// GUID
|
|
|
|
|
//
|
|
|
|
|
|
|
|
|
|
// GUID types
|
|
|
|
|
typedef uint8_t ggml_guid[16];
|
|
|
|
|
typedef ggml_guid * ggml_guid_t;
|
|
|
|
|
|
|
|
|
|
GGML_API bool ggml_guid_matches(ggml_guid_t guid_a, ggml_guid_t guid_b);
|
|
|
|
|
|
2023-04-29 09:31:52 +00:00
|
|
|
// misc
|
2022-09-25 18:23:15 +00:00
|
|
|
|
2023-04-29 09:31:52 +00:00
|
|
|
GGML_API void ggml_time_init(void); // call this once at the beginning of the program
|
|
|
|
|
GGML_API int64_t ggml_time_ms(void);
|
|
|
|
|
GGML_API int64_t ggml_time_us(void);
|
|
|
|
|
GGML_API int64_t ggml_cycles(void);
|
|
|
|
|
GGML_API int64_t ggml_cycles_per_ms(void);
|
2023-03-27 18:00:32 +00:00
|
|
|
|
2024-03-27 16:55:10 +00:00
|
|
|
// accepts a UTF-8 path, even on Windows
|
|
|
|
|
GGML_API FILE * ggml_fopen(const char * fname, const char * mode);
|
|
|
|
|
|
2023-04-29 09:31:52 +00:00
|
|
|
GGML_API void ggml_print_object (const struct ggml_object * obj);
|
|
|
|
|
GGML_API void ggml_print_objects(const struct ggml_context * ctx);
|
2022-09-25 18:23:15 +00:00
|
|
|
|
2024-10-02 23:49:47 +00:00
|
|
|
GGML_API int64_t ggml_nelements (const struct ggml_tensor * tensor);
|
|
|
|
|
GGML_API int64_t ggml_nrows (const struct ggml_tensor * tensor);
|
|
|
|
|
GGML_API size_t ggml_nbytes (const struct ggml_tensor * tensor);
|
|
|
|
|
GGML_API size_t ggml_nbytes_pad(const struct ggml_tensor * tensor); // same as ggml_nbytes() but padded to GGML_MEM_ALIGN
|
2022-09-25 18:23:15 +00:00
|
|
|
|
2024-10-02 23:49:47 +00:00
|
|
|
GGML_API int64_t ggml_blck_size(enum ggml_type type);
|
|
|
|
|
GGML_API size_t ggml_type_size(enum ggml_type type); // size in bytes for all elements in a block
|
|
|
|
|
GGML_API size_t ggml_row_size (enum ggml_type type, int64_t ne); // size in bytes for all elements in a row
|
2023-12-22 15:53:39 +00:00
|
|
|
|
|
|
|
|
GGML_DEPRECATED(
|
|
|
|
|
GGML_API double ggml_type_sizef(enum ggml_type type), // ggml_type_size()/ggml_blck_size() as float
|
|
|
|
|
"use ggml_row_size() instead");
|
2022-09-25 18:23:15 +00:00
|
|
|
|
2024-10-02 23:49:47 +00:00
|
|
|
GGML_API const char * ggml_type_name(enum ggml_type type);
|
|
|
|
|
GGML_API const char * ggml_op_name (enum ggml_op op);
|
|
|
|
|
GGML_API const char * ggml_op_symbol(enum ggml_op op);
|
2022-09-25 18:23:15 +00:00
|
|
|
|
2024-10-02 23:49:47 +00:00
|
|
|
GGML_API const char * ggml_unary_op_name(enum ggml_unary_op op);
|
|
|
|
|
GGML_API const char * ggml_op_desc(const struct ggml_tensor * t); // unary or op name
|
2023-12-07 20:27:19 +00:00
|
|
|
|
2024-10-02 23:49:47 +00:00
|
|
|
GGML_API size_t ggml_element_size(const struct ggml_tensor * tensor);
|
2023-04-29 09:31:52 +00:00
|
|
|
|
2024-10-02 23:49:47 +00:00
|
|
|
GGML_API bool ggml_is_quantized(enum ggml_type type);
|
2023-04-29 09:31:52 +00:00
|
|
|
|
2023-05-02 18:23:54 +00:00
|
|
|
// TODO: temporary until model loading of ggml examples is refactored
|
2023-04-30 15:51:57 +00:00
|
|
|
GGML_API enum ggml_type ggml_ftype_to_ggml_type(enum ggml_ftype ftype);
|
|
|
|
|
|
2024-10-02 23:49:47 +00:00
|
|
|
GGML_API bool ggml_is_transposed(const struct ggml_tensor * tensor);
|
|
|
|
|
GGML_API bool ggml_is_permuted (const struct ggml_tensor * tensor);
|
|
|
|
|
GGML_API bool ggml_is_empty (const struct ggml_tensor * tensor);
|
|
|
|
|
GGML_API bool ggml_is_scalar (const struct ggml_tensor * tensor);
|
|
|
|
|
GGML_API bool ggml_is_vector (const struct ggml_tensor * tensor);
|
|
|
|
|
GGML_API bool ggml_is_matrix (const struct ggml_tensor * tensor);
|
|
|
|
|
GGML_API bool ggml_is_3d (const struct ggml_tensor * tensor);
|
|
|
|
|
GGML_API int ggml_n_dims (const struct ggml_tensor * tensor); // returns 1 for scalars
|
2023-06-25 11:22:21 +00:00
|
|
|
|
2024-10-02 23:49:47 +00:00
|
|
|
GGML_API bool ggml_is_contiguous (const struct ggml_tensor * tensor);
|
|
|
|
|
GGML_API bool ggml_is_contiguous_0(const struct ggml_tensor * tensor); // same as ggml_is_contiguous()
|
|
|
|
|
GGML_API bool ggml_is_contiguous_1(const struct ggml_tensor * tensor); // contiguous for dims >= 1
|
|
|
|
|
GGML_API bool ggml_is_contiguous_2(const struct ggml_tensor * tensor); // contiguous for dims >= 2
|
2024-05-29 17:17:31 +00:00
|
|
|
|
2024-05-14 16:09:30 +00:00
|
|
|
GGML_API bool ggml_are_same_shape (const struct ggml_tensor * t0, const struct ggml_tensor * t1);
|
|
|
|
|
GGML_API bool ggml_are_same_stride(const struct ggml_tensor * t0, const struct ggml_tensor * t1);
|
2023-09-05 10:54:40 +00:00
|
|
|
|
2024-07-17 11:23:50 +00:00
|
|
|
GGML_API bool ggml_can_repeat(const struct ggml_tensor * t0, const struct ggml_tensor * t1);
|
|
|
|
|
|
2023-06-25 11:22:21 +00:00
|
|
|
// use this to compute the memory overhead of a tensor
|
|
|
|
|
GGML_API size_t ggml_tensor_overhead(void);
|
|
|
|
|
|
2024-04-26 16:39:58 +00:00
|
|
|
GGML_API bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbytes);
|
|
|
|
|
|
2023-04-29 09:31:52 +00:00
|
|
|
// main
|
|
|
|
|
|
2024-10-31 20:00:09 +00:00
|
|
|
GGML_API struct ggml_context * ggml_init (struct ggml_init_params params);
|
|
|
|
|
GGML_API void ggml_reset(struct ggml_context * ctx);
|
|
|
|
|
GGML_API void ggml_free (struct ggml_context * ctx);
|
2023-04-29 09:31:52 +00:00
|
|
|
|
|
|
|
|
GGML_API size_t ggml_used_mem(const struct ggml_context * ctx);
|
|
|
|
|
|
2023-09-05 10:54:40 +00:00
|
|
|
GGML_API bool ggml_get_no_alloc(struct ggml_context * ctx);
|
2023-06-25 11:22:21 +00:00
|
|
|
GGML_API void ggml_set_no_alloc(struct ggml_context * ctx, bool no_alloc);
|
|
|
|
|
|
|
|
|
|
GGML_API void * ggml_get_mem_buffer (const struct ggml_context * ctx);
|
|
|
|
|
GGML_API size_t ggml_get_mem_size (const struct ggml_context * ctx);
|
|
|
|
|
GGML_API size_t ggml_get_max_tensor_size(const struct ggml_context * ctx);
|
2023-04-29 09:31:52 +00:00
|
|
|
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_new_tensor(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
enum ggml_type type,
|
|
|
|
|
int n_dims,
|
|
|
|
|
const int64_t *ne);
|
|
|
|
|
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_new_tensor_1d(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
enum ggml_type type,
|
|
|
|
|
int64_t ne0);
|
|
|
|
|
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_new_tensor_2d(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
enum ggml_type type,
|
|
|
|
|
int64_t ne0,
|
|
|
|
|
int64_t ne1);
|
|
|
|
|
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_new_tensor_3d(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
enum ggml_type type,
|
|
|
|
|
int64_t ne0,
|
|
|
|
|
int64_t ne1,
|
|
|
|
|
int64_t ne2);
|
|
|
|
|
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_new_tensor_4d(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
enum ggml_type type,
|
|
|
|
|
int64_t ne0,
|
|
|
|
|
int64_t ne1,
|
|
|
|
|
int64_t ne2,
|
|
|
|
|
int64_t ne3);
|
|
|
|
|
|
2024-11-03 18:34:08 +00:00
|
|
|
GGML_API void * ggml_new_buffer(struct ggml_context * ctx, size_t nbytes);
|
2023-04-29 09:31:52 +00:00
|
|
|
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_dup_tensor (struct ggml_context * ctx, const struct ggml_tensor * src);
|
2023-09-05 17:57:27 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_view_tensor(struct ggml_context * ctx, struct ggml_tensor * src);
|
2023-04-29 09:31:52 +00:00
|
|
|
|
2023-11-03 19:35:05 +00:00
|
|
|
// Context tensor enumeration and lookup
|
2023-12-22 15:53:39 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_get_first_tensor(const struct ggml_context * ctx);
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_get_next_tensor (const struct ggml_context * ctx, struct ggml_tensor * tensor);
|
2023-06-25 11:22:21 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_get_tensor(struct ggml_context * ctx, const char * name);
|
|
|
|
|
|
2023-11-03 19:35:05 +00:00
|
|
|
// Converts a flat index into coordinates
|
2024-11-03 18:34:08 +00:00
|
|
|
GGML_API void ggml_unravel_index(const struct ggml_tensor * tensor, int64_t i, int64_t * i0, int64_t * i1, int64_t * i2, int64_t * i3);
|
2023-04-29 09:31:52 +00:00
|
|
|
|
2024-11-03 18:34:08 +00:00
|
|
|
GGML_API enum ggml_unary_op ggml_get_unary_op(const struct ggml_tensor * tensor);
|
2023-11-03 19:35:05 +00:00
|
|
|
|
2023-04-29 09:31:52 +00:00
|
|
|
GGML_API void * ggml_get_data (const struct ggml_tensor * tensor);
|
|
|
|
|
GGML_API float * ggml_get_data_f32(const struct ggml_tensor * tensor);
|
2022-09-25 18:23:15 +00:00
|
|
|
|
2023-09-05 10:54:40 +00:00
|
|
|
GGML_API const char * ggml_get_name (const struct ggml_tensor * tensor);
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_set_name ( struct ggml_tensor * tensor, const char * name);
|
2023-09-15 11:49:56 +00:00
|
|
|
GGML_ATTRIBUTE_FORMAT(2, 3)
|
2023-09-05 10:54:40 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_format_name( struct ggml_tensor * tensor, const char * fmt, ...);
|
2023-05-02 18:23:54 +00:00
|
|
|
|
2024-11-03 18:34:08 +00:00
|
|
|
// Tensor flags
|
|
|
|
|
GGML_API void ggml_set_input(struct ggml_tensor * tensor);
|
|
|
|
|
GGML_API void ggml_set_output(struct ggml_tensor * tensor);
|
|
|
|
|
GGML_API void ggml_set_param(struct ggml_context * ctx, struct ggml_tensor * tensor);
|
|
|
|
|
GGML_API void ggml_set_loss(struct ggml_tensor * tensor);
|
|
|
|
|
|
2022-09-25 18:23:15 +00:00
|
|
|
//
|
2023-04-29 09:31:52 +00:00
|
|
|
// operations on tensors with backpropagation
|
2022-09-25 18:23:15 +00:00
|
|
|
//
|
|
|
|
|
|
2023-04-29 09:31:52 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_dup(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a);
|
|
|
|
|
|
2023-09-05 10:54:40 +00:00
|
|
|
// in-place, returns view(a)
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_dup_inplace(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a);
|
|
|
|
|
|
2023-04-29 09:31:52 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_add(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
struct ggml_tensor * b);
|
|
|
|
|
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_add_inplace(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
struct ggml_tensor * b);
|
|
|
|
|
|
2023-11-03 19:35:05 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_add_cast(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
struct ggml_tensor * b,
|
|
|
|
|
enum ggml_type type);
|
|
|
|
|
|
2023-05-14 15:04:23 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_add1(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
struct ggml_tensor * b);
|
|
|
|
|
|
2023-06-25 11:22:21 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_add1_inplace(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
struct ggml_tensor * b);
|
|
|
|
|
|
2023-12-13 19:55:03 +00:00
|
|
|
// dst = a
|
|
|
|
|
// view(dst, nb1, nb2, nb3, offset) += b
|
|
|
|
|
// return dst
|
2023-05-14 15:04:23 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_acc(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
struct ggml_tensor * b,
|
|
|
|
|
size_t nb1,
|
|
|
|
|
size_t nb2,
|
|
|
|
|
size_t nb3,
|
|
|
|
|
size_t offset);
|
|
|
|
|
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_acc_inplace(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
struct ggml_tensor * b,
|
|
|
|
|
size_t nb1,
|
|
|
|
|
size_t nb2,
|
|
|
|
|
size_t nb3,
|
|
|
|
|
size_t offset);
|
|
|
|
|
|
2023-04-29 09:31:52 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_sub(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
struct ggml_tensor * b);
|
|
|
|
|
|
2023-06-25 11:22:21 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_sub_inplace(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
struct ggml_tensor * b);
|
|
|
|
|
|
2023-04-29 09:31:52 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_mul(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
struct ggml_tensor * b);
|
|
|
|
|
|
2023-06-25 11:22:21 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_mul_inplace(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
struct ggml_tensor * b);
|
|
|
|
|
|
2023-04-29 09:31:52 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_div(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
struct ggml_tensor * b);
|
|
|
|
|
|
2023-06-25 11:22:21 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_div_inplace(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
struct ggml_tensor * b);
|
|
|
|
|
|
2023-04-29 09:31:52 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_sqr(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a);
|
|
|
|
|
|
2023-06-25 11:22:21 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_sqr_inplace(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a);
|
|
|
|
|
|
2023-04-29 09:31:52 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_sqrt(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a);
|
|
|
|
|
|
2023-06-25 11:22:21 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_sqrt_inplace(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a);
|
|
|
|
|
|
2023-05-14 15:04:23 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_log(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a);
|
|
|
|
|
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_log_inplace(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a);
|
|
|
|
|
|
2024-08-12 13:02:08 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_sin(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a);
|
|
|
|
|
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_sin_inplace(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a);
|
|
|
|
|
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_cos(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a);
|
|
|
|
|
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_cos_inplace(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a);
|
|
|
|
|
|
2023-04-29 09:31:52 +00:00
|
|
|
// return scalar
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_sum(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a);
|
|
|
|
|
|
2023-05-14 15:04:23 +00:00
|
|
|
// sums along rows, with input shape [a,b,c,d] return shape [1,b,c,d]
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_sum_rows(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a);
|
|
|
|
|
|
2023-04-29 09:31:52 +00:00
|
|
|
// mean along rows
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_mean(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a);
|
|
|
|
|
|
2023-07-02 18:45:27 +00:00
|
|
|
// argmax along rows
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_argmax(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a);
|
|
|
|
|
|
2024-10-03 15:29:59 +00:00
|
|
|
// count number of equal elements in a and b
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_count_equal(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
struct ggml_tensor * b);
|
|
|
|
|
|
2023-04-29 09:31:52 +00:00
|
|
|
// if a is the same shape as b, and a is not parameter, return a
|
|
|
|
|
// otherwise, return a new tensor: repeat(a) to fit in b
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_repeat(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
struct ggml_tensor * b);
|
|
|
|
|
|
2023-11-03 19:35:05 +00:00
|
|
|
// sums repetitions in a into shape of b
|
2023-06-25 11:22:21 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_repeat_back(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
struct ggml_tensor * b);
|
|
|
|
|
|
2024-05-28 08:04:19 +00:00
|
|
|
// concat a and b along dim
|
2023-09-05 10:54:40 +00:00
|
|
|
// used in stable-diffusion
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_concat(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
2024-05-28 08:04:19 +00:00
|
|
|
struct ggml_tensor * b,
|
|
|
|
|
int dim);
|
2023-09-05 10:54:40 +00:00
|
|
|
|
2023-04-29 09:31:52 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_abs(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a);
|
|
|
|
|
|
2023-06-25 11:22:21 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_abs_inplace(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a);
|
|
|
|
|
|
2023-04-29 09:31:52 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_sgn(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a);
|
|
|
|
|
|
2023-06-25 11:22:21 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_sgn_inplace(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a);
|
|
|
|
|
|
2023-04-29 09:31:52 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_neg(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a);
|
|
|
|
|
|
2023-06-25 11:22:21 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_neg_inplace(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a);
|
|
|
|
|
|
2023-04-29 09:31:52 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_step(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a);
|
|
|
|
|
|
2023-06-25 11:22:21 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_step_inplace(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a);
|
|
|
|
|
|
2023-07-02 18:45:27 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_tanh(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a);
|
|
|
|
|
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_tanh_inplace(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a);
|
|
|
|
|
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_elu(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a);
|
|
|
|
|
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_elu_inplace(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a);
|
|
|
|
|
|
2023-04-29 09:31:52 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_relu(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a);
|
|
|
|
|
|
2023-12-13 19:55:03 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_leaky_relu(
|
2023-11-03 19:35:05 +00:00
|
|
|
struct ggml_context * ctx,
|
2023-12-13 19:55:03 +00:00
|
|
|
struct ggml_tensor * a, float negative_slope, bool inplace);
|
2023-11-03 19:35:05 +00:00
|
|
|
|
2023-06-25 11:22:21 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_relu_inplace(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a);
|
|
|
|
|
|
2024-05-11 13:50:54 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_sigmoid(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a);
|
|
|
|
|
|
2024-05-01 21:44:26 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_sigmoid_inplace(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a);
|
|
|
|
|
|
2023-04-29 09:31:52 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_gelu(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a);
|
|
|
|
|
|
2023-06-25 11:22:21 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_gelu_inplace(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a);
|
|
|
|
|
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_gelu_quick(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a);
|
|
|
|
|
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_gelu_quick_inplace(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a);
|
|
|
|
|
|
2023-04-29 09:31:52 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_silu(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a);
|
|
|
|
|
|
2023-06-25 11:22:21 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_silu_inplace(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a);
|
|
|
|
|
|
2023-05-14 15:04:23 +00:00
|
|
|
// a - x
|
|
|
|
|
// b - dy
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_silu_back(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
struct ggml_tensor * b);
|
|
|
|
|
|
2024-01-22 13:09:35 +00:00
|
|
|
// hardswish(x) = x * relu6(x + 3) / 6
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_hardswish(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a);
|
|
|
|
|
|
|
|
|
|
// hardsigmoid(x) = relu6(x + 3) / 6
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_hardsigmoid(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a);
|
|
|
|
|
|
2024-09-01 14:38:17 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_exp(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a);
|
|
|
|
|
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_exp_inplace(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a);
|
|
|
|
|
|
2023-04-29 09:31:52 +00:00
|
|
|
// normalize along rows
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_norm(
|
|
|
|
|
struct ggml_context * ctx,
|
2023-09-05 10:54:40 +00:00
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
float eps);
|
2023-04-29 09:31:52 +00:00
|
|
|
|
2023-06-25 11:22:21 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_norm_inplace(
|
|
|
|
|
struct ggml_context * ctx,
|
2023-09-05 10:54:40 +00:00
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
float eps);
|
2023-06-25 11:22:21 +00:00
|
|
|
|
2023-04-29 09:31:52 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_rms_norm(
|
|
|
|
|
struct ggml_context * ctx,
|
2023-09-05 10:54:40 +00:00
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
float eps);
|
2023-04-29 09:31:52 +00:00
|
|
|
|
2023-06-25 11:22:21 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_rms_norm_inplace(
|
|
|
|
|
struct ggml_context * ctx,
|
2023-09-05 10:54:40 +00:00
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
float eps);
|
|
|
|
|
|
|
|
|
|
// group normalize along ne0*ne1*n_groups
|
|
|
|
|
// used in stable-diffusion
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_group_norm(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
2024-08-06 07:26:46 +00:00
|
|
|
int n_groups,
|
|
|
|
|
float eps);
|
2023-09-05 10:54:40 +00:00
|
|
|
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_group_norm_inplace(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
2024-08-06 07:26:46 +00:00
|
|
|
int n_groups,
|
|
|
|
|
float eps);
|
2023-06-25 11:22:21 +00:00
|
|
|
|
2023-05-14 15:04:23 +00:00
|
|
|
// a - x
|
|
|
|
|
// b - dy
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_rms_norm_back(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
2023-09-05 17:57:27 +00:00
|
|
|
struct ggml_tensor * b,
|
|
|
|
|
float eps);
|
2023-05-14 15:04:23 +00:00
|
|
|
|
2023-11-03 19:35:05 +00:00
|
|
|
// A: k columns, n rows => [ne03, ne02, n, k]
|
|
|
|
|
// B: k columns, m rows (i.e. we transpose it internally) => [ne03 * x, ne02 * y, m, k]
|
|
|
|
|
// result is n columns, m rows => [ne03 * x, ne02 * y, m, n]
|
2023-04-29 09:31:52 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_mul_mat(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
struct ggml_tensor * b);
|
|
|
|
|
|
2023-12-22 15:53:39 +00:00
|
|
|
// change the precision of a matrix multiplication
|
|
|
|
|
// set to GGML_PREC_F32 for higher precision (useful for phi-2)
|
|
|
|
|
GGML_API void ggml_mul_mat_set_prec(
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
enum ggml_prec prec);
|
|
|
|
|
|
2023-12-07 20:27:19 +00:00
|
|
|
// indirect matrix multiplication
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_mul_mat_id(
|
|
|
|
|
struct ggml_context * ctx,
|
2024-04-03 13:07:05 +00:00
|
|
|
struct ggml_tensor * as,
|
2024-04-18 13:18:48 +00:00
|
|
|
struct ggml_tensor * b,
|
|
|
|
|
struct ggml_tensor * ids);
|
2023-12-07 20:27:19 +00:00
|
|
|
|
2023-06-25 11:22:21 +00:00
|
|
|
// A: m columns, n rows,
|
|
|
|
|
// B: p columns, n rows,
|
|
|
|
|
// result is m columns, p rows
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_out_prod(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
struct ggml_tensor * b);
|
|
|
|
|
|
2022-09-25 18:23:15 +00:00
|
|
|
//
|
2023-04-29 09:31:52 +00:00
|
|
|
// operations on tensors without backpropagation
|
2022-09-25 18:23:15 +00:00
|
|
|
//
|
|
|
|
|
|
2023-04-29 09:31:52 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_scale(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
2023-12-22 15:53:39 +00:00
|
|
|
float s);
|
2023-04-29 09:31:52 +00:00
|
|
|
|
2023-05-14 15:04:23 +00:00
|
|
|
// in-place, returns view(a)
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_scale_inplace(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
2023-12-22 15:53:39 +00:00
|
|
|
float s);
|
2023-05-14 15:04:23 +00:00
|
|
|
|
|
|
|
|
// b -> view(a,offset,nb1,nb2,3), return modified a
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_set(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
struct ggml_tensor * b,
|
|
|
|
|
size_t nb1,
|
|
|
|
|
size_t nb2,
|
|
|
|
|
size_t nb3,
|
2024-09-03 15:21:46 +00:00
|
|
|
size_t offset); // in bytes
|
2023-05-14 15:04:23 +00:00
|
|
|
|
|
|
|
|
// b -> view(a,offset,nb1,nb2,3), return view(a)
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_set_inplace(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
struct ggml_tensor * b,
|
|
|
|
|
size_t nb1,
|
|
|
|
|
size_t nb2,
|
|
|
|
|
size_t nb3,
|
2024-09-03 15:21:46 +00:00
|
|
|
size_t offset); // in bytes
|
2023-05-14 15:04:23 +00:00
|
|
|
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_set_1d(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
struct ggml_tensor * b,
|
2024-09-03 15:21:46 +00:00
|
|
|
size_t offset); // in bytes
|
2023-05-14 15:04:23 +00:00
|
|
|
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_set_1d_inplace(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
struct ggml_tensor * b,
|
2024-09-03 15:21:46 +00:00
|
|
|
size_t offset); // in bytes
|
2023-05-14 15:04:23 +00:00
|
|
|
|
|
|
|
|
// b -> view(a,offset,nb1,nb2,3), return modified a
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_set_2d(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
struct ggml_tensor * b,
|
|
|
|
|
size_t nb1,
|
2024-09-03 15:21:46 +00:00
|
|
|
size_t offset); // in bytes
|
2023-05-14 15:04:23 +00:00
|
|
|
|
|
|
|
|
// b -> view(a,offset,nb1,nb2,3), return view(a)
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_set_2d_inplace(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
struct ggml_tensor * b,
|
|
|
|
|
size_t nb1,
|
2024-09-03 15:21:46 +00:00
|
|
|
size_t offset); // in bytes
|
2023-05-14 15:04:23 +00:00
|
|
|
|
2023-04-29 09:31:52 +00:00
|
|
|
// a -> b, return view(b)
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_cpy(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
struct ggml_tensor * b);
|
|
|
|
|
|
2024-01-12 19:07:38 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_cast(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
enum ggml_type type);
|
|
|
|
|
|
2023-04-29 09:31:52 +00:00
|
|
|
// make contiguous
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_cont(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a);
|
|
|
|
|
|
2023-11-03 19:35:05 +00:00
|
|
|
// make contiguous, with new shape
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_cont_1d(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
int64_t ne0);
|
|
|
|
|
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_cont_2d(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
int64_t ne0,
|
|
|
|
|
int64_t ne1);
|
|
|
|
|
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_cont_3d(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
int64_t ne0,
|
|
|
|
|
int64_t ne1,
|
|
|
|
|
int64_t ne2);
|
|
|
|
|
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_cont_4d(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
int64_t ne0,
|
|
|
|
|
int64_t ne1,
|
|
|
|
|
int64_t ne2,
|
|
|
|
|
int64_t ne3);
|
|
|
|
|
|
2023-04-29 09:31:52 +00:00
|
|
|
// return view(a), b specifies the new shape
|
|
|
|
|
// TODO: when we start computing gradient, make a copy instead of view
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_reshape(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
struct ggml_tensor * b);
|
|
|
|
|
|
|
|
|
|
// return view(a)
|
|
|
|
|
// TODO: when we start computing gradient, make a copy instead of view
|
2023-05-14 15:04:23 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_reshape_1d(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
int64_t ne0);
|
|
|
|
|
|
2023-04-29 09:31:52 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_reshape_2d(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
int64_t ne0,
|
|
|
|
|
int64_t ne1);
|
|
|
|
|
|
|
|
|
|
// return view(a)
|
|
|
|
|
// TODO: when we start computing gradient, make a copy instead of view
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_reshape_3d(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
int64_t ne0,
|
|
|
|
|
int64_t ne1,
|
|
|
|
|
int64_t ne2);
|
|
|
|
|
|
2023-05-14 15:04:23 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_reshape_4d(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
int64_t ne0,
|
|
|
|
|
int64_t ne1,
|
|
|
|
|
int64_t ne2,
|
|
|
|
|
int64_t ne3);
|
|
|
|
|
|
2023-04-29 09:31:52 +00:00
|
|
|
// offset in bytes
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_view_1d(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
int64_t ne0,
|
|
|
|
|
size_t offset);
|
|
|
|
|
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_view_2d(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
int64_t ne0,
|
|
|
|
|
int64_t ne1,
|
|
|
|
|
size_t nb1, // row stride in bytes
|
|
|
|
|
size_t offset);
|
|
|
|
|
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_view_3d(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
int64_t ne0,
|
|
|
|
|
int64_t ne1,
|
|
|
|
|
int64_t ne2,
|
|
|
|
|
size_t nb1, // row stride in bytes
|
|
|
|
|
size_t nb2, // slice stride in bytes
|
|
|
|
|
size_t offset);
|
|
|
|
|
|
2023-05-14 15:04:23 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_view_4d(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
int64_t ne0,
|
|
|
|
|
int64_t ne1,
|
|
|
|
|
int64_t ne2,
|
|
|
|
|
int64_t ne3,
|
|
|
|
|
size_t nb1, // row stride in bytes
|
|
|
|
|
size_t nb2, // slice stride in bytes
|
|
|
|
|
size_t nb3,
|
|
|
|
|
size_t offset);
|
|
|
|
|
|
2023-04-29 09:31:52 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_permute(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
int axis0,
|
|
|
|
|
int axis1,
|
|
|
|
|
int axis2,
|
|
|
|
|
int axis3);
|
|
|
|
|
|
|
|
|
|
// alias for ggml_permute(ctx, a, 1, 0, 2, 3)
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_transpose(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a);
|
|
|
|
|
|
2023-12-13 19:55:03 +00:00
|
|
|
// supports 3D: a->ne[2] == b->ne[1]
|
2023-04-29 09:31:52 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_get_rows(
|
|
|
|
|
struct ggml_context * ctx,
|
2024-09-29 21:18:02 +00:00
|
|
|
struct ggml_tensor * a, // data
|
|
|
|
|
struct ggml_tensor * b); // row indices
|
2023-04-29 09:31:52 +00:00
|
|
|
|
2023-05-14 15:04:23 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_get_rows_back(
|
|
|
|
|
struct ggml_context * ctx,
|
2024-09-29 21:18:02 +00:00
|
|
|
struct ggml_tensor * a, // gradients of ggml_get_rows result
|
|
|
|
|
struct ggml_tensor * b, // row indices
|
|
|
|
|
struct ggml_tensor * c); // data for ggml_get_rows, only used for its shape
|
2023-05-14 15:04:23 +00:00
|
|
|
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_diag(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a);
|
|
|
|
|
|
2023-04-29 09:31:52 +00:00
|
|
|
// set elements above the diagonal to -INF
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_diag_mask_inf(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
int n_past);
|
|
|
|
|
|
|
|
|
|
// in-place, returns view(a)
|
2023-05-14 15:04:23 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_diag_mask_inf_inplace(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
int n_past);
|
|
|
|
|
|
|
|
|
|
// set elements above the diagonal to 0
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_diag_mask_zero(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
int n_past);
|
|
|
|
|
|
|
|
|
|
// in-place, returns view(a)
|
2023-05-20 15:56:30 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_diag_mask_zero_inplace(
|
2023-05-14 15:04:23 +00:00
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
int n_past);
|
|
|
|
|
|
2023-04-29 09:31:52 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_soft_max(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a);
|
|
|
|
|
|
|
|
|
|
// in-place, returns view(a)
|
2023-05-14 15:04:23 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_soft_max_inplace(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a);
|
|
|
|
|
|
2024-05-11 07:32:41 +00:00
|
|
|
// fused soft_max(a*scale + mask*(ALiBi slope))
|
2023-12-07 20:27:19 +00:00
|
|
|
// mask is optional
|
2024-02-19 13:18:09 +00:00
|
|
|
// max_bias = 0.0f for no ALiBi
|
2023-12-07 20:27:19 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_soft_max_ext(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
struct ggml_tensor * mask,
|
2024-02-19 13:18:09 +00:00
|
|
|
float scale,
|
|
|
|
|
float max_bias);
|
2023-12-07 20:27:19 +00:00
|
|
|
|
2023-06-25 11:22:21 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_soft_max_back(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
struct ggml_tensor * b);
|
|
|
|
|
|
|
|
|
|
// in-place, returns view(a)
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_soft_max_back_inplace(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
struct ggml_tensor * b);
|
|
|
|
|
|
2023-05-14 15:04:23 +00:00
|
|
|
// rotary position embedding
|
2024-08-13 19:13:15 +00:00
|
|
|
// if (mode & 1) - skip n_past elements (NOT SUPPORTED)
|
|
|
|
|
// if (mode & GGML_ROPE_TYPE_NEOX) - GPT-NeoX style
|
2023-11-03 19:35:05 +00:00
|
|
|
//
|
|
|
|
|
// b is an int32 vector with size a->ne[2], it contains the positions
|
2023-04-29 09:31:52 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_rope(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
2023-11-03 19:35:05 +00:00
|
|
|
struct ggml_tensor * b,
|
2023-04-29 09:31:52 +00:00
|
|
|
int n_dims,
|
2024-06-05 08:29:20 +00:00
|
|
|
int mode);
|
2023-04-29 09:31:52 +00:00
|
|
|
|
2023-05-14 15:04:23 +00:00
|
|
|
// in-place, returns view(a)
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_rope_inplace(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
2023-11-03 19:35:05 +00:00
|
|
|
struct ggml_tensor * b,
|
2023-05-14 15:04:23 +00:00
|
|
|
int n_dims,
|
2024-06-05 08:29:20 +00:00
|
|
|
int mode);
|
2023-05-14 15:04:23 +00:00
|
|
|
|
2023-09-05 10:54:40 +00:00
|
|
|
// custom RoPE
|
2024-07-29 13:06:06 +00:00
|
|
|
// c is freq factors (e.g. phi3-128k), (optional)
|
2024-05-21 20:28:32 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_rope_ext(
|
2023-09-05 10:54:40 +00:00
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
2023-11-03 19:35:05 +00:00
|
|
|
struct ggml_tensor * b,
|
2024-05-21 20:28:32 +00:00
|
|
|
struct ggml_tensor * c,
|
2023-09-05 10:54:40 +00:00
|
|
|
int n_dims,
|
|
|
|
|
int mode,
|
2024-06-05 08:29:20 +00:00
|
|
|
int n_ctx_orig,
|
2023-09-05 10:54:40 +00:00
|
|
|
float freq_base,
|
2023-11-03 19:35:05 +00:00
|
|
|
float freq_scale,
|
|
|
|
|
float ext_factor,
|
|
|
|
|
float attn_factor,
|
|
|
|
|
float beta_fast,
|
|
|
|
|
float beta_slow);
|
2023-09-05 10:54:40 +00:00
|
|
|
|
|
|
|
|
// in-place, returns view(a)
|
2024-05-21 20:28:32 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_rope_ext_inplace(
|
2023-09-05 10:54:40 +00:00
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
2023-11-03 19:35:05 +00:00
|
|
|
struct ggml_tensor * b,
|
2024-05-21 20:28:32 +00:00
|
|
|
struct ggml_tensor * c,
|
2023-09-05 10:54:40 +00:00
|
|
|
int n_dims,
|
|
|
|
|
int mode,
|
2024-06-05 08:29:20 +00:00
|
|
|
int n_ctx_orig,
|
2023-09-05 10:54:40 +00:00
|
|
|
float freq_base,
|
2023-11-03 19:35:05 +00:00
|
|
|
float freq_scale,
|
|
|
|
|
float ext_factor,
|
|
|
|
|
float attn_factor,
|
|
|
|
|
float beta_fast,
|
|
|
|
|
float beta_slow);
|
|
|
|
|
|
2024-05-21 20:28:32 +00:00
|
|
|
GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_rope_custom(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
struct ggml_tensor * b,
|
|
|
|
|
int n_dims,
|
|
|
|
|
int mode,
|
2024-06-05 08:29:20 +00:00
|
|
|
int n_ctx_orig,
|
2024-05-21 20:28:32 +00:00
|
|
|
float freq_base,
|
|
|
|
|
float freq_scale,
|
|
|
|
|
float ext_factor,
|
|
|
|
|
float attn_factor,
|
|
|
|
|
float beta_fast,
|
|
|
|
|
float beta_slow),
|
|
|
|
|
"use ggml_rope_ext instead");
|
2023-09-05 10:54:40 +00:00
|
|
|
|
2024-05-21 20:28:32 +00:00
|
|
|
GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_rope_custom_inplace(
|
2023-09-05 10:54:40 +00:00
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
2023-11-03 19:35:05 +00:00
|
|
|
struct ggml_tensor * b,
|
2023-09-05 10:54:40 +00:00
|
|
|
int n_dims,
|
2024-05-21 20:28:32 +00:00
|
|
|
int mode,
|
2024-06-05 08:29:20 +00:00
|
|
|
int n_ctx_orig,
|
2024-05-21 20:28:32 +00:00
|
|
|
float freq_base,
|
|
|
|
|
float freq_scale,
|
|
|
|
|
float ext_factor,
|
|
|
|
|
float attn_factor,
|
|
|
|
|
float beta_fast,
|
|
|
|
|
float beta_slow),
|
|
|
|
|
"use ggml_rope_ext_inplace instead");
|
|
|
|
|
|
|
|
|
|
// compute correction dims for YaRN RoPE scaling
|
2024-11-14 17:04:35 +00:00
|
|
|
GGML_API void ggml_rope_yarn_corr_dims(
|
2024-06-05 08:29:20 +00:00
|
|
|
int n_dims, int n_ctx_orig, float freq_base, float beta_fast, float beta_slow, float dims[2]);
|
2023-09-05 10:54:40 +00:00
|
|
|
|
2023-05-14 15:04:23 +00:00
|
|
|
// rotary position embedding backward, i.e compute dx from dy
|
|
|
|
|
// a - dy
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_rope_back(
|
|
|
|
|
struct ggml_context * ctx,
|
2024-09-29 21:18:02 +00:00
|
|
|
struct ggml_tensor * a, // gradients of ggml_rope result
|
|
|
|
|
struct ggml_tensor * b, // positions
|
|
|
|
|
struct ggml_tensor * c, // freq factors
|
2023-05-14 15:04:23 +00:00
|
|
|
int n_dims,
|
2023-09-05 10:54:40 +00:00
|
|
|
int mode,
|
2024-06-05 08:29:20 +00:00
|
|
|
int n_ctx_orig,
|
2023-09-05 10:54:40 +00:00
|
|
|
float freq_base,
|
|
|
|
|
float freq_scale,
|
2023-11-17 08:00:07 +00:00
|
|
|
float ext_factor,
|
|
|
|
|
float attn_factor,
|
|
|
|
|
float beta_fast,
|
2024-06-05 08:29:20 +00:00
|
|
|
float beta_slow);
|
2023-05-14 15:04:23 +00:00
|
|
|
|
2023-05-20 15:56:30 +00:00
|
|
|
// clamp
|
|
|
|
|
// in-place, returns view(a)
|
2023-11-03 19:35:05 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_clamp(
|
2023-05-20 15:56:30 +00:00
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
float min,
|
|
|
|
|
float max);
|
2023-04-29 09:31:52 +00:00
|
|
|
|
2024-07-30 13:56:35 +00:00
|
|
|
// im2col
|
|
|
|
|
// converts data into a format that effectively results in a convolution when combined with matrix multiplication
|
2023-11-12 13:31:08 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_im2col(
|
|
|
|
|
struct ggml_context * ctx,
|
2024-07-30 13:56:35 +00:00
|
|
|
struct ggml_tensor * a, // convolution kernel
|
|
|
|
|
struct ggml_tensor * b, // data
|
|
|
|
|
int s0, // stride dimension 0
|
|
|
|
|
int s1, // stride dimension 1
|
|
|
|
|
int p0, // padding dimension 0
|
|
|
|
|
int p1, // padding dimension 1
|
|
|
|
|
int d0, // dilation dimension 0
|
|
|
|
|
int d1, // dilation dimension 1
|
|
|
|
|
bool is_2D,
|
|
|
|
|
enum ggml_type dst_type);
|
|
|
|
|
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_im2col_back(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a, // convolution kernel
|
|
|
|
|
struct ggml_tensor * b, // gradient of im2col output
|
|
|
|
|
int64_t * ne, // shape of im2col input
|
|
|
|
|
int s0, // stride dimension 0
|
|
|
|
|
int s1, // stride dimension 1
|
|
|
|
|
int p0, // padding dimension 0
|
|
|
|
|
int p1, // padding dimension 1
|
|
|
|
|
int d0, // dilation dimension 0
|
|
|
|
|
int d1, // dilation dimension 1
|
|
|
|
|
bool is_2D);
|
2023-11-12 13:31:08 +00:00
|
|
|
|
2024-01-22 13:09:35 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_conv_depthwise_2d(
|
|
|
|
|
struct ggml_context * ctx,
|
2024-07-30 13:56:35 +00:00
|
|
|
struct ggml_tensor * a, // convolution kernel
|
|
|
|
|
struct ggml_tensor * b, // data
|
|
|
|
|
int s0, // stride dimension 0
|
|
|
|
|
int s1, // stride dimension 1
|
|
|
|
|
int p0, // padding dimension 0
|
|
|
|
|
int p1, // padding dimension 1
|
|
|
|
|
int d0, // dilation dimension 0
|
|
|
|
|
int d1); // dilation dimension 1
|
2024-01-22 13:09:35 +00:00
|
|
|
|
2023-07-02 18:45:27 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_conv_1d(
|
2023-04-29 09:31:52 +00:00
|
|
|
struct ggml_context * ctx,
|
2024-07-30 13:56:35 +00:00
|
|
|
struct ggml_tensor * a, // convolution kernel
|
|
|
|
|
struct ggml_tensor * b, // data
|
2023-07-02 18:45:27 +00:00
|
|
|
int s0, // stride
|
|
|
|
|
int p0, // padding
|
|
|
|
|
int d0); // dilation
|
2023-04-29 09:31:52 +00:00
|
|
|
|
2023-09-05 10:54:40 +00:00
|
|
|
// conv_1d with padding = half
|
|
|
|
|
// alias for ggml_conv_1d(a, b, s, a->ne[0]/2, d)
|
|
|
|
|
GGML_API struct ggml_tensor* ggml_conv_1d_ph(
|
|
|
|
|
struct ggml_context * ctx,
|
2024-07-30 13:56:35 +00:00
|
|
|
struct ggml_tensor * a, // convolution kernel
|
|
|
|
|
struct ggml_tensor * b, // data
|
|
|
|
|
int s, // stride
|
|
|
|
|
int d); // dilation
|
2023-09-05 10:54:40 +00:00
|
|
|
|
2023-11-03 19:35:05 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_conv_transpose_1d(
|
|
|
|
|
struct ggml_context * ctx,
|
2024-07-30 13:56:35 +00:00
|
|
|
struct ggml_tensor * a, // convolution kernel
|
|
|
|
|
struct ggml_tensor * b, // data
|
|
|
|
|
int s0, // stride
|
|
|
|
|
int p0, // padding
|
|
|
|
|
int d0); // dilation
|
2023-11-03 19:35:05 +00:00
|
|
|
|
2023-07-02 18:45:27 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_conv_2d(
|
2023-06-25 11:22:21 +00:00
|
|
|
struct ggml_context * ctx,
|
2024-07-30 13:56:35 +00:00
|
|
|
struct ggml_tensor * a, // convolution kernel
|
|
|
|
|
struct ggml_tensor * b, // data
|
|
|
|
|
int s0, // stride dimension 0
|
|
|
|
|
int s1, // stride dimension 1
|
|
|
|
|
int p0, // padding dimension 0
|
|
|
|
|
int p1, // padding dimension 1
|
|
|
|
|
int d0, // dilation dimension 0
|
|
|
|
|
int d1); // dilation dimension 1
|
2023-06-25 11:22:21 +00:00
|
|
|
|
2023-09-05 10:54:40 +00:00
|
|
|
|
|
|
|
|
// kernel size is a->ne[0] x a->ne[1]
|
|
|
|
|
// stride is equal to kernel size
|
|
|
|
|
// padding is zero
|
|
|
|
|
// example:
|
|
|
|
|
// a: 16 16 3 768
|
|
|
|
|
// b: 1024 1024 3 1
|
|
|
|
|
// res: 64 64 768 1
|
|
|
|
|
// used in sam
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_conv_2d_sk_p0(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
struct ggml_tensor * b);
|
|
|
|
|
|
|
|
|
|
// kernel size is a->ne[0] x a->ne[1]
|
|
|
|
|
// stride is 1
|
|
|
|
|
// padding is half
|
|
|
|
|
// example:
|
|
|
|
|
// a: 3 3 256 256
|
|
|
|
|
// b: 64 64 256 1
|
|
|
|
|
// res: 64 64 256 1
|
|
|
|
|
// used in sam
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_conv_2d_s1_ph(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
struct ggml_tensor * b);
|
|
|
|
|
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_conv_transpose_2d_p0(
|
2023-04-29 09:31:52 +00:00
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
2023-07-02 18:45:27 +00:00
|
|
|
struct ggml_tensor * b,
|
2023-09-05 10:54:40 +00:00
|
|
|
int stride);
|
|
|
|
|
|
|
|
|
|
enum ggml_op_pool {
|
|
|
|
|
GGML_OP_POOL_MAX,
|
|
|
|
|
GGML_OP_POOL_AVG,
|
|
|
|
|
GGML_OP_POOL_COUNT,
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_pool_1d(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
enum ggml_op_pool op,
|
|
|
|
|
int k0, // kernel size
|
|
|
|
|
int s0, // stride
|
|
|
|
|
int p0); // padding
|
|
|
|
|
|
2023-11-03 19:35:05 +00:00
|
|
|
// the result will have 2*p0 padding for the first dimension
|
|
|
|
|
// and 2*p1 padding for the second dimension
|
2023-09-05 10:54:40 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_pool_2d(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
enum ggml_op_pool op,
|
|
|
|
|
int k0,
|
|
|
|
|
int k1,
|
|
|
|
|
int s0,
|
|
|
|
|
int s1,
|
2023-11-03 19:35:05 +00:00
|
|
|
float p0,
|
|
|
|
|
float p1);
|
2023-09-05 10:54:40 +00:00
|
|
|
|
2024-07-30 13:56:35 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_pool_2d_back(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
struct ggml_tensor * af, // "a"/input used in forward pass
|
|
|
|
|
enum ggml_op_pool op,
|
|
|
|
|
int k0,
|
|
|
|
|
int k1,
|
|
|
|
|
int s0,
|
|
|
|
|
int s1,
|
|
|
|
|
float p0,
|
|
|
|
|
float p1);
|
|
|
|
|
|
2023-09-05 10:54:40 +00:00
|
|
|
// nearest interpolate
|
2024-05-15 08:52:33 +00:00
|
|
|
// multiplies ne0 and ne1 by scale factor
|
2023-09-05 10:54:40 +00:00
|
|
|
// used in stable-diffusion
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_upscale(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
int scale_factor);
|
2023-04-29 09:31:52 +00:00
|
|
|
|
2024-05-15 08:52:33 +00:00
|
|
|
// nearest interpolate
|
|
|
|
|
// nearest interpolate to specified dimensions
|
|
|
|
|
// used in tortoise.cpp
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_upscale_ext(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
int ne0,
|
|
|
|
|
int ne1,
|
|
|
|
|
int ne2,
|
|
|
|
|
int ne3);
|
|
|
|
|
|
2023-12-13 19:55:03 +00:00
|
|
|
// pad each dimension with zeros: [x, ..., x] -> [x, ..., x, 0, ..., 0]
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_pad(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
int p0,
|
|
|
|
|
int p1,
|
|
|
|
|
int p2,
|
|
|
|
|
int p3);
|
|
|
|
|
|
2024-12-03 19:20:04 +00:00
|
|
|
// pad each dimension with reflection: [a, b, c, d] -> [b, a, b, c, d, c]
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_pad_reflect_1d(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
int p0,
|
|
|
|
|
int p1);
|
|
|
|
|
|
2024-03-03 12:23:52 +00:00
|
|
|
// Ref: https://github.com/CompVis/stable-diffusion/blob/main/ldm/modules/diffusionmodules/util.py#L151
|
|
|
|
|
// timesteps: [N,]
|
|
|
|
|
// return: [N, dim]
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_timestep_embedding(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * timesteps,
|
|
|
|
|
int dim,
|
|
|
|
|
int max_period);
|
|
|
|
|
|
2023-12-07 20:27:19 +00:00
|
|
|
// sort rows
|
|
|
|
|
enum ggml_sort_order {
|
2024-02-25 10:09:09 +00:00
|
|
|
GGML_SORT_ORDER_ASC,
|
|
|
|
|
GGML_SORT_ORDER_DESC,
|
2023-12-07 20:27:19 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_argsort(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
enum ggml_sort_order order);
|
|
|
|
|
|
2024-03-03 12:23:52 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_arange(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
float start,
|
|
|
|
|
float stop,
|
|
|
|
|
float step);
|
|
|
|
|
|
2023-12-07 20:27:19 +00:00
|
|
|
// top k elements per row
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_top_k(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
int k);
|
|
|
|
|
|
ggml : add Flash Attention (llama/5021)
* ggml : add ggml_flash_attn_ext API
* ggml : fix GQA support in ggml_flash_attn_ext
* ggml : online attention (CPU)
* metal : initial implementation
* metal : f16 precision
* metal : reduce branches
* metal : specialize for head size
* wip : 8 rows per simd group
* wip : 4 rows per simd group
* wip : template for rows per warp
* metal : parallelize across KV size
* metal : parallel reduce across heads
* metal : efficient flash_attn_f16 implementation
* metal : avoid redundant loads of the attention
* metal : scale and mask in matrix form
* metal : fix comment
* llama : avoid ggml_cast, use F32 query
* metal : add parallel reduce version (disabled)
* metal : move output into local memory + optimize
- the result from each simdgroup now stays in the registers
- significantly reduced SRAM usage
- more efficient skipping of -INF blocks
- avoid simdgroup barrier in hot loop
- add comments
* metal : add tests, fix scaling, support C > 32
* metal : improve precision
* ggml : fix f16 mad
* metal : minor
* metal : support Q > 8
* tests : add ATTN tests
* metal : disable buffer allocation logs
* tests : more
* metal : faster inner loop for C == 32
* metal : fix array initialization
* tests : ifdef
* ggml : switch to padded F16 mask for ggml_soft_max, ggml_flash_attn_ext
* ggml : fix ggml_soft_max mask requirement
* cuda : fix soft_max to use correct mask size
* cuda : add flash_attn kernel (wip)
* metal : optimize softmax for C > 32
* metal : optimize softmax
* tests : minor fix
* cuda : avoid zeroing fragments
* tests : update dims
* cuda : fix __hisinf() result check
* cuda : avoid warp_reduce for smax
* cuda : use int instead of int64_t
Noticeably improves performance (thanks to Johannes)
* cuda : make loops use the same loop values
Thanks Johannes again for the tip
* cuda : unroll some of the loops
* cuda : avoid __hisinf branches
* cuda : use half2 in softmax
* cuda : switch to 1 warp for bs > 16
* cuda : speed-up reduce part of the kernel
* cuda : unroll Q*K^T loop
* cuda : fix -INF block check
* cuda : simplify softmax
* cuda : fix matrix names
* cuda : minor
* llama : adapt to F16 KQ_pos
* llama : adapt new models to F16 KQ_mask
* ggml : fix F16 store (ARM NEON)
* llama : fix type of KQ_mask and KQ_pos
* ggml : fix CPU soft_max
* tests : add hs=256
* cuda : fix build
* metal : improve perf via smaller int registers
* cuda : adapt soft_max to F16 mask and pos
* CUDA: faster FlashAttention, kernel for bs == 1
* 16 cols for Phi-2
* no vec for hs, no hs==256 ncols==32 for Volta
* adjust kernel selection logic
* 4 warps, 256 stride for all D
* no ncols == 64
* Multiple parallel blocks for batch size 1
* fix compile warnings
* fix excessive KQ_b loads
* fix cmake build
* fix KV cache padding, NaN from INFINITY (llama/6438)
* llama : flash_attn cparam + fix defrag
* server: support flash_attn param
* server: bench: enable flash_attn param
* CUDA: refactor host code, dyn. par. blocks
* fix flash_attn_vec_f16 race condition
* flush softmax exp below threshold to 0
* store temp KQ in registers
* Calculate KQ as FP32 if KQV has GGML_PREC_F32
* Add __hgt2_mask implementation for CUDA 11
* fix KQ FP32 precision fpr parallel_blocks > 1
* llama-bench : add -fa,--flash-attn arg
* metal : add BS=1 kernel for flash attention (llama/6508)
* metal : add BS=1 kernel for flash attention (wip)
* metal : support more than 1 warps
* metal : opts
* metal : opt
* metal : switch to parallel reduce
* metal : reduce registers
* metal : simplify
* metal : initial FA vec kernel
* metal : use F32 attention accumulators
* batched-bench : add fattn arg
* llama : simplify llama_build_kv_store
ggml-ci
* llama : adapt build_olmo to changes
* ggml : fix arm fp16 store on windows
* metal : clean-up
* metal : clean-up kernel code
* metal : minor
* tests : remove benchmarks
ggml-ci
* ggml : fix avx512 const correctness
ggml-ci
* ggml : fix soft_max with bias on CPU
ggml-ci
* common : print --flash-attn in help
* ggml : fix num dimensions in ggml_flash_attn_ext
* llama : force disable flash attention for incompatible models
* ggml : ggml_soft_max support F16/F32 mask/pos
ggml-ci
* cuda : uint -> uint32_t
* cuda : "constexpr dim3" -> "const dim3"
ggml-ci
* cuda : try to fix __hgt2_mask
ggml-ci
* ggml : add TODO's for F16/F32 mask/pos support in other backends
* llama : replace bool need_kq_pos with use_alibi
* llama : prep ALiBi support for BERT models
ggml-ci
* llama : fix n_batch requirements
ggml-ci
* cont
* server : add help for --flash-attn arg
* llama : disable FA for AMD
* tests : remove TMP_ATTN_BENCH
ggml-ci
* llama : support save/load state with FA enabled
ggml-ci
* ci : add CUDA save-load-state tests
ggml-ci
* llama : llama_kv_cache_clear zeroes data + fix save-load seq
ggml-ci
* llama : fix copy-paste errors, add TODO
* llama : disallow incompatible states
* llama : update llama_state_get_size after v_trans field
* metal : remove tmp log
* llama : add static reminder for llama_state_get_size
* metal : fix max nsg
ggml-ci
* ci : fix arg order
ggml-ci
---------
Co-authored-by: Johannes Gäßler <johannesg@5d6.de>
Co-authored-by: Pierrick HYMBERT <pierrick.hymbert@gmail.com>
2024-04-30 09:16:08 +00:00
|
|
|
#define GGML_KQ_MASK_PAD 32
|
|
|
|
|
|
|
|
|
|
// q: [n_embd, n_batch, n_head, 1]
|
|
|
|
|
// k: [n_embd, n_kv, n_head_kv, 1]
|
|
|
|
|
// v: [n_embd, n_kv, n_head_kv, 1] !! not transposed !!
|
|
|
|
|
// mask: [n_kv, n_batch_pad, 1, 1] !! n_batch_pad = GGML_PAD(n_batch, GGML_KQ_MASK_PAD) !!
|
|
|
|
|
// res: [n_embd, n_head, n_batch, 1] !! permuted !!
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_flash_attn_ext(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * q,
|
|
|
|
|
struct ggml_tensor * k,
|
|
|
|
|
struct ggml_tensor * v,
|
|
|
|
|
struct ggml_tensor * mask,
|
2024-05-11 07:32:41 +00:00
|
|
|
float scale,
|
2024-08-24 19:34:59 +00:00
|
|
|
float max_bias,
|
|
|
|
|
float logit_softcap);
|
ggml : add Flash Attention (llama/5021)
* ggml : add ggml_flash_attn_ext API
* ggml : fix GQA support in ggml_flash_attn_ext
* ggml : online attention (CPU)
* metal : initial implementation
* metal : f16 precision
* metal : reduce branches
* metal : specialize for head size
* wip : 8 rows per simd group
* wip : 4 rows per simd group
* wip : template for rows per warp
* metal : parallelize across KV size
* metal : parallel reduce across heads
* metal : efficient flash_attn_f16 implementation
* metal : avoid redundant loads of the attention
* metal : scale and mask in matrix form
* metal : fix comment
* llama : avoid ggml_cast, use F32 query
* metal : add parallel reduce version (disabled)
* metal : move output into local memory + optimize
- the result from each simdgroup now stays in the registers
- significantly reduced SRAM usage
- more efficient skipping of -INF blocks
- avoid simdgroup barrier in hot loop
- add comments
* metal : add tests, fix scaling, support C > 32
* metal : improve precision
* ggml : fix f16 mad
* metal : minor
* metal : support Q > 8
* tests : add ATTN tests
* metal : disable buffer allocation logs
* tests : more
* metal : faster inner loop for C == 32
* metal : fix array initialization
* tests : ifdef
* ggml : switch to padded F16 mask for ggml_soft_max, ggml_flash_attn_ext
* ggml : fix ggml_soft_max mask requirement
* cuda : fix soft_max to use correct mask size
* cuda : add flash_attn kernel (wip)
* metal : optimize softmax for C > 32
* metal : optimize softmax
* tests : minor fix
* cuda : avoid zeroing fragments
* tests : update dims
* cuda : fix __hisinf() result check
* cuda : avoid warp_reduce for smax
* cuda : use int instead of int64_t
Noticeably improves performance (thanks to Johannes)
* cuda : make loops use the same loop values
Thanks Johannes again for the tip
* cuda : unroll some of the loops
* cuda : avoid __hisinf branches
* cuda : use half2 in softmax
* cuda : switch to 1 warp for bs > 16
* cuda : speed-up reduce part of the kernel
* cuda : unroll Q*K^T loop
* cuda : fix -INF block check
* cuda : simplify softmax
* cuda : fix matrix names
* cuda : minor
* llama : adapt to F16 KQ_pos
* llama : adapt new models to F16 KQ_mask
* ggml : fix F16 store (ARM NEON)
* llama : fix type of KQ_mask and KQ_pos
* ggml : fix CPU soft_max
* tests : add hs=256
* cuda : fix build
* metal : improve perf via smaller int registers
* cuda : adapt soft_max to F16 mask and pos
* CUDA: faster FlashAttention, kernel for bs == 1
* 16 cols for Phi-2
* no vec for hs, no hs==256 ncols==32 for Volta
* adjust kernel selection logic
* 4 warps, 256 stride for all D
* no ncols == 64
* Multiple parallel blocks for batch size 1
* fix compile warnings
* fix excessive KQ_b loads
* fix cmake build
* fix KV cache padding, NaN from INFINITY (llama/6438)
* llama : flash_attn cparam + fix defrag
* server: support flash_attn param
* server: bench: enable flash_attn param
* CUDA: refactor host code, dyn. par. blocks
* fix flash_attn_vec_f16 race condition
* flush softmax exp below threshold to 0
* store temp KQ in registers
* Calculate KQ as FP32 if KQV has GGML_PREC_F32
* Add __hgt2_mask implementation for CUDA 11
* fix KQ FP32 precision fpr parallel_blocks > 1
* llama-bench : add -fa,--flash-attn arg
* metal : add BS=1 kernel for flash attention (llama/6508)
* metal : add BS=1 kernel for flash attention (wip)
* metal : support more than 1 warps
* metal : opts
* metal : opt
* metal : switch to parallel reduce
* metal : reduce registers
* metal : simplify
* metal : initial FA vec kernel
* metal : use F32 attention accumulators
* batched-bench : add fattn arg
* llama : simplify llama_build_kv_store
ggml-ci
* llama : adapt build_olmo to changes
* ggml : fix arm fp16 store on windows
* metal : clean-up
* metal : clean-up kernel code
* metal : minor
* tests : remove benchmarks
ggml-ci
* ggml : fix avx512 const correctness
ggml-ci
* ggml : fix soft_max with bias on CPU
ggml-ci
* common : print --flash-attn in help
* ggml : fix num dimensions in ggml_flash_attn_ext
* llama : force disable flash attention for incompatible models
* ggml : ggml_soft_max support F16/F32 mask/pos
ggml-ci
* cuda : uint -> uint32_t
* cuda : "constexpr dim3" -> "const dim3"
ggml-ci
* cuda : try to fix __hgt2_mask
ggml-ci
* ggml : add TODO's for F16/F32 mask/pos support in other backends
* llama : replace bool need_kq_pos with use_alibi
* llama : prep ALiBi support for BERT models
ggml-ci
* llama : fix n_batch requirements
ggml-ci
* cont
* server : add help for --flash-attn arg
* llama : disable FA for AMD
* tests : remove TMP_ATTN_BENCH
ggml-ci
* llama : support save/load state with FA enabled
ggml-ci
* ci : add CUDA save-load-state tests
ggml-ci
* llama : llama_kv_cache_clear zeroes data + fix save-load seq
ggml-ci
* llama : fix copy-paste errors, add TODO
* llama : disallow incompatible states
* llama : update llama_state_get_size after v_trans field
* metal : remove tmp log
* llama : add static reminder for llama_state_get_size
* metal : fix max nsg
ggml-ci
* ci : fix arg order
ggml-ci
---------
Co-authored-by: Johannes Gäßler <johannesg@5d6.de>
Co-authored-by: Pierrick HYMBERT <pierrick.hymbert@gmail.com>
2024-04-30 09:16:08 +00:00
|
|
|
|
|
|
|
|
GGML_API void ggml_flash_attn_ext_set_prec(
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
enum ggml_prec prec);
|
|
|
|
|
|
2024-11-08 11:47:22 +00:00
|
|
|
GGML_API enum ggml_prec ggml_flash_attn_ext_get_prec(
|
|
|
|
|
const struct ggml_tensor * a);
|
|
|
|
|
|
2024-05-23 07:00:44 +00:00
|
|
|
// TODO: needs to be adapted to ggml_flash_attn_ext
|
2023-06-25 11:22:21 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_flash_attn_back(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * q,
|
|
|
|
|
struct ggml_tensor * k,
|
|
|
|
|
struct ggml_tensor * v,
|
|
|
|
|
struct ggml_tensor * d,
|
|
|
|
|
bool masked);
|
|
|
|
|
|
llama : support Mamba Selective State Space Models (llama/5328)
* mamba : begin working on support for Mamba SSM
* mamba : begin figuring out how to (ab)use the kv cache for Mamba
* mamba : recurrent inference almost works, but incoherent
* mamba : recurrent inference WORKS!!!
* convert : optionally use d_conv and d_state from config.json for Mamba
* mamba : refactor recurrent conv, resulting in 20% perf increase
It's still slower than I'd like, but I did not really optimize `ggml_exp` yet.
I also refactored `ggml_exp` to work with tensors with more than 2 dimensions.
* ggml : parallelize ggml_exp
This results in 8% faster token generation for Mamba-130M.
* mamba : simplify the conv step with a self-overlapping view
Turns out the conv_state can be made smaller by one column.
Note that this breaks existing GGUFs of Mamba,
because the key_value_length field is tied to the conv_state size.
Convolution with a self-overlapping view is cool!
And it's much simpler than what I initially thought would be necessary
to make the convolution step work with more than 1 token at a time.
Next step is to make the SSM step work on batches of tokens too,
and thus I need to figure out a way to make a parallel selective scan
which will keep the ssm_state small and won't make it bigger
by a factor of (n_layer * batch_size).
* llama : fix Mamba KV self size wrongly displaying as f16 instead of f32
Relatedly, I also tried to see if other types than f32 worked for the states,
but they don't, because of the operators used.
It's probably better anyway to keep lots of precision there,
since the states are small anyway.
* mamba : fix self-overlapping view depth stride
* mamba : handle batches of more than 1 token
This means running Mamba no longer crashes when using the default settings!
And probably also slightly faster prompt processing.
Both batched and non-batched processing yield the same output.
Previously, the state was not cleared when starting a sequence.
Next step is to make the KV cache API work as expected for Mamba models.
* ggml: add ggml_ssm_scan to help with parallel selective scan
If the selective scan was implemented without a custom operator,
there would be waaay too many nodes in the graph. For example,
for Mamba-130M, with a batch size of 512 (the default),
a naive selective scan could add at least 24*512=12288 nodes,
which is more than LLAMA_MAX_NODES (8192),
and that's only for the smallest Mamba model.
So it's much cleaner with a custom operator.
Not sure about the name, though.
* ggml : in ggml_ssm_scan, merge multiple rows in the same vec operation
This will help with performance on CPU if ggml_vec_mul_f32
and ggml_vec_add_f32 are ever optimized with SIMD.
* mamba : very basic quantization support
Mostly works, but there is currently no difference
between the variants of a k-quant (e.g. Q4_K_S and Q4_K_M are the same).
Most of the SSM-specific weights can be kept in f32 without affecting
the size that much, since they are relatively small.
(the linear projection weights are responsible for most of Mamba's size)
Too much quantization seems to make the state degrade quite fast, and
the model begins to output gibberish.
It seems to affect bigger models to a lesser extent than small models,
but I'm not sure by how much.
Experimentation will be needed to figure out which weights are more important
for the _M (and _L?) variants of k-quants for Mamba.
* convert : fix wrong name for layer norm weight of offical Mamba models
I was using Q-bert/Mamba-* models before, which have a slighlty different
naming scheme for the weights.
(they start with "model.layers" instead of "backbone.layers")
* mamba : fuse more steps of the SSM scan in the ggml_ssm_scan operator
This increases performance on CPU by around 30% for prompt processing,
and by around 20% for text generation.
However, it also makes the ggml_exp and ggml_soft_plus operators unused.
Whether or not they should be kept will be decided later.
* convert : for Mamba, also consider the "MambaLMHeadModel" arch name
It's the name of the class of the official implementation,
though they don't use it (yet) in the "architectures" field of config.json
* mamba : fix vocab size problems with official models
The perplexity was waaaay to high for models with a non-round vocab size.
Not sure why, but it needed to be fixed in the metadata.
Note that this breaks existing GGUF-converted Mamba models,
but **only if** the vocab size was not already rounded.
* ggml : remove ggml_exp and ggml_soft_plus
They did not exist anyway outside of this branch,
and since ggml_ssm_scan fused operations together, they are unused.
It's always possible to bring them back if needed.
* mamba : remove some useless comments
No code change.
* convert : fix flake8 linter errors
* mamba : apply suggestions from code review
* mamba : remove unecessary branch for row-wise ssm_state and C multiplication
It was previously done to avoid permuting when only one token is processed
at a time (like when generating text), but permuting is cheap,
and dynamically changing the compute graph is not future-proof.
* ggml : in ggml_ssm_scan, use more appropriate asserts
* ggml : rename the destination pointer in ggml_compute_forward_ssm_scan_f32
* mamba : multiple sequences, but one at a time
This is a step towards making this Mamba implementation usable
with the server example (the way the system prompt is kept when clearing
the client slots will need to be changed before this can work, though).
The KV cache size for this kind of model is tied to the maximum number
of sequences kept at any single time.
For now, this number is obtained from n_parallel (plus one,
to have an extra sequence to dedicate to the system prompt),
but there might be a better way to do this which won't also
make the main example use 2 cells even if only 1 is really used.
(for this specific case, --parallel 0 helps)
Simultaneous sequence processing will probably require changes to
ggml_ssm_scan, and possibly a new operator for the conv step.
* mamba : support llama_kv_cache_seq_cp
This (mis)uses the logic around K shifts, because tokens in a state
can't be shifted anyway, and because inp_K_shift has the right shape and type.
Using ggml_get_rows is a nice way to do copies, but copy chains can't work.
Fortunately, copy chains don't really seem to be used in the examples.
Each KV cell is dedicated to the sequence ID corresponding to its own index.
* mamba : use a state mask
It's cleaner than the previous heuristic of
checking for the pos of the first token in the batch.
inp_KQ_mask could not be re-used for this, because it has the wrong shape
and because it seems more suited to the next step of
simultaneous sequence processing (helping with the problem of
remembering which token belongs to which sequence(s)/state(s)).
* llama : replace the usage of n_ctx with kv_self.size in many places
* mamba : use n_tokens directly instead of n_tok
* mamba : in comments, properly refer to KV cells instead of slots
* mamba : reduce memory usage of ggml_ssm_scan
From 290.37 MiB to 140.68 MiB of CPU compute buffer size
with Mamba 3B with a batch size of 512.
The result tensor of ggml_ssm_scan was previously a big part
of the CPU compute buffer size. To make it smaller,
it does not contain the intermediate ssm states anymore.
Both y and the last ssm state are combined in the result tensor,
because it seems only a single tensor can be returned by an operator
with the way the graph is built.
* mamba : simultaneous sequence processing
A batch can now contain tokens from multiple sequences.
This is necessary for at least the parallel example, the server example,
and the HellaSwag test in the perplexity example.
However, for this to be useful, uses of llama_kv_cache_seq_rm/cp
will need to be changed to work on whole sequences.
* ggml : add ggml_ssm_conv as a new operator for the conv step of Mamba
This operator makes it possible to use and update the correct states
for each token of the batch in the same way as ggml_ssm_scan.
Other solutions which use existing operators would need loops which would
add too many nodes to the graph (at least the ones I thought of).
Using this operator further reduces the size of the CPU compute buffer
from 140.68 MiB to 103.20 MiB with Mamba 3B with a batch size of 512.
And (at least on CPU), it's a bit faster than before.
Note that "ggml_ssm_conv" is probably not the most appropriate name,
and it could be changed if a better one is found.
* llama : add inp_s_seq as a new input tensor
The most convenient implementation to select the correct state (for Mamba)
for each token is to directly get the correct index from a tensor.
This is why inp_s_seq is storing int32_t and not floats.
The other, less convenient way to select the correct state would be
to have inp_KQ_mask contain 1.0f for each state used by a token
and 0.0f otherwise. This complicates quickly fetching the first used
state of a token, and is also less efficient because a whole row
of the mask would always need to be read for each token.
Using indexes makes it easy to stop searching when there are
no more sequences for a token, and the first sequence assigned
is always very quickly available (it's the first element of each row).
* mamba : support llama_kv_cache_seq_cp copy chains
* mamba : support shifting and dividing the kv cache pos
* mamba : make the server and parallel examples work with whole sequences
A seq_id is dedicated to the system prompt in both cases.
* llama : make llama_kv_cache_seq_rm return whether it succeeded or not
* mamba : dedicate an input tensor for state copy indices
This is cleaner and makes it easier to adapt when/if token positions
(and by extension, inp_K_shift) are no longer integers.
* mamba : adapt perplexity, batched, and batched-bench examples
* perplexity : limit the max number of sequences
This adapts to what the loaded model can provide.
* llama : add llama_n_max_seq to get the upper limit for seq_ids
Used by the perplexity example.
* batched : pass n_parallel to the model's context params
This should have been there already, but it wasn't.
* batched-bench : reserve sequences to support Mamba
* batched-bench : fix tokens being put in wrong sequences
Generation quality isn't what's measured in there anyway,
but at least using the correct sequences avoids using non-consecutive
token positions.
* mamba : stop abusing attention metadata
This breaks existing converted-to-GGUF Mamba models,
but will allow supporting mixed architectures like MambaFormer
without needing to break Mamba models.
This will also allow changing the size of Mamba's states
without having to reconvert models in the future.
(e.g. using something else than d_conv - 1 columns for the conv_states
will not require breaking existing converted Mamba models again)
* gguf-py : add new KV metadata key-value pairs for Mamba
* llama : add new metadata key-value pairs for Mamba
* llama : guard against divisions by zero when n_head is 0
* mamba : rename "unlimited" KV cache property to "recurrent"
* mamba : more correctly update the "used" field of the KV cache
* ggml : in ggml_ssm_scan, use a threshold for soft_plus
This is how the official Mamba implementation does it,
and it's also what torch.nn.Softplus does.
* convert : for Mamba, fallback to internal NeoX tokenizer
The resulting models are exactly the same
as if the tokenizer.json and tokenizer_config.json of GPT-NeoX were there.
* mamba : support state saving and restoring
* ggml : implicitly pass src tensors through dst for Mamba-related ops
* mamba : clarify some comments
* server : fix cache_tokens not getting correctly resized
Otherwise, when the "we have to evaluate at least 1 token" special case
was triggered, an extra token was kept in cache_tokens even if it was
removed from the KV cache.
For Mamba, this caused useless prompt reprocessing when the previous
request triggered the above case.
* convert-hf : support new metadata keys for Mamba
For the models available at
https://huggingface.co/collections/state-spaces/transformers-compatible-mamba-65e7b40ab87e5297e45ae406
* mamba : rename metadata to be more similar to transformers library
This breaks existing converted-to-GGUF models,
but the metadata names are more "standard".
* mamba : support mamba-*-hf models
These models share their token_embd.weight with their output.weight
* mamba : add missing spaces
This is purely a formatting change.
* convert-hf : omit output.weight when identical with token_embd.weight
Only for Mamba for now, but it might be relevant for other models eventually.
Most Mamba models actually share these two tensors, albeit implicitly.
* readme : add Mamba to supported models, and add recent API changes
* mamba : move state_seq and state_mask views outside layer loop
A few tensors were also missing `struct` in front of `ggml_tensor`.
2024-03-08 22:31:00 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_ssm_conv(
|
|
|
|
|
struct ggml_context * ctx,
|
2024-08-21 21:58:11 +00:00
|
|
|
struct ggml_tensor * sx,
|
|
|
|
|
struct ggml_tensor * c);
|
llama : support Mamba Selective State Space Models (llama/5328)
* mamba : begin working on support for Mamba SSM
* mamba : begin figuring out how to (ab)use the kv cache for Mamba
* mamba : recurrent inference almost works, but incoherent
* mamba : recurrent inference WORKS!!!
* convert : optionally use d_conv and d_state from config.json for Mamba
* mamba : refactor recurrent conv, resulting in 20% perf increase
It's still slower than I'd like, but I did not really optimize `ggml_exp` yet.
I also refactored `ggml_exp` to work with tensors with more than 2 dimensions.
* ggml : parallelize ggml_exp
This results in 8% faster token generation for Mamba-130M.
* mamba : simplify the conv step with a self-overlapping view
Turns out the conv_state can be made smaller by one column.
Note that this breaks existing GGUFs of Mamba,
because the key_value_length field is tied to the conv_state size.
Convolution with a self-overlapping view is cool!
And it's much simpler than what I initially thought would be necessary
to make the convolution step work with more than 1 token at a time.
Next step is to make the SSM step work on batches of tokens too,
and thus I need to figure out a way to make a parallel selective scan
which will keep the ssm_state small and won't make it bigger
by a factor of (n_layer * batch_size).
* llama : fix Mamba KV self size wrongly displaying as f16 instead of f32
Relatedly, I also tried to see if other types than f32 worked for the states,
but they don't, because of the operators used.
It's probably better anyway to keep lots of precision there,
since the states are small anyway.
* mamba : fix self-overlapping view depth stride
* mamba : handle batches of more than 1 token
This means running Mamba no longer crashes when using the default settings!
And probably also slightly faster prompt processing.
Both batched and non-batched processing yield the same output.
Previously, the state was not cleared when starting a sequence.
Next step is to make the KV cache API work as expected for Mamba models.
* ggml: add ggml_ssm_scan to help with parallel selective scan
If the selective scan was implemented without a custom operator,
there would be waaay too many nodes in the graph. For example,
for Mamba-130M, with a batch size of 512 (the default),
a naive selective scan could add at least 24*512=12288 nodes,
which is more than LLAMA_MAX_NODES (8192),
and that's only for the smallest Mamba model.
So it's much cleaner with a custom operator.
Not sure about the name, though.
* ggml : in ggml_ssm_scan, merge multiple rows in the same vec operation
This will help with performance on CPU if ggml_vec_mul_f32
and ggml_vec_add_f32 are ever optimized with SIMD.
* mamba : very basic quantization support
Mostly works, but there is currently no difference
between the variants of a k-quant (e.g. Q4_K_S and Q4_K_M are the same).
Most of the SSM-specific weights can be kept in f32 without affecting
the size that much, since they are relatively small.
(the linear projection weights are responsible for most of Mamba's size)
Too much quantization seems to make the state degrade quite fast, and
the model begins to output gibberish.
It seems to affect bigger models to a lesser extent than small models,
but I'm not sure by how much.
Experimentation will be needed to figure out which weights are more important
for the _M (and _L?) variants of k-quants for Mamba.
* convert : fix wrong name for layer norm weight of offical Mamba models
I was using Q-bert/Mamba-* models before, which have a slighlty different
naming scheme for the weights.
(they start with "model.layers" instead of "backbone.layers")
* mamba : fuse more steps of the SSM scan in the ggml_ssm_scan operator
This increases performance on CPU by around 30% for prompt processing,
and by around 20% for text generation.
However, it also makes the ggml_exp and ggml_soft_plus operators unused.
Whether or not they should be kept will be decided later.
* convert : for Mamba, also consider the "MambaLMHeadModel" arch name
It's the name of the class of the official implementation,
though they don't use it (yet) in the "architectures" field of config.json
* mamba : fix vocab size problems with official models
The perplexity was waaaay to high for models with a non-round vocab size.
Not sure why, but it needed to be fixed in the metadata.
Note that this breaks existing GGUF-converted Mamba models,
but **only if** the vocab size was not already rounded.
* ggml : remove ggml_exp and ggml_soft_plus
They did not exist anyway outside of this branch,
and since ggml_ssm_scan fused operations together, they are unused.
It's always possible to bring them back if needed.
* mamba : remove some useless comments
No code change.
* convert : fix flake8 linter errors
* mamba : apply suggestions from code review
* mamba : remove unecessary branch for row-wise ssm_state and C multiplication
It was previously done to avoid permuting when only one token is processed
at a time (like when generating text), but permuting is cheap,
and dynamically changing the compute graph is not future-proof.
* ggml : in ggml_ssm_scan, use more appropriate asserts
* ggml : rename the destination pointer in ggml_compute_forward_ssm_scan_f32
* mamba : multiple sequences, but one at a time
This is a step towards making this Mamba implementation usable
with the server example (the way the system prompt is kept when clearing
the client slots will need to be changed before this can work, though).
The KV cache size for this kind of model is tied to the maximum number
of sequences kept at any single time.
For now, this number is obtained from n_parallel (plus one,
to have an extra sequence to dedicate to the system prompt),
but there might be a better way to do this which won't also
make the main example use 2 cells even if only 1 is really used.
(for this specific case, --parallel 0 helps)
Simultaneous sequence processing will probably require changes to
ggml_ssm_scan, and possibly a new operator for the conv step.
* mamba : support llama_kv_cache_seq_cp
This (mis)uses the logic around K shifts, because tokens in a state
can't be shifted anyway, and because inp_K_shift has the right shape and type.
Using ggml_get_rows is a nice way to do copies, but copy chains can't work.
Fortunately, copy chains don't really seem to be used in the examples.
Each KV cell is dedicated to the sequence ID corresponding to its own index.
* mamba : use a state mask
It's cleaner than the previous heuristic of
checking for the pos of the first token in the batch.
inp_KQ_mask could not be re-used for this, because it has the wrong shape
and because it seems more suited to the next step of
simultaneous sequence processing (helping with the problem of
remembering which token belongs to which sequence(s)/state(s)).
* llama : replace the usage of n_ctx with kv_self.size in many places
* mamba : use n_tokens directly instead of n_tok
* mamba : in comments, properly refer to KV cells instead of slots
* mamba : reduce memory usage of ggml_ssm_scan
From 290.37 MiB to 140.68 MiB of CPU compute buffer size
with Mamba 3B with a batch size of 512.
The result tensor of ggml_ssm_scan was previously a big part
of the CPU compute buffer size. To make it smaller,
it does not contain the intermediate ssm states anymore.
Both y and the last ssm state are combined in the result tensor,
because it seems only a single tensor can be returned by an operator
with the way the graph is built.
* mamba : simultaneous sequence processing
A batch can now contain tokens from multiple sequences.
This is necessary for at least the parallel example, the server example,
and the HellaSwag test in the perplexity example.
However, for this to be useful, uses of llama_kv_cache_seq_rm/cp
will need to be changed to work on whole sequences.
* ggml : add ggml_ssm_conv as a new operator for the conv step of Mamba
This operator makes it possible to use and update the correct states
for each token of the batch in the same way as ggml_ssm_scan.
Other solutions which use existing operators would need loops which would
add too many nodes to the graph (at least the ones I thought of).
Using this operator further reduces the size of the CPU compute buffer
from 140.68 MiB to 103.20 MiB with Mamba 3B with a batch size of 512.
And (at least on CPU), it's a bit faster than before.
Note that "ggml_ssm_conv" is probably not the most appropriate name,
and it could be changed if a better one is found.
* llama : add inp_s_seq as a new input tensor
The most convenient implementation to select the correct state (for Mamba)
for each token is to directly get the correct index from a tensor.
This is why inp_s_seq is storing int32_t and not floats.
The other, less convenient way to select the correct state would be
to have inp_KQ_mask contain 1.0f for each state used by a token
and 0.0f otherwise. This complicates quickly fetching the first used
state of a token, and is also less efficient because a whole row
of the mask would always need to be read for each token.
Using indexes makes it easy to stop searching when there are
no more sequences for a token, and the first sequence assigned
is always very quickly available (it's the first element of each row).
* mamba : support llama_kv_cache_seq_cp copy chains
* mamba : support shifting and dividing the kv cache pos
* mamba : make the server and parallel examples work with whole sequences
A seq_id is dedicated to the system prompt in both cases.
* llama : make llama_kv_cache_seq_rm return whether it succeeded or not
* mamba : dedicate an input tensor for state copy indices
This is cleaner and makes it easier to adapt when/if token positions
(and by extension, inp_K_shift) are no longer integers.
* mamba : adapt perplexity, batched, and batched-bench examples
* perplexity : limit the max number of sequences
This adapts to what the loaded model can provide.
* llama : add llama_n_max_seq to get the upper limit for seq_ids
Used by the perplexity example.
* batched : pass n_parallel to the model's context params
This should have been there already, but it wasn't.
* batched-bench : reserve sequences to support Mamba
* batched-bench : fix tokens being put in wrong sequences
Generation quality isn't what's measured in there anyway,
but at least using the correct sequences avoids using non-consecutive
token positions.
* mamba : stop abusing attention metadata
This breaks existing converted-to-GGUF Mamba models,
but will allow supporting mixed architectures like MambaFormer
without needing to break Mamba models.
This will also allow changing the size of Mamba's states
without having to reconvert models in the future.
(e.g. using something else than d_conv - 1 columns for the conv_states
will not require breaking existing converted Mamba models again)
* gguf-py : add new KV metadata key-value pairs for Mamba
* llama : add new metadata key-value pairs for Mamba
* llama : guard against divisions by zero when n_head is 0
* mamba : rename "unlimited" KV cache property to "recurrent"
* mamba : more correctly update the "used" field of the KV cache
* ggml : in ggml_ssm_scan, use a threshold for soft_plus
This is how the official Mamba implementation does it,
and it's also what torch.nn.Softplus does.
* convert : for Mamba, fallback to internal NeoX tokenizer
The resulting models are exactly the same
as if the tokenizer.json and tokenizer_config.json of GPT-NeoX were there.
* mamba : support state saving and restoring
* ggml : implicitly pass src tensors through dst for Mamba-related ops
* mamba : clarify some comments
* server : fix cache_tokens not getting correctly resized
Otherwise, when the "we have to evaluate at least 1 token" special case
was triggered, an extra token was kept in cache_tokens even if it was
removed from the KV cache.
For Mamba, this caused useless prompt reprocessing when the previous
request triggered the above case.
* convert-hf : support new metadata keys for Mamba
For the models available at
https://huggingface.co/collections/state-spaces/transformers-compatible-mamba-65e7b40ab87e5297e45ae406
* mamba : rename metadata to be more similar to transformers library
This breaks existing converted-to-GGUF models,
but the metadata names are more "standard".
* mamba : support mamba-*-hf models
These models share their token_embd.weight with their output.weight
* mamba : add missing spaces
This is purely a formatting change.
* convert-hf : omit output.weight when identical with token_embd.weight
Only for Mamba for now, but it might be relevant for other models eventually.
Most Mamba models actually share these two tensors, albeit implicitly.
* readme : add Mamba to supported models, and add recent API changes
* mamba : move state_seq and state_mask views outside layer loop
A few tensors were also missing `struct` in front of `ggml_tensor`.
2024-03-08 22:31:00 +00:00
|
|
|
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_ssm_scan(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * s,
|
|
|
|
|
struct ggml_tensor * x,
|
|
|
|
|
struct ggml_tensor * dt,
|
|
|
|
|
struct ggml_tensor * A,
|
|
|
|
|
struct ggml_tensor * B,
|
2024-08-21 21:58:11 +00:00
|
|
|
struct ggml_tensor * C);
|
llama : support Mamba Selective State Space Models (llama/5328)
* mamba : begin working on support for Mamba SSM
* mamba : begin figuring out how to (ab)use the kv cache for Mamba
* mamba : recurrent inference almost works, but incoherent
* mamba : recurrent inference WORKS!!!
* convert : optionally use d_conv and d_state from config.json for Mamba
* mamba : refactor recurrent conv, resulting in 20% perf increase
It's still slower than I'd like, but I did not really optimize `ggml_exp` yet.
I also refactored `ggml_exp` to work with tensors with more than 2 dimensions.
* ggml : parallelize ggml_exp
This results in 8% faster token generation for Mamba-130M.
* mamba : simplify the conv step with a self-overlapping view
Turns out the conv_state can be made smaller by one column.
Note that this breaks existing GGUFs of Mamba,
because the key_value_length field is tied to the conv_state size.
Convolution with a self-overlapping view is cool!
And it's much simpler than what I initially thought would be necessary
to make the convolution step work with more than 1 token at a time.
Next step is to make the SSM step work on batches of tokens too,
and thus I need to figure out a way to make a parallel selective scan
which will keep the ssm_state small and won't make it bigger
by a factor of (n_layer * batch_size).
* llama : fix Mamba KV self size wrongly displaying as f16 instead of f32
Relatedly, I also tried to see if other types than f32 worked for the states,
but they don't, because of the operators used.
It's probably better anyway to keep lots of precision there,
since the states are small anyway.
* mamba : fix self-overlapping view depth stride
* mamba : handle batches of more than 1 token
This means running Mamba no longer crashes when using the default settings!
And probably also slightly faster prompt processing.
Both batched and non-batched processing yield the same output.
Previously, the state was not cleared when starting a sequence.
Next step is to make the KV cache API work as expected for Mamba models.
* ggml: add ggml_ssm_scan to help with parallel selective scan
If the selective scan was implemented without a custom operator,
there would be waaay too many nodes in the graph. For example,
for Mamba-130M, with a batch size of 512 (the default),
a naive selective scan could add at least 24*512=12288 nodes,
which is more than LLAMA_MAX_NODES (8192),
and that's only for the smallest Mamba model.
So it's much cleaner with a custom operator.
Not sure about the name, though.
* ggml : in ggml_ssm_scan, merge multiple rows in the same vec operation
This will help with performance on CPU if ggml_vec_mul_f32
and ggml_vec_add_f32 are ever optimized with SIMD.
* mamba : very basic quantization support
Mostly works, but there is currently no difference
between the variants of a k-quant (e.g. Q4_K_S and Q4_K_M are the same).
Most of the SSM-specific weights can be kept in f32 without affecting
the size that much, since they are relatively small.
(the linear projection weights are responsible for most of Mamba's size)
Too much quantization seems to make the state degrade quite fast, and
the model begins to output gibberish.
It seems to affect bigger models to a lesser extent than small models,
but I'm not sure by how much.
Experimentation will be needed to figure out which weights are more important
for the _M (and _L?) variants of k-quants for Mamba.
* convert : fix wrong name for layer norm weight of offical Mamba models
I was using Q-bert/Mamba-* models before, which have a slighlty different
naming scheme for the weights.
(they start with "model.layers" instead of "backbone.layers")
* mamba : fuse more steps of the SSM scan in the ggml_ssm_scan operator
This increases performance on CPU by around 30% for prompt processing,
and by around 20% for text generation.
However, it also makes the ggml_exp and ggml_soft_plus operators unused.
Whether or not they should be kept will be decided later.
* convert : for Mamba, also consider the "MambaLMHeadModel" arch name
It's the name of the class of the official implementation,
though they don't use it (yet) in the "architectures" field of config.json
* mamba : fix vocab size problems with official models
The perplexity was waaaay to high for models with a non-round vocab size.
Not sure why, but it needed to be fixed in the metadata.
Note that this breaks existing GGUF-converted Mamba models,
but **only if** the vocab size was not already rounded.
* ggml : remove ggml_exp and ggml_soft_plus
They did not exist anyway outside of this branch,
and since ggml_ssm_scan fused operations together, they are unused.
It's always possible to bring them back if needed.
* mamba : remove some useless comments
No code change.
* convert : fix flake8 linter errors
* mamba : apply suggestions from code review
* mamba : remove unecessary branch for row-wise ssm_state and C multiplication
It was previously done to avoid permuting when only one token is processed
at a time (like when generating text), but permuting is cheap,
and dynamically changing the compute graph is not future-proof.
* ggml : in ggml_ssm_scan, use more appropriate asserts
* ggml : rename the destination pointer in ggml_compute_forward_ssm_scan_f32
* mamba : multiple sequences, but one at a time
This is a step towards making this Mamba implementation usable
with the server example (the way the system prompt is kept when clearing
the client slots will need to be changed before this can work, though).
The KV cache size for this kind of model is tied to the maximum number
of sequences kept at any single time.
For now, this number is obtained from n_parallel (plus one,
to have an extra sequence to dedicate to the system prompt),
but there might be a better way to do this which won't also
make the main example use 2 cells even if only 1 is really used.
(for this specific case, --parallel 0 helps)
Simultaneous sequence processing will probably require changes to
ggml_ssm_scan, and possibly a new operator for the conv step.
* mamba : support llama_kv_cache_seq_cp
This (mis)uses the logic around K shifts, because tokens in a state
can't be shifted anyway, and because inp_K_shift has the right shape and type.
Using ggml_get_rows is a nice way to do copies, but copy chains can't work.
Fortunately, copy chains don't really seem to be used in the examples.
Each KV cell is dedicated to the sequence ID corresponding to its own index.
* mamba : use a state mask
It's cleaner than the previous heuristic of
checking for the pos of the first token in the batch.
inp_KQ_mask could not be re-used for this, because it has the wrong shape
and because it seems more suited to the next step of
simultaneous sequence processing (helping with the problem of
remembering which token belongs to which sequence(s)/state(s)).
* llama : replace the usage of n_ctx with kv_self.size in many places
* mamba : use n_tokens directly instead of n_tok
* mamba : in comments, properly refer to KV cells instead of slots
* mamba : reduce memory usage of ggml_ssm_scan
From 290.37 MiB to 140.68 MiB of CPU compute buffer size
with Mamba 3B with a batch size of 512.
The result tensor of ggml_ssm_scan was previously a big part
of the CPU compute buffer size. To make it smaller,
it does not contain the intermediate ssm states anymore.
Both y and the last ssm state are combined in the result tensor,
because it seems only a single tensor can be returned by an operator
with the way the graph is built.
* mamba : simultaneous sequence processing
A batch can now contain tokens from multiple sequences.
This is necessary for at least the parallel example, the server example,
and the HellaSwag test in the perplexity example.
However, for this to be useful, uses of llama_kv_cache_seq_rm/cp
will need to be changed to work on whole sequences.
* ggml : add ggml_ssm_conv as a new operator for the conv step of Mamba
This operator makes it possible to use and update the correct states
for each token of the batch in the same way as ggml_ssm_scan.
Other solutions which use existing operators would need loops which would
add too many nodes to the graph (at least the ones I thought of).
Using this operator further reduces the size of the CPU compute buffer
from 140.68 MiB to 103.20 MiB with Mamba 3B with a batch size of 512.
And (at least on CPU), it's a bit faster than before.
Note that "ggml_ssm_conv" is probably not the most appropriate name,
and it could be changed if a better one is found.
* llama : add inp_s_seq as a new input tensor
The most convenient implementation to select the correct state (for Mamba)
for each token is to directly get the correct index from a tensor.
This is why inp_s_seq is storing int32_t and not floats.
The other, less convenient way to select the correct state would be
to have inp_KQ_mask contain 1.0f for each state used by a token
and 0.0f otherwise. This complicates quickly fetching the first used
state of a token, and is also less efficient because a whole row
of the mask would always need to be read for each token.
Using indexes makes it easy to stop searching when there are
no more sequences for a token, and the first sequence assigned
is always very quickly available (it's the first element of each row).
* mamba : support llama_kv_cache_seq_cp copy chains
* mamba : support shifting and dividing the kv cache pos
* mamba : make the server and parallel examples work with whole sequences
A seq_id is dedicated to the system prompt in both cases.
* llama : make llama_kv_cache_seq_rm return whether it succeeded or not
* mamba : dedicate an input tensor for state copy indices
This is cleaner and makes it easier to adapt when/if token positions
(and by extension, inp_K_shift) are no longer integers.
* mamba : adapt perplexity, batched, and batched-bench examples
* perplexity : limit the max number of sequences
This adapts to what the loaded model can provide.
* llama : add llama_n_max_seq to get the upper limit for seq_ids
Used by the perplexity example.
* batched : pass n_parallel to the model's context params
This should have been there already, but it wasn't.
* batched-bench : reserve sequences to support Mamba
* batched-bench : fix tokens being put in wrong sequences
Generation quality isn't what's measured in there anyway,
but at least using the correct sequences avoids using non-consecutive
token positions.
* mamba : stop abusing attention metadata
This breaks existing converted-to-GGUF Mamba models,
but will allow supporting mixed architectures like MambaFormer
without needing to break Mamba models.
This will also allow changing the size of Mamba's states
without having to reconvert models in the future.
(e.g. using something else than d_conv - 1 columns for the conv_states
will not require breaking existing converted Mamba models again)
* gguf-py : add new KV metadata key-value pairs for Mamba
* llama : add new metadata key-value pairs for Mamba
* llama : guard against divisions by zero when n_head is 0
* mamba : rename "unlimited" KV cache property to "recurrent"
* mamba : more correctly update the "used" field of the KV cache
* ggml : in ggml_ssm_scan, use a threshold for soft_plus
This is how the official Mamba implementation does it,
and it's also what torch.nn.Softplus does.
* convert : for Mamba, fallback to internal NeoX tokenizer
The resulting models are exactly the same
as if the tokenizer.json and tokenizer_config.json of GPT-NeoX were there.
* mamba : support state saving and restoring
* ggml : implicitly pass src tensors through dst for Mamba-related ops
* mamba : clarify some comments
* server : fix cache_tokens not getting correctly resized
Otherwise, when the "we have to evaluate at least 1 token" special case
was triggered, an extra token was kept in cache_tokens even if it was
removed from the KV cache.
For Mamba, this caused useless prompt reprocessing when the previous
request triggered the above case.
* convert-hf : support new metadata keys for Mamba
For the models available at
https://huggingface.co/collections/state-spaces/transformers-compatible-mamba-65e7b40ab87e5297e45ae406
* mamba : rename metadata to be more similar to transformers library
This breaks existing converted-to-GGUF models,
but the metadata names are more "standard".
* mamba : support mamba-*-hf models
These models share their token_embd.weight with their output.weight
* mamba : add missing spaces
This is purely a formatting change.
* convert-hf : omit output.weight when identical with token_embd.weight
Only for Mamba for now, but it might be relevant for other models eventually.
Most Mamba models actually share these two tensors, albeit implicitly.
* readme : add Mamba to supported models, and add recent API changes
* mamba : move state_seq and state_mask views outside layer loop
A few tensors were also missing `struct` in front of `ggml_tensor`.
2024-03-08 22:31:00 +00:00
|
|
|
|
2023-06-25 11:22:21 +00:00
|
|
|
// partition into non-overlapping windows with padding if needed
|
|
|
|
|
// example:
|
|
|
|
|
// a: 768 64 64 1
|
|
|
|
|
// w: 14
|
|
|
|
|
// res: 768 14 14 25
|
|
|
|
|
// used in sam
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_win_part(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
int w);
|
|
|
|
|
|
|
|
|
|
// reverse of ggml_win_part
|
|
|
|
|
// used in sam
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_win_unpart(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
int w0,
|
|
|
|
|
int h0,
|
|
|
|
|
int w);
|
|
|
|
|
|
2023-09-05 10:54:40 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_unary(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
enum ggml_unary_op op);
|
|
|
|
|
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_unary_inplace(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
enum ggml_unary_op op);
|
|
|
|
|
|
|
|
|
|
// used in sam
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_get_rel_pos(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
int qh,
|
|
|
|
|
int kh);
|
|
|
|
|
|
|
|
|
|
// used in sam
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_add_rel_pos(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
struct ggml_tensor * pw,
|
|
|
|
|
struct ggml_tensor * ph);
|
|
|
|
|
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_add_rel_pos_inplace(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
struct ggml_tensor * pw,
|
|
|
|
|
struct ggml_tensor * ph);
|
|
|
|
|
|
2024-11-07 07:19:10 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_rwkv_wkv6(
|
2024-09-01 14:38:17 +00:00
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * k,
|
|
|
|
|
struct ggml_tensor * v,
|
|
|
|
|
struct ggml_tensor * r,
|
|
|
|
|
struct ggml_tensor * tf,
|
|
|
|
|
struct ggml_tensor * td,
|
|
|
|
|
struct ggml_tensor * state);
|
|
|
|
|
|
2023-06-25 11:22:21 +00:00
|
|
|
// custom operators
|
|
|
|
|
|
|
|
|
|
typedef void (*ggml_unary_op_f32_t) (const int, float *, const float *);
|
2023-04-29 16:30:22 +00:00
|
|
|
typedef void (*ggml_binary_op_f32_t)(const int, float *, const float *, const float *);
|
2023-04-29 09:31:52 +00:00
|
|
|
|
2023-06-25 11:22:21 +00:00
|
|
|
typedef void (*ggml_custom1_op_f32_t)(struct ggml_tensor *, const struct ggml_tensor *);
|
|
|
|
|
typedef void (*ggml_custom2_op_f32_t)(struct ggml_tensor *, const struct ggml_tensor *, const struct ggml_tensor *);
|
|
|
|
|
typedef void (*ggml_custom3_op_f32_t)(struct ggml_tensor *, const struct ggml_tensor *, const struct ggml_tensor *, const struct ggml_tensor *);
|
|
|
|
|
|
2023-09-05 10:54:40 +00:00
|
|
|
GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_unary_f32(
|
2023-04-29 09:31:52 +00:00
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
2023-09-05 10:54:40 +00:00
|
|
|
ggml_unary_op_f32_t fun),
|
|
|
|
|
"use ggml_map_custom1 instead");
|
2023-04-29 09:31:52 +00:00
|
|
|
|
2023-09-05 10:54:40 +00:00
|
|
|
GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_unary_inplace_f32(
|
2023-06-25 11:22:21 +00:00
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
2023-09-05 10:54:40 +00:00
|
|
|
ggml_unary_op_f32_t fun),
|
|
|
|
|
"use ggml_map_custom1_inplace instead");
|
2023-06-25 11:22:21 +00:00
|
|
|
|
2023-09-05 10:54:40 +00:00
|
|
|
GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_binary_f32(
|
2023-04-29 09:31:52 +00:00
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
struct ggml_tensor * b,
|
2023-09-05 10:54:40 +00:00
|
|
|
ggml_binary_op_f32_t fun),
|
|
|
|
|
"use ggml_map_custom2 instead");
|
2022-09-25 18:23:15 +00:00
|
|
|
|
2023-09-05 10:54:40 +00:00
|
|
|
GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_binary_inplace_f32(
|
2023-06-25 11:22:21 +00:00
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
struct ggml_tensor * b,
|
2023-09-05 10:54:40 +00:00
|
|
|
ggml_binary_op_f32_t fun),
|
|
|
|
|
"use ggml_map_custom2_inplace instead");
|
2023-06-25 11:22:21 +00:00
|
|
|
|
2023-09-05 10:54:40 +00:00
|
|
|
GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_custom1_f32(
|
2023-06-25 11:22:21 +00:00
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
2023-09-05 10:54:40 +00:00
|
|
|
ggml_custom1_op_f32_t fun),
|
|
|
|
|
"use ggml_map_custom1 instead");
|
2023-06-25 11:22:21 +00:00
|
|
|
|
2023-09-05 10:54:40 +00:00
|
|
|
GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_custom1_inplace_f32(
|
2023-06-25 11:22:21 +00:00
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
2023-09-05 10:54:40 +00:00
|
|
|
ggml_custom1_op_f32_t fun),
|
|
|
|
|
"use ggml_map_custom1_inplace instead");
|
2023-06-25 11:22:21 +00:00
|
|
|
|
2023-09-05 10:54:40 +00:00
|
|
|
GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_custom2_f32(
|
2023-06-25 11:22:21 +00:00
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
struct ggml_tensor * b,
|
2023-09-05 10:54:40 +00:00
|
|
|
ggml_custom2_op_f32_t fun),
|
|
|
|
|
"use ggml_map_custom2 instead");
|
2023-06-25 11:22:21 +00:00
|
|
|
|
2023-09-05 10:54:40 +00:00
|
|
|
GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_custom2_inplace_f32(
|
2023-06-25 11:22:21 +00:00
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
struct ggml_tensor * b,
|
2023-09-05 10:54:40 +00:00
|
|
|
ggml_custom2_op_f32_t fun),
|
|
|
|
|
"use ggml_map_custom2_inplace instead");
|
2023-06-25 11:22:21 +00:00
|
|
|
|
2023-09-05 10:54:40 +00:00
|
|
|
GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_custom3_f32(
|
2023-06-25 11:22:21 +00:00
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
struct ggml_tensor * b,
|
|
|
|
|
struct ggml_tensor * c,
|
2023-09-05 10:54:40 +00:00
|
|
|
ggml_custom3_op_f32_t fun),
|
|
|
|
|
"use ggml_map_custom3 instead");
|
2023-06-25 11:22:21 +00:00
|
|
|
|
2023-09-05 10:54:40 +00:00
|
|
|
GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_custom3_inplace_f32(
|
2023-06-25 11:22:21 +00:00
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
struct ggml_tensor * b,
|
|
|
|
|
struct ggml_tensor * c,
|
2023-09-05 10:54:40 +00:00
|
|
|
ggml_custom3_op_f32_t fun),
|
|
|
|
|
"use ggml_map_custom3_inplace instead");
|
|
|
|
|
|
|
|
|
|
// custom operators v2
|
|
|
|
|
|
|
|
|
|
typedef void (*ggml_custom1_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, int ith, int nth, void * userdata);
|
|
|
|
|
typedef void (*ggml_custom2_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, const struct ggml_tensor * b, int ith, int nth, void * userdata);
|
|
|
|
|
typedef void (*ggml_custom3_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, const struct ggml_tensor * b, const struct ggml_tensor * c, int ith, int nth, void * userdata);
|
|
|
|
|
|
2024-09-20 18:50:16 +00:00
|
|
|
#define GGML_N_TASKS_MAX (-1)
|
|
|
|
|
// n_tasks == GGML_N_TASKS_MAX means to use max number of tasks
|
|
|
|
|
|
2023-09-05 10:54:40 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_map_custom1(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
ggml_custom1_op_t fun,
|
|
|
|
|
int n_tasks,
|
|
|
|
|
void * userdata);
|
|
|
|
|
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_map_custom1_inplace(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
ggml_custom1_op_t fun,
|
|
|
|
|
int n_tasks,
|
|
|
|
|
void * userdata);
|
|
|
|
|
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_map_custom2(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
struct ggml_tensor * b,
|
|
|
|
|
ggml_custom2_op_t fun,
|
|
|
|
|
int n_tasks,
|
|
|
|
|
void * userdata);
|
|
|
|
|
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_map_custom2_inplace(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
struct ggml_tensor * b,
|
|
|
|
|
ggml_custom2_op_t fun,
|
|
|
|
|
int n_tasks,
|
|
|
|
|
void * userdata);
|
|
|
|
|
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_map_custom3(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
struct ggml_tensor * b,
|
|
|
|
|
struct ggml_tensor * c,
|
|
|
|
|
ggml_custom3_op_t fun,
|
|
|
|
|
int n_tasks,
|
|
|
|
|
void * userdata);
|
|
|
|
|
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_map_custom3_inplace(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
|
|
|
|
struct ggml_tensor * b,
|
|
|
|
|
struct ggml_tensor * c,
|
|
|
|
|
ggml_custom3_op_t fun,
|
|
|
|
|
int n_tasks,
|
|
|
|
|
void * userdata);
|
2023-06-25 11:22:21 +00:00
|
|
|
|
|
|
|
|
// loss function
|
|
|
|
|
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_cross_entropy_loss(
|
2024-09-29 21:18:02 +00:00
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a, // logits
|
|
|
|
|
struct ggml_tensor * b); // labels
|
2023-06-25 11:22:21 +00:00
|
|
|
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_cross_entropy_loss_back(
|
2024-09-29 21:18:02 +00:00
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a, // logits
|
|
|
|
|
struct ggml_tensor * b, // labels
|
|
|
|
|
struct ggml_tensor * c); // gradients of cross_entropy_loss result
|
2023-06-25 11:22:21 +00:00
|
|
|
|
2024-09-20 12:36:38 +00:00
|
|
|
// AdamW optimizer step
|
|
|
|
|
// Paper: https://arxiv.org/pdf/1711.05101v3.pdf
|
|
|
|
|
// PyTorch: https://pytorch.org/docs/stable/generated/torch.optim.AdamW.html
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_opt_step_adamw(
|
|
|
|
|
struct ggml_context * ctx,
|
|
|
|
|
struct ggml_tensor * a,
|
2024-09-30 07:55:23 +00:00
|
|
|
struct ggml_tensor * grad,
|
2024-11-16 12:49:35 +00:00
|
|
|
struct ggml_tensor * m,
|
|
|
|
|
struct ggml_tensor * v,
|
|
|
|
|
struct ggml_tensor * adamw_params); // parameters such a the learning rate
|
2024-09-20 12:36:38 +00:00
|
|
|
|
2023-04-29 09:31:52 +00:00
|
|
|
//
|
|
|
|
|
// automatic differentiation
|
|
|
|
|
//
|
2022-12-16 16:00:12 +00:00
|
|
|
|
2024-11-16 12:49:35 +00:00
|
|
|
GGML_API void ggml_build_forward_expand(struct ggml_cgraph * cgraph, struct ggml_tensor * tensor);
|
|
|
|
|
GGML_API void ggml_build_backward_expand(
|
|
|
|
|
struct ggml_context * ctx_static, // context for static gradients (loss + gradient accumulation)
|
|
|
|
|
struct ggml_context * ctx_compute, // context for gradient computation
|
|
|
|
|
struct ggml_cgraph * cgraph,
|
|
|
|
|
bool accumulate); // whether or not gradients should be accumulated, requires static allocation of tensors in ctx_static
|
2022-12-16 16:00:12 +00:00
|
|
|
|
2023-09-05 10:54:40 +00:00
|
|
|
// graph allocation in a context
|
2024-09-20 18:24:06 +00:00
|
|
|
GGML_API struct ggml_cgraph * ggml_new_graph (struct ggml_context * ctx); // size = GGML_DEFAULT_GRAPH_SIZE, grads = false
|
|
|
|
|
GGML_API struct ggml_cgraph * ggml_new_graph_custom(struct ggml_context * ctx, size_t size, bool grads);
|
|
|
|
|
GGML_API struct ggml_cgraph * ggml_graph_dup (struct ggml_context * ctx, struct ggml_cgraph * cgraph);
|
|
|
|
|
GGML_API void ggml_graph_cpy (struct ggml_cgraph * src, struct ggml_cgraph * dst);
|
|
|
|
|
GGML_API void ggml_graph_reset (struct ggml_cgraph * cgraph); // set regular grads + optimizer momenta to 0, set loss grad to 1
|
|
|
|
|
GGML_API void ggml_graph_clear (struct ggml_cgraph * cgraph);
|
|
|
|
|
|
|
|
|
|
GGML_API int ggml_graph_size (struct ggml_cgraph * cgraph);
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_graph_node (struct ggml_cgraph * cgraph, int i); // if i < 0, returns nodes[n_nodes + i]
|
|
|
|
|
GGML_API struct ggml_tensor ** ggml_graph_nodes (struct ggml_cgraph * cgraph);
|
|
|
|
|
GGML_API int ggml_graph_n_nodes(struct ggml_cgraph * cgraph);
|
|
|
|
|
|
|
|
|
|
GGML_API void ggml_graph_add_node(struct ggml_cgraph * cgraph, struct ggml_tensor * tensor);
|
2023-11-03 19:35:05 +00:00
|
|
|
|
2023-09-05 10:54:40 +00:00
|
|
|
GGML_API size_t ggml_graph_overhead(void);
|
2023-11-03 19:35:05 +00:00
|
|
|
GGML_API size_t ggml_graph_overhead_custom(size_t size, bool grads);
|
2023-09-05 10:54:40 +00:00
|
|
|
|
2024-11-16 12:49:35 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_graph_get_tensor (const struct ggml_cgraph * cgraph, const char * name);
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_graph_get_grad (const struct ggml_cgraph * cgraph, const struct ggml_tensor * node);
|
|
|
|
|
GGML_API struct ggml_tensor * ggml_graph_get_grad_acc(const struct ggml_cgraph * cgraph, const struct ggml_tensor * node);
|
2023-06-25 11:22:21 +00:00
|
|
|
|
2023-11-03 19:35:05 +00:00
|
|
|
GGML_API void ggml_graph_export(const struct ggml_cgraph * cgraph, const char * fname);
|
|
|
|
|
GGML_API struct ggml_cgraph * ggml_graph_import(const char * fname, struct ggml_context ** ctx_data, struct ggml_context ** ctx_eval);
|
2023-06-25 11:22:21 +00:00
|
|
|
|
2023-04-29 09:31:52 +00:00
|
|
|
// print info and performance information for the graph
|
|
|
|
|
GGML_API void ggml_graph_print(const struct ggml_cgraph * cgraph);
|
2022-09-25 18:23:15 +00:00
|
|
|
|
2023-04-29 09:31:52 +00:00
|
|
|
// dump the graph into a file using the dot format
|
|
|
|
|
GGML_API void ggml_graph_dump_dot(const struct ggml_cgraph * gb, const struct ggml_cgraph * gf, const char * filename);
|
2022-09-25 18:23:15 +00:00
|
|
|
|
2024-11-16 12:49:35 +00:00
|
|
|
// TODO these functions were sandwiched in the old optimization interface, is there a better place for them?
|
2023-11-03 19:35:05 +00:00
|
|
|
typedef void (*ggml_log_callback)(enum ggml_log_level level, const char * text, void * user_data);
|
2023-09-05 17:57:27 +00:00
|
|
|
|
2024-10-03 18:25:11 +00:00
|
|
|
// Set callback for all future logging events.
|
|
|
|
|
// If this is not called, or NULL is supplied, everything is output on stderr.
|
|
|
|
|
GGML_API void ggml_log_set(ggml_log_callback log_callback, void * user_data);
|
|
|
|
|
|
2024-11-03 18:34:08 +00:00
|
|
|
GGML_API struct ggml_tensor * ggml_set_zero(struct ggml_tensor * tensor);
|
|
|
|
|
|
2023-04-29 09:31:52 +00:00
|
|
|
//
|
|
|
|
|
// quantization
|
|
|
|
|
//
|
2022-10-25 17:18:26 +00:00
|
|
|
|
2024-01-17 16:54:56 +00:00
|
|
|
// - ggml_quantize_init can be called multiple times with the same type
|
|
|
|
|
// it will only initialize the quantization tables for the first call or after ggml_quantize_free
|
|
|
|
|
// automatically called by ggml_quantize_chunk for convenience
|
|
|
|
|
//
|
|
|
|
|
// - ggml_quantize_free will free any memory allocated by ggml_quantize_init
|
|
|
|
|
// call this at the end of the program to avoid memory leaks
|
|
|
|
|
//
|
|
|
|
|
// note: these are thread-safe
|
|
|
|
|
//
|
|
|
|
|
GGML_API void ggml_quantize_init(enum ggml_type type);
|
|
|
|
|
GGML_API void ggml_quantize_free(void);
|
|
|
|
|
|
|
|
|
|
// some quantization type cannot be used without an importance matrix
|
|
|
|
|
GGML_API bool ggml_quantize_requires_imatrix(enum ggml_type type);
|
|
|
|
|
|
|
|
|
|
// calls ggml_quantize_init internally (i.e. can allocate memory)
|
2024-03-09 13:53:59 +00:00
|
|
|
GGML_API size_t ggml_quantize_chunk(
|
|
|
|
|
enum ggml_type type,
|
|
|
|
|
const float * src,
|
|
|
|
|
void * dst,
|
2024-04-09 08:16:13 +00:00
|
|
|
int64_t start,
|
|
|
|
|
int64_t nrows,
|
|
|
|
|
int64_t n_per_row,
|
2024-03-09 13:53:59 +00:00
|
|
|
const float * imatrix);
|
2024-01-14 07:45:56 +00:00
|
|
|
|
2023-09-05 10:54:40 +00:00
|
|
|
//
|
|
|
|
|
// gguf
|
|
|
|
|
//
|
|
|
|
|
|
|
|
|
|
enum gguf_type {
|
|
|
|
|
GGUF_TYPE_UINT8 = 0,
|
|
|
|
|
GGUF_TYPE_INT8 = 1,
|
|
|
|
|
GGUF_TYPE_UINT16 = 2,
|
|
|
|
|
GGUF_TYPE_INT16 = 3,
|
|
|
|
|
GGUF_TYPE_UINT32 = 4,
|
|
|
|
|
GGUF_TYPE_INT32 = 5,
|
|
|
|
|
GGUF_TYPE_FLOAT32 = 6,
|
|
|
|
|
GGUF_TYPE_BOOL = 7,
|
|
|
|
|
GGUF_TYPE_STRING = 8,
|
|
|
|
|
GGUF_TYPE_ARRAY = 9,
|
|
|
|
|
GGUF_TYPE_UINT64 = 10,
|
|
|
|
|
GGUF_TYPE_INT64 = 11,
|
|
|
|
|
GGUF_TYPE_FLOAT64 = 12,
|
|
|
|
|
GGUF_TYPE_COUNT, // marks the end of the enum
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
struct gguf_context;
|
|
|
|
|
|
|
|
|
|
struct gguf_init_params {
|
|
|
|
|
bool no_alloc;
|
|
|
|
|
|
|
|
|
|
// if not NULL, create a ggml_context and allocate the tensor data in it
|
|
|
|
|
struct ggml_context ** ctx;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
GGML_API struct gguf_context * gguf_init_empty(void);
|
|
|
|
|
GGML_API struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_params params);
|
|
|
|
|
//GGML_API struct gguf_context * gguf_init_from_buffer(..);
|
|
|
|
|
|
|
|
|
|
GGML_API void gguf_free(struct gguf_context * ctx);
|
|
|
|
|
|
|
|
|
|
GGML_API const char * gguf_type_name(enum gguf_type type);
|
|
|
|
|
|
2023-09-15 11:49:56 +00:00
|
|
|
GGML_API int gguf_get_version (const struct gguf_context * ctx);
|
|
|
|
|
GGML_API size_t gguf_get_alignment (const struct gguf_context * ctx);
|
|
|
|
|
GGML_API size_t gguf_get_data_offset(const struct gguf_context * ctx);
|
|
|
|
|
GGML_API void * gguf_get_data (const struct gguf_context * ctx);
|
2023-09-05 10:54:40 +00:00
|
|
|
|
2023-09-15 11:49:56 +00:00
|
|
|
GGML_API int gguf_get_n_kv(const struct gguf_context * ctx);
|
|
|
|
|
GGML_API int gguf_find_key(const struct gguf_context * ctx, const char * key);
|
2023-11-03 19:35:05 +00:00
|
|
|
GGML_API const char * gguf_get_key (const struct gguf_context * ctx, int key_id);
|
|
|
|
|
|
|
|
|
|
GGML_API enum gguf_type gguf_get_kv_type (const struct gguf_context * ctx, int key_id);
|
|
|
|
|
GGML_API enum gguf_type gguf_get_arr_type(const struct gguf_context * ctx, int key_id);
|
|
|
|
|
|
|
|
|
|
// will abort if the wrong type is used for the key
|
|
|
|
|
GGML_API uint8_t gguf_get_val_u8 (const struct gguf_context * ctx, int key_id);
|
|
|
|
|
GGML_API int8_t gguf_get_val_i8 (const struct gguf_context * ctx, int key_id);
|
|
|
|
|
GGML_API uint16_t gguf_get_val_u16 (const struct gguf_context * ctx, int key_id);
|
|
|
|
|
GGML_API int16_t gguf_get_val_i16 (const struct gguf_context * ctx, int key_id);
|
|
|
|
|
GGML_API uint32_t gguf_get_val_u32 (const struct gguf_context * ctx, int key_id);
|
|
|
|
|
GGML_API int32_t gguf_get_val_i32 (const struct gguf_context * ctx, int key_id);
|
|
|
|
|
GGML_API float gguf_get_val_f32 (const struct gguf_context * ctx, int key_id);
|
|
|
|
|
GGML_API uint64_t gguf_get_val_u64 (const struct gguf_context * ctx, int key_id);
|
|
|
|
|
GGML_API int64_t gguf_get_val_i64 (const struct gguf_context * ctx, int key_id);
|
|
|
|
|
GGML_API double gguf_get_val_f64 (const struct gguf_context * ctx, int key_id);
|
|
|
|
|
GGML_API bool gguf_get_val_bool(const struct gguf_context * ctx, int key_id);
|
|
|
|
|
GGML_API const char * gguf_get_val_str (const struct gguf_context * ctx, int key_id);
|
2023-12-07 20:27:19 +00:00
|
|
|
GGML_API const void * gguf_get_val_data(const struct gguf_context * ctx, int key_id);
|
2023-11-03 19:35:05 +00:00
|
|
|
GGML_API int gguf_get_arr_n (const struct gguf_context * ctx, int key_id);
|
|
|
|
|
GGML_API const void * gguf_get_arr_data(const struct gguf_context * ctx, int key_id);
|
2023-09-15 11:49:56 +00:00
|
|
|
GGML_API const char * gguf_get_arr_str (const struct gguf_context * ctx, int key_id, int i);
|
|
|
|
|
|
2023-12-22 15:53:39 +00:00
|
|
|
GGML_API int gguf_get_n_tensors (const struct gguf_context * ctx);
|
|
|
|
|
GGML_API int gguf_find_tensor (const struct gguf_context * ctx, const char * name);
|
|
|
|
|
GGML_API size_t gguf_get_tensor_offset(const struct gguf_context * ctx, int i);
|
|
|
|
|
GGML_API char * gguf_get_tensor_name (const struct gguf_context * ctx, int i);
|
|
|
|
|
GGML_API enum ggml_type gguf_get_tensor_type (const struct gguf_context * ctx, int i);
|
2023-09-05 10:54:40 +00:00
|
|
|
|
2024-04-12 10:45:06 +00:00
|
|
|
// removes key if it exists
|
|
|
|
|
GGML_API void gguf_remove_key(struct gguf_context * ctx, const char * key);
|
|
|
|
|
|
2023-09-05 10:54:40 +00:00
|
|
|
// overrides existing values or adds a new one
|
|
|
|
|
GGML_API void gguf_set_val_u8 (struct gguf_context * ctx, const char * key, uint8_t val);
|
|
|
|
|
GGML_API void gguf_set_val_i8 (struct gguf_context * ctx, const char * key, int8_t val);
|
|
|
|
|
GGML_API void gguf_set_val_u16 (struct gguf_context * ctx, const char * key, uint16_t val);
|
|
|
|
|
GGML_API void gguf_set_val_i16 (struct gguf_context * ctx, const char * key, int16_t val);
|
|
|
|
|
GGML_API void gguf_set_val_u32 (struct gguf_context * ctx, const char * key, uint32_t val);
|
|
|
|
|
GGML_API void gguf_set_val_i32 (struct gguf_context * ctx, const char * key, int32_t val);
|
|
|
|
|
GGML_API void gguf_set_val_f32 (struct gguf_context * ctx, const char * key, float val);
|
|
|
|
|
GGML_API void gguf_set_val_u64 (struct gguf_context * ctx, const char * key, uint64_t val);
|
|
|
|
|
GGML_API void gguf_set_val_i64 (struct gguf_context * ctx, const char * key, int64_t val);
|
|
|
|
|
GGML_API void gguf_set_val_f64 (struct gguf_context * ctx, const char * key, double val);
|
|
|
|
|
GGML_API void gguf_set_val_bool(struct gguf_context * ctx, const char * key, bool val);
|
|
|
|
|
GGML_API void gguf_set_val_str (struct gguf_context * ctx, const char * key, const char * val);
|
|
|
|
|
GGML_API void gguf_set_arr_data(struct gguf_context * ctx, const char * key, enum gguf_type type, const void * data, int n);
|
|
|
|
|
GGML_API void gguf_set_arr_str (struct gguf_context * ctx, const char * key, const char ** data, int n);
|
|
|
|
|
|
|
|
|
|
// set or add KV pairs from another context
|
|
|
|
|
GGML_API void gguf_set_kv(struct gguf_context * ctx, struct gguf_context * src);
|
|
|
|
|
|
|
|
|
|
// manage tensor info
|
|
|
|
|
GGML_API void gguf_add_tensor(struct gguf_context * ctx, const struct ggml_tensor * tensor);
|
|
|
|
|
GGML_API void gguf_set_tensor_type(struct gguf_context * ctx, const char * name, enum ggml_type type);
|
|
|
|
|
GGML_API void gguf_set_tensor_data(struct gguf_context * ctx, const char * name, const void * data, size_t size);
|
|
|
|
|
|
|
|
|
|
// writing gguf files can be done in 2 ways:
|
|
|
|
|
//
|
|
|
|
|
// - write the entire gguf_context to a binary file in a single pass:
|
|
|
|
|
//
|
|
|
|
|
// gguf_write_to_file(ctx, fname);
|
|
|
|
|
//
|
|
|
|
|
// - first prepare a file with a placeholder for the meta data, write the tensor data, then write the meta data:
|
|
|
|
|
//
|
|
|
|
|
// FILE * f = fopen(fname, "wb");
|
|
|
|
|
// fseek(f, gguf_get_meta_size(ctx), SEEK_SET);
|
|
|
|
|
// fwrite(f, ...);
|
|
|
|
|
// void * data = gguf_meta_get_meta_data(ctx);
|
|
|
|
|
// fseek(f, 0, SEEK_SET);
|
|
|
|
|
// fwrite(f, data, gguf_get_meta_size(ctx));
|
|
|
|
|
// free(data);
|
|
|
|
|
// fclose(f);
|
|
|
|
|
//
|
|
|
|
|
|
|
|
|
|
// write the entire context to a binary file
|
2023-09-15 11:49:56 +00:00
|
|
|
GGML_API void gguf_write_to_file(const struct gguf_context * ctx, const char * fname, bool only_meta);
|
2023-09-05 10:54:40 +00:00
|
|
|
|
|
|
|
|
// get the size in bytes of the meta data (header, kv pairs, tensor info) including padding
|
2023-09-15 11:49:56 +00:00
|
|
|
GGML_API size_t gguf_get_meta_size(const struct gguf_context * ctx);
|
|
|
|
|
GGML_API void gguf_get_meta_data(const struct gguf_context * ctx, void * data);
|
2023-09-05 10:54:40 +00:00
|
|
|
|
2023-04-10 19:28:54 +00:00
|
|
|
#ifdef __cplusplus
|
2023-09-05 10:54:40 +00:00
|
|
|
// restrict not standard in C++
|
2023-04-10 19:28:54 +00:00
|
|
|
#define GGML_RESTRICT
|
|
|
|
|
#else
|
|
|
|
|
#define GGML_RESTRICT restrict
|
|
|
|
|
#endif
|
2024-04-09 08:16:13 +00:00
|
|
|
typedef void (*ggml_to_float_t) (const void * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
|
|
|
|
typedef void (*ggml_from_float_t)(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
|
2023-04-29 09:31:52 +00:00
|
|
|
|
2024-10-08 12:21:43 +00:00
|
|
|
struct ggml_type_traits {
|
2024-07-12 07:46:02 +00:00
|
|
|
const char * type_name;
|
|
|
|
|
int64_t blck_size;
|
|
|
|
|
int64_t blck_size_interleave; // interleave elements in blocks
|
|
|
|
|
size_t type_size;
|
|
|
|
|
bool is_quantized;
|
|
|
|
|
ggml_to_float_t to_float;
|
|
|
|
|
ggml_from_float_t from_float_ref;
|
2024-10-08 12:21:43 +00:00
|
|
|
};
|
2023-09-05 10:54:40 +00:00
|
|
|
|
2024-10-08 12:21:43 +00:00
|
|
|
GGML_API const struct ggml_type_traits * ggml_get_type_traits(enum ggml_type type);
|
2023-04-10 19:28:54 +00:00
|
|
|
|
2024-11-25 14:13:39 +00:00
|
|
|
// ggml threadpool
|
|
|
|
|
// TODO: currently, only a few functions are in the base ggml API, while the rest are in the CPU backend
|
|
|
|
|
// the goal should be to create an API that other backends can use move everything to the ggml base
|
|
|
|
|
|
|
|
|
|
// scheduling priorities
|
|
|
|
|
enum ggml_sched_priority {
|
|
|
|
|
GGML_SCHED_PRIO_NORMAL,
|
|
|
|
|
GGML_SCHED_PRIO_MEDIUM,
|
|
|
|
|
GGML_SCHED_PRIO_HIGH,
|
|
|
|
|
GGML_SCHED_PRIO_REALTIME
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
// threadpool params
|
|
|
|
|
// Use ggml_threadpool_params_default() or ggml_threadpool_params_init() to populate the defaults
|
|
|
|
|
struct ggml_threadpool_params {
|
|
|
|
|
bool cpumask[GGML_MAX_N_THREADS]; // mask of cpu cores (all-zeros means use default affinity settings)
|
|
|
|
|
int n_threads; // number of threads
|
|
|
|
|
enum ggml_sched_priority prio; // thread priority
|
|
|
|
|
uint32_t poll; // polling level (0 - no polling, 100 - aggressive polling)
|
|
|
|
|
bool strict_cpu; // strict cpu placement
|
|
|
|
|
bool paused; // start in paused state
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
struct ggml_threadpool; // forward declaration, see ggml.c
|
|
|
|
|
|
|
|
|
|
typedef struct ggml_threadpool * ggml_threadpool_t;
|
|
|
|
|
|
|
|
|
|
GGML_API struct ggml_threadpool_params ggml_threadpool_params_default(int n_threads);
|
|
|
|
|
GGML_API void ggml_threadpool_params_init (struct ggml_threadpool_params * p, int n_threads);
|
|
|
|
|
GGML_API bool ggml_threadpool_params_match (const struct ggml_threadpool_params * p0, const struct ggml_threadpool_params * p1);
|
|
|
|
|
|
2022-09-25 18:23:15 +00:00
|
|
|
#ifdef __cplusplus
|
|
|
|
|
}
|
|
|
|
|
#endif
|
