2023-11-03 19:35:05 +00:00
|
|
|
#pragma once
|
|
|
|
|
|
|
|
#include "ggml.h"
|
|
|
|
#include "ggml-alloc.h"
|
|
|
|
|
2024-11-14 17:04:35 +00:00
|
|
|
#ifdef GGML_BACKEND_SHARED
|
|
|
|
# if defined(_WIN32) && !defined(__MINGW32__)
|
|
|
|
# ifdef GGML_BACKEND_BUILD
|
|
|
|
# define GGML_BACKEND_API __declspec(dllexport) extern
|
|
|
|
# else
|
|
|
|
# define GGML_BACKEND_API __declspec(dllimport) extern
|
|
|
|
# endif
|
|
|
|
# else
|
|
|
|
# define GGML_BACKEND_API __attribute__ ((visibility ("default"))) extern
|
|
|
|
# endif
|
|
|
|
#else
|
|
|
|
# define GGML_BACKEND_API extern
|
|
|
|
#endif
|
|
|
|
|
2023-11-03 19:35:05 +00:00
|
|
|
#ifdef __cplusplus
|
|
|
|
extern "C" {
|
|
|
|
#endif
|
|
|
|
|
2023-12-07 20:27:19 +00:00
|
|
|
typedef struct ggml_backend_buffer_type * ggml_backend_buffer_type_t;
|
|
|
|
typedef struct ggml_backend_buffer * ggml_backend_buffer_t;
|
2024-03-13 17:54:21 +00:00
|
|
|
typedef struct ggml_backend_event * ggml_backend_event_t;
|
2023-12-07 20:27:19 +00:00
|
|
|
typedef struct ggml_backend * ggml_backend_t;
|
|
|
|
typedef void * ggml_backend_graph_plan_t;
|
2024-10-02 23:49:47 +00:00
|
|
|
typedef struct ggml_backend_reg * ggml_backend_reg_t;
|
|
|
|
typedef struct ggml_backend_device * ggml_backend_dev_t;
|
|
|
|
|
2023-12-07 20:27:19 +00:00
|
|
|
|
2023-11-03 19:35:05 +00:00
|
|
|
//
|
2024-10-02 23:49:47 +00:00
|
|
|
// Backend buffer type
|
2023-11-03 19:35:05 +00:00
|
|
|
//
|
|
|
|
|
2024-10-02 23:49:47 +00:00
|
|
|
GGML_API const char * ggml_backend_buft_name (ggml_backend_buffer_type_t buft);
|
|
|
|
GGML_API ggml_backend_buffer_t ggml_backend_buft_alloc_buffer (ggml_backend_buffer_type_t buft, size_t size);
|
|
|
|
GGML_API size_t ggml_backend_buft_get_alignment (ggml_backend_buffer_type_t buft);
|
|
|
|
GGML_API size_t ggml_backend_buft_get_max_size (ggml_backend_buffer_type_t buft);
|
|
|
|
GGML_API size_t ggml_backend_buft_get_alloc_size(ggml_backend_buffer_type_t buft, struct ggml_tensor * tensor);
|
|
|
|
GGML_API bool ggml_backend_buft_is_host (ggml_backend_buffer_type_t buft);
|
|
|
|
GGML_API ggml_backend_dev_t ggml_backend_buft_get_device (ggml_backend_buffer_type_t buft);
|
|
|
|
|
|
|
|
//
|
|
|
|
// Backend buffer
|
|
|
|
//
|
2023-11-03 19:35:05 +00:00
|
|
|
|
2024-01-12 19:07:38 +00:00
|
|
|
enum ggml_backend_buffer_usage {
|
|
|
|
GGML_BACKEND_BUFFER_USAGE_ANY = 0,
|
|
|
|
GGML_BACKEND_BUFFER_USAGE_WEIGHTS = 1,
|
2024-07-18 21:48:47 +00:00
|
|
|
GGML_BACKEND_BUFFER_USAGE_COMPUTE = 2,
|
2024-01-12 19:07:38 +00:00
|
|
|
};
|
|
|
|
|
2024-10-02 23:49:47 +00:00
|
|
|
GGML_API const char * ggml_backend_buffer_name (ggml_backend_buffer_t buffer);
|
|
|
|
GGML_API void ggml_backend_buffer_free (ggml_backend_buffer_t buffer);
|
|
|
|
GGML_API void * ggml_backend_buffer_get_base (ggml_backend_buffer_t buffer);
|
|
|
|
GGML_API size_t ggml_backend_buffer_get_size (ggml_backend_buffer_t buffer);
|
|
|
|
GGML_API void ggml_backend_buffer_init_tensor (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor);
|
|
|
|
GGML_API size_t ggml_backend_buffer_get_alignment (ggml_backend_buffer_t buffer);
|
|
|
|
GGML_API size_t ggml_backend_buffer_get_max_size (ggml_backend_buffer_t buffer);
|
|
|
|
GGML_API size_t ggml_backend_buffer_get_alloc_size(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor);
|
|
|
|
GGML_API void ggml_backend_buffer_clear (ggml_backend_buffer_t buffer, uint8_t value);
|
|
|
|
GGML_API bool ggml_backend_buffer_is_host (ggml_backend_buffer_t buffer);
|
|
|
|
GGML_API void ggml_backend_buffer_set_usage (ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage);
|
|
|
|
GGML_API enum ggml_backend_buffer_usage ggml_backend_buffer_get_usage (ggml_backend_buffer_t buffer);
|
|
|
|
GGML_API ggml_backend_buffer_type_t ggml_backend_buffer_get_type (ggml_backend_buffer_t buffer);
|
|
|
|
GGML_API void ggml_backend_buffer_reset (ggml_backend_buffer_t buffer);
|
|
|
|
|
|
|
|
// tensor copy between different backends
|
|
|
|
GGML_API void ggml_backend_tensor_copy(struct ggml_tensor * src, struct ggml_tensor * dst);
|
2023-11-03 19:35:05 +00:00
|
|
|
|
|
|
|
//
|
2024-10-02 23:49:47 +00:00
|
|
|
// Backend (stream)
|
2023-11-03 19:35:05 +00:00
|
|
|
//
|
|
|
|
|
2024-02-24 16:27:36 +00:00
|
|
|
GGML_API ggml_guid_t ggml_backend_guid(ggml_backend_t backend);
|
2023-11-03 19:35:05 +00:00
|
|
|
GGML_API const char * ggml_backend_name(ggml_backend_t backend);
|
|
|
|
GGML_API void ggml_backend_free(ggml_backend_t backend);
|
|
|
|
|
2023-12-07 20:27:19 +00:00
|
|
|
GGML_API ggml_backend_buffer_type_t ggml_backend_get_default_buffer_type(ggml_backend_t backend);
|
|
|
|
GGML_API ggml_backend_buffer_t ggml_backend_alloc_buffer(ggml_backend_t backend, size_t size);
|
|
|
|
GGML_API size_t ggml_backend_get_alignment(ggml_backend_t backend);
|
ggml : add Vulkan backend (llama/2059)
* Vulkan loader code
* Fix matmul kernel, continue implementation
* Continue implementation
* Vulkan memory management
* Vulkan development
* Matmul call
* Add aligned malloc and free for VMA
* Continue implementation
* First matmul success
* GEMM Kernel optimization
* 1D Blocktiling
* 2D Blocktiling
* Write coalescing
* Continue vulkan implementation and optimization
* First FP16 attempt, disabled for now
* Code abstraction, FP16 implementation, fix kernel, add FP16 to FP32 kernel
* Enable device extensions properly, restore fp16 matmul op
* Fix mulmat_f16
* Output FP32 in fp16 matmul shader
* Fix f16_to_f32 kernel
* dequant_q4_0 kernel
* Add VMA library
* Avoid requesting dedicated memory, VMA can decide that by itself
* Add bounds checking to matmul kernels, improve implementation, fix command buffers not freed properly
* add cmake commands
* Add 2d write operation, profiling code
* Fix 2d write
* Fix queue selection for AMD RADV
* Fix trailing whitespace in vk_mem_alloc.h
* Add WIP warp tile mat mul shaders
* Disable glslc optimization
* Disable glslc optimization for CMake
* Optimize warptile matmul shader, replace blocktile with it
* Add split-k optimization for small matrix multiplication
Use semaphores for synchronization instead of fences or waitidle
Rework async write/read for synchronization
* Fix validation errors, improve compatibility with AMD GPUs
* Rework command buffer handling
* Variable matmul kernel using specialization constants
* Fix synchronization on AMD, add barriers for buffer ownership transfer, add debug flag and prints
* Reuse semaphores
* Handle stage flags during command buffer submission properly
* Increase matmul test runs for consistent results
* Fix F32 matmul
* Add vectorized loading and zeropadding for matrix multiplication
* Use pinned memory for f16 preprocessing
* Don't force aligned matmul
* Don't free before queue done
* Replace VMA library with native Vulkan buffer management
* Basic offloading support with mul_f32 and dmmv for q4_0
* Run glslc commands in parallel
* Unroll loops in dmmv shader
* Reduce usage of waitIdle
* Reuse pinned allocation for f16 conversion
* Handle devices with only a single queue
* Fix trailing whitespace in CMakeLists.txt
* Allow parallel execution of kernels, parallelize third and fourth dimension calls
* Add fallback for devices only supporting one DescriptorSet per DescriptorPool
* Move to graph function similar to CUDA implementation
* Use F16 kernel for most things, replace q_f32 with mul_mat_q_f16 function
* Add F32 dmmv shaders
* Batch submissions
* Add .spv to gitignore
* Split off matrix vector multiplication for separate optimization
* Use single command buffer for matrix vector multiplication ops
* Reduce overhead of mul_f32 calls by using a single command buffer
* Add submission batching to mul_f32
* Fix tests
* Add missing barrier
* Add further missing barrier
* Add further ops
* Replace vk::QueueFamilyIgnored with VK_QUEUE_FAMILY_IGNORED to support more Vulkan header versions
* Remove unnecessary cblas link
* Fix descriptor set pre-allocation assert
* Add runtime shader compilation, start transferring shaders to this approach
* Transfer remaining shaders to header and compile on runtime
* Fix fp32 fallback if device doesn't support fp16, add force disable env var GGML_VULKAN_DISABLE_F16
* Add support for q4_1, q5_0, q5_1 and q8_0
* Remove unnecessary scalar layout extension
* Parse graph early to pre-record command buffers
* Add q6_k support
* Add multi-submit for command buffers
* Fix q6_k dequant shader for AMD
* Fix q6_k for GPUs without fp16 support
* Simplify q6_k fp16 fix
* Minor fixes
* Fix wg_denom of m-mulmat shaders
* Add Python-based Vulkan shader generator
* Replace shaderc dependency with precompiled shaders
Fix python script to generate shaders
* Clean up code
* Fix shader generator script Windows compatibility
Co-authored-by: Concedo <39025047+LostRuins@users.noreply.github.com>
* Close file before deletion
* Fix vulkan shader fp32 name
* Add q2_k and q3_k support
Add validation check to compare shader results to cpu results
* Add q4_k support
* Add q5_k support
* Bake SPIR-V bytecode into the library instead of loading shaders from file
* Switch to signal semaphores for flexibility
Prepare broadcasting support for mul mat
* Finish broadcasting mul mat support for GQA
* Clean up unused functions
Add repeat op
* Add further ops, not yet enabled. Improve semaphore code
* Reduce number of used semaphores by utilizing timelines more properly
* Remove queue information
* Reuse timeline semaphores, allow parallel operation with binary semaphores to work around nvidia driver limitations
* Add Vulkan to llama-bench
* Remove cblas dependency
* Fix matmul k-split bug
* Fix q4_k dmmv K_QUANTS_PER_ITERATION 1 shader
* Add RMS Norm shader, rework op_f32 shader setup, fix matmul bug
* Fix issues with float16 overflows in shaders
* Fix issues with older Vulkan headers on Ubuntu 22.04
* Allow multi-op partial offloading by parsing the graph to preallocate enough between-op buffers
* Implement further ops, rework op_f32 calls, fix bugs
* Finish full offloading support, add last remaining ops, fix bugs, remove redundant code
* Upload generated file ggml-vulkan-shaders.hpp, remove redundant shaders
* Merge upstream changes, fix conflicts, adapt soft_max op
* Fix Python and shader header format
* Free model gpu buffers on exit
* Use single queue per device to simplify code
* Add matmul shader support for running multiple calculations in parallel
* Switch from semaphore-synchronized multiple command buffers per op to single command buffer for multiple ops, whole graph if possible
* Fix missing event cast
* Replace uint64_t(-1) with UINT64_MAX, rename function for clarity
* Fix warning about empty C function parameters
* Fix compiler warnings
* Properly implement Vulkan backend buffer handling
* Fix oversized host staging buffers
* Simplify barrier synchronization calls
* Fix gcc warnings
* Implement max_size for backend buffer types to limit the size of a single allocation
* Use min of maxMemoryAllocationSize and maxBufferSize for device max allocation size
* refactor multi buf
* Disable unsupported ops to fix tests
* Check for maintenance4 support before using it
* Handle devices with only a single queue
* Fix single queue logic
* propagate buffer usage in multi buffers
* Implement rope_neox op
* Cleanup header and other files
* Simplify gpu_extras by removing events and putting staging memcpys into contexts
* Move queue into context
Add not-yet-enabled async backend ops
* Simplify context use, optimize matmul shader for warp size 64 (AMD GCN), fix split_k matmul shader optimization
* Add get_max_size to SYCL backend.
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* llama : fix trailing whitespace
---------
Co-authored-by: Henri Vasserman <henv@hot.ee>
Co-authored-by: Concedo <39025047+LostRuins@users.noreply.github.com>
Co-authored-by: slaren <slarengh@gmail.com>
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
2024-01-28 17:03:59 +00:00
|
|
|
GGML_API size_t ggml_backend_get_max_size(ggml_backend_t backend);
|
2023-11-03 19:35:05 +00:00
|
|
|
|
2023-12-07 20:27:19 +00:00
|
|
|
GGML_API void ggml_backend_tensor_set_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);
|
|
|
|
GGML_API void ggml_backend_tensor_get_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);
|
2023-11-03 19:35:05 +00:00
|
|
|
|
2024-11-16 12:49:35 +00:00
|
|
|
// "offset" refers to the offset in tensor->data for setting/getting data
|
2024-10-02 23:49:47 +00:00
|
|
|
GGML_API void ggml_backend_tensor_set( struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);
|
|
|
|
GGML_API void ggml_backend_tensor_get(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);
|
|
|
|
GGML_API void ggml_backend_tensor_memset( struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size);
|
2023-11-03 19:35:05 +00:00
|
|
|
|
|
|
|
GGML_API void ggml_backend_synchronize(ggml_backend_t backend);
|
|
|
|
|
2024-03-04 09:05:42 +00:00
|
|
|
GGML_API ggml_backend_graph_plan_t ggml_backend_graph_plan_create(ggml_backend_t backend, struct ggml_cgraph * cgraph);
|
|
|
|
GGML_API void ggml_backend_graph_plan_free (ggml_backend_t backend, ggml_backend_graph_plan_t plan);
|
2023-11-03 19:35:05 +00:00
|
|
|
|
2024-03-27 16:55:10 +00:00
|
|
|
GGML_API enum ggml_status ggml_backend_graph_plan_compute (ggml_backend_t backend, ggml_backend_graph_plan_t plan);
|
|
|
|
GGML_API enum ggml_status ggml_backend_graph_compute (ggml_backend_t backend, struct ggml_cgraph * cgraph);
|
|
|
|
GGML_API enum ggml_status ggml_backend_graph_compute_async(ggml_backend_t backend, struct ggml_cgraph * cgraph);
|
2024-10-02 23:49:47 +00:00
|
|
|
|
|
|
|
// NOTE: will be removed, use device version instead
|
2024-03-04 09:05:42 +00:00
|
|
|
GGML_API bool ggml_backend_supports_op(ggml_backend_t backend, const struct ggml_tensor * op);
|
2024-06-13 01:11:35 +00:00
|
|
|
GGML_API bool ggml_backend_supports_buft(ggml_backend_t backend, ggml_backend_buffer_type_t buft);
|
2024-03-27 16:55:10 +00:00
|
|
|
GGML_API bool ggml_backend_offload_op(ggml_backend_t backend, const struct ggml_tensor * op);
|
2023-11-03 19:35:05 +00:00
|
|
|
|
2024-03-13 17:54:21 +00:00
|
|
|
// asynchronous copy
|
|
|
|
// the copy is performed after all the currently queued operations in backend_src
|
|
|
|
// backend_dst will wait for the copy to complete before performing other operations
|
|
|
|
// automatic fallback to sync copy if async is not supported
|
|
|
|
GGML_API void ggml_backend_tensor_copy_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, struct ggml_tensor * src, struct ggml_tensor * dst);
|
|
|
|
|
2024-10-02 23:49:47 +00:00
|
|
|
GGML_API ggml_backend_dev_t ggml_backend_get_device(ggml_backend_t backend);
|
2023-11-03 19:35:05 +00:00
|
|
|
|
|
|
|
//
|
2024-10-02 23:49:47 +00:00
|
|
|
// Events
|
2023-11-03 19:35:05 +00:00
|
|
|
//
|
|
|
|
|
2024-10-02 23:49:47 +00:00
|
|
|
GGML_API ggml_backend_event_t ggml_backend_event_new(ggml_backend_dev_t device);
|
|
|
|
GGML_API void ggml_backend_event_free(ggml_backend_event_t event);
|
|
|
|
GGML_API void ggml_backend_event_record(ggml_backend_event_t event, ggml_backend_t backend);
|
|
|
|
GGML_API void ggml_backend_event_synchronize(ggml_backend_event_t event);
|
|
|
|
GGML_API void ggml_backend_event_wait(ggml_backend_t backend, ggml_backend_event_t event);
|
2023-11-03 19:35:05 +00:00
|
|
|
|
2024-10-02 23:49:47 +00:00
|
|
|
//
|
|
|
|
// Backend device
|
|
|
|
//
|
2023-11-03 19:35:05 +00:00
|
|
|
|
2024-10-02 23:49:47 +00:00
|
|
|
enum ggml_backend_dev_type {
|
2024-10-30 01:01:23 +00:00
|
|
|
// CPU device using system memory
|
2024-10-02 23:49:47 +00:00
|
|
|
GGML_BACKEND_DEVICE_TYPE_CPU,
|
2024-10-30 01:01:23 +00:00
|
|
|
// GPU device using dedicated memory
|
2024-10-02 23:49:47 +00:00
|
|
|
GGML_BACKEND_DEVICE_TYPE_GPU,
|
2024-10-30 01:01:23 +00:00
|
|
|
// accelerator devices intended to be used together with the CPU backend (e.g. BLAS or AMX)
|
|
|
|
GGML_BACKEND_DEVICE_TYPE_ACCEL
|
2024-10-02 23:49:47 +00:00
|
|
|
};
|
2023-12-07 20:27:19 +00:00
|
|
|
|
2024-10-02 23:49:47 +00:00
|
|
|
// functionality supported by the device
|
|
|
|
struct ggml_backend_dev_caps {
|
|
|
|
// asynchronous operations
|
|
|
|
bool async;
|
|
|
|
// pinned host buffer
|
|
|
|
bool host_buffer;
|
2024-10-07 15:27:51 +00:00
|
|
|
// creating buffers from host ptr
|
|
|
|
bool buffer_from_host_ptr;
|
2024-10-02 23:49:47 +00:00
|
|
|
// event synchronization
|
|
|
|
bool events;
|
|
|
|
};
|
2023-11-03 19:35:05 +00:00
|
|
|
|
2024-10-02 23:49:47 +00:00
|
|
|
// all the device properties
|
|
|
|
struct ggml_backend_dev_props {
|
|
|
|
const char * name;
|
|
|
|
const char * description;
|
|
|
|
size_t memory_free;
|
|
|
|
size_t memory_total;
|
|
|
|
enum ggml_backend_dev_type type;
|
|
|
|
struct ggml_backend_dev_caps caps;
|
|
|
|
};
|
|
|
|
|
|
|
|
GGML_API const char * ggml_backend_dev_name(ggml_backend_dev_t device);
|
|
|
|
GGML_API const char * ggml_backend_dev_description(ggml_backend_dev_t device);
|
|
|
|
GGML_API void ggml_backend_dev_memory(ggml_backend_dev_t device, size_t * free, size_t * total);
|
|
|
|
GGML_API enum ggml_backend_dev_type ggml_backend_dev_type(ggml_backend_dev_t device);
|
|
|
|
GGML_API void ggml_backend_dev_get_props(ggml_backend_dev_t device, struct ggml_backend_dev_props * props);
|
|
|
|
GGML_API ggml_backend_reg_t ggml_backend_dev_backend_reg(ggml_backend_dev_t device);
|
|
|
|
GGML_API ggml_backend_t ggml_backend_dev_init(ggml_backend_dev_t device, const char * params);
|
|
|
|
GGML_API ggml_backend_buffer_type_t ggml_backend_dev_buffer_type(ggml_backend_dev_t device);
|
|
|
|
GGML_API ggml_backend_buffer_type_t ggml_backend_dev_host_buffer_type(ggml_backend_dev_t device);
|
|
|
|
GGML_API ggml_backend_buffer_t ggml_backend_dev_buffer_from_host_ptr(ggml_backend_dev_t device, void * ptr, size_t size, size_t max_tensor_size);
|
|
|
|
|
|
|
|
GGML_API bool ggml_backend_dev_supports_op(ggml_backend_dev_t device, const struct ggml_tensor * op);
|
|
|
|
GGML_API bool ggml_backend_dev_supports_buft(ggml_backend_dev_t device, ggml_backend_buffer_type_t buft);
|
|
|
|
GGML_API bool ggml_backend_dev_offload_op(ggml_backend_dev_t device, const struct ggml_tensor * op);
|
|
|
|
|
|
|
|
//
|
|
|
|
// Backend (reg)
|
|
|
|
//
|
|
|
|
|
|
|
|
GGML_API const char * ggml_backend_reg_name(ggml_backend_reg_t reg);
|
|
|
|
GGML_API size_t ggml_backend_reg_dev_count(ggml_backend_reg_t reg);
|
|
|
|
GGML_API ggml_backend_dev_t ggml_backend_reg_dev_get(ggml_backend_reg_t reg, size_t index);
|
|
|
|
GGML_API void * ggml_backend_reg_get_proc_address(ggml_backend_reg_t reg, const char * name);
|
2024-10-03 18:25:11 +00:00
|
|
|
|
2024-10-30 01:01:23 +00:00
|
|
|
// Common functions that may be obtained using ggml_backend_reg_get_proc_address
|
2024-10-02 23:49:47 +00:00
|
|
|
|
2024-10-30 01:01:23 +00:00
|
|
|
// Split buffer type for tensor parallelism
|
|
|
|
typedef ggml_backend_buffer_type_t (*ggml_backend_split_buffer_type_t)(int main_device, const float * tensor_split);
|
|
|
|
// Set the number of threads for the backend
|
|
|
|
typedef void (*ggml_backend_set_n_threads_t)(ggml_backend_t backend, int n_threads);
|
|
|
|
// Get additional buffer types provided by the device (returns a NULL-terminated array)
|
|
|
|
typedef ggml_backend_buffer_type_t * (*ggml_backend_dev_get_extra_bufts_t)(ggml_backend_dev_t device);
|
2024-11-25 14:13:39 +00:00
|
|
|
// Set the abort callback for the backend
|
|
|
|
typedef void (*ggml_backend_set_abort_callback_t)(ggml_backend_t backend, ggml_abort_callback abort_callback, void * abort_callback_data);
|
|
|
|
// Get a list of feature flags supported by the backend (returns a NULL-terminated array)
|
|
|
|
struct ggml_backend_feature {
|
|
|
|
const char * name;
|
|
|
|
const char * value;
|
|
|
|
};
|
|
|
|
typedef struct ggml_backend_feature * (*ggml_backend_get_features_t)(ggml_backend_reg_t reg);
|
2023-12-22 15:53:39 +00:00
|
|
|
|
2023-12-07 20:27:19 +00:00
|
|
|
//
|
|
|
|
// Backend registry
|
|
|
|
//
|
|
|
|
|
2024-10-02 23:49:47 +00:00
|
|
|
// Backend (reg) enumeration
|
|
|
|
GGML_API size_t ggml_backend_reg_count(void);
|
|
|
|
GGML_API ggml_backend_reg_t ggml_backend_reg_get(size_t index);
|
|
|
|
GGML_API ggml_backend_reg_t ggml_backend_reg_by_name(const char * name);
|
|
|
|
|
|
|
|
// Device enumeration
|
|
|
|
GGML_API size_t ggml_backend_dev_count(void);
|
|
|
|
GGML_API ggml_backend_dev_t ggml_backend_dev_get(size_t index);
|
|
|
|
GGML_API ggml_backend_dev_t ggml_backend_dev_by_name(const char * name);
|
|
|
|
GGML_API ggml_backend_dev_t ggml_backend_dev_by_type(enum ggml_backend_dev_type type);
|
|
|
|
|
|
|
|
// Direct backend (stream) initialization
|
|
|
|
// = ggml_backend_dev_init(ggml_backend_dev_by_name(name), params)
|
|
|
|
GGML_API ggml_backend_t ggml_backend_init_by_name(const char * name, const char * params);
|
|
|
|
// = ggml_backend_dev_init(ggml_backend_dev_by_type(type), params)
|
|
|
|
GGML_API ggml_backend_t ggml_backend_init_by_type(enum ggml_backend_dev_type type, const char * params);
|
2024-10-30 01:01:23 +00:00
|
|
|
// = ggml_backend_dev_init(ggml_backend_dev_by_type(GPU) OR ggml_backend_dev_by_type(CPU), NULL)
|
2024-10-02 23:49:47 +00:00
|
|
|
GGML_API ggml_backend_t ggml_backend_init_best(void);
|
2023-11-03 19:35:05 +00:00
|
|
|
|
2024-11-25 14:13:39 +00:00
|
|
|
// Load a backend from a dynamic library and register it
|
|
|
|
GGML_API ggml_backend_reg_t ggml_backend_load(const char * path);
|
|
|
|
// Unload a backend if loaded dynamically and unregister it
|
|
|
|
GGML_API void ggml_backend_unload(ggml_backend_reg_t reg);
|
|
|
|
// Load all known backends from dynamic libraries
|
|
|
|
GGML_API void ggml_backend_load_all(void);
|
|
|
|
|
2023-11-03 19:35:05 +00:00
|
|
|
//
|
|
|
|
// Backend scheduler
|
|
|
|
//
|
|
|
|
|
2024-10-02 23:49:47 +00:00
|
|
|
// The backend scheduler allows for multiple backend devices to be used together
|
2023-11-03 19:35:05 +00:00
|
|
|
// Handles compute buffer allocation, assignment of tensors to backends, and copying of tensors between backends
|
|
|
|
// The backends are selected based on:
|
|
|
|
// - the backend that supports the operation
|
|
|
|
// - the location of the pre-allocated tensors (e.g. the weights)
|
|
|
|
/*
|
|
|
|
Example usage:
|
|
|
|
|
2024-04-03 20:57:20 +00:00
|
|
|
// operations that use tensors allocated in a buffer with USAGE_WEIGHTS will be assigned
|
2024-03-13 17:54:21 +00:00
|
|
|
// preferrably to run on the same backend as the buffer
|
|
|
|
ggml_backend_buffer_set_usage(buf_weights, GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
|
2023-11-03 19:35:05 +00:00
|
|
|
|
2024-03-13 17:54:21 +00:00
|
|
|
sched = ggml_backend_sched_new({backend_gpu, backend_gpu2, backend_cpu}, NULL, num_backends, GGML_DEFAULT_GRAPH_SIZE, false);
|
2023-11-03 19:35:05 +00:00
|
|
|
|
2024-03-13 17:54:21 +00:00
|
|
|
// initialize buffers from a max size graph (optional)
|
|
|
|
reserve_graph = build_graph(sched, max_batch_size);
|
2023-11-03 19:35:05 +00:00
|
|
|
|
2024-03-13 17:54:21 +00:00
|
|
|
// manually assign nodes to a backend (optional, should not be needed in most cases)
|
|
|
|
struct ggml_tensor * node = ggml_mul_mat(ctx, ...);
|
|
|
|
ggml_backend_sched_set_tensor_backend(sched, node, backend_gpu);
|
2023-11-03 19:35:05 +00:00
|
|
|
|
2024-03-13 17:54:21 +00:00
|
|
|
ggml_backend_sched_reserve(sched, reserve_graph);
|
2023-11-03 19:35:05 +00:00
|
|
|
|
|
|
|
// compute
|
2024-11-16 12:49:35 +00:00
|
|
|
graph = build_graph(sched); // the graph and its tensors are single-use in terms of allocation, multi-use in terms of computation
|
|
|
|
for (int i = 0; i < 10; ++i) {
|
|
|
|
ggml_backend_sched_graph_compute(sched, graph); // on the first iteration the graph is allocated automatically
|
|
|
|
}
|
2024-03-13 17:54:21 +00:00
|
|
|
|
|
|
|
// if there are graph inputs:
|
2024-11-16 12:49:35 +00:00
|
|
|
graph = build_graph(sched); // get a new graph that is not allocated (the metadata for the old graph is freed once ggml_free is called)
|
|
|
|
ggml_backend_sched_reset(sched); // clear the allocation of the previous graph
|
|
|
|
ggml_backend_sched_alloc_graph(sched, graph); // explicitly allocate the new graph but do not execute it
|
|
|
|
ggml_backend_tensor_set(input_tensor, ...); // copy data to the newly allocated graph tensors
|
|
|
|
ggml_backend_sched_graph_compute(sched, graph); // execute the graph
|
|
|
|
|
|
|
|
// as an alternative to the above it is also possible to assign the inputs to a dedicated context and
|
|
|
|
// allocate them statically via ggml_backend_alloc_ctx_tensors
|
2024-03-13 17:54:21 +00:00
|
|
|
}
|
2023-11-03 19:35:05 +00:00
|
|
|
*/
|
|
|
|
|
|
|
|
typedef struct ggml_backend_sched * ggml_backend_sched_t;
|
|
|
|
|
2024-10-02 23:49:47 +00:00
|
|
|
// Evaluation callback for each node in the graph (set with ggml_backend_sched_set_eval_callback)
|
2024-01-17 16:39:41 +00:00
|
|
|
// when ask == true, the scheduler wants to know if the user wants to observe this node
|
|
|
|
// this allows the scheduler to batch nodes together in order to evaluate them in a single call
|
|
|
|
//
|
|
|
|
// when ask == false, the scheduler is passing the node tensor to the user for observation
|
|
|
|
// if the user returns false, the scheduler will cancel the graph compute
|
|
|
|
//
|
|
|
|
typedef bool (*ggml_backend_sched_eval_callback)(struct ggml_tensor * t, bool ask, void * user_data);
|
|
|
|
|
2024-11-16 12:49:35 +00:00
|
|
|
// Initialize a backend scheduler, backends with low index are given priority over backends with high index
|
2024-03-13 17:54:21 +00:00
|
|
|
GGML_API ggml_backend_sched_t ggml_backend_sched_new(ggml_backend_t * backends, ggml_backend_buffer_type_t * bufts, int n_backends, size_t graph_size, bool parallel);
|
2024-03-04 09:05:42 +00:00
|
|
|
GGML_API void ggml_backend_sched_free(ggml_backend_sched_t sched);
|
2024-03-13 17:54:21 +00:00
|
|
|
|
2023-11-03 19:35:05 +00:00
|
|
|
// Initialize backend buffers from a measure graph
|
2024-10-02 13:32:39 +00:00
|
|
|
GGML_API bool ggml_backend_sched_reserve(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph); // returns success
|
2024-03-13 17:54:21 +00:00
|
|
|
|
2024-06-18 06:37:20 +00:00
|
|
|
GGML_API int ggml_backend_sched_get_n_backends(ggml_backend_sched_t sched);
|
|
|
|
GGML_API ggml_backend_t ggml_backend_sched_get_backend(ggml_backend_sched_t sched, int i);
|
|
|
|
|
2024-01-12 19:07:38 +00:00
|
|
|
// Get the number of splits of the last graph
|
2024-03-04 09:05:42 +00:00
|
|
|
GGML_API int ggml_backend_sched_get_n_splits(ggml_backend_sched_t sched);
|
2024-03-13 17:54:21 +00:00
|
|
|
GGML_API int ggml_backend_sched_get_n_copies(ggml_backend_sched_t sched);
|
2023-11-03 19:35:05 +00:00
|
|
|
|
2024-03-04 09:05:42 +00:00
|
|
|
GGML_API size_t ggml_backend_sched_get_buffer_size(ggml_backend_sched_t sched, ggml_backend_t backend);
|
2023-11-03 19:35:05 +00:00
|
|
|
|
2024-03-13 17:54:21 +00:00
|
|
|
GGML_API void ggml_backend_sched_set_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node, ggml_backend_t backend);
|
|
|
|
GGML_API ggml_backend_t ggml_backend_sched_get_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node);
|
2023-11-03 19:35:05 +00:00
|
|
|
|
2024-01-12 19:07:38 +00:00
|
|
|
// Allocate and compute graph on the backend scheduler
|
2024-10-02 13:32:39 +00:00
|
|
|
GGML_API bool ggml_backend_sched_alloc_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph); // returns success
|
2024-03-04 09:05:42 +00:00
|
|
|
GGML_API enum ggml_status ggml_backend_sched_graph_compute(ggml_backend_sched_t sched, struct ggml_cgraph * graph);
|
2024-03-13 17:54:21 +00:00
|
|
|
GGML_API enum ggml_status ggml_backend_sched_graph_compute_async(ggml_backend_sched_t sched, struct ggml_cgraph * graph);
|
|
|
|
GGML_API void ggml_backend_sched_synchronize(ggml_backend_sched_t sched);
|
2023-11-03 19:35:05 +00:00
|
|
|
|
2024-11-16 12:49:35 +00:00
|
|
|
// Reset all assignments and allocators - must be called before changing the node backends or allocating a new graph.
|
|
|
|
// This in effect deallocates all tensors that were previously allocated and leaves them with dangling pointers.
|
|
|
|
// The correct way to use this API is to discard the deallocated tensors and create new ones.
|
2024-03-04 09:05:42 +00:00
|
|
|
GGML_API void ggml_backend_sched_reset(ggml_backend_sched_t sched);
|
2023-12-07 20:27:19 +00:00
|
|
|
|
2024-01-17 16:39:41 +00:00
|
|
|
// Set a callback to be called for each resulting node during graph compute
|
2024-03-04 09:05:42 +00:00
|
|
|
GGML_API void ggml_backend_sched_set_eval_callback(ggml_backend_sched_t sched, ggml_backend_sched_eval_callback callback, void * user_data);
|
2024-01-17 16:39:41 +00:00
|
|
|
|
2023-12-07 20:27:19 +00:00
|
|
|
//
|
|
|
|
// Utils
|
|
|
|
//
|
|
|
|
|
|
|
|
struct ggml_backend_graph_copy {
|
|
|
|
ggml_backend_buffer_t buffer;
|
|
|
|
struct ggml_context * ctx_allocated;
|
|
|
|
struct ggml_context * ctx_unallocated;
|
|
|
|
struct ggml_cgraph * graph;
|
|
|
|
};
|
|
|
|
|
|
|
|
// Copy a graph to a different backend
|
|
|
|
GGML_API struct ggml_backend_graph_copy ggml_backend_graph_copy(ggml_backend_t backend, struct ggml_cgraph * graph);
|
|
|
|
GGML_API void ggml_backend_graph_copy_free(struct ggml_backend_graph_copy copy);
|
|
|
|
|
2024-10-02 23:49:47 +00:00
|
|
|
typedef bool (*ggml_backend_eval_callback)(int node_index, struct ggml_tensor * t1, struct ggml_tensor * t2, void * user_data);
|
2023-12-07 20:27:19 +00:00
|
|
|
|
|
|
|
// Compare the output of two backends
|
2024-01-12 19:07:38 +00:00
|
|
|
GGML_API bool ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t backend2, struct ggml_cgraph * graph, ggml_backend_eval_callback callback, void * user_data);
|
2023-12-07 20:27:19 +00:00
|
|
|
|
|
|
|
// Tensor initialization
|
|
|
|
GGML_API void ggml_backend_tensor_alloc(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, void * addr);
|
2024-06-03 17:03:26 +00:00
|
|
|
GGML_API void ggml_backend_view_init(struct ggml_tensor * tensor);
|
2023-12-07 20:27:19 +00:00
|
|
|
|
2024-11-03 18:34:08 +00:00
|
|
|
// CPU buffer types are always available
|
2024-10-02 23:49:47 +00:00
|
|
|
GGML_API ggml_backend_buffer_t ggml_backend_cpu_buffer_from_ptr(void * ptr, size_t size);
|
|
|
|
GGML_API ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void);
|
|
|
|
|
2023-11-03 19:35:05 +00:00
|
|
|
#ifdef __cplusplus
|
|
|
|
}
|
|
|
|
#endif
|