2023-09-05 10:54:40 +00:00
|
|
|
#include "ggml-alloc.h"
|
2023-11-03 19:35:05 +00:00
|
|
|
#include "ggml-backend-impl.h"
|
2023-09-05 10:54:40 +00:00
|
|
|
#include "ggml.h"
|
2023-11-03 19:35:05 +00:00
|
|
|
#include "ggml-impl.h"
|
2023-09-05 10:54:40 +00:00
|
|
|
#include <assert.h>
|
2023-11-03 19:35:05 +00:00
|
|
|
#include <limits.h>
|
2023-09-05 10:54:40 +00:00
|
|
|
#include <stdarg.h>
|
|
|
|
#include <stdio.h>
|
|
|
|
#include <stdlib.h>
|
|
|
|
#include <string.h>
|
|
|
|
|
|
|
|
#define MAX(a, b) ((a) > (b) ? (a) : (b))
|
2023-11-03 19:35:05 +00:00
|
|
|
#define MAX_FREE_BLOCKS 256
|
2023-09-05 10:54:40 +00:00
|
|
|
|
|
|
|
//#define GGML_ALLOCATOR_DEBUG
|
|
|
|
|
2023-11-03 19:35:05 +00:00
|
|
|
//#define AT_PRINTF(...) fprintf(stderr, __VA_ARGS__)
|
|
|
|
#define AT_PRINTF(...)
|
2023-09-05 10:54:40 +00:00
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
|
|
|
|
static bool ggml_is_view(const struct ggml_tensor * t) {
|
|
|
|
return t->view_src != NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool ggml_are_same_layout(const struct ggml_tensor * a, const struct ggml_tensor * b) {
|
|
|
|
if (a->type != b->type) {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
for (int i = 0; i < GGML_MAX_DIMS; i++) {
|
|
|
|
if (a->ne[i] != b->ne[i]) {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
if (a->nb[i] != b->nb[i]) {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool ggml_op_can_inplace(enum ggml_op op) {
|
|
|
|
switch (op) {
|
|
|
|
case GGML_OP_SCALE:
|
|
|
|
case GGML_OP_DIAG_MASK_ZERO:
|
|
|
|
case GGML_OP_DIAG_MASK_INF:
|
|
|
|
case GGML_OP_ADD:
|
|
|
|
case GGML_OP_ADD1:
|
|
|
|
case GGML_OP_SUB:
|
|
|
|
case GGML_OP_MUL:
|
|
|
|
case GGML_OP_DIV:
|
|
|
|
case GGML_OP_SQR:
|
|
|
|
case GGML_OP_SQRT:
|
|
|
|
case GGML_OP_LOG:
|
|
|
|
case GGML_OP_UNARY:
|
|
|
|
case GGML_OP_ROPE:
|
|
|
|
case GGML_OP_RMS_NORM:
|
|
|
|
case GGML_OP_SOFT_MAX:
|
|
|
|
return true;
|
|
|
|
|
|
|
|
default:
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-09-05 10:54:40 +00:00
|
|
|
static size_t aligned_offset(const void * buffer, size_t offset, size_t alignment) {
|
|
|
|
assert(alignment && !(alignment & (alignment - 1))); // power of 2
|
|
|
|
size_t align = (alignment - (((uintptr_t)buffer + offset) % alignment)) % alignment;
|
|
|
|
return offset + align;
|
|
|
|
}
|
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
// tallocr
|
|
|
|
|
2024-03-13 17:54:21 +00:00
|
|
|
struct ggml_tallocr ggml_tallocr_new(ggml_backend_buffer_t buffer) {
|
2024-02-11 12:37:58 +00:00
|
|
|
void * base = ggml_backend_buffer_get_base(buffer);
|
|
|
|
size_t align = ggml_backend_buffer_get_alignment(buffer);
|
|
|
|
|
|
|
|
assert(align && !(align & (align - 1))); // power of 2
|
|
|
|
|
2024-03-13 17:54:21 +00:00
|
|
|
struct ggml_tallocr talloc = (struct ggml_tallocr) {
|
2024-02-11 12:37:58 +00:00
|
|
|
/*.buffer = */ buffer,
|
|
|
|
/*.base = */ base,
|
|
|
|
/*.alignment = */ align,
|
|
|
|
/*.offset = */ aligned_offset(base, 0, align),
|
|
|
|
};
|
|
|
|
return talloc;
|
|
|
|
}
|
|
|
|
|
2024-03-13 17:54:21 +00:00
|
|
|
void ggml_tallocr_alloc(struct ggml_tallocr * talloc, struct ggml_tensor * tensor) {
|
2024-02-11 12:37:58 +00:00
|
|
|
size_t size = ggml_backend_buffer_get_alloc_size(talloc->buffer, tensor);
|
|
|
|
size = GGML_PAD(size, talloc->alignment);
|
|
|
|
|
|
|
|
if (talloc->offset + size > ggml_backend_buffer_get_size(talloc->buffer)) {
|
|
|
|
fprintf(stderr, "%s: not enough space in the buffer to allocate %s (needed %zu, available %zu)\n",
|
|
|
|
__func__, tensor->name, size, ggml_backend_buffer_get_size(talloc->buffer) - talloc->offset);
|
|
|
|
GGML_ASSERT(!"not enough space in the buffer");
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
void * addr = (char *)ggml_backend_buffer_get_base(talloc->buffer) + talloc->offset;
|
|
|
|
talloc->offset += size;
|
|
|
|
|
|
|
|
assert(((uintptr_t)addr % talloc->alignment) == 0);
|
|
|
|
|
|
|
|
ggml_backend_tensor_alloc(talloc->buffer, tensor, addr);
|
|
|
|
}
|
|
|
|
|
|
|
|
// dynamic tensor allocator
|
|
|
|
|
2023-09-05 10:54:40 +00:00
|
|
|
struct free_block {
|
2024-02-11 12:37:58 +00:00
|
|
|
size_t offset;
|
2023-09-05 10:54:40 +00:00
|
|
|
size_t size;
|
|
|
|
};
|
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
struct ggml_dyn_tallocr {
|
2023-09-05 10:54:40 +00:00
|
|
|
size_t alignment;
|
|
|
|
int n_free_blocks;
|
|
|
|
struct free_block free_blocks[MAX_FREE_BLOCKS];
|
|
|
|
size_t max_size;
|
2023-11-03 19:35:05 +00:00
|
|
|
|
2023-09-05 10:54:40 +00:00
|
|
|
#ifdef GGML_ALLOCATOR_DEBUG
|
2024-02-11 12:37:58 +00:00
|
|
|
struct {
|
|
|
|
const struct ggml_tensor * tensor;
|
|
|
|
size_t offset;
|
|
|
|
} allocated_tensors[1024];
|
2023-09-05 10:54:40 +00:00
|
|
|
#endif
|
|
|
|
};
|
|
|
|
|
|
|
|
#ifdef GGML_ALLOCATOR_DEBUG
|
2024-02-11 12:37:58 +00:00
|
|
|
static void add_allocated_tensor(struct ggml_dyn_tallocr * alloc, size_t offset, const struct ggml_tensor * tensor) {
|
2023-09-05 10:54:40 +00:00
|
|
|
for (int i = 0; i < 1024; i++) {
|
2024-02-11 12:37:58 +00:00
|
|
|
if (alloc->allocated_tensors[i].tensor == NULL) {
|
|
|
|
alloc->allocated_tensors[i].tensor = tensor;
|
|
|
|
alloc->allocated_tensors[i].offset = offset;
|
2023-09-05 10:54:40 +00:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
GGML_ASSERT(!"out of allocated_tensors");
|
|
|
|
}
|
2024-02-11 12:37:58 +00:00
|
|
|
static void remove_allocated_tensor(struct ggml_dyn_tallocr * alloc, size_t offset, const struct ggml_tensor * tensor) {
|
2023-09-05 10:54:40 +00:00
|
|
|
for (int i = 0; i < 1024; i++) {
|
2024-02-11 12:37:58 +00:00
|
|
|
if (alloc->allocated_tensors[i].offset == offset) {
|
|
|
|
alloc->allocated_tensors[i].tensor = NULL;
|
2023-09-05 10:54:40 +00:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
2024-02-11 12:37:58 +00:00
|
|
|
fprintf(stderr, "tried to free tensor %s not found\n", tensor->name);
|
2023-09-05 10:54:40 +00:00
|
|
|
GGML_ASSERT(!"tensor not found");
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
static size_t ggml_dyn_tallocr_alloc(struct ggml_dyn_tallocr * alloc, size_t size, const struct ggml_tensor * tensor) {
|
2023-09-05 10:54:40 +00:00
|
|
|
size = aligned_offset(NULL, size, alloc->alignment);
|
|
|
|
|
|
|
|
AT_PRINTF("%s: allocating %s (%zu bytes) - ", __func__, tensor->name, size);
|
|
|
|
|
|
|
|
size_t max_avail = 0;
|
|
|
|
|
|
|
|
// find the best fitting free block besides the last block
|
|
|
|
int best_fit_block = -1;
|
|
|
|
size_t best_fit_size = SIZE_MAX;
|
|
|
|
for (int i = 0; i < alloc->n_free_blocks - 1; i++) {
|
|
|
|
struct free_block * block = &alloc->free_blocks[i];
|
|
|
|
max_avail = MAX(max_avail, block->size);
|
|
|
|
if (block->size >= size && block->size <= best_fit_size) {
|
|
|
|
best_fit_block = i;
|
|
|
|
best_fit_size = block->size;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (best_fit_block == -1) {
|
|
|
|
// the last block is our last resort
|
|
|
|
struct free_block * block = &alloc->free_blocks[alloc->n_free_blocks - 1];
|
2023-09-15 09:18:18 +00:00
|
|
|
max_avail = MAX(max_avail, block->size);
|
2023-09-05 10:54:40 +00:00
|
|
|
if (block->size >= size) {
|
|
|
|
best_fit_block = alloc->n_free_blocks - 1;
|
|
|
|
} else {
|
2024-02-11 12:37:58 +00:00
|
|
|
// this should never happen
|
|
|
|
fprintf(stderr, "%s: not enough space in the buffer to allocate %zu bytes, largest block available %zu bytes\n",
|
|
|
|
__func__, size, max_avail);
|
2023-09-05 10:54:40 +00:00
|
|
|
GGML_ASSERT(!"not enough space in the buffer");
|
2024-02-11 12:37:58 +00:00
|
|
|
GGML_UNREACHABLE();
|
2023-09-05 10:54:40 +00:00
|
|
|
}
|
|
|
|
}
|
2024-01-12 19:07:38 +00:00
|
|
|
|
2023-09-05 10:54:40 +00:00
|
|
|
struct free_block * block = &alloc->free_blocks[best_fit_block];
|
2024-02-11 12:37:58 +00:00
|
|
|
size_t offset = block->offset;
|
|
|
|
block->offset = offset + size;
|
2023-09-05 10:54:40 +00:00
|
|
|
block->size -= size;
|
|
|
|
if (block->size == 0) {
|
|
|
|
// remove block if empty
|
|
|
|
alloc->n_free_blocks--;
|
|
|
|
for (int j = best_fit_block; j < alloc->n_free_blocks; j++) {
|
|
|
|
alloc->free_blocks[j] = alloc->free_blocks[j+1];
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
AT_PRINTF("block %d, offset %zu\n", best_fit_block, offset);
|
2023-09-05 10:54:40 +00:00
|
|
|
|
|
|
|
#ifdef GGML_ALLOCATOR_DEBUG
|
2024-02-11 12:37:58 +00:00
|
|
|
add_allocated_tensor(alloc, offset, tensor);
|
|
|
|
size_t cur_max = offset + size;
|
2023-09-05 10:54:40 +00:00
|
|
|
if (cur_max > alloc->max_size) {
|
2024-02-11 12:37:58 +00:00
|
|
|
// sort allocated_tensors by offset
|
|
|
|
for (int i = 0; i < 1024; i++) {
|
|
|
|
for (int j = i + 1; j < 1024; j++) {
|
|
|
|
if (alloc->allocated_tensors[i].offset > alloc->allocated_tensors[j].offset) {
|
|
|
|
const struct ggml_tensor * tmp_tensor = alloc->allocated_tensors[i].tensor;
|
|
|
|
size_t tmp_offset = alloc->allocated_tensors[i].offset;
|
|
|
|
alloc->allocated_tensors[i].tensor = alloc->allocated_tensors[j].tensor;
|
|
|
|
alloc->allocated_tensors[i].offset = alloc->allocated_tensors[j].offset;
|
|
|
|
alloc->allocated_tensors[j].tensor = tmp_tensor;
|
|
|
|
alloc->allocated_tensors[j].offset = tmp_offset;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
fprintf(stderr, "max_size = %.2f MB: tensors: ", cur_max / 1024.0 / 1024.0);
|
2023-09-05 10:54:40 +00:00
|
|
|
for (int i = 0; i < 1024; i++) {
|
2024-02-11 12:37:58 +00:00
|
|
|
if (alloc->allocated_tensors[i].tensor) {
|
|
|
|
fprintf(stderr, "%s [%zx-%zx] (%.2f MB) ", alloc->allocated_tensors[i].tensor->name,
|
|
|
|
alloc->allocated_tensors[i].offset,
|
|
|
|
alloc->allocated_tensors[i].offset + ggml_nbytes(alloc->allocated_tensors[i].tensor),
|
|
|
|
ggml_nbytes(alloc->allocated_tensors[i].tensor) / 1024.0 / 1024.0);
|
2023-09-05 10:54:40 +00:00
|
|
|
}
|
|
|
|
}
|
2024-02-11 12:37:58 +00:00
|
|
|
fprintf(stderr, "\n");
|
2023-09-05 10:54:40 +00:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
alloc->max_size = MAX(alloc->max_size, offset + size);
|
2023-09-05 10:54:40 +00:00
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
return offset;
|
2023-09-05 10:54:40 +00:00
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
GGML_UNUSED(tensor);
|
|
|
|
}
|
2023-11-03 19:35:05 +00:00
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
// this is a very naive implementation, but for our case the number of free blocks should be very small
|
|
|
|
static void ggml_dyn_tallocr_free_tensor(struct ggml_dyn_tallocr * alloc, size_t offset, size_t size, const struct ggml_tensor * tensor) {
|
2023-09-05 10:54:40 +00:00
|
|
|
size = aligned_offset(NULL, size, alloc->alignment);
|
2024-02-11 12:37:58 +00:00
|
|
|
|
|
|
|
AT_PRINTF("%s: freeing %s at %zu (%zu bytes) - n_free_blocks = %d\n", __func__, tensor->name, offset, size, alloc->n_free_blocks);
|
2023-11-03 19:35:05 +00:00
|
|
|
|
2023-09-05 10:54:40 +00:00
|
|
|
#ifdef GGML_ALLOCATOR_DEBUG
|
2024-02-11 12:37:58 +00:00
|
|
|
remove_allocated_tensor(alloc, offset, tensor);
|
2023-09-05 10:54:40 +00:00
|
|
|
#endif
|
|
|
|
|
|
|
|
// see if we can merge with an existing block
|
|
|
|
for (int i = 0; i < alloc->n_free_blocks; i++) {
|
|
|
|
struct free_block * block = &alloc->free_blocks[i];
|
|
|
|
// check if ptr is at the end of the block
|
2024-02-11 12:37:58 +00:00
|
|
|
if (block->offset + block->size == offset) {
|
2023-09-05 10:54:40 +00:00
|
|
|
block->size += size;
|
|
|
|
// check if we can merge with the next block
|
2024-02-11 12:37:58 +00:00
|
|
|
if (i < alloc->n_free_blocks - 1 && block->offset + block->size == alloc->free_blocks[i+1].offset) {
|
2023-09-05 10:54:40 +00:00
|
|
|
block->size += alloc->free_blocks[i+1].size;
|
|
|
|
alloc->n_free_blocks--;
|
|
|
|
for (int j = i+1; j < alloc->n_free_blocks; j++) {
|
|
|
|
alloc->free_blocks[j] = alloc->free_blocks[j+1];
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
// check if ptr is at the beginning of the block
|
2024-02-11 12:37:58 +00:00
|
|
|
if (offset + size == block->offset) {
|
|
|
|
block->offset = offset;
|
2023-09-05 10:54:40 +00:00
|
|
|
block->size += size;
|
|
|
|
// check if we can merge with the previous block
|
2024-02-11 12:37:58 +00:00
|
|
|
if (i > 0 && alloc->free_blocks[i-1].offset + alloc->free_blocks[i-1].size == block->offset) {
|
2023-09-05 10:54:40 +00:00
|
|
|
alloc->free_blocks[i-1].size += block->size;
|
|
|
|
alloc->n_free_blocks--;
|
|
|
|
for (int j = i; j < alloc->n_free_blocks; j++) {
|
|
|
|
alloc->free_blocks[j] = alloc->free_blocks[j+1];
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// otherwise, add a new block
|
|
|
|
GGML_ASSERT(alloc->n_free_blocks < MAX_FREE_BLOCKS && "out of free blocks");
|
|
|
|
// insert the new block in the correct position to keep the array sorted by address (to make merging blocks faster)
|
|
|
|
int insert_pos = 0;
|
2024-02-11 12:37:58 +00:00
|
|
|
while (insert_pos < alloc->n_free_blocks && alloc->free_blocks[insert_pos].offset < offset) {
|
2023-09-05 10:54:40 +00:00
|
|
|
insert_pos++;
|
|
|
|
}
|
|
|
|
// shift all blocks from insert_pos onward to make room for the new block
|
|
|
|
for (int i = alloc->n_free_blocks; i > insert_pos; i--) {
|
|
|
|
alloc->free_blocks[i] = alloc->free_blocks[i-1];
|
|
|
|
}
|
|
|
|
// insert the new block
|
2024-02-11 12:37:58 +00:00
|
|
|
alloc->free_blocks[insert_pos].offset = offset;
|
2023-09-05 10:54:40 +00:00
|
|
|
alloc->free_blocks[insert_pos].size = size;
|
|
|
|
alloc->n_free_blocks++;
|
2024-02-11 12:37:58 +00:00
|
|
|
|
|
|
|
GGML_UNUSED(tensor);
|
2023-09-05 10:54:40 +00:00
|
|
|
}
|
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
static void ggml_dyn_tallocr_reset(struct ggml_dyn_tallocr * alloc) {
|
2023-11-03 19:35:05 +00:00
|
|
|
alloc->n_free_blocks = 1;
|
2024-02-11 12:37:58 +00:00
|
|
|
alloc->free_blocks[0].offset = 0;
|
|
|
|
alloc->free_blocks[0].size = SIZE_MAX/2; // restrict maximum size of a measure allocator to half size_t max to avoid overflows
|
|
|
|
alloc->max_size = 0;
|
2023-09-05 10:54:40 +00:00
|
|
|
}
|
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
static struct ggml_dyn_tallocr * ggml_dyn_tallocr_new(size_t alignment) {
|
|
|
|
struct ggml_dyn_tallocr * alloc = (struct ggml_dyn_tallocr *)malloc(sizeof(struct ggml_dyn_tallocr));
|
2023-09-05 10:54:40 +00:00
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
*alloc = (struct ggml_dyn_tallocr) {
|
2023-09-05 10:54:40 +00:00
|
|
|
/*.alignment = */ alignment,
|
|
|
|
/*.n_free_blocks = */ 0,
|
|
|
|
/*.free_blocks = */ {{0}},
|
|
|
|
/*.max_size = */ 0,
|
|
|
|
#ifdef GGML_ALLOCATOR_DEBUG
|
2024-02-11 12:37:58 +00:00
|
|
|
/*.allocated_tensors = */ {{0}},
|
2023-09-05 10:54:40 +00:00
|
|
|
#endif
|
|
|
|
};
|
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
ggml_dyn_tallocr_reset(alloc);
|
2023-09-15 09:18:18 +00:00
|
|
|
|
2023-11-03 19:35:05 +00:00
|
|
|
return alloc;
|
2023-09-15 09:18:18 +00:00
|
|
|
}
|
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
static void ggml_dyn_tallocr_free(struct ggml_dyn_tallocr * alloc) {
|
2023-09-05 10:54:40 +00:00
|
|
|
free(alloc);
|
|
|
|
}
|
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
static size_t ggml_dyn_tallocr_max_size(struct ggml_dyn_tallocr * alloc) {
|
|
|
|
return alloc->max_size;
|
2023-09-05 10:54:40 +00:00
|
|
|
}
|
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
|
|
|
|
/////////////////////////////////////
|
2023-11-03 19:35:05 +00:00
|
|
|
|
|
|
|
// graph allocator
|
|
|
|
|
|
|
|
struct hash_node {
|
|
|
|
int n_children;
|
|
|
|
int n_views;
|
2024-02-11 12:37:58 +00:00
|
|
|
int buffer_id;
|
|
|
|
size_t offset; // offset within the buffer
|
|
|
|
bool allocated;
|
|
|
|
};
|
|
|
|
|
|
|
|
struct tensor_alloc {
|
2024-06-13 01:11:35 +00:00
|
|
|
int buffer_id;
|
2024-02-11 12:37:58 +00:00
|
|
|
size_t offset;
|
|
|
|
size_t size_max; // 0 = pre-allocated, unused, or view
|
|
|
|
};
|
|
|
|
|
2024-03-13 17:54:21 +00:00
|
|
|
struct leaf_alloc {
|
|
|
|
int buffer_id;
|
|
|
|
struct tensor_alloc leaf;
|
|
|
|
};
|
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
struct node_alloc {
|
|
|
|
struct tensor_alloc dst;
|
|
|
|
struct tensor_alloc src[GGML_MAX_SRC];
|
2023-11-03 19:35:05 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
struct ggml_gallocr {
|
2024-02-11 12:37:58 +00:00
|
|
|
ggml_backend_buffer_type_t * bufts; // [n_buffers]
|
|
|
|
ggml_backend_buffer_t * buffers; // [n_buffers]
|
|
|
|
struct ggml_dyn_tallocr ** buf_tallocs; // [n_buffers]
|
|
|
|
int n_buffers;
|
|
|
|
|
2023-11-03 19:35:05 +00:00
|
|
|
struct ggml_hash_set hash_set;
|
2024-02-11 12:37:58 +00:00
|
|
|
struct hash_node * hash_values; // [hash_set.size]
|
|
|
|
|
|
|
|
struct node_alloc * node_allocs; // [n_nodes]
|
|
|
|
int n_nodes;
|
2024-02-12 17:07:14 +00:00
|
|
|
|
2024-03-13 17:54:21 +00:00
|
|
|
struct leaf_alloc * leaf_allocs; // [n_leafs]
|
2024-02-12 17:07:14 +00:00
|
|
|
int n_leafs;
|
2023-11-03 19:35:05 +00:00
|
|
|
};
|
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
ggml_gallocr_t ggml_gallocr_new_n(ggml_backend_buffer_type_t * bufts, int n_bufs) {
|
2024-04-22 14:05:06 +00:00
|
|
|
ggml_gallocr_t galloc = (ggml_gallocr_t)calloc(1, sizeof(struct ggml_gallocr));
|
2024-02-11 12:37:58 +00:00
|
|
|
GGML_ASSERT(galloc != NULL);
|
|
|
|
|
2024-04-22 14:05:06 +00:00
|
|
|
galloc->bufts = calloc(n_bufs, sizeof(ggml_backend_buffer_type_t));
|
2024-02-11 12:37:58 +00:00
|
|
|
GGML_ASSERT(galloc->bufts != NULL);
|
|
|
|
|
2024-06-02 21:59:54 +00:00
|
|
|
galloc->buffers = calloc(n_bufs, sizeof(ggml_backend_buffer_t));
|
2024-02-11 12:37:58 +00:00
|
|
|
GGML_ASSERT(galloc->buffers != NULL);
|
|
|
|
|
2024-04-22 14:05:06 +00:00
|
|
|
galloc->buf_tallocs = calloc(n_bufs, sizeof(struct ggml_dyn_tallocr *));
|
2024-02-11 12:37:58 +00:00
|
|
|
GGML_ASSERT(galloc->buf_tallocs != NULL);
|
|
|
|
|
|
|
|
for (int i = 0; i < n_bufs; i++) {
|
|
|
|
galloc->bufts[i] = bufts[i];
|
|
|
|
galloc->buffers[i] = NULL;
|
2024-06-13 01:11:35 +00:00
|
|
|
|
|
|
|
// check if the same buffer type is used multiple times and reuse the same allocator
|
|
|
|
for (int j = 0; j < i; j++) {
|
|
|
|
if (bufts[i] == bufts[j]) {
|
|
|
|
galloc->buf_tallocs[i] = galloc->buf_tallocs[j];
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (galloc->buf_tallocs[i] == NULL) {
|
|
|
|
size_t alignment = ggml_backend_buft_get_alignment(bufts[i]);
|
|
|
|
galloc->buf_tallocs[i] = ggml_dyn_tallocr_new(alignment);
|
|
|
|
}
|
2024-02-11 12:37:58 +00:00
|
|
|
}
|
|
|
|
galloc->n_buffers = n_bufs;
|
2023-11-03 19:35:05 +00:00
|
|
|
|
|
|
|
return galloc;
|
|
|
|
}
|
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
ggml_gallocr_t ggml_gallocr_new(ggml_backend_buffer_type_t buft) {
|
|
|
|
return ggml_gallocr_new_n(&buft, 1);
|
|
|
|
}
|
|
|
|
|
2023-11-03 19:35:05 +00:00
|
|
|
void ggml_gallocr_free(ggml_gallocr_t galloc) {
|
|
|
|
if (galloc == NULL) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
for (int i = 0; i < galloc->n_buffers; i++) {
|
|
|
|
if (galloc->buffers != NULL) {
|
2024-06-13 01:11:35 +00:00
|
|
|
// skip if already freed
|
|
|
|
bool freed = false;
|
|
|
|
for (int j = 0; j < i; j++) {
|
|
|
|
if (galloc->buffers[j] == galloc->buffers[i]) {
|
|
|
|
freed = true;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (!freed) {
|
|
|
|
ggml_backend_buffer_free(galloc->buffers[i]);
|
|
|
|
}
|
2024-02-11 12:37:58 +00:00
|
|
|
}
|
|
|
|
if (galloc->buf_tallocs != NULL) {
|
2024-06-13 01:11:35 +00:00
|
|
|
// skip if already freed
|
|
|
|
bool freed = false;
|
|
|
|
for (int j = 0; j < i; j++) {
|
|
|
|
if (galloc->buf_tallocs[j] == galloc->buf_tallocs[i]) {
|
|
|
|
freed = true;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (!freed) {
|
|
|
|
ggml_dyn_tallocr_free(galloc->buf_tallocs[i]);
|
|
|
|
}
|
2024-02-11 12:37:58 +00:00
|
|
|
}
|
2023-11-03 19:35:05 +00:00
|
|
|
}
|
2024-02-11 12:37:58 +00:00
|
|
|
|
|
|
|
free(galloc->hash_set.keys);
|
|
|
|
free(galloc->hash_values);
|
|
|
|
free(galloc->bufts);
|
|
|
|
free(galloc->buffers);
|
|
|
|
free(galloc->buf_tallocs);
|
|
|
|
free(galloc->node_allocs);
|
2024-02-12 17:07:14 +00:00
|
|
|
free(galloc->leaf_allocs);
|
2023-11-03 19:35:05 +00:00
|
|
|
free(galloc);
|
|
|
|
}
|
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
typedef struct ggml_gallocr * ggml_gallocr_t;
|
2023-11-03 19:35:05 +00:00
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
static struct hash_node * ggml_gallocr_hash_get(ggml_gallocr_t galloc, struct ggml_tensor * t) {
|
2023-11-03 19:35:05 +00:00
|
|
|
size_t i = ggml_hash_find_or_insert(galloc->hash_set, t);
|
|
|
|
return &galloc->hash_values[i];
|
|
|
|
}
|
2023-09-05 10:54:40 +00:00
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
static bool ggml_gallocr_is_own(ggml_gallocr_t galloc, struct ggml_tensor * t) {
|
|
|
|
return ggml_gallocr_hash_get(galloc, t)->allocated;
|
2023-09-05 10:54:40 +00:00
|
|
|
}
|
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
static void ggml_gallocr_set_node_offset(ggml_gallocr_t galloc, struct ggml_tensor * node, int buffer_id, size_t offset) {
|
|
|
|
struct hash_node * hn = ggml_gallocr_hash_get(galloc, node);
|
|
|
|
hn->buffer_id = buffer_id;
|
|
|
|
hn->offset = offset;
|
|
|
|
hn->allocated = true;
|
2023-09-05 10:54:40 +00:00
|
|
|
}
|
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
static bool ggml_gallocr_is_allocated(ggml_gallocr_t galloc, struct ggml_tensor * t) {
|
|
|
|
return t->data != NULL || ggml_gallocr_hash_get(galloc, t)->allocated;
|
2023-11-03 19:35:05 +00:00
|
|
|
}
|
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
static void ggml_gallocr_allocate_node(ggml_gallocr_t galloc, struct ggml_tensor * node, int buffer_id) {
|
|
|
|
struct hash_node * hn = ggml_gallocr_hash_get(galloc, node);
|
2023-11-03 19:35:05 +00:00
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
if (!ggml_gallocr_is_allocated(galloc, node) && !ggml_is_view(node)) {
|
|
|
|
hn->allocated = true;
|
|
|
|
assert(hn->offset == 0);
|
2023-11-03 19:35:05 +00:00
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
// try to reuse a parent's buffer (inplace)
|
|
|
|
if (ggml_op_can_inplace(node->op)) {
|
|
|
|
for (int i = 0; i < GGML_MAX_SRC; i++) {
|
|
|
|
struct ggml_tensor * parent = node->src[i];
|
|
|
|
if (parent == NULL) {
|
2024-02-19 13:18:09 +00:00
|
|
|
continue;
|
2024-02-11 12:37:58 +00:00
|
|
|
}
|
2023-11-03 19:35:05 +00:00
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
// if the node's data is external, then we cannot re-use it
|
|
|
|
if (!ggml_gallocr_is_own(galloc, parent)) {
|
|
|
|
AT_PRINTF("not reusing parent %s for %s as %p is external\n", parent->name, node->name, parent->data);
|
|
|
|
continue;
|
|
|
|
}
|
2023-11-03 19:35:05 +00:00
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
// outputs cannot be reused
|
|
|
|
if (parent->flags & GGML_TENSOR_FLAG_OUTPUT || (parent->view_src != NULL && parent->view_src->flags & GGML_TENSOR_FLAG_OUTPUT)) {
|
|
|
|
AT_PRINTF("not reusing parent %s for %s as it is an output\n", parent->name, node->name);
|
|
|
|
continue;
|
|
|
|
}
|
2023-09-05 10:54:40 +00:00
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
if (!ggml_are_same_layout(node, parent)) {
|
|
|
|
AT_PRINTF("not reusing parent %s for %s as layouts are different\n", parent->name, node->name);
|
|
|
|
continue;
|
|
|
|
}
|
2023-09-05 10:54:40 +00:00
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
struct hash_node * p_hn = ggml_gallocr_hash_get(galloc, parent);
|
|
|
|
if (p_hn->n_children == 1 && p_hn->n_views == 0) {
|
|
|
|
if (ggml_is_view(parent)) {
|
|
|
|
struct ggml_tensor * view_src = parent->view_src;
|
|
|
|
struct hash_node * view_src_hn = ggml_gallocr_hash_get(galloc, view_src);
|
|
|
|
if (view_src_hn->n_views == 1 && view_src_hn->n_children == 0 && view_src->data == parent->data) {
|
|
|
|
AT_PRINTF("reusing view parent %s (%s) for %s\n", parent->name, view_src->name, node->name);
|
|
|
|
assert(view_src_hn->offset == p_hn->offset);
|
|
|
|
hn->buffer_id = p_hn->buffer_id;
|
|
|
|
hn->offset = p_hn->offset;
|
|
|
|
p_hn->allocated = false; // avoid freeing the parent
|
|
|
|
view_src_hn->allocated = false;
|
2023-09-05 10:54:40 +00:00
|
|
|
return;
|
|
|
|
}
|
2024-02-11 12:37:58 +00:00
|
|
|
} else {
|
|
|
|
AT_PRINTF("reusing parent %s for %s\n", parent->name, node->name);
|
|
|
|
hn->buffer_id = p_hn->buffer_id;
|
|
|
|
hn->offset = p_hn->offset;
|
|
|
|
p_hn->allocated = false; // avoid freeing the parent
|
|
|
|
return;
|
2023-09-05 10:54:40 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2024-02-11 12:37:58 +00:00
|
|
|
// allocate tensor from the buffer
|
|
|
|
struct ggml_dyn_tallocr * alloc = galloc->buf_tallocs[buffer_id];
|
|
|
|
ggml_backend_buffer_type_t buft = galloc->bufts[buffer_id];
|
|
|
|
size_t size = ggml_backend_buft_get_alloc_size(buft, node);
|
|
|
|
size_t offset = ggml_dyn_tallocr_alloc(alloc, size, node);
|
|
|
|
hn->buffer_id = buffer_id;
|
|
|
|
hn->offset = offset;
|
|
|
|
return;
|
2023-09-05 10:54:40 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2024-06-13 01:11:35 +00:00
|
|
|
static void ggml_gallocr_free_node(ggml_gallocr_t galloc, struct ggml_tensor * node) {
|
2024-02-11 12:37:58 +00:00
|
|
|
// graph outputs are never freed
|
|
|
|
if (node->flags & GGML_TENSOR_FLAG_OUTPUT) {
|
|
|
|
AT_PRINTF("not freeing output %s\n", node->name);
|
|
|
|
return;
|
|
|
|
}
|
2023-09-05 10:54:40 +00:00
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
struct hash_node * hn = ggml_gallocr_hash_get(galloc, node);
|
|
|
|
size_t offset = hn->offset;
|
2024-06-13 01:11:35 +00:00
|
|
|
int buffer_id = hn->buffer_id;
|
|
|
|
struct ggml_dyn_tallocr * alloc = galloc->buf_tallocs[buffer_id];
|
|
|
|
ggml_backend_buffer_type_t buft = galloc->bufts[buffer_id];
|
2024-02-11 12:37:58 +00:00
|
|
|
size_t size = ggml_backend_buft_get_alloc_size(buft, node);
|
|
|
|
ggml_dyn_tallocr_free_tensor(alloc, offset, size, node);
|
|
|
|
hn->allocated = false;
|
2023-11-03 19:35:05 +00:00
|
|
|
}
|
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
static int get_node_buffer_id(const int * node_buffer_ids, int i) {
|
|
|
|
return node_buffer_ids ? node_buffer_ids[i] : 0;
|
|
|
|
}
|
|
|
|
|
2024-03-13 17:54:21 +00:00
|
|
|
static void ggml_gallocr_alloc_graph_impl(ggml_gallocr_t galloc, struct ggml_cgraph * graph, const int * node_buffer_ids, const int * leaf_buffer_ids) {
|
2024-02-11 12:37:58 +00:00
|
|
|
// clear hash tables
|
|
|
|
memset(galloc->hash_set.keys, 0, galloc->hash_set.size * sizeof(struct ggml_tensor *));
|
|
|
|
memset(galloc->hash_values, 0, galloc->hash_set.size * sizeof(struct hash_node));
|
|
|
|
|
2024-03-13 17:54:21 +00:00
|
|
|
// allocate leafs
|
|
|
|
// these may be tensors that the application is not using in the graph, but may still want to allocate for other purposes
|
|
|
|
for (int i = 0; i < graph->n_leafs; i++) {
|
|
|
|
struct ggml_tensor * leaf = graph->leafs[i];
|
|
|
|
ggml_gallocr_allocate_node(galloc, leaf, get_node_buffer_id(leaf_buffer_ids, i));
|
|
|
|
}
|
|
|
|
|
2023-09-05 10:54:40 +00:00
|
|
|
// count number of children and views
|
2024-03-13 17:54:21 +00:00
|
|
|
// allocate other graph inputs and leafs first to avoid overwriting them
|
2024-02-11 12:37:58 +00:00
|
|
|
for (int i = 0; i < graph->n_nodes; i++) {
|
|
|
|
struct ggml_tensor * node = graph->nodes[i];
|
2023-11-03 19:35:05 +00:00
|
|
|
|
2024-03-27 16:55:10 +00:00
|
|
|
// TODO: better way to add external dependencies
|
|
|
|
// GGML_OP_NONE does not appear normally in the graph nodes, but is used by ggml-backend to add dependencies to
|
|
|
|
// control when some tensors are allocated and freed. in this case, the dependencies are in `src`, but the node
|
|
|
|
// itself is never used and should not be considered a dependency
|
|
|
|
if (ggml_is_view(node) && node->op != GGML_OP_NONE) {
|
2023-11-03 19:35:05 +00:00
|
|
|
struct ggml_tensor * view_src = node->view_src;
|
2024-02-11 12:37:58 +00:00
|
|
|
ggml_gallocr_hash_get(galloc, view_src)->n_views += 1;
|
2023-11-03 19:35:05 +00:00
|
|
|
}
|
2023-09-05 10:54:40 +00:00
|
|
|
|
2024-02-12 17:07:14 +00:00
|
|
|
if (node->flags & GGML_TENSOR_FLAG_INPUT) {
|
|
|
|
ggml_gallocr_allocate_node(galloc, graph->nodes[i], get_node_buffer_id(node_buffer_ids, i));
|
|
|
|
}
|
|
|
|
|
2023-11-03 19:35:05 +00:00
|
|
|
for (int j = 0; j < GGML_MAX_SRC; j++) {
|
2024-02-12 17:07:14 +00:00
|
|
|
struct ggml_tensor * src = node->src[j];
|
|
|
|
if (src == NULL) {
|
2024-02-19 13:18:09 +00:00
|
|
|
continue;
|
2023-09-05 10:54:40 +00:00
|
|
|
}
|
2024-02-12 17:07:14 +00:00
|
|
|
|
|
|
|
ggml_gallocr_hash_get(galloc, src)->n_children += 1;
|
|
|
|
|
2024-03-27 16:55:10 +00:00
|
|
|
// allocate explicit inputs
|
|
|
|
if (src->flags & GGML_TENSOR_FLAG_INPUT) {
|
2024-02-12 17:07:14 +00:00
|
|
|
ggml_gallocr_allocate_node(galloc, src, get_node_buffer_id(node_buffer_ids, i));
|
|
|
|
}
|
2023-11-03 19:35:05 +00:00
|
|
|
}
|
2024-02-12 17:07:14 +00:00
|
|
|
}
|
|
|
|
|
2023-11-03 19:35:05 +00:00
|
|
|
// allocate tensors
|
2024-02-11 12:37:58 +00:00
|
|
|
for (int i = 0; i < graph->n_nodes; i++) {
|
|
|
|
struct ggml_tensor * node = graph->nodes[i];
|
|
|
|
int buffer_id = get_node_buffer_id(node_buffer_ids, i);
|
|
|
|
|
|
|
|
// allocate parents (only leafs need to be allocated at this point)
|
|
|
|
for (int j = 0; j < GGML_MAX_SRC; j++) {
|
|
|
|
struct ggml_tensor * parent = node->src[j];
|
|
|
|
if (parent == NULL) {
|
2024-02-19 13:18:09 +00:00
|
|
|
continue;
|
2023-09-05 10:54:40 +00:00
|
|
|
}
|
2024-02-11 12:37:58 +00:00
|
|
|
ggml_gallocr_allocate_node(galloc, parent, buffer_id);
|
|
|
|
}
|
2023-09-05 10:54:40 +00:00
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
// allocate node
|
|
|
|
ggml_gallocr_allocate_node(galloc, node, buffer_id);
|
2023-11-03 19:35:05 +00:00
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
AT_PRINTF("exec: %s (%s) <= ", ggml_op_desc(node), node->name);
|
|
|
|
for (int j = 0; j < GGML_MAX_SRC; j++) {
|
|
|
|
struct ggml_tensor * parent = node->src[j];
|
|
|
|
if (parent == NULL) {
|
2024-02-19 13:18:09 +00:00
|
|
|
continue;
|
2024-02-11 12:37:58 +00:00
|
|
|
}
|
|
|
|
AT_PRINTF("%s", parent->name);
|
|
|
|
if (j < GGML_MAX_SRC - 1 && node->src[j + 1] != NULL) {
|
|
|
|
AT_PRINTF(", ");
|
2023-09-05 10:54:40 +00:00
|
|
|
}
|
|
|
|
}
|
2024-02-11 12:37:58 +00:00
|
|
|
AT_PRINTF("\n");
|
2023-09-05 10:54:40 +00:00
|
|
|
|
2023-11-03 19:35:05 +00:00
|
|
|
// update parents
|
2024-02-11 12:37:58 +00:00
|
|
|
for (int j = 0; j < GGML_MAX_SRC; j++) {
|
|
|
|
struct ggml_tensor * parent = node->src[j];
|
|
|
|
if (parent == NULL) {
|
2024-02-19 13:18:09 +00:00
|
|
|
continue;
|
2024-02-11 12:37:58 +00:00
|
|
|
}
|
|
|
|
struct hash_node * p_hn = ggml_gallocr_hash_get(galloc, parent);
|
|
|
|
p_hn->n_children -= 1;
|
|
|
|
|
|
|
|
AT_PRINTF("parent %s: %d children, %d views, allocated: %d\n",
|
|
|
|
parent->name, p_hn->n_children, p_hn->n_views, p_hn->allocated);
|
|
|
|
|
|
|
|
if (p_hn->n_children == 0 && p_hn->n_views == 0) {
|
|
|
|
if (ggml_is_view(parent)) {
|
|
|
|
struct ggml_tensor * view_src = parent->view_src;
|
|
|
|
struct hash_node * view_src_hn = ggml_gallocr_hash_get(galloc, view_src);
|
|
|
|
view_src_hn->n_views -= 1;
|
|
|
|
AT_PRINTF("view_src %s: %d children, %d views\n",
|
|
|
|
view_src->name, view_src_hn->n_children, view_src_hn->n_views);
|
|
|
|
if (view_src_hn->n_views == 0 && view_src_hn->n_children == 0 && view_src_hn->allocated) {
|
2024-06-13 01:11:35 +00:00
|
|
|
ggml_gallocr_free_node(galloc, view_src);
|
2023-09-05 10:54:40 +00:00
|
|
|
}
|
|
|
|
}
|
2024-02-11 12:37:58 +00:00
|
|
|
else if (p_hn->allocated) {
|
2024-06-13 01:11:35 +00:00
|
|
|
ggml_gallocr_free_node(galloc, parent);
|
2024-02-11 12:37:58 +00:00
|
|
|
}
|
2023-09-05 10:54:40 +00:00
|
|
|
}
|
2023-11-03 19:35:05 +00:00
|
|
|
AT_PRINTF("\n");
|
2023-09-05 10:54:40 +00:00
|
|
|
}
|
|
|
|
}
|
2023-11-03 19:35:05 +00:00
|
|
|
}
|
2023-09-05 10:54:40 +00:00
|
|
|
|
2024-03-13 17:54:21 +00:00
|
|
|
bool ggml_gallocr_reserve_n(ggml_gallocr_t galloc, struct ggml_cgraph * graph, const int * node_buffer_ids, const int * leaf_buffer_ids) {
|
2023-11-03 19:35:05 +00:00
|
|
|
size_t hash_size = graph->visited_hash_table.size;
|
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
// initialize hash table
|
2023-11-03 19:35:05 +00:00
|
|
|
if (galloc->hash_set.size < hash_size) {
|
2024-02-11 12:37:58 +00:00
|
|
|
free(galloc->hash_set.keys);
|
|
|
|
free(galloc->hash_values);
|
2023-11-03 19:35:05 +00:00
|
|
|
galloc->hash_set.size = hash_size;
|
2024-04-22 14:05:06 +00:00
|
|
|
galloc->hash_set.keys = calloc(hash_size, sizeof(struct ggml_tensor *));
|
|
|
|
galloc->hash_values = calloc(hash_size, sizeof(struct hash_node));
|
2024-02-11 12:37:58 +00:00
|
|
|
GGML_ASSERT(galloc->hash_set.keys != NULL);
|
|
|
|
GGML_ASSERT(galloc->hash_values != NULL);
|
|
|
|
} else {
|
|
|
|
// reset hash table
|
|
|
|
memset(galloc->hash_set.keys, 0, sizeof(struct ggml_tensor *) * galloc->hash_set.size);
|
|
|
|
memset(galloc->hash_values, 0, sizeof(struct hash_node) * galloc->hash_set.size);
|
2023-11-03 19:35:05 +00:00
|
|
|
}
|
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
// reset allocators
|
|
|
|
for (int i = 0; i < galloc->n_buffers; i++) {
|
|
|
|
ggml_dyn_tallocr_reset(galloc->buf_tallocs[i]);
|
|
|
|
}
|
2023-11-03 19:35:05 +00:00
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
// allocate in hash table
|
2024-03-13 17:54:21 +00:00
|
|
|
ggml_gallocr_alloc_graph_impl(galloc, graph, node_buffer_ids, leaf_buffer_ids);
|
2023-11-03 19:35:05 +00:00
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
// set the node_allocs from the hash table
|
|
|
|
if (galloc->n_nodes < graph->n_nodes) {
|
|
|
|
free(galloc->node_allocs);
|
2024-04-22 14:05:06 +00:00
|
|
|
galloc->node_allocs = calloc(graph->n_nodes, sizeof(struct node_alloc));
|
2024-02-11 12:37:58 +00:00
|
|
|
GGML_ASSERT(galloc->node_allocs != NULL);
|
2023-11-03 19:35:05 +00:00
|
|
|
}
|
2024-02-11 12:37:58 +00:00
|
|
|
galloc->n_nodes = graph->n_nodes;
|
|
|
|
for (int i = 0; i < graph->n_nodes; i++) {
|
|
|
|
struct ggml_tensor * node = graph->nodes[i];
|
|
|
|
struct node_alloc * node_alloc = &galloc->node_allocs[i];
|
|
|
|
if (node->view_src || node->data) {
|
2024-06-13 01:11:35 +00:00
|
|
|
node_alloc->dst.buffer_id = -1;
|
2024-02-11 12:37:58 +00:00
|
|
|
node_alloc->dst.offset = SIZE_MAX;
|
|
|
|
node_alloc->dst.size_max = 0;
|
|
|
|
} else {
|
|
|
|
struct hash_node * hn = ggml_gallocr_hash_get(galloc, node);
|
2024-06-13 01:11:35 +00:00
|
|
|
node_alloc->dst.buffer_id = hn->buffer_id;
|
|
|
|
node_alloc->dst.offset = hn->offset;
|
|
|
|
node_alloc->dst.size_max = ggml_backend_buft_get_alloc_size(galloc->bufts[hn->buffer_id], node);
|
2024-02-11 12:37:58 +00:00
|
|
|
}
|
|
|
|
for (int j = 0; j < GGML_MAX_SRC; j++) {
|
|
|
|
struct ggml_tensor * src = node->src[j];
|
|
|
|
if (!src || src->view_src || src->data) {
|
2024-06-13 01:11:35 +00:00
|
|
|
node_alloc->src[j].buffer_id = -1;
|
2024-02-11 12:37:58 +00:00
|
|
|
node_alloc->src[j].offset = SIZE_MAX;
|
|
|
|
node_alloc->src[j].size_max = 0;
|
|
|
|
} else {
|
|
|
|
struct hash_node * hn = ggml_gallocr_hash_get(galloc, src);
|
2024-06-13 01:11:35 +00:00
|
|
|
node_alloc->src[j].buffer_id = hn->buffer_id;
|
2024-02-11 12:37:58 +00:00
|
|
|
node_alloc->src[j].offset = hn->offset;
|
|
|
|
node_alloc->src[j].size_max = ggml_backend_buft_get_alloc_size(galloc->bufts[hn->buffer_id], src);
|
|
|
|
}
|
|
|
|
}
|
2023-11-03 19:35:05 +00:00
|
|
|
}
|
2024-02-12 17:07:14 +00:00
|
|
|
if (galloc->n_leafs < graph->n_leafs) {
|
|
|
|
free(galloc->leaf_allocs);
|
2024-04-22 14:05:06 +00:00
|
|
|
galloc->leaf_allocs = calloc(graph->n_leafs, sizeof(galloc->leaf_allocs[0]));
|
2024-02-12 17:07:14 +00:00
|
|
|
GGML_ASSERT(galloc->leaf_allocs != NULL);
|
|
|
|
}
|
|
|
|
galloc->n_leafs = graph->n_leafs;
|
|
|
|
for (int i = 0; i < graph->n_leafs; i++) {
|
|
|
|
struct ggml_tensor * leaf = graph->leafs[i];
|
|
|
|
struct hash_node * hn = ggml_gallocr_hash_get(galloc, leaf);
|
2024-03-13 17:54:21 +00:00
|
|
|
galloc->leaf_allocs[i].buffer_id = hn->buffer_id;
|
2024-03-16 14:47:14 +00:00
|
|
|
if (leaf->view_src || leaf->data) {
|
2024-06-13 01:11:35 +00:00
|
|
|
galloc->leaf_allocs[i].leaf.buffer_id = -1;
|
2024-03-16 14:47:14 +00:00
|
|
|
galloc->leaf_allocs[i].leaf.offset = SIZE_MAX;
|
|
|
|
galloc->leaf_allocs[i].leaf.size_max = 0;
|
|
|
|
} else {
|
2024-06-13 01:11:35 +00:00
|
|
|
galloc->leaf_allocs[i].leaf.buffer_id = hn->buffer_id;
|
2024-03-16 14:47:14 +00:00
|
|
|
galloc->leaf_allocs[i].leaf.offset = hn->offset;
|
|
|
|
galloc->leaf_allocs[i].leaf.size_max = ggml_backend_buft_get_alloc_size(galloc->bufts[hn->buffer_id], leaf);
|
|
|
|
}
|
2024-02-12 17:07:14 +00:00
|
|
|
}
|
2023-11-03 19:35:05 +00:00
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
// reallocate buffers if needed
|
|
|
|
for (int i = 0; i < galloc->n_buffers; i++) {
|
2024-06-13 01:11:35 +00:00
|
|
|
// if the buffer type is used multiple times, we reuse the same buffer
|
|
|
|
for (int j = 0; j < i; j++) {
|
|
|
|
if (galloc->buf_tallocs[j] == galloc->buf_tallocs[i]) {
|
|
|
|
galloc->buffers[i] = galloc->buffers[j];
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
size_t cur_size = galloc->buffers[i] ? ggml_backend_buffer_get_size(galloc->buffers[i]) : 0;
|
|
|
|
size_t new_size = ggml_dyn_tallocr_max_size(galloc->buf_tallocs[i]);
|
2023-11-03 19:35:05 +00:00
|
|
|
|
2024-03-13 17:54:21 +00:00
|
|
|
// even if there are no tensors allocated in this buffer, we still need to allocate it to initialize views
|
|
|
|
if (new_size > cur_size || galloc->buffers[i] == NULL) {
|
2024-02-11 12:37:58 +00:00
|
|
|
#ifndef NDEBUG
|
|
|
|
fprintf(stderr, "%s: reallocating %s buffer from size %.02f MiB to %.02f MiB\n", __func__, ggml_backend_buft_name(galloc->bufts[i]), cur_size / 1024.0 / 1024.0, new_size / 1024.0 / 1024.0);
|
|
|
|
#endif
|
2024-06-13 01:11:35 +00:00
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
ggml_backend_buffer_free(galloc->buffers[i]);
|
|
|
|
galloc->buffers[i] = ggml_backend_buft_alloc_buffer(galloc->bufts[i], new_size);
|
|
|
|
if (galloc->buffers[i] == NULL) {
|
|
|
|
fprintf(stderr, "%s: failed to allocate %s buffer of size %zu\n", __func__, ggml_backend_buft_name(galloc->bufts[i]), new_size);
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2023-11-03 19:35:05 +00:00
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
return true;
|
2023-11-03 19:35:05 +00:00
|
|
|
}
|
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
bool ggml_gallocr_reserve(ggml_gallocr_t galloc, struct ggml_cgraph *graph) {
|
2024-03-13 17:54:21 +00:00
|
|
|
return ggml_gallocr_reserve_n(galloc, graph, NULL, NULL);
|
2023-11-03 19:35:05 +00:00
|
|
|
}
|
|
|
|
|
2024-06-13 01:11:35 +00:00
|
|
|
static void ggml_gallocr_init_tensor(ggml_gallocr_t galloc, struct ggml_tensor * tensor, struct tensor_alloc * tensor_alloc) {
|
|
|
|
int buffer_id = tensor_alloc->buffer_id;
|
2024-03-13 17:54:21 +00:00
|
|
|
assert(tensor->data || tensor->view_src || ggml_backend_buffer_get_alloc_size(galloc->buffers[buffer_id], tensor) <= tensor_alloc->size_max);
|
2023-11-03 19:35:05 +00:00
|
|
|
|
2024-03-13 17:54:21 +00:00
|
|
|
if (tensor->view_src != NULL) {
|
|
|
|
if (tensor->buffer == NULL) {
|
2024-02-11 12:37:58 +00:00
|
|
|
assert(tensor_alloc->offset == SIZE_MAX);
|
2024-03-13 17:54:21 +00:00
|
|
|
if (tensor->view_src->buffer == NULL) {
|
2024-02-11 12:37:58 +00:00
|
|
|
// this tensor was allocated without ggml-backend
|
|
|
|
return;
|
|
|
|
}
|
2024-06-03 17:03:26 +00:00
|
|
|
ggml_backend_view_init(tensor);
|
2024-02-11 12:37:58 +00:00
|
|
|
}
|
|
|
|
} else {
|
2024-03-13 17:54:21 +00:00
|
|
|
if (tensor->data == NULL) {
|
2024-02-11 12:37:58 +00:00
|
|
|
assert(tensor_alloc->offset != SIZE_MAX);
|
2024-03-13 17:54:21 +00:00
|
|
|
assert(ggml_backend_buffer_get_alloc_size(galloc->buffers[buffer_id], tensor) <= tensor_alloc->size_max);
|
2024-02-12 17:07:14 +00:00
|
|
|
void * base = ggml_backend_buffer_get_base(galloc->buffers[buffer_id]);
|
2024-02-11 12:37:58 +00:00
|
|
|
void * addr = (char *)base + tensor_alloc->offset;
|
2024-03-13 17:54:21 +00:00
|
|
|
ggml_backend_tensor_alloc(galloc->buffers[buffer_id], tensor, addr);
|
2024-02-11 12:37:58 +00:00
|
|
|
} else {
|
2024-03-13 17:54:21 +00:00
|
|
|
if (tensor->buffer == NULL) {
|
2024-02-11 12:37:58 +00:00
|
|
|
// this tensor was allocated without ggml-backend
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2023-11-03 19:35:05 +00:00
|
|
|
}
|
|
|
|
|
2024-06-13 01:11:35 +00:00
|
|
|
static bool ggml_gallocr_node_needs_realloc(ggml_gallocr_t galloc, struct ggml_tensor * node, struct tensor_alloc * talloc) {
|
|
|
|
ggml_backend_buffer_type_t buft = talloc->buffer_id != -1 ? galloc->bufts[talloc->buffer_id] : NULL;
|
2024-02-11 12:37:58 +00:00
|
|
|
size_t node_size = (node->data || node->view_src) ? 0 : ggml_backend_buft_get_alloc_size(buft, node);
|
|
|
|
return talloc->size_max >= node_size;
|
2023-11-03 19:35:05 +00:00
|
|
|
}
|
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
static bool ggml_gallocr_needs_realloc(ggml_gallocr_t galloc, struct ggml_cgraph * graph) {
|
|
|
|
if (galloc->n_nodes != graph->n_nodes) {
|
|
|
|
#ifndef NDEBUG
|
|
|
|
fprintf(stderr, "%s: graph has different number of nodes\n", __func__);
|
|
|
|
#endif
|
|
|
|
return true;
|
|
|
|
}
|
2023-11-03 19:35:05 +00:00
|
|
|
|
2024-02-12 17:07:14 +00:00
|
|
|
if (galloc->n_leafs != graph->n_leafs) {
|
|
|
|
#ifndef NDEBUG
|
|
|
|
fprintf(stderr, "%s: graph has different number of leafs\n", __func__);
|
|
|
|
#endif
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
for (int i = 0; i < graph->n_nodes; i++) {
|
|
|
|
struct ggml_tensor * node = graph->nodes[i];
|
|
|
|
struct node_alloc * node_alloc = &galloc->node_allocs[i];
|
2023-11-03 19:35:05 +00:00
|
|
|
|
2024-06-13 01:11:35 +00:00
|
|
|
if (!ggml_gallocr_node_needs_realloc(galloc, node, &node_alloc->dst)) {
|
2024-02-11 12:37:58 +00:00
|
|
|
#ifndef NDEBUG
|
|
|
|
fprintf(stderr, "%s: node %s is not valid\n", __func__, node->name);
|
|
|
|
#endif
|
|
|
|
return true;
|
|
|
|
}
|
2023-11-03 19:35:05 +00:00
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
for (int j = 0; j < GGML_MAX_SRC; j++) {
|
|
|
|
struct ggml_tensor * src = node->src[j];
|
|
|
|
if (src == NULL) {
|
2024-02-19 13:18:09 +00:00
|
|
|
continue;
|
2024-02-11 12:37:58 +00:00
|
|
|
}
|
2024-06-13 01:11:35 +00:00
|
|
|
if (!ggml_gallocr_node_needs_realloc(galloc, src, &node_alloc->src[j])) {
|
2024-02-11 12:37:58 +00:00
|
|
|
#ifndef NDEBUG
|
|
|
|
fprintf(stderr, "%s: src %d (%s) of node %s is not valid\n", __func__, j, src->name, node->name);
|
|
|
|
#endif
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
}
|
2023-12-22 15:53:39 +00:00
|
|
|
}
|
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
return false;
|
2023-11-03 19:35:05 +00:00
|
|
|
}
|
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
bool ggml_gallocr_alloc_graph(ggml_gallocr_t galloc, struct ggml_cgraph * graph) {
|
|
|
|
if (ggml_gallocr_needs_realloc(galloc, graph)) {
|
|
|
|
if (galloc->n_buffers == 1) {
|
|
|
|
#ifndef NDEBUG
|
|
|
|
fprintf(stderr, "%s: reallocating buffers automatically\n", __func__);
|
|
|
|
#endif
|
|
|
|
if (!ggml_gallocr_reserve(galloc, graph)) {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
#ifndef NDEBUG
|
|
|
|
fprintf(stderr, "%s: cannot reallocate multi buffer graph automatically, call reserve\n", __func__);
|
|
|
|
#endif
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
2023-11-03 19:35:05 +00:00
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
// reset buffers
|
|
|
|
for (int i = 0; i < galloc->n_buffers; i++) {
|
|
|
|
if (galloc->buffers[i] != NULL) {
|
|
|
|
ggml_backend_buffer_reset(galloc->buffers[i]);
|
|
|
|
}
|
|
|
|
}
|
2023-11-03 19:35:05 +00:00
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
// allocate the graph tensors from the previous assignments
|
2024-03-13 17:54:21 +00:00
|
|
|
// leafs
|
|
|
|
for (int i = 0; i < graph->n_leafs; i++) {
|
|
|
|
struct ggml_tensor * leaf = graph->leafs[i];
|
|
|
|
struct leaf_alloc * leaf_alloc = &galloc->leaf_allocs[i];
|
2024-06-13 01:11:35 +00:00
|
|
|
ggml_gallocr_init_tensor(galloc, leaf, &leaf_alloc->leaf);
|
2024-03-13 17:54:21 +00:00
|
|
|
}
|
2024-02-12 17:07:14 +00:00
|
|
|
// nodes
|
2024-02-11 12:37:58 +00:00
|
|
|
for (int i = 0; i < graph->n_nodes; i++) {
|
|
|
|
struct ggml_tensor * node = graph->nodes[i];
|
|
|
|
struct node_alloc * node_alloc = &galloc->node_allocs[i];
|
|
|
|
for (int j = 0; j < GGML_MAX_SRC; j++) {
|
|
|
|
struct ggml_tensor * src = node->src[j];
|
|
|
|
if (src == NULL) {
|
2024-02-19 13:18:09 +00:00
|
|
|
continue;
|
2024-02-11 12:37:58 +00:00
|
|
|
}
|
2024-06-13 01:11:35 +00:00
|
|
|
ggml_gallocr_init_tensor(galloc, src, &node_alloc->src[j]);
|
2024-02-11 12:37:58 +00:00
|
|
|
}
|
2024-06-13 01:11:35 +00:00
|
|
|
ggml_gallocr_init_tensor(galloc, node, &node_alloc->dst);
|
2024-02-12 17:07:14 +00:00
|
|
|
}
|
2023-11-03 19:35:05 +00:00
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
return true;
|
2023-09-05 10:54:40 +00:00
|
|
|
}
|
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
size_t ggml_gallocr_get_buffer_size(ggml_gallocr_t galloc, int buffer_id) {
|
|
|
|
GGML_ASSERT(buffer_id >= 0 && buffer_id < galloc->n_buffers);
|
|
|
|
|
|
|
|
if (galloc->buffers[buffer_id] == NULL) {
|
|
|
|
return 0;
|
|
|
|
}
|
2024-06-13 01:11:35 +00:00
|
|
|
|
|
|
|
for (int i = 0; i < buffer_id; i++) {
|
|
|
|
if (galloc->buffers[i] == galloc->buffers[buffer_id]) {
|
|
|
|
// this buffer is the same as a previous one due to the same buffer type being used multiple times
|
|
|
|
// only return the buffer size the first time it appears to avoid double counting
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2024-02-11 12:37:58 +00:00
|
|
|
return ggml_backend_buffer_get_size(galloc->buffers[buffer_id]);
|
2023-09-05 10:54:40 +00:00
|
|
|
}
|
2023-12-07 20:27:19 +00:00
|
|
|
|
|
|
|
// utils
|
|
|
|
|
ggml : add Vulkan backend (llama/2059)
* Vulkan loader code
* Fix matmul kernel, continue implementation
* Continue implementation
* Vulkan memory management
* Vulkan development
* Matmul call
* Add aligned malloc and free for VMA
* Continue implementation
* First matmul success
* GEMM Kernel optimization
* 1D Blocktiling
* 2D Blocktiling
* Write coalescing
* Continue vulkan implementation and optimization
* First FP16 attempt, disabled for now
* Code abstraction, FP16 implementation, fix kernel, add FP16 to FP32 kernel
* Enable device extensions properly, restore fp16 matmul op
* Fix mulmat_f16
* Output FP32 in fp16 matmul shader
* Fix f16_to_f32 kernel
* dequant_q4_0 kernel
* Add VMA library
* Avoid requesting dedicated memory, VMA can decide that by itself
* Add bounds checking to matmul kernels, improve implementation, fix command buffers not freed properly
* add cmake commands
* Add 2d write operation, profiling code
* Fix 2d write
* Fix queue selection for AMD RADV
* Fix trailing whitespace in vk_mem_alloc.h
* Add WIP warp tile mat mul shaders
* Disable glslc optimization
* Disable glslc optimization for CMake
* Optimize warptile matmul shader, replace blocktile with it
* Add split-k optimization for small matrix multiplication
Use semaphores for synchronization instead of fences or waitidle
Rework async write/read for synchronization
* Fix validation errors, improve compatibility with AMD GPUs
* Rework command buffer handling
* Variable matmul kernel using specialization constants
* Fix synchronization on AMD, add barriers for buffer ownership transfer, add debug flag and prints
* Reuse semaphores
* Handle stage flags during command buffer submission properly
* Increase matmul test runs for consistent results
* Fix F32 matmul
* Add vectorized loading and zeropadding for matrix multiplication
* Use pinned memory for f16 preprocessing
* Don't force aligned matmul
* Don't free before queue done
* Replace VMA library with native Vulkan buffer management
* Basic offloading support with mul_f32 and dmmv for q4_0
* Run glslc commands in parallel
* Unroll loops in dmmv shader
* Reduce usage of waitIdle
* Reuse pinned allocation for f16 conversion
* Handle devices with only a single queue
* Fix trailing whitespace in CMakeLists.txt
* Allow parallel execution of kernels, parallelize third and fourth dimension calls
* Add fallback for devices only supporting one DescriptorSet per DescriptorPool
* Move to graph function similar to CUDA implementation
* Use F16 kernel for most things, replace q_f32 with mul_mat_q_f16 function
* Add F32 dmmv shaders
* Batch submissions
* Add .spv to gitignore
* Split off matrix vector multiplication for separate optimization
* Use single command buffer for matrix vector multiplication ops
* Reduce overhead of mul_f32 calls by using a single command buffer
* Add submission batching to mul_f32
* Fix tests
* Add missing barrier
* Add further missing barrier
* Add further ops
* Replace vk::QueueFamilyIgnored with VK_QUEUE_FAMILY_IGNORED to support more Vulkan header versions
* Remove unnecessary cblas link
* Fix descriptor set pre-allocation assert
* Add runtime shader compilation, start transferring shaders to this approach
* Transfer remaining shaders to header and compile on runtime
* Fix fp32 fallback if device doesn't support fp16, add force disable env var GGML_VULKAN_DISABLE_F16
* Add support for q4_1, q5_0, q5_1 and q8_0
* Remove unnecessary scalar layout extension
* Parse graph early to pre-record command buffers
* Add q6_k support
* Add multi-submit for command buffers
* Fix q6_k dequant shader for AMD
* Fix q6_k for GPUs without fp16 support
* Simplify q6_k fp16 fix
* Minor fixes
* Fix wg_denom of m-mulmat shaders
* Add Python-based Vulkan shader generator
* Replace shaderc dependency with precompiled shaders
Fix python script to generate shaders
* Clean up code
* Fix shader generator script Windows compatibility
Co-authored-by: Concedo <39025047+LostRuins@users.noreply.github.com>
* Close file before deletion
* Fix vulkan shader fp32 name
* Add q2_k and q3_k support
Add validation check to compare shader results to cpu results
* Add q4_k support
* Add q5_k support
* Bake SPIR-V bytecode into the library instead of loading shaders from file
* Switch to signal semaphores for flexibility
Prepare broadcasting support for mul mat
* Finish broadcasting mul mat support for GQA
* Clean up unused functions
Add repeat op
* Add further ops, not yet enabled. Improve semaphore code
* Reduce number of used semaphores by utilizing timelines more properly
* Remove queue information
* Reuse timeline semaphores, allow parallel operation with binary semaphores to work around nvidia driver limitations
* Add Vulkan to llama-bench
* Remove cblas dependency
* Fix matmul k-split bug
* Fix q4_k dmmv K_QUANTS_PER_ITERATION 1 shader
* Add RMS Norm shader, rework op_f32 shader setup, fix matmul bug
* Fix issues with float16 overflows in shaders
* Fix issues with older Vulkan headers on Ubuntu 22.04
* Allow multi-op partial offloading by parsing the graph to preallocate enough between-op buffers
* Implement further ops, rework op_f32 calls, fix bugs
* Finish full offloading support, add last remaining ops, fix bugs, remove redundant code
* Upload generated file ggml-vulkan-shaders.hpp, remove redundant shaders
* Merge upstream changes, fix conflicts, adapt soft_max op
* Fix Python and shader header format
* Free model gpu buffers on exit
* Use single queue per device to simplify code
* Add matmul shader support for running multiple calculations in parallel
* Switch from semaphore-synchronized multiple command buffers per op to single command buffer for multiple ops, whole graph if possible
* Fix missing event cast
* Replace uint64_t(-1) with UINT64_MAX, rename function for clarity
* Fix warning about empty C function parameters
* Fix compiler warnings
* Properly implement Vulkan backend buffer handling
* Fix oversized host staging buffers
* Simplify barrier synchronization calls
* Fix gcc warnings
* Implement max_size for backend buffer types to limit the size of a single allocation
* Use min of maxMemoryAllocationSize and maxBufferSize for device max allocation size
* refactor multi buf
* Disable unsupported ops to fix tests
* Check for maintenance4 support before using it
* Handle devices with only a single queue
* Fix single queue logic
* propagate buffer usage in multi buffers
* Implement rope_neox op
* Cleanup header and other files
* Simplify gpu_extras by removing events and putting staging memcpys into contexts
* Move queue into context
Add not-yet-enabled async backend ops
* Simplify context use, optimize matmul shader for warp size 64 (AMD GCN), fix split_k matmul shader optimization
* Add get_max_size to SYCL backend.
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* llama : fix trailing whitespace
---------
Co-authored-by: Henri Vasserman <henv@hot.ee>
Co-authored-by: Concedo <39025047+LostRuins@users.noreply.github.com>
Co-authored-by: slaren <slarengh@gmail.com>
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
2024-01-28 17:03:59 +00:00
|
|
|
static bool alloc_tensor_range(struct ggml_context * ctx,
|
|
|
|
struct ggml_tensor * first, struct ggml_tensor * last,
|
|
|
|
ggml_backend_buffer_type_t buft, size_t size,
|
|
|
|
ggml_backend_buffer_t ** buffers, size_t * n_buffers) {
|
|
|
|
ggml_backend_buffer_t buffer = ggml_backend_buft_alloc_buffer(buft, size);
|
2024-01-12 19:07:38 +00:00
|
|
|
if (buffer == NULL) {
|
|
|
|
#ifndef NDEBUG
|
ggml : add Vulkan backend (llama/2059)
* Vulkan loader code
* Fix matmul kernel, continue implementation
* Continue implementation
* Vulkan memory management
* Vulkan development
* Matmul call
* Add aligned malloc and free for VMA
* Continue implementation
* First matmul success
* GEMM Kernel optimization
* 1D Blocktiling
* 2D Blocktiling
* Write coalescing
* Continue vulkan implementation and optimization
* First FP16 attempt, disabled for now
* Code abstraction, FP16 implementation, fix kernel, add FP16 to FP32 kernel
* Enable device extensions properly, restore fp16 matmul op
* Fix mulmat_f16
* Output FP32 in fp16 matmul shader
* Fix f16_to_f32 kernel
* dequant_q4_0 kernel
* Add VMA library
* Avoid requesting dedicated memory, VMA can decide that by itself
* Add bounds checking to matmul kernels, improve implementation, fix command buffers not freed properly
* add cmake commands
* Add 2d write operation, profiling code
* Fix 2d write
* Fix queue selection for AMD RADV
* Fix trailing whitespace in vk_mem_alloc.h
* Add WIP warp tile mat mul shaders
* Disable glslc optimization
* Disable glslc optimization for CMake
* Optimize warptile matmul shader, replace blocktile with it
* Add split-k optimization for small matrix multiplication
Use semaphores for synchronization instead of fences or waitidle
Rework async write/read for synchronization
* Fix validation errors, improve compatibility with AMD GPUs
* Rework command buffer handling
* Variable matmul kernel using specialization constants
* Fix synchronization on AMD, add barriers for buffer ownership transfer, add debug flag and prints
* Reuse semaphores
* Handle stage flags during command buffer submission properly
* Increase matmul test runs for consistent results
* Fix F32 matmul
* Add vectorized loading and zeropadding for matrix multiplication
* Use pinned memory for f16 preprocessing
* Don't force aligned matmul
* Don't free before queue done
* Replace VMA library with native Vulkan buffer management
* Basic offloading support with mul_f32 and dmmv for q4_0
* Run glslc commands in parallel
* Unroll loops in dmmv shader
* Reduce usage of waitIdle
* Reuse pinned allocation for f16 conversion
* Handle devices with only a single queue
* Fix trailing whitespace in CMakeLists.txt
* Allow parallel execution of kernels, parallelize third and fourth dimension calls
* Add fallback for devices only supporting one DescriptorSet per DescriptorPool
* Move to graph function similar to CUDA implementation
* Use F16 kernel for most things, replace q_f32 with mul_mat_q_f16 function
* Add F32 dmmv shaders
* Batch submissions
* Add .spv to gitignore
* Split off matrix vector multiplication for separate optimization
* Use single command buffer for matrix vector multiplication ops
* Reduce overhead of mul_f32 calls by using a single command buffer
* Add submission batching to mul_f32
* Fix tests
* Add missing barrier
* Add further missing barrier
* Add further ops
* Replace vk::QueueFamilyIgnored with VK_QUEUE_FAMILY_IGNORED to support more Vulkan header versions
* Remove unnecessary cblas link
* Fix descriptor set pre-allocation assert
* Add runtime shader compilation, start transferring shaders to this approach
* Transfer remaining shaders to header and compile on runtime
* Fix fp32 fallback if device doesn't support fp16, add force disable env var GGML_VULKAN_DISABLE_F16
* Add support for q4_1, q5_0, q5_1 and q8_0
* Remove unnecessary scalar layout extension
* Parse graph early to pre-record command buffers
* Add q6_k support
* Add multi-submit for command buffers
* Fix q6_k dequant shader for AMD
* Fix q6_k for GPUs without fp16 support
* Simplify q6_k fp16 fix
* Minor fixes
* Fix wg_denom of m-mulmat shaders
* Add Python-based Vulkan shader generator
* Replace shaderc dependency with precompiled shaders
Fix python script to generate shaders
* Clean up code
* Fix shader generator script Windows compatibility
Co-authored-by: Concedo <39025047+LostRuins@users.noreply.github.com>
* Close file before deletion
* Fix vulkan shader fp32 name
* Add q2_k and q3_k support
Add validation check to compare shader results to cpu results
* Add q4_k support
* Add q5_k support
* Bake SPIR-V bytecode into the library instead of loading shaders from file
* Switch to signal semaphores for flexibility
Prepare broadcasting support for mul mat
* Finish broadcasting mul mat support for GQA
* Clean up unused functions
Add repeat op
* Add further ops, not yet enabled. Improve semaphore code
* Reduce number of used semaphores by utilizing timelines more properly
* Remove queue information
* Reuse timeline semaphores, allow parallel operation with binary semaphores to work around nvidia driver limitations
* Add Vulkan to llama-bench
* Remove cblas dependency
* Fix matmul k-split bug
* Fix q4_k dmmv K_QUANTS_PER_ITERATION 1 shader
* Add RMS Norm shader, rework op_f32 shader setup, fix matmul bug
* Fix issues with float16 overflows in shaders
* Fix issues with older Vulkan headers on Ubuntu 22.04
* Allow multi-op partial offloading by parsing the graph to preallocate enough between-op buffers
* Implement further ops, rework op_f32 calls, fix bugs
* Finish full offloading support, add last remaining ops, fix bugs, remove redundant code
* Upload generated file ggml-vulkan-shaders.hpp, remove redundant shaders
* Merge upstream changes, fix conflicts, adapt soft_max op
* Fix Python and shader header format
* Free model gpu buffers on exit
* Use single queue per device to simplify code
* Add matmul shader support for running multiple calculations in parallel
* Switch from semaphore-synchronized multiple command buffers per op to single command buffer for multiple ops, whole graph if possible
* Fix missing event cast
* Replace uint64_t(-1) with UINT64_MAX, rename function for clarity
* Fix warning about empty C function parameters
* Fix compiler warnings
* Properly implement Vulkan backend buffer handling
* Fix oversized host staging buffers
* Simplify barrier synchronization calls
* Fix gcc warnings
* Implement max_size for backend buffer types to limit the size of a single allocation
* Use min of maxMemoryAllocationSize and maxBufferSize for device max allocation size
* refactor multi buf
* Disable unsupported ops to fix tests
* Check for maintenance4 support before using it
* Handle devices with only a single queue
* Fix single queue logic
* propagate buffer usage in multi buffers
* Implement rope_neox op
* Cleanup header and other files
* Simplify gpu_extras by removing events and putting staging memcpys into contexts
* Move queue into context
Add not-yet-enabled async backend ops
* Simplify context use, optimize matmul shader for warp size 64 (AMD GCN), fix split_k matmul shader optimization
* Add get_max_size to SYCL backend.
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* llama : fix trailing whitespace
---------
Co-authored-by: Henri Vasserman <henv@hot.ee>
Co-authored-by: Concedo <39025047+LostRuins@users.noreply.github.com>
Co-authored-by: slaren <slarengh@gmail.com>
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
2024-01-28 17:03:59 +00:00
|
|
|
fprintf(stderr, "%s: failed to allocate %s buffer of size %zu\n", __func__, ggml_backend_buft_name(buft), size);
|
2024-01-12 19:07:38 +00:00
|
|
|
#endif
|
ggml : add Vulkan backend (llama/2059)
* Vulkan loader code
* Fix matmul kernel, continue implementation
* Continue implementation
* Vulkan memory management
* Vulkan development
* Matmul call
* Add aligned malloc and free for VMA
* Continue implementation
* First matmul success
* GEMM Kernel optimization
* 1D Blocktiling
* 2D Blocktiling
* Write coalescing
* Continue vulkan implementation and optimization
* First FP16 attempt, disabled for now
* Code abstraction, FP16 implementation, fix kernel, add FP16 to FP32 kernel
* Enable device extensions properly, restore fp16 matmul op
* Fix mulmat_f16
* Output FP32 in fp16 matmul shader
* Fix f16_to_f32 kernel
* dequant_q4_0 kernel
* Add VMA library
* Avoid requesting dedicated memory, VMA can decide that by itself
* Add bounds checking to matmul kernels, improve implementation, fix command buffers not freed properly
* add cmake commands
* Add 2d write operation, profiling code
* Fix 2d write
* Fix queue selection for AMD RADV
* Fix trailing whitespace in vk_mem_alloc.h
* Add WIP warp tile mat mul shaders
* Disable glslc optimization
* Disable glslc optimization for CMake
* Optimize warptile matmul shader, replace blocktile with it
* Add split-k optimization for small matrix multiplication
Use semaphores for synchronization instead of fences or waitidle
Rework async write/read for synchronization
* Fix validation errors, improve compatibility with AMD GPUs
* Rework command buffer handling
* Variable matmul kernel using specialization constants
* Fix synchronization on AMD, add barriers for buffer ownership transfer, add debug flag and prints
* Reuse semaphores
* Handle stage flags during command buffer submission properly
* Increase matmul test runs for consistent results
* Fix F32 matmul
* Add vectorized loading and zeropadding for matrix multiplication
* Use pinned memory for f16 preprocessing
* Don't force aligned matmul
* Don't free before queue done
* Replace VMA library with native Vulkan buffer management
* Basic offloading support with mul_f32 and dmmv for q4_0
* Run glslc commands in parallel
* Unroll loops in dmmv shader
* Reduce usage of waitIdle
* Reuse pinned allocation for f16 conversion
* Handle devices with only a single queue
* Fix trailing whitespace in CMakeLists.txt
* Allow parallel execution of kernels, parallelize third and fourth dimension calls
* Add fallback for devices only supporting one DescriptorSet per DescriptorPool
* Move to graph function similar to CUDA implementation
* Use F16 kernel for most things, replace q_f32 with mul_mat_q_f16 function
* Add F32 dmmv shaders
* Batch submissions
* Add .spv to gitignore
* Split off matrix vector multiplication for separate optimization
* Use single command buffer for matrix vector multiplication ops
* Reduce overhead of mul_f32 calls by using a single command buffer
* Add submission batching to mul_f32
* Fix tests
* Add missing barrier
* Add further missing barrier
* Add further ops
* Replace vk::QueueFamilyIgnored with VK_QUEUE_FAMILY_IGNORED to support more Vulkan header versions
* Remove unnecessary cblas link
* Fix descriptor set pre-allocation assert
* Add runtime shader compilation, start transferring shaders to this approach
* Transfer remaining shaders to header and compile on runtime
* Fix fp32 fallback if device doesn't support fp16, add force disable env var GGML_VULKAN_DISABLE_F16
* Add support for q4_1, q5_0, q5_1 and q8_0
* Remove unnecessary scalar layout extension
* Parse graph early to pre-record command buffers
* Add q6_k support
* Add multi-submit for command buffers
* Fix q6_k dequant shader for AMD
* Fix q6_k for GPUs without fp16 support
* Simplify q6_k fp16 fix
* Minor fixes
* Fix wg_denom of m-mulmat shaders
* Add Python-based Vulkan shader generator
* Replace shaderc dependency with precompiled shaders
Fix python script to generate shaders
* Clean up code
* Fix shader generator script Windows compatibility
Co-authored-by: Concedo <39025047+LostRuins@users.noreply.github.com>
* Close file before deletion
* Fix vulkan shader fp32 name
* Add q2_k and q3_k support
Add validation check to compare shader results to cpu results
* Add q4_k support
* Add q5_k support
* Bake SPIR-V bytecode into the library instead of loading shaders from file
* Switch to signal semaphores for flexibility
Prepare broadcasting support for mul mat
* Finish broadcasting mul mat support for GQA
* Clean up unused functions
Add repeat op
* Add further ops, not yet enabled. Improve semaphore code
* Reduce number of used semaphores by utilizing timelines more properly
* Remove queue information
* Reuse timeline semaphores, allow parallel operation with binary semaphores to work around nvidia driver limitations
* Add Vulkan to llama-bench
* Remove cblas dependency
* Fix matmul k-split bug
* Fix q4_k dmmv K_QUANTS_PER_ITERATION 1 shader
* Add RMS Norm shader, rework op_f32 shader setup, fix matmul bug
* Fix issues with float16 overflows in shaders
* Fix issues with older Vulkan headers on Ubuntu 22.04
* Allow multi-op partial offloading by parsing the graph to preallocate enough between-op buffers
* Implement further ops, rework op_f32 calls, fix bugs
* Finish full offloading support, add last remaining ops, fix bugs, remove redundant code
* Upload generated file ggml-vulkan-shaders.hpp, remove redundant shaders
* Merge upstream changes, fix conflicts, adapt soft_max op
* Fix Python and shader header format
* Free model gpu buffers on exit
* Use single queue per device to simplify code
* Add matmul shader support for running multiple calculations in parallel
* Switch from semaphore-synchronized multiple command buffers per op to single command buffer for multiple ops, whole graph if possible
* Fix missing event cast
* Replace uint64_t(-1) with UINT64_MAX, rename function for clarity
* Fix warning about empty C function parameters
* Fix compiler warnings
* Properly implement Vulkan backend buffer handling
* Fix oversized host staging buffers
* Simplify barrier synchronization calls
* Fix gcc warnings
* Implement max_size for backend buffer types to limit the size of a single allocation
* Use min of maxMemoryAllocationSize and maxBufferSize for device max allocation size
* refactor multi buf
* Disable unsupported ops to fix tests
* Check for maintenance4 support before using it
* Handle devices with only a single queue
* Fix single queue logic
* propagate buffer usage in multi buffers
* Implement rope_neox op
* Cleanup header and other files
* Simplify gpu_extras by removing events and putting staging memcpys into contexts
* Move queue into context
Add not-yet-enabled async backend ops
* Simplify context use, optimize matmul shader for warp size 64 (AMD GCN), fix split_k matmul shader optimization
* Add get_max_size to SYCL backend.
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* llama : fix trailing whitespace
---------
Co-authored-by: Henri Vasserman <henv@hot.ee>
Co-authored-by: Concedo <39025047+LostRuins@users.noreply.github.com>
Co-authored-by: slaren <slarengh@gmail.com>
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
2024-01-28 17:03:59 +00:00
|
|
|
for (size_t i = 0; i < *n_buffers; i++) {
|
2024-06-11 19:20:29 +00:00
|
|
|
ggml_backend_buffer_free((*buffers)[i]);
|
ggml : add Vulkan backend (llama/2059)
* Vulkan loader code
* Fix matmul kernel, continue implementation
* Continue implementation
* Vulkan memory management
* Vulkan development
* Matmul call
* Add aligned malloc and free for VMA
* Continue implementation
* First matmul success
* GEMM Kernel optimization
* 1D Blocktiling
* 2D Blocktiling
* Write coalescing
* Continue vulkan implementation and optimization
* First FP16 attempt, disabled for now
* Code abstraction, FP16 implementation, fix kernel, add FP16 to FP32 kernel
* Enable device extensions properly, restore fp16 matmul op
* Fix mulmat_f16
* Output FP32 in fp16 matmul shader
* Fix f16_to_f32 kernel
* dequant_q4_0 kernel
* Add VMA library
* Avoid requesting dedicated memory, VMA can decide that by itself
* Add bounds checking to matmul kernels, improve implementation, fix command buffers not freed properly
* add cmake commands
* Add 2d write operation, profiling code
* Fix 2d write
* Fix queue selection for AMD RADV
* Fix trailing whitespace in vk_mem_alloc.h
* Add WIP warp tile mat mul shaders
* Disable glslc optimization
* Disable glslc optimization for CMake
* Optimize warptile matmul shader, replace blocktile with it
* Add split-k optimization for small matrix multiplication
Use semaphores for synchronization instead of fences or waitidle
Rework async write/read for synchronization
* Fix validation errors, improve compatibility with AMD GPUs
* Rework command buffer handling
* Variable matmul kernel using specialization constants
* Fix synchronization on AMD, add barriers for buffer ownership transfer, add debug flag and prints
* Reuse semaphores
* Handle stage flags during command buffer submission properly
* Increase matmul test runs for consistent results
* Fix F32 matmul
* Add vectorized loading and zeropadding for matrix multiplication
* Use pinned memory for f16 preprocessing
* Don't force aligned matmul
* Don't free before queue done
* Replace VMA library with native Vulkan buffer management
* Basic offloading support with mul_f32 and dmmv for q4_0
* Run glslc commands in parallel
* Unroll loops in dmmv shader
* Reduce usage of waitIdle
* Reuse pinned allocation for f16 conversion
* Handle devices with only a single queue
* Fix trailing whitespace in CMakeLists.txt
* Allow parallel execution of kernels, parallelize third and fourth dimension calls
* Add fallback for devices only supporting one DescriptorSet per DescriptorPool
* Move to graph function similar to CUDA implementation
* Use F16 kernel for most things, replace q_f32 with mul_mat_q_f16 function
* Add F32 dmmv shaders
* Batch submissions
* Add .spv to gitignore
* Split off matrix vector multiplication for separate optimization
* Use single command buffer for matrix vector multiplication ops
* Reduce overhead of mul_f32 calls by using a single command buffer
* Add submission batching to mul_f32
* Fix tests
* Add missing barrier
* Add further missing barrier
* Add further ops
* Replace vk::QueueFamilyIgnored with VK_QUEUE_FAMILY_IGNORED to support more Vulkan header versions
* Remove unnecessary cblas link
* Fix descriptor set pre-allocation assert
* Add runtime shader compilation, start transferring shaders to this approach
* Transfer remaining shaders to header and compile on runtime
* Fix fp32 fallback if device doesn't support fp16, add force disable env var GGML_VULKAN_DISABLE_F16
* Add support for q4_1, q5_0, q5_1 and q8_0
* Remove unnecessary scalar layout extension
* Parse graph early to pre-record command buffers
* Add q6_k support
* Add multi-submit for command buffers
* Fix q6_k dequant shader for AMD
* Fix q6_k for GPUs without fp16 support
* Simplify q6_k fp16 fix
* Minor fixes
* Fix wg_denom of m-mulmat shaders
* Add Python-based Vulkan shader generator
* Replace shaderc dependency with precompiled shaders
Fix python script to generate shaders
* Clean up code
* Fix shader generator script Windows compatibility
Co-authored-by: Concedo <39025047+LostRuins@users.noreply.github.com>
* Close file before deletion
* Fix vulkan shader fp32 name
* Add q2_k and q3_k support
Add validation check to compare shader results to cpu results
* Add q4_k support
* Add q5_k support
* Bake SPIR-V bytecode into the library instead of loading shaders from file
* Switch to signal semaphores for flexibility
Prepare broadcasting support for mul mat
* Finish broadcasting mul mat support for GQA
* Clean up unused functions
Add repeat op
* Add further ops, not yet enabled. Improve semaphore code
* Reduce number of used semaphores by utilizing timelines more properly
* Remove queue information
* Reuse timeline semaphores, allow parallel operation with binary semaphores to work around nvidia driver limitations
* Add Vulkan to llama-bench
* Remove cblas dependency
* Fix matmul k-split bug
* Fix q4_k dmmv K_QUANTS_PER_ITERATION 1 shader
* Add RMS Norm shader, rework op_f32 shader setup, fix matmul bug
* Fix issues with float16 overflows in shaders
* Fix issues with older Vulkan headers on Ubuntu 22.04
* Allow multi-op partial offloading by parsing the graph to preallocate enough between-op buffers
* Implement further ops, rework op_f32 calls, fix bugs
* Finish full offloading support, add last remaining ops, fix bugs, remove redundant code
* Upload generated file ggml-vulkan-shaders.hpp, remove redundant shaders
* Merge upstream changes, fix conflicts, adapt soft_max op
* Fix Python and shader header format
* Free model gpu buffers on exit
* Use single queue per device to simplify code
* Add matmul shader support for running multiple calculations in parallel
* Switch from semaphore-synchronized multiple command buffers per op to single command buffer for multiple ops, whole graph if possible
* Fix missing event cast
* Replace uint64_t(-1) with UINT64_MAX, rename function for clarity
* Fix warning about empty C function parameters
* Fix compiler warnings
* Properly implement Vulkan backend buffer handling
* Fix oversized host staging buffers
* Simplify barrier synchronization calls
* Fix gcc warnings
* Implement max_size for backend buffer types to limit the size of a single allocation
* Use min of maxMemoryAllocationSize and maxBufferSize for device max allocation size
* refactor multi buf
* Disable unsupported ops to fix tests
* Check for maintenance4 support before using it
* Handle devices with only a single queue
* Fix single queue logic
* propagate buffer usage in multi buffers
* Implement rope_neox op
* Cleanup header and other files
* Simplify gpu_extras by removing events and putting staging memcpys into contexts
* Move queue into context
Add not-yet-enabled async backend ops
* Simplify context use, optimize matmul shader for warp size 64 (AMD GCN), fix split_k matmul shader optimization
* Add get_max_size to SYCL backend.
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* llama : fix trailing whitespace
---------
Co-authored-by: Henri Vasserman <henv@hot.ee>
Co-authored-by: Concedo <39025047+LostRuins@users.noreply.github.com>
Co-authored-by: slaren <slarengh@gmail.com>
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
2024-01-28 17:03:59 +00:00
|
|
|
}
|
2024-01-29 22:19:29 +00:00
|
|
|
free(*buffers);
|
ggml : add Vulkan backend (llama/2059)
* Vulkan loader code
* Fix matmul kernel, continue implementation
* Continue implementation
* Vulkan memory management
* Vulkan development
* Matmul call
* Add aligned malloc and free for VMA
* Continue implementation
* First matmul success
* GEMM Kernel optimization
* 1D Blocktiling
* 2D Blocktiling
* Write coalescing
* Continue vulkan implementation and optimization
* First FP16 attempt, disabled for now
* Code abstraction, FP16 implementation, fix kernel, add FP16 to FP32 kernel
* Enable device extensions properly, restore fp16 matmul op
* Fix mulmat_f16
* Output FP32 in fp16 matmul shader
* Fix f16_to_f32 kernel
* dequant_q4_0 kernel
* Add VMA library
* Avoid requesting dedicated memory, VMA can decide that by itself
* Add bounds checking to matmul kernels, improve implementation, fix command buffers not freed properly
* add cmake commands
* Add 2d write operation, profiling code
* Fix 2d write
* Fix queue selection for AMD RADV
* Fix trailing whitespace in vk_mem_alloc.h
* Add WIP warp tile mat mul shaders
* Disable glslc optimization
* Disable glslc optimization for CMake
* Optimize warptile matmul shader, replace blocktile with it
* Add split-k optimization for small matrix multiplication
Use semaphores for synchronization instead of fences or waitidle
Rework async write/read for synchronization
* Fix validation errors, improve compatibility with AMD GPUs
* Rework command buffer handling
* Variable matmul kernel using specialization constants
* Fix synchronization on AMD, add barriers for buffer ownership transfer, add debug flag and prints
* Reuse semaphores
* Handle stage flags during command buffer submission properly
* Increase matmul test runs for consistent results
* Fix F32 matmul
* Add vectorized loading and zeropadding for matrix multiplication
* Use pinned memory for f16 preprocessing
* Don't force aligned matmul
* Don't free before queue done
* Replace VMA library with native Vulkan buffer management
* Basic offloading support with mul_f32 and dmmv for q4_0
* Run glslc commands in parallel
* Unroll loops in dmmv shader
* Reduce usage of waitIdle
* Reuse pinned allocation for f16 conversion
* Handle devices with only a single queue
* Fix trailing whitespace in CMakeLists.txt
* Allow parallel execution of kernels, parallelize third and fourth dimension calls
* Add fallback for devices only supporting one DescriptorSet per DescriptorPool
* Move to graph function similar to CUDA implementation
* Use F16 kernel for most things, replace q_f32 with mul_mat_q_f16 function
* Add F32 dmmv shaders
* Batch submissions
* Add .spv to gitignore
* Split off matrix vector multiplication for separate optimization
* Use single command buffer for matrix vector multiplication ops
* Reduce overhead of mul_f32 calls by using a single command buffer
* Add submission batching to mul_f32
* Fix tests
* Add missing barrier
* Add further missing barrier
* Add further ops
* Replace vk::QueueFamilyIgnored with VK_QUEUE_FAMILY_IGNORED to support more Vulkan header versions
* Remove unnecessary cblas link
* Fix descriptor set pre-allocation assert
* Add runtime shader compilation, start transferring shaders to this approach
* Transfer remaining shaders to header and compile on runtime
* Fix fp32 fallback if device doesn't support fp16, add force disable env var GGML_VULKAN_DISABLE_F16
* Add support for q4_1, q5_0, q5_1 and q8_0
* Remove unnecessary scalar layout extension
* Parse graph early to pre-record command buffers
* Add q6_k support
* Add multi-submit for command buffers
* Fix q6_k dequant shader for AMD
* Fix q6_k for GPUs without fp16 support
* Simplify q6_k fp16 fix
* Minor fixes
* Fix wg_denom of m-mulmat shaders
* Add Python-based Vulkan shader generator
* Replace shaderc dependency with precompiled shaders
Fix python script to generate shaders
* Clean up code
* Fix shader generator script Windows compatibility
Co-authored-by: Concedo <39025047+LostRuins@users.noreply.github.com>
* Close file before deletion
* Fix vulkan shader fp32 name
* Add q2_k and q3_k support
Add validation check to compare shader results to cpu results
* Add q4_k support
* Add q5_k support
* Bake SPIR-V bytecode into the library instead of loading shaders from file
* Switch to signal semaphores for flexibility
Prepare broadcasting support for mul mat
* Finish broadcasting mul mat support for GQA
* Clean up unused functions
Add repeat op
* Add further ops, not yet enabled. Improve semaphore code
* Reduce number of used semaphores by utilizing timelines more properly
* Remove queue information
* Reuse timeline semaphores, allow parallel operation with binary semaphores to work around nvidia driver limitations
* Add Vulkan to llama-bench
* Remove cblas dependency
* Fix matmul k-split bug
* Fix q4_k dmmv K_QUANTS_PER_ITERATION 1 shader
* Add RMS Norm shader, rework op_f32 shader setup, fix matmul bug
* Fix issues with float16 overflows in shaders
* Fix issues with older Vulkan headers on Ubuntu 22.04
* Allow multi-op partial offloading by parsing the graph to preallocate enough between-op buffers
* Implement further ops, rework op_f32 calls, fix bugs
* Finish full offloading support, add last remaining ops, fix bugs, remove redundant code
* Upload generated file ggml-vulkan-shaders.hpp, remove redundant shaders
* Merge upstream changes, fix conflicts, adapt soft_max op
* Fix Python and shader header format
* Free model gpu buffers on exit
* Use single queue per device to simplify code
* Add matmul shader support for running multiple calculations in parallel
* Switch from semaphore-synchronized multiple command buffers per op to single command buffer for multiple ops, whole graph if possible
* Fix missing event cast
* Replace uint64_t(-1) with UINT64_MAX, rename function for clarity
* Fix warning about empty C function parameters
* Fix compiler warnings
* Properly implement Vulkan backend buffer handling
* Fix oversized host staging buffers
* Simplify barrier synchronization calls
* Fix gcc warnings
* Implement max_size for backend buffer types to limit the size of a single allocation
* Use min of maxMemoryAllocationSize and maxBufferSize for device max allocation size
* refactor multi buf
* Disable unsupported ops to fix tests
* Check for maintenance4 support before using it
* Handle devices with only a single queue
* Fix single queue logic
* propagate buffer usage in multi buffers
* Implement rope_neox op
* Cleanup header and other files
* Simplify gpu_extras by removing events and putting staging memcpys into contexts
* Move queue into context
Add not-yet-enabled async backend ops
* Simplify context use, optimize matmul shader for warp size 64 (AMD GCN), fix split_k matmul shader optimization
* Add get_max_size to SYCL backend.
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* llama : fix trailing whitespace
---------
Co-authored-by: Henri Vasserman <henv@hot.ee>
Co-authored-by: Concedo <39025047+LostRuins@users.noreply.github.com>
Co-authored-by: slaren <slarengh@gmail.com>
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
2024-01-28 17:03:59 +00:00
|
|
|
return false;
|
2024-01-12 19:07:38 +00:00
|
|
|
}
|
|
|
|
|
2024-03-13 17:54:21 +00:00
|
|
|
struct ggml_tallocr tallocr = ggml_tallocr_new(buffer);
|
2023-12-07 20:27:19 +00:00
|
|
|
|
ggml : add Vulkan backend (llama/2059)
* Vulkan loader code
* Fix matmul kernel, continue implementation
* Continue implementation
* Vulkan memory management
* Vulkan development
* Matmul call
* Add aligned malloc and free for VMA
* Continue implementation
* First matmul success
* GEMM Kernel optimization
* 1D Blocktiling
* 2D Blocktiling
* Write coalescing
* Continue vulkan implementation and optimization
* First FP16 attempt, disabled for now
* Code abstraction, FP16 implementation, fix kernel, add FP16 to FP32 kernel
* Enable device extensions properly, restore fp16 matmul op
* Fix mulmat_f16
* Output FP32 in fp16 matmul shader
* Fix f16_to_f32 kernel
* dequant_q4_0 kernel
* Add VMA library
* Avoid requesting dedicated memory, VMA can decide that by itself
* Add bounds checking to matmul kernels, improve implementation, fix command buffers not freed properly
* add cmake commands
* Add 2d write operation, profiling code
* Fix 2d write
* Fix queue selection for AMD RADV
* Fix trailing whitespace in vk_mem_alloc.h
* Add WIP warp tile mat mul shaders
* Disable glslc optimization
* Disable glslc optimization for CMake
* Optimize warptile matmul shader, replace blocktile with it
* Add split-k optimization for small matrix multiplication
Use semaphores for synchronization instead of fences or waitidle
Rework async write/read for synchronization
* Fix validation errors, improve compatibility with AMD GPUs
* Rework command buffer handling
* Variable matmul kernel using specialization constants
* Fix synchronization on AMD, add barriers for buffer ownership transfer, add debug flag and prints
* Reuse semaphores
* Handle stage flags during command buffer submission properly
* Increase matmul test runs for consistent results
* Fix F32 matmul
* Add vectorized loading and zeropadding for matrix multiplication
* Use pinned memory for f16 preprocessing
* Don't force aligned matmul
* Don't free before queue done
* Replace VMA library with native Vulkan buffer management
* Basic offloading support with mul_f32 and dmmv for q4_0
* Run glslc commands in parallel
* Unroll loops in dmmv shader
* Reduce usage of waitIdle
* Reuse pinned allocation for f16 conversion
* Handle devices with only a single queue
* Fix trailing whitespace in CMakeLists.txt
* Allow parallel execution of kernels, parallelize third and fourth dimension calls
* Add fallback for devices only supporting one DescriptorSet per DescriptorPool
* Move to graph function similar to CUDA implementation
* Use F16 kernel for most things, replace q_f32 with mul_mat_q_f16 function
* Add F32 dmmv shaders
* Batch submissions
* Add .spv to gitignore
* Split off matrix vector multiplication for separate optimization
* Use single command buffer for matrix vector multiplication ops
* Reduce overhead of mul_f32 calls by using a single command buffer
* Add submission batching to mul_f32
* Fix tests
* Add missing barrier
* Add further missing barrier
* Add further ops
* Replace vk::QueueFamilyIgnored with VK_QUEUE_FAMILY_IGNORED to support more Vulkan header versions
* Remove unnecessary cblas link
* Fix descriptor set pre-allocation assert
* Add runtime shader compilation, start transferring shaders to this approach
* Transfer remaining shaders to header and compile on runtime
* Fix fp32 fallback if device doesn't support fp16, add force disable env var GGML_VULKAN_DISABLE_F16
* Add support for q4_1, q5_0, q5_1 and q8_0
* Remove unnecessary scalar layout extension
* Parse graph early to pre-record command buffers
* Add q6_k support
* Add multi-submit for command buffers
* Fix q6_k dequant shader for AMD
* Fix q6_k for GPUs without fp16 support
* Simplify q6_k fp16 fix
* Minor fixes
* Fix wg_denom of m-mulmat shaders
* Add Python-based Vulkan shader generator
* Replace shaderc dependency with precompiled shaders
Fix python script to generate shaders
* Clean up code
* Fix shader generator script Windows compatibility
Co-authored-by: Concedo <39025047+LostRuins@users.noreply.github.com>
* Close file before deletion
* Fix vulkan shader fp32 name
* Add q2_k and q3_k support
Add validation check to compare shader results to cpu results
* Add q4_k support
* Add q5_k support
* Bake SPIR-V bytecode into the library instead of loading shaders from file
* Switch to signal semaphores for flexibility
Prepare broadcasting support for mul mat
* Finish broadcasting mul mat support for GQA
* Clean up unused functions
Add repeat op
* Add further ops, not yet enabled. Improve semaphore code
* Reduce number of used semaphores by utilizing timelines more properly
* Remove queue information
* Reuse timeline semaphores, allow parallel operation with binary semaphores to work around nvidia driver limitations
* Add Vulkan to llama-bench
* Remove cblas dependency
* Fix matmul k-split bug
* Fix q4_k dmmv K_QUANTS_PER_ITERATION 1 shader
* Add RMS Norm shader, rework op_f32 shader setup, fix matmul bug
* Fix issues with float16 overflows in shaders
* Fix issues with older Vulkan headers on Ubuntu 22.04
* Allow multi-op partial offloading by parsing the graph to preallocate enough between-op buffers
* Implement further ops, rework op_f32 calls, fix bugs
* Finish full offloading support, add last remaining ops, fix bugs, remove redundant code
* Upload generated file ggml-vulkan-shaders.hpp, remove redundant shaders
* Merge upstream changes, fix conflicts, adapt soft_max op
* Fix Python and shader header format
* Free model gpu buffers on exit
* Use single queue per device to simplify code
* Add matmul shader support for running multiple calculations in parallel
* Switch from semaphore-synchronized multiple command buffers per op to single command buffer for multiple ops, whole graph if possible
* Fix missing event cast
* Replace uint64_t(-1) with UINT64_MAX, rename function for clarity
* Fix warning about empty C function parameters
* Fix compiler warnings
* Properly implement Vulkan backend buffer handling
* Fix oversized host staging buffers
* Simplify barrier synchronization calls
* Fix gcc warnings
* Implement max_size for backend buffer types to limit the size of a single allocation
* Use min of maxMemoryAllocationSize and maxBufferSize for device max allocation size
* refactor multi buf
* Disable unsupported ops to fix tests
* Check for maintenance4 support before using it
* Handle devices with only a single queue
* Fix single queue logic
* propagate buffer usage in multi buffers
* Implement rope_neox op
* Cleanup header and other files
* Simplify gpu_extras by removing events and putting staging memcpys into contexts
* Move queue into context
Add not-yet-enabled async backend ops
* Simplify context use, optimize matmul shader for warp size 64 (AMD GCN), fix split_k matmul shader optimization
* Add get_max_size to SYCL backend.
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* llama : fix trailing whitespace
---------
Co-authored-by: Henri Vasserman <henv@hot.ee>
Co-authored-by: Concedo <39025047+LostRuins@users.noreply.github.com>
Co-authored-by: slaren <slarengh@gmail.com>
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
2024-01-28 17:03:59 +00:00
|
|
|
for (struct ggml_tensor * t = first; t != last; t = ggml_get_next_tensor(ctx, t)) {
|
2023-12-07 20:27:19 +00:00
|
|
|
if (t->data == NULL) {
|
|
|
|
if (t->view_src == NULL) {
|
2024-03-13 17:54:21 +00:00
|
|
|
ggml_tallocr_alloc(&tallocr, t);
|
2024-02-11 12:37:58 +00:00
|
|
|
} else if (t->buffer == NULL) {
|
2024-06-03 17:03:26 +00:00
|
|
|
ggml_backend_view_init(t);
|
2023-12-07 20:27:19 +00:00
|
|
|
}
|
2023-12-22 15:53:39 +00:00
|
|
|
} else {
|
2024-02-11 12:37:58 +00:00
|
|
|
if (t->view_src != NULL && t->buffer == NULL) {
|
2023-12-22 15:53:39 +00:00
|
|
|
// view of a pre-allocated tensor
|
2024-06-03 17:03:26 +00:00
|
|
|
ggml_backend_view_init(t);
|
2023-12-22 15:53:39 +00:00
|
|
|
}
|
2023-12-07 20:27:19 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
ggml : add Vulkan backend (llama/2059)
* Vulkan loader code
* Fix matmul kernel, continue implementation
* Continue implementation
* Vulkan memory management
* Vulkan development
* Matmul call
* Add aligned malloc and free for VMA
* Continue implementation
* First matmul success
* GEMM Kernel optimization
* 1D Blocktiling
* 2D Blocktiling
* Write coalescing
* Continue vulkan implementation and optimization
* First FP16 attempt, disabled for now
* Code abstraction, FP16 implementation, fix kernel, add FP16 to FP32 kernel
* Enable device extensions properly, restore fp16 matmul op
* Fix mulmat_f16
* Output FP32 in fp16 matmul shader
* Fix f16_to_f32 kernel
* dequant_q4_0 kernel
* Add VMA library
* Avoid requesting dedicated memory, VMA can decide that by itself
* Add bounds checking to matmul kernels, improve implementation, fix command buffers not freed properly
* add cmake commands
* Add 2d write operation, profiling code
* Fix 2d write
* Fix queue selection for AMD RADV
* Fix trailing whitespace in vk_mem_alloc.h
* Add WIP warp tile mat mul shaders
* Disable glslc optimization
* Disable glslc optimization for CMake
* Optimize warptile matmul shader, replace blocktile with it
* Add split-k optimization for small matrix multiplication
Use semaphores for synchronization instead of fences or waitidle
Rework async write/read for synchronization
* Fix validation errors, improve compatibility with AMD GPUs
* Rework command buffer handling
* Variable matmul kernel using specialization constants
* Fix synchronization on AMD, add barriers for buffer ownership transfer, add debug flag and prints
* Reuse semaphores
* Handle stage flags during command buffer submission properly
* Increase matmul test runs for consistent results
* Fix F32 matmul
* Add vectorized loading and zeropadding for matrix multiplication
* Use pinned memory for f16 preprocessing
* Don't force aligned matmul
* Don't free before queue done
* Replace VMA library with native Vulkan buffer management
* Basic offloading support with mul_f32 and dmmv for q4_0
* Run glslc commands in parallel
* Unroll loops in dmmv shader
* Reduce usage of waitIdle
* Reuse pinned allocation for f16 conversion
* Handle devices with only a single queue
* Fix trailing whitespace in CMakeLists.txt
* Allow parallel execution of kernels, parallelize third and fourth dimension calls
* Add fallback for devices only supporting one DescriptorSet per DescriptorPool
* Move to graph function similar to CUDA implementation
* Use F16 kernel for most things, replace q_f32 with mul_mat_q_f16 function
* Add F32 dmmv shaders
* Batch submissions
* Add .spv to gitignore
* Split off matrix vector multiplication for separate optimization
* Use single command buffer for matrix vector multiplication ops
* Reduce overhead of mul_f32 calls by using a single command buffer
* Add submission batching to mul_f32
* Fix tests
* Add missing barrier
* Add further missing barrier
* Add further ops
* Replace vk::QueueFamilyIgnored with VK_QUEUE_FAMILY_IGNORED to support more Vulkan header versions
* Remove unnecessary cblas link
* Fix descriptor set pre-allocation assert
* Add runtime shader compilation, start transferring shaders to this approach
* Transfer remaining shaders to header and compile on runtime
* Fix fp32 fallback if device doesn't support fp16, add force disable env var GGML_VULKAN_DISABLE_F16
* Add support for q4_1, q5_0, q5_1 and q8_0
* Remove unnecessary scalar layout extension
* Parse graph early to pre-record command buffers
* Add q6_k support
* Add multi-submit for command buffers
* Fix q6_k dequant shader for AMD
* Fix q6_k for GPUs without fp16 support
* Simplify q6_k fp16 fix
* Minor fixes
* Fix wg_denom of m-mulmat shaders
* Add Python-based Vulkan shader generator
* Replace shaderc dependency with precompiled shaders
Fix python script to generate shaders
* Clean up code
* Fix shader generator script Windows compatibility
Co-authored-by: Concedo <39025047+LostRuins@users.noreply.github.com>
* Close file before deletion
* Fix vulkan shader fp32 name
* Add q2_k and q3_k support
Add validation check to compare shader results to cpu results
* Add q4_k support
* Add q5_k support
* Bake SPIR-V bytecode into the library instead of loading shaders from file
* Switch to signal semaphores for flexibility
Prepare broadcasting support for mul mat
* Finish broadcasting mul mat support for GQA
* Clean up unused functions
Add repeat op
* Add further ops, not yet enabled. Improve semaphore code
* Reduce number of used semaphores by utilizing timelines more properly
* Remove queue information
* Reuse timeline semaphores, allow parallel operation with binary semaphores to work around nvidia driver limitations
* Add Vulkan to llama-bench
* Remove cblas dependency
* Fix matmul k-split bug
* Fix q4_k dmmv K_QUANTS_PER_ITERATION 1 shader
* Add RMS Norm shader, rework op_f32 shader setup, fix matmul bug
* Fix issues with float16 overflows in shaders
* Fix issues with older Vulkan headers on Ubuntu 22.04
* Allow multi-op partial offloading by parsing the graph to preallocate enough between-op buffers
* Implement further ops, rework op_f32 calls, fix bugs
* Finish full offloading support, add last remaining ops, fix bugs, remove redundant code
* Upload generated file ggml-vulkan-shaders.hpp, remove redundant shaders
* Merge upstream changes, fix conflicts, adapt soft_max op
* Fix Python and shader header format
* Free model gpu buffers on exit
* Use single queue per device to simplify code
* Add matmul shader support for running multiple calculations in parallel
* Switch from semaphore-synchronized multiple command buffers per op to single command buffer for multiple ops, whole graph if possible
* Fix missing event cast
* Replace uint64_t(-1) with UINT64_MAX, rename function for clarity
* Fix warning about empty C function parameters
* Fix compiler warnings
* Properly implement Vulkan backend buffer handling
* Fix oversized host staging buffers
* Simplify barrier synchronization calls
* Fix gcc warnings
* Implement max_size for backend buffer types to limit the size of a single allocation
* Use min of maxMemoryAllocationSize and maxBufferSize for device max allocation size
* refactor multi buf
* Disable unsupported ops to fix tests
* Check for maintenance4 support before using it
* Handle devices with only a single queue
* Fix single queue logic
* propagate buffer usage in multi buffers
* Implement rope_neox op
* Cleanup header and other files
* Simplify gpu_extras by removing events and putting staging memcpys into contexts
* Move queue into context
Add not-yet-enabled async backend ops
* Simplify context use, optimize matmul shader for warp size 64 (AMD GCN), fix split_k matmul shader optimization
* Add get_max_size to SYCL backend.
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* llama : fix trailing whitespace
---------
Co-authored-by: Henri Vasserman <henv@hot.ee>
Co-authored-by: Concedo <39025047+LostRuins@users.noreply.github.com>
Co-authored-by: slaren <slarengh@gmail.com>
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
2024-01-28 17:03:59 +00:00
|
|
|
*buffers = realloc(*buffers, sizeof(ggml_backend_buffer_t) * (*n_buffers + 1));
|
|
|
|
(*buffers)[(*n_buffers)++] = buffer;
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
ggml_backend_buffer_t ggml_backend_alloc_ctx_tensors_from_buft(struct ggml_context * ctx, ggml_backend_buffer_type_t buft) {
|
|
|
|
GGML_ASSERT(ggml_get_no_alloc(ctx) == true);
|
|
|
|
|
|
|
|
size_t alignment = ggml_backend_buft_get_alignment(buft);
|
|
|
|
size_t max_size = ggml_backend_buft_get_max_size(buft);
|
|
|
|
|
|
|
|
ggml_backend_buffer_t * buffers = NULL;
|
|
|
|
size_t n_buffers = 0;
|
|
|
|
|
|
|
|
size_t cur_buf_size = 0;
|
|
|
|
struct ggml_tensor * first = ggml_get_first_tensor(ctx);
|
|
|
|
for (struct ggml_tensor * t = first; t != NULL; t = ggml_get_next_tensor(ctx, t)) {
|
|
|
|
size_t this_size = 0;
|
|
|
|
if (t->data == NULL && t->view_src == NULL) {
|
|
|
|
this_size = GGML_PAD(ggml_backend_buft_get_alloc_size(buft, t), alignment);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (this_size > max_size) {
|
|
|
|
fprintf(stderr, "%s: tensor %s is too large to fit in a %s buffer (tensor size: %zu, max buffer size: %zu)\n",
|
|
|
|
__func__, t->name,
|
|
|
|
ggml_backend_buft_name(buft),
|
|
|
|
this_size, max_size);
|
|
|
|
for (size_t i = 0; i < n_buffers; i++) {
|
|
|
|
ggml_backend_buffer_free(buffers[i]);
|
|
|
|
}
|
|
|
|
free(buffers);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
if ((cur_buf_size + this_size) > max_size) {
|
|
|
|
// allocate tensors in the current buffer
|
|
|
|
if (!alloc_tensor_range(ctx, first, t, buft, cur_buf_size, &buffers, &n_buffers)) {
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
first = t;
|
|
|
|
cur_buf_size = this_size;
|
|
|
|
} else {
|
|
|
|
cur_buf_size += this_size;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// allocate remaining tensors
|
|
|
|
if (cur_buf_size > 0) {
|
|
|
|
if (!alloc_tensor_range(ctx, first, NULL, buft, cur_buf_size, &buffers, &n_buffers)) {
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (n_buffers == 0) {
|
|
|
|
#ifndef NDEBUG
|
|
|
|
fprintf(stderr, "%s: all tensors in the context are already allocated\n", __func__);
|
|
|
|
#endif
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
ggml_backend_buffer_t buffer;
|
|
|
|
if (n_buffers == 1) {
|
|
|
|
buffer = buffers[0];
|
|
|
|
} else {
|
|
|
|
buffer = ggml_backend_multi_buffer_alloc_buffer(buffers, n_buffers);
|
|
|
|
}
|
|
|
|
free(buffers);
|
2023-12-07 20:27:19 +00:00
|
|
|
return buffer;
|
|
|
|
}
|
|
|
|
|
|
|
|
ggml_backend_buffer_t ggml_backend_alloc_ctx_tensors(struct ggml_context * ctx, ggml_backend_t backend) {
|
|
|
|
return ggml_backend_alloc_ctx_tensors_from_buft(ctx, ggml_backend_get_default_buffer_type(backend));
|
|
|
|
}
|