mirror of
https://github.com/ggerganov/whisper.cpp.git
synced 2025-04-29 07:19:51 +00:00
ggml: Re-enable CUDA graphs in presence of CONT and DUP nodes (llama/12970)
This commit is contained in:
parent
314ce5981e
commit
4e936e2afa
@ -551,7 +551,7 @@ static void ggml_cpy_f16_f16_cuda(
|
|||||||
(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, cdst_indirect, graph_cpynode_index++);
|
(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, cdst_indirect, graph_cpynode_index++);
|
||||||
}
|
}
|
||||||
|
|
||||||
void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, ggml_tensor * src1) {
|
void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, ggml_tensor * src1, bool disable_indirection_for_this_node) {
|
||||||
const int64_t ne = ggml_nelements(src0);
|
const int64_t ne = ggml_nelements(src0);
|
||||||
GGML_ASSERT(ne == ggml_nelements(src1));
|
GGML_ASSERT(ne == ggml_nelements(src1));
|
||||||
|
|
||||||
@ -588,7 +588,7 @@ void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, gg
|
|||||||
char ** dest_ptrs_d = nullptr;
|
char ** dest_ptrs_d = nullptr;
|
||||||
int graph_cpynode_index = -1;
|
int graph_cpynode_index = -1;
|
||||||
#if defined(GGML_CUDA_USE_GRAPHS) || defined(GGML_HIP_GRAPHS)
|
#if defined(GGML_CUDA_USE_GRAPHS) || defined(GGML_HIP_GRAPHS)
|
||||||
if(ctx.cuda_graph->use_cpy_indirection) {
|
if(ctx.cuda_graph->use_cpy_indirection && !disable_indirection_for_this_node) {
|
||||||
dest_ptrs_d = ctx.cuda_graph->dest_ptrs_d;
|
dest_ptrs_d = ctx.cuda_graph->dest_ptrs_d;
|
||||||
graph_cpynode_index = ctx.cuda_graph->graph_cpynode_index;
|
graph_cpynode_index = ctx.cuda_graph->graph_cpynode_index;
|
||||||
}
|
}
|
||||||
@ -636,7 +636,7 @@ void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, gg
|
|||||||
ggml_type_name(src0->type), ggml_type_name(src1->type));
|
ggml_type_name(src0->type), ggml_type_name(src1->type));
|
||||||
}
|
}
|
||||||
#if defined(GGML_CUDA_USE_GRAPHS) || defined(GGML_HIP_GRAPHS)
|
#if defined(GGML_CUDA_USE_GRAPHS) || defined(GGML_HIP_GRAPHS)
|
||||||
if(ctx.cuda_graph->use_cpy_indirection) {
|
if(ctx.cuda_graph->use_cpy_indirection && !disable_indirection_for_this_node) {
|
||||||
ctx.cuda_graph->graph_cpynode_index = graph_cpynode_index;
|
ctx.cuda_graph->graph_cpynode_index = graph_cpynode_index;
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
@ -645,7 +645,8 @@ void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, gg
|
|||||||
|
|
||||||
void ggml_cuda_dup(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
void ggml_cuda_dup(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||||
const ggml_tensor * src0 = dst->src[0];
|
const ggml_tensor * src0 = dst->src[0];
|
||||||
ggml_cuda_cpy(ctx, src0, dst);
|
bool disable_indirection = true;
|
||||||
|
ggml_cuda_cpy(ctx, src0, dst, disable_indirection);
|
||||||
}
|
}
|
||||||
|
|
||||||
void* ggml_cuda_cpy_fn(const ggml_tensor * src0, ggml_tensor * src1) {
|
void* ggml_cuda_cpy_fn(const ggml_tensor * src0, ggml_tensor * src1) {
|
||||||
|
@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
#define CUDA_CPY_BLOCK_SIZE 64
|
#define CUDA_CPY_BLOCK_SIZE 64
|
||||||
|
|
||||||
void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, ggml_tensor * src1);
|
void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, ggml_tensor * src1, bool disable_indirection = false);
|
||||||
|
|
||||||
void ggml_cuda_dup(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
void ggml_cuda_dup(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
||||||
|
|
||||||
|
@ -2489,7 +2489,7 @@ static bool check_node_graph_compatibility_and_refresh_copy_ops(ggml_backend_cud
|
|||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
if (node->op == GGML_OP_MUL_MAT_ID || node->op == GGML_OP_CONT || node->op == GGML_OP_DUP) {
|
if (node->op == GGML_OP_MUL_MAT_ID) {
|
||||||
use_cuda_graph = false; // This node type is not supported by CUDA graph capture
|
use_cuda_graph = false; // This node type is not supported by CUDA graph capture
|
||||||
#ifndef NDEBUG
|
#ifndef NDEBUG
|
||||||
GGML_LOG_DEBUG("%s: disabling CUDA graphs due to unsupported node type\n", __func__);
|
GGML_LOG_DEBUG("%s: disabling CUDA graphs due to unsupported node type\n", __func__);
|
||||||
|
Loading…
x
Reference in New Issue
Block a user