mirror of
https://github.com/ggerganov/whisper.cpp.git
synced 2024-12-18 20:27:53 +00:00
7711 lines
224 KiB
C
7711 lines
224 KiB
C
#include "ggml.h"
|
|
|
|
#include <alloca.h>
|
|
#include <assert.h>
|
|
#include <time.h>
|
|
#include <math.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
#include <stdint.h>
|
|
#include <stdio.h>
|
|
#include <stdatomic.h>
|
|
|
|
#include <pthread.h>
|
|
|
|
#define GGML_DEBUG 0
|
|
|
|
#if UINTPTR_MAX == 0xFFFFFFFF
|
|
#define GGML_MEM_ALIGN 4
|
|
#else
|
|
#define GGML_MEM_ALIGN 16
|
|
#endif
|
|
|
|
#define MAX(a, b) ((a) > (b) ? (a) : (b))
|
|
#define MIN(a, b) ((a) < (b) ? (a) : (b))
|
|
|
|
#define UNUSED(x) (void)(x)
|
|
#define SWAP(x, y, T) do { T SWAP = x; x = y; y = SWAP; } while (0)
|
|
|
|
#define GGML_ASSERT(x) \
|
|
do { \
|
|
if (!(x)) { \
|
|
fprintf(stderr, "GGML_ASSERT: %s:%d: %s\n", __FILE__, __LINE__, #x); \
|
|
abort(); \
|
|
} \
|
|
} while (0)
|
|
|
|
#ifdef GGML_USE_ACCELERATE
|
|
#include <Accelerate/Accelerate.h>
|
|
#endif
|
|
|
|
// floating point type used to accumulate sums
|
|
typedef double ggml_float;
|
|
|
|
// 16-bit float
|
|
// on Arm, we use __fp16
|
|
// on x86, we use uint16_t
|
|
#ifdef __ARM_NEON
|
|
|
|
// if YCM cannot find <arm_neon.h>, make a symbolic link to it, for example:
|
|
//
|
|
// $ ln -sfn /Library/Developer/CommandLineTools/usr/lib/clang/13.1.6/include/arm_neon.h ./src/
|
|
//
|
|
#include <arm_neon.h>
|
|
|
|
float ggml_fp16_to_fp32(ggml_fp16_t x) {
|
|
return x;
|
|
}
|
|
|
|
ggml_fp16_t ggml_fp32_to_fp16(float x) {
|
|
return x;
|
|
}
|
|
|
|
#else
|
|
|
|
#include <immintrin.h>
|
|
|
|
static inline float fp32_from_bits(uint32_t w) {
|
|
union {
|
|
uint32_t as_bits;
|
|
float as_value;
|
|
} fp32 = { w };
|
|
return fp32.as_value;
|
|
}
|
|
|
|
static inline uint32_t fp32_to_bits(float f) {
|
|
union {
|
|
float as_value;
|
|
uint32_t as_bits;
|
|
} fp32 = { f };
|
|
return fp32.as_bits;
|
|
}
|
|
|
|
float ggml_fp16_to_fp32(ggml_fp16_t h) {
|
|
const uint32_t w = (uint32_t) h << 16;
|
|
const uint32_t sign = w & UINT32_C(0x80000000);
|
|
const uint32_t two_w = w + w;
|
|
|
|
const uint32_t exp_offset = UINT32_C(0xE0) << 23;
|
|
#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) || defined(__GNUC__) && !defined(__STRICT_ANSI__)
|
|
const float exp_scale = 0x1.0p-112f;
|
|
#else
|
|
const float exp_scale = fp32_from_bits(UINT32_C(0x7800000));
|
|
#endif
|
|
const float normalized_value = fp32_from_bits((two_w >> 4) + exp_offset) * exp_scale;
|
|
|
|
const uint32_t magic_mask = UINT32_C(126) << 23;
|
|
const float magic_bias = 0.5f;
|
|
const float denormalized_value = fp32_from_bits((two_w >> 17) | magic_mask) - magic_bias;
|
|
|
|
const uint32_t denormalized_cutoff = UINT32_C(1) << 27;
|
|
const uint32_t result = sign |
|
|
(two_w < denormalized_cutoff ? fp32_to_bits(denormalized_value) : fp32_to_bits(normalized_value));
|
|
return fp32_from_bits(result);
|
|
}
|
|
|
|
ggml_fp16_t ggml_fp32_to_fp16(float f) {
|
|
#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) || defined(__GNUC__) && !defined(__STRICT_ANSI__)
|
|
const float scale_to_inf = 0x1.0p+112f;
|
|
const float scale_to_zero = 0x1.0p-110f;
|
|
#else
|
|
const float scale_to_inf = fp32_from_bits(UINT32_C(0x77800000));
|
|
const float scale_to_zero = fp32_from_bits(UINT32_C(0x08800000));
|
|
#endif
|
|
float base = (fabsf(f) * scale_to_inf) * scale_to_zero;
|
|
|
|
const uint32_t w = fp32_to_bits(f);
|
|
const uint32_t shl1_w = w + w;
|
|
const uint32_t sign = w & UINT32_C(0x80000000);
|
|
uint32_t bias = shl1_w & UINT32_C(0xFF000000);
|
|
if (bias < UINT32_C(0x71000000)) {
|
|
bias = UINT32_C(0x71000000);
|
|
}
|
|
|
|
base = fp32_from_bits((bias >> 1) + UINT32_C(0x07800000)) + base;
|
|
const uint32_t bits = fp32_to_bits(base);
|
|
const uint32_t exp_bits = (bits >> 13) & UINT32_C(0x00007C00);
|
|
const uint32_t mantissa_bits = bits & UINT32_C(0x00000FFF);
|
|
const uint32_t nonsign = exp_bits + mantissa_bits;
|
|
return (sign >> 16) | (shl1_w > UINT32_C(0xFF000000) ? UINT16_C(0x7E00) : nonsign);
|
|
}
|
|
#endif
|
|
|
|
//
|
|
// global data
|
|
//
|
|
|
|
// precomputed gelu table for f16 (128 KB)
|
|
static ggml_fp16_t table_gelu_f16[1 << 16];
|
|
|
|
// precomputed exp table for f16 (128 KB)
|
|
static ggml_fp16_t table_exp_f16[1 << 16];
|
|
|
|
//
|
|
// timing
|
|
//
|
|
|
|
int64_t ggml_time_ms(void) {
|
|
struct timespec ts;
|
|
clock_gettime(CLOCK_MONOTONIC, &ts);
|
|
return (int64_t)ts.tv_sec*1000 + (int64_t)ts.tv_nsec/1000000;
|
|
}
|
|
|
|
int64_t ggml_time_us(void) {
|
|
struct timespec ts;
|
|
clock_gettime(CLOCK_MONOTONIC, &ts);
|
|
return (int64_t)ts.tv_sec*1000000 + (int64_t)ts.tv_nsec/1000;
|
|
}
|
|
|
|
int64_t ggml_cycles(void) {
|
|
return clock();
|
|
}
|
|
|
|
int64_t ggml_cycles_per_ms(void) {
|
|
return CLOCKS_PER_SEC/1000;
|
|
}
|
|
|
|
#ifdef GGML_PERF
|
|
#define ggml_perf_time_ms() ggml_time_ms()
|
|
#define ggml_perf_time_us() ggml_time_us()
|
|
#define ggml_perf_cycles() ggml_cycles()
|
|
#define ggml_perf_cycles_per_ms() ggml_cycles_per_ms()
|
|
#else
|
|
#define ggml_perf_time_ms() 0
|
|
#define ggml_perf_time_us() 0
|
|
#define ggml_perf_cycles() 0
|
|
#define ggml_perf_cycles_per_ms() 0
|
|
#endif
|
|
|
|
//
|
|
// cache line
|
|
//
|
|
|
|
#if defined(__cpp_lib_hardware_interference_size)
|
|
const size_t CACHE_LINE_SIZE = hardware_destructive_interference_size;
|
|
#else
|
|
const size_t CACHE_LINE_SIZE = 64;
|
|
#endif
|
|
|
|
const size_t CACHE_LINE_SIZE_F32 = CACHE_LINE_SIZE/sizeof(float);
|
|
|
|
//
|
|
// fundamental operations
|
|
//
|
|
|
|
inline static void ggml_vec_set_i8(const int n, int8_t * x, const int8_t v) { for (int i = 0; i < n; ++i) x[i] = v; }
|
|
|
|
inline static void ggml_vec_set_i16(const int n, int16_t * x, const int16_t v) { for (int i = 0; i < n; ++i) x[i] = v; }
|
|
|
|
inline static void ggml_vec_set_i32(const int n, int32_t * x, const int32_t v) { for (int i = 0; i < n; ++i) x[i] = v; }
|
|
|
|
inline static void ggml_vec_set_f16(const int n, ggml_fp16_t * x, const int32_t v) { for (int i = 0; i < n; ++i) x[i] = v; }
|
|
|
|
inline static void ggml_vec_add_f32 (const int n, float * z, const float * x, const float * y) { for (int i = 0; i < n; ++i) z[i] = x[i] + y[i]; }
|
|
inline static void ggml_vec_acc_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] += x[i]; }
|
|
inline static void ggml_vec_acc1_f32(const int n, float * y, const float v) { for (int i = 0; i < n; ++i) y[i] += v; }
|
|
inline static void ggml_vec_sub_f32 (const int n, float * z, const float * x, const float * y) { for (int i = 0; i < n; ++i) z[i] = x[i] - y[i]; }
|
|
inline static void ggml_vec_set_f32 (const int n, float * x, const float v) { for (int i = 0; i < n; ++i) x[i] = v; }
|
|
inline static void ggml_vec_cpy_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = x[i]; }
|
|
inline static void ggml_vec_neg_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = -x[i]; }
|
|
inline static void ggml_vec_mul_f32 (const int n, float * z, const float * x, const float * y) { for (int i = 0; i < n; ++i) z[i] = x[i]*y[i]; }
|
|
inline static void ggml_vec_div_f32 (const int n, float * z, const float * x, const float * y) { for (int i = 0; i < n; ++i) z[i] = x[i]/y[i]; }
|
|
|
|
inline static void ggml_vec_dot_f32(const int n, float * restrict s, const float * restrict x, const float * restrict y) {
|
|
ggml_float sumf = 0.0;
|
|
#ifdef __ARM_NEON
|
|
// NEON 128-bit
|
|
const int n16 = (n & ~15);
|
|
|
|
float32x4_t sum0 = vdupq_n_f32(0);
|
|
float32x4_t sum1 = vdupq_n_f32(0);
|
|
float32x4_t sum2 = vdupq_n_f32(0);
|
|
float32x4_t sum3 = vdupq_n_f32(0);
|
|
|
|
float32x4_t x0, x1, x2, x3;
|
|
float32x4_t y0, y1, y2, y3;
|
|
|
|
for (int i = 0; i < n16; i += 16) {
|
|
x0 = vld1q_f32(x + i + 0);
|
|
x1 = vld1q_f32(x + i + 4);
|
|
x2 = vld1q_f32(x + i + 8);
|
|
x3 = vld1q_f32(x + i + 12);
|
|
|
|
y0 = vld1q_f32(y + i + 0);
|
|
y1 = vld1q_f32(y + i + 4);
|
|
y2 = vld1q_f32(y + i + 8);
|
|
y3 = vld1q_f32(y + i + 12);
|
|
|
|
sum0 = vfmaq_f32(sum0, x0, y0);
|
|
sum1 = vfmaq_f32(sum1, x1, y1);
|
|
sum2 = vfmaq_f32(sum2, x2, y2);
|
|
sum3 = vfmaq_f32(sum3, x3, y3);
|
|
}
|
|
|
|
// reduce sum0..sum3 to sum0
|
|
sum0 = vaddq_f32(sum0, sum1);
|
|
sum2 = vaddq_f32(sum2, sum3);
|
|
sum0 = vaddq_f32(sum0, sum2);
|
|
|
|
float32x2_t sumf32 = vadd_f32(vget_low_f32(sum0), vget_high_f32(sum0));
|
|
sumf = vget_lane_f32(sumf32, 0) + vget_lane_f32(sumf32, 1);
|
|
|
|
// leftovers
|
|
for (int i = n16; i < n; ++i) {
|
|
sumf += x[i]*y[i];
|
|
}
|
|
#elif defined(__AVX2__)
|
|
// AVX 256-bit (unroll 4)
|
|
const int n32 = (n & ~31);
|
|
|
|
__m256 sum0 = _mm256_setzero_ps();
|
|
__m256 sum1 = _mm256_setzero_ps();
|
|
__m256 sum2 = _mm256_setzero_ps();
|
|
__m256 sum3 = _mm256_setzero_ps();
|
|
|
|
__m256 x0, x1, x2, x3;
|
|
__m256 y0, y1, y2, y3;
|
|
|
|
for (int i = 0; i < n32; i += 32) {
|
|
x0 = _mm256_loadu_ps(x + i + 0);
|
|
x1 = _mm256_loadu_ps(x + i + 8);
|
|
x2 = _mm256_loadu_ps(x + i + 16);
|
|
x3 = _mm256_loadu_ps(x + i + 24);
|
|
|
|
y0 = _mm256_loadu_ps(y + i + 0);
|
|
y1 = _mm256_loadu_ps(y + i + 8);
|
|
y2 = _mm256_loadu_ps(y + i + 16);
|
|
y3 = _mm256_loadu_ps(y + i + 24);
|
|
|
|
sum0 = _mm256_fmadd_ps(x0, y0, sum0);
|
|
sum1 = _mm256_fmadd_ps(x1, y1, sum1);
|
|
sum2 = _mm256_fmadd_ps(x2, y2, sum2);
|
|
sum3 = _mm256_fmadd_ps(x3, y3, sum3);
|
|
}
|
|
|
|
sum0 = _mm256_add_ps(sum0, sum1);
|
|
sum2 = _mm256_add_ps(sum2, sum3);
|
|
sum0 = _mm256_add_ps(sum0, sum2);
|
|
|
|
const __m128 r4 = _mm_add_ps(_mm256_castps256_ps128(sum0), _mm256_extractf128_ps(sum0, 1));
|
|
const __m128 r2 = _mm_add_ps(r4, _mm_movehl_ps(r4, r4));
|
|
const __m128 r1 = _mm_add_ss(r2, _mm_movehdup_ps(r2));
|
|
|
|
sumf = _mm_cvtss_f32(r1);
|
|
|
|
// leftovers
|
|
for (int i = n32; i < n; ++i) {
|
|
sumf += x[i]*y[i];
|
|
}
|
|
#else
|
|
// scalar
|
|
for (int i = 0; i < n; ++i) {
|
|
sumf += x[i]*y[i];
|
|
}
|
|
#endif
|
|
|
|
*s = sumf;
|
|
}
|
|
|
|
inline static void ggml_vec_dot_f16(const int n, float * restrict s, ggml_fp16_t * restrict x, ggml_fp16_t * restrict y) {
|
|
ggml_float sumf = 0.0;
|
|
#ifdef __ARM_NEON
|
|
const int n32 = (n & ~31);
|
|
|
|
#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
|
|
float16x8_t sum0 = vdupq_n_f16(0);
|
|
float16x8_t sum1 = vdupq_n_f16(0);
|
|
float16x8_t sum2 = vdupq_n_f16(0);
|
|
float16x8_t sum3 = vdupq_n_f16(0);
|
|
|
|
float16x8_t x0, x1, x2, x3;
|
|
float16x8_t y0, y1, y2, y3;
|
|
|
|
for (int i = 0; i < n32; i += 32) {
|
|
x0 = vld1q_f16(x + i + 0 );
|
|
x1 = vld1q_f16(x + i + 8 );
|
|
x2 = vld1q_f16(x + i + 16);
|
|
x3 = vld1q_f16(x + i + 24);
|
|
|
|
y0 = vld1q_f16(y + i + 0 );
|
|
y1 = vld1q_f16(y + i + 8 );
|
|
y2 = vld1q_f16(y + i + 16);
|
|
y3 = vld1q_f16(y + i + 24);
|
|
|
|
sum0 = vfmaq_f16(sum0, x0, y0);
|
|
sum1 = vfmaq_f16(sum1, x1, y1);
|
|
sum2 = vfmaq_f16(sum2, x2, y2);
|
|
sum3 = vfmaq_f16(sum3, x3, y3);
|
|
}
|
|
|
|
// reduce sum0..sum3 to sum0
|
|
sum0 = vaddq_f16(sum0, sum1);
|
|
sum2 = vaddq_f16(sum2, sum3);
|
|
sum0 = vaddq_f16(sum0, sum2);
|
|
|
|
// load sum0 into 2 float32x4_t
|
|
float32x4_t sum0f32 = vcvt_f32_f16(vget_low_f16(sum0));
|
|
float32x4_t sum1f32 = vcvt_f32_f16(vget_high_f16(sum0));
|
|
|
|
// reduce sum0f32 and sum1f32 to sumf
|
|
sum0f32 = vaddq_f32(sum0f32, sum1f32);
|
|
|
|
float32x2_t sumf32 = vadd_f32(vget_low_f32(sum0f32), vget_high_f32(sum0f32));
|
|
sumf = vget_lane_f32(sumf32, 0) + vget_lane_f32(sumf32, 1);
|
|
#else
|
|
float32x4_t sum0 = vdupq_n_f32(0);
|
|
float32x4_t sum1 = vdupq_n_f32(0);
|
|
float32x4_t sum2 = vdupq_n_f32(0);
|
|
float32x4_t sum3 = vdupq_n_f32(0);
|
|
float32x4_t sum4 = vdupq_n_f32(0);
|
|
float32x4_t sum5 = vdupq_n_f32(0);
|
|
float32x4_t sum6 = vdupq_n_f32(0);
|
|
float32x4_t sum7 = vdupq_n_f32(0);
|
|
|
|
float32x4_t x0, x1, x2, x3, x4, x5, x6, x7;
|
|
float32x4_t y0, y1, y2, y3, y4, y5, y6, y7;
|
|
|
|
for (int i = 0; i < n32; i += 32) {
|
|
x0 = vcvt_f32_f16(vld1_f16(x + i + 0 ));
|
|
x1 = vcvt_f32_f16(vld1_f16(x + i + 4 ));
|
|
x2 = vcvt_f32_f16(vld1_f16(x + i + 8 ));
|
|
x3 = vcvt_f32_f16(vld1_f16(x + i + 12));
|
|
x4 = vcvt_f32_f16(vld1_f16(x + i + 16));
|
|
x5 = vcvt_f32_f16(vld1_f16(x + i + 20));
|
|
x6 = vcvt_f32_f16(vld1_f16(x + i + 24));
|
|
x7 = vcvt_f32_f16(vld1_f16(x + i + 28));
|
|
|
|
y0 = vcvt_f32_f16(vld1_f16(y + i + 0 ));
|
|
y1 = vcvt_f32_f16(vld1_f16(y + i + 4 ));
|
|
y2 = vcvt_f32_f16(vld1_f16(y + i + 8 ));
|
|
y3 = vcvt_f32_f16(vld1_f16(y + i + 12));
|
|
y4 = vcvt_f32_f16(vld1_f16(y + i + 16));
|
|
y5 = vcvt_f32_f16(vld1_f16(y + i + 20));
|
|
y6 = vcvt_f32_f16(vld1_f16(y + i + 24));
|
|
y7 = vcvt_f32_f16(vld1_f16(y + i + 28));
|
|
|
|
sum0 = vfmaq_f32(sum0, x0, y0);
|
|
sum1 = vfmaq_f32(sum1, x1, y1);
|
|
sum2 = vfmaq_f32(sum2, x2, y2);
|
|
sum3 = vfmaq_f32(sum3, x3, y3);
|
|
sum4 = vfmaq_f32(sum4, x4, y4);
|
|
sum5 = vfmaq_f32(sum5, x5, y5);
|
|
sum6 = vfmaq_f32(sum6, x6, y6);
|
|
sum7 = vfmaq_f32(sum7, x7, y7);
|
|
}
|
|
|
|
// reduce sum0..sum7 to sum0
|
|
sum0 = vaddq_f32(sum0, sum1);
|
|
sum2 = vaddq_f32(sum2, sum3);
|
|
sum4 = vaddq_f32(sum4, sum5);
|
|
sum6 = vaddq_f32(sum6, sum7);
|
|
sum0 = vaddq_f32(sum0, sum2);
|
|
sum4 = vaddq_f32(sum4, sum6);
|
|
sum0 = vaddq_f32(sum0, sum4);
|
|
|
|
// reduce sum0 to sumf
|
|
float32x2_t sumf32 = vadd_f32(vget_low_f32(sum0), vget_high_f32(sum0));
|
|
sumf = vget_lane_f32(sumf32, 0) + vget_lane_f32(sumf32, 1);
|
|
#endif
|
|
|
|
// leftovers
|
|
for (int i = n32; i < n; ++i) {
|
|
sumf += ggml_fp16_to_fp32(x[i])*ggml_fp16_to_fp32(y[i]);
|
|
}
|
|
#elif defined(__AVX2__)
|
|
// AVX 256-bit (unroll 4)
|
|
const int n32 = (n & ~31);
|
|
|
|
__m256 sum0 = _mm256_setzero_ps();
|
|
__m256 sum1 = _mm256_setzero_ps();
|
|
__m256 sum2 = _mm256_setzero_ps();
|
|
__m256 sum3 = _mm256_setzero_ps();
|
|
|
|
__m256 x0, x1, x2, x3;
|
|
__m256 y0, y1, y2, y3;
|
|
|
|
for (int i = 0; i < n32; i += 32) {
|
|
x0 = _mm256_cvtph_ps(_mm_loadu_si128((__m128i*)(x + i + 0 )));
|
|
x1 = _mm256_cvtph_ps(_mm_loadu_si128((__m128i*)(x + i + 8 )));
|
|
x2 = _mm256_cvtph_ps(_mm_loadu_si128((__m128i*)(x + i + 16)));
|
|
x3 = _mm256_cvtph_ps(_mm_loadu_si128((__m128i*)(x + i + 24)));
|
|
|
|
y0 = _mm256_cvtph_ps(_mm_loadu_si128((__m128i*)(y + i + 0 )));
|
|
y1 = _mm256_cvtph_ps(_mm_loadu_si128((__m128i*)(y + i + 8 )));
|
|
y2 = _mm256_cvtph_ps(_mm_loadu_si128((__m128i*)(y + i + 16)));
|
|
y3 = _mm256_cvtph_ps(_mm_loadu_si128((__m128i*)(y + i + 24)));
|
|
|
|
sum0 = _mm256_fmadd_ps(x0, y0, sum0);
|
|
sum1 = _mm256_fmadd_ps(x1, y1, sum1);
|
|
sum2 = _mm256_fmadd_ps(x2, y2, sum2);
|
|
sum3 = _mm256_fmadd_ps(x3, y3, sum3);
|
|
}
|
|
|
|
const __m256 sum01 = _mm256_add_ps(sum0, sum1);
|
|
const __m256 sum23 = _mm256_add_ps(sum2, sum3);
|
|
const __m256 sum0123 = _mm256_add_ps(sum01, sum23);
|
|
|
|
const __m128 r4 = _mm_add_ps(_mm256_castps256_ps128(sum0123), _mm256_extractf128_ps(sum0123, 1));
|
|
const __m128 r2 = _mm_add_ps(r4, _mm_movehl_ps(r4, r4));
|
|
const __m128 r1 = _mm_add_ss(r2, _mm_movehdup_ps(r2));
|
|
|
|
sumf = _mm_cvtss_f32(r1);
|
|
|
|
// leftovers
|
|
for (int i = n32; i < n; ++i) {
|
|
//GGML_ASSERT(false);
|
|
sumf += ggml_fp16_to_fp32(x[i])*ggml_fp16_to_fp32(y[i]);
|
|
}
|
|
#else
|
|
for (int i = 0; i < n; ++i) {
|
|
sumf += ggml_fp16_to_fp32(x[i])*ggml_fp16_to_fp32(y[i]);
|
|
}
|
|
#endif
|
|
|
|
*s = sumf;
|
|
}
|
|
|
|
inline static void ggml_vec_mad_f32(const int n, float * restrict y, const float * restrict x, const float v) {
|
|
#ifdef __ARM_NEON
|
|
// NEON 128-bit
|
|
const int n16 = (n & ~15);
|
|
|
|
const float32x4_t v4 = vdupq_n_f32(v);
|
|
|
|
float32x4_t x0, x1, x2, x3;
|
|
float32x4_t y0, y1, y2, y3;
|
|
|
|
for (int i = 0; i < n16; i += 16) {
|
|
x0 = vld1q_f32(x + i + 0);
|
|
x1 = vld1q_f32(x + i + 4);
|
|
x2 = vld1q_f32(x + i + 8);
|
|
x3 = vld1q_f32(x + i + 12);
|
|
|
|
y0 = vld1q_f32(y + i + 0);
|
|
y1 = vld1q_f32(y + i + 4);
|
|
y2 = vld1q_f32(y + i + 8);
|
|
y3 = vld1q_f32(y + i + 12);
|
|
|
|
y0 = vfmaq_f32(y0, x0, v4);
|
|
y1 = vfmaq_f32(y1, x1, v4);
|
|
y2 = vfmaq_f32(y2, x2, v4);
|
|
y3 = vfmaq_f32(y3, x3, v4);
|
|
|
|
vst1q_f32(y + i + 0, y0);
|
|
vst1q_f32(y + i + 4, y1);
|
|
vst1q_f32(y + i + 8, y2);
|
|
vst1q_f32(y + i + 12, y3);
|
|
}
|
|
|
|
// leftovers
|
|
for (int i = n16; i < n; ++i) {
|
|
y[i] += x[i]*v;
|
|
}
|
|
#elif defined(__AVX2__)
|
|
// AVX 256-bit (unroll 4)
|
|
const int n32 = (n & ~31);
|
|
|
|
const __m256 v4 = _mm256_set1_ps(v);
|
|
|
|
__m256 x0, x1, x2, x3;
|
|
__m256 y0, y1, y2, y3;
|
|
|
|
for (int i = 0; i < n32; i += 32) {
|
|
x0 = _mm256_loadu_ps(x + i + 0);
|
|
x1 = _mm256_loadu_ps(x + i + 8);
|
|
x2 = _mm256_loadu_ps(x + i + 16);
|
|
x3 = _mm256_loadu_ps(x + i + 24);
|
|
|
|
y0 = _mm256_loadu_ps(y + i + 0);
|
|
y1 = _mm256_loadu_ps(y + i + 8);
|
|
y2 = _mm256_loadu_ps(y + i + 16);
|
|
y3 = _mm256_loadu_ps(y + i + 24);
|
|
|
|
y0 = _mm256_fmadd_ps(x0, v4, y0);
|
|
y1 = _mm256_fmadd_ps(x1, v4, y1);
|
|
y2 = _mm256_fmadd_ps(x2, v4, y2);
|
|
y3 = _mm256_fmadd_ps(x3, v4, y3);
|
|
|
|
_mm256_storeu_ps(y + i + 0, y0);
|
|
_mm256_storeu_ps(y + i + 8, y1);
|
|
_mm256_storeu_ps(y + i + 16, y2);
|
|
_mm256_storeu_ps(y + i + 24, y3);
|
|
}
|
|
|
|
// leftovers
|
|
for (int i = n32; i < n; ++i) {
|
|
y[i] += x[i]*v;
|
|
}
|
|
#else
|
|
// scalar
|
|
for (int i = 0; i < n; ++i) {
|
|
y[i] += x[i]*v;
|
|
}
|
|
#endif
|
|
}
|
|
|
|
inline static void ggml_vec_mad_f16(const int n, ggml_fp16_t * restrict y, ggml_fp16_t * restrict x, const float v) {
|
|
#ifdef __ARM_NEON
|
|
// NEON 128-bit
|
|
const int n32 = (n & ~31);
|
|
|
|
#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
|
|
const float16x8_t v8 = vdupq_n_f16(v);
|
|
|
|
float16x8_t x0, x1, x2, x3;
|
|
float16x8_t y0, y1, y2, y3;
|
|
|
|
for (int i = 0; i < n32; i += 32) {
|
|
y0 = vld1q_f16(y + i + 0 );
|
|
y1 = vld1q_f16(y + i + 8 );
|
|
y2 = vld1q_f16(y + i + 16);
|
|
y3 = vld1q_f16(y + i + 24);
|
|
|
|
x0 = vld1q_f16(x + i + 0 );
|
|
x1 = vld1q_f16(x + i + 8 );
|
|
x2 = vld1q_f16(x + i + 16);
|
|
x3 = vld1q_f16(x + i + 24);
|
|
|
|
y0 = vfmaq_f16(y0, x0, v8);
|
|
y1 = vfmaq_f16(y1, x1, v8);
|
|
y2 = vfmaq_f16(y2, x2, v8);
|
|
y3 = vfmaq_f16(y3, x3, v8);
|
|
|
|
vst1q_f16(y + i + 0 , y0);
|
|
vst1q_f16(y + i + 8 , y1);
|
|
vst1q_f16(y + i + 16, y2);
|
|
vst1q_f16(y + i + 24, y3);
|
|
}
|
|
#else
|
|
const float32x4_t v40 = vdupq_n_f32(v);
|
|
const float32x4_t v41 = vdupq_n_f32(v);
|
|
|
|
float32x4_t x0, x1, x2, x3, x4, x5, x6, x7;
|
|
float32x4_t y0, y1, y2, y3, y4, y5, y6, y7;
|
|
|
|
for (int i = 0; i < n32; i += 32) {
|
|
y0 = vcvt_f32_f16(vld1_f16(y + i + 0 ));
|
|
y1 = vcvt_f32_f16(vld1_f16(y + i + 4 ));
|
|
y2 = vcvt_f32_f16(vld1_f16(y + i + 8 ));
|
|
y3 = vcvt_f32_f16(vld1_f16(y + i + 12));
|
|
y4 = vcvt_f32_f16(vld1_f16(y + i + 16));
|
|
y5 = vcvt_f32_f16(vld1_f16(y + i + 20));
|
|
y6 = vcvt_f32_f16(vld1_f16(y + i + 24));
|
|
y7 = vcvt_f32_f16(vld1_f16(y + i + 28));
|
|
|
|
x0 = vcvt_f32_f16(vld1_f16(x + i + 0 ));
|
|
x1 = vcvt_f32_f16(vld1_f16(x + i + 4 ));
|
|
x2 = vcvt_f32_f16(vld1_f16(x + i + 8 ));
|
|
x3 = vcvt_f32_f16(vld1_f16(x + i + 12));
|
|
x4 = vcvt_f32_f16(vld1_f16(x + i + 16));
|
|
x5 = vcvt_f32_f16(vld1_f16(x + i + 20));
|
|
x6 = vcvt_f32_f16(vld1_f16(x + i + 24));
|
|
x7 = vcvt_f32_f16(vld1_f16(x + i + 28));
|
|
|
|
y0 = vfmaq_f32(y0, x0, v40);
|
|
y1 = vfmaq_f32(y1, x1, v40);
|
|
y2 = vfmaq_f32(y2, x2, v40);
|
|
y3 = vfmaq_f32(y3, x3, v40);
|
|
y4 = vfmaq_f32(y4, x4, v41);
|
|
y5 = vfmaq_f32(y5, x5, v41);
|
|
y6 = vfmaq_f32(y6, x6, v41);
|
|
y7 = vfmaq_f32(y7, x7, v41);
|
|
|
|
vst1_f16(y + i + 0 , vcvt_f16_f32(y0));
|
|
vst1_f16(y + i + 4 , vcvt_f16_f32(y1));
|
|
vst1_f16(y + i + 8 , vcvt_f16_f32(y2));
|
|
vst1_f16(y + i + 12, vcvt_f16_f32(y3));
|
|
vst1_f16(y + i + 16, vcvt_f16_f32(y4));
|
|
vst1_f16(y + i + 20, vcvt_f16_f32(y5));
|
|
vst1_f16(y + i + 24, vcvt_f16_f32(y6));
|
|
vst1_f16(y + i + 28, vcvt_f16_f32(y7));
|
|
}
|
|
#endif
|
|
|
|
// leftovers
|
|
for (int i = n32; i < n; ++i) {
|
|
GGML_ASSERT(false);
|
|
y[i] = ggml_fp32_to_fp16(ggml_fp16_to_fp32(y[i]) + ggml_fp16_to_fp32(x[i])*v);
|
|
}
|
|
#elif defined(__AVX2__)
|
|
// AVX 256-bit
|
|
const int n32 = (n & ~31);
|
|
|
|
const __m256 v8 = _mm256_set1_ps(v);
|
|
|
|
__m256 x0, x1, x2, x3;
|
|
__m256 y0, y1, y2, y3;
|
|
|
|
for (int i = 0; i < n32; i += 32) {
|
|
y0 = _mm256_cvtph_ps(_mm_loadu_si128((__m128i*)(y + i + 0 )));
|
|
y1 = _mm256_cvtph_ps(_mm_loadu_si128((__m128i*)(y + i + 8 )));
|
|
y2 = _mm256_cvtph_ps(_mm_loadu_si128((__m128i*)(y + i + 16)));
|
|
y3 = _mm256_cvtph_ps(_mm_loadu_si128((__m128i*)(y + i + 24)));
|
|
|
|
x0 = _mm256_cvtph_ps(_mm_loadu_si128((__m128i*)(x + i + 0 )));
|
|
x1 = _mm256_cvtph_ps(_mm_loadu_si128((__m128i*)(x + i + 8 )));
|
|
x2 = _mm256_cvtph_ps(_mm_loadu_si128((__m128i*)(x + i + 16)));
|
|
x3 = _mm256_cvtph_ps(_mm_loadu_si128((__m128i*)(x + i + 24)));
|
|
|
|
y0 = _mm256_fmadd_ps(x0, v8, y0);
|
|
y1 = _mm256_fmadd_ps(x1, v8, y1);
|
|
y2 = _mm256_fmadd_ps(x2, v8, y2);
|
|
y3 = _mm256_fmadd_ps(x3, v8, y3);
|
|
|
|
_mm_storeu_si128((__m128i*)(y + i + 0 ), _mm256_cvtps_ph(y0, 0));
|
|
_mm_storeu_si128((__m128i*)(y + i + 8 ), _mm256_cvtps_ph(y1, 0));
|
|
_mm_storeu_si128((__m128i*)(y + i + 16), _mm256_cvtps_ph(y2, 0));
|
|
_mm_storeu_si128((__m128i*)(y + i + 24), _mm256_cvtps_ph(y3, 0));
|
|
}
|
|
|
|
// leftovers
|
|
for (int i = n32; i < n; ++i) {
|
|
GGML_ASSERT(false);
|
|
y[i] = ggml_fp32_to_fp16(ggml_fp16_to_fp32(y[i]) + ggml_fp16_to_fp32(x[i])*v);
|
|
}
|
|
#else
|
|
for (int i = 0; i < n; ++i) {
|
|
y[i] = ggml_fp32_to_fp16(ggml_fp16_to_fp32(y[i]) + ggml_fp16_to_fp32(x[i])*v);
|
|
}
|
|
#endif
|
|
}
|
|
|
|
inline static void ggml_vec_scale_f32(const int n, float * y, const float v) { for (int i = 0; i < n; ++i) y[i] *= v; }
|
|
inline static void ggml_vec_norm_f32 (const int n, float * s, const float * x) { ggml_vec_dot_f32(n, s, x, x); *s = sqrt(*s); }
|
|
inline static void ggml_vec_sqr_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = x[i]*x[i]; }
|
|
inline static void ggml_vec_sqrt_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = sqrt(x[i]); }
|
|
inline static void ggml_vec_abs_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = fabsf(x[i]); }
|
|
inline static void ggml_vec_sgn_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = (x[i] > 0.f) ? 1.f : ((x[i] < 0.f) ? -1.f : 0.f); }
|
|
inline static void ggml_vec_step_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = (x[i] > 0.f) ? 1.f : 0.f; }
|
|
inline static void ggml_vec_relu_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = (x[i] > 0.f) ? x[i] : 0.f; }
|
|
|
|
const ggml_float GELU_COEF_A = 0.044715;
|
|
const ggml_float SQRT_2_OVER_PI = 0.79788456080286535587989211986876;
|
|
|
|
inline static float ggml_gelu_f32(float x) {
|
|
return 0.5*x*(1.0 + tanh(SQRT_2_OVER_PI*x*(1.0 + GELU_COEF_A*x*x)));
|
|
}
|
|
|
|
inline static void ggml_vec_gelu_f32(const int n, float * y, const float * x) {
|
|
for (int i = 0; i < n; ++i) {
|
|
y[i] = ggml_gelu_f32(x[i]);
|
|
}
|
|
}
|
|
|
|
inline static void ggml_vec_gelu_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {
|
|
const uint16_t * i16 = (const uint16_t *) x;
|
|
for (int i = 0; i < n; ++i) {
|
|
y[i] = table_gelu_f16[i16[i]];
|
|
}
|
|
}
|
|
|
|
inline static void ggml_vec_sum_f32 (const int n, float * s, const float * x) { ggml_float sum = 0.0; for (int i = 0; i < n; ++i) sum += x[i]; *s += sum; }
|
|
inline static void ggml_vec_norm_inv_f32(const int n, float * s, const float * x) { ggml_vec_norm_f32(n, s, x); *s = 1./(*s); }
|
|
|
|
//
|
|
// logging
|
|
//
|
|
|
|
#if (GGML_DEBUG >= 1)
|
|
#define GGML_PRINT_DEBUG(...) printf(__VA_ARGS__)
|
|
#else
|
|
#define GGML_PRINT_DEBUG(...)
|
|
#endif
|
|
|
|
#if (GGML_DEBUG >= 5)
|
|
#define GGML_PRINT_DEBUG_5(...) printf(__VA_ARGS__)
|
|
#else
|
|
#define GGML_PRINT_DEBUG_5(...)
|
|
#endif
|
|
|
|
#if (GGML_DEBUG >= 10)
|
|
#define GGML_PRINT_DEBUG_10(...) printf(__VA_ARGS__)
|
|
#else
|
|
#define GGML_PRINT_DEBUG_10(...)
|
|
#endif
|
|
|
|
#define GGML_PRINT(...) printf(__VA_ARGS__)
|
|
|
|
//
|
|
// data types
|
|
//
|
|
|
|
const size_t GGML_TYPE_SIZE[GGML_TYPE_COUNT] = {
|
|
sizeof(int8_t ),
|
|
sizeof(int16_t),
|
|
sizeof(int32_t),
|
|
sizeof(ggml_fp16_t),
|
|
sizeof(float ),
|
|
};
|
|
|
|
const char * GGML_OP_LABEL[GGML_OP_COUNT] = {
|
|
"NONE",
|
|
|
|
"DUP",
|
|
"ADD",
|
|
"SUB",
|
|
"MUL",
|
|
"DIV",
|
|
"SQR",
|
|
"SQRT",
|
|
"SUM",
|
|
"MEAN",
|
|
"REPEAT",
|
|
"ABS",
|
|
"SGN",
|
|
"NEG",
|
|
"STEP",
|
|
"RELU",
|
|
"GELU",
|
|
"NORM",
|
|
|
|
"MUL_MAT",
|
|
|
|
"SCALE",
|
|
"CPY",
|
|
"RESHAPE",
|
|
"VIEW",
|
|
"PERMUTE",
|
|
"TRANSPOSE",
|
|
"GET_ROWS",
|
|
"DIAG_MASK_INF",
|
|
"SOFT_MAX",
|
|
"ROPE",
|
|
"CONV_1D_1S",
|
|
"CONV_1D_2S",
|
|
|
|
"FLASH_ATTN",
|
|
"FLASH_FF",
|
|
};
|
|
|
|
const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = {
|
|
"none",
|
|
|
|
"x",
|
|
"x+y",
|
|
"x-y",
|
|
"x*y",
|
|
"x/y",
|
|
"x^2",
|
|
"√x",
|
|
"Σx",
|
|
"Σx/n",
|
|
"repeat(x)",
|
|
"abs(x)",
|
|
"sgn(x)",
|
|
"-x",
|
|
"step(x)",
|
|
"relu(x)",
|
|
"gelu(x)",
|
|
"norm(x)",
|
|
|
|
"X*Y",
|
|
|
|
"x*v",
|
|
"x-\\>y",
|
|
"reshape(x)",
|
|
"view(x)",
|
|
"permute(x)",
|
|
"transpose(x)",
|
|
"get_rows(x)",
|
|
"diag_mask_inf(x)",
|
|
"soft_max(x)",
|
|
"rope(x)",
|
|
"conv_1d_1s(x)",
|
|
"conv_1d_2s(x)",
|
|
|
|
"flash_attn(x)",
|
|
"flash_ff(x)",
|
|
};
|
|
|
|
//
|
|
// ggml object
|
|
//
|
|
|
|
struct ggml_object {
|
|
size_t offset;
|
|
size_t size;
|
|
|
|
struct ggml_object * next;
|
|
|
|
char padding[8];
|
|
};
|
|
|
|
const size_t GGML_OBJECT_SIZE = sizeof(struct ggml_object);
|
|
|
|
static_assert(sizeof(struct ggml_object)%GGML_MEM_ALIGN == 0, "ggml_object size must be a multiple of GGML_MEM_ALIGN");
|
|
static_assert(sizeof(struct ggml_tensor)%GGML_MEM_ALIGN == 0, "ggml_tensor size must be a multiple of GGML_MEM_ALIGN");
|
|
|
|
//
|
|
// ggml context
|
|
//
|
|
|
|
struct ggml_context {
|
|
size_t mem_size;
|
|
void * mem_buffer;
|
|
bool mem_buffer_owned;
|
|
|
|
int n_objects;
|
|
|
|
struct ggml_object * objects_begin;
|
|
struct ggml_object * objects_end;
|
|
};
|
|
|
|
struct ggml_context_container {
|
|
bool used;
|
|
|
|
struct ggml_context context;
|
|
};
|
|
|
|
//
|
|
// compute types
|
|
//
|
|
|
|
enum ggml_task_type {
|
|
GGML_TASK_INIT = 0,
|
|
GGML_TASK_COMPUTE,
|
|
GGML_TASK_FINALIZE,
|
|
};
|
|
|
|
struct ggml_compute_params {
|
|
enum ggml_task_type type;
|
|
|
|
int ith, nth;
|
|
|
|
// work buffer for all threads
|
|
size_t wsize;
|
|
void * wdata;
|
|
};
|
|
|
|
//
|
|
// ggml state
|
|
//
|
|
|
|
struct ggml_state {
|
|
struct ggml_context_container contexts[GGML_MAX_CONTEXTS];
|
|
};
|
|
|
|
// global state
|
|
struct ggml_state g_state;
|
|
|
|
////////////////////////////////////////////////////////////////////////////////
|
|
|
|
void ggml_print_object(const struct ggml_object * obj) {
|
|
GGML_PRINT(" - ggml_object: offset = %zu, size = %zu, next = %p\n",
|
|
obj->offset, obj->size, (const void *) obj->next);
|
|
}
|
|
|
|
void ggml_print_objects(const struct ggml_context * ctx) {
|
|
struct ggml_object * obj = ctx->objects_begin;
|
|
|
|
GGML_PRINT("%s: objects in context %p:\n", __func__, (const void *) ctx);
|
|
|
|
while (obj != NULL) {
|
|
ggml_print_object(obj);
|
|
obj = obj->next;
|
|
}
|
|
|
|
GGML_PRINT("%s: --- end ---\n", __func__);
|
|
}
|
|
|
|
int ggml_nelements(const struct ggml_tensor * tensor) {
|
|
static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function");
|
|
|
|
return tensor->ne[0]*tensor->ne[1]*tensor->ne[2]*tensor->ne[3];
|
|
}
|
|
|
|
int ggml_nrows(const struct ggml_tensor * tensor) {
|
|
static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function");
|
|
|
|
return tensor->ne[1]*tensor->ne[2]*tensor->ne[3];
|
|
}
|
|
|
|
size_t ggml_nbytes(const struct ggml_tensor * tensor) {
|
|
static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function");
|
|
|
|
return ggml_nelements(tensor)*GGML_TYPE_SIZE[tensor->type];
|
|
}
|
|
|
|
size_t ggml_type_size(enum ggml_type type) {
|
|
return GGML_TYPE_SIZE[type];
|
|
}
|
|
|
|
size_t ggml_element_size(const struct ggml_tensor * tensor) {
|
|
return GGML_TYPE_SIZE[tensor->type];
|
|
}
|
|
|
|
bool ggml_is_scalar(const struct ggml_tensor * tensor) {
|
|
static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function");
|
|
|
|
return tensor->ne[0] == 1 && tensor->ne[1] == 1 && tensor->ne[2] == 1 && tensor->ne[3] == 1;
|
|
}
|
|
|
|
bool ggml_is_vector(const struct ggml_tensor * tensor) {
|
|
static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function");
|
|
|
|
return tensor->ne[1] == 1 && tensor->ne[2] == 1 && tensor->ne[3] == 1;
|
|
}
|
|
|
|
bool ggml_is_matrix(const struct ggml_tensor * tensor) {
|
|
static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function");
|
|
|
|
return tensor->ne[2] == 1 && tensor->ne[3] == 1;
|
|
}
|
|
|
|
bool ggml_can_mul_mat(const struct ggml_tensor * t0, const struct ggml_tensor * t1) {
|
|
static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function");
|
|
|
|
return
|
|
(t0->ne[0] == t1->ne[0]) &&
|
|
(t0->ne[2] == t1->ne[2]) &&
|
|
(t0->ne[3] == t1->ne[3]);
|
|
}
|
|
|
|
bool ggml_is_contiguous(const struct ggml_tensor * tensor) {
|
|
static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function");
|
|
|
|
return
|
|
tensor->nb[0] == GGML_TYPE_SIZE[tensor->type] &&
|
|
tensor->nb[1] == tensor->nb[0]*tensor->ne[0] &&
|
|
tensor->nb[2] == tensor->nb[1]*tensor->ne[1] &&
|
|
tensor->nb[3] == tensor->nb[2]*tensor->ne[2];
|
|
}
|
|
|
|
bool ggml_is_padded_1d(const struct ggml_tensor * tensor) {
|
|
static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function");
|
|
|
|
return
|
|
tensor->nb[0] == GGML_TYPE_SIZE[tensor->type] &&
|
|
tensor->nb[2] == tensor->nb[1]*tensor->ne[1] &&
|
|
tensor->nb[3] == tensor->nb[2]*tensor->ne[2];;
|
|
}
|
|
|
|
bool ggml_are_same_shape(const struct ggml_tensor * t0, const struct ggml_tensor * t1) {
|
|
static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function");
|
|
|
|
return
|
|
(t0->ne[0] == t1->ne[0] ) &&
|
|
(t0->ne[1] == t1->ne[1] ) &&
|
|
(t0->ne[2] == t1->ne[2] ) &&
|
|
(t0->ne[3] == t1->ne[3] );
|
|
}
|
|
|
|
// check if t1 can be represented as a repeatition of t0
|
|
bool ggml_can_repeat(const struct ggml_tensor * t0, const struct ggml_tensor * t1) {
|
|
static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function");
|
|
|
|
return
|
|
(t1->ne[0]%t0->ne[0] == 0) &&
|
|
(t1->ne[1]%t0->ne[1] == 0) &&
|
|
(t1->ne[2]%t0->ne[2] == 0) &&
|
|
(t1->ne[3]%t0->ne[3] == 0);
|
|
}
|
|
|
|
int ggml_up32(int n) {
|
|
return (n + 31) & ~31;
|
|
}
|
|
|
|
int ggml_up64(int n) {
|
|
return (n + 63) & ~63;
|
|
}
|
|
|
|
// assert that pointer is aligned to GGML_MEM_ALIGN
|
|
#define ggml_assert_aligned(ptr) \
|
|
assert(((uintptr_t) (ptr))%GGML_MEM_ALIGN == 0)
|
|
|
|
////////////////////////////////////////////////////////////////////////////////
|
|
|
|
struct ggml_context * ggml_init(struct ggml_init_params params) {
|
|
static bool is_first_call = true;
|
|
if (is_first_call) {
|
|
const uint64_t t_start = ggml_time_us(); UNUSED(t_start);
|
|
|
|
ggml_fp16_t ii;
|
|
for (int i = 0; i < (1 << 16); ++i) {
|
|
uint16_t ui = i;
|
|
memcpy(&ii, &ui, sizeof(ii));
|
|
const float f = ggml_fp16_to_fp32(ii);
|
|
table_gelu_f16[i] = ggml_fp32_to_fp16(ggml_gelu_f32(f));
|
|
table_exp_f16[i] = ggml_fp32_to_fp16(exp(f));
|
|
}
|
|
|
|
const uint64_t t_end = ggml_time_us(); UNUSED(t_end);
|
|
|
|
GGML_PRINT_DEBUG("%s: GELU and EXP tables initialized in %f ms\n", __func__, (t_end - t_start)/1000.0f);
|
|
|
|
is_first_call = false;
|
|
}
|
|
|
|
// find non-used context in g_state
|
|
struct ggml_context * ctx = NULL;
|
|
|
|
static bool first_time = true;
|
|
if (first_time) {
|
|
for (int i = 0; i < GGML_MAX_CONTEXTS; i++) {
|
|
g_state.contexts[i].used = false;
|
|
}
|
|
first_time = false;
|
|
}
|
|
|
|
for (int i = 0; i < GGML_MAX_CONTEXTS; i++) {
|
|
if (!g_state.contexts[i].used) {
|
|
g_state.contexts[i].used = true;
|
|
ctx = &g_state.contexts[i].context;
|
|
|
|
GGML_PRINT_DEBUG("%s: found unused context %d\n", __func__, i);
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (ctx == NULL) {
|
|
GGML_PRINT_DEBUG("%s: no unused context found\n", __func__);
|
|
return NULL;
|
|
}
|
|
|
|
*ctx = (struct ggml_context) {
|
|
.mem_size = params.mem_size,
|
|
.mem_buffer = params.mem_buffer ? params.mem_buffer : malloc(params.mem_size),
|
|
.mem_buffer_owned = params.mem_buffer ? false : true,
|
|
.n_objects = 0,
|
|
.objects_begin = NULL,
|
|
.objects_end = NULL,
|
|
};
|
|
|
|
ggml_assert_aligned(ctx->mem_buffer);
|
|
|
|
return ctx;
|
|
}
|
|
|
|
void ggml_free(struct ggml_context * ctx) {
|
|
for (int i = 0; i < GGML_MAX_CONTEXTS; i++) {
|
|
if (&g_state.contexts[i].context == ctx) {
|
|
g_state.contexts[i].used = false;
|
|
|
|
GGML_PRINT_DEBUG("%s: context %d with %d objects has been freed. memory used = %zu\n",
|
|
__func__, i, ctx->n_objects, ctx->objects_end->offset + ctx->objects_end->size);
|
|
|
|
if (ctx->mem_buffer_owned) {
|
|
free(ctx->mem_buffer);
|
|
}
|
|
|
|
return;
|
|
}
|
|
}
|
|
|
|
GGML_PRINT_DEBUG("%s: context not found\n", __func__);
|
|
}
|
|
|
|
size_t ggml_used_mem(const struct ggml_context * ctx) {
|
|
return ctx->objects_end->offset + ctx->objects_end->size;
|
|
}
|
|
|
|
////////////////////////////////////////////////////////////////////////////////
|
|
|
|
struct ggml_tensor * ggml_new_tensor_impl(
|
|
struct ggml_context * ctx,
|
|
enum ggml_type type,
|
|
int n_dims,
|
|
const int* ne,
|
|
void* data) {
|
|
// always insert objects at the end of the context's memory pool
|
|
struct ggml_object * obj_cur = ctx->objects_end;
|
|
|
|
const size_t cur_offset = obj_cur == NULL ? 0 : obj_cur->offset;
|
|
const size_t cur_size = obj_cur == NULL ? 0 : obj_cur->size;
|
|
const size_t cur_end = cur_offset + cur_size;
|
|
|
|
size_t size_needed = 0;
|
|
|
|
if (data == NULL) {
|
|
size_needed += GGML_TYPE_SIZE[type];
|
|
for (int i = 0; i < n_dims; i++) {
|
|
size_needed *= ne[i];
|
|
}
|
|
// align to GGML_MEM_ALIGN
|
|
size_needed = ((size_needed + GGML_MEM_ALIGN - 1)/GGML_MEM_ALIGN)*GGML_MEM_ALIGN;
|
|
|
|
}
|
|
size_needed += sizeof(struct ggml_tensor);
|
|
|
|
if (cur_end + size_needed + GGML_OBJECT_SIZE > ctx->mem_size) {
|
|
GGML_PRINT("%s: not enough space in the context's memory pool\n", __func__);
|
|
assert(false);
|
|
return NULL;
|
|
}
|
|
|
|
char * const mem_buffer = ctx->mem_buffer;
|
|
|
|
struct ggml_object * const obj_new = (struct ggml_object *)(mem_buffer + cur_end);
|
|
|
|
*obj_new = (struct ggml_object) {
|
|
.offset = cur_end + GGML_OBJECT_SIZE,
|
|
.size = size_needed,
|
|
.next = NULL,
|
|
};
|
|
|
|
if (obj_cur != NULL) {
|
|
obj_cur->next = obj_new;
|
|
} else {
|
|
// this is the first object in this context
|
|
ctx->objects_begin = obj_new;
|
|
}
|
|
|
|
ctx->objects_end = obj_new;
|
|
|
|
//GGML_PRINT_DEBUG("%s: inserted new object at %zu\n", __func__, cur_end);
|
|
|
|
struct ggml_tensor * const result = (struct ggml_tensor *)(mem_buffer + obj_new->offset);
|
|
|
|
ggml_assert_aligned(result);
|
|
|
|
*result = (struct ggml_tensor) {
|
|
/*.type =*/ type,
|
|
/*.n_dims =*/ n_dims,
|
|
/*.ne =*/ { 1, 1, 1, 1 },
|
|
/*.nb =*/ { 0, 0, 0, 0 },
|
|
/*.op =*/ GGML_OP_NONE,
|
|
/*.is_param =*/ false,
|
|
/*.grad =*/ NULL,
|
|
/*.src0 =*/ NULL,
|
|
/*.src1 =*/ NULL,
|
|
/*.opt =*/ { NULL },
|
|
/*.n_tasks =*/ 0,
|
|
/*.perf_runs =*/ 0,
|
|
/*.perf_cycles =*/ 0,
|
|
/*.perf_time_us =*/ 0,
|
|
/*.data =*/ data == NULL ? (void *)(result + 1) : data,
|
|
/*.pad =*/ { 0 },
|
|
};
|
|
|
|
ggml_assert_aligned(result->data);
|
|
|
|
for (int i = 0; i < n_dims; i++) {
|
|
result->ne[i] = ne[i];
|
|
}
|
|
|
|
result->nb[0] = GGML_TYPE_SIZE[type];
|
|
for (int i = 1; i < GGML_MAX_DIMS; i++) {
|
|
result->nb[i] = result->nb[i - 1]*result->ne[i - 1];
|
|
}
|
|
|
|
ctx->n_objects++;
|
|
|
|
return result;
|
|
}
|
|
|
|
struct ggml_tensor * ggml_new_tensor(
|
|
struct ggml_context * ctx,
|
|
enum ggml_type type,
|
|
int n_dims,
|
|
const int* ne) {
|
|
return ggml_new_tensor_impl(ctx, type, n_dims, ne, NULL);
|
|
}
|
|
|
|
struct ggml_tensor * ggml_new_tensor_1d(
|
|
struct ggml_context * ctx,
|
|
enum ggml_type type,
|
|
int ne0) {
|
|
return ggml_new_tensor(ctx, type, 1, &ne0);
|
|
}
|
|
|
|
struct ggml_tensor * ggml_new_tensor_2d(
|
|
struct ggml_context * ctx,
|
|
enum ggml_type type,
|
|
int ne0,
|
|
int ne1) {
|
|
const int ne[2] = { ne0, ne1 };
|
|
return ggml_new_tensor(ctx, type, 2, ne);
|
|
}
|
|
|
|
struct ggml_tensor * ggml_new_tensor_3d(
|
|
struct ggml_context * ctx,
|
|
enum ggml_type type,
|
|
int ne0,
|
|
int ne1,
|
|
int ne2) {
|
|
const int ne[3] = { ne0, ne1, ne2 };
|
|
return ggml_new_tensor(ctx, type, 3, ne);
|
|
}
|
|
|
|
struct ggml_tensor * ggml_new_tensor_4d(
|
|
struct ggml_context * ctx,
|
|
enum ggml_type type,
|
|
int ne0,
|
|
int ne1,
|
|
int ne2,
|
|
int ne3) {
|
|
const int ne[4] = { ne0, ne1, ne2, ne3 };
|
|
return ggml_new_tensor(ctx, type, 4, ne);
|
|
}
|
|
|
|
struct ggml_tensor * ggml_new_i32(struct ggml_context * ctx, int32_t value) {
|
|
struct ggml_tensor * result = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 1);
|
|
|
|
ggml_set_i32(result, value);
|
|
|
|
return result;
|
|
}
|
|
|
|
struct ggml_tensor * ggml_new_f32(struct ggml_context * ctx, float value) {
|
|
struct ggml_tensor * result = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1);
|
|
|
|
ggml_set_f32(result, value);
|
|
|
|
return result;
|
|
}
|
|
|
|
struct ggml_tensor * ggml_dup_tensor(struct ggml_context * ctx, const struct ggml_tensor * src) {
|
|
return ggml_new_tensor_impl(ctx, src->type, src->n_dims, src->ne, NULL);
|
|
}
|
|
|
|
struct ggml_tensor * ggml_set_zero(struct ggml_tensor * tensor) {
|
|
memset(tensor->data, 0, ggml_nbytes(tensor));
|
|
return tensor;
|
|
}
|
|
|
|
struct ggml_tensor * ggml_set_i32 (struct ggml_tensor * tensor, int32_t value) {
|
|
const int n = ggml_nrows(tensor);
|
|
const int nc = tensor->ne[0];
|
|
const size_t n1 = tensor->nb[1];
|
|
|
|
char * const data = tensor->data;
|
|
|
|
switch (tensor->type) {
|
|
case GGML_TYPE_I8:
|
|
{
|
|
assert(tensor->nb[0] == sizeof(int8_t));
|
|
for (int i = 0; i < n; i++) {
|
|
ggml_vec_set_i8(nc, (int8_t *)(data + i*n1), value);
|
|
}
|
|
} break;
|
|
case GGML_TYPE_I16:
|
|
{
|
|
assert(tensor->nb[0] == sizeof(int16_t));
|
|
for (int i = 0; i < n; i++) {
|
|
ggml_vec_set_i16(nc, (int16_t *)(data + i*n1), value);
|
|
}
|
|
} break;
|
|
case GGML_TYPE_I32:
|
|
{
|
|
assert(tensor->nb[0] == sizeof(int32_t));
|
|
for (int i = 0; i < n; i++) {
|
|
ggml_vec_set_i32(nc, (int32_t *)(data + i*n1), value);
|
|
}
|
|
} break;
|
|
case GGML_TYPE_F16:
|
|
{
|
|
assert(tensor->nb[0] == sizeof(ggml_fp16_t));
|
|
for (int i = 0; i < n; i++) {
|
|
ggml_vec_set_f16(nc, (ggml_fp16_t *)(data + i*n1), value);
|
|
}
|
|
} break;
|
|
case GGML_TYPE_F32:
|
|
{
|
|
assert(tensor->nb[0] == sizeof(float));
|
|
for (int i = 0; i < n; i++) {
|
|
ggml_vec_set_f32(nc, (float *)(data + i*n1), value);
|
|
}
|
|
} break;
|
|
case GGML_TYPE_COUNT:
|
|
{
|
|
assert(false);
|
|
} break;
|
|
}
|
|
|
|
return tensor;
|
|
}
|
|
|
|
struct ggml_tensor * ggml_set_f32(struct ggml_tensor * tensor, float value) {
|
|
const int n = ggml_nrows(tensor);
|
|
const int nc = tensor->ne[0];
|
|
const size_t n1 = tensor->nb[1];
|
|
|
|
char * const data = tensor->data;
|
|
|
|
switch (tensor->type) {
|
|
case GGML_TYPE_I8:
|
|
{
|
|
assert(tensor->nb[0] == sizeof(int8_t));
|
|
for (int i = 0; i < n; i++) {
|
|
ggml_vec_set_i8(nc, (int8_t *)(data + i*n1), value);
|
|
}
|
|
} break;
|
|
case GGML_TYPE_I16:
|
|
{
|
|
assert(tensor->nb[0] == sizeof(int16_t));
|
|
for (int i = 0; i < n; i++) {
|
|
ggml_vec_set_i16(nc, (int16_t *)(data + i*n1), value);
|
|
}
|
|
} break;
|
|
case GGML_TYPE_I32:
|
|
{
|
|
assert(tensor->nb[0] == sizeof(int32_t));
|
|
for (int i = 0; i < n; i++) {
|
|
ggml_vec_set_i32(nc, (int32_t *)(data + i*n1), value);
|
|
}
|
|
} break;
|
|
case GGML_TYPE_F16:
|
|
{
|
|
assert(tensor->nb[0] == sizeof(ggml_fp16_t));
|
|
for (int i = 0; i < n; i++) {
|
|
ggml_vec_set_f16(nc, (ggml_fp16_t *)(data + i*n1), value);
|
|
}
|
|
} break;
|
|
case GGML_TYPE_F32:
|
|
{
|
|
assert(tensor->nb[0] == sizeof(float));
|
|
for (int i = 0; i < n; i++) {
|
|
ggml_vec_set_f32(nc, (float *)(data + i*n1), value);
|
|
}
|
|
} break;
|
|
case GGML_TYPE_COUNT:
|
|
{
|
|
assert(false);
|
|
} break;
|
|
}
|
|
|
|
return tensor;
|
|
}
|
|
|
|
int32_t ggml_get_i32_1d(const struct ggml_tensor * tensor, int i) {
|
|
switch (tensor->type) {
|
|
case GGML_TYPE_I8:
|
|
{
|
|
GGML_ASSERT(tensor->nb[0] == sizeof(int8_t));
|
|
return ((int8_t *)(tensor->data))[i];
|
|
} break;
|
|
case GGML_TYPE_I16:
|
|
{
|
|
GGML_ASSERT(tensor->nb[0] == sizeof(int16_t));
|
|
return ((int16_t *)(tensor->data))[i];
|
|
} break;
|
|
case GGML_TYPE_I32:
|
|
{
|
|
GGML_ASSERT(tensor->nb[0] == sizeof(int32_t));
|
|
return ((int32_t *)(tensor->data))[i];
|
|
} break;
|
|
case GGML_TYPE_F16:
|
|
{
|
|
GGML_ASSERT(tensor->nb[0] == sizeof(ggml_fp16_t));
|
|
return ggml_fp16_to_fp32(((ggml_fp16_t *)(tensor->data))[i]);
|
|
} break;
|
|
case GGML_TYPE_F32:
|
|
{
|
|
GGML_ASSERT(tensor->nb[0] == sizeof(float));
|
|
return ((float *)(tensor->data))[i];
|
|
} break;
|
|
case GGML_TYPE_COUNT:
|
|
{
|
|
GGML_ASSERT(false);
|
|
} break;
|
|
}
|
|
|
|
return 0.0f;
|
|
}
|
|
|
|
void ggml_set_i32_1d(const struct ggml_tensor * tensor, int i, int32_t value) {
|
|
switch (tensor->type) {
|
|
case GGML_TYPE_I8:
|
|
{
|
|
GGML_ASSERT(tensor->nb[0] == sizeof(int8_t));
|
|
((int8_t *)(tensor->data))[i] = value;
|
|
} break;
|
|
case GGML_TYPE_I16:
|
|
{
|
|
GGML_ASSERT(tensor->nb[0] == sizeof(int16_t));
|
|
((int16_t *)(tensor->data))[i] = value;
|
|
} break;
|
|
case GGML_TYPE_I32:
|
|
{
|
|
GGML_ASSERT(tensor->nb[0] == sizeof(int32_t));
|
|
((int32_t *)(tensor->data))[i] = value;
|
|
} break;
|
|
case GGML_TYPE_F16:
|
|
{
|
|
GGML_ASSERT(tensor->nb[0] == sizeof(ggml_fp16_t));
|
|
((ggml_fp16_t *)(tensor->data))[i] = ggml_fp32_to_fp16(value);
|
|
} break;
|
|
case GGML_TYPE_F32:
|
|
{
|
|
GGML_ASSERT(tensor->nb[0] == sizeof(float));
|
|
((float *)(tensor->data))[i] = value;
|
|
} break;
|
|
case GGML_TYPE_COUNT:
|
|
{
|
|
GGML_ASSERT(false);
|
|
} break;
|
|
}
|
|
}
|
|
|
|
float ggml_get_f32_1d(const struct ggml_tensor * tensor, int i) {
|
|
switch (tensor->type) {
|
|
case GGML_TYPE_I8:
|
|
{
|
|
GGML_ASSERT(tensor->nb[0] == sizeof(int8_t));
|
|
return ((int8_t *)(tensor->data))[i];
|
|
} break;
|
|
case GGML_TYPE_I16:
|
|
{
|
|
GGML_ASSERT(tensor->nb[0] == sizeof(int16_t));
|
|
return ((int16_t *)(tensor->data))[i];
|
|
} break;
|
|
case GGML_TYPE_I32:
|
|
{
|
|
GGML_ASSERT(tensor->nb[0] == sizeof(int32_t));
|
|
return ((int32_t *)(tensor->data))[i];
|
|
} break;
|
|
case GGML_TYPE_F16:
|
|
{
|
|
GGML_ASSERT(tensor->nb[0] == sizeof(ggml_fp16_t));
|
|
return ggml_fp16_to_fp32(((ggml_fp16_t *)(tensor->data))[i]);
|
|
} break;
|
|
case GGML_TYPE_F32:
|
|
{
|
|
GGML_ASSERT(tensor->nb[0] == sizeof(float));
|
|
return ((float *)(tensor->data))[i];
|
|
} break;
|
|
case GGML_TYPE_COUNT:
|
|
{
|
|
GGML_ASSERT(false);
|
|
} break;
|
|
}
|
|
|
|
return 0.0f;
|
|
}
|
|
|
|
void ggml_set_f32_1d(const struct ggml_tensor * tensor, int i, float value) {
|
|
switch (tensor->type) {
|
|
case GGML_TYPE_I8:
|
|
{
|
|
GGML_ASSERT(tensor->nb[0] == sizeof(int8_t));
|
|
((int8_t *)(tensor->data))[i] = value;
|
|
} break;
|
|
case GGML_TYPE_I16:
|
|
{
|
|
GGML_ASSERT(tensor->nb[0] == sizeof(int16_t));
|
|
((int16_t *)(tensor->data))[i] = value;
|
|
} break;
|
|
case GGML_TYPE_I32:
|
|
{
|
|
GGML_ASSERT(tensor->nb[0] == sizeof(int32_t));
|
|
((int32_t *)(tensor->data))[i] = value;
|
|
} break;
|
|
case GGML_TYPE_F16:
|
|
{
|
|
GGML_ASSERT(tensor->nb[0] == sizeof(ggml_fp16_t));
|
|
((ggml_fp16_t *)(tensor->data))[i] = ggml_fp32_to_fp16(value);
|
|
} break;
|
|
case GGML_TYPE_F32:
|
|
{
|
|
GGML_ASSERT(tensor->nb[0] == sizeof(float));
|
|
((float *)(tensor->data))[i] = value;
|
|
} break;
|
|
case GGML_TYPE_COUNT:
|
|
{
|
|
GGML_ASSERT(false);
|
|
} break;
|
|
}
|
|
}
|
|
|
|
void * ggml_get_data(const struct ggml_tensor * tensor) {
|
|
return tensor->data;
|
|
}
|
|
|
|
float * ggml_get_data_f32(const struct ggml_tensor * tensor) {
|
|
assert(tensor->type == GGML_TYPE_F32);
|
|
return (float *)(tensor->data);
|
|
}
|
|
|
|
struct ggml_tensor * ggml_view_tensor(
|
|
struct ggml_context * ctx,
|
|
const struct ggml_tensor * src) {
|
|
return ggml_new_tensor_impl(ctx, src->type, src->n_dims, src->ne, src->data);
|
|
}
|
|
|
|
////////////////////////////////////////////////////////////////////////////////
|
|
|
|
// ggml_dup
|
|
|
|
struct ggml_tensor * ggml_dup_impl(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a,
|
|
bool inplace) {
|
|
bool is_node = false;
|
|
|
|
if (!inplace && (a->grad)) {
|
|
is_node = true;
|
|
}
|
|
|
|
struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
|
|
|
|
result->op = GGML_OP_DUP;
|
|
result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
|
|
result->src0 = a;
|
|
result->src1 = NULL;
|
|
|
|
return result;
|
|
}
|
|
|
|
struct ggml_tensor * ggml_dup(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a) {
|
|
return ggml_dup_impl(ctx, a, false);
|
|
}
|
|
|
|
struct ggml_tensor * ggml_dup_inplace(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a) {
|
|
return ggml_dup_impl(ctx, a, true);
|
|
}
|
|
|
|
// ggml_add
|
|
|
|
struct ggml_tensor * ggml_add_impl(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a,
|
|
struct ggml_tensor * b,
|
|
bool inplace) {
|
|
assert(ggml_are_same_shape(a, b));
|
|
|
|
bool is_node = false;
|
|
|
|
if (!inplace && (a->grad || b->grad)) {
|
|
is_node = true;
|
|
}
|
|
|
|
struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
|
|
|
|
result->op = GGML_OP_ADD;
|
|
result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
|
|
result->src0 = a;
|
|
result->src1 = b;
|
|
|
|
return result;
|
|
}
|
|
|
|
struct ggml_tensor * ggml_add(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a,
|
|
struct ggml_tensor * b) {
|
|
return ggml_add_impl(ctx, a, b, false);
|
|
}
|
|
|
|
struct ggml_tensor * ggml_add_inplace(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a,
|
|
struct ggml_tensor * b) {
|
|
return ggml_add_impl(ctx, a, b, true);
|
|
}
|
|
|
|
// ggml_sub
|
|
|
|
struct ggml_tensor * ggml_sub_impl(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a,
|
|
struct ggml_tensor * b,
|
|
bool inplace) {
|
|
assert(ggml_are_same_shape(a, b));
|
|
|
|
bool is_node = false;
|
|
|
|
if (!inplace && (a->grad || b->grad)) {
|
|
is_node = true;
|
|
}
|
|
|
|
struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
|
|
|
|
result->op = GGML_OP_SUB;
|
|
result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
|
|
result->src0 = a;
|
|
result->src1 = b;
|
|
|
|
return result;
|
|
}
|
|
|
|
struct ggml_tensor * ggml_sub(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a,
|
|
struct ggml_tensor * b) {
|
|
return ggml_sub_impl(ctx, a, b, false);
|
|
}
|
|
|
|
struct ggml_tensor * ggml_sub_inplace(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a,
|
|
struct ggml_tensor * b) {
|
|
return ggml_sub_impl(ctx, a, b, true);
|
|
}
|
|
|
|
// ggml_mul
|
|
|
|
struct ggml_tensor * ggml_mul_impl(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a,
|
|
struct ggml_tensor * b,
|
|
bool inplace) {
|
|
assert(ggml_are_same_shape(a, b));
|
|
|
|
bool is_node = false;
|
|
|
|
if (!inplace && (a->grad || b->grad)) {
|
|
is_node = true;
|
|
}
|
|
|
|
if (inplace) {
|
|
assert(is_node == false);
|
|
}
|
|
|
|
struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
|
|
|
|
result->op = GGML_OP_MUL;
|
|
result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
|
|
result->src0 = a;
|
|
result->src1 = b;
|
|
|
|
return result;
|
|
}
|
|
|
|
struct ggml_tensor * ggml_mul(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a,
|
|
struct ggml_tensor * b) {
|
|
return ggml_mul_impl(ctx, a, b, false);
|
|
}
|
|
|
|
struct ggml_tensor * ggml_mul_inplace(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a,
|
|
struct ggml_tensor * b) {
|
|
return ggml_mul_impl(ctx, a, b, true);
|
|
}
|
|
|
|
// ggml_div
|
|
|
|
struct ggml_tensor * ggml_div_impl(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a,
|
|
struct ggml_tensor * b,
|
|
bool inplace) {
|
|
assert(ggml_are_same_shape(a, b));
|
|
|
|
bool is_node = false;
|
|
|
|
if (!inplace && (a->grad || b->grad)) {
|
|
is_node = true;
|
|
}
|
|
|
|
if (inplace) {
|
|
assert(is_node == false);
|
|
}
|
|
|
|
struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
|
|
|
|
result->op = GGML_OP_DIV;
|
|
result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
|
|
result->src0 = a;
|
|
result->src1 = b;
|
|
|
|
return result;
|
|
}
|
|
|
|
struct ggml_tensor * ggml_div(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a,
|
|
struct ggml_tensor * b) {
|
|
return ggml_div_impl(ctx, a, b, false);
|
|
}
|
|
|
|
struct ggml_tensor * ggml_div_inplace(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a,
|
|
struct ggml_tensor * b) {
|
|
return ggml_div_impl(ctx, a, b, true);
|
|
}
|
|
|
|
// ggml_sqr
|
|
|
|
struct ggml_tensor * ggml_sqr_impl(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a,
|
|
bool inplace) {
|
|
bool is_node = false;
|
|
|
|
if (!inplace && (a->grad)) {
|
|
is_node = true;
|
|
}
|
|
|
|
struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
|
|
|
|
result->op = GGML_OP_SQR;
|
|
result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
|
|
result->src0 = a;
|
|
result->src1 = NULL;
|
|
|
|
return result;
|
|
}
|
|
|
|
struct ggml_tensor * ggml_sqr(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a) {
|
|
return ggml_sqr_impl(ctx, a, false);
|
|
}
|
|
|
|
struct ggml_tensor * ggml_sqr_inplace(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a) {
|
|
return ggml_sqr_impl(ctx, a, true);
|
|
}
|
|
|
|
// ggml_sqrt
|
|
|
|
struct ggml_tensor * ggml_sqrt_impl(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a,
|
|
bool inplace) {
|
|
bool is_node = false;
|
|
|
|
if (!inplace && (a->grad)) {
|
|
is_node = true;
|
|
}
|
|
|
|
struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
|
|
|
|
result->op = GGML_OP_SQRT;
|
|
result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
|
|
result->src0 = a;
|
|
result->src1 = NULL;
|
|
|
|
return result;
|
|
}
|
|
|
|
struct ggml_tensor * ggml_sqrt(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a) {
|
|
return ggml_sqrt_impl(ctx, a, false);
|
|
}
|
|
|
|
struct ggml_tensor * ggml_sqrt_inplace(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a) {
|
|
return ggml_sqrt_impl(ctx, a, true);
|
|
}
|
|
|
|
// ggml_sum
|
|
|
|
struct ggml_tensor * ggml_sum(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a) {
|
|
bool is_node = false;
|
|
|
|
if (a->grad) {
|
|
is_node = true;
|
|
}
|
|
|
|
struct ggml_tensor * result = ggml_new_tensor_1d(ctx, a->type, 1);
|
|
|
|
result->op = GGML_OP_SUM;
|
|
result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
|
|
result->src0 = a;
|
|
result->src1 = NULL;
|
|
|
|
return result;
|
|
}
|
|
|
|
// ggml_mean
|
|
|
|
struct ggml_tensor * ggml_mean(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a) {
|
|
bool is_node = false;
|
|
|
|
if (a->grad) {
|
|
assert(false); // TODO: implement
|
|
is_node = true;
|
|
}
|
|
|
|
int ne[GGML_MAX_DIMS] = { 1, a->ne[1], a->ne[2], a->ne[3] };
|
|
struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, a->n_dims, ne);
|
|
|
|
result->op = GGML_OP_MEAN;
|
|
result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
|
|
result->src0 = a;
|
|
result->src1 = NULL;
|
|
|
|
return result;
|
|
}
|
|
|
|
// ggml_repeat
|
|
|
|
struct ggml_tensor * ggml_repeat(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a,
|
|
struct ggml_tensor * b) {
|
|
assert(ggml_can_repeat(a, b));
|
|
|
|
bool is_node = false;
|
|
|
|
if (a->grad) {
|
|
is_node = true;
|
|
}
|
|
|
|
if (ggml_are_same_shape(a, b) && !is_node) {
|
|
return a;
|
|
}
|
|
|
|
struct ggml_tensor * result = ggml_new_tensor(ctx, a->type, b->n_dims, b->ne);
|
|
|
|
result->op = GGML_OP_REPEAT;
|
|
result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
|
|
result->src0 = a;
|
|
result->src1 = NULL;
|
|
|
|
return result;
|
|
}
|
|
|
|
// ggml_abs
|
|
|
|
struct ggml_tensor * ggml_abs_impl(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a,
|
|
bool inplace) {
|
|
bool is_node = false;
|
|
|
|
if (!inplace && (a->grad)) {
|
|
is_node = true;
|
|
}
|
|
|
|
struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
|
|
|
|
result->op = GGML_OP_ABS;
|
|
result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
|
|
result->src0 = a;
|
|
result->src1 = NULL;
|
|
|
|
return result;
|
|
}
|
|
|
|
struct ggml_tensor * ggml_abs(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a) {
|
|
return ggml_abs_impl(ctx, a, false);
|
|
}
|
|
|
|
struct ggml_tensor * ggml_abs_inplace(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a) {
|
|
return ggml_abs_impl(ctx, a, true);
|
|
}
|
|
|
|
|
|
// ggml_sgn
|
|
|
|
struct ggml_tensor * ggml_sgn_impl(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a,
|
|
bool inplace) {
|
|
bool is_node = false;
|
|
|
|
if (!inplace && (a->grad)) {
|
|
is_node = true;
|
|
}
|
|
|
|
struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
|
|
|
|
result->op = GGML_OP_SGN;
|
|
result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
|
|
result->src0 = a;
|
|
result->src1 = NULL;
|
|
|
|
return result;
|
|
}
|
|
|
|
struct ggml_tensor * ggml_sgn(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a) {
|
|
return ggml_sgn_impl(ctx, a, false);
|
|
}
|
|
|
|
struct ggml_tensor * ggml_sgn_inplace(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a) {
|
|
return ggml_sgn_impl(ctx, a, true);
|
|
}
|
|
|
|
// ggml_neg
|
|
|
|
struct ggml_tensor * ggml_neg_impl(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a,
|
|
bool inplace) {
|
|
bool is_node = false;
|
|
|
|
if (!inplace && (a->grad)) {
|
|
is_node = true;
|
|
}
|
|
|
|
struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
|
|
|
|
result->op = GGML_OP_NEG;
|
|
result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
|
|
result->src0 = a;
|
|
result->src1 = NULL;
|
|
|
|
return result;
|
|
}
|
|
|
|
struct ggml_tensor * ggml_neg(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a) {
|
|
return ggml_neg_impl(ctx, a, false);
|
|
}
|
|
|
|
struct ggml_tensor * ggml_neg_inplace(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a) {
|
|
return ggml_neg_impl(ctx, a, true);
|
|
}
|
|
|
|
// ggml_step
|
|
|
|
struct ggml_tensor * ggml_step_impl(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a,
|
|
bool inplace) {
|
|
bool is_node = false;
|
|
|
|
if (!inplace && (a->grad)) {
|
|
is_node = true;
|
|
}
|
|
|
|
struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
|
|
|
|
result->op = GGML_OP_STEP;
|
|
result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
|
|
result->src0 = a;
|
|
result->src1 = NULL;
|
|
|
|
return result;
|
|
}
|
|
|
|
struct ggml_tensor * ggml_step(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a) {
|
|
return ggml_step_impl(ctx, a, false);
|
|
}
|
|
|
|
struct ggml_tensor * ggml_step_inplace(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a) {
|
|
return ggml_step_impl(ctx, a, true);
|
|
}
|
|
|
|
// ggml_relu
|
|
|
|
struct ggml_tensor * ggml_relu_impl(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a,
|
|
bool inplace) {
|
|
bool is_node = false;
|
|
|
|
if (!inplace && (a->grad)) {
|
|
is_node = true;
|
|
}
|
|
|
|
struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
|
|
|
|
result->op = GGML_OP_RELU;
|
|
result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
|
|
result->src0 = a;
|
|
result->src1 = NULL;
|
|
|
|
return result;
|
|
}
|
|
|
|
struct ggml_tensor * ggml_relu(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a) {
|
|
return ggml_relu_impl(ctx, a, false);
|
|
}
|
|
|
|
struct ggml_tensor * ggml_relu_inplace(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a) {
|
|
return ggml_relu_impl(ctx, a, true);
|
|
}
|
|
|
|
// ggml_gelu
|
|
|
|
struct ggml_tensor * ggml_gelu_impl(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a,
|
|
bool inplace) {
|
|
bool is_node = false;
|
|
|
|
if (!inplace && (a->grad)) {
|
|
is_node = true;
|
|
}
|
|
|
|
struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
|
|
|
|
result->op = GGML_OP_GELU;
|
|
result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
|
|
result->src0 = a;
|
|
result->src1 = NULL;
|
|
|
|
return result;
|
|
}
|
|
|
|
struct ggml_tensor * ggml_gelu(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a) {
|
|
return ggml_gelu_impl(ctx, a, false);
|
|
}
|
|
|
|
struct ggml_tensor * ggml_gelu_inplace(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a) {
|
|
return ggml_gelu_impl(ctx, a, true);
|
|
}
|
|
|
|
// ggml_norm
|
|
|
|
struct ggml_tensor * ggml_norm_impl(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a,
|
|
bool inplace) {
|
|
bool is_node = false;
|
|
|
|
if (!inplace && (a->grad)) {
|
|
assert(false); // TODO: implement backward
|
|
is_node = true;
|
|
}
|
|
|
|
struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
|
|
|
|
result->op = GGML_OP_NORM;
|
|
result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
|
|
result->src0 = a;
|
|
result->src1 = NULL; // TODO: maybe store epsilon here?
|
|
|
|
return result;
|
|
}
|
|
|
|
struct ggml_tensor * ggml_norm(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a) {
|
|
return ggml_norm_impl(ctx, a, false);
|
|
}
|
|
|
|
struct ggml_tensor * ggml_norm_inplace(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a) {
|
|
return ggml_norm_impl(ctx, a, true);
|
|
}
|
|
|
|
// ggml_mul_mat
|
|
|
|
struct ggml_tensor * ggml_mul_mat(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a,
|
|
struct ggml_tensor * b) {
|
|
assert(ggml_can_mul_mat(a, b));
|
|
|
|
bool is_node = false;
|
|
|
|
if (a->grad || b->grad) {
|
|
is_node = true;
|
|
}
|
|
|
|
const int ne[4] = { a->ne[1], b->ne[1], a->ne[2], b->ne[3] };
|
|
struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, MIN(a->n_dims, b->n_dims), ne);
|
|
|
|
result->op = GGML_OP_MUL_MAT;
|
|
result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
|
|
result->src0 = a;
|
|
result->src1 = b;
|
|
|
|
return result;
|
|
}
|
|
|
|
// ggml_scale
|
|
|
|
struct ggml_tensor * ggml_scale_impl(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a,
|
|
struct ggml_tensor * b,
|
|
bool inplace) {
|
|
assert(ggml_is_scalar(b));
|
|
assert(ggml_is_padded_1d(a));
|
|
|
|
bool is_node = false;
|
|
|
|
if (!inplace && (a->grad || b->grad)) {
|
|
assert(false); // TODO: implement backward
|
|
is_node = true;
|
|
}
|
|
|
|
// TODO: when implement backward, fix this:
|
|
//struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
|
|
struct ggml_tensor * result = ggml_view_tensor(ctx, a);
|
|
|
|
result->op = GGML_OP_SCALE;
|
|
result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
|
|
result->src0 = a;
|
|
result->src1 = b;
|
|
|
|
return result;
|
|
}
|
|
|
|
struct ggml_tensor * ggml_scale(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a,
|
|
struct ggml_tensor * b) {
|
|
return ggml_scale_impl(ctx, a, b, false);
|
|
}
|
|
|
|
struct ggml_tensor * ggml_scale_inplace(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a,
|
|
struct ggml_tensor * b) {
|
|
return ggml_scale_impl(ctx, a, b, true);
|
|
}
|
|
|
|
// ggml_cpy
|
|
|
|
struct ggml_tensor * ggml_cpy_impl(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a,
|
|
struct ggml_tensor * b,
|
|
bool inplace) {
|
|
assert(ggml_nelements(a) == ggml_nelements(b));
|
|
|
|
bool is_node = false;
|
|
|
|
if (!inplace && (a->grad || b->grad)) {
|
|
assert(false); // TODO: implement backward
|
|
is_node = true;
|
|
}
|
|
|
|
// make a view of the destination
|
|
struct ggml_tensor * result = ggml_view_tensor(ctx, b);
|
|
|
|
result->op = GGML_OP_CPY;
|
|
result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
|
|
result->src0 = a;
|
|
result->src1 = b;
|
|
|
|
return result;
|
|
}
|
|
|
|
struct ggml_tensor * ggml_cpy(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a,
|
|
struct ggml_tensor * b) {
|
|
return ggml_cpy_impl(ctx, a, b, false);
|
|
}
|
|
|
|
struct ggml_tensor * ggml_cpy_inplace(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a,
|
|
struct ggml_tensor * b) {
|
|
return ggml_cpy_impl(ctx, a, b, true);
|
|
}
|
|
|
|
// ggml_reshape
|
|
|
|
struct ggml_tensor * ggml_reshape(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a,
|
|
struct ggml_tensor * b) {
|
|
assert(ggml_is_contiguous(a));
|
|
assert(ggml_is_contiguous(b));
|
|
assert(ggml_nelements(a) == ggml_nelements(b));
|
|
|
|
bool is_node = false;
|
|
|
|
if (a->grad || b->grad) {
|
|
assert(false); // TODO: implement backward
|
|
is_node = true;
|
|
}
|
|
|
|
struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, b->n_dims, b->ne, a->data);
|
|
|
|
result->op = GGML_OP_RESHAPE;
|
|
result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
|
|
result->src0 = a;
|
|
result->src1 = NULL;
|
|
|
|
return result;
|
|
}
|
|
|
|
struct ggml_tensor * ggml_reshape_2d(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a,
|
|
int ne0,
|
|
int ne1) {
|
|
assert(ggml_is_contiguous(a));
|
|
assert(ggml_nelements(a) == ne0*ne1);
|
|
|
|
bool is_node = false;
|
|
|
|
if (a->grad) {
|
|
assert(false); // TODO: implement backward
|
|
is_node = true;
|
|
}
|
|
|
|
const int ne[2] = { ne0, ne1 };
|
|
struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, 2, ne, a->data);
|
|
|
|
result->op = GGML_OP_RESHAPE;
|
|
result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
|
|
result->src0 = a;
|
|
result->src1 = NULL;
|
|
|
|
return result;
|
|
}
|
|
|
|
struct ggml_tensor * ggml_reshape_3d(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a,
|
|
int ne0,
|
|
int ne1,
|
|
int ne2) {
|
|
assert(ggml_is_contiguous(a));
|
|
assert(ggml_nelements(a) == ne0*ne1*ne2);
|
|
|
|
bool is_node = false;
|
|
|
|
if (a->grad) {
|
|
assert(false); // TODO: implement backward
|
|
is_node = true;
|
|
}
|
|
|
|
const int ne[3] = { ne0, ne1, ne2 };
|
|
struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, 3, ne, a->data);
|
|
|
|
result->op = GGML_OP_RESHAPE;
|
|
result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
|
|
result->src0 = a;
|
|
result->src1 = NULL;
|
|
|
|
return result;
|
|
}
|
|
|
|
// ggml_view_1d
|
|
|
|
struct ggml_tensor * ggml_view_1d(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a,
|
|
int ne0,
|
|
size_t offset) {
|
|
if (a->grad) {
|
|
assert(false); // gradient propagation is not supported
|
|
}
|
|
|
|
struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, 1, &ne0, (char *) a->data + offset);
|
|
|
|
result->op = GGML_OP_VIEW;
|
|
result->grad = NULL;
|
|
result->src0 = a;
|
|
result->src1 = NULL; // TODO: maybe store the offset here?
|
|
|
|
return result;
|
|
}
|
|
|
|
// ggml_view_2d
|
|
|
|
struct ggml_tensor * ggml_view_2d(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a,
|
|
int ne0,
|
|
int ne1,
|
|
size_t nb1,
|
|
size_t offset) {
|
|
if (a->grad) {
|
|
assert(false); // gradient propagation is not supported
|
|
}
|
|
|
|
const int ne[GGML_MAX_DIMS] = { ne0, ne1, 1, 1 };
|
|
|
|
struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, 2, ne, (char *) a->data + offset);
|
|
|
|
result->nb[1] = nb1;
|
|
result->nb[2] = result->nb[1]*ne1;
|
|
result->nb[3] = result->nb[2];
|
|
|
|
result->op = GGML_OP_VIEW;
|
|
result->grad = NULL;
|
|
result->src0 = a;
|
|
result->src1 = NULL; // TODO: maybe store the offset here?
|
|
|
|
return result;
|
|
}
|
|
|
|
// ggml_permute
|
|
|
|
struct ggml_tensor * ggml_permute(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a,
|
|
int axis0,
|
|
int axis1,
|
|
int axis2,
|
|
int axis3) {
|
|
assert(axis0 >= 0 && axis0 < GGML_MAX_DIMS);
|
|
assert(axis1 >= 0 && axis1 < GGML_MAX_DIMS);
|
|
assert(axis2 >= 0 && axis2 < GGML_MAX_DIMS);
|
|
assert(axis3 >= 0 && axis3 < GGML_MAX_DIMS);
|
|
|
|
assert(axis0 != axis1);
|
|
assert(axis0 != axis2);
|
|
assert(axis0 != axis3);
|
|
assert(axis1 != axis2);
|
|
assert(axis1 != axis3);
|
|
assert(axis2 != axis3);
|
|
|
|
bool is_node = false;
|
|
|
|
if (a->grad) {
|
|
assert(false); // TODO: implement backward
|
|
is_node = true;
|
|
}
|
|
|
|
struct ggml_tensor * result = ggml_view_tensor(ctx, a);
|
|
|
|
int ne[GGML_MAX_DIMS];
|
|
int nb[GGML_MAX_DIMS];
|
|
|
|
ne[axis0] = a->ne[0];
|
|
ne[axis1] = a->ne[1];
|
|
ne[axis2] = a->ne[2];
|
|
ne[axis3] = a->ne[3];
|
|
|
|
nb[axis0] = a->nb[0];
|
|
nb[axis1] = a->nb[1];
|
|
nb[axis2] = a->nb[2];
|
|
nb[axis3] = a->nb[3];
|
|
|
|
result->ne[0] = ne[0];
|
|
result->ne[1] = ne[1];
|
|
result->ne[2] = ne[2];
|
|
result->ne[3] = ne[3];
|
|
|
|
result->nb[0] = nb[0];
|
|
result->nb[1] = nb[1];
|
|
result->nb[2] = nb[2];
|
|
result->nb[3] = nb[3];
|
|
|
|
result->op = GGML_OP_PERMUTE;
|
|
result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
|
|
result->src0 = a;
|
|
result->src1 = NULL; // TODO: maybe store the permutation here?
|
|
|
|
return result;
|
|
}
|
|
|
|
// ggml_transpose
|
|
|
|
struct ggml_tensor * ggml_transpose(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a) {
|
|
bool is_node = false;
|
|
|
|
if (a->grad) {
|
|
assert(false); // TODO: implement backward
|
|
is_node = true;
|
|
}
|
|
|
|
struct ggml_tensor * result = ggml_view_tensor(ctx, a);
|
|
|
|
result->ne[0] = a->ne[1];
|
|
result->ne[1] = a->ne[0];
|
|
|
|
result->nb[0] = a->nb[1];
|
|
result->nb[1] = a->nb[0];
|
|
|
|
result->op = GGML_OP_TRANSPOSE;
|
|
result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
|
|
result->src0 = a;
|
|
result->src1 = NULL;
|
|
|
|
return result;
|
|
}
|
|
|
|
// ggml_get_rows
|
|
|
|
struct ggml_tensor * ggml_get_rows(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a,
|
|
struct ggml_tensor * b) {
|
|
assert(ggml_is_matrix(a) && ggml_is_vector(b) && b->type == GGML_TYPE_I32);
|
|
|
|
bool is_node = false;
|
|
|
|
if (a->grad || b->grad) {
|
|
assert(false); // TODO: implement backward
|
|
is_node = true;
|
|
}
|
|
|
|
// TODO: implement non F32 return
|
|
//struct ggml_tensor * result = ggml_new_tensor_2d(ctx, a->type, a->ne[0], b->ne[0]);
|
|
struct ggml_tensor * result = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, a->ne[0], b->ne[0]);
|
|
|
|
result->op = GGML_OP_GET_ROWS;
|
|
result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
|
|
result->src0 = a;
|
|
result->src1 = b;
|
|
|
|
return result;
|
|
}
|
|
|
|
// ggml_diag_mask_inf
|
|
|
|
struct ggml_tensor * ggml_diag_mask_inf(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a,
|
|
int n_past) {
|
|
bool is_node = false;
|
|
|
|
if (a->grad) {
|
|
assert(false); // TODO: implement backward
|
|
is_node = true;
|
|
}
|
|
|
|
// TODO: when implement backward, fix this:
|
|
//struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
|
|
struct ggml_tensor * result = ggml_view_tensor(ctx, a);
|
|
|
|
struct ggml_tensor * b = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 1);
|
|
((int32_t *) b->data)[0] = n_past;
|
|
|
|
result->op = GGML_OP_DIAG_MASK_INF;
|
|
result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
|
|
result->src0 = a;
|
|
result->src1 = b;
|
|
|
|
return result;
|
|
}
|
|
|
|
// ggml_soft_max
|
|
|
|
struct ggml_tensor * ggml_soft_max(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a) {
|
|
bool is_node = false;
|
|
|
|
if (a->grad) {
|
|
assert(false); // TODO: implement backward
|
|
is_node = true;
|
|
}
|
|
|
|
// TODO: when implement backward, fix this:
|
|
//struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
|
|
struct ggml_tensor * result = ggml_view_tensor(ctx, a);
|
|
|
|
result->op = GGML_OP_SOFT_MAX;
|
|
result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
|
|
result->src0 = a;
|
|
result->src1 = NULL;
|
|
|
|
return result;
|
|
}
|
|
|
|
// ggml_rope
|
|
|
|
struct ggml_tensor * ggml_rope(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a,
|
|
int n_past,
|
|
int n_dims,
|
|
int mode) {
|
|
assert(n_past >= 0);
|
|
bool is_node = false;
|
|
|
|
if (a->grad) {
|
|
assert(false); // TODO: implement backward
|
|
is_node = true;
|
|
}
|
|
|
|
// TODO: when implement backward, fix this:
|
|
//struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
|
|
struct ggml_tensor * result = ggml_view_tensor(ctx, a);
|
|
|
|
struct ggml_tensor * b = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 3);
|
|
((int32_t *) b->data)[0] = n_past;
|
|
((int32_t *) b->data)[1] = n_dims;
|
|
((int32_t *) b->data)[2] = mode;
|
|
|
|
result->op = GGML_OP_ROPE;
|
|
result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
|
|
result->src0 = a;
|
|
result->src1 = b;
|
|
|
|
return result;
|
|
}
|
|
|
|
// ggml_conv_1d_1s
|
|
|
|
struct ggml_tensor * ggml_conv_1d_1s(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a,
|
|
struct ggml_tensor * b) {
|
|
assert(ggml_is_matrix(b));
|
|
assert(a->ne[1] == b->ne[1]);
|
|
assert(a->ne[3] == 1);
|
|
bool is_node = false;
|
|
|
|
if (a->grad || b->grad) {
|
|
assert(false); // TODO: implement backward
|
|
is_node = true;
|
|
}
|
|
|
|
const int ne[4] = { b->ne[0], a->ne[2], 1, 1, };
|
|
struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 2, ne);
|
|
|
|
result->op = GGML_OP_CONV_1D_1S;
|
|
result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
|
|
result->src0 = a;
|
|
result->src1 = b;
|
|
|
|
return result;
|
|
}
|
|
|
|
// ggml_conv_1d_2s
|
|
|
|
struct ggml_tensor * ggml_conv_1d_2s(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a,
|
|
struct ggml_tensor * b) {
|
|
assert(ggml_is_matrix(b));
|
|
assert(a->ne[1] == b->ne[1]);
|
|
assert(a->ne[3] == 1);
|
|
bool is_node = false;
|
|
|
|
if (a->grad || b->grad) {
|
|
assert(false); // TODO: implement backward
|
|
is_node = true;
|
|
}
|
|
|
|
const int ne[4] = { b->ne[0]/2, a->ne[2], 1, 1, };
|
|
struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 2, ne);
|
|
|
|
result->op = GGML_OP_CONV_1D_2S;
|
|
result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
|
|
result->src0 = a;
|
|
result->src1 = b;
|
|
|
|
return result;
|
|
}
|
|
|
|
// ggml_flash_attn
|
|
|
|
struct ggml_tensor * ggml_flash_attn(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * q,
|
|
struct ggml_tensor * k,
|
|
struct ggml_tensor * v,
|
|
bool masked) {
|
|
assert(ggml_can_mul_mat(k, q));
|
|
// TODO: check if vT can be multiplied by (k*qT)
|
|
|
|
bool is_node = false;
|
|
|
|
if (q->grad || k->grad || v->grad) {
|
|
GGML_ASSERT(false); // TODO: implement backward
|
|
is_node = true;
|
|
}
|
|
|
|
//struct ggml_tensor * result = ggml_dup_tensor(ctx, q);
|
|
struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, q->ne);
|
|
|
|
result->op = GGML_OP_FLASH_ATTN;
|
|
result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
|
|
result->src0 = q;
|
|
result->src1 = k;
|
|
result->opt[0] = v;
|
|
result->opt[1] = ggml_new_i32(ctx, masked ? 1 : 0);
|
|
|
|
return result;
|
|
}
|
|
|
|
// ggml_flash_ff
|
|
|
|
struct ggml_tensor * ggml_flash_ff(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * a,
|
|
struct ggml_tensor * b0,
|
|
struct ggml_tensor * b1,
|
|
struct ggml_tensor * c0,
|
|
struct ggml_tensor * c1) {
|
|
assert(ggml_can_mul_mat(b0, a));
|
|
// TODO: more checks
|
|
|
|
bool is_node = false;
|
|
|
|
if (a->grad || b0->grad || b1->grad || c0->grad || c1->grad) {
|
|
GGML_ASSERT(false); // TODO: implement backward
|
|
is_node = true;
|
|
}
|
|
|
|
//struct ggml_tensor * result = ggml_dup_tensor(ctx, a);
|
|
struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, a->ne);
|
|
|
|
result->op = GGML_OP_FLASH_FF;
|
|
result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
|
|
result->src0 = a;
|
|
result->src1 = b0;
|
|
result->opt[0] = b1;
|
|
result->opt[1] = c0;
|
|
result->opt[2] = c1;
|
|
|
|
return result;
|
|
}
|
|
|
|
////////////////////////////////////////////////////////////////////////////////
|
|
|
|
void ggml_set_param(
|
|
struct ggml_context * ctx,
|
|
struct ggml_tensor * tensor) {
|
|
tensor->is_param = true;
|
|
|
|
assert(tensor->grad == NULL);
|
|
tensor->grad = ggml_dup_tensor(ctx, tensor);
|
|
}
|
|
|
|
// ggml_compute_forward_dup
|
|
|
|
void ggml_compute_forward_dup_f16(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
struct ggml_tensor * dst) {
|
|
assert(params->ith == 0);
|
|
assert(ggml_is_contiguous(dst));
|
|
assert(ggml_nelements(dst) == ggml_nelements(src0));
|
|
|
|
if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {
|
|
return;
|
|
}
|
|
|
|
//const int ne00 = src0->ne[0];
|
|
//const int ne01 = src0->ne[1];
|
|
//const int ne02 = src0->ne[2];
|
|
//const int ne03 = src0->ne[3];
|
|
|
|
//const size_t nb00 = src0->nb[0];
|
|
//const size_t nb01 = src0->nb[1];
|
|
//const size_t nb02 = src0->nb[2];
|
|
//const size_t nb03 = src0->nb[3];
|
|
|
|
if (ggml_is_contiguous(src0) && src0->type == dst->type) {
|
|
memcpy(dst->data, src0->data, ggml_nelements(dst) * GGML_TYPE_SIZE[src0->type]);
|
|
return;
|
|
}
|
|
|
|
GGML_ASSERT(false); // TODO: implement
|
|
}
|
|
|
|
void ggml_compute_forward_dup_f32(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
struct ggml_tensor * dst) {
|
|
GGML_ASSERT(params->ith == 0);
|
|
GGML_ASSERT(ggml_is_contiguous(dst));
|
|
GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0));
|
|
|
|
if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {
|
|
return;
|
|
}
|
|
|
|
const int ne00 = src0->ne[0];
|
|
const int ne01 = src0->ne[1];
|
|
const int ne02 = src0->ne[2];
|
|
const int ne03 = src0->ne[3];
|
|
|
|
const size_t nb00 = src0->nb[0];
|
|
const size_t nb01 = src0->nb[1];
|
|
const size_t nb02 = src0->nb[2];
|
|
const size_t nb03 = src0->nb[3];
|
|
|
|
if (ggml_is_contiguous(src0) && src0->type == dst->type) {
|
|
memcpy(dst->data, src0->data, ggml_nelements(dst) * GGML_TYPE_SIZE[src0->type]);
|
|
return;
|
|
}
|
|
|
|
if (src0->nb[0] == sizeof(float)) {
|
|
if (dst->type == GGML_TYPE_F32) {
|
|
int id = 0;
|
|
const size_t rs = ne00*nb00;
|
|
|
|
for (int i03 = 0; i03 < ne03; i03++) {
|
|
for (int i02 = 0; i02 < ne02; i02++) {
|
|
for (int i01 = 0; i01 < ne01; i01++) {
|
|
const char * src0_ptr = (char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03;
|
|
char * dst_ptr = (char *) dst->data + id*rs;
|
|
|
|
memcpy(dst_ptr, src0_ptr, rs);
|
|
|
|
id++;
|
|
}
|
|
}
|
|
}
|
|
} else if (dst->type == GGML_TYPE_F16) {
|
|
int id = 0;
|
|
ggml_fp16_t * dst_ptr = (ggml_fp16_t *) dst->data;
|
|
|
|
for (int i03 = 0; i03 < ne03; i03++) {
|
|
for (int i02 = 0; i02 < ne02; i02++) {
|
|
for (int i01 = 0; i01 < ne01; i01++) {
|
|
for (int i00 = 0; i00 < ne00; i00++) {
|
|
const float * src0_ptr = (float *) ((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);
|
|
|
|
dst_ptr[id] = ggml_fp32_to_fp16(*src0_ptr);
|
|
id++;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
GGML_ASSERT(false); // TODO: implement
|
|
}
|
|
} else {
|
|
//printf("%s: this is not optimal - fix me\n", __func__);
|
|
|
|
if (dst->type == GGML_TYPE_F32) {
|
|
int id = 0;
|
|
float * dst_ptr = (float *) dst->data;
|
|
|
|
for (int i03 = 0; i03 < ne03; i03++) {
|
|
for (int i02 = 0; i02 < ne02; i02++) {
|
|
for (int i01 = 0; i01 < ne01; i01++) {
|
|
for (int i00 = 0; i00 < ne00; i00++) {
|
|
const float * src0_ptr = (float *) ((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);
|
|
|
|
dst_ptr[id] = *src0_ptr;
|
|
id++;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
} else if (dst->type == GGML_TYPE_F16) {
|
|
int id = 0;
|
|
ggml_fp16_t * dst_ptr = (ggml_fp16_t *) dst->data;
|
|
|
|
for (int i03 = 0; i03 < ne03; i03++) {
|
|
for (int i02 = 0; i02 < ne02; i02++) {
|
|
for (int i01 = 0; i01 < ne01; i01++) {
|
|
for (int i00 = 0; i00 < ne00; i00++) {
|
|
const float * src0_ptr = (float *) ((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);
|
|
|
|
dst_ptr[id] = ggml_fp32_to_fp16(*src0_ptr);
|
|
id++;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
GGML_ASSERT(false); // TODO: implement
|
|
}
|
|
}
|
|
}
|
|
|
|
void ggml_compute_forward_dup(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
struct ggml_tensor * dst) {
|
|
switch (src0->type) {
|
|
case GGML_TYPE_F16:
|
|
{
|
|
ggml_compute_forward_dup_f16(params, src0, dst);
|
|
} break;
|
|
case GGML_TYPE_F32:
|
|
{
|
|
ggml_compute_forward_dup_f32(params, src0, dst);
|
|
} break;
|
|
case GGML_TYPE_I8:
|
|
case GGML_TYPE_I16:
|
|
case GGML_TYPE_I32:
|
|
case GGML_TYPE_COUNT:
|
|
{
|
|
GGML_ASSERT(false);
|
|
} break;
|
|
}
|
|
}
|
|
|
|
// ggml_compute_forward_add
|
|
|
|
void ggml_compute_forward_add_f32(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
const struct ggml_tensor * src1,
|
|
struct ggml_tensor * dst) {
|
|
GGML_ASSERT(params->ith == 0);
|
|
GGML_ASSERT(ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst));
|
|
|
|
if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {
|
|
return;
|
|
}
|
|
|
|
const int n = ggml_nrows(src0);
|
|
const int nc = src0->ne[0];
|
|
|
|
const size_t nb00 = src0->nb[0];
|
|
const size_t nb01 = src0->nb[1];
|
|
|
|
const size_t nb10 = src1->nb[0];
|
|
const size_t nb11 = src1->nb[1];
|
|
|
|
const size_t nb0 = dst->nb[0];
|
|
const size_t nb1 = dst->nb[1];
|
|
|
|
GGML_ASSERT( nb0 == sizeof(float));
|
|
GGML_ASSERT(nb00 == sizeof(float));
|
|
|
|
if (nb10 == sizeof(float)) {
|
|
for (int j = 0; j < n; j++) {
|
|
ggml_vec_add_f32(nc,
|
|
(float *) ((char *) dst->data + j*nb1),
|
|
(float *) ((char *) src0->data + j*nb01),
|
|
(float *) ((char *) src1->data + j*nb11));
|
|
}
|
|
} else {
|
|
// src1 is not contiguous
|
|
for (int j = 0; j < n; j++) {
|
|
float * dst_ptr = (float *) ((char *) dst->data + j*nb1);
|
|
float * src0_ptr = (float *) ((char *) src0->data + j*nb01);
|
|
for (int i = 0; i < nc; i++) {
|
|
float * src1_ptr = (float *) ((char *) src1->data + j*nb11 + i*nb10);
|
|
|
|
dst_ptr[i] = src0_ptr[i] + *src1_ptr;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void ggml_compute_forward_add(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
const struct ggml_tensor * src1,
|
|
struct ggml_tensor * dst) {
|
|
switch (src0->type) {
|
|
case GGML_TYPE_F32:
|
|
{
|
|
ggml_compute_forward_add_f32(params, src0, src1, dst);
|
|
} break;
|
|
case GGML_TYPE_I8:
|
|
case GGML_TYPE_I16:
|
|
case GGML_TYPE_I32:
|
|
case GGML_TYPE_F16:
|
|
case GGML_TYPE_COUNT:
|
|
{
|
|
assert(false);
|
|
} break;
|
|
}
|
|
}
|
|
|
|
// ggml_compute_forward_sub
|
|
|
|
void ggml_compute_forward_sub_f32(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
const struct ggml_tensor * src1,
|
|
struct ggml_tensor * dst) {
|
|
assert(params->ith == 0);
|
|
assert(ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst));
|
|
|
|
if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {
|
|
return;
|
|
}
|
|
|
|
const int n = ggml_nrows(src0);
|
|
const int nc = src0->ne[0];
|
|
|
|
assert( dst->nb[0] == sizeof(float));
|
|
assert(src0->nb[0] == sizeof(float));
|
|
assert(src1->nb[0] == sizeof(float));
|
|
|
|
for (int i = 0; i < n; i++) {
|
|
ggml_vec_sub_f32(nc,
|
|
(float *) ((char *) dst->data + i*( dst->nb[1])),
|
|
(float *) ((char *) src0->data + i*(src0->nb[1])),
|
|
(float *) ((char *) src1->data + i*(src1->nb[1])));
|
|
}
|
|
}
|
|
|
|
void ggml_compute_forward_sub(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
const struct ggml_tensor * src1,
|
|
struct ggml_tensor * dst) {
|
|
switch (src0->type) {
|
|
case GGML_TYPE_F32:
|
|
{
|
|
ggml_compute_forward_sub_f32(params, src0, src1, dst);
|
|
} break;
|
|
case GGML_TYPE_I8:
|
|
case GGML_TYPE_I16:
|
|
case GGML_TYPE_I32:
|
|
case GGML_TYPE_F16:
|
|
case GGML_TYPE_COUNT:
|
|
{
|
|
assert(false);
|
|
} break;
|
|
}
|
|
}
|
|
|
|
// ggml_compute_forward_mul
|
|
|
|
void ggml_compute_forward_mul_f32(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
const struct ggml_tensor * src1,
|
|
struct ggml_tensor * dst) {
|
|
assert(params->ith == 0);
|
|
assert(ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst));
|
|
|
|
if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {
|
|
return;
|
|
}
|
|
|
|
const int n = ggml_nrows(src0);
|
|
const int nc = src0->ne[0];
|
|
|
|
assert( dst->nb[0] == sizeof(float));
|
|
assert(src0->nb[0] == sizeof(float));
|
|
assert(src1->nb[0] == sizeof(float));
|
|
|
|
for (int i = 0; i < n; i++) {
|
|
ggml_vec_mul_f32(nc,
|
|
(float *) ((char *) dst->data + i*( dst->nb[1])),
|
|
(float *) ((char *) src0->data + i*(src0->nb[1])),
|
|
(float *) ((char *) src1->data + i*(src1->nb[1])));
|
|
}
|
|
}
|
|
|
|
void ggml_compute_forward_mul(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
const struct ggml_tensor * src1,
|
|
struct ggml_tensor * dst) {
|
|
switch (src0->type) {
|
|
case GGML_TYPE_F32:
|
|
{
|
|
ggml_compute_forward_mul_f32(params, src0, src1, dst);
|
|
} break;
|
|
case GGML_TYPE_I8:
|
|
case GGML_TYPE_I16:
|
|
case GGML_TYPE_I32:
|
|
case GGML_TYPE_F16:
|
|
case GGML_TYPE_COUNT:
|
|
{
|
|
assert(false);
|
|
} break;
|
|
}
|
|
}
|
|
|
|
// ggml_compute_forward_div
|
|
|
|
void ggml_compute_forward_div_f32(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
const struct ggml_tensor * src1,
|
|
struct ggml_tensor * dst) {
|
|
assert(params->ith == 0);
|
|
assert(ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst));
|
|
|
|
if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {
|
|
return;
|
|
}
|
|
|
|
const int n = ggml_nrows(src0);
|
|
const int nc = src0->ne[0];
|
|
|
|
assert( dst->nb[0] == sizeof(float));
|
|
assert(src0->nb[0] == sizeof(float));
|
|
assert(src1->nb[0] == sizeof(float));
|
|
|
|
for (int i = 0; i < n; i++) {
|
|
ggml_vec_div_f32(nc,
|
|
(float *) ((char *) dst->data + i*( dst->nb[1])),
|
|
(float *) ((char *) src0->data + i*(src0->nb[1])),
|
|
(float *) ((char *) src1->data + i*(src1->nb[1])));
|
|
}
|
|
}
|
|
|
|
void ggml_compute_forward_div(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
const struct ggml_tensor * src1,
|
|
struct ggml_tensor * dst) {
|
|
switch (src0->type) {
|
|
case GGML_TYPE_F32:
|
|
{
|
|
ggml_compute_forward_div_f32(params, src0, src1, dst);
|
|
} break;
|
|
case GGML_TYPE_I8:
|
|
case GGML_TYPE_I16:
|
|
case GGML_TYPE_I32:
|
|
case GGML_TYPE_F16:
|
|
case GGML_TYPE_COUNT:
|
|
{
|
|
assert(false);
|
|
} break;
|
|
}
|
|
}
|
|
|
|
// ggml_compute_forward_sqr
|
|
|
|
void ggml_compute_forward_sqr_f32(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
struct ggml_tensor * dst) {
|
|
assert(params->ith == 0);
|
|
assert(ggml_are_same_shape(src0, dst));
|
|
|
|
if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {
|
|
return;
|
|
}
|
|
|
|
const int n = ggml_nrows(src0);
|
|
const int nc = src0->ne[0];
|
|
|
|
assert( dst->nb[0] == sizeof(float));
|
|
assert(src0->nb[0] == sizeof(float));
|
|
|
|
for (int i = 0; i < n; i++) {
|
|
ggml_vec_sqr_f32(nc,
|
|
(float *) ((char *) dst->data + i*( dst->nb[1])),
|
|
(float *) ((char *) src0->data + i*(src0->nb[1])));
|
|
}
|
|
}
|
|
|
|
void ggml_compute_forward_sqr(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
struct ggml_tensor * dst) {
|
|
switch (src0->type) {
|
|
case GGML_TYPE_F32:
|
|
{
|
|
ggml_compute_forward_sqr_f32(params, src0, dst);
|
|
} break;
|
|
case GGML_TYPE_I8:
|
|
case GGML_TYPE_I16:
|
|
case GGML_TYPE_I32:
|
|
case GGML_TYPE_F16:
|
|
case GGML_TYPE_COUNT:
|
|
{
|
|
assert(false);
|
|
} break;
|
|
}
|
|
}
|
|
|
|
// ggml_compute_forward_sqrt
|
|
|
|
void ggml_compute_forward_sqrt_f32(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
struct ggml_tensor * dst) {
|
|
assert(params->ith == 0);
|
|
assert(ggml_are_same_shape(src0, dst));
|
|
|
|
if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {
|
|
return;
|
|
}
|
|
|
|
const int n = ggml_nrows(src0);
|
|
const int nc = src0->ne[0];
|
|
|
|
assert( dst->nb[0] == sizeof(float));
|
|
assert(src0->nb[0] == sizeof(float));
|
|
|
|
for (int i = 0; i < n; i++) {
|
|
ggml_vec_sqrt_f32(nc,
|
|
(float *) ((char *) dst->data + i*( dst->nb[1])),
|
|
(float *) ((char *) src0->data + i*(src0->nb[1])));
|
|
}
|
|
}
|
|
|
|
void ggml_compute_forward_sqrt(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
struct ggml_tensor * dst) {
|
|
switch (src0->type) {
|
|
case GGML_TYPE_F32:
|
|
{
|
|
ggml_compute_forward_sqrt_f32(params, src0, dst);
|
|
} break;
|
|
case GGML_TYPE_I8:
|
|
case GGML_TYPE_I16:
|
|
case GGML_TYPE_I32:
|
|
case GGML_TYPE_F16:
|
|
case GGML_TYPE_COUNT:
|
|
{
|
|
assert(false);
|
|
} break;
|
|
}
|
|
}
|
|
|
|
// ggml_compute_forward_sum
|
|
|
|
void ggml_compute_forward_sum_f32(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
struct ggml_tensor * dst) {
|
|
assert(params->ith == 0);
|
|
assert(ggml_is_scalar(dst));
|
|
|
|
if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {
|
|
return;
|
|
}
|
|
|
|
assert(ggml_is_scalar(dst));
|
|
assert(src0->nb[0] == sizeof(float));
|
|
|
|
*(float *) (dst->data) = 0.0f;
|
|
|
|
const int ne00 = src0->ne[0];
|
|
const int ne01 = src0->ne[1];
|
|
const int ne02 = src0->ne[2];
|
|
const int ne03 = src0->ne[3];
|
|
|
|
const size_t nb01 = src0->nb[1];
|
|
const size_t nb02 = src0->nb[2];
|
|
const size_t nb03 = src0->nb[3];
|
|
|
|
for (int i03 = 0; i03 < ne03; i03++) {
|
|
for (int i02 = 0; i02 < ne02; i02++) {
|
|
for (int i01 = 0; i01 < ne01; i01++) {
|
|
ggml_vec_sum_f32(ne00,
|
|
(float *) (dst->data),
|
|
(float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void ggml_compute_forward_sum(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
struct ggml_tensor * dst) {
|
|
switch (src0->type) {
|
|
case GGML_TYPE_F32:
|
|
{
|
|
ggml_compute_forward_sum_f32(params, src0, dst);
|
|
} break;
|
|
case GGML_TYPE_I8:
|
|
case GGML_TYPE_I16:
|
|
case GGML_TYPE_I32:
|
|
case GGML_TYPE_F16:
|
|
case GGML_TYPE_COUNT:
|
|
{
|
|
assert(false);
|
|
} break;
|
|
}
|
|
}
|
|
|
|
// ggml_compute_forward_mean
|
|
|
|
void ggml_compute_forward_mean_f32(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
struct ggml_tensor * dst) {
|
|
assert(params->ith == 0);
|
|
|
|
if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {
|
|
return;
|
|
}
|
|
|
|
assert(src0->nb[0] == sizeof(float));
|
|
|
|
const int ne00 = src0->ne[0];
|
|
const int ne01 = src0->ne[1];
|
|
const int ne02 = src0->ne[2];
|
|
const int ne03 = src0->ne[3];
|
|
|
|
const size_t nb01 = src0->nb[1];
|
|
const size_t nb02 = src0->nb[2];
|
|
const size_t nb03 = src0->nb[3];
|
|
|
|
const int ne0 = dst->ne[0];
|
|
const int ne1 = dst->ne[1];
|
|
const int ne2 = dst->ne[2];
|
|
const int ne3 = dst->ne[3];
|
|
|
|
assert(ne0 == 1);
|
|
assert(ne1 == ne01);
|
|
assert(ne2 == ne02);
|
|
assert(ne3 == ne03);
|
|
|
|
UNUSED(ne0);
|
|
UNUSED(ne1);
|
|
UNUSED(ne2);
|
|
UNUSED(ne3);
|
|
|
|
const size_t nb1 = dst->nb[1];
|
|
const size_t nb2 = dst->nb[2];
|
|
const size_t nb3 = dst->nb[3];
|
|
|
|
for (int i03 = 0; i03 < ne03; i03++) {
|
|
for (int i02 = 0; i02 < ne02; i02++) {
|
|
for (int i01 = 0; i01 < ne01; i01++) {
|
|
*(float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3) = 0.0f;
|
|
|
|
ggml_vec_sum_f32(ne00,
|
|
(float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3),
|
|
(float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03));
|
|
|
|
*(float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3) /= (float) ne00;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void ggml_compute_forward_mean(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
struct ggml_tensor * dst) {
|
|
switch (src0->type) {
|
|
case GGML_TYPE_F32:
|
|
{
|
|
ggml_compute_forward_mean_f32(params, src0, dst);
|
|
} break;
|
|
case GGML_TYPE_I8:
|
|
case GGML_TYPE_I16:
|
|
case GGML_TYPE_I32:
|
|
case GGML_TYPE_F16:
|
|
case GGML_TYPE_COUNT:
|
|
{
|
|
assert(false);
|
|
} break;
|
|
}
|
|
}
|
|
|
|
// ggml_compute_forward_repeat
|
|
|
|
void ggml_compute_forward_repeat_f32(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
struct ggml_tensor * dst) {
|
|
assert(params->ith == 0);
|
|
assert(ggml_can_repeat(src0, dst));
|
|
|
|
if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {
|
|
return;
|
|
}
|
|
|
|
// TODO: implement support for rank > 2 tensors
|
|
assert(src0->ne[2] == 1);
|
|
assert(src0->ne[3] == 1);
|
|
assert( dst->ne[2] == 1);
|
|
assert( dst->ne[3] == 1);
|
|
|
|
const int nc = dst->ne[0];
|
|
const int nr = dst->ne[1];
|
|
const int nc0 = src0->ne[0];
|
|
const int nr0 = src0->ne[1];
|
|
const int ncr = nc/nc0; // guaranteed to be an integer due to the check in ggml_can_repeat
|
|
const int nrr = nr/nr0; // guaranteed to be an integer due to the check in ggml_can_repeat
|
|
|
|
// TODO: support for transposed / permuted tensors
|
|
assert( dst->nb[0] == sizeof(float));
|
|
assert(src0->nb[0] == sizeof(float));
|
|
|
|
// TODO: maybe this is not optimal?
|
|
for (int i = 0; i < nrr; i++) {
|
|
for (int j = 0; j < ncr; j++) {
|
|
for (int k = 0; k < nr0; k++) {
|
|
ggml_vec_cpy_f32(nc0,
|
|
(float *) ((char *) dst->data + (i*nr0 + k)*( dst->nb[1]) + j*nc0*( dst->nb[0])),
|
|
(float *) ((char *) src0->data + ( k)*(src0->nb[1])));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void ggml_compute_forward_repeat(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
struct ggml_tensor * dst) {
|
|
switch (src0->type) {
|
|
case GGML_TYPE_F32:
|
|
{
|
|
ggml_compute_forward_repeat_f32(params, src0, dst);
|
|
} break;
|
|
case GGML_TYPE_I8:
|
|
case GGML_TYPE_I16:
|
|
case GGML_TYPE_I32:
|
|
case GGML_TYPE_F16:
|
|
case GGML_TYPE_COUNT:
|
|
{
|
|
assert(false);
|
|
} break;
|
|
}
|
|
}
|
|
|
|
// ggml_compute_forward_abs
|
|
|
|
void ggml_compute_forward_abs_f32(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
struct ggml_tensor * dst) {
|
|
assert(params->ith == 0);
|
|
assert(ggml_are_same_shape(src0, dst));
|
|
|
|
if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {
|
|
return;
|
|
}
|
|
|
|
const int n = ggml_nrows(src0);
|
|
const int nc = src0->ne[0];
|
|
|
|
assert(dst->nb[0] == sizeof(float));
|
|
assert(src0->nb[0] == sizeof(float));
|
|
|
|
for (int i = 0; i < n; i++) {
|
|
ggml_vec_abs_f32(nc,
|
|
(float *) ((char *) dst->data + i*( dst->nb[1])),
|
|
(float *) ((char *) src0->data + i*(src0->nb[1])));
|
|
}
|
|
}
|
|
|
|
void ggml_compute_forward_abs(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
struct ggml_tensor * dst) {
|
|
switch (src0->type) {
|
|
case GGML_TYPE_F32:
|
|
{
|
|
ggml_compute_forward_abs_f32(params, src0, dst);
|
|
} break;
|
|
case GGML_TYPE_I8:
|
|
case GGML_TYPE_I16:
|
|
case GGML_TYPE_I32:
|
|
case GGML_TYPE_F16:
|
|
case GGML_TYPE_COUNT:
|
|
{
|
|
assert(false);
|
|
} break;
|
|
}
|
|
}
|
|
|
|
// ggml_compute_forward_sgn
|
|
|
|
void ggml_compute_forward_sgn_f32(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
struct ggml_tensor * dst) {
|
|
assert(params->ith == 0);
|
|
assert(ggml_are_same_shape(src0, dst));
|
|
|
|
if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {
|
|
return;
|
|
}
|
|
|
|
const int n = ggml_nrows(src0);
|
|
const int nc = src0->ne[0];
|
|
|
|
assert(dst->nb[0] == sizeof(float));
|
|
assert(src0->nb[0] == sizeof(float));
|
|
|
|
for (int i = 0; i < n; i++) {
|
|
ggml_vec_sgn_f32(nc,
|
|
(float *) ((char *) dst->data + i*( dst->nb[1])),
|
|
(float *) ((char *) src0->data + i*(src0->nb[1])));
|
|
}
|
|
}
|
|
|
|
void ggml_compute_forward_sgn(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
struct ggml_tensor * dst) {
|
|
switch (src0->type) {
|
|
case GGML_TYPE_F32:
|
|
{
|
|
ggml_compute_forward_sgn_f32(params, src0, dst);
|
|
} break;
|
|
case GGML_TYPE_I8:
|
|
case GGML_TYPE_I16:
|
|
case GGML_TYPE_I32:
|
|
case GGML_TYPE_F16:
|
|
case GGML_TYPE_COUNT:
|
|
{
|
|
assert(false);
|
|
} break;
|
|
}
|
|
}
|
|
|
|
// ggml_compute_forward_neg
|
|
|
|
void ggml_compute_forward_neg_f32(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
struct ggml_tensor * dst) {
|
|
assert(params->ith == 0);
|
|
assert(ggml_are_same_shape(src0, dst));
|
|
|
|
if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {
|
|
return;
|
|
}
|
|
|
|
const int n = ggml_nrows(src0);
|
|
const int nc = src0->ne[0];
|
|
|
|
assert(dst->nb[0] == sizeof(float));
|
|
assert(src0->nb[0] == sizeof(float));
|
|
|
|
for (int i = 0; i < n; i++) {
|
|
ggml_vec_neg_f32(nc,
|
|
(float *) ((char *) dst->data + i*( dst->nb[1])),
|
|
(float *) ((char *) src0->data + i*(src0->nb[1])));
|
|
}
|
|
}
|
|
|
|
void ggml_compute_forward_neg(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
struct ggml_tensor * dst) {
|
|
switch (src0->type) {
|
|
case GGML_TYPE_F32:
|
|
{
|
|
ggml_compute_forward_neg_f32(params, src0, dst);
|
|
} break;
|
|
case GGML_TYPE_I8:
|
|
case GGML_TYPE_I16:
|
|
case GGML_TYPE_I32:
|
|
case GGML_TYPE_F16:
|
|
case GGML_TYPE_COUNT:
|
|
{
|
|
assert(false);
|
|
} break;
|
|
}
|
|
}
|
|
|
|
// ggml_compute_forward_step
|
|
|
|
void ggml_compute_forward_step_f32(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
struct ggml_tensor * dst) {
|
|
assert(params->ith == 0);
|
|
assert(ggml_are_same_shape(src0, dst));
|
|
|
|
if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {
|
|
return;
|
|
}
|
|
|
|
const int n = ggml_nrows(src0);
|
|
const int nc = src0->ne[0];
|
|
|
|
assert(dst->nb[0] == sizeof(float));
|
|
assert(src0->nb[0] == sizeof(float));
|
|
|
|
for (int i = 0; i < n; i++) {
|
|
ggml_vec_step_f32(nc,
|
|
(float *) ((char *) dst->data + i*( dst->nb[1])),
|
|
(float *) ((char *) src0->data + i*(src0->nb[1])));
|
|
}
|
|
}
|
|
|
|
void ggml_compute_forward_step(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
struct ggml_tensor * dst) {
|
|
switch (src0->type) {
|
|
case GGML_TYPE_F32:
|
|
{
|
|
ggml_compute_forward_step_f32(params, src0, dst);
|
|
} break;
|
|
case GGML_TYPE_I8:
|
|
case GGML_TYPE_I16:
|
|
case GGML_TYPE_I32:
|
|
case GGML_TYPE_F16:
|
|
case GGML_TYPE_COUNT:
|
|
{
|
|
assert(false);
|
|
} break;
|
|
}
|
|
}
|
|
|
|
// ggml_compute_forward_relu
|
|
|
|
void ggml_compute_forward_relu_f32(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
struct ggml_tensor * dst) {
|
|
assert(params->ith == 0);
|
|
assert(ggml_are_same_shape(src0, dst));
|
|
|
|
if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {
|
|
return;
|
|
}
|
|
|
|
const int n = ggml_nrows(src0);
|
|
const int nc = src0->ne[0];
|
|
|
|
assert(dst->nb[0] == sizeof(float));
|
|
assert(src0->nb[0] == sizeof(float));
|
|
|
|
for (int i = 0; i < n; i++) {
|
|
ggml_vec_relu_f32(nc,
|
|
(float *) ((char *) dst->data + i*( dst->nb[1])),
|
|
(float *) ((char *) src0->data + i*(src0->nb[1])));
|
|
}
|
|
}
|
|
|
|
void ggml_compute_forward_relu(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
struct ggml_tensor * dst) {
|
|
switch (src0->type) {
|
|
case GGML_TYPE_F32:
|
|
{
|
|
ggml_compute_forward_relu_f32(params, src0, dst);
|
|
} break;
|
|
case GGML_TYPE_I8:
|
|
case GGML_TYPE_I16:
|
|
case GGML_TYPE_I32:
|
|
case GGML_TYPE_F16:
|
|
case GGML_TYPE_COUNT:
|
|
{
|
|
assert(false);
|
|
} break;
|
|
}
|
|
}
|
|
|
|
// ggml_compute_forward_gelu
|
|
|
|
void ggml_compute_forward_gelu_f32(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
struct ggml_tensor * dst) {
|
|
GGML_ASSERT(ggml_is_contiguous(src0));
|
|
GGML_ASSERT(ggml_is_contiguous(dst));
|
|
GGML_ASSERT(ggml_are_same_shape(src0, dst));
|
|
|
|
if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {
|
|
return;
|
|
}
|
|
|
|
const int ith = params->ith;
|
|
const int nth = params->nth;
|
|
|
|
const int nc = src0->ne[0];
|
|
const int nr = ggml_nrows(src0);
|
|
|
|
// rows per thread
|
|
const int dr = (nr + nth - 1)/nth;
|
|
|
|
// row range for this thread
|
|
const int ir0 = dr*ith;
|
|
const int ir1 = MIN(ir0 + dr, nr);
|
|
|
|
for (int i1 = ir0; i1 < ir1; i1++) {
|
|
ggml_vec_gelu_f32(nc,
|
|
(float *) ((char *) dst->data + i1*( dst->nb[1])),
|
|
(float *) ((char *) src0->data + i1*(src0->nb[1])));
|
|
|
|
#ifndef NDEBUG
|
|
for (int k = 0; k < nc; k++) {
|
|
const float x = ((float *) ((char *) dst->data + i1*( dst->nb[1])))[k];
|
|
UNUSED(x);
|
|
assert(!isnan(x));
|
|
assert(!isinf(x));
|
|
}
|
|
#endif
|
|
}
|
|
}
|
|
|
|
void ggml_compute_forward_gelu(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
struct ggml_tensor * dst) {
|
|
switch (src0->type) {
|
|
case GGML_TYPE_F32:
|
|
{
|
|
ggml_compute_forward_gelu_f32(params, src0, dst);
|
|
} break;
|
|
case GGML_TYPE_I8:
|
|
case GGML_TYPE_I16:
|
|
case GGML_TYPE_I32:
|
|
case GGML_TYPE_F16:
|
|
case GGML_TYPE_COUNT:
|
|
{
|
|
assert(false);
|
|
} break;
|
|
}
|
|
}
|
|
|
|
// ggml_compute_forward_norm
|
|
|
|
void ggml_compute_forward_norm_f32(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
struct ggml_tensor * dst) {
|
|
assert(params->ith == 0);
|
|
assert(ggml_are_same_shape(src0, dst));
|
|
|
|
if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {
|
|
return;
|
|
}
|
|
|
|
assert(src0->nb[0] == sizeof(float));
|
|
|
|
const int ne00 = src0->ne[0];
|
|
const int ne01 = src0->ne[1];
|
|
const int ne02 = src0->ne[2];
|
|
const int ne03 = src0->ne[3];
|
|
|
|
const size_t nb01 = src0->nb[1];
|
|
const size_t nb02 = src0->nb[2];
|
|
const size_t nb03 = src0->nb[3];
|
|
|
|
const size_t nb1 = dst->nb[1];
|
|
const size_t nb2 = dst->nb[2];
|
|
const size_t nb3 = dst->nb[3];
|
|
|
|
const ggml_float eps = 1e-5f; // TODO: make this a parameter
|
|
|
|
// TODO: optimize
|
|
for (int i03 = 0; i03 < ne03; i03++) {
|
|
for (int i02 = 0; i02 < ne02; i02++) {
|
|
for (int i01 = 0; i01 < ne01; i01++) {
|
|
const float * x = (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);
|
|
|
|
ggml_float mean = 0.0;
|
|
for (int i00 = 0; i00 < ne00; i00++) {
|
|
mean += x[i00];
|
|
}
|
|
|
|
mean /= ne00;
|
|
|
|
float * y = (float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3);
|
|
|
|
ggml_float sum2 = 0.0;
|
|
for (int i00 = 0; i00 < ne00; i00++) {
|
|
ggml_float v = x[i00] - mean;
|
|
y[i00] = v;
|
|
sum2 += v*v;
|
|
}
|
|
|
|
const float scale = 1.0/sqrt(sum2/ne00 + eps);
|
|
|
|
ggml_vec_scale_f32(ne00, y, scale);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void ggml_compute_forward_norm(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
struct ggml_tensor * dst) {
|
|
switch (src0->type) {
|
|
case GGML_TYPE_F32:
|
|
{
|
|
ggml_compute_forward_norm_f32(params, src0, dst);
|
|
} break;
|
|
case GGML_TYPE_I8:
|
|
case GGML_TYPE_I16:
|
|
case GGML_TYPE_I32:
|
|
case GGML_TYPE_F16:
|
|
case GGML_TYPE_COUNT:
|
|
{
|
|
assert(false);
|
|
} break;
|
|
}
|
|
}
|
|
|
|
// ggml_compute_forward_mul_mat
|
|
|
|
void ggml_compute_forward_mul_mat_f32(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
const struct ggml_tensor * src1,
|
|
struct ggml_tensor * dst) {
|
|
int64_t t0 = ggml_perf_time_us();
|
|
UNUSED(t0);
|
|
|
|
const int ne00 = src0->ne[0];
|
|
const int ne01 = src0->ne[1];
|
|
const int ne02 = src0->ne[2];
|
|
const int ne03 = src0->ne[3];
|
|
|
|
const int ne10 = src1->ne[0];
|
|
const int ne11 = src1->ne[1];
|
|
const int ne12 = src1->ne[2];
|
|
const int ne13 = src1->ne[3];
|
|
|
|
const int ne0 = dst->ne[0];
|
|
const int ne1 = dst->ne[1];
|
|
const int ne2 = dst->ne[2];
|
|
const int ne3 = dst->ne[3];
|
|
const int ne = ne0*ne1*ne2*ne3;
|
|
|
|
const int nb00 = src0->nb[0];
|
|
const int nb01 = src0->nb[1];
|
|
const int nb02 = src0->nb[2];
|
|
const int nb03 = src0->nb[3];
|
|
|
|
const int nb10 = src1->nb[0];
|
|
const int nb11 = src1->nb[1];
|
|
const int nb12 = src1->nb[2];
|
|
const int nb13 = src1->nb[3];
|
|
|
|
const int nb0 = dst->nb[0];
|
|
const int nb1 = dst->nb[1];
|
|
const int nb2 = dst->nb[2];
|
|
const int nb3 = dst->nb[3];
|
|
|
|
const int ith = params->ith;
|
|
const int nth = params->nth;
|
|
|
|
assert(ne02 == ne12);
|
|
assert(ne03 == ne13);
|
|
assert(ne2 == ne12);
|
|
assert(ne3 == ne13);
|
|
|
|
// TODO: we don't support permuted src0
|
|
assert(nb00 == sizeof(float) || nb01 == sizeof(float));
|
|
|
|
// dst cannot be transposed or permuted
|
|
assert(nb0 == sizeof(float));
|
|
assert(nb0 <= nb1);
|
|
assert(nb1 <= nb2);
|
|
assert(nb2 <= nb3);
|
|
|
|
assert(ne0 == ne01);
|
|
assert(ne1 == ne11);
|
|
assert(ne2 == ne02);
|
|
assert(ne3 == ne03);
|
|
|
|
// nb01 >= nb00 - src0 is not transposed
|
|
// compute by src0 rows
|
|
//
|
|
// nb00 < nb01 - src0 is transposed
|
|
// compute by src0 columns
|
|
|
|
if (params->type == GGML_TASK_INIT) {
|
|
if (nb01 >= nb00) {
|
|
return;
|
|
}
|
|
|
|
// TODO: fix this memset (wsize is overestimated)
|
|
memset(params->wdata, 0, params->wsize);
|
|
return;
|
|
}
|
|
|
|
if (params->type == GGML_TASK_FINALIZE) {
|
|
if (nb01 >= nb00) {
|
|
return;
|
|
}
|
|
|
|
// TODO: fix this memset (wsize is overestimated)
|
|
//assert(params->wsize == (ggml_nbytes(dst) + CACHE_LINE_SIZE)*nth);
|
|
|
|
float * const wdata = params->wdata;
|
|
|
|
// cols per thread
|
|
const int dc = (ne + nth - 1)/nth;
|
|
|
|
// col range for this thread
|
|
const int ic0 = dc*ith;
|
|
const int ic1 = MIN(ic0 + dc, ne);
|
|
|
|
ggml_vec_cpy_f32(ic1 - ic0, (float *) dst->data + ic0, wdata + ic0);
|
|
|
|
for (int k = 1; k < nth; k++) {
|
|
ggml_vec_acc_f32(ic1 - ic0, (float *) dst->data + ic0, wdata + (ne + CACHE_LINE_SIZE_F32)*k + ic0);
|
|
}
|
|
|
|
return;
|
|
}
|
|
|
|
//#ifdef GGML_USE_ACCELERATE
|
|
// // try to use BLAS
|
|
//
|
|
// if (nb01 >= nb00 && ne0 > 1024 && ne1 > 1024) {
|
|
// if (params->ith != 0) return;
|
|
// printf("XXXXXXXX\n");
|
|
//
|
|
// GGML_ASSERT(ggml_is_contiguous(src0));
|
|
// GGML_ASSERT(ggml_is_contiguous(src1));
|
|
//
|
|
// printf("ne00 = %d, ne01 = %d, ne02 = %d, ne03 = %d\n", ne00, ne01, ne02, ne03);
|
|
// printf("ne10 = %d, ne11 = %d, ne12 = %d, ne13 = %d\n", ne10, ne11, ne12, ne13);
|
|
// printf("ne0 = %d, ne1 = %d, ne2 = %d, ne3 = %d\n", ne0, ne1, ne2, ne3);
|
|
//
|
|
// printf("nb00 = %d, nb01 = %d, nb02 = %d, nb03 = %d\n", nb00, nb01, nb02, nb03);
|
|
// printf("nb10 = %d, nb11 = %d, nb12 = %d, nb13 = %d\n", nb10, nb11, nb12, nb13);
|
|
// printf("nb0 = %d, nb1 = %d, nb2 = %d, nb3 = %d\n", nb0, nb1, nb2, nb3);
|
|
//
|
|
// float * const wdata = params->wdata;
|
|
//
|
|
// int64_t tsum = 0.0;
|
|
// for (int i03 = 0; i03 < ne03; i03++) {
|
|
// for (int i02 = 0; i02 < ne02; i02++) {
|
|
// const float * x = (float *) ((char *) src0->data + i02*nb02 + i03*nb03);
|
|
// const float * y = (float *) ((char *) src1->data + i02*nb12 + i03*nb13);
|
|
// float * z = (float *) ((char *) dst->data + i02*nb2 + i03*nb3);
|
|
//
|
|
// // transpose src1
|
|
// for (int j = 0; j < ne11; ++j) {
|
|
// for (int i = 0; i < ne10; ++i) {
|
|
// wdata[i*ne11 + j] = y[j*ne10 + i];
|
|
// }
|
|
// }
|
|
//
|
|
// {
|
|
// const int64_t tt0 = ggml_time_us();
|
|
// cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans,
|
|
// 1500, 1500, 64,
|
|
// 1.0, x, 64,
|
|
// wdata, 1500,
|
|
// 0.0, z, 1500);
|
|
// const int64_t tt1 = ggml_time_us();
|
|
// tsum += tt1 - tt0;
|
|
// }
|
|
//
|
|
// // transpose z
|
|
// for (int j = 0; j < ne1; ++j) {
|
|
// for (int i = 0; i < ne0; ++i) {
|
|
// wdata[i*ne1 + j] = z[j*ne0 + i];
|
|
// }
|
|
// }
|
|
//
|
|
// memcpy(z, wdata, ne0*ne1*sizeof(float));
|
|
//
|
|
// //cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans,
|
|
// // ne0, ne1, 64,
|
|
// // 1.0f,
|
|
// // x, ne00,
|
|
// // y, ne11,
|
|
// // 0.0f,
|
|
// // z, 1500);
|
|
// }
|
|
// }
|
|
// printf("time = %f ms\n", tsum/1000.0);
|
|
// return;
|
|
// } else {
|
|
// //cblas_sgemv(CblasRowMajor, CblasTrans, ne00, ne01, 1.0, src0->data, ne01, src1->data, 1, 0.0, dst->data, 1);
|
|
// }
|
|
//
|
|
//#endif
|
|
|
|
|
|
if (nb01 >= nb00) {
|
|
// TODO: do not support transposed src1
|
|
assert(nb10 == sizeof(float));
|
|
|
|
// parallelize by src0 rows using ggml_vec_dot_f32
|
|
|
|
// total rows in src0
|
|
const int nr = ne01*ne02*ne03;
|
|
|
|
// rows per thread
|
|
const int dr = (nr + nth - 1)/nth;
|
|
|
|
// row range for this thread
|
|
const int ir0 = dr*ith;
|
|
const int ir1 = MIN(ir0 + dr, nr);
|
|
|
|
for (int ir = ir0; ir < ir1; ++ir) {
|
|
// src0 indices
|
|
const int i03 = ir/(ne02*ne01);
|
|
const int i02 = (ir - i03*ne02*ne01)/ne01;
|
|
const int i01 = (ir - i03*ne02*ne01 - i02*ne01);
|
|
|
|
for (int ic = 0; ic < ne11; ++ic) {
|
|
// src1 indices
|
|
const int i13 = i03;
|
|
const int i12 = i02;
|
|
const int i11 = ic;
|
|
|
|
// dst indices
|
|
const int i0 = i01;
|
|
const int i1 = i11;
|
|
const int i2 = i02;
|
|
const int i3 = i03;
|
|
|
|
ggml_vec_dot_f32(ne00,
|
|
(float *) ((char *) dst->data + (i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3)),
|
|
(float *) ((char *) src0->data + (i01*nb01 + i02*nb02 + i03*nb03)),
|
|
(float *) ((char *) src1->data + (i11*nb11 + i12*nb12 + i13*nb13)));
|
|
}
|
|
}
|
|
} else {
|
|
// parallelize by src1 columns using ggml_vec_mad_f32
|
|
// each thread has its own work data
|
|
// during FINALIZE we accumulate all work data into dst
|
|
|
|
// total columns in src1
|
|
const int nc = ne10;
|
|
|
|
// columns per thread
|
|
const int dc = (nc + nth - 1)/nth;
|
|
|
|
// column range for this thread
|
|
const int ic0 = dc*ith;
|
|
const int ic1 = MIN(ic0 + dc, nc);
|
|
|
|
// work data for thread
|
|
const int wo = (ne + CACHE_LINE_SIZE_F32)*ith;
|
|
float * const wdata = params->wdata;
|
|
|
|
for (int i13 = 0; i13 < ne13; ++i13) {
|
|
for (int i12 = 0; i12 < ne12; ++i12) {
|
|
for (int i11 = 0; i11 < ne11; ++i11) {
|
|
for (int ic = ic0; ic < ic1; ++ic) {
|
|
// src1 indices
|
|
const int i10 = ic;
|
|
|
|
// src0 indices
|
|
const int i03 = i13;
|
|
const int i02 = i12;
|
|
const int i00 = ic;
|
|
|
|
// dst indices
|
|
const int i1 = i11;
|
|
const int i2 = i12;
|
|
const int i3 = i13;
|
|
|
|
assert(sizeof(float)*(wo + i3*ne2*ne1*ne0 + i2*ne1*ne0 + i1*ne0 + ne01) <= params->wsize);
|
|
|
|
ggml_vec_mad_f32(ne01,
|
|
(float *) (wdata + wo + i3*ne2*ne1*ne0 + i2*ne1*ne0 + i1*ne0),
|
|
(float *) ((char *) src0->data + (i00*nb00 + i02*nb02 + i03*nb03)),
|
|
*(float *) ((char *) src1->data + (i10*nb10 + i11*nb11 + i12*nb12 + i13*nb13)));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
//int64_t t1 = ggml_perf_time_us();
|
|
//static int64_t acc = 0;
|
|
//acc += t1 - t0;
|
|
//if (t1 - t0 > 10) {
|
|
// printf("\n");
|
|
// printf("ne00 = %5d, ne01 = %5d, ne02 = %5d, ne03 = %5d\n", ne00, ne01, ne02, ne03);
|
|
// printf("nb00 = %5d, nb01 = %5d, nb02 = %5d, nb03 = %5d\n", nb00, nb01, nb02, nb03);
|
|
// printf("ne10 = %5d, ne11 = %5d, ne12 = %5d, ne13 = %5d\n", ne10, ne11, ne12, ne13);
|
|
// printf("nb10 = %5d, nb11 = %5d, nb12 = %5d, nb13 = %5d\n", nb10, nb11, nb12, nb13);
|
|
|
|
// printf("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX task %d/%d: %d us, acc = %d\n", ith, nth, (int) (t1 - t0), (int) acc);
|
|
//}
|
|
}
|
|
|
|
void ggml_compute_forward_mul_mat_f16_f32(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
const struct ggml_tensor * src1,
|
|
struct ggml_tensor * dst) {
|
|
int64_t t0 = ggml_perf_time_us();
|
|
UNUSED(t0);
|
|
|
|
const int ne00 = src0->ne[0];
|
|
const int ne01 = src0->ne[1];
|
|
const int ne02 = src0->ne[2];
|
|
const int ne03 = src0->ne[3];
|
|
|
|
const int ne10 = src1->ne[0];
|
|
const int ne11 = src1->ne[1];
|
|
const int ne12 = src1->ne[2];
|
|
const int ne13 = src1->ne[3];
|
|
|
|
const int ne0 = dst->ne[0];
|
|
const int ne1 = dst->ne[1];
|
|
const int ne2 = dst->ne[2];
|
|
const int ne3 = dst->ne[3];
|
|
const int ne = ne0*ne1*ne2*ne3;
|
|
|
|
const int nb00 = src0->nb[0];
|
|
const int nb01 = src0->nb[1];
|
|
const int nb02 = src0->nb[2];
|
|
const int nb03 = src0->nb[3];
|
|
|
|
const int nb10 = src1->nb[0];
|
|
const int nb11 = src1->nb[1];
|
|
const int nb12 = src1->nb[2];
|
|
const int nb13 = src1->nb[3];
|
|
|
|
const int nb0 = dst->nb[0];
|
|
const int nb1 = dst->nb[1];
|
|
const int nb2 = dst->nb[2];
|
|
const int nb3 = dst->nb[3];
|
|
|
|
const int ith = params->ith;
|
|
const int nth = params->nth;
|
|
|
|
assert(ne02 == ne12);
|
|
assert(ne03 == ne13);
|
|
assert(ne2 == ne12);
|
|
assert(ne3 == ne13);
|
|
|
|
// TODO: we don't support permuted src0
|
|
assert(nb00 == sizeof(ggml_fp16_t) || nb01 == sizeof(ggml_fp16_t));
|
|
|
|
// dst cannot be transposed or permuted
|
|
assert(nb0 == sizeof(float));
|
|
assert(nb0 <= nb1);
|
|
assert(nb1 <= nb2);
|
|
assert(nb2 <= nb3);
|
|
|
|
assert(ne0 == ne01);
|
|
assert(ne1 == ne11);
|
|
assert(ne2 == ne02);
|
|
assert(ne3 == ne03);
|
|
|
|
// nb01 >= nb00 - src0 is not transposed
|
|
// compute by src0 rows
|
|
//
|
|
// nb00 < nb01 - src0 is transposed
|
|
// compute by src0 columns
|
|
|
|
if (params->type == GGML_TASK_INIT) {
|
|
if (nb01 >= nb00) {
|
|
ggml_fp16_t * const wdata = params->wdata;
|
|
|
|
int id = 0;
|
|
for (int i13 = 0; i13 < ne13; ++i13) {
|
|
for (int i12 = 0; i12 < ne12; ++i12) {
|
|
for (int i11 = 0; i11 < ne11; ++i11) {
|
|
for (int i10 = 0; i10 < ne10; ++i10) {
|
|
wdata[id++] = ggml_fp32_to_fp16(*(float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11 + i10*nb10));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
GGML_ASSERT(id*sizeof(ggml_fp16_t) <= params->wsize);
|
|
|
|
return;
|
|
}
|
|
|
|
// TODO: fix this memset (wsize is overestimated)
|
|
memset(params->wdata, 0, params->wsize);
|
|
return;
|
|
}
|
|
|
|
if (params->type == GGML_TASK_FINALIZE) {
|
|
if (nb01 >= nb00) {
|
|
return;
|
|
}
|
|
|
|
// TODO: fix this memset (wsize is overestimated)
|
|
//assert(params->wsize == (ggml_nbytes(dst) + CACHE_LINE_SIZE)*nth);
|
|
|
|
ggml_fp16_t * const wdata = params->wdata;
|
|
|
|
// cols per thread
|
|
const int dc = (ne + nth - 1)/nth;
|
|
|
|
// col range for this thread
|
|
const int ic0 = dc*ith;
|
|
const int ic1 = MIN(ic0 + dc, ne);
|
|
|
|
for (int i = ic0; i < ic1; ++i) {
|
|
((float *) dst->data)[i] = ggml_fp16_to_fp32(wdata[i]);
|
|
}
|
|
|
|
for (int k = 1; k < nth; k++) {
|
|
for (int i = ic0; i < ic1; ++i) {
|
|
((float *) dst->data)[i] += ggml_fp16_to_fp32(wdata[(ne + CACHE_LINE_SIZE_F32)*k + i]);
|
|
}
|
|
}
|
|
|
|
return;
|
|
}
|
|
|
|
if (nb01 >= nb00) {
|
|
// fp16 -> half the size, so divide by 2
|
|
// TODO: do not support transposed src1
|
|
assert(nb10/2 == sizeof(ggml_fp16_t));
|
|
|
|
// parallelize by src0 rows using ggml_vec_dot_f32
|
|
|
|
// total rows in src0
|
|
const int nr = ne01*ne02*ne03;
|
|
|
|
// rows per thread
|
|
const int dr = (nr + nth - 1)/nth;
|
|
|
|
// row range for this thread
|
|
const int ir0 = dr*ith;
|
|
const int ir1 = MIN(ir0 + dr, nr);
|
|
|
|
ggml_fp16_t * wdata = params->wdata;
|
|
|
|
for (int ir = ir0; ir < ir1; ++ir) {
|
|
// src0 indices
|
|
const int i03 = ir/(ne02*ne01);
|
|
const int i02 = (ir - i03*ne02*ne01)/ne01;
|
|
const int i01 = (ir - i03*ne02*ne01 - i02*ne01);
|
|
|
|
const int i13 = i03;
|
|
const int i12 = i02;
|
|
|
|
const int i0 = i01;
|
|
const int i2 = i02;
|
|
const int i3 = i03;
|
|
|
|
ggml_fp16_t * src0_row = (ggml_fp16_t *) ((char *) src0->data + (i01*nb01 + i02*nb02 + i03*nb03));
|
|
ggml_fp16_t * src1_col = wdata + (i13*ne12*ne11 + i12*ne11 + 0)*ne00;
|
|
|
|
float * dst_col = (float *) ((char *) dst->data + (i0*nb0 + 0*nb1 + i2*nb2 + i3*nb3));
|
|
|
|
for (int ic = 0; ic < ne11; ++ic) {
|
|
assert(ne00 % 32 == 0);
|
|
|
|
ggml_vec_dot_f16(ne00, &dst_col[ic*ne0], src0_row, src1_col + ic*ne00);
|
|
}
|
|
}
|
|
} else {
|
|
// parallelize by src1 columns using ggml_vec_mad_f32
|
|
// each thread has its own work data
|
|
// during FINALIZE we accumulate all work data into dst
|
|
|
|
// total columns in src1
|
|
const int nc = ne10;
|
|
|
|
// columns per thread
|
|
const int dc = (nc + nth - 1)/nth;
|
|
|
|
// column range for this thread
|
|
const int ic0 = dc*ith;
|
|
const int ic1 = MIN(ic0 + dc, nc);
|
|
|
|
// work data for thread
|
|
const int wo = (ne + CACHE_LINE_SIZE_F32)*ith;
|
|
ggml_fp16_t * const wdata = params->wdata;
|
|
|
|
for (int i13 = 0; i13 < ne13; ++i13) {
|
|
for (int i12 = 0; i12 < ne12; ++i12) {
|
|
for (int i11 = 0; i11 < ne11; ++i11) {
|
|
// dst indices
|
|
const int i1 = i11;
|
|
const int i2 = i12;
|
|
const int i3 = i13;
|
|
|
|
ggml_fp16_t * dst_row = wdata + wo + i3*ne2*ne1*ne0 + i2*ne1*ne0 + i1*ne0;
|
|
|
|
for (int ic = ic0; ic < ic1; ++ic) {
|
|
// src1 indices
|
|
const int i10 = ic;
|
|
|
|
// src0 indices
|
|
const int i03 = i13;
|
|
const int i02 = i12;
|
|
const int i00 = ic;
|
|
|
|
assert(sizeof(ggml_fp16_t)*(wo + i3*ne2*ne1*ne0 + i2*ne1*ne0 + i1*ne0 + ne01) <= params->wsize);
|
|
|
|
ggml_fp16_t * src0_col = (ggml_fp16_t *) ((char *) src0->data + (i00*nb00 + i02*nb02 + i03*nb03));
|
|
float src1_val = * (float *) ((char *) src1->data + (i10*nb10 + i11*nb11 + i12*nb12 + i13*nb13));
|
|
|
|
ggml_vec_mad_f16(ne01, dst_row, src0_col, src1_val);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
//int64_t t1 = ggml_time_us();
|
|
//static int64_t acc = 0;
|
|
//acc += t1 - t0;
|
|
//if (t1 - t0 > 10) {
|
|
// printf("\n");
|
|
// printf("ne00 = %5d, ne01 = %5d, ne02 = %5d, ne03 = %5d\n", ne00, ne01, ne02, ne03);
|
|
// printf("nb00 = %5d, nb01 = %5d, nb02 = %5d, nb03 = %5d\n", nb00, nb01, nb02, nb03);
|
|
// printf("ne10 = %5d, ne11 = %5d, ne12 = %5d, ne13 = %5d\n", ne10, ne11, ne12, ne13);
|
|
|
|
// printf("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX task %d/%d: %d us, acc = %d\n", ith, nth, (int) (t1 - t0), (int) acc);
|
|
//}
|
|
}
|
|
|
|
void ggml_compute_forward_mul_mat(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
const struct ggml_tensor * src1,
|
|
struct ggml_tensor * dst) {
|
|
switch (src0->type) {
|
|
case GGML_TYPE_F16:
|
|
{
|
|
ggml_compute_forward_mul_mat_f16_f32(params, src0, src1, dst);
|
|
} break;
|
|
case GGML_TYPE_F32:
|
|
{
|
|
ggml_compute_forward_mul_mat_f32(params, src0, src1, dst);
|
|
} break;
|
|
case GGML_TYPE_I8:
|
|
case GGML_TYPE_I16:
|
|
case GGML_TYPE_I32:
|
|
case GGML_TYPE_COUNT:
|
|
{
|
|
assert(false);
|
|
} break;
|
|
}
|
|
}
|
|
|
|
// ggml_compute_forward_scale
|
|
|
|
void ggml_compute_forward_scale_f32(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
const struct ggml_tensor * src1,
|
|
struct ggml_tensor * dst) {
|
|
GGML_ASSERT(ggml_is_contiguous(src0));
|
|
GGML_ASSERT(ggml_is_contiguous(dst));
|
|
GGML_ASSERT(ggml_are_same_shape(src0, dst));
|
|
GGML_ASSERT(ggml_is_scalar(src1));
|
|
|
|
if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {
|
|
return;
|
|
}
|
|
|
|
// scale factor
|
|
const float v = *(float *) src1->data;
|
|
|
|
const int ith = params->ith;
|
|
const int nth = params->nth;
|
|
|
|
const int nc = src0->ne[0];
|
|
const int nr = ggml_nrows(src0);
|
|
|
|
// rows per thread
|
|
const int dr = (nr + nth - 1)/nth;
|
|
|
|
// row range for this thread
|
|
const int ir0 = dr*ith;
|
|
const int ir1 = MIN(ir0 + dr, nr);
|
|
|
|
for (int i1 = ir0; i1 < ir1; i1++) {
|
|
ggml_vec_scale_f32(nc, (float *) ((char *) dst->data + i1*(dst->nb[1])), v);
|
|
}
|
|
}
|
|
|
|
void ggml_compute_forward_scale(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
const struct ggml_tensor * src1,
|
|
struct ggml_tensor * dst) {
|
|
switch (src0->type) {
|
|
case GGML_TYPE_F32:
|
|
{
|
|
ggml_compute_forward_scale_f32(params, src0, src1, dst);
|
|
} break;
|
|
case GGML_TYPE_I8:
|
|
case GGML_TYPE_I16:
|
|
case GGML_TYPE_I32:
|
|
case GGML_TYPE_F16:
|
|
case GGML_TYPE_COUNT:
|
|
{
|
|
assert(false);
|
|
} break;
|
|
}
|
|
}
|
|
|
|
// ggml_compute_forward_cpy
|
|
|
|
void ggml_compute_forward_cpy(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
struct ggml_tensor * dst) {
|
|
ggml_compute_forward_dup(params, src0, dst);
|
|
}
|
|
|
|
// ggml_compute_forward_reshape
|
|
|
|
void ggml_compute_forward_reshape(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
struct ggml_tensor * dst) {
|
|
// NOP
|
|
UNUSED(params);
|
|
UNUSED(src0);
|
|
UNUSED(dst);
|
|
}
|
|
|
|
// ggml_compute_forward_view
|
|
|
|
void ggml_compute_forward_view(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0) {
|
|
// NOP
|
|
UNUSED(params);
|
|
UNUSED(src0);
|
|
}
|
|
|
|
// ggml_compute_forward_permute
|
|
|
|
void ggml_compute_forward_permute(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0) {
|
|
// NOP
|
|
UNUSED(params);
|
|
UNUSED(src0);
|
|
}
|
|
|
|
// ggml_compute_forward_transpose
|
|
|
|
void ggml_compute_forward_transpose(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0) {
|
|
// NOP
|
|
UNUSED(params);
|
|
UNUSED(src0);
|
|
}
|
|
|
|
// ggml_compute_forward_get_rows
|
|
|
|
void ggml_compute_forward_get_rows_f16(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
const struct ggml_tensor * src1,
|
|
struct ggml_tensor * dst) {
|
|
assert(params->ith == 0);
|
|
|
|
if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {
|
|
return;
|
|
}
|
|
|
|
const int nc = src0->ne[0];
|
|
const int nr = ggml_nelements(src1);
|
|
|
|
assert( dst->ne[0] == nc);
|
|
assert( dst->ne[1] == nr);
|
|
assert(src0->nb[0] == sizeof(ggml_fp16_t));
|
|
|
|
for (int i = 0; i < nr; ++i) {
|
|
const int r = ((int32_t *) src1->data)[i];
|
|
|
|
for (int j = 0; j < nc; ++j) {
|
|
ggml_fp16_t v = ((ggml_fp16_t *) ((char *) src0->data + r*src0->nb[1]))[j];
|
|
((float *) ((char *) dst->data + i*dst->nb[1]))[j] = ggml_fp16_to_fp32(v);
|
|
}
|
|
}
|
|
}
|
|
|
|
void ggml_compute_forward_get_rows_f32(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
const struct ggml_tensor * src1,
|
|
struct ggml_tensor * dst) {
|
|
assert(params->ith == 0);
|
|
|
|
if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {
|
|
return;
|
|
}
|
|
|
|
const int nc = src0->ne[0];
|
|
const int nr = ggml_nelements(src1);
|
|
|
|
assert( dst->ne[0] == nc);
|
|
assert( dst->ne[1] == nr);
|
|
assert(src0->nb[0] == sizeof(float));
|
|
|
|
for (int i = 0; i < nr; ++i) {
|
|
const int r = ((int32_t *) src1->data)[i];
|
|
|
|
ggml_vec_cpy_f32(nc,
|
|
(float *) ((char *) dst->data + i*dst->nb[1]),
|
|
(float *) ((char *) src0->data + r*src0->nb[1]));
|
|
}
|
|
}
|
|
|
|
void ggml_compute_forward_get_rows(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
const struct ggml_tensor * src1,
|
|
struct ggml_tensor * dst) {
|
|
switch (src0->type) {
|
|
case GGML_TYPE_F16:
|
|
{
|
|
ggml_compute_forward_get_rows_f16(params, src0, src1, dst);
|
|
} break;
|
|
case GGML_TYPE_F32:
|
|
{
|
|
ggml_compute_forward_get_rows_f32(params, src0, src1, dst);
|
|
} break;
|
|
case GGML_TYPE_I8:
|
|
case GGML_TYPE_I16:
|
|
case GGML_TYPE_I32:
|
|
case GGML_TYPE_COUNT:
|
|
{
|
|
assert(false);
|
|
} break;
|
|
}
|
|
}
|
|
|
|
// ggml_compute_forward_diag_mask_inf
|
|
|
|
void ggml_compute_forward_diag_mask_inf_f32(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
const struct ggml_tensor * src1,
|
|
struct ggml_tensor * dst) {
|
|
assert(params->ith == 0);
|
|
assert(src1->type == GGML_TYPE_I32);
|
|
assert(ggml_nelements(src1) == 1);
|
|
|
|
if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {
|
|
return;
|
|
}
|
|
|
|
const int n_past = ((int32_t *) src1->data)[0];
|
|
|
|
// TODO: handle transposed/permuted matrices
|
|
|
|
const int n = ggml_nrows(src0);
|
|
const int nc = src0->ne[0];
|
|
const int nr = src0->ne[1];
|
|
const int nz = n/nr;
|
|
|
|
assert( dst->nb[0] == sizeof(float));
|
|
assert(src0->nb[0] == sizeof(float));
|
|
|
|
for (int k = 0; k < nz; k++) {
|
|
for (int j = 0; j < nr; j++) {
|
|
for (int i = n_past; i < nc; i++) {
|
|
if (i > n_past + j) {
|
|
*(float *)((char *) dst->data + k*dst->nb[2] + j*dst->nb[1] + i*dst->nb[0]) = -INFINITY;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void ggml_compute_forward_diag_mask_inf(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
const struct ggml_tensor * src1,
|
|
struct ggml_tensor * dst) {
|
|
switch (src0->type) {
|
|
case GGML_TYPE_F32:
|
|
{
|
|
ggml_compute_forward_diag_mask_inf_f32(params, src0, src1, dst);
|
|
} break;
|
|
case GGML_TYPE_I8:
|
|
case GGML_TYPE_I16:
|
|
case GGML_TYPE_I32:
|
|
case GGML_TYPE_F16:
|
|
case GGML_TYPE_COUNT:
|
|
{
|
|
assert(false);
|
|
} break;
|
|
}
|
|
}
|
|
|
|
// ggml_compute_forward_soft_max
|
|
|
|
void ggml_compute_forward_soft_max_f32(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
struct ggml_tensor * dst) {
|
|
GGML_ASSERT(ggml_is_contiguous(src0));
|
|
GGML_ASSERT(ggml_is_contiguous(dst));
|
|
GGML_ASSERT(ggml_are_same_shape(src0, dst));
|
|
|
|
if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {
|
|
return;
|
|
}
|
|
|
|
// TODO: handle transposed/permuted matrices
|
|
|
|
const int ith = params->ith;
|
|
const int nth = params->nth;
|
|
|
|
const int nc = src0->ne[0];
|
|
const int nr = ggml_nrows(src0);
|
|
|
|
// rows per thread
|
|
const int dr = (nr + nth - 1)/nth;
|
|
|
|
// row range for this thread
|
|
const int ir0 = dr*ith;
|
|
const int ir1 = MIN(ir0 + dr, nr);
|
|
|
|
for (int i1 = ir0; i1 < ir1; i1++) {
|
|
float *p = (float *)((char *) dst->data + i1*dst->nb[1]);
|
|
|
|
#ifndef NDEBUG
|
|
for (int i = 0; i < nc; ++i) {
|
|
assert(!isnan(p[i]));
|
|
}
|
|
#endif
|
|
|
|
float max = -INFINITY;
|
|
for (int i = 0; i < nc; i++) {
|
|
max = MAX(max, p[i]);
|
|
}
|
|
|
|
ggml_float sum = 0.0;
|
|
|
|
uint16_t ss;
|
|
for (int i = 0; i < nc; i++) {
|
|
if (p[i] == -INFINITY) {
|
|
p[i] = 0.0;
|
|
} else {
|
|
//const float val = (p[i] == -INFINITY) ? 0.0 : exp(p[i] - max);
|
|
ggml_fp16_t s = ggml_fp32_to_fp16(p[i] - max);
|
|
memcpy(&ss, &s, sizeof(ss));
|
|
const float val = ggml_fp16_to_fp32(table_exp_f16[ss]);
|
|
sum += val;
|
|
p[i] = val;
|
|
}
|
|
}
|
|
|
|
assert(sum > 0.0f);
|
|
|
|
sum = 1.0/sum;
|
|
ggml_vec_scale_f32(nc, p, sum);
|
|
|
|
#ifndef NDEBUG
|
|
for (int i = 0; i < nc; ++i) {
|
|
assert(!isnan(p[i]));
|
|
assert(!isinf(p[i]));
|
|
}
|
|
#endif
|
|
}
|
|
}
|
|
|
|
void ggml_compute_forward_soft_max(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
struct ggml_tensor * dst) {
|
|
switch (src0->type) {
|
|
case GGML_TYPE_F32:
|
|
{
|
|
ggml_compute_forward_soft_max_f32(params, src0, dst);
|
|
} break;
|
|
case GGML_TYPE_I8:
|
|
case GGML_TYPE_I16:
|
|
case GGML_TYPE_I32:
|
|
case GGML_TYPE_F16:
|
|
case GGML_TYPE_COUNT:
|
|
{
|
|
assert(false);
|
|
} break;
|
|
}
|
|
}
|
|
|
|
// ggml_compute_forward_rope
|
|
|
|
void ggml_compute_forward_rope_f32(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
const struct ggml_tensor * src1,
|
|
struct ggml_tensor * dst) {
|
|
assert(params->ith == 0);
|
|
assert(src1->type == GGML_TYPE_I32);
|
|
assert(ggml_nelements(src1) == 3);
|
|
|
|
if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {
|
|
return;
|
|
}
|
|
|
|
const int n_past = ((int32_t *) src1->data)[0];
|
|
const int n_dims = ((int32_t *) src1->data)[1];
|
|
const int mode = ((int32_t *) src1->data)[2];
|
|
|
|
//const int ne0 = src0->ne[0];
|
|
const int ne1 = src0->ne[1];
|
|
const int ne2 = src0->ne[2];
|
|
const int ne3 = src0->ne[3];
|
|
|
|
const int nb0 = src0->nb[0];
|
|
const int nb1 = src0->nb[1];
|
|
const int nb2 = src0->nb[2];
|
|
const int nb3 = src0->nb[3];
|
|
|
|
//printf("ne0: %d, ne1: %d, ne2: %d, ne3: %d\n", ne0, ne1, ne2, ne3);
|
|
//printf("n_past = %d, ne2 = %d\n", n_past, ne2);
|
|
|
|
assert(nb0 == sizeof(float));
|
|
|
|
// TODO: optimize
|
|
for (int i3 = 0; i3 < ne3; i3++) {
|
|
for (int i2 = (mode == 0 ? 0 : n_past); i2 < ne2; i2++) {
|
|
const int p = (mode == 0 ? n_past + i2 : i2);
|
|
for (int i1 = 0; i1 < ne1; i1++) {
|
|
for (int i0 = 0; i0 < n_dims; i0 += 2) {
|
|
const double theta = pow(10000.0, ((double)-i0)/n_dims);
|
|
|
|
const double cos_theta = cos(p*theta);
|
|
const double sin_theta = sin(p*theta);
|
|
|
|
const float * const src = (float *)((char *) src0->data + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
|
|
float * dst_data = (float *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
|
|
|
|
double x0 = src[0];
|
|
double x1 = src[1];
|
|
|
|
dst_data[0] = x0*cos_theta - x1*sin_theta;
|
|
dst_data[1] = x0*sin_theta + x1*cos_theta;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void ggml_compute_forward_rope(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
const struct ggml_tensor * src1,
|
|
struct ggml_tensor * dst) {
|
|
switch (src0->type) {
|
|
case GGML_TYPE_F32:
|
|
{
|
|
ggml_compute_forward_rope_f32(params, src0, src1, dst);
|
|
} break;
|
|
case GGML_TYPE_I8:
|
|
case GGML_TYPE_I16:
|
|
case GGML_TYPE_I32:
|
|
case GGML_TYPE_F16:
|
|
case GGML_TYPE_COUNT:
|
|
{
|
|
assert(false);
|
|
} break;
|
|
}
|
|
}
|
|
|
|
// ggml_compute_forward_conv_1d_1s
|
|
|
|
void ggml_compute_forward_conv_1d_1s_f16_f32(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
const struct ggml_tensor * src1,
|
|
struct ggml_tensor * dst) {
|
|
GGML_ASSERT(src0->type == GGML_TYPE_F16);
|
|
GGML_ASSERT(src1->type == GGML_TYPE_F32);
|
|
GGML_ASSERT( dst->type == GGML_TYPE_F32);
|
|
|
|
int64_t t0 = ggml_perf_time_us();
|
|
UNUSED(t0);
|
|
|
|
const int ne00 = src0->ne[0];
|
|
const int ne01 = src0->ne[1];
|
|
const int ne02 = src0->ne[2];
|
|
//const int ne03 = src0->ne[3];
|
|
|
|
const int ne10 = src1->ne[0];
|
|
const int ne11 = src1->ne[1];
|
|
//const int ne12 = src1->ne[2];
|
|
//const int ne13 = src1->ne[3];
|
|
|
|
//const int ne0 = dst->ne[0];
|
|
//const int ne1 = dst->ne[1];
|
|
//const int ne2 = dst->ne[2];
|
|
//const int ne3 = dst->ne[3];
|
|
//const int ne = ne0*ne1*ne2*ne3;
|
|
|
|
const int nb00 = src0->nb[0];
|
|
const int nb01 = src0->nb[1];
|
|
const int nb02 = src0->nb[2];
|
|
//const int nb03 = src0->nb[3];
|
|
|
|
const int nb10 = src1->nb[0];
|
|
const int nb11 = src1->nb[1];
|
|
//const int nb12 = src1->nb[2];
|
|
//const int nb13 = src1->nb[3];
|
|
|
|
//const int nb0 = dst->nb[0];
|
|
const int nb1 = dst->nb[1];
|
|
//const int nb2 = dst->nb[2];
|
|
//const int nb3 = dst->nb[3];
|
|
|
|
const int ith = params->ith;
|
|
const int nth = params->nth;
|
|
|
|
const int nk = ne00;
|
|
const int nh = nk/2;
|
|
|
|
const int ew0 = ggml_up32(ne01);
|
|
|
|
GGML_ASSERT(ne00 % 2 == 1); // TODO: support even kernel sizes
|
|
GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));
|
|
GGML_ASSERT(nb10 == sizeof(float));
|
|
|
|
if (params->type == GGML_TASK_INIT) {
|
|
// TODO: fix this memset (wsize is overestimated)
|
|
memset(params->wdata, 0, params->wsize);
|
|
|
|
// prepare kernel data (src0)
|
|
{
|
|
ggml_fp16_t * const wdata = (ggml_fp16_t *) params->wdata + 0;
|
|
|
|
for (int i02 = 0; i02 < ne02; i02++) {
|
|
for (int i01 = 0; i01 < ne01; i01++) {
|
|
const ggml_fp16_t * const src = (ggml_fp16_t *)((char *) src0->data + i02*nb02 + i01*nb01);
|
|
ggml_fp16_t * dst_data = wdata + i02*ew0*ne00;
|
|
for (int i00 = 0; i00 < ne00; i00++) {
|
|
dst_data[i00*ew0 + i01] = src[i00];
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// prepare source data (src1)
|
|
{
|
|
ggml_fp16_t * const wdata = (ggml_fp16_t *) params->wdata + ne02*ew0*ne00;
|
|
|
|
for (int i11 = 0; i11 < ne11; i11++) {
|
|
const float * const src = (float *)((char *) src1->data + i11*nb11);
|
|
ggml_fp16_t * dst_data = wdata;
|
|
for (int i10 = 0; i10 < ne10; i10++) {
|
|
dst_data[(i10 + nh)*ew0 + i11] = ggml_fp32_to_fp16(src[i10]);
|
|
}
|
|
}
|
|
}
|
|
|
|
return;
|
|
}
|
|
|
|
if (params->type == GGML_TASK_FINALIZE) {
|
|
return;
|
|
}
|
|
|
|
// total rows in dst
|
|
const int nr = ne02;
|
|
|
|
// rows per thread
|
|
const int dr = (nr + nth - 1)/nth;
|
|
|
|
// row range for this thread
|
|
const int ir0 = dr*ith;
|
|
const int ir1 = MIN(ir0 + dr, nr);
|
|
|
|
for (int i1 = ir0; i1 < ir1; i1++) {
|
|
float * dst_data = (float *)((char *) dst->data + i1*nb1);
|
|
for (int i0 = 0; i0 < ne10; ++i0) {
|
|
dst_data[i0] = 0;
|
|
for (int k = -nh; k <= nh; k++) {
|
|
float v = 0.0f;
|
|
ggml_vec_dot_f16(ew0, &v,
|
|
(ggml_fp16_t *) params->wdata + i1*ew0*ne00 + (nh + k)*ew0,
|
|
(ggml_fp16_t *) params->wdata + ne02*ew0*ne00 + (i0 + nh + k)*ew0);
|
|
|
|
dst_data[i0] += v;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void ggml_compute_forward_conv_1d_1s_f32(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
const struct ggml_tensor * src1,
|
|
struct ggml_tensor * dst) {
|
|
GGML_ASSERT(src0->type == GGML_TYPE_F32);
|
|
GGML_ASSERT(src1->type == GGML_TYPE_F32);
|
|
GGML_ASSERT( dst->type == GGML_TYPE_F32);
|
|
|
|
int64_t t0 = ggml_perf_time_us();
|
|
UNUSED(t0);
|
|
|
|
const int ne00 = src0->ne[0];
|
|
const int ne01 = src0->ne[1];
|
|
const int ne02 = src0->ne[2];
|
|
//const int ne03 = src0->ne[3];
|
|
|
|
const int ne10 = src1->ne[0];
|
|
const int ne11 = src1->ne[1];
|
|
//const int ne12 = src1->ne[2];
|
|
//const int ne13 = src1->ne[3];
|
|
|
|
//const int ne0 = dst->ne[0];
|
|
//const int ne1 = dst->ne[1];
|
|
//const int ne2 = dst->ne[2];
|
|
//const int ne3 = dst->ne[3];
|
|
//const int ne = ne0*ne1*ne2*ne3;
|
|
|
|
const int nb00 = src0->nb[0];
|
|
const int nb01 = src0->nb[1];
|
|
const int nb02 = src0->nb[2];
|
|
//const int nb03 = src0->nb[3];
|
|
|
|
const int nb10 = src1->nb[0];
|
|
const int nb11 = src1->nb[1];
|
|
//const int nb12 = src1->nb[2];
|
|
//const int nb13 = src1->nb[3];
|
|
|
|
//const int nb0 = dst->nb[0];
|
|
const int nb1 = dst->nb[1];
|
|
//const int nb2 = dst->nb[2];
|
|
//const int nb3 = dst->nb[3];
|
|
|
|
const int ith = params->ith;
|
|
const int nth = params->nth;
|
|
|
|
const int nk = ne00;
|
|
const int nh = nk/2;
|
|
|
|
const int ew0 = ggml_up32(ne01);
|
|
|
|
GGML_ASSERT(ne00 % 2 == 1); // TODO: support even kernel sizes
|
|
GGML_ASSERT(nb00 == sizeof(float));
|
|
GGML_ASSERT(nb10 == sizeof(float));
|
|
|
|
if (params->type == GGML_TASK_INIT) {
|
|
// TODO: fix this memset (wsize is overestimated)
|
|
memset(params->wdata, 0, params->wsize);
|
|
|
|
// prepare kernel data (src0)
|
|
{
|
|
float * const wdata = (float *) params->wdata + 0;
|
|
|
|
for (int i02 = 0; i02 < ne02; i02++) {
|
|
for (int i01 = 0; i01 < ne01; i01++) {
|
|
const float * const src = (float *)((char *) src0->data + i02*nb02 + i01*nb01);
|
|
float * dst_data = wdata + i02*ew0*ne00;
|
|
for (int i00 = 0; i00 < ne00; i00++) {
|
|
dst_data[i00*ew0 + i01] = src[i00];
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// prepare source data (src1)
|
|
{
|
|
float * const wdata = (float *) params->wdata + ne02*ew0*ne00;
|
|
|
|
for (int i11 = 0; i11 < ne11; i11++) {
|
|
const float * const src = (float *)((char *) src1->data + i11*nb11);
|
|
float * dst_data = wdata;
|
|
for (int i10 = 0; i10 < ne10; i10++) {
|
|
dst_data[(i10 + nh)*ew0 + i11] = src[i10];
|
|
}
|
|
}
|
|
}
|
|
|
|
return;
|
|
}
|
|
|
|
if (params->type == GGML_TASK_FINALIZE) {
|
|
return;
|
|
}
|
|
|
|
// total rows in dst
|
|
const int nr = ne02;
|
|
|
|
// rows per thread
|
|
const int dr = (nr + nth - 1)/nth;
|
|
|
|
// row range for this thread
|
|
const int ir0 = dr*ith;
|
|
const int ir1 = MIN(ir0 + dr, nr);
|
|
|
|
for (int i1 = ir0; i1 < ir1; i1++) {
|
|
float * dst_data = (float *)((char *) dst->data + i1*nb1);
|
|
for (int i0 = 0; i0 < ne10; ++i0) {
|
|
dst_data[i0] = 0;
|
|
for (int k = -nh; k <= nh; k++) {
|
|
float v = 0.0f;
|
|
ggml_vec_dot_f32(ew0, &v,
|
|
(float *) params->wdata + i1*ew0*ne00 + (nh + k)*ew0,
|
|
(float *) params->wdata + ne02*ew0*ne00 + (i0 + nh + k)*ew0);
|
|
|
|
dst_data[i0] += v;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void ggml_compute_forward_conv_1d_1s(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
const struct ggml_tensor * src1,
|
|
struct ggml_tensor * dst) {
|
|
switch (src0->type) {
|
|
case GGML_TYPE_F16:
|
|
{
|
|
ggml_compute_forward_conv_1d_1s_f16_f32(params, src0, src1, dst);
|
|
} break;
|
|
case GGML_TYPE_F32:
|
|
{
|
|
ggml_compute_forward_conv_1d_1s_f32(params, src0, src1, dst);
|
|
} break;
|
|
case GGML_TYPE_I8:
|
|
case GGML_TYPE_I16:
|
|
case GGML_TYPE_I32:
|
|
case GGML_TYPE_COUNT:
|
|
{
|
|
GGML_ASSERT(false);
|
|
} break;
|
|
}
|
|
}
|
|
|
|
// ggml_compute_forward_conv_1d_2s
|
|
|
|
void ggml_compute_forward_conv_1d_2s_f16_f32(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
const struct ggml_tensor * src1,
|
|
struct ggml_tensor * dst) {
|
|
GGML_ASSERT(src0->type == GGML_TYPE_F16);
|
|
GGML_ASSERT(src1->type == GGML_TYPE_F32);
|
|
GGML_ASSERT( dst->type == GGML_TYPE_F32);
|
|
|
|
int64_t t0 = ggml_perf_time_us();
|
|
UNUSED(t0);
|
|
|
|
const int ne00 = src0->ne[0];
|
|
const int ne01 = src0->ne[1];
|
|
const int ne02 = src0->ne[2];
|
|
//const int ne03 = src0->ne[3];
|
|
|
|
const int ne10 = src1->ne[0];
|
|
const int ne11 = src1->ne[1];
|
|
//const int ne12 = src1->ne[2];
|
|
//const int ne13 = src1->ne[3];
|
|
|
|
//const int ne0 = dst->ne[0];
|
|
//const int ne1 = dst->ne[1];
|
|
//const int ne2 = dst->ne[2];
|
|
//const int ne3 = dst->ne[3];
|
|
//const int ne = ne0*ne1*ne2*ne3;
|
|
|
|
const int nb00 = src0->nb[0];
|
|
const int nb01 = src0->nb[1];
|
|
const int nb02 = src0->nb[2];
|
|
//const int nb03 = src0->nb[3];
|
|
|
|
const int nb10 = src1->nb[0];
|
|
const int nb11 = src1->nb[1];
|
|
//const int nb12 = src1->nb[2];
|
|
//const int nb13 = src1->nb[3];
|
|
|
|
//const int nb0 = dst->nb[0];
|
|
const int nb1 = dst->nb[1];
|
|
//const int nb2 = dst->nb[2];
|
|
//const int nb3 = dst->nb[3];
|
|
|
|
const int ith = params->ith;
|
|
const int nth = params->nth;
|
|
|
|
const int nk = ne00;
|
|
const int nh = nk/2;
|
|
|
|
const int ew0 = ggml_up32(ne01);
|
|
|
|
GGML_ASSERT(ne00 % 2 == 1); // TODO: support even kernel sizes
|
|
GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));
|
|
GGML_ASSERT(nb10 == sizeof(float));
|
|
|
|
if (params->type == GGML_TASK_INIT) {
|
|
// TODO: fix this memset (wsize is overestimated)
|
|
memset(params->wdata, 0, params->wsize);
|
|
|
|
// prepare kernel data (src0)
|
|
{
|
|
ggml_fp16_t * const wdata = (ggml_fp16_t *) params->wdata + 0;
|
|
|
|
for (int i02 = 0; i02 < ne02; i02++) {
|
|
for (int i01 = 0; i01 < ne01; i01++) {
|
|
const ggml_fp16_t * const src = (ggml_fp16_t *)((char *) src0->data + i02*nb02 + i01*nb01);
|
|
ggml_fp16_t * dst_data = wdata + i02*ew0*ne00;
|
|
for (int i00 = 0; i00 < ne00; i00++) {
|
|
dst_data[i00*ew0 + i01] = src[i00];
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// prepare source data (src1)
|
|
{
|
|
ggml_fp16_t * const wdata = (ggml_fp16_t *) params->wdata + ne02*ew0*ne00;
|
|
|
|
for (int i11 = 0; i11 < ne11; i11++) {
|
|
const float * const src = (float *)((char *) src1->data + i11*nb11);
|
|
ggml_fp16_t * dst_data = wdata;
|
|
for (int i10 = 0; i10 < ne10; i10++) {
|
|
dst_data[(i10 + nh)*ew0 + i11] = ggml_fp32_to_fp16(src[i10]);
|
|
}
|
|
}
|
|
}
|
|
|
|
return;
|
|
}
|
|
|
|
if (params->type == GGML_TASK_FINALIZE) {
|
|
return;
|
|
}
|
|
|
|
// total rows in dst
|
|
const int nr = ne02;
|
|
|
|
// rows per thread
|
|
const int dr = (nr + nth - 1)/nth;
|
|
|
|
// row range for this thread
|
|
const int ir0 = dr*ith;
|
|
const int ir1 = MIN(ir0 + dr, nr);
|
|
|
|
for (int i1 = ir0; i1 < ir1; i1++) {
|
|
float * dst_data = (float *)((char *) dst->data + i1*nb1);
|
|
for (int i0 = 0; i0 < ne10; i0 += 2) {
|
|
dst_data[i0/2] = 0;
|
|
for (int k = -nh; k <= nh; k++) {
|
|
float v = 0.0f;
|
|
ggml_vec_dot_f16(ew0, &v,
|
|
(ggml_fp16_t *) params->wdata + i1*ew0*ne00 + (nh + k)*ew0,
|
|
(ggml_fp16_t *) params->wdata + ne02*ew0*ne00 + (i0 + nh + k)*ew0);
|
|
|
|
dst_data[i0/2] += v;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void ggml_compute_forward_conv_1d_2s_f32(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
const struct ggml_tensor * src1,
|
|
struct ggml_tensor * dst) {
|
|
GGML_ASSERT(src0->type == GGML_TYPE_F32);
|
|
GGML_ASSERT(src1->type == GGML_TYPE_F32);
|
|
GGML_ASSERT( dst->type == GGML_TYPE_F32);
|
|
|
|
int64_t t0 = ggml_perf_time_us();
|
|
UNUSED(t0);
|
|
|
|
const int ne00 = src0->ne[0];
|
|
const int ne01 = src0->ne[1];
|
|
const int ne02 = src0->ne[2];
|
|
//const int ne03 = src0->ne[3];
|
|
|
|
const int ne10 = src1->ne[0];
|
|
const int ne11 = src1->ne[1];
|
|
//const int ne12 = src1->ne[2];
|
|
//const int ne13 = src1->ne[3];
|
|
|
|
//const int ne0 = dst->ne[0];
|
|
//const int ne1 = dst->ne[1];
|
|
//const int ne2 = dst->ne[2];
|
|
//const int ne3 = dst->ne[3];
|
|
//const int ne = ne0*ne1*ne2*ne3;
|
|
|
|
const int nb00 = src0->nb[0];
|
|
const int nb01 = src0->nb[1];
|
|
const int nb02 = src0->nb[2];
|
|
//const int nb03 = src0->nb[3];
|
|
|
|
const int nb10 = src1->nb[0];
|
|
const int nb11 = src1->nb[1];
|
|
//const int nb12 = src1->nb[2];
|
|
//const int nb13 = src1->nb[3];
|
|
|
|
//const int nb0 = dst->nb[0];
|
|
const int nb1 = dst->nb[1];
|
|
//const int nb2 = dst->nb[2];
|
|
//const int nb3 = dst->nb[3];
|
|
|
|
const int ith = params->ith;
|
|
const int nth = params->nth;
|
|
|
|
const int nk = ne00;
|
|
const int nh = nk/2;
|
|
|
|
const int ew0 = ggml_up32(ne01);
|
|
|
|
GGML_ASSERT(ne00 % 2 == 1); // TODO: support even kernel sizes
|
|
GGML_ASSERT(nb00 == sizeof(float));
|
|
GGML_ASSERT(nb10 == sizeof(float));
|
|
|
|
if (params->type == GGML_TASK_INIT) {
|
|
// TODO: fix this memset (wsize is overestimated)
|
|
memset(params->wdata, 0, params->wsize);
|
|
|
|
// prepare kernel data (src0)
|
|
{
|
|
float * const wdata = (float *) params->wdata + 0;
|
|
|
|
for (int i02 = 0; i02 < ne02; i02++) {
|
|
for (int i01 = 0; i01 < ne01; i01++) {
|
|
const float * const src = (float *)((char *) src0->data + i02*nb02 + i01*nb01);
|
|
float * dst_data = wdata + i02*ew0*ne00;
|
|
for (int i00 = 0; i00 < ne00; i00++) {
|
|
dst_data[i00*ew0 + i01] = src[i00];
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// prepare source data (src1)
|
|
{
|
|
float * const wdata = (float *) params->wdata + ne02*ew0*ne00;
|
|
|
|
for (int i11 = 0; i11 < ne11; i11++) {
|
|
const float * const src = (float *)((char *) src1->data + i11*nb11);
|
|
float * dst_data = wdata;
|
|
for (int i10 = 0; i10 < ne10; i10++) {
|
|
dst_data[(i10 + nh)*ew0 + i11] = src[i10];
|
|
}
|
|
}
|
|
}
|
|
|
|
return;
|
|
}
|
|
|
|
if (params->type == GGML_TASK_FINALIZE) {
|
|
return;
|
|
}
|
|
|
|
// total rows in dst
|
|
const int nr = ne02;
|
|
|
|
// rows per thread
|
|
const int dr = (nr + nth - 1)/nth;
|
|
|
|
// row range for this thread
|
|
const int ir0 = dr*ith;
|
|
const int ir1 = MIN(ir0 + dr, nr);
|
|
|
|
for (int i1 = ir0; i1 < ir1; i1++) {
|
|
float * dst_data = (float *)((char *) dst->data + i1*nb1);
|
|
for (int i0 = 0; i0 < ne10; i0 += 2) {
|
|
dst_data[i0/2] = 0;
|
|
for (int k = -nh; k <= nh; k++) {
|
|
float v = 0.0f;
|
|
ggml_vec_dot_f32(ew0, &v,
|
|
(float *) params->wdata + i1*ew0*ne00 + (nh + k)*ew0,
|
|
(float *) params->wdata + ne02*ew0*ne00 + (i0 + nh + k)*ew0);
|
|
|
|
dst_data[i0/2] += v;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void ggml_compute_forward_conv_1d_2s(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * src0,
|
|
const struct ggml_tensor * src1,
|
|
struct ggml_tensor * dst) {
|
|
switch (src0->type) {
|
|
case GGML_TYPE_F16:
|
|
{
|
|
ggml_compute_forward_conv_1d_2s_f16_f32(params, src0, src1, dst);
|
|
} break;
|
|
case GGML_TYPE_F32:
|
|
{
|
|
ggml_compute_forward_conv_1d_2s_f32(params, src0, src1, dst);
|
|
} break;
|
|
case GGML_TYPE_I8:
|
|
case GGML_TYPE_I16:
|
|
case GGML_TYPE_I32:
|
|
case GGML_TYPE_COUNT:
|
|
{
|
|
GGML_ASSERT(false);
|
|
} break;
|
|
}
|
|
}
|
|
|
|
// ggml_compute_forward_flash_attn
|
|
|
|
void ggml_compute_forward_flash_attn_f32(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * q,
|
|
const struct ggml_tensor * k,
|
|
const struct ggml_tensor * v,
|
|
const bool masked,
|
|
struct ggml_tensor * dst) {
|
|
int64_t t0 = ggml_perf_time_us();
|
|
UNUSED(t0);
|
|
|
|
const int neq0 = q->ne[0];
|
|
const int neq1 = q->ne[1];
|
|
const int neq2 = q->ne[2];
|
|
const int neq3 = q->ne[3];
|
|
|
|
const int nek0 = k->ne[0];
|
|
const int nek1 = k->ne[1];
|
|
//const int nek2 = k->ne[2];
|
|
//const int nek3 = k->ne[3];
|
|
|
|
//const int nev0 = v->ne[0];
|
|
const int nev1 = v->ne[1];
|
|
//const int nev2 = v->ne[2];
|
|
//const int nev3 = v->ne[3];
|
|
|
|
const int ne0 = dst->ne[0];
|
|
const int ne1 = dst->ne[1];
|
|
//const int ne2 = dst->ne[2];
|
|
//const int ne3 = dst->ne[3];
|
|
|
|
const int nbk0 = k->nb[0];
|
|
const int nbk1 = k->nb[1];
|
|
const int nbk2 = k->nb[2];
|
|
const int nbk3 = k->nb[3];
|
|
|
|
const int nbq0 = q->nb[0];
|
|
const int nbq1 = q->nb[1];
|
|
const int nbq2 = q->nb[2];
|
|
const int nbq3 = q->nb[3];
|
|
|
|
const int nbv0 = v->nb[0];
|
|
const int nbv1 = v->nb[1];
|
|
const int nbv2 = v->nb[2];
|
|
const int nbv3 = v->nb[3];
|
|
|
|
const int nb0 = dst->nb[0];
|
|
const int nb1 = dst->nb[1];
|
|
const int nb2 = dst->nb[2];
|
|
const int nb3 = dst->nb[3];
|
|
|
|
const int ith = params->ith;
|
|
const int nth = params->nth;
|
|
|
|
const int D = neq0;
|
|
const int N = neq1;
|
|
const int P = nek1 - N;
|
|
const int M = P + N;
|
|
|
|
GGML_ASSERT(ne0 == D);
|
|
GGML_ASSERT(ne1 == N);
|
|
GGML_ASSERT(P >= 0);
|
|
|
|
GGML_ASSERT(nbq0 == sizeof(float));
|
|
GGML_ASSERT(nbk0 == sizeof(float));
|
|
GGML_ASSERT(nbv0 == sizeof(float));
|
|
|
|
GGML_ASSERT(neq0 == D);
|
|
GGML_ASSERT(nek0 == D);
|
|
GGML_ASSERT(nev1 == D);
|
|
|
|
GGML_ASSERT(neq1 == N);
|
|
GGML_ASSERT(nek1 == N + P);
|
|
GGML_ASSERT(nev1 == D);
|
|
|
|
// dst cannot be transposed or permuted
|
|
GGML_ASSERT(nb0 == sizeof(float));
|
|
GGML_ASSERT(nb0 <= nb1);
|
|
GGML_ASSERT(nb1 <= nb2);
|
|
GGML_ASSERT(nb2 <= nb3);
|
|
|
|
if (params->type == GGML_TASK_INIT) {
|
|
return;
|
|
}
|
|
|
|
if (params->type == GGML_TASK_FINALIZE) {
|
|
return;
|
|
}
|
|
|
|
// parallelize by q rows using ggml_vec_dot_f32
|
|
|
|
// total rows in q
|
|
const int nr = neq1*neq2*neq3;
|
|
|
|
// rows per thread
|
|
const int dr = (nr + nth - 1)/nth;
|
|
|
|
// row range for this thread
|
|
const int ir0 = dr*ith;
|
|
const int ir1 = MIN(ir0 + dr, nr);
|
|
|
|
const float scale = 1.0/sqrt((double) D);
|
|
|
|
//printf("P=%d N=%d D=%d ir0=%d ir1=%d scale = %f\n", P, N, D, ir0, ir1, scale);
|
|
|
|
for (int ir = ir0; ir < ir1; ++ir) {
|
|
// q indices
|
|
const int iq3 = ir/(neq2*neq1);
|
|
const int iq2 = (ir - iq3*neq2*neq1)/neq1;
|
|
const int iq1 = (ir - iq3*neq2*neq1 - iq2*neq1);
|
|
|
|
float * S = (float *) params->wdata + ith*(M + CACHE_LINE_SIZE_F32);
|
|
|
|
for (int ic = 0; ic < nek1; ++ic) {
|
|
// k indices
|
|
const int ik3 = iq3;
|
|
const int ik2 = iq2;
|
|
const int ik1 = ic;
|
|
|
|
// S indices
|
|
const int i1 = ik1;
|
|
|
|
ggml_vec_dot_f32(neq0,
|
|
S + i1,
|
|
(float *) ((char *) k->data + (ik1*nbk1 + ik2*nbk2 + ik3*nbk3)),
|
|
(float *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3)));
|
|
}
|
|
|
|
// scale
|
|
ggml_vec_scale_f32(nek1, S, scale);
|
|
|
|
if (masked) {
|
|
for (int i = P; i < M; i++) {
|
|
if (i > P + iq1) {
|
|
S[i] = -INFINITY;
|
|
}
|
|
}
|
|
}
|
|
|
|
// softmax
|
|
{
|
|
float max = -INFINITY;
|
|
for (int i = 0; i < M; i++) {
|
|
max = MAX(max, S[i]);
|
|
}
|
|
|
|
ggml_float sum = 0.0;
|
|
|
|
uint16_t ss;
|
|
for (int i = 0; i < M; i++) {
|
|
if (S[i] == -INFINITY) {
|
|
S[i] = 0.0;
|
|
} else {
|
|
//const float val = (S[i] == -INFINITY) ? 0.0 : exp(S[i] - max);
|
|
ggml_fp16_t s = ggml_fp32_to_fp16(S[i] - max);
|
|
memcpy(&ss, &s, sizeof(ss));
|
|
const float val = ggml_fp16_to_fp32(table_exp_f16[ss]);
|
|
sum += val;
|
|
S[i] = val;
|
|
}
|
|
}
|
|
|
|
assert(sum > 0.0f);
|
|
|
|
sum = 1.0/sum;
|
|
ggml_vec_scale_f32(M, S, sum);
|
|
}
|
|
|
|
for (int ic = 0; ic < nev1; ++ic) {
|
|
// dst indices
|
|
const int i1 = iq1;
|
|
const int i2 = iq2;
|
|
const int i3 = iq3;
|
|
|
|
ggml_vec_dot_f32(nek1,
|
|
(float *) ((char *) dst->data + (ic*nb0 + i1*nb1 + i2*nb2 + i3*nb3)),
|
|
(float *) ((char *) v->data + ( ic*nbv1 + i2*nbv2 + i3*nbv3)),
|
|
S);
|
|
}
|
|
}
|
|
}
|
|
|
|
void ggml_compute_forward_flash_attn_f16(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * q,
|
|
const struct ggml_tensor * k,
|
|
const struct ggml_tensor * v,
|
|
const bool masked,
|
|
struct ggml_tensor * dst) {
|
|
int64_t t0 = ggml_perf_time_us();
|
|
UNUSED(t0);
|
|
|
|
const int neq0 = q->ne[0];
|
|
const int neq1 = q->ne[1];
|
|
const int neq2 = q->ne[2];
|
|
const int neq3 = q->ne[3];
|
|
|
|
const int nek0 = k->ne[0];
|
|
const int nek1 = k->ne[1];
|
|
//const int nek2 = k->ne[2];
|
|
//const int nek3 = k->ne[3];
|
|
|
|
//const int nev0 = v->ne[0];
|
|
const int nev1 = v->ne[1];
|
|
//const int nev2 = v->ne[2];
|
|
//const int nev3 = v->ne[3];
|
|
|
|
const int ne0 = dst->ne[0];
|
|
const int ne1 = dst->ne[1];
|
|
//const int ne2 = dst->ne[2];
|
|
//const int ne3 = dst->ne[3];
|
|
|
|
const int nbk0 = k->nb[0];
|
|
const int nbk1 = k->nb[1];
|
|
const int nbk2 = k->nb[2];
|
|
const int nbk3 = k->nb[3];
|
|
|
|
const int nbq0 = q->nb[0];
|
|
const int nbq1 = q->nb[1];
|
|
const int nbq2 = q->nb[2];
|
|
const int nbq3 = q->nb[3];
|
|
|
|
const int nbv0 = v->nb[0];
|
|
const int nbv1 = v->nb[1];
|
|
const int nbv2 = v->nb[2];
|
|
const int nbv3 = v->nb[3];
|
|
|
|
const int nb0 = dst->nb[0];
|
|
const int nb1 = dst->nb[1];
|
|
const int nb2 = dst->nb[2];
|
|
const int nb3 = dst->nb[3];
|
|
|
|
const int ith = params->ith;
|
|
const int nth = params->nth;
|
|
|
|
const int D = neq0;
|
|
const int N = neq1;
|
|
const int P = nek1 - N;
|
|
const int M = P + N;
|
|
|
|
GGML_ASSERT(ne0 == D);
|
|
GGML_ASSERT(ne1 == N);
|
|
GGML_ASSERT(P >= 0);
|
|
|
|
GGML_ASSERT(nbq0 == sizeof(ggml_fp16_t));
|
|
GGML_ASSERT(nbk0 == sizeof(ggml_fp16_t));
|
|
GGML_ASSERT(nbv0 == sizeof(ggml_fp16_t));
|
|
|
|
GGML_ASSERT(neq0 == D);
|
|
GGML_ASSERT(nek0 == D);
|
|
GGML_ASSERT(nev1 == D);
|
|
|
|
GGML_ASSERT(neq1 == N);
|
|
GGML_ASSERT(nek1 == N + P);
|
|
GGML_ASSERT(nev1 == D);
|
|
|
|
// dst cannot be transposed or permuted
|
|
GGML_ASSERT(nb0 == sizeof(float));
|
|
GGML_ASSERT(nb0 <= nb1);
|
|
GGML_ASSERT(nb1 <= nb2);
|
|
GGML_ASSERT(nb2 <= nb3);
|
|
|
|
if (params->type == GGML_TASK_INIT) {
|
|
return;
|
|
}
|
|
|
|
if (params->type == GGML_TASK_FINALIZE) {
|
|
return;
|
|
}
|
|
|
|
// parallelize by q rows using ggml_vec_dot_f32
|
|
|
|
// total rows in q
|
|
const int nr = neq1*neq2*neq3;
|
|
|
|
// rows per thread
|
|
const int dr = (nr + nth - 1)/nth;
|
|
|
|
// row range for this thread
|
|
const int ir0 = dr*ith;
|
|
const int ir1 = MIN(ir0 + dr, nr);
|
|
|
|
const float scale = 1.0/sqrt((double) D);
|
|
|
|
//printf("P=%d N=%d D=%d ir0=%d ir1=%d scale = %f\n", P, N, D, ir0, ir1, scale);
|
|
|
|
for (int ir = ir0; ir < ir1; ++ir) {
|
|
// q indices
|
|
const int iq3 = ir/(neq2*neq1);
|
|
const int iq2 = (ir - iq3*neq2*neq1)/neq1;
|
|
const int iq1 = (ir - iq3*neq2*neq1 - iq2*neq1);
|
|
|
|
float * S = (float *) params->wdata + ith*(2*M + CACHE_LINE_SIZE_F32);
|
|
|
|
for (int ic = 0; ic < nek1; ++ic) {
|
|
// k indices
|
|
const int ik3 = iq3;
|
|
const int ik2 = iq2;
|
|
const int ik1 = ic;
|
|
|
|
// S indices
|
|
const int i1 = ik1;
|
|
|
|
ggml_vec_dot_f16(neq0,
|
|
S + i1,
|
|
(ggml_fp16_t *) ((char *) k->data + (ik1*nbk1 + ik2*nbk2 + ik3*nbk3)),
|
|
(ggml_fp16_t *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3)));
|
|
}
|
|
|
|
// scale
|
|
ggml_vec_scale_f32(nek1, S, scale);
|
|
|
|
if (masked) {
|
|
for (int i = P; i < M; i++) {
|
|
if (i > P + iq1) {
|
|
S[i] = -INFINITY;
|
|
}
|
|
}
|
|
}
|
|
|
|
// softmax
|
|
{
|
|
float max = -INFINITY;
|
|
for (int i = 0; i < M; i++) {
|
|
max = MAX(max, S[i]);
|
|
}
|
|
|
|
ggml_float sum = 0.0;
|
|
|
|
uint16_t ss;
|
|
for (int i = 0; i < M; i++) {
|
|
if (S[i] == -INFINITY) {
|
|
S[i] = 0.0;
|
|
} else {
|
|
//const float val = (S[i] == -INFINITY) ? 0.0 : exp(S[i] - max);
|
|
ggml_fp16_t s = ggml_fp32_to_fp16(S[i] - max);
|
|
memcpy(&ss, &s, sizeof(ss));
|
|
const float val = ggml_fp16_to_fp32(table_exp_f16[ss]);
|
|
sum += val;
|
|
S[i] = val;
|
|
}
|
|
}
|
|
|
|
assert(sum > 0.0f);
|
|
|
|
sum = 1.0/sum;
|
|
ggml_vec_scale_f32(M, S, sum);
|
|
}
|
|
|
|
ggml_fp16_t * S16 = (ggml_fp16_t *) ((float *) params->wdata + ith*(2*M + CACHE_LINE_SIZE_F32) + M);
|
|
|
|
for (int i = 0; i < M; i++) {
|
|
S16[i] = ggml_fp32_to_fp16(S[i]);
|
|
}
|
|
|
|
for (int ic = 0; ic < nev1; ++ic) {
|
|
// dst indices
|
|
const int i1 = iq1;
|
|
const int i2 = iq2;
|
|
const int i3 = iq3;
|
|
|
|
ggml_vec_dot_f16(nek1,
|
|
(float *) ((char *) dst->data + (ic*nb0 + i1*nb1 + i2*nb2 + i3*nb3)),
|
|
(ggml_fp16_t *) ((char *) v->data + ( ic*nbv1 + i2*nbv2 + i3*nbv3)),
|
|
S16);
|
|
}
|
|
}
|
|
}
|
|
|
|
void ggml_compute_forward_flash_attn(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * q,
|
|
const struct ggml_tensor * k,
|
|
const struct ggml_tensor * v,
|
|
const bool masked,
|
|
struct ggml_tensor * dst) {
|
|
switch (q->type) {
|
|
case GGML_TYPE_F16:
|
|
{
|
|
ggml_compute_forward_flash_attn_f16(params, q, k, v, masked, dst);
|
|
} break;
|
|
case GGML_TYPE_F32:
|
|
{
|
|
ggml_compute_forward_flash_attn_f32(params, q, k, v, masked, dst);
|
|
} break;
|
|
case GGML_TYPE_I8:
|
|
case GGML_TYPE_I16:
|
|
case GGML_TYPE_I32:
|
|
case GGML_TYPE_COUNT:
|
|
{
|
|
assert(false);
|
|
} break;
|
|
}
|
|
}
|
|
|
|
// ggml_compute_forward_flash_ff
|
|
|
|
void ggml_compute_forward_flash_ff_f16(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * a, // F16
|
|
const struct ggml_tensor * b0, // F16 fc_w
|
|
const struct ggml_tensor * b1, // F32 fc_b
|
|
const struct ggml_tensor * c0, // F16 proj_w
|
|
const struct ggml_tensor * c1, // F32 proj_b
|
|
struct ggml_tensor * dst) {
|
|
int64_t t0 = ggml_perf_time_us();
|
|
UNUSED(t0);
|
|
|
|
const int nea0 = a->ne[0];
|
|
const int nea1 = a->ne[1];
|
|
const int nea2 = a->ne[2];
|
|
const int nea3 = a->ne[3];
|
|
|
|
const int neb00 = b0->ne[0];
|
|
const int neb01 = b0->ne[1];
|
|
//const int neb02 = b0->ne[2];
|
|
//const int neb03 = b0->ne[3];
|
|
|
|
const int neb10 = b1->ne[0];
|
|
const int neb11 = b1->ne[1];
|
|
//const int neb12 = b1->ne[2];
|
|
//const int neb13 = b1->ne[3];
|
|
|
|
const int nec00 = c0->ne[0];
|
|
const int nec01 = c0->ne[1];
|
|
//const int nec02 = c0->ne[2];
|
|
//const int nec03 = c0->ne[3];
|
|
|
|
const int nec10 = c1->ne[0];
|
|
const int nec11 = c1->ne[1];
|
|
//const int nec12 = c1->ne[2];
|
|
//const int nec13 = c1->ne[3];
|
|
|
|
const int ne0 = dst->ne[0];
|
|
const int ne1 = dst->ne[1];
|
|
const int ne2 = dst->ne[2];
|
|
//const int ne3 = dst->ne[3];
|
|
|
|
const int nba0 = a->nb[0];
|
|
const int nba1 = a->nb[1];
|
|
const int nba2 = a->nb[2];
|
|
const int nba3 = a->nb[3];
|
|
|
|
const int nbb00 = b0->nb[0];
|
|
const int nbb01 = b0->nb[1];
|
|
const int nbb02 = b0->nb[2];
|
|
const int nbb03 = b0->nb[3];
|
|
|
|
const int nbb10 = b1->nb[0];
|
|
//const int nbb11 = b1->nb[1];
|
|
//const int nbb12 = b1->nb[2];
|
|
//const int nbb13 = b1->nb[3];
|
|
|
|
const int nbc00 = c0->nb[0];
|
|
const int nbc01 = c0->nb[1];
|
|
const int nbc02 = c0->nb[2];
|
|
const int nbc03 = c0->nb[3];
|
|
|
|
const int nbc10 = c1->nb[0];
|
|
//const int nbc11 = c1->nb[1];
|
|
//const int nbc12 = c1->nb[2];
|
|
//const int nbc13 = c1->nb[3];
|
|
|
|
const int nb0 = dst->nb[0];
|
|
const int nb1 = dst->nb[1];
|
|
const int nb2 = dst->nb[2];
|
|
const int nb3 = dst->nb[3];
|
|
|
|
const int ith = params->ith;
|
|
const int nth = params->nth;
|
|
|
|
const int D = nea0;
|
|
//const int N = nea1;
|
|
const int M = neb01;
|
|
|
|
GGML_ASSERT(ne0 == nea0);
|
|
GGML_ASSERT(ne1 == nea1);
|
|
GGML_ASSERT(ne2 == nea2);
|
|
|
|
GGML_ASSERT(nba0 == sizeof(ggml_fp16_t));
|
|
GGML_ASSERT(nbb00 == sizeof(ggml_fp16_t));
|
|
GGML_ASSERT(nbb10 == sizeof(float));
|
|
GGML_ASSERT(nbc00 == sizeof(ggml_fp16_t));
|
|
GGML_ASSERT(nbc10 == sizeof(float));
|
|
|
|
GGML_ASSERT(neb00 == D);
|
|
GGML_ASSERT(neb01 == M);
|
|
GGML_ASSERT(neb10 == M);
|
|
GGML_ASSERT(neb11 == 1);
|
|
|
|
GGML_ASSERT(nec00 == M);
|
|
GGML_ASSERT(nec01 == D);
|
|
GGML_ASSERT(nec10 == D);
|
|
GGML_ASSERT(nec11 == 1);
|
|
|
|
// dst cannot be transposed or permuted
|
|
GGML_ASSERT(nb0 == sizeof(float));
|
|
GGML_ASSERT(nb0 <= nb1);
|
|
GGML_ASSERT(nb1 <= nb2);
|
|
GGML_ASSERT(nb2 <= nb3);
|
|
|
|
if (params->type == GGML_TASK_INIT) {
|
|
return;
|
|
}
|
|
|
|
if (params->type == GGML_TASK_FINALIZE) {
|
|
return;
|
|
}
|
|
|
|
// parallelize by a rows using ggml_vec_dot_f32
|
|
|
|
// total rows in a
|
|
const int nr = nea1*nea2*nea3;
|
|
|
|
// rows per thread
|
|
const int dr = (nr + nth - 1)/nth;
|
|
|
|
// row range for this thread
|
|
const int ir0 = dr*ith;
|
|
const int ir1 = MIN(ir0 + dr, nr);
|
|
|
|
for (int ir = ir0; ir < ir1; ++ir) {
|
|
// a indices
|
|
const int ia3 = ir/(nea2*nea1);
|
|
const int ia2 = (ir - ia3*nea2*nea1)/nea1;
|
|
const int ia1 = (ir - ia3*nea2*nea1 - ia2*nea1);
|
|
|
|
float * S = (float *) params->wdata + ith*(2*M + CACHE_LINE_SIZE_F32);
|
|
|
|
for (int ic = 0; ic < neb01; ++ic) {
|
|
// b0 indices
|
|
const int ib03 = ia3;
|
|
const int ib02 = ia2;
|
|
const int ib01 = ic;
|
|
|
|
// S indices
|
|
const int i1 = ib01;
|
|
|
|
ggml_vec_dot_f16(nea0,
|
|
S + i1,
|
|
(ggml_fp16_t *) ((char *) b0->data + (ib01*nbb01 + ib02*nbb02 + ib03*nbb03)),
|
|
(ggml_fp16_t *) ((char *) a->data + ( ia1*nba1 + ia2*nba2 + ia3*nba3)));
|
|
}
|
|
|
|
ggml_vec_add_f32(neb01, S, S, (float *) b1->data);
|
|
//ggml_vec_gelu_f32(neb01, S, S);
|
|
|
|
ggml_fp16_t * S16 = (ggml_fp16_t *) ((float *) params->wdata + ith*(2*M + CACHE_LINE_SIZE_F32) + M);
|
|
|
|
for (int i = 0; i < M; i++) {
|
|
S16[i] = ggml_fp32_to_fp16(S[i]);
|
|
}
|
|
|
|
ggml_vec_gelu_f16(neb01, S16, S16);
|
|
|
|
{
|
|
// dst indices
|
|
const int i1 = ia1;
|
|
const int i2 = ia2;
|
|
const int i3 = ia3;
|
|
|
|
for (int ic = 0; ic < nec01; ++ic) {
|
|
|
|
ggml_vec_dot_f16(neb01,
|
|
(float *) ((char *) dst->data + (ic*nb0 + i1*nb1 + i2*nb2 + i3*nb3)),
|
|
(ggml_fp16_t *) ((char *) c0->data + ( ic*nbc01 + i2*nbc02 + i3*nbc03)),
|
|
S16);
|
|
}
|
|
|
|
ggml_vec_add_f32(nec01,
|
|
(float *) ((char *) dst->data + (i1*nb1 + i2*nb2 + i3*nb3)),
|
|
(float *) ((char *) dst->data + (i1*nb1 + i2*nb2 + i3*nb3)),
|
|
(float *) c1->data);
|
|
}
|
|
}
|
|
}
|
|
|
|
void ggml_compute_forward_flash_ff(
|
|
const struct ggml_compute_params * params,
|
|
const struct ggml_tensor * a,
|
|
const struct ggml_tensor * b0,
|
|
const struct ggml_tensor * b1,
|
|
const struct ggml_tensor * c0,
|
|
const struct ggml_tensor * c1,
|
|
struct ggml_tensor * dst) {
|
|
switch (b0->type) {
|
|
case GGML_TYPE_F16:
|
|
{
|
|
ggml_compute_forward_flash_ff_f16(params, a, b0, b1, c0, c1, dst);
|
|
} break;
|
|
case GGML_TYPE_F32:
|
|
{
|
|
GGML_ASSERT(false); // TODO
|
|
} break;
|
|
case GGML_TYPE_I8:
|
|
case GGML_TYPE_I16:
|
|
case GGML_TYPE_I32:
|
|
case GGML_TYPE_COUNT:
|
|
{
|
|
assert(false);
|
|
} break;
|
|
}
|
|
}
|
|
|
|
/////////////////////////////////
|
|
|
|
void ggml_compute_forward(struct ggml_compute_params * params, struct ggml_tensor * tensor) {
|
|
assert(params);
|
|
|
|
switch (tensor->op) {
|
|
case GGML_OP_DUP:
|
|
{
|
|
ggml_compute_forward_dup(params, tensor->src0, tensor);
|
|
} break;
|
|
case GGML_OP_ADD:
|
|
{
|
|
ggml_compute_forward_add(params, tensor->src0, tensor->src1, tensor);
|
|
} break;
|
|
case GGML_OP_SUB:
|
|
{
|
|
ggml_compute_forward_sub(params, tensor->src0, tensor->src1, tensor);
|
|
} break;
|
|
case GGML_OP_MUL:
|
|
{
|
|
ggml_compute_forward_mul(params, tensor->src0, tensor->src1, tensor);
|
|
} break;
|
|
case GGML_OP_DIV:
|
|
{
|
|
ggml_compute_forward_div(params, tensor->src0, tensor->src1, tensor);
|
|
} break;
|
|
case GGML_OP_SQR:
|
|
{
|
|
ggml_compute_forward_sqr(params, tensor->src0, tensor);
|
|
} break;
|
|
case GGML_OP_SQRT:
|
|
{
|
|
ggml_compute_forward_sqrt(params, tensor->src0, tensor);
|
|
} break;
|
|
case GGML_OP_SUM:
|
|
{
|
|
ggml_compute_forward_sum(params, tensor->src0, tensor);
|
|
} break;
|
|
case GGML_OP_MEAN:
|
|
{
|
|
ggml_compute_forward_mean(params, tensor->src0, tensor);
|
|
} break;
|
|
case GGML_OP_REPEAT:
|
|
{
|
|
ggml_compute_forward_repeat(params, tensor->src0, tensor);
|
|
} break;
|
|
case GGML_OP_ABS:
|
|
{
|
|
ggml_compute_forward_abs(params, tensor->src0, tensor);
|
|
} break;
|
|
case GGML_OP_SGN:
|
|
{
|
|
ggml_compute_forward_sgn(params, tensor->src0, tensor);
|
|
} break;
|
|
case GGML_OP_NEG:
|
|
{
|
|
ggml_compute_forward_neg(params, tensor->src0, tensor);
|
|
} break;
|
|
case GGML_OP_STEP:
|
|
{
|
|
ggml_compute_forward_step(params, tensor->src0, tensor);
|
|
} break;
|
|
case GGML_OP_RELU:
|
|
{
|
|
ggml_compute_forward_relu(params, tensor->src0, tensor);
|
|
} break;
|
|
case GGML_OP_GELU:
|
|
{
|
|
ggml_compute_forward_gelu(params, tensor->src0, tensor);
|
|
} break;
|
|
case GGML_OP_NORM:
|
|
{
|
|
ggml_compute_forward_norm(params, tensor->src0, tensor);
|
|
} break;
|
|
case GGML_OP_MUL_MAT:
|
|
{
|
|
ggml_compute_forward_mul_mat(params, tensor->src0, tensor->src1, tensor);
|
|
} break;
|
|
case GGML_OP_SCALE:
|
|
{
|
|
ggml_compute_forward_scale(params, tensor->src0, tensor->src1, tensor);
|
|
} break;
|
|
case GGML_OP_CPY:
|
|
{
|
|
ggml_compute_forward_cpy(params, tensor->src0, tensor);
|
|
} break;
|
|
case GGML_OP_RESHAPE:
|
|
{
|
|
ggml_compute_forward_reshape(params, tensor->src0, tensor);
|
|
} break;
|
|
case GGML_OP_VIEW:
|
|
{
|
|
ggml_compute_forward_view(params, tensor->src0);
|
|
} break;
|
|
case GGML_OP_PERMUTE:
|
|
{
|
|
ggml_compute_forward_permute(params, tensor->src0);
|
|
} break;
|
|
case GGML_OP_TRANSPOSE:
|
|
{
|
|
ggml_compute_forward_transpose(params, tensor->src0);
|
|
} break;
|
|
case GGML_OP_GET_ROWS:
|
|
{
|
|
ggml_compute_forward_get_rows(params, tensor->src0, tensor->src1, tensor);
|
|
} break;
|
|
case GGML_OP_DIAG_MASK_INF:
|
|
{
|
|
ggml_compute_forward_diag_mask_inf(params, tensor->src0, tensor->src1, tensor);
|
|
} break;
|
|
case GGML_OP_SOFT_MAX:
|
|
{
|
|
ggml_compute_forward_soft_max(params, tensor->src0, tensor);
|
|
} break;
|
|
case GGML_OP_ROPE:
|
|
{
|
|
ggml_compute_forward_rope(params, tensor->src0, tensor->src1, tensor);
|
|
} break;
|
|
case GGML_OP_CONV_1D_1S:
|
|
{
|
|
ggml_compute_forward_conv_1d_1s(params, tensor->src0, tensor->src1, tensor);
|
|
} break;
|
|
case GGML_OP_CONV_1D_2S:
|
|
{
|
|
ggml_compute_forward_conv_1d_2s(params, tensor->src0, tensor->src1, tensor);
|
|
} break;
|
|
case GGML_OP_FLASH_ATTN:
|
|
{
|
|
int32_t t = ggml_get_i32_1d(tensor->opt[1], 0);
|
|
GGML_ASSERT(t == 0 || t == 1);
|
|
bool masked = t != 0;
|
|
ggml_compute_forward_flash_attn(params, tensor->src0, tensor->src1, tensor->opt[0], masked, tensor);
|
|
} break;
|
|
case GGML_OP_FLASH_FF:
|
|
{
|
|
ggml_compute_forward_flash_ff(params, tensor->src0, tensor->src1, tensor->opt[0], tensor->opt[1], tensor->opt[2], tensor);
|
|
} break;
|
|
case GGML_OP_NONE:
|
|
{
|
|
// nop
|
|
} break;
|
|
case GGML_OP_COUNT:
|
|
{
|
|
GGML_ASSERT(false);
|
|
} break;
|
|
};
|
|
}
|
|
|
|
////////////////////////////////////////////////////////////////////////////////
|
|
|
|
void ggml_compute_backward(struct ggml_context * ctx, struct ggml_tensor * tensor, bool inplace) {
|
|
struct ggml_tensor * src0 = tensor->src0;
|
|
struct ggml_tensor * src1 = tensor->src1;
|
|
|
|
switch (tensor->op) {
|
|
case GGML_OP_DUP:
|
|
{
|
|
if (src0->grad) {
|
|
src0->grad = ggml_add_impl(ctx, src0->grad, tensor->grad, inplace);
|
|
}
|
|
} break;
|
|
case GGML_OP_ADD:
|
|
{
|
|
if (src0->grad) {
|
|
src0->grad = ggml_add_impl(ctx, src0->grad, tensor->grad, inplace);
|
|
}
|
|
if (src1->grad) {
|
|
src1->grad = ggml_add_impl(ctx, src1->grad, tensor->grad, inplace);
|
|
}
|
|
} break;
|
|
case GGML_OP_SUB:
|
|
{
|
|
if (src0->grad) {
|
|
src0->grad = ggml_add_impl(ctx, src0->grad, tensor->grad, inplace);
|
|
}
|
|
if (src1->grad) {
|
|
src1->grad = ggml_sub_impl(ctx, src1->grad, tensor->grad, inplace);
|
|
}
|
|
} break;
|
|
case GGML_OP_MUL:
|
|
{
|
|
if (src0->grad) {
|
|
src0->grad =
|
|
ggml_add_impl(ctx,
|
|
src0->grad,
|
|
ggml_mul(ctx, src1, tensor->grad),
|
|
inplace);
|
|
}
|
|
if (src1->grad) {
|
|
src1->grad =
|
|
ggml_add_impl(ctx,
|
|
src1->grad,
|
|
ggml_mul(ctx, src0, tensor->grad),
|
|
inplace);
|
|
}
|
|
} break;
|
|
case GGML_OP_DIV:
|
|
{
|
|
if (src0->grad) {
|
|
src0->grad =
|
|
ggml_add_impl(ctx,
|
|
src0->grad,
|
|
ggml_div(ctx, tensor->grad, src1),
|
|
inplace);
|
|
}
|
|
if (src1->grad) {
|
|
src1->grad =
|
|
ggml_sub_impl(ctx,
|
|
src1->grad,
|
|
ggml_mul(ctx,
|
|
tensor->grad,
|
|
ggml_div(ctx, tensor, src1)),
|
|
inplace);
|
|
}
|
|
} break;
|
|
case GGML_OP_SQR:
|
|
{
|
|
if (src0->grad) {
|
|
src0->grad =
|
|
ggml_add_impl(ctx,
|
|
src0->grad,
|
|
ggml_mul(ctx,
|
|
ggml_mul(ctx, src0, tensor->grad),
|
|
ggml_repeat(ctx, ggml_new_f32(ctx, 2.0f), src0)),
|
|
inplace);
|
|
}
|
|
} break;
|
|
case GGML_OP_SQRT:
|
|
{
|
|
if (src0->grad) {
|
|
src0->grad =
|
|
ggml_add_impl(ctx,
|
|
src0->grad,
|
|
ggml_div(ctx,
|
|
ggml_repeat(ctx, ggml_new_f32(ctx, 0.5f), tensor),
|
|
tensor),
|
|
inplace);
|
|
}
|
|
} break;
|
|
case GGML_OP_SUM:
|
|
{
|
|
if (src0->grad) {
|
|
src0->grad =
|
|
ggml_add_impl(ctx,
|
|
src0->grad,
|
|
ggml_repeat(ctx, tensor->grad, src0->grad),
|
|
inplace);
|
|
}
|
|
} break;
|
|
case GGML_OP_MEAN:
|
|
{
|
|
assert(false); // TODO: implement
|
|
} break;
|
|
case GGML_OP_REPEAT:
|
|
{
|
|
if (src0->grad) {
|
|
src0->grad =
|
|
ggml_add_impl(ctx,
|
|
src0->grad,
|
|
ggml_sum(ctx, tensor->grad),
|
|
inplace);
|
|
}
|
|
} break;
|
|
case GGML_OP_ABS:
|
|
{
|
|
if (src0->grad) {
|
|
src0->grad =
|
|
ggml_add_impl(ctx,
|
|
src0->grad,
|
|
ggml_mul(ctx,
|
|
ggml_sgn(ctx, src0),
|
|
tensor->grad),
|
|
inplace);
|
|
}
|
|
} break;
|
|
case GGML_OP_SGN:
|
|
{
|
|
if (src0->grad) {
|
|
// noop
|
|
}
|
|
} break;
|
|
case GGML_OP_NEG:
|
|
{
|
|
if (src0->grad) {
|
|
src0->grad = ggml_sub_impl(ctx, src0->grad, tensor->grad, inplace);
|
|
}
|
|
} break;
|
|
case GGML_OP_STEP:
|
|
{
|
|
if (src0->grad) {
|
|
// noop
|
|
}
|
|
} break;
|
|
case GGML_OP_RELU:
|
|
{
|
|
if (src0->grad) {
|
|
src0->grad = ggml_sub_impl(ctx,
|
|
src0->grad,
|
|
ggml_mul(ctx,
|
|
ggml_step(ctx, src0),
|
|
tensor->grad),
|
|
inplace);
|
|
}
|
|
} break;
|
|
case GGML_OP_GELU:
|
|
{
|
|
assert(false); // TODO: not implemented
|
|
} break;
|
|
case GGML_OP_NORM:
|
|
{
|
|
assert(false); // TODO: not implemented
|
|
} break;
|
|
case GGML_OP_MUL_MAT:
|
|
{
|
|
if (src0->grad) {
|
|
// TODO: this requires outer product - ggml_out_prod(ctx, src1, tensor->grad);
|
|
assert(false);
|
|
}
|
|
if (src1->grad) {
|
|
src1->grad =
|
|
ggml_add_impl(ctx,
|
|
src1->grad,
|
|
// TODO: fix transpose, the node will break the graph connections
|
|
ggml_mul_mat(ctx, ggml_transpose(ctx, src0), tensor->grad),
|
|
inplace);
|
|
}
|
|
} break;
|
|
case GGML_OP_SCALE:
|
|
{
|
|
GGML_ASSERT(false); // TODO: not implemented
|
|
} break;
|
|
case GGML_OP_CPY:
|
|
{
|
|
GGML_ASSERT(false); // TODO: not implemented
|
|
} break;
|
|
case GGML_OP_RESHAPE:
|
|
{
|
|
GGML_ASSERT(false); // TODO: not implemented
|
|
} break;
|
|
case GGML_OP_VIEW:
|
|
{
|
|
GGML_ASSERT(false); // not supported
|
|
} break;
|
|
case GGML_OP_PERMUTE:
|
|
{
|
|
GGML_ASSERT(false); // TODO: not implemented
|
|
} break;
|
|
case GGML_OP_TRANSPOSE:
|
|
{
|
|
GGML_ASSERT(false); // TODO: not implemented
|
|
} break;
|
|
case GGML_OP_GET_ROWS:
|
|
{
|
|
GGML_ASSERT(false); // TODO: not implemented
|
|
} break;
|
|
case GGML_OP_DIAG_MASK_INF:
|
|
{
|
|
GGML_ASSERT(false); // TODO: not implemented
|
|
} break;
|
|
case GGML_OP_SOFT_MAX:
|
|
{
|
|
GGML_ASSERT(false); // TODO: not implemented
|
|
} break;
|
|
case GGML_OP_ROPE:
|
|
{
|
|
GGML_ASSERT(false); // TODO: not implemented
|
|
} break;
|
|
case GGML_OP_CONV_1D_1S:
|
|
{
|
|
GGML_ASSERT(false); // TODO: not implemented
|
|
} break;
|
|
case GGML_OP_CONV_1D_2S:
|
|
{
|
|
GGML_ASSERT(false); // TODO: not implemented
|
|
} break;
|
|
case GGML_OP_FLASH_ATTN:
|
|
{
|
|
GGML_ASSERT(false); // not supported
|
|
} break;
|
|
case GGML_OP_FLASH_FF:
|
|
{
|
|
GGML_ASSERT(false); // not supported
|
|
} break;
|
|
case GGML_OP_NONE:
|
|
{
|
|
// nop
|
|
} break;
|
|
case GGML_OP_COUNT:
|
|
{
|
|
GGML_ASSERT(false);
|
|
} break;
|
|
};
|
|
}
|
|
|
|
void ggml_visit_parents(struct ggml_cgraph * cgraph, struct ggml_tensor * node) {
|
|
if (node->grad == NULL) {
|
|
// this usually happens when we generate intermediate nodes from constants in the backward pass
|
|
// it can also happen during forward pass, if the user performs computations with constants
|
|
if (node->op != GGML_OP_NONE) {
|
|
//GGML_PRINT_DEBUG("%s: warning: node %p has no grad, but op %d\n", __func__, (void *) node, node->op);
|
|
}
|
|
}
|
|
|
|
// check if already visited
|
|
for (int i = 0; i < cgraph->n_nodes; i++) {
|
|
if (cgraph->nodes[i] == node) {
|
|
return;
|
|
}
|
|
}
|
|
|
|
for (int i = 0; i < cgraph->n_leafs; i++) {
|
|
if (cgraph->leafs[i] == node) {
|
|
return;
|
|
}
|
|
}
|
|
|
|
if (node->src0) {
|
|
ggml_visit_parents(cgraph, node->src0);
|
|
}
|
|
|
|
if (node->src1) {
|
|
ggml_visit_parents(cgraph, node->src1);
|
|
}
|
|
|
|
for (int i = 0; i < GGML_MAX_OPT; ++i) {
|
|
if (node->opt[i]) {
|
|
ggml_visit_parents(cgraph, node->opt[i]);
|
|
}
|
|
}
|
|
|
|
if (node->op == GGML_OP_NONE && node->grad == NULL) {
|
|
// reached a leaf node, not part of the gradient graph (e.g. a constant)
|
|
assert(cgraph->n_leafs < GGML_MAX_NODES);
|
|
|
|
cgraph->leafs[cgraph->n_leafs] = node;
|
|
cgraph->n_leafs++;
|
|
} else {
|
|
assert(cgraph->n_nodes < GGML_MAX_NODES);
|
|
|
|
cgraph->nodes[cgraph->n_nodes] = node;
|
|
cgraph->grads[cgraph->n_nodes] = node->grad;
|
|
cgraph->n_nodes++;
|
|
}
|
|
}
|
|
|
|
void ggml_build_forward_impl(struct ggml_cgraph * cgraph, struct ggml_tensor * tensor, bool expand) {
|
|
if (!expand) {
|
|
cgraph->n_nodes = 0;
|
|
cgraph->n_leafs = 0;
|
|
}
|
|
|
|
const int n0 = cgraph->n_nodes;
|
|
UNUSED(n0);
|
|
|
|
ggml_visit_parents(cgraph, tensor);
|
|
|
|
const int n_new = cgraph->n_nodes - n0;
|
|
GGML_PRINT_DEBUG("%s: visited %d new nodes\n", __func__, n_new);
|
|
|
|
if (n_new > 0) {
|
|
// the last added node should always be starting point
|
|
assert(cgraph->nodes[cgraph->n_nodes - 1] == tensor);
|
|
}
|
|
}
|
|
|
|
void ggml_build_forward_expand(struct ggml_cgraph * cgraph, struct ggml_tensor * tensor) {
|
|
ggml_build_forward_impl(cgraph, tensor, true);
|
|
}
|
|
|
|
struct ggml_cgraph ggml_build_forward(struct ggml_tensor * tensor) {
|
|
struct ggml_cgraph result = {
|
|
/*.n_nodes =*/ 0,
|
|
/*.n_leafs =*/ 0,
|
|
/*.n_threads =*/ 0,
|
|
/*.work_size =*/ 0,
|
|
/*.work =*/ NULL,
|
|
/*.nodes =*/ { NULL },
|
|
/*.grads =*/ { NULL },
|
|
/*.leafs =*/ { NULL },
|
|
/*.perf_runs =*/ 0,
|
|
/*.perf_cycles =*/ 0,
|
|
/*.perf_time_us =*/ 0,
|
|
};
|
|
|
|
ggml_build_forward_impl(&result, tensor, false);
|
|
|
|
return result;
|
|
}
|
|
|
|
struct ggml_cgraph ggml_build_backward(struct ggml_context * ctx, struct ggml_cgraph * gf, bool keep) {
|
|
struct ggml_cgraph result = *gf;
|
|
|
|
assert(gf->n_nodes > 0);
|
|
|
|
// if we are keeping the gradient graph, we have to detach the gradient nodes from the original graph
|
|
if (keep) {
|
|
for (int i = 0; i < gf->n_nodes; i++) {
|
|
struct ggml_tensor * node = gf->nodes[i];
|
|
|
|
if (node->grad) {
|
|
node->grad = ggml_dup_tensor(ctx, node);
|
|
gf->grads[i] = node->grad;
|
|
}
|
|
}
|
|
}
|
|
|
|
for (int i = gf->n_nodes - 1; i >= 0; i--) {
|
|
struct ggml_tensor * node = gf->nodes[i];
|
|
|
|
// because we detached the grad nodes from the original graph, we can afford inplace operations
|
|
if (node->grad) {
|
|
ggml_compute_backward(ctx, node, keep);
|
|
}
|
|
}
|
|
|
|
for (int i = gf->n_nodes - 1; i >= 0; i--) {
|
|
struct ggml_tensor * node = gf->nodes[i];
|
|
|
|
if (node->is_param) {
|
|
GGML_PRINT_DEBUG("%s: found root node %p\n", __func__, (void *) node);
|
|
ggml_build_forward_impl(&result, node->grad, true);
|
|
}
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
//
|
|
// thread data
|
|
//
|
|
// synchronization is done via busy loops
|
|
// I tried using spin locks, but not sure how to use them correctly - the things I tried were slower than busy loops
|
|
//
|
|
|
|
#ifdef __APPLE__
|
|
|
|
//#include <os/lock.h>
|
|
|
|
//typedef os_unfair_lock ggml_lock_t;
|
|
//
|
|
//#define ggml_lock_init(x) UNUSED(x)
|
|
//#define ggml_lock_destroy(x) UNUSED(x)
|
|
//#define ggml_lock_lock os_unfair_lock_lock
|
|
//#define ggml_lock_unlock os_unfair_lock_unlock
|
|
//
|
|
//#define GGML_LOCK_INITIALIZER OS_UNFAIR_LOCK_INIT
|
|
|
|
typedef int ggml_lock_t;
|
|
|
|
#define ggml_lock_init(x) UNUSED(x)
|
|
#define ggml_lock_destroy(x) UNUSED(x)
|
|
#define ggml_lock_lock(x) UNUSED(x)
|
|
#define ggml_lock_unlock(x) UNUSED(x)
|
|
|
|
#define GGML_LOCK_INITIALIZER 0
|
|
|
|
#else
|
|
|
|
//typedef pthread_spinlock_t ggml_lock_t;
|
|
|
|
//#define ggml_lock_init(x) pthread_spin_init(x, PTHREAD_PROCESS_PRIVATE)
|
|
//#define ggml_lock_destroy pthread_spin_destroy
|
|
//#define ggml_lock_lock pthread_spin_lock
|
|
//#define ggml_lock_unlock pthread_spin_unlock
|
|
|
|
typedef int ggml_lock_t;
|
|
|
|
#define ggml_lock_init(x) UNUSED(x)
|
|
#define ggml_lock_destroy(x) UNUSED(x)
|
|
#define ggml_lock_lock(x) UNUSED(x)
|
|
#define ggml_lock_unlock(x) UNUSED(x)
|
|
|
|
#define GGML_LOCK_INITIALIZER 0
|
|
|
|
#endif
|
|
|
|
struct ggml_compute_state_shared {
|
|
ggml_lock_t spin;
|
|
|
|
int n_threads;
|
|
|
|
// synchronization primitives
|
|
atomic_int n_ready;
|
|
atomic_bool has_work;
|
|
atomic_bool stop; // stop all threads
|
|
};
|
|
|
|
struct ggml_compute_state {
|
|
pthread_t thrd;
|
|
|
|
struct ggml_compute_params params;
|
|
struct ggml_tensor * node;
|
|
|
|
struct ggml_compute_state_shared * shared;
|
|
};
|
|
|
|
// function used by each compute thread
|
|
void * ggml_graph_compute_one(void * data) {
|
|
struct ggml_compute_state * state = (struct ggml_compute_state *) data;
|
|
|
|
ggml_compute_forward(&state->params, state->node);
|
|
|
|
return NULL;
|
|
}
|
|
|
|
void * ggml_graph_compute_thread(void * data) {
|
|
struct ggml_compute_state * state = (struct ggml_compute_state *) data;
|
|
|
|
const int n_threads = state->shared->n_threads;
|
|
|
|
while (true) {
|
|
if (atomic_fetch_add(&state->shared->n_ready, 1) == n_threads - 1) {
|
|
atomic_store(&state->shared->has_work, false);
|
|
} else {
|
|
while (atomic_load(&state->shared->has_work)) {
|
|
if (atomic_load(&state->shared->stop)) {
|
|
return NULL;
|
|
}
|
|
ggml_lock_lock (&state->shared->spin);
|
|
ggml_lock_unlock(&state->shared->spin);
|
|
}
|
|
}
|
|
|
|
atomic_fetch_sub(&state->shared->n_ready, 1);
|
|
|
|
// wait for work
|
|
while (!atomic_load(&state->shared->has_work)) {
|
|
if (atomic_load(&state->shared->stop)) {
|
|
return NULL;
|
|
}
|
|
ggml_lock_lock (&state->shared->spin);
|
|
ggml_lock_unlock(&state->shared->spin);
|
|
}
|
|
|
|
// check if we should stop
|
|
if (atomic_load(&state->shared->stop)) {
|
|
break;
|
|
}
|
|
|
|
if (state->node) {
|
|
ggml_compute_forward(&state->params, state->node);
|
|
state->node = NULL;
|
|
} else {
|
|
break;
|
|
}
|
|
}
|
|
|
|
return NULL;
|
|
}
|
|
|
|
void ggml_graph_compute(struct ggml_context * ctx, struct ggml_cgraph * cgraph) {
|
|
if (cgraph->n_threads <= 0) {
|
|
cgraph->n_threads = 8;
|
|
}
|
|
|
|
const int n_threads = cgraph->n_threads;
|
|
|
|
struct ggml_compute_state_shared state_shared = {
|
|
/*.spin =*/ GGML_LOCK_INITIALIZER,
|
|
/*.n_threads =*/ n_threads,
|
|
/*.n_ready =*/ 0,
|
|
/*.has_work =*/ false,
|
|
/*.stop =*/ false,
|
|
};
|
|
struct ggml_compute_state * workers = n_threads > 1 ? alloca(sizeof(struct ggml_compute_state)*(n_threads - 1)) : NULL;
|
|
|
|
// create thread pool
|
|
if (n_threads > 1) {
|
|
ggml_lock_init(&state_shared.spin);
|
|
|
|
atomic_store(&state_shared.has_work, true);
|
|
|
|
for (int j = 0; j < n_threads - 1; j++) {
|
|
workers[j] = (struct ggml_compute_state) {
|
|
.thrd = 0,
|
|
.params = {
|
|
.type = GGML_TASK_COMPUTE,
|
|
.ith = j + 1,
|
|
.nth = n_threads,
|
|
.wsize = cgraph->work ? ggml_nbytes(cgraph->work) : 0,
|
|
.wdata = cgraph->work ? cgraph->work->data : NULL,
|
|
},
|
|
.node = NULL,
|
|
.shared = &state_shared,
|
|
};
|
|
int rc = pthread_create(&workers[j].thrd, NULL, ggml_graph_compute_thread, &workers[j]);
|
|
assert(rc == 0);
|
|
UNUSED(rc);
|
|
}
|
|
}
|
|
|
|
// initialize tasks + work buffer
|
|
{
|
|
size_t work_size = 0;
|
|
|
|
// thread scheduling for the different operations
|
|
for (int i = 0; i < cgraph->n_nodes; i++) {
|
|
struct ggml_tensor * node = cgraph->nodes[i];
|
|
|
|
switch (node->op) {
|
|
case GGML_OP_DUP:
|
|
case GGML_OP_ADD:
|
|
case GGML_OP_SUB:
|
|
case GGML_OP_MUL:
|
|
case GGML_OP_DIV:
|
|
case GGML_OP_SQR:
|
|
case GGML_OP_SQRT:
|
|
case GGML_OP_SUM:
|
|
case GGML_OP_MEAN:
|
|
case GGML_OP_REPEAT:
|
|
case GGML_OP_ABS:
|
|
case GGML_OP_SGN:
|
|
case GGML_OP_NEG:
|
|
case GGML_OP_STEP:
|
|
case GGML_OP_RELU:
|
|
{
|
|
node->n_tasks = 1;
|
|
} break;
|
|
case GGML_OP_GELU:
|
|
{
|
|
node->n_tasks = MIN(n_threads, ggml_nrows(node->src0));
|
|
} break;
|
|
case GGML_OP_NORM:
|
|
{
|
|
node->n_tasks = 1;
|
|
} break;
|
|
case GGML_OP_MUL_MAT:
|
|
{
|
|
// TODO: use different scheduling for different matrix sizes
|
|
node->n_tasks = n_threads;
|
|
|
|
size_t cur = 0;
|
|
|
|
// TODO: better way to determine if the matrix is transposed
|
|
if (node->src0->nb[1] < node->src0->nb[0]) {
|
|
cur = ggml_nbytes(node)*node->n_tasks; // TODO: this can become (n_tasks-1)
|
|
} else {
|
|
if (node->src0->type == GGML_TYPE_F16 &&
|
|
node->src1->type == GGML_TYPE_F32) {
|
|
cur = sizeof(ggml_fp16_t)*ggml_nelements(node->src1);
|
|
} else if (node->src0->type == GGML_TYPE_F32 &&
|
|
node->src1->type == GGML_TYPE_F32) {
|
|
cur = 0;
|
|
} else {
|
|
GGML_ASSERT(false);
|
|
}
|
|
}
|
|
|
|
work_size = MAX(work_size, cur);
|
|
} break;
|
|
case GGML_OP_SCALE:
|
|
{
|
|
node->n_tasks = MIN(n_threads, ggml_nrows(node->src0));
|
|
} break;
|
|
case GGML_OP_CPY:
|
|
case GGML_OP_RESHAPE:
|
|
case GGML_OP_VIEW:
|
|
case GGML_OP_PERMUTE:
|
|
case GGML_OP_TRANSPOSE:
|
|
case GGML_OP_GET_ROWS:
|
|
case GGML_OP_DIAG_MASK_INF:
|
|
{
|
|
node->n_tasks = 1;
|
|
} break;
|
|
case GGML_OP_SOFT_MAX:
|
|
{
|
|
node->n_tasks = MIN(n_threads, ggml_nrows(node->src0));
|
|
} break;
|
|
case GGML_OP_ROPE:
|
|
{
|
|
node->n_tasks = 1;
|
|
} break;
|
|
case GGML_OP_CONV_1D_1S:
|
|
case GGML_OP_CONV_1D_2S:
|
|
{
|
|
node->n_tasks = n_threads;
|
|
|
|
GGML_ASSERT(node->src0->ne[3] == 1);
|
|
GGML_ASSERT(node->src1->ne[2] == 1);
|
|
GGML_ASSERT(node->src1->ne[3] == 1);
|
|
|
|
size_t cur = 0;
|
|
const int nk = node->src0->ne[0];
|
|
|
|
if (node->src0->type == GGML_TYPE_F16 &&
|
|
node->src1->type == GGML_TYPE_F32) {
|
|
cur = sizeof(ggml_fp16_t)*(
|
|
nk*ggml_up32(node->src0->ne[1])*node->src0->ne[2] +
|
|
( 2*(nk/2) + node->src1->ne[0])*node->src1->ne[1]
|
|
);
|
|
} else if (node->src0->type == GGML_TYPE_F32 &&
|
|
node->src1->type == GGML_TYPE_F32) {
|
|
cur = sizeof(float)*(
|
|
nk*ggml_up32(node->src0->ne[1])*node->src0->ne[2] +
|
|
( 2*(nk/2) + node->src1->ne[0])*node->src1->ne[1]
|
|
);
|
|
} else {
|
|
GGML_ASSERT(false);
|
|
}
|
|
|
|
work_size = MAX(work_size, cur);
|
|
} break;
|
|
case GGML_OP_FLASH_ATTN:
|
|
{
|
|
node->n_tasks = n_threads;
|
|
|
|
size_t cur = 0;
|
|
|
|
if (node->src1->type == GGML_TYPE_F32) {
|
|
cur = sizeof(float)*node->src1->ne[1]*node->n_tasks; // TODO: this can become (n_tasks-1)
|
|
cur += sizeof(float)*node->src1->ne[1]*node->n_tasks; // this is overestimated by x2
|
|
}
|
|
|
|
if (node->src1->type == GGML_TYPE_F16) {
|
|
cur = sizeof(float)*node->src1->ne[1]*node->n_tasks; // TODO: this can become (n_tasks-1)
|
|
cur += sizeof(float)*node->src1->ne[1]*node->n_tasks; // this is overestimated by x2
|
|
}
|
|
|
|
work_size = MAX(work_size, cur);
|
|
} break;
|
|
case GGML_OP_FLASH_FF:
|
|
{
|
|
node->n_tasks = n_threads;
|
|
|
|
size_t cur = 0;
|
|
|
|
if (node->src1->type == GGML_TYPE_F32) {
|
|
cur = sizeof(float)*node->src1->ne[1]*node->n_tasks; // TODO: this can become (n_tasks-1)
|
|
cur += sizeof(float)*node->src1->ne[1]*node->n_tasks; // this is overestimated by x2
|
|
}
|
|
|
|
if (node->src1->type == GGML_TYPE_F16) {
|
|
cur = sizeof(float)*node->src1->ne[1]*node->n_tasks; // TODO: this can become (n_tasks-1)
|
|
cur += sizeof(float)*node->src1->ne[1]*node->n_tasks; // this is overestimated by x2
|
|
}
|
|
|
|
work_size = MAX(work_size, cur);
|
|
} break;
|
|
case GGML_OP_NONE:
|
|
{
|
|
node->n_tasks = 1;
|
|
} break;
|
|
case GGML_OP_COUNT:
|
|
{
|
|
assert(false);
|
|
} break;
|
|
};
|
|
}
|
|
|
|
if (cgraph->work != NULL && work_size > cgraph->work_size) {
|
|
assert(false); // TODO: better handling
|
|
}
|
|
|
|
if (work_size > 0 && cgraph->work == NULL) {
|
|
cgraph->work_size = work_size + CACHE_LINE_SIZE*(n_threads - 1);
|
|
|
|
GGML_PRINT_DEBUG("%s: allocating work buffer for graph (%zu bytes)\n", __func__, cgraph->work_size);
|
|
cgraph->work = ggml_new_tensor_1d(ctx, GGML_TYPE_I8, cgraph->work_size);
|
|
}
|
|
}
|
|
|
|
const int64_t perf_start_cycles = ggml_perf_cycles();
|
|
const int64_t perf_start_time_us = ggml_perf_time_us();
|
|
|
|
for (int i = 0; i < cgraph->n_nodes; i++) {
|
|
GGML_PRINT_DEBUG_5("%s: %d/%d\n", __func__, i, cgraph->n_nodes);
|
|
|
|
struct ggml_tensor * node = cgraph->nodes[i];
|
|
|
|
// TODO: this could be used to avoid unnecessary computations, but it needs to be improved
|
|
//if (node->grad == NULL && node->perf_runs > 0) {
|
|
// continue;
|
|
//}
|
|
|
|
const int64_t perf_node_start_cycles = ggml_perf_cycles();
|
|
const int64_t perf_node_start_time_us = ggml_perf_time_us();
|
|
|
|
// INIT
|
|
struct ggml_compute_params params = {
|
|
/*.type =*/ GGML_TASK_INIT,
|
|
/*.ith =*/ 0,
|
|
/*.nth =*/ n_threads,
|
|
/*.wsize =*/ cgraph->work ? ggml_nbytes(cgraph->work) : 0,
|
|
/*.wdata =*/ cgraph->work ? cgraph->work->data : NULL,
|
|
};
|
|
|
|
ggml_compute_forward(¶ms, node);
|
|
|
|
// COMPUTE
|
|
if (node->n_tasks > 1) {
|
|
if (atomic_fetch_add(&state_shared.n_ready, 1) == n_threads - 1) {
|
|
atomic_store(&state_shared.has_work, false);
|
|
}
|
|
|
|
while (atomic_load(&state_shared.has_work)) {
|
|
ggml_lock_lock (&state_shared.spin);
|
|
ggml_lock_unlock(&state_shared.spin);
|
|
}
|
|
|
|
// launch thread pool
|
|
for (int j = 0; j < n_threads - 1; j++) {
|
|
workers[j].params = (struct ggml_compute_params) {
|
|
.type = GGML_TASK_COMPUTE,
|
|
.ith = j + 1,
|
|
.nth = n_threads,
|
|
.wsize = cgraph->work ? ggml_nbytes(cgraph->work) : 0,
|
|
.wdata = cgraph->work ? cgraph->work->data : NULL,
|
|
};
|
|
workers[j].node = node;
|
|
}
|
|
|
|
atomic_fetch_sub(&state_shared.n_ready, 1);
|
|
|
|
while (atomic_load(&state_shared.n_ready) > 0) {
|
|
ggml_lock_lock (&state_shared.spin);
|
|
ggml_lock_unlock(&state_shared.spin);
|
|
}
|
|
|
|
atomic_store(&state_shared.has_work, true);
|
|
}
|
|
|
|
params.type = GGML_TASK_COMPUTE;
|
|
ggml_compute_forward(¶ms, node);
|
|
|
|
// wait for thread pool
|
|
if (node->n_tasks > 1) {
|
|
if (atomic_fetch_add(&state_shared.n_ready, 1) == n_threads - 1) {
|
|
atomic_store(&state_shared.has_work, false);
|
|
}
|
|
|
|
while (atomic_load(&state_shared.has_work)) {
|
|
ggml_lock_lock (&state_shared.spin);
|
|
ggml_lock_unlock(&state_shared.spin);
|
|
}
|
|
|
|
atomic_fetch_sub(&state_shared.n_ready, 1);
|
|
|
|
while (atomic_load(&state_shared.n_ready) != 0) {
|
|
ggml_lock_lock (&state_shared.spin);
|
|
ggml_lock_unlock(&state_shared.spin);
|
|
}
|
|
}
|
|
|
|
// FINALIZE
|
|
if (node->n_tasks > 1) {
|
|
if (atomic_fetch_add(&state_shared.n_ready, 1) == n_threads - 1) {
|
|
atomic_store(&state_shared.has_work, false);
|
|
}
|
|
|
|
while (atomic_load(&state_shared.has_work)) {
|
|
ggml_lock_lock (&state_shared.spin);
|
|
ggml_lock_unlock(&state_shared.spin);
|
|
}
|
|
|
|
// launch thread pool
|
|
for (int j = 0; j < n_threads - 1; j++) {
|
|
workers[j].params = (struct ggml_compute_params) {
|
|
.type = GGML_TASK_FINALIZE,
|
|
.ith = j + 1,
|
|
.nth = n_threads,
|
|
.wsize = cgraph->work ? ggml_nbytes(cgraph->work) : 0,
|
|
.wdata = cgraph->work ? cgraph->work->data : NULL,
|
|
};
|
|
workers[j].node = node;
|
|
}
|
|
|
|
atomic_fetch_sub(&state_shared.n_ready, 1);
|
|
|
|
while (atomic_load(&state_shared.n_ready) > 0) {
|
|
ggml_lock_lock (&state_shared.spin);
|
|
ggml_lock_unlock(&state_shared.spin);
|
|
}
|
|
|
|
atomic_store(&state_shared.has_work, true);
|
|
}
|
|
|
|
params.type = GGML_TASK_FINALIZE;
|
|
ggml_compute_forward(¶ms, node);
|
|
|
|
// wait for thread pool
|
|
if (node->n_tasks > 1) {
|
|
if (atomic_fetch_add(&state_shared.n_ready, 1) == n_threads - 1) {
|
|
atomic_store(&state_shared.has_work, false);
|
|
}
|
|
|
|
while (atomic_load(&state_shared.has_work)) {
|
|
ggml_lock_lock (&state_shared.spin);
|
|
ggml_lock_unlock(&state_shared.spin);
|
|
}
|
|
|
|
atomic_fetch_sub(&state_shared.n_ready, 1);
|
|
|
|
while (atomic_load(&state_shared.n_ready) != 0) {
|
|
ggml_lock_lock (&state_shared.spin);
|
|
ggml_lock_unlock(&state_shared.spin);
|
|
}
|
|
}
|
|
|
|
// performance stats (node)
|
|
{
|
|
int64_t perf_cycles_cur = ggml_perf_cycles() - perf_node_start_cycles;
|
|
int64_t perf_time_us_cur = ggml_perf_time_us() - perf_node_start_time_us;
|
|
|
|
node->perf_runs++;
|
|
node->perf_cycles += perf_cycles_cur;
|
|
node->perf_time_us += perf_time_us_cur;
|
|
}
|
|
}
|
|
|
|
// join thread pool
|
|
if (n_threads > 1) {
|
|
atomic_store(&state_shared.stop, true);
|
|
atomic_store(&state_shared.has_work, true);
|
|
|
|
for (int j = 0; j < n_threads - 1; j++) {
|
|
int rc = pthread_join(workers[j].thrd, NULL);
|
|
assert(rc == 0);
|
|
UNUSED(rc);
|
|
}
|
|
|
|
ggml_lock_destroy(&state_shared.spin);
|
|
}
|
|
|
|
// performance stats (graph)
|
|
{
|
|
int64_t perf_cycles_cur = ggml_perf_cycles() - perf_start_cycles;
|
|
int64_t perf_time_us_cur = ggml_perf_time_us() - perf_start_time_us;
|
|
|
|
cgraph->perf_runs++;
|
|
cgraph->perf_cycles += perf_cycles_cur;
|
|
cgraph->perf_time_us += perf_time_us_cur;
|
|
|
|
GGML_PRINT_DEBUG("%s: perf (%d) - cpu = %.3f / %.3f ms, wall = %.3f / %.3f ms\n",
|
|
__func__, cgraph->perf_runs,
|
|
(double) perf_cycles_cur / (double) ggml_cycles_per_ms(),
|
|
(double) cgraph->perf_cycles / (double) ggml_cycles_per_ms() / (double) cgraph->perf_runs,
|
|
(double) perf_time_us_cur / 1000.0,
|
|
(double) cgraph->perf_time_us / 1000.0 / cgraph->perf_runs);
|
|
}
|
|
}
|
|
|
|
void ggml_graph_reset(struct ggml_cgraph * cgraph) {
|
|
for (int i = 0; i < cgraph->n_nodes; i++) {
|
|
struct ggml_tensor * grad = cgraph->grads[i];
|
|
|
|
if (grad) {
|
|
ggml_set_zero(grad);
|
|
}
|
|
}
|
|
}
|
|
|
|
void ggml_graph_print(const struct ggml_cgraph * cgraph) {
|
|
int64_t perf_total_per_op_us[GGML_OP_COUNT] = {0};
|
|
|
|
GGML_PRINT("=== GRAPH ===\n");
|
|
|
|
GGML_PRINT_DEBUG("n_threads = %d\n", cgraph->n_threads);
|
|
GGML_PRINT_DEBUG("total work size = %zu bytes\n",cgraph->work_size);
|
|
|
|
GGML_PRINT("n_nodes = %d\n", cgraph->n_nodes);
|
|
for (int i = 0; i < cgraph->n_nodes; i++) {
|
|
struct ggml_tensor * node = cgraph->nodes[i];
|
|
|
|
perf_total_per_op_us[node->op] += node->perf_time_us;
|
|
|
|
GGML_PRINT(" - %3d: [ %6d, %6d] %16s %s (%3d) cpu = %7.3f / %7.3f ms, wall = %7.3f / %7.3f ms\n",
|
|
i,
|
|
node->ne[0], node->ne[1],
|
|
GGML_OP_LABEL[node->op], node->is_param ? "x" : node->grad ? "g" : " ", node->perf_runs,
|
|
(double) node->perf_cycles / (double) ggml_cycles_per_ms(),
|
|
(double) node->perf_cycles / (double) ggml_cycles_per_ms() / (double) node->perf_runs,
|
|
(double) node->perf_time_us / 1000.0,
|
|
(double) node->perf_time_us / 1000.0 / node->perf_runs);
|
|
}
|
|
|
|
GGML_PRINT("n_leafs = %d\n", cgraph->n_leafs);
|
|
for (int i = 0; i < cgraph->n_leafs; i++) {
|
|
struct ggml_tensor * node = cgraph->leafs[i];
|
|
|
|
GGML_PRINT(" - %3d: [ %6d, %6d] %8s\n",
|
|
i,
|
|
node->ne[0], node->ne[1],
|
|
GGML_OP_LABEL[node->op]);
|
|
}
|
|
|
|
for (int i = 0; i < GGML_OP_COUNT; i++) {
|
|
GGML_PRINT("perf_total_per_op_us[%16s] = %7.3f ms\n", GGML_OP_LABEL[i], (double) perf_total_per_op_us[i] / 1000.0);
|
|
}
|
|
|
|
GGML_PRINT("========================================\n");
|
|
}
|
|
|
|
// check if node is part of the graph
|
|
bool ggml_graph_find(const struct ggml_cgraph * cgraph, const struct ggml_tensor * node) {
|
|
if (cgraph == NULL) {
|
|
return true;
|
|
}
|
|
|
|
for (int i = 0; i < cgraph->n_nodes; i++) {
|
|
if (cgraph->nodes[i] == node) {
|
|
return true;
|
|
}
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
struct ggml_tensor * ggml_graph_get_parent(const struct ggml_cgraph * cgraph, const struct ggml_tensor * node) {
|
|
for (int i = 0; i < cgraph->n_nodes; i++) {
|
|
struct ggml_tensor * parent = cgraph->nodes[i];
|
|
|
|
if (parent->grad == node) {
|
|
return parent;
|
|
}
|
|
}
|
|
|
|
return NULL;
|
|
}
|
|
|
|
void ggml_graph_dump_dot(const struct ggml_cgraph * gb, const struct ggml_cgraph * gf, const char * filename) {
|
|
char color[16];
|
|
|
|
FILE * fp = fopen(filename, "w");
|
|
assert(fp);
|
|
|
|
fprintf(fp, "digraph G {\n");
|
|
fprintf(fp, " newrank = true;\n");
|
|
fprintf(fp, " rankdir = LR;\n");
|
|
|
|
for (int i = 0; i < gb->n_nodes; i++) {
|
|
struct ggml_tensor * node = gb->nodes[i];
|
|
|
|
if (ggml_graph_get_parent(gb, node) != NULL) {
|
|
continue;
|
|
}
|
|
|
|
if (node->is_param) {
|
|
snprintf(color, sizeof(color), "yellow");
|
|
} else if (node->grad) {
|
|
if (ggml_graph_find(gf, node)) {
|
|
snprintf(color, sizeof(color), "green");
|
|
} else {
|
|
snprintf(color, sizeof(color), "lightblue");
|
|
}
|
|
} else {
|
|
snprintf(color, sizeof(color), "white");
|
|
}
|
|
|
|
fprintf(fp, " \"%p\" [ \
|
|
style = filled; fillcolor = %s; shape = record; \
|
|
label=\"%d [%d, %d] | <x>%s",
|
|
(void *) node, color,
|
|
i, node->ne[0], node->ne[1],
|
|
GGML_OP_SYMBOL[node->op]);
|
|
|
|
if (node->grad) {
|
|
fprintf(fp, " | <g>%s\"; ]\n", GGML_OP_SYMBOL[node->grad->op]);
|
|
} else {
|
|
fprintf(fp, "\"; ]\n");
|
|
}
|
|
}
|
|
|
|
for (int i = 0; i < gb->n_leafs; i++) {
|
|
struct ggml_tensor * node = gb->leafs[i];
|
|
|
|
snprintf(color, sizeof(color), "pink");
|
|
|
|
if (ggml_nelements(node) == 1) {
|
|
fprintf(fp, " \"%p\" [ \
|
|
style = filled; fillcolor = %s; shape = record; \
|
|
label=\"<x>%.1e\"; ]\n",
|
|
(void *) node, color, ggml_get_f32_1d(node, 0));
|
|
} else {
|
|
fprintf(fp, " \"%p\" [ \
|
|
style = filled; fillcolor = %s; shape = record; \
|
|
label=\"<x>CONST %d [%d, %d]\"; ]\n",
|
|
(void *) node, color,
|
|
i, node->ne[0], node->ne[1]);
|
|
}
|
|
}
|
|
|
|
for (int i = 0; i < gb->n_nodes; i++) {
|
|
struct ggml_tensor * node = gb->nodes[i];
|
|
|
|
struct ggml_tensor * parent = ggml_graph_get_parent(gb, node);
|
|
|
|
if (node->src0) {
|
|
struct ggml_tensor * parent0 = ggml_graph_get_parent(gb, node->src0);
|
|
|
|
fprintf(fp, " \"%p\":%s -> \"%p\":%s [ arrowhead = %s; style = %s; label = \"x\"; ]\n",
|
|
parent0 ? (void *) parent0 : (void *) node->src0,
|
|
parent0 ? "g" : "x",
|
|
parent ? (void *) parent : (void *) node,
|
|
parent ? "g" : "x",
|
|
parent ? "empty" : "vee",
|
|
parent ? "dashed" : "solid");
|
|
}
|
|
|
|
if (node->src1) {
|
|
struct ggml_tensor * parent1 = ggml_graph_get_parent(gb, node->src1);
|
|
|
|
fprintf(fp, " \"%p\":%s -> \"%p\":%s [ arrowhead = %s; style = %s; label = \"y\"; ]\n",
|
|
parent1 ? (void *) parent1 : (void *) node->src1,
|
|
parent1 ? "g" : "x",
|
|
parent ? (void *) parent : (void *) node,
|
|
parent ? "g" : "x",
|
|
parent ? "empty" : "vee",
|
|
parent ? "dashed" : "solid");
|
|
}
|
|
}
|
|
|
|
for (int i = 0; i < gb->n_leafs; i++) {
|
|
struct ggml_tensor * node = gb->leafs[i];
|
|
|
|
if (node->src0) {
|
|
fprintf(fp, " \"%p\":%s -> \"%p\":%s [ label = \"x\"; ]\n",
|
|
(void *) node->src0, "x",
|
|
(void *) node, "x");
|
|
}
|
|
|
|
if (node->src1) {
|
|
fprintf(fp, " \"%p\":%s -> \"%p\":%s [ label = \"y\"; ]\n",
|
|
(void *) node->src1, "x",
|
|
(void *) node, "x");
|
|
}
|
|
}
|
|
|
|
fprintf(fp, "}\n");
|
|
|
|
fclose(fp);
|
|
|
|
GGML_PRINT("%s: dot -Tpng %s -o %s.png && open %s.png\n", __func__, filename, filename, filename);
|
|
}
|
|
|
|
////////////////////////////////////////////////////////////////////////////////
|
|
|
|
void ggml_opt_set_params(int np, struct ggml_tensor * const ps[], const float * x) {
|
|
int i = 0;
|
|
for (int p = 0; p < np; ++p) {
|
|
const int ne = ggml_nelements(ps[p]) ;
|
|
// TODO: add function to set tensor from array
|
|
for (int j = 0; j < ne; ++j) {
|
|
ggml_set_f32_1d(ps[p], j, x[i++]);
|
|
}
|
|
}
|
|
}
|
|
|
|
void ggml_opt_get_params(int np, struct ggml_tensor * const ps[], float * x) {
|
|
int i = 0;
|
|
for (int p = 0; p < np; ++p) {
|
|
const int ne = ggml_nelements(ps[p]) ;
|
|
// TODO: add function to get all elements at once
|
|
for (int j = 0; j < ne; ++j) {
|
|
x[i++] = ggml_get_f32_1d(ps[p], j);
|
|
}
|
|
}
|
|
}
|
|
|
|
void ggml_opt_get_grad(int np, struct ggml_tensor * const ps[], float * g) {
|
|
int i = 0;
|
|
for (int p = 0; p < np; ++p) {
|
|
const int ne = ggml_nelements(ps[p]) ;
|
|
// TODO: add function to get all elements at once
|
|
for (int j = 0; j < ne; ++j) {
|
|
g[i++] = ggml_get_f32_1d(ps[p]->grad, j);
|
|
}
|
|
}
|
|
}
|
|
|
|
//
|
|
// ADAM
|
|
//
|
|
// ref: https://arxiv.org/pdf/1412.6980.pdf
|
|
//
|
|
|
|
enum ggml_opt_result ggml_opt_adam(
|
|
struct ggml_context * ctx,
|
|
struct ggml_opt_params params,
|
|
struct ggml_tensor * f,
|
|
struct ggml_cgraph * gf,
|
|
struct ggml_cgraph * gb) {
|
|
assert(ggml_is_scalar(f));
|
|
|
|
gf->n_threads = params.n_threads;
|
|
gb->n_threads = params.n_threads;
|
|
|
|
// these will store the parameters we want to optimize
|
|
struct ggml_tensor * ps[GGML_MAX_PARAMS];
|
|
|
|
int np = 0;
|
|
int nx = 0;
|
|
for (int i = 0; i < gf->n_nodes; ++i) {
|
|
if (gf->nodes[i]->is_param) {
|
|
GGML_PRINT_DEBUG("found param %d: grad->op = %d\n", np, gf->nodes[i]->grad->op);
|
|
|
|
assert(np < GGML_MAX_PARAMS);
|
|
|
|
ps[np++] = gf->nodes[i];
|
|
nx += ggml_nelements(gf->nodes[i]);
|
|
}
|
|
}
|
|
|
|
// constants
|
|
const float alpha = params.adam.alpha;
|
|
const float beta1 = params.adam.beta1;
|
|
const float beta2 = params.adam.beta2;
|
|
const float eps = params.adam.eps;
|
|
|
|
float * x = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, nx)->data; // view of the parameters
|
|
float * g1 = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, nx)->data; // gradient
|
|
float * g2 = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, nx)->data; // gradient squared
|
|
float * m = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, nx)->data; // first moment
|
|
float * v = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, nx)->data; // second moment
|
|
float * mh = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, nx)->data; // first moment hat
|
|
float * vh = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, nx)->data; // second moment hat
|
|
|
|
float * pf = params.past > 0 ? ggml_new_tensor_1d(ctx, GGML_TYPE_F32, params.past)->data : NULL; // past function values
|
|
|
|
// initialize
|
|
ggml_vec_set_f32(nx, m, 0.0f);
|
|
ggml_vec_set_f32(nx, v, 0.0f);
|
|
|
|
// update view
|
|
ggml_opt_get_params(np, ps, x);
|
|
|
|
// compute the function value
|
|
ggml_graph_reset (gf);
|
|
ggml_set_f32 (f->grad, 1.0f);
|
|
ggml_graph_compute(ctx, gb);
|
|
|
|
float fx_prev = ggml_get_f32_1d(f, 0);
|
|
if (pf) {
|
|
pf[0] = fx_prev;
|
|
}
|
|
|
|
int n_no_improvement = 0;
|
|
float fx_best = fx_prev;
|
|
|
|
// run the optimizer
|
|
for (int t = 0; t < params.adam.n_iter; ++t) {
|
|
GGML_PRINT_DEBUG ("=== iter %d ===\n", t);
|
|
|
|
GGML_PRINT_DEBUG ("f = %10.6f\n", ggml_get_f32_1d(f, 0));
|
|
GGML_PRINT_DEBUG_5("df/dx0 = %10.6f\n", ggml_get_f32_1d(ps[0]->grad, 0));
|
|
GGML_PRINT_DEBUG_5("df/dx1 = %10.6f\n", ggml_get_f32_1d(ps[1]->grad, 0));
|
|
|
|
for (int i = 0; i < np; ++i) {
|
|
GGML_PRINT_DEBUG("param %d: %10.6f, g = %10.6f\n", i,
|
|
ggml_get_f32_1d(ps[i], 0), ggml_get_f32_1d(ps[i]->grad, 0));
|
|
}
|
|
|
|
const int64_t t_start_wall = ggml_time_us();
|
|
const int64_t t_start_cpu = ggml_cycles();
|
|
UNUSED(t_start_wall);
|
|
UNUSED(t_start_cpu);
|
|
|
|
{
|
|
// update the gradient
|
|
ggml_opt_get_grad(np, ps, g1);
|
|
|
|
// m_t = beta1*m_t-1 + (1 - beta1)*g_t
|
|
ggml_vec_scale_f32(nx, m, beta1);
|
|
ggml_vec_mad_f32 (nx, m, g1, 1.0f - beta1);
|
|
|
|
// g2 = g1^2
|
|
ggml_vec_sqr_f32 (nx, g2, g1);
|
|
|
|
// v_t = beta2*v_t-1 + (1 - beta2)*g_t^2
|
|
ggml_vec_scale_f32(nx, v, beta2);
|
|
ggml_vec_mad_f32 (nx, v, g2, 1.0f - beta2);
|
|
|
|
// m^hat = m_t / (1 - beta1^t)
|
|
// v^hat = v_t / (1 - beta2^t)
|
|
// x_t = x_t-1 - alpha*m^hat/(sqrt(v^hat) + eps)
|
|
ggml_vec_cpy_f32 (nx, mh, m);
|
|
ggml_vec_cpy_f32 (nx, vh, v);
|
|
|
|
ggml_vec_scale_f32(nx, mh, alpha/(1.0f - powf(beta1, t + 1)));
|
|
ggml_vec_scale_f32(nx, vh, 1.0f/(1.0f - powf(beta2, t + 1)));
|
|
|
|
ggml_vec_sqrt_f32 (nx, vh, vh);
|
|
ggml_vec_acc1_f32 (nx, vh, eps);
|
|
|
|
ggml_vec_div_f32 (nx, mh, mh, vh);
|
|
ggml_vec_sub_f32 (nx, x, x, mh);
|
|
|
|
// update the parameters
|
|
ggml_opt_set_params(np, ps, x);
|
|
}
|
|
|
|
ggml_graph_reset (gf);
|
|
ggml_set_f32 (f->grad, 1.0f);
|
|
ggml_graph_compute(ctx, gb);
|
|
|
|
const float fx = ggml_get_f32_1d(f, 0);
|
|
|
|
// check convergence
|
|
if (fabsf(fx - fx_prev)/fx < params.adam.eps_f) {
|
|
GGML_PRINT_DEBUG("converged\n");
|
|
|
|
return GGML_OPT_OK;
|
|
}
|
|
|
|
// delta-based convergence test
|
|
if (pf != NULL) {
|
|
// need at least params.past iterations to start checking for convergence
|
|
if (params.past <= t) {
|
|
const float rate = (pf[t%params.past] - fx)/fx;
|
|
|
|
if (fabs(rate) < params.delta) {
|
|
return GGML_OPT_OK;
|
|
}
|
|
}
|
|
|
|
pf[t%params.past] = fx;
|
|
}
|
|
|
|
// check for improvement
|
|
if (params.max_no_improvement > 0) {
|
|
if (fx_best > fx) {
|
|
fx_best = fx;
|
|
n_no_improvement = 0;
|
|
} else {
|
|
++n_no_improvement;
|
|
|
|
if (n_no_improvement >= params.max_no_improvement) {
|
|
return GGML_OPT_OK;
|
|
}
|
|
}
|
|
}
|
|
|
|
fx_prev = fx;
|
|
|
|
{
|
|
const int64_t t_end_cpu = ggml_cycles();
|
|
GGML_PRINT_DEBUG("time iter: %5.3f s\n", (t_end_cpu - t_start_cpu)/CLOCKS_PER_SEC);
|
|
UNUSED(t_end_cpu);
|
|
|
|
const int64_t t_end_wall = ggml_time_us();
|
|
GGML_PRINT_DEBUG("wall time iter: %5.3f s\n", (t_end_wall - t_start_wall)/1e6);
|
|
UNUSED(t_end_wall);
|
|
}
|
|
}
|
|
|
|
return GGML_OPT_DID_NOT_CONVERGE;
|
|
}
|
|
|
|
//
|
|
// L-BFGS
|
|
//
|
|
// the L-BFGS implementation below is based on the following implementation:
|
|
//
|
|
// https://github.com/chokkan/liblbfgs
|
|
//
|
|
|
|
struct ggml_lbfgs_iteration_data {
|
|
float alpha;
|
|
float ys;
|
|
float * s;
|
|
float * y;
|
|
};
|
|
|
|
static enum ggml_opt_result linesearch_backtracking(
|
|
struct ggml_context * ctx,
|
|
const struct ggml_opt_params * params,
|
|
int nx,
|
|
float * x,
|
|
float * fx,
|
|
float * g,
|
|
float * d,
|
|
float * step,
|
|
const float * xp,
|
|
struct ggml_tensor * f,
|
|
struct ggml_cgraph * gf,
|
|
struct ggml_cgraph * gb,
|
|
const int np,
|
|
struct ggml_tensor * ps[]) {
|
|
int count = 0;
|
|
|
|
float width = 0.0f;
|
|
float dg = 0.0f;
|
|
float finit = 0.0f;
|
|
float dginit = 0.0f;
|
|
float dgtest = 0.0f;
|
|
|
|
const float dec = 0.5f;
|
|
const float inc = 2.1f;
|
|
|
|
if (*step <= 0.) {
|
|
return GGML_LINESEARCH_INVALID_PARAMETERS;
|
|
}
|
|
|
|
// compute the initial gradient in the search direction
|
|
ggml_vec_dot_f32(nx, &dginit, g, d);
|
|
|
|
// make sure that d points to a descent direction
|
|
if (0 < dginit) {
|
|
return GGML_LINESEARCH_FAIL;
|
|
}
|
|
|
|
// initialize local variables
|
|
finit = *fx;
|
|
dgtest = params->lbfgs.ftol*dginit;
|
|
|
|
while (true) {
|
|
ggml_vec_cpy_f32(nx, x, xp);
|
|
ggml_vec_mad_f32(nx, x, d, *step);
|
|
|
|
// evaluate the function and gradient values
|
|
{
|
|
ggml_opt_set_params(np, ps, x);
|
|
|
|
ggml_graph_reset (gf);
|
|
ggml_set_f32 (f->grad, 1.0f);
|
|
ggml_graph_compute(ctx, gb);
|
|
|
|
ggml_opt_get_grad(np, ps, g);
|
|
|
|
*fx = ggml_get_f32_1d(f, 0);
|
|
}
|
|
|
|
++count;
|
|
|
|
if (*fx > finit + (*step)*dgtest) {
|
|
width = dec;
|
|
} else {
|
|
// Armijo condition is satisfied
|
|
if (params->lbfgs.linesearch == GGML_LINESEARCH_BACKTRACKING_ARMIJO) {
|
|
return count;
|
|
}
|
|
|
|
ggml_vec_dot_f32(nx, &dg, g, d);
|
|
|
|
// check the Wolfe condition
|
|
if (dg < params->lbfgs.wolfe * dginit) {
|
|
width = inc;
|
|
} else {
|
|
if(params->lbfgs.linesearch == GGML_LINESEARCH_BACKTRACKING_WOLFE) {
|
|
// regular Wolfe conditions
|
|
return count;
|
|
}
|
|
|
|
if(dg > -params->lbfgs.wolfe*dginit) {
|
|
width = dec;
|
|
} else {
|
|
// strong Wolfe condition (GGML_LINESEARCH_BACKTRACKING_STRONG_WOLFE)
|
|
return count;
|
|
}
|
|
return count;
|
|
}
|
|
}
|
|
|
|
if (*step < params->lbfgs.min_step) {
|
|
return GGML_LINESEARCH_MINIMUM_STEP;
|
|
}
|
|
if (*step > params->lbfgs.max_step) {
|
|
return GGML_LINESEARCH_MAXIMUM_STEP;
|
|
}
|
|
if (params->lbfgs.max_linesearch <= count) {
|
|
return GGML_LINESEARCH_MAXIMUM_ITERATIONS;
|
|
}
|
|
|
|
(*step) *= width;
|
|
}
|
|
|
|
return GGML_LINESEARCH_FAIL;
|
|
}
|
|
|
|
enum ggml_opt_result ggml_opt_lbfgs(
|
|
struct ggml_context * ctx,
|
|
struct ggml_opt_params params,
|
|
struct ggml_tensor * f,
|
|
struct ggml_cgraph * gf,
|
|
struct ggml_cgraph * gb) {
|
|
if (params.lbfgs.linesearch == GGML_LINESEARCH_BACKTRACKING_WOLFE ||
|
|
params.lbfgs.linesearch == GGML_LINESEARCH_BACKTRACKING_STRONG_WOLFE) {
|
|
if (params.lbfgs.wolfe <= params.lbfgs.ftol || 1. <= params.lbfgs.wolfe) {
|
|
return GGML_OPT_INVALID_WOLFE;
|
|
}
|
|
}
|
|
|
|
gf->n_threads = params.n_threads;
|
|
gb->n_threads = params.n_threads;
|
|
|
|
const int m = params.lbfgs.m;
|
|
|
|
// these will store the parameters we want to optimize
|
|
struct ggml_tensor * ps[GGML_MAX_PARAMS];
|
|
|
|
int np = 0;
|
|
int nx = 0;
|
|
for (int i = 0; i < gf->n_nodes; ++i) {
|
|
if (gf->nodes[i]->is_param) {
|
|
GGML_PRINT_DEBUG("found param %d: grad->op = %d\n", np, gf->nodes[i]->grad->op);
|
|
|
|
assert(np < GGML_MAX_PARAMS);
|
|
|
|
ps[np++] = gf->nodes[i];
|
|
nx += ggml_nelements(gf->nodes[i]);
|
|
}
|
|
}
|
|
|
|
float * x = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, nx)->data; // current parameters
|
|
float * xp = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, nx)->data; // previous parameters
|
|
float * g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, nx)->data; // current gradient
|
|
float * gp = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, nx)->data; // previous gradient
|
|
float * d = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, nx)->data; // search direction
|
|
|
|
float * pf = params.past > 0 ? ggml_new_tensor_1d(ctx, GGML_TYPE_F32, params.past)->data : NULL; // past function values
|
|
|
|
float fx = 0.0f; // cost function value
|
|
float xnorm = 0.0f; // ||x||
|
|
float gnorm = 0.0f; // ||g||
|
|
float step = 0.0f;
|
|
|
|
// initialize x from the graph nodes
|
|
ggml_opt_get_params(np, ps, x);
|
|
|
|
// the L-BFGS memory
|
|
struct ggml_lbfgs_iteration_data * lm = alloca(sizeof(struct ggml_lbfgs_iteration_data)*m);
|
|
|
|
for (int i = 0; i < m; ++i) {
|
|
lm[i].alpha = 0.0f;
|
|
lm[i].ys = 0.0f;
|
|
lm[i].s = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, nx)->data;
|
|
lm[i].y = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, nx)->data;
|
|
}
|
|
|
|
// evaluate the function value and its gradient
|
|
{
|
|
ggml_opt_set_params(np, ps, x);
|
|
|
|
ggml_graph_reset (gf);
|
|
ggml_set_f32 (f->grad, 1.0f);
|
|
ggml_graph_compute(ctx, gb);
|
|
|
|
ggml_opt_get_grad(np, ps, g);
|
|
|
|
fx = ggml_get_f32_1d(f, 0);
|
|
}
|
|
|
|
if (pf) {
|
|
pf[0] = fx;
|
|
}
|
|
|
|
float fx_best = fx;
|
|
|
|
// search direction = -gradient
|
|
ggml_vec_neg_f32(nx, d, g);
|
|
|
|
// ||x||, ||g||
|
|
ggml_vec_norm_f32(nx, &xnorm, x);
|
|
ggml_vec_norm_f32(nx, &gnorm, g);
|
|
|
|
if (xnorm < 1.0f) {
|
|
xnorm = 1.0f;
|
|
}
|
|
|
|
// already optimized
|
|
if (gnorm/xnorm <= params.lbfgs.eps) {
|
|
return GGML_OPT_OK;
|
|
}
|
|
|
|
// initial step
|
|
ggml_vec_norm_inv_f32(nx, &step, d);
|
|
|
|
int j = 0;
|
|
int k = 1;
|
|
int ls = 0;
|
|
int end = 0;
|
|
int bound = 0;
|
|
int n_no_improvement = 0;
|
|
|
|
float ys = 0.0f;
|
|
float yy = 0.0f;
|
|
float beta = 0.0f;
|
|
|
|
while (true) {
|
|
// store the current position and gradient vectors
|
|
ggml_vec_cpy_f32(nx, xp, x);
|
|
ggml_vec_cpy_f32(nx, gp, g);
|
|
|
|
ls = linesearch_backtracking(ctx, ¶ms, nx, x, &fx, g, d, &step, xp, f, gf, gb, np, ps);
|
|
|
|
if (ls < 0) {
|
|
// linesearch failed - go back to the previous point and return
|
|
ggml_vec_cpy_f32(nx, x, xp);
|
|
ggml_vec_cpy_f32(nx, g, gp);
|
|
|
|
return ls;
|
|
}
|
|
|
|
ggml_vec_norm_f32(nx, &xnorm, x);
|
|
ggml_vec_norm_f32(nx, &gnorm, g);
|
|
|
|
GGML_PRINT_DEBUG("f = %10.6f\n", ggml_get_f32_1d(f, 0));
|
|
|
|
if (xnorm < 1.0) {
|
|
xnorm = 1.0;
|
|
}
|
|
if (gnorm/xnorm <= params.lbfgs.eps) {
|
|
// converged
|
|
return GGML_OPT_OK;
|
|
}
|
|
|
|
// delta-based convergence test
|
|
if (pf != NULL) {
|
|
// need at least params.past iterations to start checking for convergence
|
|
if (params.past <= k) {
|
|
const float rate = (pf[k%params.past] - fx)/fx;
|
|
|
|
if (fabs(rate) < params.delta) {
|
|
return GGML_OPT_OK;
|
|
}
|
|
}
|
|
|
|
pf[k%params.past] = fx;
|
|
}
|
|
|
|
// check for improvement
|
|
if (params.max_no_improvement > 0) {
|
|
if (fx < fx_best) {
|
|
fx_best = fx;
|
|
n_no_improvement = 0;
|
|
} else {
|
|
n_no_improvement++;
|
|
|
|
if (n_no_improvement >= params.max_no_improvement) {
|
|
return GGML_OPT_OK;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (params.lbfgs.n_iter != 0 && params.lbfgs.n_iter < k + 1) {
|
|
// reached the maximum number of iterations
|
|
return GGML_OPT_DID_NOT_CONVERGE;
|
|
}
|
|
|
|
// update vectors s and y:
|
|
// s_{k+1} = x_{k+1} - x_{k} = \step * d_{k}.
|
|
// y_{k+1} = g_{k+1} - g_{k}.
|
|
//
|
|
ggml_vec_sub_f32(nx, lm[end].s, x, xp);
|
|
ggml_vec_sub_f32(nx, lm[end].y, g, gp);
|
|
|
|
// compute scalars ys and yy:
|
|
// ys = y^t \cdot s -> 1 / \rho.
|
|
// yy = y^t \cdot y.
|
|
//
|
|
ggml_vec_dot_f32(nx, &ys, lm[end].y, lm[end].s);
|
|
ggml_vec_dot_f32(nx, &yy, lm[end].y, lm[end].y);
|
|
|
|
lm[end].ys = ys;
|
|
|
|
// find new search direction
|
|
// ref: https://en.wikipedia.org/wiki/Limited-memory_BFGS
|
|
|
|
bound = (m <= k) ? m : k;
|
|
k++;
|
|
end = (end + 1)%m;
|
|
|
|
// initialize search direction with -g
|
|
ggml_vec_neg_f32(nx, d, g);
|
|
|
|
j = end;
|
|
for (int i = 0; i < bound; ++i) {
|
|
j = (j + m - 1) % m;
|
|
// \alpha_{j} = \rho_{j} s^{t}_{j} \cdot q_{k+1}
|
|
ggml_vec_dot_f32(nx, &lm[j].alpha, lm[j].s, d);
|
|
lm[j].alpha /= lm[j].ys;
|
|
// q_{i} = q_{i+1} - \alpha_{i} y_{i}
|
|
ggml_vec_mad_f32(nx, d, lm[j].y, -lm[j].alpha);
|
|
}
|
|
|
|
ggml_vec_scale_f32(nx, d, ys/yy);
|
|
|
|
for (int i = 0; i < bound; ++i) {
|
|
// \beta_{j} = \rho_{j} y^t_{j} \cdot \gamma_{i}
|
|
ggml_vec_dot_f32(nx, &beta, lm[j].y, d);
|
|
beta /= lm[j].ys;
|
|
// \gamma_{i+1} = \gamma_{i} + (\alpha_{j} - \beta_{j}) s_{j}
|
|
ggml_vec_mad_f32(nx, d, lm[j].s, lm[j].alpha - beta);
|
|
j = (j + 1)%m;
|
|
}
|
|
|
|
step = 1.0;
|
|
}
|
|
|
|
return GGML_OPT_DID_NOT_CONVERGE;
|
|
}
|
|
|
|
struct ggml_opt_params ggml_opt_default_params(enum ggml_opt_type type) {
|
|
struct ggml_opt_params result;
|
|
|
|
switch (type) {
|
|
case GGML_OPT_ADAM:
|
|
{
|
|
result = (struct ggml_opt_params) {
|
|
.type = GGML_OPT_ADAM,
|
|
.n_threads = 1,
|
|
.past = 0,
|
|
.delta = 1e-5f,
|
|
|
|
.max_no_improvement = 100,
|
|
|
|
.print_forward_graph = true,
|
|
.print_backward_graph = true,
|
|
|
|
.adam = {
|
|
.n_iter = 10000,
|
|
.alpha = 0.001f,
|
|
.beta1 = 0.9f,
|
|
.beta2 = 0.999f,
|
|
.eps = 1e-8f,
|
|
.eps_f = 1e-5f,
|
|
.eps_g = 1e-3f,
|
|
},
|
|
};
|
|
} break;
|
|
case GGML_OPT_LBFGS:
|
|
{
|
|
result = (struct ggml_opt_params) {
|
|
.type = GGML_OPT_LBFGS,
|
|
.n_threads = 1,
|
|
.past = 0,
|
|
.delta = 1e-5f,
|
|
|
|
.max_no_improvement = 0,
|
|
|
|
.print_forward_graph = true,
|
|
.print_backward_graph = true,
|
|
|
|
.lbfgs = {
|
|
.m = 6,
|
|
.n_iter = 100,
|
|
.max_linesearch = 20,
|
|
|
|
.eps = 1e-5f,
|
|
.ftol = 1e-4f,
|
|
.wolfe = 0.9f,
|
|
.min_step = 1e-20f,
|
|
.max_step = 1e+20f,
|
|
|
|
.linesearch = GGML_LINESEARCH_DEFAULT,
|
|
},
|
|
};
|
|
} break;
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
enum ggml_opt_result ggml_opt(
|
|
struct ggml_context * ctx,
|
|
struct ggml_opt_params params,
|
|
struct ggml_tensor * f) {
|
|
bool free_ctx = false;
|
|
if (ctx == NULL) {
|
|
struct ggml_init_params params_ctx = {
|
|
.mem_size = 16*1024*1024,
|
|
.mem_buffer = NULL,
|
|
};
|
|
|
|
ctx = ggml_init(params_ctx);
|
|
if (ctx == NULL) {
|
|
return GGML_OPT_NO_CONTEXT;
|
|
}
|
|
|
|
free_ctx = true;
|
|
}
|
|
|
|
enum ggml_opt_result result = GGML_OPT_OK;
|
|
|
|
// build forward + backward compute graphs
|
|
struct ggml_cgraph gf = ggml_build_forward (f);
|
|
struct ggml_cgraph gb = ggml_build_backward(ctx, &gf, false);
|
|
|
|
switch (params.type) {
|
|
case GGML_OPT_ADAM:
|
|
{
|
|
result = ggml_opt_adam(ctx, params, f, &gf, &gb);
|
|
} break;
|
|
case GGML_OPT_LBFGS:
|
|
{
|
|
result = ggml_opt_lbfgs(ctx, params, f, &gf, &gb);
|
|
} break;
|
|
}
|
|
|
|
if (params.print_forward_graph) {
|
|
ggml_graph_print (&gf);
|
|
ggml_graph_dump_dot(&gf, NULL, "opt-forward.dot");
|
|
}
|
|
|
|
if (params.print_backward_graph) {
|
|
ggml_graph_print (&gb);
|
|
ggml_graph_dump_dot(&gb, &gf, "opt-backward.dot");
|
|
}
|
|
|
|
if (free_ctx) {
|
|
ggml_free(ctx);
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
////////////////////////////////////////////////////////////////////////////////
|