Commit de5122d87 for llama.cpp
commit de5122d879d00d2895c36be7e0cee94105aeeabd
Author: Georgi Gerganov <ggerganov@gmail.com>
Date: Tue Oct 27 07:55:31 2026 +0200
ggml : relax scale to non-contiguous views (#31146)
* ggml : relax scale to non-contiguous views
SCALE requires src0 to be 1D-contiguous. Relax this to dim-0
contiguity with arbitrary row strides (ggml_is_contiguous_rows),
same as the unary ops in the Metal backend.
- ggml: relax the assert in ggml_scale_impl
- cpu: address src0 rows through their byte strides (f32, bf16)
- cuda: stride-aware kernel, fastdiv row decomposition
- metal: support bf16 scale (guarded unary instantiations)
- sycl: keep the gate at contiguous, the kernel is flat f32
Add view variants to test_scale. Verified on CPU, Metal and CUDA.
Assisted-by: dsh:llama.cpp/ggml-org/Qwen3.8-Flash-Next-GGUF (RTX 5090 32GB, DDR5 128GB)
* cuda : skip the softcap fusion on non-contiguous views
The fused softcap kernel reads src and writes dst as dense arrays,
so a view reaching SCALE -> TANH -> SCALE now that SCALE accepts
non-contiguous rows would be read wrongly. Keep such graphs on the
strided kernels until the fusion becomes stride aware.
---------
Co-authored-by: Pascal <admin@serveurperso.com>
diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp
index afeda2aff..85e17018c 100644
--- a/ggml/src/ggml-cpu/ops.cpp
+++ b/ggml/src/ggml-cpu/ops.cpp
@@ -4846,8 +4846,8 @@ static void ggml_compute_forward_scale_f32(
const ggml_tensor * src0 = dst->src[0];
- GGML_ASSERT(ggml_is_contiguous(src0));
- GGML_ASSERT(ggml_is_contiguous(dst));
+ GGML_ASSERT(ggml_is_contiguous_rows(src0));
+ GGML_ASSERT(ggml_is_contiguous_rows(dst));
GGML_ASSERT(ggml_are_same_shape(src0, dst));
float s; // scale factor
@@ -4862,6 +4862,8 @@ static void ggml_compute_forward_scale_f32(
const int nc = src0->ne[0];
const int nr = ggml_nrows(src0);
+ GGML_TENSOR_UNARY_OP_LOCALS
+
// rows per thread
const int dr = (nr + nth - 1)/nth;
@@ -4869,25 +4871,23 @@ static void ggml_compute_forward_scale_f32(
const int ir0 = dr*ith;
const int ir1 = MIN(ir0 + dr, nr);
- const size_t nb01 = src0->nb[1];
+ for (int i = ir0; i < ir1; i++) {
+ const int64_t i1 = i % ne01;
+ const int64_t i2 = (i / ne01) % ne02;
+ const int64_t i3 = i / (ne01*ne02);
- const size_t nb1 = dst->nb[1];
+ float * y = (float *) ((char *) dst->data + i1*nb1 + i2*nb2 + i3*nb3);
+ const float * x = (const float *) ((char *) src0->data + i1*nb01 + i2*nb02 + i3*nb03);
- if (b == 0.0f) {
- for (int i1 = ir0; i1 < ir1; i1++) {
+ if (b == 0.0f) {
if (dst->data != src0->data) {
// src0 is same shape as dst => same indices
// TODO: add x parameter to ggml_vec_scale_f32 and remove this memcpy
- memcpy((char *)dst->data + i1*nb1, (char *)src0->data + i1*nb01, nc * sizeof(float));
+ memcpy(y, x, nc * sizeof(float));
}
- ggml_vec_scale_f32(nc, (float *) ((char *) dst->data + i1*nb1), s);
- }
- } else {
- for (int i1 = ir0; i1 < ir1; i1++) {
- ggml_vec_mad1_f32(nc,
- (float *) ((char *) dst->data + i1*nb1),
- (float *) ((char *) src0->data + i1*nb1),
- s, b);
+ ggml_vec_scale_f32(nc, y, s);
+ } else {
+ ggml_vec_mad1_f32(nc, y, x, s, b);
}
}
}
@@ -4898,8 +4898,8 @@ static void ggml_compute_forward_scale_bf16(
const ggml_tensor * src0 = dst->src[0];
- GGML_ASSERT(ggml_is_contiguous(src0));
- GGML_ASSERT(ggml_is_contiguous(dst));
+ GGML_ASSERT(ggml_is_contiguous_rows(src0));
+ GGML_ASSERT(ggml_is_contiguous_rows(dst));
GGML_ASSERT(ggml_are_same_shape(src0, dst));
float s; // scale factor
@@ -4914,6 +4914,8 @@ static void ggml_compute_forward_scale_bf16(
const int nc = src0->ne[0];
const int nr = ggml_nrows(src0);
+ GGML_TENSOR_UNARY_OP_LOCALS
+
// rows per thread
const int dr = (nr + nth - 1)/nth;
@@ -4921,9 +4923,13 @@ static void ggml_compute_forward_scale_bf16(
const int ir0 = dr*ith;
const int ir1 = MIN(ir0 + dr, nr);
- for (int i1 = ir0; i1 < ir1; i1++) {
- const ggml_bf16_t * x = (const ggml_bf16_t *) ((const char *) src0->data + i1*src0->nb[1]);
- ggml_bf16_t * y = (ggml_bf16_t *) ((char *) dst->data + i1*dst->nb[1]);
+ for (int i = ir0; i < ir1; i++) {
+ const int64_t i1 = i % ne01;
+ const int64_t i2 = (i / ne01) % ne02;
+ const int64_t i3 = i / (ne01*ne02);
+
+ const ggml_bf16_t * x = (const ggml_bf16_t *) ((const char *) src0->data + i1*nb01 + i2*nb02 + i3*nb03);
+ ggml_bf16_t * y = (ggml_bf16_t *) ((char *) dst->data + i1*nb1 + i2*nb2 + i3*nb3);
for (int k = 0; k < nc; k++) {
y[k] = GGML_FP32_TO_BF16(s*GGML_BF16_TO_FP32(x[k]) + b);
}
diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
index b795951b2..aba21524f 100644
--- a/ggml/src/ggml-cuda/ggml-cuda.cu
+++ b/ggml/src/ggml-cuda/ggml-cuda.cu
@@ -3822,6 +3822,11 @@ static bool ggml_cuda_can_fuse(const struct ggml_cgraph * cgraph,
return false;
}
+ // TODO: make the fused softcap kernel stride aware, it reads src and writes dst as dense
+ if (!ggml_is_contiguous(scale->src[0]) || !ggml_is_contiguous(scale2)) {
+ return false;
+ }
+
if (ggml_get_unary_op(tanh) != GGML_UNARY_OP_TANH) {
return false;
}
@@ -5972,7 +5977,8 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
case GGML_OP_LOG:
return true;
case GGML_OP_SCALE:
- return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_BF16) && op->type == op->src[0]->type;
+ return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_BF16) && op->type == op->src[0]->type &&
+ ggml_is_contiguous_rows(op->src[0]);
case GGML_OP_CLAMP:
return ggml_is_contiguous_rows(op->src[0]);
case GGML_OP_ADD:
diff --git a/ggml/src/ggml-cuda/scale.cu b/ggml/src/ggml-cuda/scale.cu
index 5648b9261..d53f906c5 100644
--- a/ggml/src/ggml-cuda/scale.cu
+++ b/ggml/src/ggml-cuda/scale.cu
@@ -3,23 +3,45 @@
#define MAX_GRIDDIM_X 0x7FFFFFFF
+// src and dst may be views: rows are contiguous, dims 1..3 follow the strides (in elements).
template <typename T>
-static __global__ void scale_f32(const T * x, T * dst, const float scale, const float bias, const int64_t nelements) {
+static __global__ void scale_f32(const T * x, T * dst, const float scale, const float bias, const uint32_t k,
+ const uint3 ne0, const uint3 ne1, const uint3 ne2,
+ const uint32_t s01, const uint32_t s02, const uint32_t s03,
+ const uint32_t s1, const uint32_t s2, const uint32_t s3) {
ggml_cuda_pdl_lc();
- int64_t tid = (int64_t)blockIdx.x * (int64_t)blockDim.x + (int64_t)threadIdx.x;
- int64_t stride = (int64_t)blockDim.x * (int64_t)gridDim.x;
+ const uint32_t i = blockIdx.x * blockDim.x + threadIdx.x;
ggml_cuda_pdl_sync();
- for (int64_t i = tid; i < nelements; i += stride) {
- dst[i] = ggml_cuda_cast<T>(scale * ggml_cuda_cast<float>(x[i]) + bias);
+ if (i >= k) {
+ return;
}
+
+ const uint2 d0 = fast_div_modulo(i, ne0); // <i / ne0, i0>
+ const uint2 d1 = fast_div_modulo(d0.x, ne1); // <i / (ne0*ne1), i1>
+ const uint2 d2 = fast_div_modulo(d1.x, ne2); // <i3, i2>
+
+ const size_t i_src = d0.y + size_t(d1.y)*s01 + size_t(d2.y)*s02 + size_t(d2.x)*s03;
+ const size_t i_dst = d0.y + size_t(d1.y)*s1 + size_t(d2.y)*s2 + size_t(d2.x)*s3;
+
+ dst[i_dst] = ggml_cuda_cast<T>(scale * ggml_cuda_cast<float>(x[i_src]) + bias);
}
template <typename T>
-static void scale_f32_cuda(const T * x, T * dst, const float scale, const float bias, const int64_t nelements, cudaStream_t stream) {
- const int64_t num_blocks = (nelements + CUDA_SCALE_BLOCK_SIZE - 1) / CUDA_SCALE_BLOCK_SIZE;
+static void scale_f32_cuda(const T * x, T * dst, const float scale, const float bias, const ggml_tensor * src0, const ggml_tensor * t, cudaStream_t stream) {
+ const int64_t k = ggml_nelements(src0);
+ const size_t ts = sizeof(T);
+ GGML_ASSERT(k <= std::numeric_limits<uint32_t>::max());
+
+ const uint3 ne0 = init_fastdiv_values(src0->ne[0]);
+ const uint3 ne1 = init_fastdiv_values(src0->ne[1]);
+ const uint3 ne2 = init_fastdiv_values(src0->ne[2]);
+
+ const int64_t num_blocks = (k + CUDA_SCALE_BLOCK_SIZE - 1) / CUDA_SCALE_BLOCK_SIZE;
const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(MIN(MAX_GRIDDIM_X, num_blocks), CUDA_SCALE_BLOCK_SIZE, 0, stream);
- ggml_cuda_kernel_launch(scale_f32<T>, launch_params, x, dst, scale, bias, nelements);
+ ggml_cuda_kernel_launch(scale_f32<T>, launch_params, x, dst, scale, bias, (uint32_t) k, ne0, ne1, ne2,
+ src0->nb[1]/ts, src0->nb[2]/ts, src0->nb[3]/ts,
+ t->nb[1]/ts, t->nb[2]/ts, t->nb[3]/ts);
}
void ggml_cuda_op_scale(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
@@ -28,6 +50,7 @@ void ggml_cuda_op_scale(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_BF16);
GGML_ASSERT(dst->type == src0->type);
+ GGML_ASSERT(ggml_is_contiguous_rows(src0) && ggml_is_contiguous_rows(dst));
float scale;
float bias;
@@ -35,8 +58,8 @@ void ggml_cuda_op_scale(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
memcpy(&bias, (float *) dst->op_params + 1, sizeof(float));
if (src0->type == GGML_TYPE_BF16) {
- scale_f32_cuda((const nv_bfloat16 *) src0->data, (nv_bfloat16 *) dst->data, scale, bias, ggml_nelements(src0), stream);
+ scale_f32_cuda((const nv_bfloat16 *) src0->data, (nv_bfloat16 *) dst->data, scale, bias, src0, dst, stream);
} else {
- scale_f32_cuda((const float *) src0->data, (float *) dst->data, scale, bias, ggml_nelements(src0), stream);
+ scale_f32_cuda((const float *) src0->data, (float *) dst->data, scale, bias, src0, dst, stream);
}
}
diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.m
index d1bce8bad..31ffe0585 100644
--- a/ggml/src/ggml-metal/ggml-metal-device.m
+++ b/ggml/src/ggml-metal/ggml-metal-device.m
@@ -1555,6 +1555,7 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
switch (op->op) {
case GGML_OP_SCALE:
+ return ggml_is_contiguous_rows(op->src[0]) && (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_BF16);
case GGML_OP_FILL:
case GGML_OP_CLAMP:
case GGML_OP_SQR:
diff --git a/ggml/src/ggml-metal/kernels/unary.metal b/ggml/src/ggml-metal/kernels/unary.metal
index e50a64863..8c2c18ebd 100644
--- a/ggml/src/ggml-metal/kernels/unary.metal
+++ b/ggml/src/ggml-metal/kernels/unary.metal
@@ -3,6 +3,24 @@
constant short FC_unary_op [[function_constant(FC_UNARY + 0)]];
constant bool FC_unary_cnt[[function_constant(FC_UNARY + 1)]];
+template <typename T, typename U>
+inline constexpr T ggml_metal_unary_cast(U x) {
+ return (T) x;
+}
+
+#if defined(GGML_METAL_HAS_BF16)
+// MSL has no C-style cast from float vectors to bfloat vectors
+template <>
+inline constexpr bfloat4 ggml_metal_unary_cast<bfloat4, float4>(float4 x) {
+ return bfloat4(bfloat(x[0]), bfloat(x[1]), bfloat(x[2]), bfloat(x[3]));
+}
+
+template <>
+inline constexpr bfloat4 ggml_metal_unary_cast<bfloat4, float>(float x) {
+ return bfloat4(bfloat(x), bfloat(x), bfloat(x), bfloat(x));
+}
+#endif
+
template <typename T0, typename T, typename TC>
kernel void kernel_unary_impl(
constant ggml_metal_kargs_unary & args,
@@ -48,79 +66,79 @@ kernel void kernel_unary_impl(
const TC x = (TC) src0_ptr[i0];
if (FC_OP == OP_UNARY_NUM_SCALE) {
- dst_ptr[i0] = (T) (args.scale * x + args.bias);
+ dst_ptr[i0] = ggml_metal_unary_cast<T>(args.scale * x + args.bias);
}
if (FC_OP == OP_UNARY_NUM_FILL) {
- dst_ptr[i0] = (T) args.val;
+ dst_ptr[i0] = ggml_metal_unary_cast<T>(args.val);
}
if (FC_OP == OP_UNARY_NUM_CLAMP) {
- dst_ptr[i0] = (T) clamp(x, args.min, args.max);
+ dst_ptr[i0] = ggml_metal_unary_cast<T>(clamp(x, args.min, args.max));
}
if (FC_OP == OP_UNARY_NUM_SQR) {
- dst_ptr[i0] = (T) (x * x);
+ dst_ptr[i0] = ggml_metal_unary_cast<T>(x * x);
}
if (FC_OP == OP_UNARY_NUM_SQRT) {
- dst_ptr[i0] = (T) sqrt(x);
+ dst_ptr[i0] = ggml_metal_unary_cast<T>(sqrt(x));
}
if (FC_OP == OP_UNARY_NUM_SIN) {
- dst_ptr[i0] = (T) sin(x);
+ dst_ptr[i0] = ggml_metal_unary_cast<T>(sin(x));
}
if (FC_OP == OP_UNARY_NUM_COS) {
- dst_ptr[i0] = (T) cos(x);
+ dst_ptr[i0] = ggml_metal_unary_cast<T>(cos(x));
}
if (FC_OP == OP_UNARY_NUM_LOG) {
- dst_ptr[i0] = (T) log(x);
+ dst_ptr[i0] = ggml_metal_unary_cast<T>(log(x));
}
if (FC_OP == OP_UNARY_NUM_LEAKY_RELU) {
- dst_ptr[i0] = (T) (TC(x > 0)*x + TC(x <= 0)*(x * args.slope));
+ dst_ptr[i0] = ggml_metal_unary_cast<T>(TC(x > 0)*x + TC(x <= 0)*(x * args.slope));
}
if (FC_OP == OP_UNARY_NUM_TANH) {
- dst_ptr[i0] = (T) precise::tanh(x);
+ dst_ptr[i0] = ggml_metal_unary_cast<T>(precise::tanh(x));
}
if (FC_OP == OP_UNARY_NUM_RELU) {
- dst_ptr[i0] = (T) fmax(0, x);
+ dst_ptr[i0] = ggml_metal_unary_cast<T>(fmax(0, x));
}
if (FC_OP == OP_UNARY_NUM_SIGMOID) {
- dst_ptr[i0] = (T) (1 / (1 + exp(-x)));
+ dst_ptr[i0] = ggml_metal_unary_cast<T>(1 / (1 + exp(-x)));
}
if (FC_OP == OP_UNARY_NUM_GELU) {
- dst_ptr[i0] = (T) (0.5*x*(1 + precise::tanh(SQRT_2_OVER_PI*x*(1 + GELU_COEF_A*x*x))));
+ dst_ptr[i0] = ggml_metal_unary_cast<T>(0.5*x*(1 + precise::tanh(SQRT_2_OVER_PI*x*(1 + GELU_COEF_A*x*x))));
}
if (FC_OP == OP_UNARY_NUM_GELU_ERF) {
- dst_ptr[i0] = (T) (0.5*x*(1 + erf_approx(SQRT_2_INV*x)));
+ dst_ptr[i0] = ggml_metal_unary_cast<T>(0.5*x*(1 + erf_approx(SQRT_2_INV*x)));
}
if (FC_OP == OP_UNARY_NUM_GELU_QUICK) {
- dst_ptr[i0] = (T) (x * (1/(1 + exp(GELU_QUICK_COEF*x))));
+ dst_ptr[i0] = ggml_metal_unary_cast<T>(x * (1/(1 + exp(GELU_QUICK_COEF*x))));
}
if (FC_OP == OP_UNARY_NUM_SILU) {
- dst_ptr[i0] = (T) (x / (1 + exp(-x)));
+ dst_ptr[i0] = ggml_metal_unary_cast<T>(x / (1 + exp(-x)));
}
if (FC_OP == OP_UNARY_NUM_ELU) {
- dst_ptr[i0] = (T) elu_approx(x);
+ dst_ptr[i0] = ggml_metal_unary_cast<T>(elu_approx(x));
}
if (FC_OP == OP_UNARY_NUM_NEG) {
- dst_ptr[i0] = (T) -x;
+ dst_ptr[i0] = ggml_metal_unary_cast<T>(-x);
}
if (FC_OP == OP_UNARY_NUM_ABS) {
- dst_ptr[i0] = (T) fabs(x);
+ dst_ptr[i0] = ggml_metal_unary_cast<T>(fabs(x));
}
if (FC_OP == OP_UNARY_NUM_SGN) {
@@ -132,40 +150,40 @@ kernel void kernel_unary_impl(
}
if (FC_OP == OP_UNARY_NUM_HARDSWISH) {
- dst_ptr[i0] = (T) (x * fmax(0, fmin(1, x/6 + 0.5)));
+ dst_ptr[i0] = ggml_metal_unary_cast<T>(x * fmax(0, fmin(1, x/6 + 0.5)));
}
if (FC_OP == OP_UNARY_NUM_HARDSIGMOID) {
- dst_ptr[i0] = (T) fmax(0, fmin(1, x/6 + 0.5));
+ dst_ptr[i0] = ggml_metal_unary_cast<T>(fmax(0, fmin(1, x/6 + 0.5)));
}
if (FC_OP == OP_UNARY_NUM_EXP) {
- dst_ptr[i0] = (T) exp(x);
+ dst_ptr[i0] = ggml_metal_unary_cast<T>(exp(x));
}
if (FC_OP == OP_UNARY_NUM_SOFTPLUS) {
- dst_ptr[i0] = (T) select(log(1 + exp(x)), x, x > 20);
+ dst_ptr[i0] = ggml_metal_unary_cast<T>(select(log(1 + exp(x)), x, x > 20));
}
if (FC_OP == OP_UNARY_NUM_EXPM1) {
// TODO: precise implementation
- dst_ptr[i0] = (T) (exp(x) - 1);
+ dst_ptr[i0] = ggml_metal_unary_cast<T>(exp(x) - 1);
}
if (FC_OP == OP_UNARY_NUM_FLOOR) {
- dst_ptr[i0] = (T) floor(x);
+ dst_ptr[i0] = ggml_metal_unary_cast<T>(floor(x));
}
if (FC_OP == OP_UNARY_NUM_CEIL) {
- dst_ptr[i0] = (T) ceil(x);
+ dst_ptr[i0] = ggml_metal_unary_cast<T>(ceil(x));
}
if (FC_OP == OP_UNARY_NUM_ROUND) {
- dst_ptr[i0] = (T) round(x);
+ dst_ptr[i0] = ggml_metal_unary_cast<T>(round(x));
}
if (FC_OP == OP_UNARY_NUM_TRUNC) {
- dst_ptr[i0] = (T) trunc(x);
+ dst_ptr[i0] = ggml_metal_unary_cast<T>(trunc(x));
}
if (FC_OP == OP_UNARY_NUM_XIELU) {
@@ -174,7 +192,7 @@ kernel void kernel_unary_impl(
const TC clamped = fmin(xi, TC(args.val));
const TC y_pos = TC(args.scale) * xi * xi + TC(args.bias) * xi;
const TC y_neg = (exp(clamped) - TC(1.0f) - xi) * TC(args.slope) + TC(args.bias) * xi;
- dst_ptr[i0] = (T) (gate * y_pos + (TC(1.0f) - gate) * y_neg);
+ dst_ptr[i0] = ggml_metal_unary_cast<T>(gate * y_pos + (TC(1.0f) - gate) * y_neg);
}
}
@@ -188,6 +206,10 @@ template [[host_name("kernel_unary_f32_f32")]] kernel kernel_unary_t kernel_un
template [[host_name("kernel_unary_f32_f32_4")]] kernel kernel_unary_t kernel_unary_impl<float4, float4, float4>;
template [[host_name("kernel_unary_f16_f16")]] kernel kernel_unary_t kernel_unary_impl<half, half, float>;
template [[host_name("kernel_unary_f16_f16_4")]] kernel kernel_unary_t kernel_unary_impl<half4, half4, float4>;
+#if defined(GGML_METAL_HAS_BF16)
+template [[host_name("kernel_unary_bf16_bf16")]] kernel kernel_unary_t kernel_unary_impl<bfloat, bfloat, float>;
+template [[host_name("kernel_unary_bf16_bf16_4")]] kernel kernel_unary_t kernel_unary_impl<bfloat4, bfloat4, float4>;
+#endif
kernel void kernel_silu_back_f32(
constant ggml_metal_kargs_silu_back & args,
diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp
index ae0f291c4..97ad492e5 100644
--- a/ggml/src/ggml-sycl/ggml-sycl.cpp
+++ b/ggml/src/ggml-sycl/ggml-sycl.cpp
@@ -7503,8 +7503,8 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons
case GGML_OP_RMS_NORM_BACK:
return ggml_is_contiguous(op->src[0]);
case GGML_OP_SCALE:
- // the scale kernel is f32 only
- return op->type == GGML_TYPE_F32 && op->src[0]->type == GGML_TYPE_F32;
+ // the scale kernel is f32 only, and reads src0 flat
+ return op->type == GGML_TYPE_F32 && op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]);
case GGML_OP_CONT:
return true;
case GGML_OP_TRI:
diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c
index ed8554b14..5d8449a88 100644
--- a/ggml/src/ggml.c
+++ b/ggml/src/ggml.c
@@ -3503,7 +3503,7 @@ static struct ggml_tensor * ggml_scale_impl(
float s,
float b,
bool inplace) {
- GGML_ASSERT(ggml_is_padded_1d(a));
+ GGML_ASSERT(ggml_is_contiguous_rows(a));
struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index 011b74613..0ccde35eb 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -3679,23 +3679,37 @@ struct test_scale : public test_case {
float scale;
float bias;
bool inplace;
+ int v; // view (1 : non-contiguous a)
std::string vars() override {
- return VARS_TO_STR5(type, ne, scale, bias, inplace);
+ return VARS_TO_STR6(type, ne, scale, bias, inplace, v);
}
test_scale(ggml_type type = GGML_TYPE_F32,
std::array<int64_t, 4> ne = {10, 10, 10, 10},
float scale = 2.0f,
float bias = 0.0f,
- bool inplace = false)
- : type(type), ne(ne), scale(scale), bias(bias), inplace(inplace) {}
+ bool inplace = false,
+ int v = 0)
+ : type(type), ne(ne), scale(scale), bias(bias), inplace(inplace), v(v) {}
ggml_tensor * build_graph(ggml_context * ctx) override {
- ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data());
+ auto ne_a = ne;
+ if (v & 1) {
+ ne_a[0] *= 3;
+ ne_a[1] *= 2;
+ ne_a[2] *= 5;
+ ne_a[3] *= 4;
+ }
+ ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne_a.data());
ggml_set_param(a);
ggml_set_name(a, "a");
+ if (v & 1) {
+ a = ggml_view_4d(ctx, a, ne[0], ne[1], ne[2], ne[3], a->nb[1], a->nb[2], a->nb[3], 0);
+ ggml_set_name(a, "view_of_a");
+ }
+
ggml_tensor * out;
if (inplace) {
out = ggml_scale_bias_inplace(ctx, a, scale, bias);
@@ -10861,6 +10875,10 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_scale(GGML_TYPE_BF16, {10, 10, 10, 10}, 2.0f, 1.0f));
test_cases.emplace_back(new test_scale(GGML_TYPE_BF16, {10, 10, 10, 10}, 2.0f, 1.0f, true)); // inplace test
test_cases.emplace_back(new test_scale(GGML_TYPE_BF16, {100, 10, 10, 10}, 0.5f, 0.0f));
+ test_cases.emplace_back(new test_scale(GGML_TYPE_F32, {10, 10, 10, 10}, 2.0f, 0.0f, false, 1)); // view test
+ test_cases.emplace_back(new test_scale(GGML_TYPE_F32, {10, 10, 10, 10}, 2.0f, 1.0f, false, 1)); // view test
+ test_cases.emplace_back(new test_scale(GGML_TYPE_F32, {10, 10, 10, 10}, 2.0f, 1.0f, true, 1)); // view test, inplace
+ test_cases.emplace_back(new test_scale(GGML_TYPE_BF16, {10, 10, 10, 10}, 2.0f, 1.0f, true, 1)); // view test, inplace
test_cases.emplace_back(new test_softcap(GGML_TYPE_F32, {10, 10, 10, 10}, 50.0f));
test_cases.emplace_back(new test_silu_back());