Commit 970b51ce7 for llama.cpp
commit 970b51ce7c9ba5fff0726c5e9994f1c2f0e4eedc
Author: kurquhar <kurquhar@qti.qualcomm.com>
Date: Thu Oct 22 10:54:24 2026 -0700
opencl: add depthwise conv2d (#30900)
* opencl: add Adreno depthwise conv2d
Assisted-by: OpenCode
* opencl: use local program for depthwise conv2d
Assisted-by: OpenCode
* opencl: use singed type for args
---------
Co-authored-by: Li He <lih@qti.qualcomm.com>
diff --git a/ggml/src/ggml-opencl/CMakeLists.txt b/ggml/src/ggml-opencl/CMakeLists.txt
index 977013b94..1c6edc036 100644
--- a/ggml/src/ggml-opencl/CMakeLists.txt
+++ b/ggml/src/ggml-opencl/CMakeLists.txt
@@ -241,6 +241,7 @@ set(GGML_OPENCL_KERNELS
mul_mm_f16_f32_kq_kqv
conv2d
conv2d_f16_f32
+ conv2_dw
flash_attn_repack
flash_attn_pre_f16
flash_attn_f32_f16
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp
index d56b3488d..1fafe8e33 100644
--- a/ggml/src/ggml-opencl/ggml-opencl.cpp
+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp
@@ -1005,6 +1005,7 @@ struct ggml_backend_opencl_context {
cl_kernel kernel_conv_2d_f16;
cl_kernel kernel_conv_2d_f32;
cl_kernel kernel_conv_2d_f16_f32;
+ cl_kernel kernel_conv2d_dw_f32_f32, kernel_conv2d_dwf16_f32;
cl_kernel kernel_ssm_conv_f32_f32, kernel_ssm_conv_f32_f32_4;
// [size_idx][kda][tgpp] where size_idx: 0=S_V=16, 1=32, 2=64, 3=128; kda: 0 or 1.
// tgpp 0 = TG variant (COLS_PER_LANE_GROUP=1), tgpp 1 = prefill variant (COLS_PER_LANE_GROUP=4).
@@ -3761,6 +3762,23 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
}
}
+ // conv2_dw
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "conv2_dw.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("conv2_dw.cl");
+#endif
+ cl_program prog =
+ build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
+ CL_CHECK((backend_ctx->kernel_conv2d_dw_f32_f32 = clCreateKernel(prog, "kernel_conv2d_dw_f32_f32", &err), err));
+ CL_CHECK((backend_ctx->kernel_conv2d_dwf16_f32 = clCreateKernel(prog, "kernel_conv2d_dwf16_f32", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ GGML_LOG_CONT(".");
+ }
+
// ssm_conv
{
#ifdef GGML_OPENCL_EMBED_KERNELS
@@ -9572,6 +9590,11 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
return (op->src[0]->type == GGML_TYPE_F16 && op->src[1]->type == GGML_TYPE_F16 && op->type == GGML_TYPE_F16) ||
(op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32) ||
(op->src[0]->type == GGML_TYPE_F16 && op->src[1]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32);
+ case GGML_OP_CONV_2D_DW:
+ return op->src[1]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32 &&
+ op->src[0]->nb[0] == ggml_type_size(op->src[0]->type) &&
+ op->src[1]->nb[0] == sizeof(float) && op->nb[0] == sizeof(float) &&
+ (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16);
case GGML_OP_SSM_CONV:
return (op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32);
case GGML_OP_SSM_SCAN: {
@@ -19715,6 +19738,90 @@ static void ggml_cl_conv_2d(ggml_backend_t backend, const ggml_tensor * src0, co
backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size, local_work_size, dst);
}
+static void ggml_cl_conv2_dw(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
+ GGML_TENSOR_BINARY_OP_LOCALS;
+ ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *) backend->context;
+
+ ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *) src0->extra;
+ ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *) src1->extra;
+ ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *) dst->extra;
+
+ const cl_long offset0 = extra0->offset + src0->view_offs;
+ const cl_long offset1 = extra1->offset + src1->view_offs;
+ const cl_long offsetd = extrad->offset + dst->view_offs;
+
+ const cl_long C = ne12;
+ const cl_long N = ne13;
+ const cl_long KW = ne00;
+ const cl_long KH = ne01;
+ const cl_long W = ne10;
+ const cl_long H = ne11;
+ const cl_long OW = ne0;
+ const cl_long OH = ne1;
+
+ const cl_int s0 = dst->op_params[0];
+ const cl_int s1 = dst->op_params[1];
+ const cl_int p0 = dst->op_params[2];
+ const cl_int p1 = dst->op_params[3];
+ const cl_int d0 = dst->op_params[4];
+ const cl_int d1 = dst->op_params[5];
+
+ const cl_long cl_nb01 = nb01/ggml_type_size(src0->type);
+ const cl_long cl_nb03 = nb03/ggml_type_size(src0->type);
+ const cl_long cl_nb11 = nb11/ggml_type_size(src1->type);
+ const cl_long cl_nb12 = nb12/ggml_type_size(src1->type);
+ const cl_long cl_nb13 = nb13/ggml_type_size(src1->type);
+ const cl_long cl_nb1 = nb1/ggml_type_size(dst->type);
+ const cl_long cl_nb2 = nb2/ggml_type_size(dst->type);
+ const cl_long cl_nb3 = nb3/ggml_type_size(dst->type);
+
+ cl_kernel kernel;
+ if (src0->type == GGML_TYPE_F32) {
+ kernel = backend_ctx->kernel_conv2d_dw_f32_f32;
+ } else if (src0->type == GGML_TYPE_F16) {
+ kernel = backend_ctx->kernel_conv2d_dwf16_f32;
+ } else {
+ GGML_ASSERT(false && "Unsupported data type combination for conv2_dw");
+ }
+
+ cl_uint idx = 0;
+ CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_mem), &extra0->data_device));
+ CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &offset0));
+ CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_mem), &extra1->data_device));
+ CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &offset1));
+ CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &offsetd));
+ CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &C));
+ CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &N));
+ CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &KW));
+ CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &KH));
+ CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &W));
+ CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &H));
+ CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &OW));
+ CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &OH));
+ CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_int), &s0));
+ CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_int), &s1));
+ CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_int), &p0));
+ CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_int), &p1));
+ CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_int), &d0));
+ CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_int), &d1));
+ CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &cl_nb01));
+ CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &cl_nb03));
+ CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &cl_nb11));
+ CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &cl_nb12));
+ CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &cl_nb13));
+ CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &cl_nb1));
+ CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &cl_nb2));
+ CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &cl_nb3));
+
+ const size_t total = (size_t) N * C * OH * OW;
+ const size_t local_size = MIN((size_t) 256, backend_ctx->get_kernel_workgroup_size(kernel));
+ size_t global_work_size[] = { CEIL_DIV(total, local_size) * local_size };
+ size_t local_work_size[] = { local_size };
+
+ backend_ctx->enqueue_ndrange_kernel(kernel, 1, global_work_size, local_work_size, dst);
+}
+
// is_kq selects which of the two products this call is, and it is decided by the
// CALLER -- the two admission arms in ggml_cl_mul_mat, each of which knows which
// one it matched. It used to be re-derived here from nb01 > nb02, i.e. "K is
@@ -31030,6 +31137,12 @@ bool ggml_cl_compute_forward(ggml_backend_t backend, struct ggml_tensor * tensor
}
func = ggml_cl_conv_2d;
break;
+ case GGML_OP_CONV_2D_DW:
+ if (!any_on_device) {
+ return false;
+ }
+ func = ggml_cl_conv2_dw;
+ break;
case GGML_OP_SSM_CONV:
if (!any_on_device) {
return false;
diff --git a/ggml/src/ggml-opencl/kernels/conv2_dw.cl b/ggml/src/ggml-opencl/kernels/conv2_dw.cl
new file mode 100644
index 000000000..2a35b9c96
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/conv2_dw.cl
@@ -0,0 +1,139 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+kernel void kernel_conv2d_dw_f32_f32(
+ global const float * weights,
+ long weights_offset,
+ global const float * input,
+ long input_offset,
+ global float * output,
+ long output_offset,
+ long C,
+ long N,
+ long KW,
+ long KH,
+ long W,
+ long H,
+ long OW,
+ long OH,
+ int s0,
+ int s1,
+ int p0,
+ int p1,
+ int d0,
+ int d1,
+ long nb01,
+ long nb03,
+ long nb11,
+ long nb12,
+ long nb13,
+ long nb1,
+ long nb2,
+ long nb3
+) {
+ weights = (global const float *) ((global const char *) weights + weights_offset);
+ input = (global const float *) ((global const char *) input + input_offset);
+ output = (global float *) ((global char *) output + output_offset);
+
+ const long gid = get_global_id(0);
+ const long OHW = OH * OW;
+ const long COHW = C * OHW;
+ const long total = N * COHW;
+ if (gid >= total) {
+ return;
+ }
+
+ const long n = gid / COHW;
+ const long c_rem = gid - n * COHW;
+ const long c = c_rem / OHW;
+ const long oh = (c_rem - c * OHW) / OW;
+ const long ow = c_rem - c * OHW - oh * OW;
+
+ float sum = 0.0f;
+ for (long kh = 0; kh < KH; ++kh) {
+ const long ih = oh * s1 + kh * d1 - p1;
+ if (ih < 0 || ih >= H) {
+ continue;
+ }
+ for (long kw = 0; kw < KW; ++kw) {
+ const long iw = ow * s0 + kw * d0 - p0;
+ if (iw < 0 || iw >= W) {
+ continue;
+ }
+ const long weight_idx = kw + kh * nb01 + c * nb03;
+ const long input_idx = iw + ih * nb11 + c * nb12 + n * nb13;
+ sum += weights[weight_idx] * input[input_idx];
+ }
+ }
+
+ const long output_idx = ow + oh * nb1 + c * nb2 + n * nb3;
+ output[output_idx] = sum;
+}
+
+kernel void kernel_conv2d_dwf16_f32(
+ global const half * weights,
+ long weights_offset,
+ global const float * input,
+ long input_offset,
+ global float * output,
+ long output_offset,
+ long C,
+ long N,
+ long KW,
+ long KH,
+ long W,
+ long H,
+ long OW,
+ long OH,
+ int s0,
+ int s1,
+ int p0,
+ int p1,
+ int d0,
+ int d1,
+ long nb01,
+ long nb03,
+ long nb11,
+ long nb12,
+ long nb13,
+ long nb1,
+ long nb2,
+ long nb3
+) {
+ weights = (global const half *) ((global const char *) weights + weights_offset);
+ input = (global const float *) ((global const char *) input + input_offset);
+ output = (global float *) ((global char *) output + output_offset);
+
+ const long gid = get_global_id(0);
+ const long OHW = OH * OW;
+ const long COHW = C * OHW;
+ const long total = N * COHW;
+ if (gid >= total) {
+ return;
+ }
+
+ const long n = gid / COHW;
+ const long c_rem = gid - n * COHW;
+ const long c = c_rem / OHW;
+ const long oh = (c_rem - c * OHW) / OW;
+ const long ow = c_rem - c * OHW - oh * OW;
+
+ float sum = 0.0f;
+ for (long kh = 0; kh < KH; ++kh) {
+ const long ih = oh * s1 + kh * d1 - p1;
+ if (ih < 0 || ih >= H) {
+ continue;
+ }
+ for (long kw = 0; kw < KW; ++kw) {
+ const long iw = ow * s0 + kw * d0 - p0;
+ if (iw < 0 || iw >= W) {
+ continue;
+ }
+ const long weight_idx = kw + kh * nb01 + c * nb03;
+ const long input_idx = iw + ih * nb11 + c * nb12 + n * nb13;
+ sum += (float) weights[weight_idx] * input[input_idx];
+ }
+ }
+
+ const long output_idx = ow + oh * nb1 + c * nb2 + n * nb3;
+ output[output_idx] = sum;
+}