Commit 8a199ad31 for llama.cpp
commit 8a199ad31a7eb27d5a46a6b08026176d82bc9894
Author: Max Krasnyansky <maxk@qti.qualcomm.com>
Date: Thu Oct 29 00:22:34 2026 -0700
hexagon: support for UPSCALE op (#31331)
* hex-upscale: basic upscale ops (not fully vectorized)
* hex-upscale: optimize nearest 2x, 4x and bilinear 4x
* hex-upscale: add support for non-contig rows and dims collapsing
* hex-upscale: use vgather instead of scalar staging
* hex-upscale: move all kernel select logic to the host, review comments and cleanups
* tests: add permuted UPSCALE test to cover qwen3-vl image->text scenarios
diff --git a/ggml/src/ggml-hexagon/htp-kparams.cpp b/ggml/src/ggml-hexagon/htp-kparams.cpp
index b9ae9f9db..c8c1947e6 100644
--- a/ggml/src/ggml-hexagon/htp-kparams.cpp
+++ b/ggml/src/ggml-hexagon/htp-kparams.cpp
@@ -60,6 +60,7 @@ htp_op_code htp_op_remap(const struct ggml_tensor * t) {
case GGML_OP_CONV_2D: return HTP_OP_CONV_2D;
case GGML_OP_CONV_2D_DW: return HTP_OP_CONV_2D_DW;
case GGML_OP_ROLL: return HTP_OP_ROLL;
+ case GGML_OP_UPSCALE: return HTP_OP_UPSCALE;
case GGML_OP_UNARY:
switch (ggml_get_unary_op(t)) {
@@ -2373,6 +2374,183 @@ static bool htp_kparams_solve_pool_2d(
return true;
}
+bool htp_kparams_solve_upscale(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * src0,
+ const struct ggml_tensor * dst,
+ struct htp_upscale_kernel_params * kparams
+) {
+ memset(kparams, 0, sizeof(*kparams));
+
+ if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) {
+ return false;
+ }
+
+ const uint32_t src_x = (uint32_t) src0->ne[0];
+ const uint32_t src_y = (uint32_t) src0->ne[1];
+ const uint32_t src_z = (uint32_t) src0->ne[2];
+ const uint32_t src_w = (uint32_t) src0->ne[3];
+
+ const uint32_t dst_x = (uint32_t) dst->ne[0];
+ const uint32_t dst_y = (uint32_t) dst->ne[1];
+ const uint32_t dst_z = (uint32_t) dst->ne[2];
+ const uint32_t dst_w = (uint32_t) dst->ne[3];
+
+ if (src_x == 0 || src_y == 0 || src_z == 0 || src_w == 0 ||
+ dst_x == 0 || dst_y == 0 || dst_z == 0 || dst_w == 0) {
+ return false;
+ }
+
+ const bool is_src_contig_row = (src0->nb[0] == sizeof(float));
+ const bool is_src_strided_2d = (src0->nb[0] > sizeof(float)) &&
+ (src0->nb[0] % sizeof(float) == 0) &&
+ (src0->nb[1] == (size_t) src_x * src0->nb[0]) &&
+ ((size_t) src_x * src_y <= 0xFFFFu);
+
+ if ((!is_src_contig_row && !is_src_strided_2d) || dst->nb[0] != sizeof(float)) {
+ return false;
+ }
+
+ const int32_t mode_flags = dst->op_params[0];
+ const uint32_t mode_raw = (uint32_t) (mode_flags & 0xFF);
+ if (mode_raw >= GGML_SCALE_MODE_COUNT) {
+ return false;
+ }
+
+ const bool align_corners = (mode_flags & GGML_SCALE_FLAG_ALIGN_CORNERS) != 0;
+ const bool antialias = (mode_flags & GGML_SCALE_FLAG_ANTIALIAS) != 0;
+
+ float sf0 = (float) dst_x / (float) src_x;
+ float sf1 = (float) dst_y / (float) src_y;
+ float sf2 = (float) dst_z / (float) src_z;
+ float sf3 = (float) dst_w / (float) src_w;
+ float pixel_offset = 0.5f;
+
+ if (align_corners) {
+ pixel_offset = 0.0f;
+ sf0 = (dst_x > 1 && src_x > 1) ? (float) (dst_x - 1) / (float) (src_x - 1) : sf0;
+ sf1 = (dst_y > 1 && src_y > 1) ? (float) (dst_y - 1) / (float) (src_y - 1) : sf1;
+ }
+
+ const float inv_sf0 = 1.0f / sf0;
+ const float inv_sf1 = 1.0f / sf1;
+ const float inv_sf2 = 1.0f / sf2;
+ const float inv_sf3 = 1.0f / sf3;
+
+ const float filter_radius0 = antialias ? (std::max)(1.0f, inv_sf0) : 0.0f;
+ const float filter_radius1 = antialias ? (std::max)(1.0f, inv_sf1) : 0.0f;
+ const float invscale0 = antialias ? (1.0f / filter_radius0) : 0.0f;
+ const float invscale1 = antialias ? (1.0f / filter_radius1) : 0.0f;
+
+ const uint32_t total_slices = dst_z * dst_w;
+ uint32_t n_threads = htp_session_n_threads(sess);
+
+ struct htp_upscale_vtcm_layout layout;
+ uint32_t dst_nrows_per_chunk = 0;
+ while (n_threads > 0) {
+ if (htp_upscale_solve_layout(&layout, src_x, src_y, dst_x, dst_y, inv_sf1, filter_radius1, n_threads,
+ htp_session_vtcm_size(sess), &dst_nrows_per_chunk)) {
+ break;
+ }
+ n_threads--;
+ }
+ if (n_threads == 0) {
+ return false;
+ }
+
+ size_t lut_x_size = 0;
+ size_t elem_y_sz = sizeof(uint32_t);
+ if (mode_raw == HTP_UPSCALE_MODE_BICUBIC) {
+ lut_x_size = hex_round_up(dst_x * sizeof(struct upscale_bicubic_x), 128);
+ elem_y_sz = sizeof(struct upscale_bicubic_y);
+ } else if (antialias) {
+ lut_x_size = hex_round_up(dst_x * sizeof(struct upscale_antialias_x), 128);
+ elem_y_sz = sizeof(struct upscale_antialias_y);
+ } else if (mode_raw == HTP_UPSCALE_MODE_BILINEAR) {
+ const uint32_t n_chunks = (dst_x + 31) / 32;
+ lut_x_size = hex_round_up(n_chunks * sizeof(struct upscale_bilinear_x_chunk), 128);
+ elem_y_sz = sizeof(struct upscale_bilinear_y);
+ } else {
+ lut_x_size = hex_round_up(dst_x * sizeof(uint32_t), 128);
+ }
+ const size_t total_lut_size = lut_x_size + hex_round_up(dst_y * elem_y_sz, 128);
+ if (total_lut_size > HTP_DDR_SPAD_SIZE) {
+ return false;
+ }
+
+ const uint32_t chunks_per_slice = (dst_y + dst_nrows_per_chunk - 1) / dst_nrows_per_chunk;
+ const uint32_t total_chunks = total_slices * chunks_per_slice;
+
+ n_threads = (std::min)(n_threads, total_chunks > 0 ? total_chunks : 1);
+ htp_upscale_vtcm_layout_build(&layout, src_x, src_y, dst_x, dst_y, dst_nrows_per_chunk, inv_sf1, filter_radius1, n_threads);
+
+ const uint32_t slice_dst_bytes = dst_x * dst_y * sizeof(float);
+ const uint32_t chunk_dst_bytes = dst_x * dst_nrows_per_chunk * sizeof(float);
+ const bool dst_contig = (dst->nb[1] == (size_t) dst_x * sizeof(float)) &&
+ (dst_z <= 1 || dst->nb[2] == (size_t) dst_y * dst->nb[1]) &&
+ (dst_w <= 1 || dst->nb[3] == (size_t) dst_z * dst->nb[2]);
+
+ uint32_t chunks_per_group = 0;
+ if (dst_contig) {
+ const uint32_t slices_per_group = (slice_dst_bytes > 0)
+ ? (HEX_L2_LINE_SIZE / hex_gcd_u32(slice_dst_bytes, HEX_L2_LINE_SIZE))
+ : 1;
+ if (slices_per_group == 1 && (chunk_dst_bytes % HEX_L2_LINE_SIZE == 0)) {
+ chunks_per_group = 1;
+ } else {
+ chunks_per_group = slices_per_group * chunks_per_slice;
+ }
+ }
+
+ uint8_t kernel_type = HTP_UPSCALE_KERNEL_BILINEAR;
+ if (mode_raw == HTP_UPSCALE_MODE_NEAREST) {
+ if (dst_x == 2 * src_x && dst_y == 2 * src_y && !align_corners) {
+ kernel_type = HTP_UPSCALE_KERNEL_NEAREST_2X;
+ } else if (dst_x == 4 * src_x && dst_y == 4 * src_y && !align_corners) {
+ kernel_type = HTP_UPSCALE_KERNEL_NEAREST_4X;
+ } else {
+ kernel_type = HTP_UPSCALE_KERNEL_NEAREST;
+ }
+ } else if (mode_raw == HTP_UPSCALE_MODE_BICUBIC) {
+ kernel_type = HTP_UPSCALE_KERNEL_BICUBIC;
+ } else if (antialias) {
+ kernel_type = HTP_UPSCALE_KERNEL_BILINEAR_AA;
+ } else if (dst_x == 2 * src_x && dst_y == 2 * src_y && !align_corners && (src_x % 32 == 0)) {
+ kernel_type = HTP_UPSCALE_KERNEL_BILINEAR_2X;
+ } else {
+ kernel_type = HTP_UPSCALE_KERNEL_BILINEAR;
+ }
+
+ kparams->kernel_type = kernel_type;
+ kparams->mode = (uint8_t) mode_raw;
+ kparams->align_corners = align_corners ? 1 : 0;
+ kparams->antialias = antialias ? 1 : 0;
+
+ kparams->inv_sf0 = inv_sf0;
+ kparams->inv_sf1 = inv_sf1;
+ kparams->inv_sf2 = inv_sf2;
+ kparams->inv_sf3 = inv_sf3;
+ kparams->pixel_offset = pixel_offset;
+
+ kparams->filter_radius0 = filter_radius0;
+ kparams->filter_radius1 = filter_radius1;
+ kparams->invscale0 = invscale0;
+ kparams->invscale1 = invscale1;
+
+ kparams->n_threads = n_threads;
+ kparams->chunks_per_group = chunks_per_group;
+
+ kparams->dst_nrows_per_chunk = dst_nrows_per_chunk;
+ kparams->chunks_per_slice = chunks_per_slice;
+ kparams->total_chunks = total_chunks;
+ kparams->vtcm_size = (uint32_t) layout.total_bytes;
+
+ kparams->div_chunks_per_slice = init_fastdiv_values(chunks_per_slice > 0 ? chunks_per_slice : 1);
+ kparams->div_dst_z = init_fastdiv_values(dst_z > 0 ? dst_z : 1);
+
+ return true;
+}
+
static bool htp_kparams_solve_concat(
const struct ggml_hexagon_session * sess,
const struct ggml_tensor * op,
@@ -3917,6 +4095,31 @@ void htp_opnode_collapse(const struct ggml_hexagon_session * sess, htp_opnode &
}
return;
}
+
+ if (node.opcode == HTP_OP_UPSCALE) {
+ if (node.inputs.empty() || !node.inputs[0] || node.outputs.empty() || !node.outputs[0]) {
+ return;
+ }
+ const struct ggml_tensor * src0 = node.inputs[0];
+ const struct ggml_tensor * dst = node.outputs[0];
+
+ if ((src0->ne[3] > 1 || dst->ne[3] > 1) &&
+ src0->ne[2] == dst->ne[2] && src0->ne[3] == dst->ne[3] &&
+ src0->nb[3] == (size_t) src0->ne[2] * src0->nb[2] &&
+ dst->nb[3] == (size_t) dst->ne[2] * dst->nb[2]) {
+ struct ggml_tensor src0_c = *src0;
+ struct ggml_tensor dst_c = *dst;
+ src0_c.ne[2] *= src0_c.ne[3];
+ src0_c.ne[3] = 1;
+ src0_c.nb[3] = (size_t) src0_c.ne[2] * src0_c.nb[2];
+ dst_c.ne[2] *= dst_c.ne[3];
+ dst_c.ne[3] = 1;
+ dst_c.nb[3] = (size_t) dst_c.ne[2] * dst_c.nb[2];
+ node.inputs[0] = node.add_dummy(src0_c);
+ node.outputs[0] = node.add_dummy(dst_c);
+ }
+ return;
+ }
}
bool htp_kparams_solve(
@@ -3946,6 +4149,9 @@ bool htp_kparams_solve(
const size_t src2_size = src2 ? hex_round_up(ggml_nbytes(src2), 128) : 0;
return htp_kparams_solve_matmul_impl(sess, src0, src1, dst, src2 ? src2->nb[1] : 0, src2_size, node.kparams<htp_mm_kernel_params>());
}
+ if (node.opcode == HTP_OP_UPSCALE) {
+ return htp_kparams_solve_upscale(sess, src0, dst, node.kparams<htp_upscale_kernel_params>());
+ }
return htp_kparams_solve(sess, node.node, node.kernel_params);
}
@@ -3956,7 +4162,7 @@ bool htp_kparams_solve(
void * kparams
) {
const htp_op_code opcode = htp_op_remap(op);
- if (htp_op_is_unary(opcode) || htp_op_is_glu(opcode)) {
+ if (htp_op_is_unary(opcode) || htp_op_is_glu(opcode) || opcode == HTP_OP_UPSCALE) {
htp_opnode tmp(opcode, op);
if (!htp_kparams_solve(sess, tmp)) {
return false;
diff --git a/ggml/src/ggml-hexagon/htp-kparams.h b/ggml/src/ggml-hexagon/htp-kparams.h
index fdf751d99..b757f430b 100644
--- a/ggml/src/ggml-hexagon/htp-kparams.h
+++ b/ggml/src/ggml-hexagon/htp-kparams.h
@@ -29,6 +29,7 @@
#include "htp/ssm-conv.h"
#include "htp/sum-rows-ops.h"
#include "htp/unary-ops.h"
+#include "htp/upscale-ops.h"
#include <type_traits>
#include <utility>
@@ -134,4 +135,19 @@ bool htp_kparams_solve_unary_mul_glu(
struct htp_glu_kernel_params * kparams
);
+bool htp_kparams_solve_upscale(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * src0,
+ const struct ggml_tensor * dst,
+ struct htp_upscale_kernel_params * kparams
+);
+
+static inline bool htp_kparams_solve_upscale(
+ const struct ggml_hexagon_session * sess,
+ const struct ggml_tensor * op,
+ struct htp_upscale_kernel_params * kparams
+) {
+ return htp_kparams_solve_upscale(sess, op->src[0], op, kparams);
+}
+
#endif // HTP_KPARAMS_H
diff --git a/ggml/src/ggml-hexagon/htp-opnode.cpp b/ggml/src/ggml-hexagon/htp-opnode.cpp
index 3509ffeee..d4e5150c0 100644
--- a/ggml/src/ggml-hexagon/htp-opnode.cpp
+++ b/ggml/src/ggml-hexagon/htp-opnode.cpp
@@ -19,6 +19,7 @@
#include "htp/ssm-conv.h"
#include "htp/sum-rows-ops.h"
#include "htp/unary-ops.h"
+#include "htp/upscale-ops.h"
#include <stdio.h>
@@ -224,6 +225,19 @@ static std::string format_kernel_params(const htp_opnode & node) {
} else if (node.opcode == HTP_OP_SOLVE_TRI) {
const auto * kparams = node.kparams<htp_solve_tri_kernel_params>();
snprintf(str, max_size, "%s nth %d", kparams->batched ? "batched" : "chunked", (int) kparams->n_threads);
+ } else if (node.opcode == HTP_OP_UPSCALE) {
+ const auto * kparams = node.kparams<htp_upscale_kernel_params>();
+ const char * ktype_str = "unknown";
+ switch (kparams->kernel_type) {
+ case HTP_UPSCALE_KERNEL_NEAREST: ktype_str = "nearest"; break;
+ case HTP_UPSCALE_KERNEL_NEAREST_2X: ktype_str = "nearest-2x"; break;
+ case HTP_UPSCALE_KERNEL_NEAREST_4X: ktype_str = "nearest-4x"; break;
+ case HTP_UPSCALE_KERNEL_BILINEAR: ktype_str = kparams->align_corners ? "bilinear-ac" : "bilinear"; break;
+ case HTP_UPSCALE_KERNEL_BILINEAR_2X: ktype_str = "bilinear-2x"; break;
+ case HTP_UPSCALE_KERNEL_BILINEAR_AA: ktype_str = "bilinear-aa"; break;
+ case HTP_UPSCALE_KERNEL_BICUBIC: ktype_str = "bicubic"; break;
+ }
+ snprintf(str, max_size, "%s nth %d chk %d vtcm %u", ktype_str, (int) kparams->n_threads, (int) kparams->dst_nrows_per_chunk, (unsigned int) kparams->vtcm_size);
} else {
snprintf(str, max_size, "----");
}
diff --git a/ggml/src/ggml-hexagon/htp/CMakeLists.txt b/ggml/src/ggml-hexagon/htp/CMakeLists.txt
index 5aec23a85..b5d6d55da 100644
--- a/ggml/src/ggml-hexagon/htp/CMakeLists.txt
+++ b/ggml/src/ggml-hexagon/htp/CMakeLists.txt
@@ -47,6 +47,7 @@ add_library(${HTP_LIB} SHARED
conv2d-ops.c
roll-ops.c
pool-ops.c
+ upscale-ops.c
allreduce-ops.c
)
diff --git a/ggml/src/ggml-hexagon/htp/htp-ctx.h b/ggml/src/ggml-hexagon/htp/htp-ctx.h
index 4a05bb974..0c455141d 100644
--- a/ggml/src/ggml-hexagon/htp/htp-ctx.h
+++ b/ggml/src/ggml-hexagon/htp/htp-ctx.h
@@ -188,5 +188,6 @@ int op_pool_2d(struct htp_ops_context * octx);
int op_pool_1d(struct htp_ops_context * octx);
int op_allreduce(struct htp_ops_context * octx);
int op_roll(struct htp_ops_context * octx);
+int op_upscale(struct htp_ops_context * octx);
#endif /* HTP_CTX_H */
diff --git a/ggml/src/ggml-hexagon/htp/htp-ops.h b/ggml/src/ggml-hexagon/htp/htp-ops.h
index e2e7edfa6..b8df01830 100644
--- a/ggml/src/ggml-hexagon/htp/htp-ops.h
+++ b/ggml/src/ggml-hexagon/htp/htp-ops.h
@@ -116,6 +116,7 @@ enum htp_op_code {
HTP_OP_GLU_SIGMOID,
HTP_OP_UNARY_CEIL,
HTP_OP_UNARY_SGN,
+ HTP_OP_UPSCALE,
HTP_OP_INVALID
};
diff --git a/ggml/src/ggml-hexagon/htp/main.c b/ggml/src/ggml-hexagon/htp/main.c
index 0ab9fbd85..78adc9a49 100644
--- a/ggml/src/ggml-hexagon/htp/main.c
+++ b/ggml/src/ggml-hexagon/htp/main.c
@@ -1049,6 +1049,9 @@ static int execute_op(struct htp_ops_context * octx) {
case HTP_OP_TRI:
return op_unary(octx);
+ case HTP_OP_UPSCALE:
+ return op_upscale(octx);
+
case HTP_OP_INVALID:
break;
}
diff --git a/ggml/src/ggml-hexagon/htp/upscale-ops.c b/ggml/src/ggml-hexagon/htp/upscale-ops.c
new file mode 100644
index 000000000..62070cf5d
--- /dev/null
+++ b/ggml/src/ggml-hexagon/htp/upscale-ops.c
@@ -0,0 +1,958 @@
+#pragma clang diagnostic ignored "-Wunused-variable"
+
+#include <float.h>
+#include <math.h>
+#include <HAP_farf.h>
+
+#include "hex-common.h"
+#include "dma-queue.h"
+#include "hex-profile.h"
+#include "htp-ctx.h"
+#include "htp-ops.h"
+#include "htp-tensor.h"
+#include "hvx-types.h"
+#include "hvx-base.h"
+#include "hvx-utils.h"
+#include "upscale-ops.h"
+
+static inline int32_t clamp_i32(int32_t v, int32_t lo, int32_t hi) {
+ return v < lo ? lo : (v > hi ? hi : v);
+}
+
+static inline float clamp_f32(float v, float lo, float hi) {
+ return v < lo ? lo : (v > hi ? hi : v);
+}
+
+static inline float bicubic_weight1(float x) {
+ const float a = -0.75f;
+ return ((a + 2.0f) * x - (a + 3.0f)) * x * x + 1.0f;
+}
+
+static inline float bicubic_weight2(float x) {
+ const float a = -0.75f;
+ return ((a * x - 5.0f * a) * x + 8.0f * a) * x - 4.0f * a;
+}
+
+static inline void bicubic_weights(float x, float w[4]) {
+ w[0] = bicubic_weight2(x + 1.0f);
+ w[1] = bicubic_weight1(x);
+ w[2] = bicubic_weight1(1.0f - x);
+ w[3] = bicubic_weight2(2.0f - x);
+}
+
+static void upscale_init_luts_nearest(
+ uint32_t * lut_x,
+ uint32_t * lut_y,
+ const struct htp_tensor * src0,
+ const struct htp_tensor * dst,
+ const struct htp_upscale_kernel_params * p
+) {
+ const uint32_t src_x = src0->ne[0];
+ const uint32_t src_y = src0->ne[1];
+ const uint32_t dst_x = dst->ne[0];
+ const uint32_t dst_y = dst->ne[1];
+ const uint32_t dst_x_aligned = (dst_x + VLEN_FP32 - 1) & ~(VLEN_FP32 - 1);
+
+ for (uint32_t ox = 0; ox < dst_x_aligned; ++ox) {
+ if (ox < dst_x) {
+ uint32_t x0 = (uint32_t) ((float) ox * p->inv_sf0);
+ lut_x[ox] = MIN(x0, src_x - 1) * sizeof(float);
+ } else {
+ lut_x[ox] = 0;
+ }
+ }
+ for (uint32_t oy = 0; oy < dst_y; ++oy) {
+ uint32_t y0 = (uint32_t) ((float) oy * p->inv_sf1);
+ lut_y[oy] = MIN(y0, src_y - 1);
+ }
+}
+
+static void upscale_init_luts_bilinear(
+ struct upscale_bilinear_x_chunk * lut_x,
+ struct upscale_bilinear_y * lut_y,
+ const struct htp_tensor * src0,
+ const struct htp_tensor * dst,
+ const struct htp_upscale_kernel_params * p
+) {
+ const uint32_t src_x = src0->ne[0];
+ const uint32_t src_y = src0->ne[1];
+ const uint32_t dst_x = dst->ne[0];
+ const uint32_t dst_y = dst->ne[1];
+ const uint32_t n_chunks = (dst_x + VLEN_FP32 - 1) / VLEN_FP32;
+
+ for (uint32_t c = 0; c < n_chunks; ++c) {
+ struct upscale_bilinear_x_chunk * chunk = &lut_x[c];
+ const uint32_t ox_start = c * VLEN_FP32;
+ for (uint32_t l = 0; l < VLEN_FP32; ++l) {
+ const uint32_t ox = ox_start + l;
+ if (ox < dst_x) {
+ float xf = ((float) ox + p->pixel_offset) * p->inv_sf0 - p->pixel_offset;
+ int32_t x0 = (int32_t) floorf(xf);
+ int32_t x1 = x0 + 1;
+ x0 = clamp_i32(x0, 0, (int32_t) src_x - 1);
+ x1 = clamp_i32(x1, 0, (int32_t) src_x - 1);
+ float dx = clamp_f32(xf - (float) x0, 0.0f, 1.0f);
+
+ chunk->off0[l] = x0 * (int32_t) sizeof(float);
+ chunk->off1[l] = x1 * (int32_t) sizeof(float);
+ chunk->w0[l] = 1.0f - dx;
+ chunk->dx[l] = dx;
+ } else {
+ chunk->off0[l] = 0;
+ chunk->off1[l] = 0;
+ chunk->w0[l] = 0.0f;
+ chunk->dx[l] = 0.0f;
+ }
+ }
+ }
+ for (uint32_t oy = 0; oy < dst_y; ++oy) {
+ float yf = ((float) oy + p->pixel_offset) * p->inv_sf1 - p->pixel_offset;
+ int32_t y0 = (int32_t) floorf(yf);
+ int32_t y1 = y0 + 1;
+ y0 = clamp_i32(y0, 0, (int32_t) src_y - 1);
+ y1 = clamp_i32(y1, 0, (int32_t) src_y - 1);
+ float dy = clamp_f32(yf - (float) y0, 0.0f, 1.0f);
+
+ lut_y[oy].y0 = (uint32_t) y0;
+ lut_y[oy].y1 = (uint32_t) y1;
+ lut_y[oy].wy1 = dy;
+ lut_y[oy].wy0 = 1.0f - dy;
+ }
+}
+
+static void upscale_init_luts_bicubic(
+ struct upscale_bicubic_x * lut_x,
+ struct upscale_bicubic_y * lut_y,
+ const struct htp_tensor * src0,
+ const struct htp_tensor * dst,
+ const struct htp_upscale_kernel_params * p
+) {
+ const uint32_t src_x = src0->ne[0];
+ const uint32_t src_y = src0->ne[1];
+ const uint32_t dst_x = dst->ne[0];
+ const uint32_t dst_y = dst->ne[1];
+
+ for (uint32_t ox = 0; ox < dst_x; ++ox) {
+ float xf = ((float) ox + p->pixel_offset) * p->inv_sf0 - p->pixel_offset;
+ int32_t x0 = (int32_t) floorf(xf);
+ float dx = xf - (float) x0;
+ bicubic_weights(dx, lut_x[ox].w);
+
+ lut_x[ox].x[0] = clamp_i32(x0 - 1, 0, (int32_t) src_x - 1);
+ lut_x[ox].x[1] = clamp_i32(x0 + 0, 0, (int32_t) src_x - 1);
+ lut_x[ox].x[2] = clamp_i32(x0 + 1, 0, (int32_t) src_x - 1);
+ lut_x[ox].x[3] = clamp_i32(x0 + 2, 0, (int32_t) src_x - 1);
+ }
+ for (uint32_t oy = 0; oy < dst_y; ++oy) {
+ float yf = ((float) oy + p->pixel_offset) * p->inv_sf1 - p->pixel_offset;
+ int32_t y0 = (int32_t) floorf(yf);
+ float dy = yf - (float) y0;
+ bicubic_weights(dy, lut_y[oy].w);
+
+ lut_y[oy].y[0] = clamp_i32(y0 - 1, 0, (int32_t) src_y - 1);
+ lut_y[oy].y[1] = clamp_i32(y0 + 0, 0, (int32_t) src_y - 1);
+ lut_y[oy].y[2] = clamp_i32(y0 + 1, 0, (int32_t) src_y - 1);
+ lut_y[oy].y[3] = clamp_i32(y0 + 2, 0, (int32_t) src_y - 1);
+ }
+}
+
+static void upscale_init_luts_antialias(
+ struct upscale_antialias_x * lut_x,
+ struct upscale_antialias_y * lut_y,
+ const struct htp_tensor * src0,
+ const struct htp_tensor * dst,
+ const struct htp_upscale_kernel_params * p
+) {
+ const uint32_t src_x = src0->ne[0];
+ const uint32_t src_y = src0->ne[1];
+ const uint32_t dst_x = dst->ne[0];
+ const uint32_t dst_y = dst->ne[1];
+
+ const float filter_radius0 = p->filter_radius0;
+ const float filter_radius1 = p->filter_radius1;
+
+ for (uint32_t ox = 0; ox < dst_x; ++ox) {
+ const float xf = ((float) ox + p->pixel_offset) * p->inv_sf0;
+ int32_t x_min = (int32_t) MAX(xf - filter_radius0 + p->pixel_offset, 0.0f);
+ int32_t x_max = (int32_t) MIN(xf + filter_radius0 + p->pixel_offset, (float) src_x);
+ lut_x[ox].x_min = clamp_i32(x_min, 0, (int32_t) src_x);
+ lut_x[ox].x_max = clamp_i32(x_max, 0, (int32_t) src_x);
+ lut_x[ox].xf = xf;
+ }
+ for (uint32_t oy = 0; oy < dst_y; ++oy) {
+ const float yf = ((float) oy + p->pixel_offset) * p->inv_sf1;
+ int32_t y_min = (int32_t) MAX(yf - filter_radius1 + p->pixel_offset, 0.0f);
+ int32_t y_max = (int32_t) MIN(yf + filter_radius1 + p->pixel_offset, (float) src_y);
+ lut_y[oy].y_min = clamp_i32(y_min, 0, (int32_t) src_y);
+ lut_y[oy].y_max = clamp_i32(y_max, 0, (int32_t) src_y);
+ lut_y[oy].yf = yf;
+ }
+}
+
+static inline void get_src_y_window(
+ const struct htp_tensor * src0,
+ const struct htp_tensor * dst,
+ const struct htp_upscale_kernel_params * p,
+ const void * lut_y,
+ uint32_t dst_y_start,
+ uint32_t dst_y_count,
+ uint32_t * out_src_y_start,
+ uint32_t * out_src_y_count
+) {
+ const uint32_t src_y = src0->ne[1];
+ const uint32_t dst_y = dst->ne[1];
+
+ if (dst_y_count == dst_y) {
+ *out_src_y_start = 0;
+ *out_src_y_count = src_y;
+ return;
+ }
+
+ const uint32_t last_oy = dst_y_start + dst_y_count - 1;
+
+ if (p->mode == HTP_UPSCALE_MODE_NEAREST) {
+ const uint32_t * ly = (const uint32_t *) lut_y;
+ const uint32_t y0 = ly[dst_y_start];
+ const uint32_t y1 = ly[last_oy];
+ *out_src_y_start = y0;
+ *out_src_y_count = y1 - y0 + 1;
+ } else if (p->mode == HTP_UPSCALE_MODE_BICUBIC) {
+ const struct upscale_bicubic_y * ly = (const struct upscale_bicubic_y *) lut_y;
+ const int32_t y0 = ly[dst_y_start].y[0];
+ const int32_t y1 = ly[last_oy].y[3];
+ *out_src_y_start = (uint32_t) y0;
+ *out_src_y_count = (uint32_t) (y1 - y0 + 1);
+ } else if (p->antialias) {
+ const struct upscale_antialias_y * ly = (const struct upscale_antialias_y *) lut_y;
+ const int32_t y0 = ly[dst_y_start].y_min;
+ const int32_t y1 = ly[last_oy].y_max;
+ *out_src_y_start = (uint32_t) y0;
+ *out_src_y_count = (uint32_t) (y1 - y0);
+ } else {
+ const struct upscale_bilinear_y * ly = (const struct upscale_bilinear_y *) lut_y;
+ const uint32_t y0 = ly[dst_y_start].y0;
+ const uint32_t y1 = ly[last_oy].y1;
+ *out_src_y_start = y0;
+ *out_src_y_count = y1 - y0 + 1;
+ }
+}
+
+static inline void resolve_dst_slice(
+ const struct htp_tensor * dst,
+ const struct htp_upscale_kernel_params * p,
+ uint32_t chunk_idx,
+ dma_addr_t * out_slice_dst_off,
+ uint32_t * out_dst_ys,
+ uint32_t * out_dst_yc
+) {
+ const uint32_t s_idx = fastdiv(chunk_idx, &p->div_chunks_per_slice);
+ const uint32_t c_idx = chunk_idx - s_idx * p->chunks_per_slice;
+ const uint32_t d_ys = c_idx * p->dst_nrows_per_chunk;
+ const uint32_t d_yc = MIN(p->dst_nrows_per_chunk, dst->ne[1] - d_ys);
+ const uint32_t i3 = fastdiv(s_idx, &p->div_dst_z);
+ const uint32_t i2 = s_idx - i3 * dst->ne[2];
+
+ *out_slice_dst_off = (dma_addr_t) i2 * dst->nb[2] + (dma_addr_t) i3 * dst->nb[3];
+ *out_dst_ys = d_ys;
+ *out_dst_yc = d_yc;
+}
+
+static inline void resolve_src_slice(
+ const struct htp_tensor * src0,
+ const struct htp_tensor * dst,
+ const struct htp_upscale_kernel_params * p,
+ const void * lut_y,
+ uint32_t chunk_idx,
+ dma_addr_t * out_slice_src_off,
+ uint32_t * out_src_ys,
+ uint32_t * out_src_yc
+) {
+ const uint32_t s_idx = fastdiv(chunk_idx, &p->div_chunks_per_slice);
+ const uint32_t c_idx = chunk_idx - s_idx * p->chunks_per_slice;
+ const uint32_t d_ys = c_idx * p->dst_nrows_per_chunk;
+ const uint32_t d_yc = MIN(p->dst_nrows_per_chunk, dst->ne[1] - d_ys);
+ uint32_t s_ys, s_yc;
+ get_src_y_window(src0, dst, p, lut_y, d_ys, d_yc, &s_ys, &s_yc);
+
+ const uint32_t i3 = fastdiv(s_idx, &p->div_dst_z);
+ const uint32_t i2 = s_idx - i3 * dst->ne[2];
+ const uint32_t i03 = (uint32_t) ((float) i3 * p->inv_sf3);
+ const uint32_t i02 = (uint32_t) ((float) i2 * p->inv_sf2);
+
+ *out_slice_src_off = (dma_addr_t) i02 * src0->nb[2] + (dma_addr_t) i03 * src0->nb[3];
+ *out_src_ys = s_ys;
+ *out_src_yc = s_yc;
+}
+
+struct upscale_context;
+
+typedef void (*upscale_chunk_fn_t)(
+ const struct upscale_context * ctx,
+ const float * srcb,
+ float * dstb,
+ float * R_buf,
+ float * gather_tmp,
+ uint32_t dst_y_start,
+ uint32_t dst_y_count,
+ uint32_t src_y_start
+);
+
+struct upscale_context {
+ struct htp_ops_context * octx;
+ const struct htp_upscale_kernel_params * kparams;
+ struct htp_upscale_vtcm_layout layout;
+ upscale_chunk_fn_t compute_chunk;
+ const void * lut_x;
+ const void * lut_y;
+ uint32_t chunk_start;
+ uint32_t chunk_count;
+ uint32_t chunks_per_thread;
+};
+
+static void compute_nearest_chunk(
+ const struct upscale_context * ctx,
+ const float * srcb,
+ float * dstb,
+ float * R_buf,
+ float * gather_tmp,
+ uint32_t dst_y_start,
+ uint32_t dst_y_count,
+ uint32_t src_y_start
+) {
+ const uint32_t src_x = ctx->octx->src[0]->ne[0];
+ const uint32_t dst_x = ctx->octx->dst->ne[0];
+ const uint32_t * lut_x = (const uint32_t *) ctx->lut_x;
+ const uint32_t * lut_y = (const uint32_t *) ctx->lut_y;
+ const uint32_t region_size = hex_round_up(src_x * sizeof(float), 128);
+
+ for (uint32_t oy = dst_y_start; oy < dst_y_start + dst_y_count; ++oy) {
+ const uint32_t y0 = lut_y[oy];
+ const float * r0 = srcb + (y0 - src_y_start) * src_x;
+ float * dst_row = dstb + (oy - dst_y_start) * dst_x;
+
+ for (uint32_t ix = 0; ix < src_x; ix += VLEN_FP32) {
+ *(HVX_Vector *) (R_buf + ix) = hvx_vmemu(r0 + ix);
+ }
+
+ for (uint32_t ox = 0; ox < dst_x; ox += VLEN_FP32) {
+ const uint32_t rem = dst_x - ox;
+ const uint32_t nbytes = (rem < VLEN_FP32) ? (rem * sizeof(float)) : VLEN;
+ const HVX_Vector voff = hvx_vmem(lut_x + ox);
+ Q6_vgather_ARMVw((HVX_Vector *) gather_tmp, (size_t) R_buf, region_size, voff);
+ hvx_gather_sync(gather_tmp);
+ hvx_vec_store_u(dst_row + ox, nbytes, hvx_vmem(gather_tmp));
+ }
+ }
+}
+
+static void compute_nearest_2x_chunk(
+ const struct upscale_context * ctx,
+ const float * srcb,
+ float * dstb,
+ float * R_buf,
+ float * gather_tmp,
+ uint32_t dst_y_start,
+ uint32_t dst_y_count,
+ uint32_t src_y_start
+) {
+ (void) R_buf;
+ (void) gather_tmp;
+ const uint32_t src_x = ctx->octx->src[0]->ne[0];
+ const uint32_t dst_x = ctx->octx->dst->ne[0];
+
+ for (uint32_t oy = dst_y_start; oy < dst_y_start + dst_y_count; ++oy) {
+ const uint32_t sy = oy >> 1;
+ const float * r0 = srcb + (sy - src_y_start) * src_x;
+ float * dst_row = dstb + (oy - dst_y_start) * dst_x;
+
+ for (uint32_t ix = 0; ix < src_x; ix += VLEN_FP32) {
+ const HVX_Vector vin = hvx_vmemu(r0 + ix);
+
+ const HVX_VectorPair p = Q6_W_vshuff_VVR(vin, vin, -4);
+ const HVX_Vector v_lo = Q6_V_lo_W(p);
+ const HVX_Vector v_hi = Q6_V_hi_W(p);
+
+ const uint32_t ox0 = 2 * ix;
+ const uint32_t rem_out0 = (dst_x > ox0) ? (dst_x - ox0) : 0;
+ const uint32_t nbytes_out0 = (rem_out0 < VLEN_FP32) ? (rem_out0 * sizeof(float)) : VLEN;
+ hvx_vec_store_u(dst_row + ox0, nbytes_out0, v_lo);
+
+ const uint32_t ox1 = ox0 + VLEN_FP32;
+ if (dst_x > ox1) {
+ const uint32_t rem_out1 = dst_x - ox1;
+ const uint32_t nbytes_out1 = (rem_out1 < VLEN_FP32) ? (rem_out1 * sizeof(float)) : VLEN;
+ hvx_vec_store_u(dst_row + ox1, nbytes_out1, v_hi);
+ }
+ }
+ }
+}
+
+static void compute_nearest_4x_chunk(
+ const struct upscale_context * ctx,
+ const float * srcb,
+ float * dstb,
+ float * R_buf,
+ float * gather_tmp,
+ uint32_t dst_y_start,
+ uint32_t dst_y_count,
+ uint32_t src_y_start
+) {
+ (void) R_buf;
+ (void) gather_tmp;
+ const uint32_t src_x = ctx->octx->src[0]->ne[0];
+ const uint32_t dst_x = ctx->octx->dst->ne[0];
+
+ for (uint32_t oy = dst_y_start; oy < dst_y_start + dst_y_count; ++oy) {
+ const uint32_t sy = oy >> 2;
+ const float * r0 = srcb + (sy - src_y_start) * src_x;
+ float * dst_row = dstb + (oy - dst_y_start) * dst_x;
+
+ for (uint32_t ix = 0; ix < src_x; ix += VLEN_FP32) {
+ const HVX_Vector vin = hvx_vmemu(r0 + ix);
+
+ const HVX_VectorPair p = Q6_W_vshuff_VVR(vin, vin, -4);
+ const HVX_Vector v_lo = Q6_V_lo_W(p);
+ const HVX_Vector v_hi = Q6_V_hi_W(p);
+
+ const HVX_VectorPair p0 = Q6_W_vshuff_VVR(v_lo, v_lo, -4);
+ const HVX_VectorPair p1 = Q6_W_vshuff_VVR(v_hi, v_hi, -4);
+
+ const HVX_Vector out0 = Q6_V_lo_W(p0);
+ const HVX_Vector out1 = Q6_V_hi_W(p0);
+ const HVX_Vector out2 = Q6_V_lo_W(p1);
+ const HVX_Vector out3 = Q6_V_hi_W(p1);
+
+ const uint32_t ox = 4 * ix;
+ if (dst_x > ox) {
+ const uint32_t rem = dst_x - ox;
+ hvx_vec_store_u(dst_row + ox, (rem < VLEN_FP32) ? (rem * sizeof(float)) : VLEN, out0);
+ }
+ if (dst_x > ox + VLEN_FP32) {
+ const uint32_t rem = dst_x - (ox + VLEN_FP32);
+ hvx_vec_store_u(dst_row + ox + VLEN_FP32, (rem < VLEN_FP32) ? (rem * sizeof(float)) : VLEN, out1);
+ }
+ if (dst_x > ox + 2 * VLEN_FP32) {
+ const uint32_t rem = dst_x - (ox + 2 * VLEN_FP32);
+ hvx_vec_store_u(dst_row + ox + 2 * VLEN_FP32, (rem < VLEN_FP32) ? (rem * sizeof(float)) : VLEN, out2);
+ }
+ if (dst_x > ox + 3 * VLEN_FP32) {
+ const uint32_t rem = dst_x - (ox + 3 * VLEN_FP32);
+ hvx_vec_store_u(dst_row + ox + 3 * VLEN_FP32, (rem < VLEN_FP32) ? (rem * sizeof(float)) : VLEN, out3);
+ }
+ }
+ }
+}
+
+static inline void compute_bilinear_chunk_32(
+ float * restrict dst_chunk,
+ const float * restrict R_buf,
+ float * restrict tmp_a,
+ float * restrict tmp_b,
+ uint32_t region_size,
+ const struct upscale_bilinear_x_chunk * restrict chunk,
+ uint32_t nbytes
+) {
+ const HVX_Vector voff0 = hvx_vmem(chunk->off0);
+ const HVX_Vector voff1 = hvx_vmem(chunk->off1);
+ const HVX_Vector vw0 = hvx_vmem(chunk->w0);
+ const HVX_Vector vdx = hvx_vmem(chunk->dx);
+
+ Q6_vgather_ARMVw((HVX_Vector *) tmp_a, (size_t) R_buf, region_size, voff0);
+ Q6_vgather_ARMVw((HVX_Vector *) tmp_b, (size_t) R_buf, region_size, voff1);
+ hvx_gather_sync(tmp_b);
+
+ const HVX_Vector va = hvx_vmem(tmp_a);
+ const HVX_Vector vb = hvx_vmem(tmp_b);
+
+ const HVX_Vector val = hvx_vec_add_f32_f32(
+ hvx_vec_mul_f32_f32(vw0, va),
+ hvx_vec_mul_f32_f32(vdx, vb)
+ );
+
+ hvx_vec_store_u(dst_chunk, nbytes, val);
+}
+
+static void compute_bilinear_chunk(
+ const struct upscale_context * ctx,
+ const float * srcb,
+ float * dstb,
+ float * R_buf,
+ float * gather_tmp,
+ uint32_t dst_y_start,
+ uint32_t dst_y_count,
+ uint32_t src_y_start
+) {
+ const uint32_t src_x = ctx->octx->src[0]->ne[0];
+ const uint32_t dst_x = ctx->octx->dst->ne[0];
+ const struct upscale_bilinear_x_chunk * lut_x = (const struct upscale_bilinear_x_chunk *) ctx->lut_x;
+ const struct upscale_bilinear_y * lut_y = (const struct upscale_bilinear_y *) ctx->lut_y;
+ const uint32_t n_chunks = (dst_x + VLEN_FP32 - 1) / VLEN_FP32;
+ const uint32_t region_size = hex_round_up(src_x * sizeof(float), 128);
+ float * tmp_a = gather_tmp;
+ float * tmp_b = gather_tmp + VLEN_FP32;
+
+ for (uint32_t oy = dst_y_start; oy < dst_y_start + dst_y_count; ++oy) {
+ const struct upscale_bilinear_y * ey = &lut_y[oy];
+ const float * r0 = srcb + (ey->y0 - src_y_start) * src_x;
+ const float * r1 = srcb + (ey->y1 - src_y_start) * src_x;
+ float * dst_row = dstb + (oy - dst_y_start) * dst_x;
+
+ if (ey->y0 == ey->y1) {
+ for (uint32_t ix = 0; ix < src_x; ix += VLEN_FP32) {
+ *(HVX_Vector *) (R_buf + ix) = hvx_vmemu(r0 + ix);
+ }
+ } else {
+ const HVX_Vector vwy0 = hvx_vec_splat_f32(ey->wy0);
+ const HVX_Vector vwy1 = hvx_vec_splat_f32(ey->wy1);
+ for (uint32_t ix = 0; ix < src_x; ix += VLEN_FP32) {
+ const HVX_Vector vr0 = hvx_vmemu(r0 + ix);
+ const HVX_Vector vr1 = hvx_vmemu(r1 + ix);
+ *(HVX_Vector *) (R_buf + ix) = hvx_vec_add_f32_f32(
+ hvx_vec_mul_f32_f32(vwy0, vr0),
+ hvx_vec_mul_f32_f32(vwy1, vr1)
+ );
+ }
+ }
+
+ for (uint32_t c = 0; c < n_chunks; ++c) {
+ const uint32_t ox = c * VLEN_FP32;
+ const uint32_t rem = dst_x - ox;
+ const uint32_t nbytes = (rem < VLEN_FP32) ? (rem * sizeof(float)) : VLEN;
+ compute_bilinear_chunk_32(dst_row + ox, R_buf, tmp_a, tmp_b, region_size, &lut_x[c], nbytes);
+ }
+ }
+}
+
+static void compute_bilinear_2x_chunk(
+ const struct upscale_context * ctx,
+ const float * srcb,
+ float * dstb,
+ float * R_buf,
+ float * gather_tmp,
+ uint32_t dst_y_start,
+ uint32_t dst_y_count,
+ uint32_t src_y_start
+) {
+ (void) R_buf;
+ (void) gather_tmp;
+ const uint32_t src_x = ctx->octx->src[0]->ne[0];
+ const uint32_t dst_x = ctx->octx->dst->ne[0];
+ const struct upscale_bilinear_y * lut_y = (const struct upscale_bilinear_y *) ctx->lut_y;
+ const HVX_Vector c_075 = hvx_vec_splat_f32(0.75f);
+ const HVX_Vector c_025 = hvx_vec_splat_f32(0.25f);
+
+ for (uint32_t oy = dst_y_start; oy < dst_y_start + dst_y_count; ++oy) {
+ const struct upscale_bilinear_y * ey = &lut_y[oy];
+ const float * r0 = srcb + (ey->y0 - src_y_start) * src_x;
+ const float * r1 = srcb + (ey->y1 - src_y_start) * src_x;
+ float * dst_row = dstb + (oy - dst_y_start) * dst_x;
+
+ const bool same_row = (ey->y0 == ey->y1);
+ const HVX_Vector vwy0 = hvx_vec_splat_f32(ey->wy0);
+ const HVX_Vector vwy1 = hvx_vec_splat_f32(ey->wy1);
+
+ HVX_Vector v_curr = hvx_vmem(r0);
+ if (!same_row) {
+ const HVX_Vector v1 = hvx_vmem(r1);
+ v_curr = hvx_vec_add_f32_f32(hvx_vec_mul_f32_f32(vwy0, v_curr), hvx_vec_mul_f32_f32(vwy1, v1));
+ }
+
+ HVX_Vector v_prev = Q6_V_vror_VR(v_curr, 4);
+
+ for (uint32_t ix = 0; ix < src_x; ix += VLEN_FP32) {
+ HVX_Vector v_next_chunk;
+ const uint32_t next_ix = ix + VLEN_FP32;
+ if (next_ix < src_x) {
+ v_next_chunk = hvx_vmem(r0 + next_ix);
+ if (!same_row) {
+ const HVX_Vector v1 = hvx_vmem(r1 + next_ix);
+ v_next_chunk = hvx_vec_add_f32_f32(hvx_vec_mul_f32_f32(vwy0, v_next_chunk), hvx_vec_mul_f32_f32(vwy1, v1));
+ }
+ } else {
+ v_next_chunk = Q6_V_vror_VR(v_curr, 124);
+ }
+
+ const HVX_Vector v_shl = Q6_V_valign_VVR(v_curr, v_prev, 124);
+ const HVX_Vector v_shr = Q6_V_valign_VVR(v_next_chunk, v_curr, 4);
+
+ const HVX_Vector ve = hvx_vec_add_f32_f32(hvx_vec_mul_f32_f32(c_075, v_curr), hvx_vec_mul_f32_f32(c_025, v_shl));
+ const HVX_Vector vo = hvx_vec_add_f32_f32(hvx_vec_mul_f32_f32(c_075, v_curr), hvx_vec_mul_f32_f32(c_025, v_shr));
+
+ const HVX_VectorPair p = Q6_W_vshuff_VVR(vo, ve, -4);
+ const HVX_Vector out0 = Q6_V_lo_W(p);
+ const HVX_Vector out1 = Q6_V_hi_W(p);
+
+ const uint32_t ox0 = 2 * ix;
+ hvx_vmem(dst_row + ox0) = out0;
+ hvx_vmem(dst_row + ox0 + VLEN_FP32) = out1;
+
+ v_prev = v_curr;
+ v_curr = v_next_chunk;
+ }
+ }
+}
+
+static void compute_bilinear_antialias_chunk(
+ const struct upscale_context * ctx,
+ const float * srcb,
+ float * dstb,
+ float * R_buf,
+ float * gather_tmp,
+ uint32_t dst_y_start,
+ uint32_t dst_y_count,
+ uint32_t src_y_start
+) {
+ (void) R_buf;
+ (void) gather_tmp;
+ const uint32_t src_x = ctx->octx->src[0]->ne[0];
+ const uint32_t dst_x = ctx->octx->dst->ne[0];
+ const struct htp_upscale_kernel_params * p = ctx->kparams;
+ const struct upscale_antialias_x * lut_x = (const struct upscale_antialias_x *) ctx->lut_x;
+ const struct upscale_antialias_y * lut_y = (const struct upscale_antialias_y *) ctx->lut_y;
+
+ const float invscale1 = p->invscale1;
+ const float invscale0 = p->invscale0;
+
+ for (uint32_t oy = dst_y_start; oy < dst_y_start + dst_y_count; ++oy) {
+ const struct upscale_antialias_y * ey = &lut_y[oy];
+ const float yf = ey->yf;
+ const int32_t y_min = ey->y_min;
+ const int32_t y_max = ey->y_max;
+
+ float * dst_row = dstb + (oy - dst_y_start) * dst_x;
+
+ for (uint32_t ox = 0; ox < dst_x; ox += VLEN_FP32) {
+ const uint32_t rem = dst_x - ox;
+ const uint32_t nbytes = (rem < VLEN_FP32) ? (rem * sizeof(float)) : VLEN;
+ float tmp[VLEN_FP32] __attribute__((aligned(128)));
+ const uint32_t cnt = rem < VLEN_FP32 ? rem : VLEN_FP32;
+
+ for (uint32_t l = 0; l < cnt; ++l) {
+ const uint32_t cur_ox = ox + l;
+ const struct upscale_antialias_x * ex = &lut_x[cur_ox];
+ const float xf = ex->xf;
+ const int32_t x_min = ex->x_min;
+ const int32_t x_max = ex->x_max;
+
+ float val = 0.0f;
+ float total_weight = 0.0f;
+
+ for (int32_t sy = y_min; sy < y_max; ++sy) {
+ const float diff_y = fabsf((float) sy - yf + p->pixel_offset) * invscale1;
+ const float weight_y = MAX(1.0f - diff_y, 0.0f);
+ if (weight_y <= 0.0f) {
+ continue;
+ }
+ const float * r = srcb + (uint32_t) (sy - (int32_t) src_y_start) * src_x;
+
+ for (int32_t sx = x_min; sx < x_max; ++sx) {
+ const float diff_x = fabsf((float) sx - xf + p->pixel_offset) * invscale0;
+ const float weight_x = MAX(1.0f - diff_x, 0.0f);
+ const float weight = weight_x * weight_y;
+ if (weight <= 0.0f) {
+ continue;
+ }
+ val += r[sx] * weight;
+ total_weight += weight;
+ }
+ }
+
+ tmp[l] = (total_weight > 0.0f) ? (val / total_weight) : 0.0f;
+ }
+
+ hvx_vec_store_u(dst_row + ox, nbytes, *(const HVX_Vector *) tmp);
+ }
+ }
+}
+
+static void compute_bicubic_chunk(
+ const struct upscale_context * ctx,
+ const float * srcb,
+ float * dstb,
+ float * R_buf,
+ float * gather_tmp,
+ uint32_t dst_y_start,
+ uint32_t dst_y_count,
+ uint32_t src_y_start
+) {
+ (void) R_buf;
+ (void) gather_tmp;
+ const uint32_t src_x = ctx->octx->src[0]->ne[0];
+ const uint32_t dst_x = ctx->octx->dst->ne[0];
+ const struct upscale_bicubic_x * lut_x = (const struct upscale_bicubic_x *) ctx->lut_x;
+ const struct upscale_bicubic_y * lut_y = (const struct upscale_bicubic_y *) ctx->lut_y;
+
+ for (uint32_t oy = dst_y_start; oy < dst_y_start + dst_y_count; ++oy) {
+ const struct upscale_bicubic_y * ey = &lut_y[oy];
+ const float * r0 = srcb + (uint32_t) (ey->y[0] - (int32_t) src_y_start) * src_x;
+ const float * r1 = srcb + (uint32_t) (ey->y[1] - (int32_t) src_y_start) * src_x;
+ const float * r2 = srcb + (uint32_t) (ey->y[2] - (int32_t) src_y_start) * src_x;
+ const float * r3 = srcb + (uint32_t) (ey->y[3] - (int32_t) src_y_start) * src_x;
+
+ float * dst_row = dstb + (oy - dst_y_start) * dst_x;
+
+ const HVX_Vector vwy0 = hvx_vec_splat_f32(ey->w[0]);
+ const HVX_Vector vwy1 = hvx_vec_splat_f32(ey->w[1]);
+ const HVX_Vector vwy2 = hvx_vec_splat_f32(ey->w[2]);
+ const HVX_Vector vwy3 = hvx_vec_splat_f32(ey->w[3]);
+
+ for (uint32_t ox = 0; ox < dst_x; ox += VLEN_FP32) {
+ const uint32_t rem = dst_x - ox;
+ const uint32_t nbytes = (rem < VLEN_FP32) ? (rem * sizeof(float)) : VLEN;
+
+ float v0[VLEN_FP32] __attribute__((aligned(128)));
+ float v1[VLEN_FP32] __attribute__((aligned(128)));
+ float v2[VLEN_FP32] __attribute__((aligned(128)));
+ float v3[VLEN_FP32] __attribute__((aligned(128)));
+
+ const uint32_t cnt = rem < VLEN_FP32 ? rem : VLEN_FP32;
+ for (uint32_t l = 0; l < cnt; ++l) {
+ const struct upscale_bicubic_x * e = &lut_x[ox + l];
+ v0[l] = r0[e->x[0]] * e->w[0] + r0[e->x[1]] * e->w[1] + r0[e->x[2]] * e->w[2] + r0[e->x[3]] * e->w[3];
+ v1[l] = r1[e->x[0]] * e->w[0] + r1[e->x[1]] * e->w[1] + r1[e->x[2]] * e->w[2] + r1[e->x[3]] * e->w[3];
+ v2[l] = r2[e->x[0]] * e->w[0] + r2[e->x[1]] * e->w[1] + r2[e->x[2]] * e->w[2] + r2[e->x[3]] * e->w[3];
+ v3[l] = r3[e->x[0]] * e->w[0] + r3[e->x[1]] * e->w[1] + r3[e->x[2]] * e->w[2] + r3[e->x[3]] * e->w[3];
+ }
+ for (uint32_t l = cnt; l < VLEN_FP32; ++l) {
+ v0[l] = v1[l] = v2[l] = v3[l] = 0.0f;
+ }
+
+ const HVX_Vector vv0 = *(const HVX_Vector *) v0;
+ const HVX_Vector vv1 = *(const HVX_Vector *) v1;
+ const HVX_Vector vv2 = *(const HVX_Vector *) v2;
+ const HVX_Vector vv3 = *(const HVX_Vector *) v3;
+
+ const HVX_Vector acc0 = hvx_vec_add_f32_f32(hvx_vec_mul_f32_f32(vwy0, vv0), hvx_vec_mul_f32_f32(vwy1, vv1));
+ const HVX_Vector acc1 = hvx_vec_add_f32_f32(hvx_vec_mul_f32_f32(vwy2, vv2), hvx_vec_mul_f32_f32(vwy3, vv3));
+ const HVX_Vector val = hvx_vec_add_f32_f32(acc0, acc1);
+
+ hvx_vec_store_u(dst_row + ox, nbytes, val);
+ }
+ }
+}
+
+static inline void dma_push_src_chunk(
+ dma_queue * dma_q,
+ float * spad,
+ const struct htp_tensor * src0,
+ dma_addr_t slice_src_off,
+ uint32_t src_ys,
+ uint32_t src_yc
+) {
+ const uint32_t src_x = src0->ne[0];
+ const dma_addr_t src_addr = src0->data + slice_src_off + (dma_addr_t) src_ys * src0->nb[1];
+
+ if (src0->nb[0] == sizeof(float)) {
+ const uint32_t row_size = src_x * sizeof(float);
+ const uint32_t dst_stride = row_size;
+ const uint32_t src_stride = src0->nb[1];
+ dma_queue_push(dma_q, dma_make_data(spad, src_addr), dst_stride, src_stride, row_size, src_yc);
+ } else {
+ const uint32_t row_size = sizeof(float);
+ const uint32_t dst_stride = sizeof(float);
+ const uint32_t src_stride = src0->nb[0];
+ const uint32_t nrows = src_x * src_yc;
+ dma_queue_push(dma_q, dma_make_data(spad, src_addr), dst_stride, src_stride, row_size, nrows);
+ }
+}
+
+static void upscale_worker(unsigned int nth, unsigned int ith, void * data) {
+ struct upscale_context * ctx = (struct upscale_context *) data;
+ const struct htp_upscale_kernel_params * p = ctx->kparams;
+ const struct htp_upscale_vtcm_layout * L = &ctx->layout;
+ const struct htp_tensor * src0 = ctx->octx->src[0];
+ const struct htp_tensor * dst = ctx->octx->dst;
+
+ const uint32_t first = ctx->chunk_start + ith * ctx->chunks_per_thread;
+ const uint32_t last = MIN(first + ctx->chunks_per_thread, ctx->chunk_start + ctx->chunk_count);
+
+ if (first >= last) {
+ return;
+ }
+
+ struct htp_thread_trace * tr = &ctx->octx->ctx->trace[ith];
+ dma_queue * dma_q = ctx->octx->ctx->dma[ith];
+
+ const uint32_t src_spad_half = (uint32_t) L->src_spad_half_size;
+ const uint32_t dst_spad_half = (uint32_t) L->dst_spad_half_size;
+
+ uint8_t * vtcm_base = (uint8_t *) ctx->octx->ctx->vtcm_base;
+ uint8_t * src_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_src) + ith * L->src_bytes_per_thread;
+ uint8_t * dst_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_dst) + ith * L->dst_bytes_per_thread;
+ float * R_buf = VTCM_LAYOUT_PTR(float, vtcm_base, L->off_r) + ith * (L->r_bytes_per_thread / sizeof(float));
+ float * gather_tmp = VTCM_LAYOUT_PTR(float, vtcm_base, L->off_gather) + ith * (L->gather_bytes_per_thread / sizeof(float));
+ const void * lut_y = ctx->lut_y;
+
+ float * srcb2[2] = { (float *) src_spad, (float *) (src_spad + src_spad_half) };
+ float * dstb2[2] = { (float *) dst_spad, (float *) (dst_spad + dst_spad_half) };
+
+ const uint32_t dst_row_size = dst->ne[0] * sizeof(float);
+ const uint32_t dst_stride = dst->nb[1];
+
+ const uint32_t total = last - first;
+
+ // Warmup: prime pipeline with initial transfers
+ for (uint32_t i = 0; i < total && i < 2; ++i) {
+ dma_queue_push(dma_q, dma_make_data(dst->data, dstb2[i]), dst_stride, dst_row_size, dst_row_size, 0);
+
+ dma_addr_t slice_src_off;
+ uint32_t src_ys, src_yc;
+ resolve_src_slice(src0, dst, p, lut_y, first + i, &slice_src_off, &src_ys, &src_yc);
+
+ dma_push_src_chunk(dma_q, srcb2[i], src0, slice_src_off, src_ys, src_yc);
+ }
+
+ // Steady state loop
+ for (uint32_t i = 0; i < total; ++i) {
+ const uint32_t buf = i & 1u;
+ float * srcb = srcb2[buf];
+ float * dstb = dstb2[buf];
+
+ dma_queue_pop(dma_q);
+ dma_queue_pop(dma_q);
+
+ dma_addr_t slice_dst_off;
+ uint32_t dst_ys, dst_yc;
+ resolve_dst_slice(dst, p, first + i, &slice_dst_off, &dst_ys, &dst_yc);
+
+ uint32_t src_ys, src_yc;
+ get_src_y_window(src0, dst, p, lut_y, dst_ys, dst_yc, &src_ys, &src_yc);
+
+ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) (first + i));
+ ctx->compute_chunk(ctx, srcb, dstb, R_buf, gather_tmp, dst_ys, dst_yc, src_ys);
+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) (first + i));
+
+ const dma_addr_t dst_addr = dst->data + slice_dst_off + (dma_addr_t) dst_ys * dst_stride;
+ dma_queue_push(dma_q, dma_make_data(dst_addr, dstb), dst_stride, dst_row_size, dst_row_size, dst_yc);
+
+ const uint32_t next = i + 2;
+ if (next < total) {
+ dma_addr_t next_src_off;
+ uint32_t next_src_ys, next_src_yc;
+ resolve_src_slice(src0, dst, p, lut_y, first + next, &next_src_off, &next_src_ys, &next_src_yc);
+
+ dma_push_src_chunk(dma_q, srcb, src0, next_src_off, next_src_ys, next_src_yc);
+ }
+ }
+
+ dma_queue_flush(dma_q);
+
+ (void) nth;
+}
+
+int op_upscale(struct htp_ops_context * octx) {
+ const struct htp_tensor * src0 = octx->src[0];
+ const struct htp_tensor * dst = octx->dst;
+ const struct htp_upscale_kernel_params * p =
+ (const struct htp_upscale_kernel_params *) octx->kernel_params;
+
+ const uint32_t src_x = src0->ne[0];
+ const uint32_t src_y = src0->ne[1];
+ const uint32_t dst_x = dst->ne[0];
+ const uint32_t dst_y = dst->ne[1];
+
+ uint32_t chunk_start = 0;
+ uint32_t chunk_count = p->total_chunks;
+
+ if (octx->ctx->mdev.count > 1) {
+ const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(
+ chunk_count, htp_tensor_mdev_data_aligned(dst) ? p->chunks_per_group : 0,
+ octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
+ chunk_start = range.start;
+ chunk_count = range.count;
+ }
+
+ if (chunk_count == 0) {
+ return HTP_STATUS_OK;
+ }
+
+ const uint32_t n_threads = MIN(p->n_threads, chunk_count);
+ if (!htp_ops_context_set_n_threads(octx, n_threads)) {
+ return HTP_STATUS_INVAL_PARAMS;
+ }
+
+ const uint32_t chunks_per_thread = fastdiv(chunk_count + n_threads - 1, &octx->n_threads_div);
+
+ size_t lut_x_size = 0;
+ size_t elem_y_sz = sizeof(uint32_t);
+
+ if (p->mode == HTP_UPSCALE_MODE_BICUBIC) {
+ lut_x_size = hex_round_up(dst_x * sizeof(struct upscale_bicubic_x), 128);
+ elem_y_sz = sizeof(struct upscale_bicubic_y);
+ } else if (p->antialias) {
+ lut_x_size = hex_round_up(dst_x * sizeof(struct upscale_antialias_x), 128);
+ elem_y_sz = sizeof(struct upscale_antialias_y);
+ } else if (p->mode == HTP_UPSCALE_MODE_BILINEAR) {
+ const uint32_t n_chunks = (dst_x + VLEN_FP32 - 1) / VLEN_FP32;
+ lut_x_size = hex_round_up(n_chunks * sizeof(struct upscale_bilinear_x_chunk), 128);
+ elem_y_sz = sizeof(struct upscale_bilinear_y);
+ } else {
+ lut_x_size = hex_round_up(dst_x * sizeof(uint32_t), 128);
+ }
+
+ const size_t lut_y_size = hex_round_up(dst_y * elem_y_sz, 128);
+ const size_t total_lut_size = lut_x_size + lut_y_size;
+
+ if (total_lut_size > octx->ctx->ddr_spad_size) {
+ return HTP_STATUS_NO_SUPPORT;
+ }
+
+ void * base_scratch = octx->ctx->ddr_spad_base;
+ void * lut_x = base_scratch;
+ void * lut_y = (uint8_t *) base_scratch + lut_x_size;
+
+ if (p->mode == HTP_UPSCALE_MODE_NEAREST) {
+ upscale_init_luts_nearest((uint32_t *) lut_x, (uint32_t *) lut_y, src0, dst, p);
+ } else if (p->mode == HTP_UPSCALE_MODE_BICUBIC) {
+ upscale_init_luts_bicubic((struct upscale_bicubic_x *) lut_x, (struct upscale_bicubic_y *) lut_y, src0, dst, p);
+ } else if (p->antialias) {
+ upscale_init_luts_antialias((struct upscale_antialias_x *) lut_x, (struct upscale_antialias_y *) lut_y, src0, dst, p);
+ } else {
+ upscale_init_luts_bilinear((struct upscale_bilinear_x_chunk *) lut_x, (struct upscale_bilinear_y *) lut_y, src0, dst, p);
+ }
+
+ upscale_chunk_fn_t compute_chunk;
+ switch (p->kernel_type) {
+ case HTP_UPSCALE_KERNEL_NEAREST:
+ compute_chunk = compute_nearest_chunk;
+ break;
+ case HTP_UPSCALE_KERNEL_NEAREST_2X:
+ compute_chunk = compute_nearest_2x_chunk;
+ break;
+ case HTP_UPSCALE_KERNEL_NEAREST_4X:
+ compute_chunk = compute_nearest_4x_chunk;
+ break;
+ case HTP_UPSCALE_KERNEL_BILINEAR:
+ compute_chunk = compute_bilinear_chunk;
+ break;
+ case HTP_UPSCALE_KERNEL_BILINEAR_2X:
+ compute_chunk = compute_bilinear_2x_chunk;
+ break;
+ case HTP_UPSCALE_KERNEL_BILINEAR_AA:
+ compute_chunk = compute_bilinear_antialias_chunk;
+ break;
+ case HTP_UPSCALE_KERNEL_BICUBIC:
+ compute_chunk = compute_bicubic_chunk;
+ break;
+ default:
+ return HTP_STATUS_INVAL_PARAMS;
+ }
+
+ struct upscale_context ctx = {
+ .octx = octx,
+ .kparams = p,
+ .compute_chunk = compute_chunk,
+ .lut_x = lut_x,
+ .lut_y = lut_y,
+ .chunk_start = chunk_start,
+ .chunk_count = chunk_count,
+ .chunks_per_thread = chunks_per_thread,
+ };
+
+ htp_upscale_vtcm_layout_build(&ctx.layout, src_x, src_y, dst_x, dst_y,
+ p->dst_nrows_per_chunk, p->inv_sf1, p->filter_radius1, n_threads);
+
+ work_queue_run(octx->ctx->work_queue, upscale_worker, &ctx, n_threads);
+
+ return HTP_STATUS_OK;
+}
diff --git a/ggml/src/ggml-hexagon/htp/upscale-ops.h b/ggml/src/ggml-hexagon/htp/upscale-ops.h
new file mode 100644
index 000000000..aff7854b5
--- /dev/null
+++ b/ggml/src/ggml-hexagon/htp/upscale-ops.h
@@ -0,0 +1,204 @@
+#ifndef HTP_UPSCALE_OPS_H
+#define HTP_UPSCALE_OPS_H
+
+#include <stdint.h>
+#include <stddef.h>
+#include <stdbool.h>
+#include <math.h>
+
+#include "hex-common.h"
+#include "hex-fastdiv.h"
+#include "htp-vtcm.h"
+
+enum htp_upscale_mode {
+ HTP_UPSCALE_MODE_NEAREST = 0,
+ HTP_UPSCALE_MODE_BILINEAR = 1,
+ HTP_UPSCALE_MODE_BICUBIC = 2,
+};
+
+enum htp_upscale_kernel_type {
+ HTP_UPSCALE_KERNEL_NEAREST = 0,
+ HTP_UPSCALE_KERNEL_NEAREST_2X = 1,
+ HTP_UPSCALE_KERNEL_NEAREST_4X = 2,
+ HTP_UPSCALE_KERNEL_BILINEAR = 3,
+ HTP_UPSCALE_KERNEL_BILINEAR_2X = 4,
+ HTP_UPSCALE_KERNEL_BILINEAR_AA = 5,
+ HTP_UPSCALE_KERNEL_BICUBIC = 6,
+};
+
+struct htp_upscale_kernel_params {
+ uint8_t kernel_type;
+ uint8_t mode;
+ uint8_t align_corners;
+ uint8_t antialias;
+
+ float inv_sf0;
+ float inv_sf1;
+ float inv_sf2;
+ float inv_sf3;
+ float pixel_offset;
+
+ float filter_radius0;
+ float filter_radius1;
+ float invscale0;
+ float invscale1;
+
+ uint32_t n_threads;
+ uint32_t chunks_per_group;
+
+ uint32_t dst_nrows_per_chunk;
+ uint32_t chunks_per_slice;
+ uint32_t total_chunks;
+ uint32_t vtcm_size;
+
+ struct fastdiv_values div_chunks_per_slice;
+ struct fastdiv_values div_dst_z;
+};
+
+#if defined(__cplusplus)
+static_assert(sizeof(struct htp_upscale_kernel_params) <= 128, "htp_upscale_kernel_params is too large");
+#else
+_Static_assert(sizeof(struct htp_upscale_kernel_params) <= 128, "htp_upscale_kernel_params is too large");
+#endif
+
+struct __attribute__((aligned(128))) upscale_bilinear_x_chunk {
+ int32_t off0[32];
+ int32_t off1[32];
+ float w0[32];
+ float dx[32];
+};
+
+struct upscale_bilinear_y {
+ uint32_t y0;
+ uint32_t y1;
+ float wy0;
+ float wy1;
+};
+
+struct upscale_bicubic_x {
+ int32_t x[4];
+ float w[4];
+};
+
+struct upscale_bicubic_y {
+ int32_t y[4];
+ float w[4];
+};
+
+struct upscale_antialias_x {
+ int32_t x_min;
+ int32_t x_max;
+ float xf;
+};
+
+struct upscale_antialias_y {
+ int32_t y_min;
+ int32_t y_max;
+ float yf;
+};
+
+struct htp_upscale_vtcm_layout {
+ size_t total_bytes;
+ size_t off_src;
+ size_t off_dst;
+ size_t off_r;
+ size_t off_gather;
+ size_t src_bytes_per_thread;
+ size_t dst_bytes_per_thread;
+ size_t r_bytes_per_thread;
+ size_t gather_bytes_per_thread;
+ size_t src_spad_half_size;
+ size_t dst_spad_half_size;
+ uint32_t max_src_chunk_rows;
+ uint32_t dst_chunk_rows;
+};
+
+static inline void htp_upscale_vtcm_layout_build(
+ struct htp_upscale_vtcm_layout * L,
+ uint32_t src_x,
+ uint32_t src_y,
+ uint32_t dst_x,
+ uint32_t dst_y,
+ uint32_t dst_chunk_rows,
+ float inv_sf1,
+ float filter_radius1,
+ uint32_t n_threads
+) {
+ uint32_t max_src_rows;
+ if (dst_chunk_rows >= dst_y) {
+ max_src_rows = src_y;
+ } else {
+ const float est = (float) dst_chunk_rows * inv_sf1 + 2.0f * filter_radius1;
+ max_src_rows = (uint32_t) ceilf(est) + 4;
+ if (max_src_rows > src_y) {
+ max_src_rows = src_y;
+ }
+ }
+
+ const uint32_t src_chunk_bytes = src_x * max_src_rows * sizeof(float);
+ const uint32_t dst_chunk_bytes = dst_x * dst_chunk_rows * sizeof(float);
+
+ L->max_src_chunk_rows = max_src_rows;
+ L->dst_chunk_rows = dst_chunk_rows;
+ L->src_spad_half_size = hex_round_up(src_chunk_bytes + 256, 128);
+ L->dst_spad_half_size = hex_round_up(dst_chunk_bytes, 128);
+ L->src_bytes_per_thread = 2 * L->src_spad_half_size;
+ L->dst_bytes_per_thread = 2 * L->dst_spad_half_size;
+ L->r_bytes_per_thread = hex_round_up(src_x * sizeof(float), 128);
+ L->gather_bytes_per_thread = 256;
+
+ size_t off = 0;
+ VTCM_LAYOUT_ALLOC(off, off_src, L->src_bytes_per_thread * n_threads);
+ VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads);
+ VTCM_LAYOUT_ALLOC(off, off_r, L->r_bytes_per_thread * n_threads);
+ VTCM_LAYOUT_ALLOC(off, off_gather, L->gather_bytes_per_thread * n_threads);
+ L->total_bytes = off;
+}
+
+static inline bool htp_upscale_solve_layout(
+ struct htp_upscale_vtcm_layout * layout,
+ uint32_t src_x,
+ uint32_t src_y,
+ uint32_t dst_x,
+ uint32_t dst_y,
+ float inv_sf1,
+ float filter_radius1,
+ uint32_t n_threads,
+ size_t vtcm_budget,
+ uint32_t * out_dst_chunk_rows
+) {
+ // 1. Try full plane
+ uint32_t chunk_rows = dst_y;
+ htp_upscale_vtcm_layout_build(layout, src_x, src_y, dst_x, dst_y, chunk_rows, inv_sf1, filter_radius1, n_threads);
+ if (layout->total_bytes <= vtcm_budget) {
+ *out_dst_chunk_rows = chunk_rows;
+ return true;
+ }
+
+ // 2. Try smaller chunk bands: 64, 32, 16, 8, 4, 2, 1
+ chunk_rows = 64;
+ while (chunk_rows > 1) {
+ if (chunk_rows < dst_y) {
+ htp_upscale_vtcm_layout_build(layout, src_x, src_y, dst_x, dst_y, chunk_rows, inv_sf1, filter_radius1, n_threads);
+ if (layout->total_bytes <= vtcm_budget) {
+ *out_dst_chunk_rows = chunk_rows;
+ return true;
+ }
+ }
+ chunk_rows /= 2;
+ }
+
+ // 3. Fallback to 1 row
+ chunk_rows = 1;
+ htp_upscale_vtcm_layout_build(layout, src_x, src_y, dst_x, dst_y, chunk_rows, inv_sf1, filter_radius1, n_threads);
+ if (layout->total_bytes <= vtcm_budget) {
+ *out_dst_chunk_rows = chunk_rows;
+ return true;
+ }
+
+ return false;
+}
+
+int op_upscale(struct htp_ops_context * octx);
+
+#endif // HTP_UPSCALE_OPS_H
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index deddef330..b4740f7b6 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -8179,21 +8179,32 @@ struct test_interpolate : public test_case {
const std::array<int64_t, 4> ne;
const std::array<int64_t, 4> ne_tgt;
const ggml_scale_mode mode = GGML_SCALE_MODE_NEAREST;
+ const std::array<int64_t, 4> permute;
+ bool _use_permute;
std::string vars() override {
- return VARS_TO_STR4(type, ne, ne_tgt, mode);
+ std::string v = VARS_TO_STR4(type, ne, ne_tgt, mode);
+ if (_use_permute) v += "," + VAR_TO_STR(permute);
+ return v;
}
test_interpolate(ggml_type type = GGML_TYPE_F32,
- std::array<int64_t, 4> ne = {2, 5, 7, 11},
- std::array<int64_t, 4> ne_tgt = {5, 7, 11, 13},
- ggml_scale_mode mode = GGML_SCALE_MODE_NEAREST)
- : type(type), ne(ne), ne_tgt(ne_tgt), mode(mode) {}
+ std::array<int64_t, 4> ne = {2, 5, 7, 11},
+ std::array<int64_t, 4> ne_tgt = {5, 7, 11, 13},
+ ggml_scale_mode mode = GGML_SCALE_MODE_NEAREST,
+ std::array<int64_t, 4> permute = {0, 0, 0, 0})
+ : type(type), ne(ne), ne_tgt(ne_tgt), mode(mode), permute(permute),
+ _use_permute(permute[0] + permute[1] + permute[2] + permute[3] > 0) {}
ggml_tensor * build_graph(ggml_context * ctx) override {
ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data());
ggml_set_name(a, "a");
+ if (_use_permute) {
+ a = ggml_permute(ctx, a, permute[0], permute[1], permute[2], permute[3]);
+ ggml_set_name(a, "a_permuted");
+ }
+
ggml_tensor * out = ggml_interpolate(ctx, a, ne_tgt[0], ne_tgt[1],ne_tgt[2], ne_tgt[3], mode);
ggml_set_name(out, "out");
@@ -11969,6 +11980,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_interpolate(GGML_TYPE_F32, {2, 5, 7, 11}, {5, 7, 11, 13}, (ggml_scale_mode)(mode | GGML_SCALE_FLAG_ALIGN_CORNERS)));
test_cases.emplace_back(new test_interpolate(GGML_TYPE_F32, {1, 4, 3, 2}, {2, 8, 3, 2}, (ggml_scale_mode)(mode | GGML_SCALE_FLAG_ALIGN_CORNERS)));
test_cases.emplace_back(new test_interpolate(GGML_TYPE_F32, {4, 1, 3, 2}, {1, 1, 3, 2}, (ggml_scale_mode)(mode | GGML_SCALE_FLAG_ALIGN_CORNERS)));
+ test_cases.emplace_back(new test_interpolate(GGML_TYPE_F32, {64, 48, 48, 1}, {156, 104, 64, 1}, (ggml_scale_mode)(mode | GGML_SCALE_FLAG_ALIGN_CORNERS), {2, 0, 1, 3}));
}
test_cases.emplace_back(new test_sum());