Commit ef4247f35d for openssl.org
commit ef4247f35d4c1e2f710cfaa4be7ea85eed48fb22
Author: Julian Zhu <julian.oerv@isrc.iscas.ac.cn>
Date: Mon May 11 23:30:12 2026 +0800
RISC-V: GHASH: Zvbc multi-block aggregation
The initial Zvbc GHASH correctly computes GF(2^128) products via
hardware clmul but processes one block per iteration each time,
leaving register resources underutilized.
Expand on the initial implementation: precompute H^2/H^3/H^4 and
batch 2 or 4 products into shared register pairs before a single
reduction.
Signed-off-by: Julian Zhu <julian.oerv@isrc.iscas.ac.cn>
Reviewed-by: Neil Horman <nhorman@openssl.org>
Reviewed-by: Mounir Idrassi <mounir.idrassi@idrix.fr>
Merge-date: Thu Sep 17 12:32:48 2026
Merged-from: https://github.com/openssl/openssl/pull/31178
diff --git a/crypto/modes/asm/ghash-riscv64-zvkb-zvbc.pl b/crypto/modes/asm/ghash-riscv64-zvkb-zvbc.pl
index 5eb748bdc2..c62a7e3296 100644
--- a/crypto/modes/asm/ghash-riscv64-zvkb-zvbc.pl
+++ b/crypto/modes/asm/ghash-riscv64-zvkb-zvbc.pl
@@ -2,7 +2,7 @@
# This file is dual-licensed, meaning that you can use it under your
# choice of either of the following two licenses:
#
-# Copyright 2023 The OpenSSL Project Authors. All Rights Reserved.
+# Copyright 2023-2026 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the Apache License 2.0 (the "License"). You can obtain
# a copy in the file LICENSE in the source distribution or at
@@ -11,6 +11,7 @@
# or
#
# Copyright (c) 2023, Christoph Müllner <christoph.muellner@vrull.eu>
+# Copyright (c) 2026, Julian Zhu <julian.oerv@isrc.iscas.ac.cn>
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
@@ -58,6 +59,12 @@ my $code=<<___;
.text
___
+my ($V0, $V1, $V2, $V3, $V4, $V5, $V6, $V7,
+ $V8, $V9, $V10, $V11, $V12, $V13, $V14, $V15,
+ $V16, $V17, $V18, $V19, $V20, $V21, $V22, $V23,
+ $V24, $V25, $V26, $V27, $V28, $V29, $V30, $V31,
+) = map("v$_",(0..31));
+
################################################################################
# void gcm_init_rv64i_zvkb_zvbc(u128 Htable[16], const u64 H[2]);
#
@@ -65,8 +72,7 @@ ___
# output: Htable: Preprocessed key data for gcm_gmult_rv64i_zvkb_zvbc and
# gcm_ghash_rv64i_zvkb_zvbc
{
-my ($Htable,$H,$TMP0,$TMP1,$TMP2) = ("a0","a1","t0","t1","t2");
-my ($V0,$V1,$V2,$V3,$V4,$V5,$V6) = ("v0","v1","v2","v3","v4","v5","v6");
+my ($Htable,$H,$TMP0,$TMP1,$TMP2,$TMP3) = ("a0","a1","t0","t1","t2","t3");
$code .= <<___;
.p2align 3
@@ -108,7 +114,139 @@ gcm_init_rv64i_zvkb_zvbc:
@{[vxor_vv_v0t $V1, $V1, $V2]} # vxor.vv v1, v1, v2, v0.t
- @{[vse64_v $V1, $Htable]} # vse64.v v1, (a0)
+ @{[vse64_v $V1, $Htable]} # vse64.v v1, (a0) -- store H
+
+ # --- Compute H^2 = H * H for multi-block aggregation ---
+ ld $TMP0, 0($Htable)
+ ld $TMP1, 8($Htable)
+ la $TMP3, Lpolymod
+ ld $TMP3, 8($TMP3)
+
+ @{[vmv_v_v $V5, $V1]} # copy H to v5
+
+ # Schoolbook multiply: H * H
+ @{[vclmul_vx $V1, $V5, $TMP0]}
+ @{[vclmulh_vx $V3, $V5, $TMP0]}
+ @{[vclmul_vx $V4, $V5, $TMP1]}
+ @{[vclmulh_vx $V2, $V5, $TMP1]}
+
+ @{[vslideup_vi $V5, $V3, 1]}
+ @{[vslideup_vi $V6, $V4, 1]}
+ @{[vslidedown_vi $V3, $V3, 1]}
+ @{[vslidedown_vi $V4, $V4, 1]}
+
+ @{[vmv_v_i $V0, 1]}
+ @{[vxor_vv_v0t $V2, $V2, $V3]}
+ @{[vxor_vv_v0t $V2, $V2, $V4]}
+
+ @{[vmv_v_i $V0, 2]}
+ @{[vxor_vv_v0t $V1, $V1, $V5]}
+ @{[vxor_vv_v0t $V1, $V1, $V6]}
+
+ # Gueron reduction
+ @{[vslideup_vi_v0t $V3, $V1, 1]}
+ @{[vclmul_vx_v0t $V3, $V3, $TMP3]}
+ @{[vxor_vv_v0t $V1, $V1, $V3]}
+
+ @{[vclmul_vx_v0t $V3, $V1, $TMP3]}
+ @{[vclmulh_vx $V4, $V1, $TMP3]}
+
+ @{[vmv_v_i $V0, 1]}
+ @{[vslidedown_vi $V3, $V3, 1]}
+
+ @{[vxor_vv $V1, $V1, $V4]}
+ @{[vxor_vv_v0t $V1, $V1, $V3]}
+
+ @{[vxor_vv $V2, $V2, $V1]}
+
+ # Store H^2 at Htable+16
+ addi $Htable, $Htable, 16
+ @{[vse64_v $V2, $Htable]}
+
+ # --- Compute H^3 = H^2 * H ---
+ @{[vmv_v_v $V5, $V2]} # v5 = H^2
+
+ @{[vclmul_vx $V1, $V5, $TMP0]}
+ @{[vclmulh_vx $V3, $V5, $TMP0]}
+ @{[vclmul_vx $V4, $V5, $TMP1]}
+ @{[vclmulh_vx $V2, $V5, $TMP1]}
+
+ @{[vslideup_vi $V5, $V3, 1]}
+ @{[vslideup_vi $V6, $V4, 1]}
+ @{[vslidedown_vi $V3, $V3, 1]}
+ @{[vslidedown_vi $V4, $V4, 1]}
+
+ @{[vmv_v_i $V0, 1]}
+ @{[vxor_vv_v0t $V2, $V2, $V3]}
+ @{[vxor_vv_v0t $V2, $V2, $V4]}
+
+ @{[vmv_v_i $V0, 2]}
+ @{[vxor_vv_v0t $V1, $V1, $V5]}
+ @{[vxor_vv_v0t $V1, $V1, $V6]}
+
+ @{[vslideup_vi_v0t $V3, $V1, 1]}
+ @{[vclmul_vx_v0t $V3, $V3, $TMP3]}
+ @{[vxor_vv_v0t $V1, $V1, $V3]}
+
+ @{[vclmul_vx_v0t $V3, $V1, $TMP3]}
+ @{[vclmulh_vx $V4, $V1, $TMP3]}
+
+ @{[vmv_v_i $V0, 1]}
+ @{[vslidedown_vi $V3, $V3, 1]}
+
+ @{[vxor_vv $V1, $V1, $V4]}
+ @{[vxor_vv_v0t $V1, $V1, $V3]}
+
+ @{[vxor_vv $V2, $V2, $V1]}
+
+ # Store H^3 at Htable+32
+ addi $Htable, $Htable, 16
+ @{[vse64_v $V2, $Htable]}
+
+ # --- Compute H^4 = H^2 * H^2 ---
+ # Load H^2 vector and scalar halves from Htable-16
+ addi $TMP2, $Htable, -16
+ ld $TMP0, 0($TMP2)
+ ld $TMP1, 8($TMP2)
+ @{[vle64_v $V5, $TMP2]}
+
+ @{[vclmul_vx $V1, $V5, $TMP0]}
+ @{[vclmulh_vx $V3, $V5, $TMP0]}
+ @{[vclmul_vx $V4, $V5, $TMP1]}
+ @{[vclmulh_vx $V2, $V5, $TMP1]}
+
+ @{[vslideup_vi $V5, $V3, 1]}
+ @{[vslideup_vi $V6, $V4, 1]}
+ @{[vslidedown_vi $V3, $V3, 1]}
+ @{[vslidedown_vi $V4, $V4, 1]}
+
+ @{[vmv_v_i $V0, 1]}
+ @{[vxor_vv_v0t $V2, $V2, $V3]}
+ @{[vxor_vv_v0t $V2, $V2, $V4]}
+
+ @{[vmv_v_i $V0, 2]}
+ @{[vxor_vv_v0t $V1, $V1, $V5]}
+ @{[vxor_vv_v0t $V1, $V1, $V6]}
+
+ @{[vslideup_vi_v0t $V3, $V1, 1]}
+ @{[vclmul_vx_v0t $V3, $V3, $TMP3]}
+ @{[vxor_vv_v0t $V1, $V1, $V3]}
+
+ @{[vclmul_vx_v0t $V3, $V1, $TMP3]}
+ @{[vclmulh_vx $V4, $V1, $TMP3]}
+
+ @{[vmv_v_i $V0, 1]}
+ @{[vslidedown_vi $V3, $V3, 1]}
+
+ @{[vxor_vv $V1, $V1, $V4]}
+ @{[vxor_vv_v0t $V1, $V1, $V3]}
+
+ @{[vxor_vv $V2, $V2, $V1]}
+
+ # Store H^4 at Htable+48
+ addi $Htable, $Htable, 16
+ @{[vse64_v $V2, $Htable]}
+
ret
.size gcm_init_rv64i_zvkb_zvbc,.-gcm_init_rv64i_zvkb_zvbc
___
@@ -122,7 +260,6 @@ ___
# output: Xi: next hash value Xi = (Xi * H mod f)
{
my ($Xi,$Htable,$TMP0,$TMP1,$TMP2,$TMP3,$TMP4) = ("a0","a1","t0","t1","t2","t3","t4");
-my ($V0,$V1,$V2,$V3,$V4,$V5,$V6) = ("v0","v1","v2","v3","v4","v5","v6");
$code .= <<___;
.text
@@ -243,123 +380,324 @@ ___
# output: Xi: Xi+1 (next hash value Xi)
{
my ($Xi,$Htable,$inp,$len,$TMP0,$TMP1,$TMP2,$TMP3,$M8,$TMP5,$TMP6) = ("a0","a1","a2","a3","t0","t1","t2","t3","t4","t5","t6");
-my ($V0,$V1,$V2,$V3,$V4,$V5,$V6,$Vinp) = ("v0","v1","v2","v3","v4","v5","v6","v7");
+my ($s0,$s1,$s2,$s3) = ("s0","s1","s2","s3");
$code .= <<___;
.p2align 3
.globl gcm_ghash_rv64i_zvkb_zvbc
.type gcm_ghash_rv64i_zvkb_zvbc,\@function
gcm_ghash_rv64i_zvkb_zvbc:
+ # Load H (for single-block and second multiply in 2-block)
ld $TMP0, ($Htable)
ld $TMP1, 8($Htable)
- li $TMP2, 63
+ # Load H^2 (for first multiply in 2-block)
+ ld $TMP2, 16($Htable)
+ ld $TMP5, 24($Htable)
la $TMP3, Lpolymod
ld $TMP3, 8($TMP3)
- # Load/store data in reverse order.
- # This is needed as a part of endianness swap.
+ # Load/store data in reverse order for endianness swap.
add $Xi, $Xi, 8
add $inp, $inp, 8
li $M8, -8
@{[vsetivli__x0_2_e64_m1_tu_mu]} # vsetivli x0, 2, e64, m1, tu, mu
- @{[vlse64_v $V5, $Xi, $M8]} # vlse64.v v5, (a0), t4
+ @{[vlse64_v $V5, $Xi, $M8]} # load Xi (word-swapped)
+
+ # Check for 4-block path (len >= 64)
+ li $TMP6, 64
+ blt $len, $TMP6, Lcheck_2x
+
+ # --- 4-block aggregation path ---
+ # Save callee-saved registers
+ addi sp, sp, -32
+ sd s0, 0(sp)
+ sd s1, 8(sp)
+ sd s2, 16(sp)
+ sd s3, 24(sp)
+
+ # Load H^3, H^4
+ ld s0, 32($Htable) # H^3_lo
+ ld s1, 40($Htable) # H^3_hi
+ ld s2, 48($Htable) # H^4_lo
+ ld s3, 56($Htable) # H^4_hi
+
+Lstep_4x:
+ # === Block 1: (Xi ^ C1) * H^4 ===
+ @{[vlse64_v $V7, $inp, $M8]}
+ add $inp, $inp, 16
+ @{[vxor_vv $V5, $V5, $V7]}
+ @{[vrev8_v $V5, $V5]}
+
+ @{[vclmul_vx $V1, $V5, $s2]}
+ @{[vclmulh_vx $V3, $V5, $s2]}
+ @{[vclmul_vx $V4, $V5, $s3]}
+ @{[vclmulh_vx $V2, $V5, $s3]}
+
+ @{[vslideup_vi $V5, $V3, 1]}
+ @{[vslideup_vi $V6, $V4, 1]}
+ @{[vslidedown_vi $V3, $V3, 1]}
+ @{[vslidedown_vi $V4, $V4, 1]}
+
+ @{[vmv_v_i $V0, 1]}
+ @{[vxor_vv_v0t $V2, $V2, $V3]}
+ @{[vxor_vv_v0t $V2, $V2, $V4]}
+ @{[vmv_v_i $V0, 2]}
+ @{[vxor_vv_v0t $V1, $V1, $V5]}
+ @{[vxor_vv_v0t $V1, $V1, $V6]}
+
+ # Save product 1
+ @{[vmv_v_v $V8, $V2]}
+ @{[vmv_v_v $V9, $V1]}
+
+ # === Block 2: C2 * H^3 ===
+ @{[vlse64_v $V7, $inp, $M8]}
+ add $inp, $inp, 16
+ @{[vrev8_v $V5, $V7]}
+
+ @{[vclmul_vx $V1, $V5, $s0]}
+ @{[vclmulh_vx $V3, $V5, $s0]}
+ @{[vclmul_vx $V4, $V5, $s1]}
+ @{[vclmulh_vx $V2, $V5, $s1]}
+
+ @{[vslideup_vi $V5, $V3, 1]}
+ @{[vslideup_vi $V6, $V4, 1]}
+ @{[vslidedown_vi $V3, $V3, 1]}
+ @{[vslidedown_vi $V4, $V4, 1]}
+
+ @{[vmv_v_i $V0, 1]}
+ @{[vxor_vv_v0t $V2, $V2, $V3]}
+ @{[vxor_vv_v0t $V2, $V2, $V4]}
+ @{[vmv_v_i $V0, 2]}
+ @{[vxor_vv_v0t $V1, $V1, $V5]}
+ @{[vxor_vv_v0t $V1, $V1, $V6]}
+
+ # Accumulate product 2
+ @{[vxor_vv $V8, $V8, $V2]}
+ @{[vxor_vv $V9, $V9, $V1]}
+
+ # === Block 3: C3 * H^2 ===
+ @{[vlse64_v $V7, $inp, $M8]}
+ add $inp, $inp, 16
+ @{[vrev8_v $V5, $V7]}
+
+ @{[vclmul_vx $V1, $V5, $TMP2]}
+ @{[vclmulh_vx $V3, $V5, $TMP2]}
+ @{[vclmul_vx $V4, $V5, $TMP5]}
+ @{[vclmulh_vx $V2, $V5, $TMP5]}
+
+ @{[vslideup_vi $V5, $V3, 1]}
+ @{[vslideup_vi $V6, $V4, 1]}
+ @{[vslidedown_vi $V3, $V3, 1]}
+ @{[vslidedown_vi $V4, $V4, 1]}
+
+ @{[vmv_v_i $V0, 1]}
+ @{[vxor_vv_v0t $V2, $V2, $V3]}
+ @{[vxor_vv_v0t $V2, $V2, $V4]}
+ @{[vmv_v_i $V0, 2]}
+ @{[vxor_vv_v0t $V1, $V1, $V5]}
+ @{[vxor_vv_v0t $V1, $V1, $V6]}
+
+ # Accumulate product 3
+ @{[vxor_vv $V8, $V8, $V2]}
+ @{[vxor_vv $V9, $V9, $V1]}
+
+ # === Block 4: C4 * H ===
+ @{[vlse64_v $V7, $inp, $M8]}
+ add $inp, $inp, 16
+ add $len, $len, -64
+ @{[vrev8_v $V5, $V7]}
-Lstep:
- # Read input data
- @{[vlse64_v $Vinp, $inp, $M8]} # vle64.v v0, (a2)
+ @{[vclmul_vx $V1, $V5, $TMP0]}
+ @{[vclmulh_vx $V3, $V5, $TMP0]}
+ @{[vclmul_vx $V4, $V5, $TMP1]}
+ @{[vclmulh_vx $V2, $V5, $TMP1]}
+
+ @{[vslideup_vi $V5, $V3, 1]}
+ @{[vslideup_vi $V6, $V4, 1]}
+ @{[vslidedown_vi $V3, $V3, 1]}
+ @{[vslidedown_vi $V4, $V4, 1]}
+
+ @{[vmv_v_i $V0, 1]}
+ @{[vxor_vv_v0t $V2, $V2, $V3]}
+ @{[vxor_vv_v0t $V2, $V2, $V4]}
+ @{[vmv_v_i $V0, 2]}
+ @{[vxor_vv_v0t $V1, $V1, $V5]}
+ @{[vxor_vv_v0t $V1, $V1, $V6]}
+
+ # Combine all 4 products
+ @{[vxor_vv $V2, $V2, $V8]}
+ @{[vxor_vv $V1, $V1, $V9]}
+
+ # Single Gueron reduction for all 4 blocks
+ # v0 = 2 from above
+ @{[vslideup_vi_v0t $V3, $V1, 1]}
+ @{[vclmul_vx_v0t $V3, $V3, $TMP3]}
+ @{[vxor_vv_v0t $V1, $V1, $V3]}
+
+ @{[vclmul_vx_v0t $V3, $V1, $TMP3]}
+ @{[vclmulh_vx $V4, $V1, $TMP3]}
+
+ @{[vmv_v_i $V0, 1]}
+ @{[vslidedown_vi $V3, $V3, 1]}
+
+ @{[vxor_vv $V1, $V1, $V4]}
+ @{[vxor_vv_v0t $V1, $V1, $V3]}
+
+ @{[vxor_vv $V2, $V2, $V1]}
+
+ @{[vrev8_v $V5, $V2]}
+
+ li $TMP6, 64
+ bge $len, $TMP6, Lstep_4x
+
+ # Restore callee-saved registers
+ ld s0, 0(sp)
+ ld s1, 8(sp)
+ ld s2, 16(sp)
+ ld s3, 24(sp)
+ addi sp, sp, 32
+
+Lcheck_2x:
+ # Check for 2-block path (len >= 32)
+ li $TMP6, 32
+ blt $len, $TMP6, Lcheck_1x
+
+Lstep_2x:
+ # === 2-block iteration: (Xi ^ C1) * H^2 + C2 * H ===
+
+ # Block 1: load C1, XOR with Xi
+ @{[vlse64_v $V7, $inp, $M8]}
add $inp, $inp, 16
add $len, $len, -16
- # XOR them into Xi
- @{[vxor_vv $V5, $V5, $Vinp]} # vxor.vv v0, v0, v1
+ @{[vxor_vv $V5, $V5, $V7]}
+ @{[vrev8_v $V5, $V5]}
+
+ # Schoolbook multiply (Xi ^ C1) * H^2 -> product in (v2, v1)
+ @{[vclmul_vx $V1, $V5, $TMP2]}
+ @{[vclmulh_vx $V3, $V5, $TMP2]}
+ @{[vclmul_vx $V4, $V5, $TMP5]}
+ @{[vclmulh_vx $V2, $V5, $TMP5]}
+
+ @{[vslideup_vi $V5, $V3, 1]}
+ @{[vslideup_vi $V6, $V4, 1]}
+ @{[vslidedown_vi $V3, $V3, 1]}
+ @{[vslidedown_vi $V4, $V4, 1]}
+
+ @{[vmv_v_i $V0, 1]}
+ @{[vxor_vv_v0t $V2, $V2, $V3]}
+ @{[vxor_vv_v0t $V2, $V2, $V4]}
+ @{[vmv_v_i $V0, 2]}
+ @{[vxor_vv_v0t $V1, $V1, $V5]}
+ @{[vxor_vv_v0t $V1, $V1, $V6]}
+
+ # Save first product
+ @{[vmv_v_v $V8, $V2]}
+ @{[vmv_v_v $V9, $V1]}
+
+ # Block 2: load C2 (no XOR with Xi)
+ @{[vlse64_v $V7, $inp, $M8]}
+ add $inp, $inp, 16
+ add $len, $len, -16
+ @{[vrev8_v $V5, $V7]}
- @{[vrev8_v $V5, $V5]} # vrev8.v v5, v5
+ # Schoolbook multiply C2 * H -> product in (v2, v1)
+ @{[vclmul_vx $V1, $V5, $TMP0]}
+ @{[vclmulh_vx $V3, $V5, $TMP0]}
+ @{[vclmul_vx $V4, $V5, $TMP1]}
+ @{[vclmulh_vx $V2, $V5, $TMP1]}
- # Multiplication
+ @{[vslideup_vi $V5, $V3, 1]}
+ @{[vslideup_vi $V6, $V4, 1]}
+ @{[vslidedown_vi $V3, $V3, 1]}
+ @{[vslidedown_vi $V4, $V4, 1]}
- # Do two 64x64 multiplications in one go to save some time
- # and simplify things.
+ @{[vmv_v_i $V0, 1]}
+ @{[vxor_vv_v0t $V2, $V2, $V3]}
+ @{[vxor_vv_v0t $V2, $V2, $V4]}
+ @{[vmv_v_i $V0, 2]}
+ @{[vxor_vv_v0t $V1, $V1, $V5]}
+ @{[vxor_vv_v0t $V1, $V1, $V6]}
- # A = a1a0 (t1, t0)
- # B = b1b0 (v5)
- # C = c1c0 (256 bit)
- # c1 = a1b1 + (a0b1)h + (a1b0)h
- # c0 = a0b0 + (a0b1)l + (a1b0)h
+ # Combine products: (v2,v1) += (v8,v9)
+ @{[vxor_vv $V2, $V2, $V8]}
+ @{[vxor_vv $V1, $V1, $V9]}
- # v1 = (a0b1)l,(a0b0)l
- @{[vclmul_vx $V1, $V5, $TMP0]} # vclmul.vx v1, v5, t0
- # v3 = (a0b1)h,(a0b0)h
- @{[vclmulh_vx $V3, $V5, $TMP0]} # vclmulh.vx v3, v5, t0
+ # Single Gueron reduction for both blocks
+ # v0 = 2 from above
+ @{[vslideup_vi_v0t $V3, $V1, 1]}
+ @{[vclmul_vx_v0t $V3, $V3, $TMP3]}
+ @{[vxor_vv_v0t $V1, $V1, $V3]}
- # v4 = (a1b1)l,(a1b0)l
- @{[vclmul_vx $V4, $V5, $TMP1]} # vclmul.vx v4, v5, t1
- # v2 = (a1b1)h,(a1b0)h
- @{[vclmulh_vx $V2, $V5, $TMP1]} # vclmulh.vx v2, v5, t1
+ @{[vclmul_vx_v0t $V3, $V1, $TMP3]}
+ @{[vclmulh_vx $V4, $V1, $TMP3]}
- # Is there a better way to do this?
- # Would need to swap the order of elements within a vector register.
- @{[vslideup_vi $V5, $V3, 1]} # vslideup.vi v5, v3, 1
- @{[vslideup_vi $V6, $V4, 1]} # vslideup.vi v6, v4, 1
- @{[vslidedown_vi $V3, $V3, 1]} # vslidedown.vi v3, v3, 1
- @{[vslidedown_vi $V4, $V4, 1]} # vslidedown.vi v4, v4, 1
+ @{[vmv_v_i $V0, 1]}
+ @{[vslidedown_vi $V3, $V3, 1]}
- @{[vmv_v_i $V0, 1]} # vmv.v.i v0, 1
- # v2 += (a0b1)h
- @{[vxor_vv_v0t $V2, $V2, $V3]} # vxor.vv v2, v2, v3, v0.t
- # v2 += (a1b1)l
- @{[vxor_vv_v0t $V2, $V2, $V4]} # vxor.vv v2, v2, v4, v0.t
+ @{[vxor_vv $V1, $V1, $V4]}
+ @{[vxor_vv_v0t $V1, $V1, $V3]}
- @{[vmv_v_i $V0, 2]} # vmv.v.i v0, 2
- # v1 += (a0b0)h,0
- @{[vxor_vv_v0t $V1, $V1, $V5]} # vxor.vv v1, v1, v5, v0.t
- # v1 += (a1b0)l,0
- @{[vxor_vv_v0t $V1, $V1, $V6]} # vxor.vv v1, v1, v6, v0.t
+ @{[vxor_vv $V2, $V2, $V1]}
- # Now the 256bit product should be stored in (v2,v1)
- # v1 = (a0b1)l + (a0b0)h + (a1b0)l, (a0b0)l
- # v2 = (a1b1)h, (a1b0)h + (a0b1)h + (a1b1)l
+ @{[vrev8_v $V5, $V2]}
- # Reduction
- # Let C := A*B = c3,c2,c1,c0 = v2[1],v2[0],v1[1],v1[0]
- # This is a slight variation of the Gueron's Montgomery reduction.
- # The difference being the order of some operations has been changed,
- # to make a better use of vclmul(h) instructions.
+ li $TMP6, 32
+ bge $len, $TMP6, Lstep_2x
- # First step:
- # c1 += (c0 * P)l
- # vmv.v.i v0, 2
- @{[vslideup_vi_v0t $V3, $V1, 1]} # vslideup.vi v3, v1, 1, v0.t
- @{[vclmul_vx_v0t $V3, $V3, $TMP3]} # vclmul.vx v3, v3, t3, v0.t
- @{[vxor_vv_v0t $V1, $V1, $V3]} # vxor.vv v1, v1, v3, v0.t
+Lcheck_1x:
+ # Check for remaining single block
+ beqz $len, Ldone
- # Second step:
- # D = d1,d0 is final result
- # We want:
- # m1 = c1 + (c1 * P)h
- # m0 = (c1 * P)l + (c0 * P)h + c0
- # d1 = c3 + m1
- # d0 = c2 + m0
+Lstep:
+ # === Single-block: (Xi ^ block) * H ===
+ @{[vlse64_v $V7, $inp, $M8]}
+ add $inp, $inp, 16
+ add $len, $len, -16
+ @{[vxor_vv $V5, $V5, $V7]}
+ @{[vrev8_v $V5, $V5]}
- #v3 = (c1 * P)l, 0
- @{[vclmul_vx_v0t $V3, $V1, $TMP3]} # vclmul.vx v3, v1, t3, v0.t
- #v4 = (c1 * P)h, (c0 * P)h
- @{[vclmulh_vx $V4, $V1, $TMP3]} # vclmulh.vx v4, v1, t3
+ # Schoolbook multiply * H
+ @{[vclmul_vx $V1, $V5, $TMP0]}
+ @{[vclmulh_vx $V3, $V5, $TMP0]}
+ @{[vclmul_vx $V4, $V5, $TMP1]}
+ @{[vclmulh_vx $V2, $V5, $TMP1]}
- @{[vmv_v_i $V0, 1]} # vmv.v.i v0, 1
- @{[vslidedown_vi $V3, $V3, 1]} # vslidedown.vi v3, v3, 1
+ @{[vslideup_vi $V5, $V3, 1]}
+ @{[vslideup_vi $V6, $V4, 1]}
+ @{[vslidedown_vi $V3, $V3, 1]}
+ @{[vslidedown_vi $V4, $V4, 1]}
- @{[vxor_vv $V1, $V1, $V4]} # vxor.vv v1, v1, v4
- @{[vxor_vv_v0t $V1, $V1, $V3]} # vxor.vv v1, v1, v3, v0.t
+ @{[vmv_v_i $V0, 1]}
+ @{[vxor_vv_v0t $V2, $V2, $V3]}
+ @{[vxor_vv_v0t $V2, $V2, $V4]}
+ @{[vmv_v_i $V0, 2]}
+ @{[vxor_vv_v0t $V1, $V1, $V5]}
+ @{[vxor_vv_v0t $V1, $V1, $V6]}
- # XOR in the upper upper part of the product
- @{[vxor_vv $V2, $V2, $V1]} # vxor.vv v2, v2, v1
+ # Gueron reduction
+ @{[vslideup_vi_v0t $V3, $V1, 1]}
+ @{[vclmul_vx_v0t $V3, $V3, $TMP3]}
+ @{[vxor_vv_v0t $V1, $V1, $V3]}
+
+ @{[vclmul_vx_v0t $V3, $V1, $TMP3]}
+ @{[vclmulh_vx $V4, $V1, $TMP3]}
+
+ @{[vmv_v_i $V0, 1]}
+ @{[vslidedown_vi $V3, $V3, 1]}
+
+ @{[vxor_vv $V1, $V1, $V4]}
+ @{[vxor_vv_v0t $V1, $V1, $V3]}
+
+ @{[vxor_vv $V2, $V2, $V1]}
- @{[vrev8_v $V5, $V2]} # vrev8.v v2, v2
+ @{[vrev8_v $V5, $V2]}
bnez $len, Lstep
- @{[vsse64_v $V5, $Xi, $M8]} # vsse64.v v2, (a0), t4
+Ldone:
+ @{[vsse64_v $V5, $Xi, $M8]}
ret
.size gcm_ghash_rv64i_zvkb_zvbc,.-gcm_ghash_rv64i_zvkb_zvbc
___