Commit a1bd8501eb for openssl.org

commit a1bd8501eb5d3d265285f62206742e72c7410895
Author: slontis <shane.lontis@oracle.com>
Date:   Tue Sep 29 17:53:08 2026 +1000

    ARM: Add ML-KEM NTT optimizations.

    Assisted-by: OpenAI Codex:gpt-5.6-sol

    The code was generated by AI by referencing the c code and the FIPS standard.
    It was told to ignore additional references. An iterative approach was then
    used to improve the performance of the instructions performed in each layer.
    It uses the multiplication method from "Neon NTT: Faster Dilithium, Kyber, and Saber
    on Cortex-A72 and Apple M" Section 3.2.1 Barrett multiplication in 3-instructions.

    Reviewed-by: Paul Dale <paul.dale@oracle.com>
    Reviewed-by: Mounir Idrassi <mounir.idrassi@idrix.fr>
    Reviewed-by: Tom Cosgrove <tom.cosgrove@arm.com>
    Merge-date: Tue Nov 17 20:59:03 2026
    Merged-from: https://github.com/openssl/openssl/pull/33026

diff --git a/CHANGES.md b/CHANGES.md
index 8cac4bde01..adae608d14 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -53,6 +53,10 @@ OpenSSL 4.2

    *Matt Caswell*

+ * Added optimized ML-KEM NTT and inverse NTT implementations for AArch64.
+
+   *Shane Lontis*
+
  * Added `BIO_dgram_pair_set0_pcap_file()` which intercepts and writes UDP
    datagrams exchanged of over `BIO_dgram_pair` bio memory object to file
    in libpcap format. To enable `BIO_dgram_pair_set0_pcap_file()` the build
diff --git a/crypto/ml_kem/asm/mlkem-armv8.pl b/crypto/ml_kem/asm/mlkem-armv8.pl
new file mode 100644
index 0000000000..cf7c22a51a
--- /dev/null
+++ b/crypto/ml_kem/asm/mlkem-armv8.pl
@@ -0,0 +1,739 @@
+#! /usr/bin/env perl
+# Copyright 2026 The OpenSSL Project Authors. All Rights Reserved.
+#
+# Licensed under the Apache License 2.0 (the "License").  You may not use
+# this file except in compliance with the License.  You can obtain a copy
+# in the file LICENSE in the source distribution or at
+# https://www.openssl.org/source/license.html
+
+# AArch64 ML-KEM NTT/INTT, following FIPS 203 Section 4.3, Algorithms 9/10
+# and the interfaces in crypto/ml_kem/ml_kem.c.
+#
+# The transforms operate on 256 coefficients modulo q = 3329.  Seven layers
+# leave 128 coefficient pairs: forward lengths 128..2, inverse lengths 2..128.
+# Twiddle factors are z = 17^bitrev7(i) mod q; inverse layers use their inverses.
+# Each z has a reduction constant c = round(2^15*z/q).
+# Inputs and outputs are canonical [0,q); intermediate lanes are signed 16-bit.
+# Twiddle multiplication: r = low16(x*z) - q*round(x*c/2^15).
+#
+# Eight halfword lanes are processed in parallel.  Lengths >= 16 use paired
+# vector butterflies; length 8 uses adjacent vectors.  Lengths 4 and 2 batch
+# several groups with lane-packed roots.
+#
+# Forward lazy-reduction bounds after layers 1 through 7:
+#
+#   [-1702,5030], [-3575,6903], [-5546,8874], [-7625,10953],
+#   [-9800,13128], [-12096,15424], [-14520,17848].
+#
+# These fit signed 16-bit lanes and require only one final canonicalization.
+# Inverse sum paths are center-reduced after layers 3 and 6, bounding the final
+# layer by [-4421,4421].  Scaling by the inverse of 128 modulo q (3303, or -26)
+# is fused with canonicalization.
+#
+# Long layers are unrolled; lengths 4 and 2 each loop over sixteen batches.
+# Final reduction/scaling passes each use eight 64-byte iterations.
+# Addresses and branch conditions depend only on public offsets and counters;
+# canonicalization uses sign masks.
+#
+# Only little-endian AArch64 is supported.  Big-endian builds retain the
+# generic C transforms because the vector loads and packing are endian-sensitive.
+#
+# Performance on Apple M3 Pro, macOS 26.7.1, Apple clang 21.0.0:
+#   ./Configure darwin64-arm64-cc no-shared --strict-warnings
+#   (default -O3), single-threaded, default provider.
+#   openssl speed -elapsed -seconds 2 ml-kem-512 ml-kem-768 ml-kem-1024
+# Median operations/second from three runs per backend, alternating backend
+# order between runs.  The generic baseline uses the same build with
+# MLKEM_NTT_ARMV8_ASM undefined only when compiling ml_kem.c; all other
+# assembly remains enabled.  These are full KEM operation rates, not isolated
+# transform timings.  Higher is better; gain is the increase in throughput.
+#
+# Parameter set   Operation     Generic C    AArch64    Gain
+# ML-KEM-512      Keygen           99,901    112,726   12.8%
+#                 Encaps          152,875    219,129   43.3%
+#                 Decaps          102,990    148,347   44.0%
+# ML-KEM-768      Keygen           61,821     68,922   11.5%
+#                 Encaps          105,366    145,407   38.0%
+#                 Decaps           71,698    101,336   41.3%
+# ML-KEM-1024     Keygen           39,804     43,923   10.4%
+#                 Encaps           74,128     98,779   33.3%
+#                 Decaps           52,570     70,457   34.0%
+use strict;
+use warnings;
+
+my $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
+my $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
+
+$0 =~ m|(.*[\\/])[^\\/]+$|;
+my $dir = $1;
+my $xlate;
+($xlate = "${dir}arm-xlate.pl" and -f $xlate) or
+($xlate = "${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
+die "can't locate arm-xlate.pl";
+
+open OUT, "| \"$^X\" $xlate $flavour \"$output\""
+    or die "can't call $xlate: $!";
+*STDOUT = *OUT;
+
+my $q = 3329;
+my $code = '';
+
+# Register aliases.
+my $coeff_base = 'x0';
+
+my ($packed_zc, $q_reg) = ('w9', 'w11');
+my $twiddle_ptr = 'x12';
+my ($coeff_ptr, $batches_left) = ('x1', 'w2');
+my ($a_v, $b_v, $product_v, $tmp_v, $sum_v, $batch_a_v) =
+    map("v$_", (0, 1, 2, 3, 5, 4));
+my ($a2_v, $b2_v, $product2_v, $tmp2_v, $sum2_v) =
+    map("v$_", (16, 17, 18, 19, 20));
+my ($a_q, $b_q, $product_q, $sum_q) =
+    map("q$_", (0, 1, 2, 5));
+my ($a2_q, $b2_q, $product2_q, $sum2_q) =
+    map("q$_", (16, 17, 18, 20));
+my ($barrett_v, $c_v, $z_v, $q_v) =
+    map("v$_", (28, 29, 30, 31));
+my ($c_q, $z_q) = map("q$_", (29, 30));
+# Indexed halfword operands must use v0-v15.  v6 holds z and c in lanes 0/1.
+my ($intt_zc_v, $intt_zc_s) = ('v6', 's6');
+
+# Inverse (z,c) pairs for lengths >= 8; c = round(2^15*z/q).
+my %intt_zc_constants = (
+    # Layer 3
+    8 => [
+        -821, -8081, 1355, 13338, 450, 4429, 936, 9213,
+        447, 4400, -535, -5266, 1235, 12156, -1426, -14036,
+        -1333, -13121, 1089, 10719, -56, -551, 283, 2786,
+        -1476, -14529, -1339, -13180, 882, 8682, -296, -2914,
+    ],
+    # Layer 4
+    16 => [
+        1583, 15582, -569, -5601, 69, 679, 543, 5345,
+        -797, -7845, -193, -1900, 1410, 13879, -1062, -10453,
+    ],
+    # Layer 5
+    32 => [
+        -848, -8347, 1432, 14095, -630, -6201, 687, 6762,
+    ],
+    # Layer 6
+    64 => [
+        40, 394, 749, 7373,
+    ],
+    # Layer 7
+    128 => [
+        1600, 15749,
+    ],
+);
+
+# Center the first len coefficients of each stride-sized inverse block.
+# Only these sum paths can exceed the later signed-16-bit bound.
+sub intt_reduce {
+    my ($len, $stride) = @_;
+    # Product halves are already reduced by twiddle multiplication.
+    for (my $base = 0; $base < 256; $base += $stride) {
+        for (my $j = 0; $j < $len; $j += 8) {
+            my $off = 2 * ($base + $j);
+            $code .= <<___;
+        ldr     $a_q, [$coeff_base, #$off]
+
+        sqrdmulh $tmp_v.8h, $a_v.8h, $barrett_v.8h
+        mls     $a_v.8h, $tmp_v.8h, $q_v.8h
+
+        str     $a_q, [$coeff_base, #$off]
+___
+        }
+    }
+}
+
+# Length-8 forward butterflies use one vector per half of each group.
+# (a,b) = (a+b*z,a-b*z) modulo q.
+sub ntt_layer8 {
+    for (my $start = 0; $start < 256; $start += 16) {
+        $code .= <<___;
+        ldp     $z_q, $c_q, [$twiddle_ptr], #32
+___
+        my $a = 2 * $start;
+        $code .= <<___;
+        ldp     $a_q, $b_q, [$coeff_base, #$a]
+
+        // Twiddle multiplication.
+        mul     $product_v.8h, $b_v.8h, $z_v.8h
+        sqrdmulh $tmp_v.8h, $b_v.8h, $c_v.8h
+        mls     $product_v.8h, $tmp_v.8h, $q_v.8h
+
+        // Butterfly sum/difference.
+        sub     $sum_v.8h, $a_v.8h, $product_v.8h
+        add     $a_v.8h, $a_v.8h, $product_v.8h
+
+        stp     $a_q, $sum_q, [$coeff_base, #$a]
+___
+    }
+}
+
+# Forward Cooley-Tukey butterflies, unrolled for lengths >= 16.
+# For each group with twiddle z:
+#   for each butterfly pair (a,b):
+#       t = b*z mod q
+#       (a,b) = (a+t,a-t)
+sub ntt_vector_layer {
+    my ($len) = @_;
+
+    for (my $start = 0; $start < 256; $start += 2 * $len) {
+        $code .= <<___;
+        ldp     $z_q, $c_q, [$twiddle_ptr], #32
+___
+        for (my $j = 0; $j < $len; $j += 16) {
+            my $a = 2 * ($start + $j);
+            my $b = 2 * ($start + $len + $j);
+
+            $code .= <<___;
+        ldp     $a_q, $a2_q, [$coeff_base, #$a]
+        ldp     $b_q, $b2_q, [$coeff_base, #$b]
+
+        // Twiddle multiplication.
+        mul     $product_v.8h, $b_v.8h, $z_v.8h
+        mul     $product2_v.8h, $b2_v.8h, $z_v.8h
+        sqrdmulh $tmp_v.8h, $b_v.8h, $c_v.8h
+        sqrdmulh $tmp2_v.8h, $b2_v.8h, $c_v.8h
+        mls     $product_v.8h, $tmp_v.8h, $q_v.8h
+        mls     $product2_v.8h, $tmp2_v.8h, $q_v.8h
+
+        // Butterfly sum/difference.
+        sub     $sum_v.8h, $a_v.8h, $product_v.8h
+        sub     $sum2_v.8h, $a2_v.8h, $product2_v.8h
+        add     $a_v.8h, $a_v.8h, $product_v.8h
+        add     $a2_v.8h, $a2_v.8h, $product2_v.8h
+
+        stp     $a_q, $a2_q, [$coeff_base, #$a]
+        stp     $sum_q, $sum2_q, [$coeff_base, #$b]
+___
+        }
+    }
+}
+
+# Length 2 batches four [a0,a1 | b0,b1] groups as 32-bit blocks;
+# length 4 batches two [a0..a3 | b0..b3] groups as 64-bit blocks.
+# For each 32-byte batch:
+#   (a,b) = deinterleave(load_batch())
+#   z = load_lane_twiddles()
+#   t = b*z mod q
+#   store_interleaved(a+t,a-t)
+sub ntt_interleaved_layer {
+    my ($len) = @_;
+    my $shape = $len == 2 ? '.4s' : '.2d';
+    my $loop = ".Lossl_ml_kem_ntt_armv8_length${len}_loop";
+    $code .= <<___;
+        mov     $coeff_ptr, $coeff_base
+        mov     $batches_left, #16
+$loop:
+        ldp     $z_q, $c_q, [$twiddle_ptr], #32
+        ldp     $a_q, $b_q, [$coeff_ptr]
+        uzp1    $batch_a_v$shape, $a_v$shape, $b_v$shape
+        uzp2    $b_v$shape, $a_v$shape, $b_v$shape
+
+        // Twiddle multiplication.
+        mul     $product_v.8h, $b_v.8h, $z_v.8h
+        sqrdmulh $tmp_v.8h, $b_v.8h, $c_v.8h
+        mls     $product_v.8h, $tmp_v.8h, $q_v.8h
+
+        // Butterfly sum/difference.
+        sub     $sum_v.8h, $batch_a_v.8h, $product_v.8h
+        add     $batch_a_v.8h, $batch_a_v.8h, $product_v.8h
+
+        zip1    $a_v$shape, $batch_a_v$shape, $sum_v$shape
+        zip2    $b_v$shape, $batch_a_v$shape, $sum_v$shape
+        stp     $a_q, $b_q, [$coeff_ptr], #32
+
+        subs    $batches_left, $batches_left, #1
+        b.ne    $loop
+___
+}
+
+# Barrett-reduce all 256 coefficients in place, then correct negative
+# residues to give canonical output in [0,q).
+sub ntt_reduce {
+    $code .= <<___;
+        mov     $coeff_ptr, $coeff_base
+        mov     $batches_left, #8
+.Lntt_canonical_loop:
+___
+    for (1..2) {
+        $code .= <<___;
+        ldp     $a_q, $b_q, [$coeff_ptr]
+
+        sqrdmulh $tmp_v.8h, $a_v.8h, $barrett_v.8h
+        mls     $a_v.8h, $tmp_v.8h, $q_v.8h
+
+        sshr    $tmp_v.8h, $a_v.8h, #15
+        mls     $a_v.8h, $tmp_v.8h, $q_v.8h
+
+        sqrdmulh $tmp_v.8h, $b_v.8h, $barrett_v.8h
+        mls     $b_v.8h, $tmp_v.8h, $q_v.8h
+
+        sshr    $tmp_v.8h, $b_v.8h, #15
+        mls     $b_v.8h, $tmp_v.8h, $q_v.8h
+
+        stp     $a_q, $b_q, [$coeff_ptr], #32
+___
+    }
+    $code .= <<___;
+        subs    $batches_left, $batches_left, #1
+        b.ne    .Lntt_canonical_loop
+___
+}
+
+# void ossl_ml_kem_ntt_armv8(uint16_t *c);
+# Input: x0 points to 256 polynomial coefficients in [0,3329).
+# Output: c is overwritten with its forward NTT, also in [0,3329).
+# No return value.
+sub ntt {
+    my $name = 'ossl_ml_kem_ntt_armv8';
+    $code .= <<___;
+.globl  $name
+.type   $name,%function
+.align  4
+$name:
+        AARCH64_VALID_CALL_TARGET
+        adrp    $twiddle_ptr, .Lntt_constants
+        add     $twiddle_ptr, $twiddle_ptr, #:lo12:.Lntt_constants
+        mov     $q_reg, #$q
+        dup     $q_v.8h, $q_reg
+        movi    $barrett_v.8h, #10
+___
+    ntt_vector_layer(128);
+    ntt_vector_layer(64);
+    ntt_vector_layer(32);
+    ntt_vector_layer(16);
+    ntt_layer8();
+    ntt_interleaved_layer(4);
+    ntt_interleaved_layer(2);
+
+    ntt_reduce();
+    $code .= <<___;
+        ret
+.size   $name,.-$name
+
+___
+}
+
+# Length-8 inverse butterflies use one vector per half of each group.
+# (a,b) = (a+b,(a-b)*z) modulo q, using inverse twiddles.
+sub intt_layer8 {
+    my $group = 0;
+
+    for (my $start = 0; $start < 256; $start += 16, ++$group) {
+        my ($z, $c) = @{$intt_zc_constants{8}}[2 * $group, 2 * $group + 1];
+        my $z_bits = $z & 0xffff;
+        my $c_bits = $c & 0xffff;
+        $code .= <<___;
+        mov     $packed_zc, #$z_bits
+        movk    $packed_zc, #$c_bits, lsl #16
+        fmov    $intt_zc_s, $packed_zc
+___
+        my $a = 2 * $start;
+        $code .= <<___;
+        ldp     $a_q, $b_q, [$coeff_base, #$a]
+
+        // Butterfly sum/difference.
+        add     $sum_v.8h, $a_v.8h, $b_v.8h
+        sub     $b_v.8h, $a_v.8h, $b_v.8h
+
+        // Twiddle multiplication.
+        mul     $product_v.8h, $b_v.8h, $intt_zc_v.h[0]
+        sqrdmulh $tmp_v.8h, $b_v.8h, $intt_zc_v.h[1]
+        mls     $product_v.8h, $tmp_v.8h, $q_v.8h
+
+        stp     $sum_q, $product_q, [$coeff_base, #$a]
+___
+    }
+}
+
+# Inverse Gentleman-Sande butterflies, unrolled for lengths >= 16.
+# For each group with inverse twiddle z:
+#   for each butterfly pair (a,b):
+#       (a,b) = (a+b,(a-b)*z mod q)
+sub intt_vector_layer {
+    my ($len) = @_;
+    my $group = 0;
+
+    for (my $start = 0; $start < 256; $start += 2 * $len, ++$group) {
+        my ($z, $c) = @{$intt_zc_constants{$len}}[2 * $group, 2 * $group + 1];
+        my $z_bits = $z & 0xffff;
+        my $c_bits = $c & 0xffff;
+        $code .= <<___;
+        mov     $packed_zc, #$z_bits
+        movk    $packed_zc, #$c_bits, lsl #16
+        fmov    $intt_zc_s, $packed_zc
+___
+        for (my $j = 0; $j < $len; $j += 16) {
+            my $a = 2 * ($start + $j);
+            my $b = 2 * ($start + $len + $j);
+
+            $code .= <<___;
+        ldp     $a_q, $a2_q, [$coeff_base, #$a]
+        ldp     $b_q, $b2_q, [$coeff_base, #$b]
+
+        // Butterfly sum/difference.
+        add     $sum_v.8h, $a_v.8h, $b_v.8h
+        add     $sum2_v.8h, $a2_v.8h, $b2_v.8h
+        sub     $b_v.8h, $a_v.8h, $b_v.8h
+        sub     $b2_v.8h, $a2_v.8h, $b2_v.8h
+
+        // Twiddle multiplication.
+        mul     $product_v.8h, $b_v.8h, $intt_zc_v.h[0]
+        mul     $product2_v.8h, $b2_v.8h, $intt_zc_v.h[0]
+        sqrdmulh $tmp_v.8h, $b_v.8h, $intt_zc_v.h[1]
+        sqrdmulh $tmp2_v.8h, $b2_v.8h, $intt_zc_v.h[1]
+        mls     $product_v.8h, $tmp_v.8h, $q_v.8h
+        mls     $product2_v.8h, $tmp2_v.8h, $q_v.8h
+
+        stp     $sum_q, $sum2_q, [$coeff_base, #$a]
+        stp     $product_q, $product2_q, [$coeff_base, #$b]
+___
+        }
+    }
+}
+
+# Same lane layout as ntt_interleaved_layer().  For each 32-byte batch:
+#   (a,b) = deinterleave(load_batch())
+#   z = load_lane_twiddles()
+#   (a,b) = (a+b,(a-b)*z mod q)
+#   store_interleaved(a,b)
+sub intt_interleaved_layer {
+    my ($len) = @_;
+    my $shape = $len == 2 ? '.4s' : '.2d';
+    my $loop = ".Lossl_ml_kem_intt_armv8_length${len}_loop";
+    $code .= <<___;
+        mov     $coeff_ptr, $coeff_base
+        mov     $batches_left, #16
+$loop:
+        ldp     $z_q, $c_q, [$twiddle_ptr], #32
+        ldp     $a_q, $b_q, [$coeff_ptr]
+        uzp1    $batch_a_v$shape, $a_v$shape, $b_v$shape
+        uzp2    $b_v$shape, $a_v$shape, $b_v$shape
+        // Butterfly sum/difference.
+        add     $sum_v.8h, $batch_a_v.8h, $b_v.8h
+        sub     $b_v.8h, $batch_a_v.8h, $b_v.8h
+
+        // Twiddle multiplication.
+        mul     $product_v.8h, $b_v.8h, $z_v.8h
+        sqrdmulh $tmp_v.8h, $b_v.8h, $c_v.8h
+        mls     $product_v.8h, $tmp_v.8h, $q_v.8h
+
+        zip1    $a_v$shape, $sum_v$shape, $product_v$shape
+        zip2    $b_v$shape, $sum_v$shape, $product_v$shape
+        stp     $a_q, $b_q, [$coeff_ptr], #32
+
+        subs    $batches_left, $batches_left, #1
+        b.ne    $loop
+___
+}
+
+# Normalize all 256 coefficients in place by multiplying by the inverse
+# of 128 modulo q, then correct negative residues to give output in [0,q).
+sub intt_normalize {
+    # Scale by the inverse of 128 modulo q (-26), with reduction constant -256.
+    my $z_bits = -26 & 0xffff;
+    my $c_bits = -256 & 0xffff;
+    $code .= <<___;
+        mov     $packed_zc, #$z_bits
+        movk    $packed_zc, #$c_bits, lsl #16
+        fmov    $intt_zc_s, $packed_zc
+        mov     $coeff_ptr, $coeff_base
+        mov     $batches_left, #8
+.Lintt_scale_loop:
+___
+    for (1..2) {
+        $code .= <<___;
+        ldp     $b_q, $b2_q, [$coeff_ptr]
+
+        // Inverse normalization.
+        mul     $product_v.8h, $b_v.8h, $intt_zc_v.h[0]
+        mul     $product2_v.8h, $b2_v.8h, $intt_zc_v.h[0]
+        sqrdmulh $tmp_v.8h, $b_v.8h, $intt_zc_v.h[1]
+        sqrdmulh $tmp2_v.8h, $b2_v.8h, $intt_zc_v.h[1]
+        mls     $product_v.8h, $tmp_v.8h, $q_v.8h
+        mls     $product2_v.8h, $tmp2_v.8h, $q_v.8h
+
+        sshr    $tmp_v.8h, $product_v.8h, #15
+        sshr    $tmp2_v.8h, $product2_v.8h, #15
+        mls     $product_v.8h, $tmp_v.8h, $q_v.8h
+        mls     $product2_v.8h, $tmp2_v.8h, $q_v.8h
+
+        stp     $product_q, $product2_q, [$coeff_ptr], #32
+___
+    }
+    $code .= <<___;
+        subs    $batches_left, $batches_left, #1
+        b.ne    .Lintt_scale_loop
+___
+}
+
+# void ossl_ml_kem_intt_armv8(uint16_t *c);
+# Input: x0 points to 256 NTT coefficients in [0,3329).
+# Output: c is overwritten with its inverse NTT, also in [0,3329).
+# No return value.
+sub intt {
+    my $name = 'ossl_ml_kem_intt_armv8';
+    $code .= <<___;
+.globl  $name
+.type   $name,%function
+.align  4
+$name:
+        AARCH64_VALID_CALL_TARGET
+        adrp    $twiddle_ptr, .Lintt_short_constants
+        add     $twiddle_ptr, $twiddle_ptr, #:lo12:.Lintt_short_constants
+        mov     $q_reg, #$q
+        dup     $q_v.8h, $q_reg
+        movi    $barrett_v.8h, #10
+___
+    intt_interleaved_layer(2);
+    intt_interleaved_layer(4);
+    intt_layer8();
+    # After layer 3, center sum paths to preserve signed 16-bit headroom.
+    intt_reduce(8, 16);
+    intt_vector_layer(16);
+    intt_vector_layer(32);
+    intt_vector_layer(64);
+    # After layer 6, center sums before the final layer to avoid 16-bit overflow.
+    intt_reduce(64, 128);
+    intt_vector_layer(128);
+
+    intt_normalize();
+    $code .= <<___;
+        ret
+.size   $name,.-$name
+
+___
+}
+
+$code .= <<___;
+#include "arch/arm_arch.h"
+
+.text
+___
+ntt();
+intt();
+
+# Each 32-byte entry holds eight twiddles followed by eight reduction
+# constants, c = round(2^15*z/q).
+# Entries follow transform consumption order:
+#
+#   Forward   0..30: lengths 128..8, one root broadcast per entry.
+#            31..46: length 4, two roots repeated four times each.
+#            47..62: length 2, four roots repeated twice each.
+#   Inverse   0..15: length 2, four roots repeated twice each.
+#            16..31: length 4, two roots repeated four times each.
+$code .= <<___;
+.rodata
+.align  4
+// Forward NTT: broadcast lengths 128..8, then lane-packed lengths 4 and 2.
+// 8 twiddles followed by 8 reduction constants per 32-byte entry.
+.Lntt_constants:
+// Length 128
+        .short  -1600, -1600, -1600, -1600, -1600, -1600, -1600, -1600
+        .short  -15749, -15749, -15749, -15749, -15749, -15749, -15749, -15749
+// Length 64
+        .short  -749, -749, -749, -749, -749, -749, -749, -749
+        .short  -7373, -7373, -7373, -7373, -7373, -7373, -7373, -7373
+        .short  -40, -40, -40, -40, -40, -40, -40, -40
+        .short  -394, -394, -394, -394, -394, -394, -394, -394
+// Length 32
+        .short  -687, -687, -687, -687, -687, -687, -687, -687
+        .short  -6762, -6762, -6762, -6762, -6762, -6762, -6762, -6762
+        .short  630, 630, 630, 630, 630, 630, 630, 630
+        .short  6201, 6201, 6201, 6201, 6201, 6201, 6201, 6201
+        .short  -1432, -1432, -1432, -1432, -1432, -1432, -1432, -1432
+        .short  -14095, -14095, -14095, -14095, -14095, -14095, -14095, -14095
+        .short  848, 848, 848, 848, 848, 848, 848, 848
+        .short  8347, 8347, 8347, 8347, 8347, 8347, 8347, 8347
+// Length 16
+        .short  1062, 1062, 1062, 1062, 1062, 1062, 1062, 1062
+        .short  10453, 10453, 10453, 10453, 10453, 10453, 10453, 10453
+        .short  -1410, -1410, -1410, -1410, -1410, -1410, -1410, -1410
+        .short  -13879, -13879, -13879, -13879, -13879, -13879, -13879, -13879
+        .short  193, 193, 193, 193, 193, 193, 193, 193
+        .short  1900, 1900, 1900, 1900, 1900, 1900, 1900, 1900
+        .short  797, 797, 797, 797, 797, 797, 797, 797
+        .short  7845, 7845, 7845, 7845, 7845, 7845, 7845, 7845
+        .short  -543, -543, -543, -543, -543, -543, -543, -543
+        .short  -5345, -5345, -5345, -5345, -5345, -5345, -5345, -5345
+        .short  -69, -69, -69, -69, -69, -69, -69, -69
+        .short  -679, -679, -679, -679, -679, -679, -679, -679
+        .short  569, 569, 569, 569, 569, 569, 569, 569
+        .short  5601, 5601, 5601, 5601, 5601, 5601, 5601, 5601
+        .short  -1583, -1583, -1583, -1583, -1583, -1583, -1583, -1583
+        .short  -15582, -15582, -15582, -15582, -15582, -15582, -15582, -15582
+// Length 8
+        .short  296, 296, 296, 296, 296, 296, 296, 296
+        .short  2914, 2914, 2914, 2914, 2914, 2914, 2914, 2914
+        .short  -882, -882, -882, -882, -882, -882, -882, -882
+        .short  -8682, -8682, -8682, -8682, -8682, -8682, -8682, -8682
+        .short  1339, 1339, 1339, 1339, 1339, 1339, 1339, 1339
+        .short  13180, 13180, 13180, 13180, 13180, 13180, 13180, 13180
+        .short  1476, 1476, 1476, 1476, 1476, 1476, 1476, 1476
+        .short  14529, 14529, 14529, 14529, 14529, 14529, 14529, 14529
+        .short  -283, -283, -283, -283, -283, -283, -283, -283
+        .short  -2786, -2786, -2786, -2786, -2786, -2786, -2786, -2786
+        .short  56, 56, 56, 56, 56, 56, 56, 56
+        .short  551, 551, 551, 551, 551, 551, 551, 551
+        .short  -1089, -1089, -1089, -1089, -1089, -1089, -1089, -1089
+        .short  -10719, -10719, -10719, -10719, -10719, -10719, -10719, -10719
+        .short  1333, 1333, 1333, 1333, 1333, 1333, 1333, 1333
+        .short  13121, 13121, 13121, 13121, 13121, 13121, 13121, 13121
+        .short  1426, 1426, 1426, 1426, 1426, 1426, 1426, 1426
+        .short  14036, 14036, 14036, 14036, 14036, 14036, 14036, 14036
+        .short  -1235, -1235, -1235, -1235, -1235, -1235, -1235, -1235
+        .short  -12156, -12156, -12156, -12156, -12156, -12156, -12156, -12156
+        .short  535, 535, 535, 535, 535, 535, 535, 535
+        .short  5266, 5266, 5266, 5266, 5266, 5266, 5266, 5266
+        .short  -447, -447, -447, -447, -447, -447, -447, -447
+        .short  -4400, -4400, -4400, -4400, -4400, -4400, -4400, -4400
+        .short  -936, -936, -936, -936, -936, -936, -936, -936
+        .short  -9213, -9213, -9213, -9213, -9213, -9213, -9213, -9213
+        .short  -450, -450, -450, -450, -450, -450, -450, -450
+        .short  -4429, -4429, -4429, -4429, -4429, -4429, -4429, -4429
+        .short  -1355, -1355, -1355, -1355, -1355, -1355, -1355, -1355
+        .short  -13338, -13338, -13338, -13338, -13338, -13338, -13338, -13338
+        .short  821, 821, 821, 821, 821, 821, 821, 821
+        .short  8081, 8081, 8081, 8081, 8081, 8081, 8081, 8081
+// Length 4
+        .short  289, 289, 289, 289, 331, 331, 331, 331
+        .short  2845, 2845, 2845, 2845, 3258, 3258, 3258, 3258
+        .short  -76, -76, -76, -76, -1573, -1573, -1573, -1573
+        .short  -748, -748, -748, -748, -15483, -15483, -15483, -15483
+        .short  1197, 1197, 1197, 1197, -1025, -1025, -1025, -1025
+        .short  11782, 11782, 11782, 11782, -10089, -10089, -10089, -10089
+        .short  -1052, -1052, -1052, -1052, -1274, -1274, -1274, -1274
+        .short  -10355, -10355, -10355, -10355, -12540, -12540, -12540, -12540
+        .short  650, 650, 650, 650, -1352, -1352, -1352, -1352
+        .short  6398, 6398, 6398, 6398, -13308, -13308, -13308, -13308
+        .short  -816, -816, -816, -816, 632, 632, 632, 632
+        .short  -8032, -8032, -8032, -8032, 6221, 6221, 6221, 6221
+        .short  -464, -464, -464, -464, 33, 33, 33, 33
+        .short  -4567, -4567, -4567, -4567, 325, 325, 325, 325
+        .short  1320, 1320, 1320, 1320, -1414, -1414, -1414, -1414
+        .short  12993, 12993, 12993, 12993, -13918, -13918, -13918, -13918
+        .short  -1010, -1010, -1010, -1010, 1435, 1435, 1435, 1435
+        .short  -9942, -9942, -9942, -9942, 14125, 14125, 14125, 14125
+        .short  807, 807, 807, 807, 452, 452, 452, 452
+        .short  7943, 7943, 7943, 7943, 4449, 4449, 4449, 4449
+        .short  1438, 1438, 1438, 1438, -461, -461, -461, -461
+        .short  14155, 14155, 14155, 14155, -4538, -4538, -4538, -4538
+        .short  1534, 1534, 1534, 1534, -927, -927, -927, -927
+        .short  15099, 15099, 15099, 15099, -9125, -9125, -9125, -9125
+        .short  -682, -682, -682, -682, -712, -712, -712, -712
+        .short  -6713, -6713, -6713, -6713, -7008, -7008, -7008, -7008
+        .short  1481, 1481, 1481, 1481, 648, 648, 648, 648
+        .short  14578, 14578, 14578, 14578, 6378, 6378, 6378, 6378
+        .short  -855, -855, -855, -855, -219, -219, -219, -219
+        .short  -8416, -8416, -8416, -8416, -2156, -2156, -2156, -2156
+        .short  1227, 1227, 1227, 1227, 910, 910, 910, 910
+        .short  12078, 12078, 12078, 12078, 8957, 8957, 8957, 8957
+// Length 2
+        .short  17, 17, -568, -568, 583, 583, -680, -680
+        .short  167, 167, -5591, -5591, 5739, 5739, -6693, -6693
+        .short  1637, 1637, 723, 723, -1041, -1041, 1100, 1100
+        .short  16113, 16113, 7117, 7117, -10247, -10247, 10828, 10828
+        .short  1409, 1409, -667, -667, -48, -48, 233, 233
+        .short  13869, 13869, -6565, -6565, -472, -472, 2293, 2293
+        .short  756, 756, -1173, -1173, -314, -314, -279, -279
+        .short  7441, 7441, -11546, -11546, -3091, -3091, -2746, -2746
+        .short  -1626, -1626, 1651, 1651, -540, -540, -1540, -1540
+        .short  -16005, -16005, 16251, 16251, -5315, -5315, -15159, -15159
+        .short  -1482, -1482, 952, 952, 1461, 1461, -642, -642
+        .short  -14588, -14588, 9371, 9371, 14381, 14381, -6319, -6319
+        .short  939, 939, -1021, -1021, -892, -892, -941, -941
+        .short  9243, 9243, -10050, -10050, -8780, -8780, -9262, -9262
+        .short  733, 733, -992, -992, 268, 268, 641, 641
+        .short  7215, 7215, -9764, -9764, 2638, 2638, 6309, 6309
+        .short  1584, 1584, -1031, -1031, -1292, -1292, -109, -109
+        .short  15592, 15592, -10148, -10148, -12717, -12717, -1073, -1073
+        .short  375, 375, -780, -780, -1239, -1239, 1645, 1645
+        .short  3691, 3691, -7678, -7678, -12196, -12196, 16192, 16192
+        .short  1063, 1063, 319, 319, -556, -556, 757, 757
+        .short  10463, 10463, 3140, 3140, -5473, -5473, 7451, 7451
+        .short  -1230, -1230, 561, 561, -863, -863, -735, -735
+        .short  -12107, -12107, 5522, 5522, -8495, -8495, -7235, -7235
+        .short  -525, -525, 1092, 1092, 403, 403, 1026, 1026
+        .short  -5168, -5168, 10749, 10749, 3967, 3967, 10099, 10099
+        .short  1143, 1143, -1179, -1179, -554, -554, 886, 886
+        .short  11251, 11251, -11605, -11605, -5453, -5453, 8721, 8721
+        .short  -1607, -1607, 1212, 1212, -1455, -1455, 1029, 1029
+        .short  -15818, -15818, 11930, 11930, -14322, -14322, 10129, 10129
+        .short  -1219, -1219, -394, -394, 885, 885, -1175, -1175
+        .short  -11999, -11999, -3878, -3878, 8711, 8711, -11566, -11566
+.align  4
+// Inverse NTT: lane-packed length-2 entries followed by length-4 entries.
+// 8 inverse twiddles followed by 8 reduction constants per 32-byte entry.
+.Lintt_short_constants:
+// Length 2
+        .short  1175, 1175, -885, -885, 394, 394, 1219, 1219
+        .short  11566, 11566, -8711, -8711, 3878, 3878, 11999, 11999
+        .short  -1029, -1029, 1455, 1455, -1212, -1212, 1607, 1607
+        .short  -10129, -10129, 14322, 14322, -11930, -11930, 15818, 15818
+        .short  -886, -886, 554, 554, 1179, 1179, -1143, -1143
+        .short  -8721, -8721, 5453, 5453, 11605, 11605, -11251, -11251
+        .short  -1026, -1026, -403, -403, -1092, -1092, 525, 525
+        .short  -10099, -10099, -3967, -3967, -10749, -10749, 5168, 5168
+        .short  735, 735, 863, 863, -561, -561, 1230, 1230
+        .short  7235, 7235, 8495, 8495, -5522, -5522, 12107, 12107
+        .short  -757, -757, 556, 556, -319, -319, -1063, -1063
+        .short  -7451, -7451, 5473, 5473, -3140, -3140, -10463, -10463
+        .short  -1645, -1645, 1239, 1239, 780, 780, -375, -375
+        .short  -16192, -16192, 12196, 12196, 7678, 7678, -3691, -3691
+        .short  109, 109, 1292, 1292, 1031, 1031, -1584, -1584
+        .short  1073, 1073, 12717, 12717, 10148, 10148, -15592, -15592
+        .short  -641, -641, -268, -268, 992, 992, -733, -733
+        .short  -6309, -6309, -2638, -2638, 9764, 9764, -7215, -7215
+        .short  941, 941, 892, 892, 1021, 1021, -939, -939
+        .short  9262, 9262, 8780, 8780, 10050, 10050, -9243, -9243
+        .short  642, 642, -1461, -1461, -952, -952, 1482, 1482
+        .short  6319, 6319, -14381, -14381, -9371, -9371, 14588, 14588
+        .short  1540, 1540, 540, 540, -1651, -1651, 1626, 1626
+        .short  15159, 15159, 5315, 5315, -16251, -16251, 16005, 16005
+        .short  279, 279, 314, 314, 1173, 1173, -756, -756
+        .short  2746, 2746, 3091, 3091, 11546, 11546, -7441, -7441
+        .short  -233, -233, 48, 48, 667, 667, -1409, -1409
+        .short  -2293, -2293, 472, 472, 6565, 6565, -13869, -13869
+        .short  -1100, -1100, 1041, 1041, -723, -723, -1637, -1637
+        .short  -10828, -10828, 10247, 10247, -7117, -7117, -16113, -16113
+        .short  680, 680, -583, -583, 568, 568, -17, -17
+        .short  6693, 6693, -5739, -5739, 5591, 5591, -167, -167
+// Length 4
+        .short  -910, -910, -910, -910, -1227, -1227, -1227, -1227
+        .short  -8957, -8957, -8957, -8957, -12078, -12078, -12078, -12078
+        .short  219, 219, 219, 219, 855, 855, 855, 855
+        .short  2156, 2156, 2156, 2156, 8416, 8416, 8416, 8416
+        .short  -648, -648, -648, -648, -1481, -1481, -1481, -1481
+        .short  -6378, -6378, -6378, -6378, -14578, -14578, -14578, -14578
+        .short  712, 712, 712, 712, 682, 682, 682, 682
+        .short  7008, 7008, 7008, 7008, 6713, 6713, 6713, 6713
+        .short  927, 927, 927, 927, -1534, -1534, -1534, -1534
+        .short  9125, 9125, 9125, 9125, -15099, -15099, -15099, -15099
+        .short  461, 461, 461, 461, -1438, -1438, -1438, -1438
+        .short  4538, 4538, 4538, 4538, -14155, -14155, -14155, -14155
+        .short  -452, -452, -452, -452, -807, -807, -807, -807
+        .short  -4449, -4449, -4449, -4449, -7943, -7943, -7943, -7943
+        .short  -1435, -1435, -1435, -1435, 1010, 1010, 1010, 1010
+        .short  -14125, -14125, -14125, -14125, 9942, 9942, 9942, 9942
+        .short  1414, 1414, 1414, 1414, -1320, -1320, -1320, -1320
+        .short  13918, 13918, 13918, 13918, -12993, -12993, -12993, -12993
+        .short  -33, -33, -33, -33, 464, 464, 464, 464
+        .short  -325, -325, -325, -325, 4567, 4567, 4567, 4567
+        .short  -632, -632, -632, -632, 816, 816, 816, 816
+        .short  -6221, -6221, -6221, -6221, 8032, 8032, 8032, 8032
+        .short  1352, 1352, 1352, 1352, -650, -650, -650, -650
+        .short  13308, 13308, 13308, 13308, -6398, -6398, -6398, -6398
+        .short  1274, 1274, 1274, 1274, 1052, 1052, 1052, 1052
+        .short  12540, 12540, 12540, 12540, 10355, 10355, 10355, 10355
+        .short  1025, 1025, 1025, 1025, -1197, -1197, -1197, -1197
+        .short  10089, 10089, 10089, 10089, -11782, -11782, -11782, -11782
+        .short  1573, 1573, 1573, 1573, 76, 76, 76, 76
+        .short  15483, 15483, 15483, 15483, 748, 748, 748, 748
+        .short  -331, -331, -331, -331, -289, -289, -289, -289
+        .short  -3258, -3258, -3258, -3258, -2845, -2845, -2845, -2845
+___
+print $code;
+close STDOUT or die "error closing STDOUT: $!";
diff --git a/crypto/ml_kem/build.info b/crypto/ml_kem/build.info
index 78c9804bda..ae0ada918c 100644
--- a/crypto/ml_kem/build.info
+++ b/crypto/ml_kem/build.info
@@ -4,6 +4,8 @@ $MLKEMASM=
 IF[{- !$disabled{asm} -}]
   $MLKEMASM_x86_64=ml_kem_avx2.s
   $MLKEMDEF_x86_64=MLKEM_AVX2_X86_64_ASM
+  $MLKEMASM_aarch64=mlkem-armv8.S
+  $MLKEMDEF_aarch64=MLKEM_NTT_ARMV8_ASM
   $MLKEMDEF_ppc64=MLKEM_NTT_PPC_ASM

   IF[{- $target{sys_id} ne "AIX" && $target{sys_id} ne "MACOSX" -}]
@@ -41,3 +43,4 @@ DEFINE[../../providers/libdefault.a]=$MLKEMDEF
 DEFINE[../../providers/libfips.a]=$MLKEMDEF

 GENERATE[ml_kem_avx2.s]=asm/ml_kem_avx2.pl
+GENERATE[mlkem-armv8.S]=asm/mlkem-armv8.pl
diff --git a/crypto/ml_kem/ml_kem.c b/crypto/ml_kem/ml_kem.c
index 5fbf8b41a3..172e09e844 100644
--- a/crypto/ml_kem/ml_kem.c
+++ b/crypto/ml_kem/ml_kem.c
@@ -431,6 +431,9 @@ static CRYPTO_ONCE ml_kem_ntt_once = CRYPTO_ONCE_STATIC_INIT;
 #if defined(_ARCH_PPC64)
 #include "arch/ppc_arch.h"
 #endif
+#if defined(MLKEM_NTT_ARMV8_ASM) && !defined(__AARCH64EB__)
+#include "arch/arm_arch.h"
+#endif

 /*
  * Function pointer types for NTT dispatch.
@@ -447,9 +450,9 @@ static CRYPTO_ONCE ml_kem_ntt_once = CRYPTO_ONCE_STATIC_INIT;
  *
  *  scalar_inverse_ntt_demontgomerize
  *      Inverse NTT whose input may still carry an inverse-Montgomery factor
- *      R^-1 from a preceding inner_product_montgomery call.  On the generic
- *      PPC, and AVX2 paths the two pointers are identical (both point at the same
- *      fully-reduced implementation), because the generic inner_product
+ *      R^-1 from a preceding inner_product_montgomery call.  On the generic,
+ *      PPC, AVX2, and AArch64 paths the two pointers are identical (both point at
+ *      the same fully-reduced implementation), because the generic inner_product
  *      already produces fully-reduced outputs via Barrett reduction.  On the
  *      s390x/vec128 path the two are distinct:
  *        - inner_product_montgomery_vec128 accumulates the dot-product using
@@ -482,9 +485,9 @@ static ml_kem_scalar_ntt_fn scalar_ntt = ossl_ml_kem_scalar_ntt_generic;
 static ml_kem_scalar_inverse_ntt_fn scalar_inverse_ntt = ossl_ml_kem_scalar_inverse_ntt_generic;
 /*
  * scalar_inverse_ntt_demontgomerize: used after inner_product_montgomery.  On
- * generic/PPC/AVX2 this is the same function as scalar_inverse_ntt.  On s390x it is
- * a specialised variant that also removes the inverse-Montgomery factor R^-1
- * left by inner_product_montgomery_vec128.
+ * generic/PPC/AVX2/AArch64 this is the same function as scalar_inverse_ntt.
+ * On s390x it is a specialised variant that also removes the inverse-Montgomery
+ * factor R^-1 left by inner_product_montgomery_vec128.
  */
 static ml_kem_scalar_inverse_ntt_demontgomerize_fn scalar_inverse_ntt_demontgomerize = ossl_ml_kem_scalar_inverse_ntt_generic;

@@ -522,6 +525,22 @@ static void scalar_inverse_ntt_ppc(scalar *s)
 }
 #endif

+/* The assembly uses little-endian vector lane packing. */
+#if defined(MLKEM_NTT_ARMV8_ASM) && !defined(__AARCH64EB__)
+void ossl_ml_kem_ntt_armv8(uint16_t *c);
+void ossl_ml_kem_intt_armv8(uint16_t *c);
+
+static void scalar_ntt_armv8(scalar *s)
+{
+    ossl_ml_kem_ntt_armv8(s->c);
+}
+
+static void scalar_inverse_ntt_armv8(scalar *s)
+{
+    ossl_ml_kem_intt_armv8(s->c);
+}
+#endif
+
 /*
  * VX_COMPILER_SUPPORT_VEC128 is now defined (via include/crypto/ml_kem.h)
  * whenever OPENSSL_ML_KEM_S390X && __s390x__, without requiring __VX__.
@@ -632,6 +651,13 @@ static void ml_kem_ntt_init(void)
         matrix_mult_transpose_add = matrix_mult_transpose_add_avx2;
     }
 #endif
+#if defined(MLKEM_NTT_ARMV8_ASM) && !defined(__AARCH64EB__)
+    if (OPENSSL_armcap_P & ARMV7_NEON) {
+        scalar_ntt = scalar_ntt_armv8;
+        scalar_inverse_ntt = scalar_inverse_ntt_armv8;
+        scalar_inverse_ntt_demontgomerize = scalar_inverse_ntt_armv8;
+    }
+#endif

 /*
  * Initialize NTT function pointers to PPC64le implementations if available.