Commit 917cf4ebb7 for openssl.org
commit 917cf4ebb7c3eeaa29e3abcd17383a37ff6180f4
Author: Mounir IDRASSI <mounir.idrassi@idrix.fr>
Date: Wed Sep 23 20:56:18 2026 +0900
Bound DTLS 1.3 ACK bookkeeping and fit ACKs to records
Repeated fragments of an incomplete handshake message can arrive in fresh
records without making reassembly progress. Each record adds an ACK entry
and scans all previous entries, allowing unbounded storage and quadratic
work. Limit the list to 1023 record numbers, the capacity of a maximum-sized
plaintext ACK, and check this limit before the duplicate scan.
Even a bounded list may exceed the current MTU or configured fragment
limit. dtls1_do_write() currently splits the encoded ACK vector without
updating its length prefix, so a valid fragmented ticket can cause the peer
to abort with SSL_R_LENGTH_TOO_LONG when it reads the ACK.
Send complete ACK vectors containing whole 16-byte record numbers within
the existing output limits. Reuse two already-sent bytes for each next
vector's length prefix, preserving progress across write retries. Report
each emitted ACK through the message callback. All retained record numbers
are sent, without requiring a state-machine change.
Add regressions for repeated fragments while reassembly stays incomplete,
and completed ticket exchanges at a small MTU, under a configured fragment
limit, and with a write retry between ACK records. The growth test fails
on the original code; all three output cases also fail with only the
storage cap applied.
Validation: 12 DTLS and bad-DTLS test recipes, strict warning compilation
of both changed files, and clang-format checks. Additional local diagnostics
verified padding, per-record callbacks, and retransmission recovery after
ACK-list saturation and completion of a valid ticket.
Fixes #32957
Assisted-by: Codex:gpt-6-astra
Reviewed-by: Ryan Hooper <ryanh@openssl.foundation>
Reviewed-by: Frederik Wedel-Heinen <fwh.openssl@gmail.com>
Reviewed-by: Tomas Mraz <tomas@openssl.foundation>
Merge-date: Wed Sep 30 08:19:46 2026
Merged-from: https://github.com/openssl/openssl/pull/32958
diff --git a/ssl/ssl_local.h b/ssl/ssl_local.h
index 8e2a7e9ce0..9d40fcb0f4 100644
--- a/ssl/ssl_local.h
+++ b/ssl/ssl_local.h
@@ -2178,6 +2178,14 @@ typedef struct dtls_msg_info_st {
unsigned short msg_seq;
} dtls_msg_info;
+/* RFC 9147, section 4: a 64-bit epoch and a 64-bit sequence number. */
+#define DTLS13_RECORD_NUMBER_LEN 16
+/* RFC 9147, section 7: the ACK vector's two-byte length prefix. */
+#define DTLS13_ACK_HEADER_LEN 2
+/* A nonempty ACK contains the prefix and at least one record number. */
+#define DTLS13_ACK_MIN_NONEMPTY_LEN \
+ (DTLS13_ACK_HEADER_LEN + DTLS13_RECORD_NUMBER_LEN)
+
/* rfc9147, section 4 */
typedef struct dtls1_record_number_st DTLS1_RECORD_NUMBER;
diff --git a/ssl/statem/statem_dtls.c b/ssl/statem/statem_dtls.c
index bf1e4a62f2..4e7a91cff9 100644
--- a/ssl/statem/statem_dtls.c
+++ b/ssl/statem/statem_dtls.c
@@ -193,8 +193,8 @@ static int dtls1_write_hm_header(unsigned char *msgheaderstart,
}
/*
- * send s->init_buf in records of type 'type' (SSL3_RT_HANDSHAKE or
- * SSL3_RT_CHANGE_CIPHER_SPEC)
+ * send s->init_buf in records of type 'type' (SSL3_RT_HANDSHAKE,
+ * SSL3_RT_CHANGE_CIPHER_SPEC or SSL3_RT_ACK)
*
* When sending a fragmented handshake message this function will re-use
* s->init_buf->data but overwrite previously sent data to fill out the handshake
@@ -221,6 +221,9 @@ int dtls1_do_write(SSL_CONNECTION *s, uint8_t recordtype)
const size_t msg_len = s->d1->w_msg.msg_body_len;
const unsigned short msg_seq = s->d1->w_msg.msg_seq;
const unsigned char msg_type = s->d1->w_msg.msg_type;
+ const size_t min_len = recordtype == SSL3_RT_ACK
+ ? DTLS13_ACK_MIN_NONEMPTY_LEN
+ : DTLS1_HM_HEADER_LENGTH + 1;
if (!dtls1_query_mtu(s))
return -1;
@@ -272,7 +275,7 @@ int dtls1_do_write(SSL_CONNECTION *s, uint8_t recordtype)
else
curr_mtu = 0;
- if (curr_mtu <= DTLS1_HM_HEADER_LENGTH) {
+ if (curr_mtu < min_len) {
/*
* grr.. we could get an error if MTU picked was wrong
*/
@@ -281,7 +284,7 @@ int dtls1_do_write(SSL_CONNECTION *s, uint8_t recordtype)
s->rwstate = SSL_WRITING;
return ret;
}
- if (s->d1->mtu > overhead + DTLS1_HM_HEADER_LENGTH) {
+ if (s->d1->mtu >= overhead + min_len) {
curr_mtu = s->d1->mtu - overhead;
} else {
/* Shouldn't happen */
@@ -306,6 +309,17 @@ int dtls1_do_write(SSL_CONNECTION *s, uint8_t recordtype)
msgstart = (unsigned char *)&s->init_buf->data[s->init_off];
+ if (recordtype == SSL3_RT_ACK) {
+ /* Each record needs a complete vector of record numbers. */
+ if (!ossl_assert(len >= DTLS13_ACK_HEADER_LEN))
+ return -1;
+ len = DTLS13_ACK_HEADER_LEN
+ + ((len - DTLS13_ACK_HEADER_LEN) / DTLS13_RECORD_NUMBER_LEN)
+ * DTLS13_RECORD_NUMBER_LEN;
+ msgstart[0] = (unsigned char)((len - DTLS13_ACK_HEADER_LEN) >> 8);
+ msgstart[1] = (unsigned char)(len - DTLS13_ACK_HEADER_LEN);
+ }
+
if (recordtype == SSL3_RT_HANDSHAKE) {
const size_t fragoff = s->init_off;
const size_t fraglen = len - DTLS1_HM_HEADER_LENGTH;
@@ -407,8 +421,12 @@ int dtls1_do_write(SSL_CONNECTION *s, uint8_t recordtype)
if (!ssl3_finish_mac(s, msgstart, xlen))
return -1;
}
+ if (recordtype == SSL3_RT_ACK && s->msg_callback != NULL)
+ s->msg_callback(1, s->version, recordtype, msgstart, written,
+ ussl, s->msg_callback_arg);
+
if (written == s->init_num) {
- if (s->msg_callback)
+ if (s->msg_callback && recordtype != SSL3_RT_ACK)
s->msg_callback(1, s->version, recordtype, s->init_buf->data,
s->init_off + s->init_num, ussl,
s->msg_callback_arg);
@@ -418,6 +436,9 @@ int dtls1_do_write(SSL_CONNECTION *s, uint8_t recordtype)
return 1;
}
+ /* Reuse the last two sent bytes for the next ACK vector length. */
+ if (recordtype == SSL3_RT_ACK)
+ written -= DTLS13_ACK_HEADER_LEN;
s->init_off += written;
s->init_num -= written;
}
@@ -569,6 +590,15 @@ static int add_record_to_ack_list(SSL_CONNECTION *sc)
uint64_t epoch = sc->s3.tmp.record_epoch;
uint64_t sequence = sc->s3.tmp.record_seq_num;
+ /*
+ * Retain at most one maximum-sized ACK record's worth of record numbers.
+ * Check before the duplicate scan to bound the work once the list is full.
+ * Excess records may be omitted from ACKs (RFC 9147, section 7.1).
+ */
+ if (ossl_list_record_number_num(&sc->d1->ack_rec_num)
+ >= (SSL3_RT_MAX_PLAIN_LENGTH - DTLS13_ACK_HEADER_LEN) / DTLS13_RECORD_NUMBER_LEN)
+ return 1;
+
for (recnum = ossl_list_record_number_head(&sc->d1->ack_rec_num);
recnum != NULL;
recnum = ossl_list_record_number_next(recnum)) {
diff --git a/test/dtls13_internal_test.c b/test/dtls13_internal_test.c
index 3d78654dac..7ef5d7b4a4 100644
--- a/test/dtls13_internal_test.c
+++ b/test/dtls13_internal_test.c
@@ -624,6 +624,163 @@ end:
SSL_CTX_free(cctx);
return testresult;
}
+
+static int test_dtls13_ack_list_bound(void)
+{
+ SSL_CTX *sctx = NULL, *cctx = NULL;
+ SSL *server = NULL, *client = NULL;
+ SSL_CONNECTION *sc, *cc;
+ /*
+ * Declare two body bytes but supply only one, keeping reassembly incomplete.
+ */
+ unsigned char frag[] = {
+ SSL3_MT_KEY_UPDATE, /* msg_type */
+ 0, 0, 2, /* msg_body_len */
+ 0, 0, /* msg_seq */
+ 0, 0, 0, /* fragment_offset */
+ 0, 0, 1, /* fragment_length */
+ 0 /* fragment data */
+ };
+ unsigned char buf;
+ const size_t max_records = (SSL3_RT_MAX_PLAIN_LENGTH - DTLS13_ACK_HEADER_LEN)
+ / DTLS13_RECORD_NUMBER_LEN;
+ size_t i, written;
+ int ret, testresult = 0;
+
+ if (!TEST_true(create_ssl_ctx_pair(NULL, DTLS_server_method(),
+ DTLS_client_method(), DTLS1_3_VERSION, DTLS1_3_VERSION,
+ &sctx, &cctx, cert, privkey))
+ || !TEST_true(SSL_CTX_set_num_tickets(sctx, 0)))
+ goto end;
+ if (!TEST_true(create_ssl_objects(sctx, cctx, &server, &client, NULL, NULL))
+ || !TEST_true(create_ssl_connection(server, client, SSL_ERROR_NONE)))
+ goto end;
+ sc = SSL_CONNECTION_FROM_SSL(server);
+ cc = SSL_CONNECTION_FROM_SSL(client);
+
+ if (!TEST_size_t_eq(ossl_list_record_number_num(&sc->d1->ack_rec_num), 0))
+ goto end;
+
+ /*
+ * Repeat the first byte of a two-byte message in fresh records, leaving
+ * reassembly incomplete. Consume each record before sending the next.
+ */
+ frag[4] = (unsigned char)(cc->d1->next_handshake_write_seq >> 8);
+ frag[5] = (unsigned char)cc->d1->next_handshake_write_seq;
+ for (i = 0; i < 2 * max_records; i++) {
+ if (!TEST_int_eq(dtls1_write_bytes(cc, SSL3_RT_HANDSHAKE, frag,
+ sizeof(frag), &written),
+ 1)
+ || !TEST_size_t_eq(written, sizeof(frag))
+ || !TEST_int_gt(BIO_flush(SSL_get_wbio(client)), 0))
+ goto end;
+ ret = SSL_read(server, &buf, sizeof(buf));
+ if (!TEST_int_eq(SSL_get_error(server, ret), SSL_ERROR_WANT_READ)
+ || !TEST_size_t_eq(BIO_ctrl_pending(SSL_get_rbio(server)), 0)
+ || !TEST_size_t_eq(ossl_list_record_number_num(&sc->d1->ack_rec_num),
+ i < max_records ? i + 1 : max_records))
+ goto end;
+ }
+
+ testresult = 1;
+end:
+ SSL_free(server);
+ SSL_free(client);
+ SSL_CTX_free(sctx);
+ SSL_CTX_free(cctx);
+ return testresult;
+}
+static int large_ticket_cb(SSL *ssl, void *arg)
+{
+ static const unsigned char appdata[8192] = { 0 };
+
+ return SSL_SESSION_set1_ticket_appdata(SSL_get_session(ssl), appdata,
+ sizeof(appdata));
+}
+
+/* Split ACKs at the MTU or fragment limit, including a retry between records. */
+static int test_dtls13_ack_records(int idx)
+{
+ SSL_CTX *sctx = NULL, *cctx = NULL;
+ SSL *server = NULL, *client = NULL;
+ SSL_CONNECTION *sc, *cc;
+ BIO *retry = NULL;
+ pitem *item;
+ dtls_sent_msg *msg;
+ unsigned char buf;
+ size_t limit;
+ int ret, testresult = 0;
+
+ if (!TEST_true(create_ssl_ctx_pair(NULL, DTLS_server_method(),
+ DTLS_client_method(), DTLS1_3_VERSION, DTLS1_3_VERSION,
+ &sctx, &cctx, cert, privkey))
+ || !TEST_true(SSL_CTX_set_num_tickets(sctx, 0))
+ || !TEST_true(SSL_CTX_set_session_ticket_cb(sctx, large_ticket_cb, NULL, NULL))
+ || !TEST_true(create_ssl_objects(sctx, cctx, &server, &client, NULL, NULL))
+ || !TEST_true(create_ssl_connection(server, client, SSL_ERROR_NONE)))
+ goto end;
+ sc = SSL_CONNECTION_FROM_SSL(server);
+ cc = SSL_CONNECTION_FROM_SSL(client);
+
+ SSL_set_options(server, SSL_OP_NO_QUERY_MTU);
+ SSL_set_options(client, SSL_OP_NO_QUERY_MTU);
+ if (!TEST_long_gt(SSL_set_mtu(server, 256), 0)
+ || !TEST_long_gt(SSL_set_mtu(client, idx == 1 ? 1500 : 256), 0)
+ || !TEST_true(SSL_set_max_send_fragment(client, 512))
+ || !TEST_true(SSL_new_session_ticket(server))
+ || !TEST_int_eq(SSL_write(server, "s", 1), 1)
+ || !TEST_ptr(item = pqueue_peek(&sc->d1->sent_messages)))
+ goto end;
+
+ /* The ticket must require more ACK entries than fit in one record. */
+ msg = item->data;
+ limit = idx == 1 ? 512 : DTLS_get_data_mtu(client);
+ if (!TEST_size_t_gt(DTLS13_ACK_HEADER_LEN
+ + DTLS13_RECORD_NUMBER_LEN * ossl_list_record_number_num(&msg->rec_nums),
+ limit))
+ goto end;
+ sc->d1->next_timeout = ossl_time_add(ossl_time_now(), ossl_seconds2time(3600));
+
+ if (idx == 2) {
+ if (!TEST_ptr(retry = BIO_new(bio_s_maybe_retry()))
+ || !TEST_true(BIO_up_ref(SSL_get_wbio(client))))
+ goto end;
+ SSL_set0_wbio(client, BIO_push(retry, SSL_get_wbio(client)));
+ retry = NULL;
+ if (!TEST_long_eq(BIO_ctrl(SSL_get_wbio(client),
+ MAYBE_RETRY_CTRL_SET_RETRY_AFTER_CNT, 1, NULL),
+ 1))
+ goto end;
+ ret = SSL_read(client, &buf, 1);
+ if (!TEST_int_eq(SSL_get_error(client, ret), SSL_ERROR_WANT_WRITE)
+ || !TEST_size_t_gt(cc->init_off, 0)
+ || !TEST_long_eq(BIO_ctrl(SSL_get_wbio(client),
+ MAYBE_RETRY_CTRL_SET_RETRY_AFTER_CNT, 100, NULL),
+ 1))
+ goto end;
+ }
+
+ if (!TEST_int_eq(SSL_read(client, &buf, 1), 1)
+ || !TEST_uchar_eq(buf, 's'))
+ goto end;
+ ret = SSL_read(server, &buf, 1);
+ if (!TEST_int_eq(SSL_get_error(server, ret), SSL_ERROR_WANT_READ)
+ || !TEST_size_t_eq(pqueue_size(&sc->d1->sent_messages), 0)
+ || !TEST_true(ossl_time_is_zero(sc->d1->next_timeout))
+ || !TEST_int_eq(SSL_write(client, "c", 1), 1)
+ || !TEST_int_eq(SSL_read(server, &buf, 1), 1)
+ || !TEST_uchar_eq(buf, 'c'))
+ goto end;
+
+ testresult = 1;
+end:
+ BIO_free(retry);
+ SSL_free(server);
+ SSL_free(client);
+ SSL_CTX_free(sctx);
+ SSL_CTX_free(cctx);
+ return testresult;
+}
#endif /* OPENSSL_NO_DTLS1_3 */
int setup_tests(void)
@@ -640,6 +797,8 @@ int setup_tests(void)
ADD_ALL_TESTS(test_dtls13_ack_coverage, 2);
ADD_ALL_TESTS(test_dtls13_ticket_ack_retransmit, 8);
ADD_TEST(test_dtls13_pha_ack_retransmit);
+ ADD_TEST(test_dtls13_ack_list_bound);
+ ADD_ALL_TESTS(test_dtls13_ack_records, 3);
#endif
return 1;
}