Commit d1ebd2a7 for tesseract
commit d1ebd2a789f8220209d675c1d4bc18d7a25c8e26
Author: Stefan Weil <sw@weilnetz.de>
Date: Mon Nov 2 09:00:07 2026 +0100
serialis: require declared sizes to be backed by remaining bytes (#4675)
A crafted .traineddata file could declare tens of millions of elements
for a std::vector, a std::string, or a std::vector<char> while carrying
only a few trailing bytes. The deserializers resized the container
before reading, forcing a multi-gigabyte allocation from a tiny input
(denial of service; see GHSA-fxpc-5x59-r9px, reported via the recoder
encoder_ path: 50,000,000 RecodedCharID declares ~2.2 GB).
The declared size is now checked against the bytes actually remaining
in the stream, using the minimum serialized size per element:
- 4 for a string (the uint32 length prefix is always present),
- 1 for a class or a pointer (a field, or the non-null flag),
- sizeof(T) for a plain type.
sizeof(T) deliberately is not used for the check: a class like
RecodedCharID serializes to fewer bytes than its sizeof (5 to 41 vs 44)
and a null pointer to a single byte, so a sizeof-based bound would
reject valid models.
Add regression tests for all three affected deserializers, including
the exact std::vector<RecodedCharID> case, plus fully-backed round-trips
that prove valid data is still accepted.
Assisted-by: OpenCode / qwen3.8-27b-thinking (Alibaba Cloud)
Signed-off-by: Stefan Weil <sw@weilnetz.de>
diff --git a/src/ccutil/serialis.cpp b/src/ccutil/serialis.cpp
index b8914c0b..cd542262 100644
--- a/src/ccutil/serialis.cpp
+++ b/src/ccutil/serialis.cpp
@@ -103,6 +103,12 @@ bool TFile::DeSerialize(std::string &data) {
// Arbitrarily limit the size to protect against bad data.
return false;
} else if (size > 0) {
+ // The declared size must be backed by the bytes that actually remain in
+ // the stream, so a crafted file cannot declare a huge string while
+ // carrying only a few trailing bytes (denial of service).
+ if (static_cast<uint64_t>(size) > RemainingBytes()) {
+ return false;
+ }
// TODO: optimize.
data.resize(size);
return DeSerialize(&data[0], size);
@@ -124,6 +130,12 @@ bool TFile::DeSerialize(std::vector<char> &data) {
// Arbitrarily limit the size to protect against bad data.
return false;
} else if (size > 0) {
+ // The declared size must be backed by the bytes that actually remain in
+ // the stream, so a crafted file cannot declare a huge vector while
+ // carrying only a few trailing bytes (denial of service).
+ if (static_cast<uint64_t>(size) > RemainingBytes()) {
+ return false;
+ }
// TODO: optimize.
data.resize(size);
return DeSerialize(&data[0], data.size());
diff --git a/src/ccutil/serialis.h b/src/ccutil/serialis.h
index 59a4e1f1..c9ece996 100644
--- a/src/ccutil/serialis.h
+++ b/src/ccutil/serialis.h
@@ -20,6 +20,7 @@
#define SERIALIS_H
#include <tesseract/baseapi.h> // FileReader
+#include <cstddef> // for size_t
#include <cstdint> // uint8_t
#include <cstdio>
#include <cstdlib>
@@ -97,53 +98,78 @@ public:
return false;
} else if (size == 0) {
data.clear();
+ return true;
} else if (size > 50000000) {
// Arbitrarily limit the number of elements to protect against bad data.
return false;
- } else if constexpr (std::is_same<T, std::string>::value) {
- // Deserialize a string.
- // TODO: optimize.
- data.resize(size);
- for (auto &item : data) {
- if (!DeSerialize(item)) {
- return false;
- }
+ } else {
+ // The numeric cap alone is not enough: a crafted file could declare
+ // tens of millions of elements while carrying only a few trailing
+ // bytes, forcing a huge allocation (denial of service) from a tiny
+ // input. The declared size must be backed by the bytes that actually
+ // remain in the stream. The bound must use the *minimum serialized*
+ // size per element, not sizeof(T): a class like RecodedCharID
+ // serializes to fewer bytes than its sizeof (5 to 41 vs 44), and a
+ // null pointer serializes to a single byte, so a sizeof-based check
+ // would reject valid models. The minimum per element is 4 for a
+ // string (the uint32 length prefix is present even when empty), 1 for
+ // a class or a pointer (a field, or the non-null flag), and exactly
+ // sizeof(T) for a plain type.
+ constexpr size_t kMinBytesPerElement =
+ std::is_same<T, std::string>::value
+ ? static_cast<size_t>(4)
+ : (std::is_class<T>::value || std::is_pointer<T>::value)
+ ? static_cast<size_t>(1)
+ : sizeof(T);
+ if (static_cast<uint64_t>(size) * kMinBytesPerElement >
+ RemainingBytes()) {
+ return false;
}
- } else if constexpr (std::is_class<T>::value) {
- // Deserialize a tesseract class.
- // TODO: optimize.
- data.resize(size);
- for (auto &item : data) {
- if (!item.DeSerialize(this)) {
- return false;
+ if constexpr (std::is_same<T, std::string>::value) {
+ // Deserialize a string.
+ // TODO: optimize.
+ data.resize(size);
+ for (auto &item : data) {
+ if (!DeSerialize(item)) {
+ return false;
+ }
}
- }
- } else if constexpr (std::is_pointer<T>::value) {
- // Deserialize pointers.
- // TODO: optimize.
- data.resize(size);
- for (uint32_t i = 0; i < size; i++) {
- uint8_t non_null;
- if (!DeSerialize(&non_null)) {
- return false;
- }
- if (non_null) {
- typedef typename std::remove_pointer<T>::type ST;
- auto item = new ST;
- if (!item->DeSerialize(this)) {
- delete item;
+ } else if constexpr (std::is_class<T>::value) {
+ // Deserialize a tesseract class.
+ // TODO: optimize.
+ data.resize(size);
+ for (auto &item : data) {
+ if (!item.DeSerialize(this)) {
+ return false;
+ }
+ }
+ } else if constexpr (std::is_pointer<T>::value) {
+ // Deserialize pointers.
+ // TODO: optimize.
+ data.resize(size);
+ for (uint32_t i = 0; i < size; i++) {
+ uint8_t non_null;
+ if (!DeSerialize(&non_null)) {
return false;
}
- data[i] = item;
+ if (non_null) {
+ typedef typename std::remove_pointer<T>::type ST;
+ auto item = new ST;
+ if (!item->DeSerialize(this)) {
+ delete item;
+ return false;
+ }
+ data[i] = item;
+ }
}
+ } else {
+ // Deserialize a non-class.
+ // TODO: optimize.
+ data.resize(size);
+ return DeSerialize(&data[0], size);
}
- } else {
- // Deserialize a non-class.
- // TODO: optimize.
- data.resize(size);
- return DeSerialize(&data[0], size);
+ return true;
}
- return true;
}
// Serialize data.
diff --git a/unittest/tfile_test.cc b/unittest/tfile_test.cc
index aeb77744..6a180ec5 100644
--- a/unittest/tfile_test.cc
+++ b/unittest/tfile_test.cc
@@ -10,9 +10,12 @@
// limitations under the License.
#include "serialis.h"
+#include "unicharcompress.h" // for RecodedCharID
#include "include_gunit.h"
+#include <cstdint> // for uint32_t
+
namespace tesseract {
// Tests TFile and std::vector serialization by serializing and
@@ -212,4 +215,142 @@ TEST_F(TfileTest, BigEndian) {
m3.ExpectEq(m2);
}
+// Builds a little-endian uint32 followed by the given number of zero
+// trailing bytes, mirroring the count-prefixed layout the vector, string
+// and char-vector deserializers expect.
+std::vector<char> MakeCountBuffer(uint32_t count, int trailing_bytes) {
+ std::vector<char> buf;
+ for (int i = 0; i < 4; ++i) {
+ buf.push_back(static_cast<char>((count >> (8 * i)) & 0xFF));
+ }
+ buf.resize(buf.size() + trailing_bytes, 0);
+ return buf;
+}
+
+// A declared element count that is not backed by the bytes remaining in
+// the stream must be rejected before the vector is resized, so a tiny
+// crafted file cannot force a multi-gigabyte allocation (denial of
+// service). This is the case for a recoder encoder_ of 50,000,000
+// RecodedCharID: 50,000,000 * sizeof(RecodedCharID) is ~2.2 GB.
+TEST_F(TfileTest, RejectsUnbackedVectorCount) {
+ std::vector<char> data = MakeCountBuffer(50000000, 8);
+ TFile fp;
+ EXPECT_TRUE(fp.Open(&data[0], data.size()));
+ std::vector<int> vec;
+ EXPECT_FALSE(fp.DeSerialize(vec));
+ // The vector must not have been resized before the rejection.
+ EXPECT_TRUE(vec.empty());
+}
+
+TEST_F(TfileTest, RejectsUnbackedStringVectorCount) {
+ std::vector<char> data = MakeCountBuffer(50000000, 8);
+ TFile fp;
+ EXPECT_TRUE(fp.Open(&data[0], data.size()));
+ std::vector<std::string> vec;
+ EXPECT_FALSE(fp.DeSerialize(vec));
+ EXPECT_TRUE(vec.empty());
+}
+
+TEST_F(TfileTest, RejectsUnbackedCharVectorSize) {
+ std::vector<char> data = MakeCountBuffer(50000000, 8);
+ TFile fp;
+ EXPECT_TRUE(fp.Open(&data[0], data.size()));
+ std::vector<char> vec;
+ EXPECT_FALSE(fp.DeSerialize(vec));
+ EXPECT_TRUE(vec.empty());
+}
+
+TEST_F(TfileTest, RejectsUnbackedStringSize) {
+ std::vector<char> data = MakeCountBuffer(50000000, 8);
+ TFile fp;
+ EXPECT_TRUE(fp.Open(&data[0], data.size()));
+ std::string str;
+ EXPECT_FALSE(fp.DeSerialize(str));
+ EXPECT_TRUE(str.empty());
+}
+
+// A declared element count of zero must be accepted (it clears the vector).
+// This is the only path that falls out of the branch chain without an
+// explicit read, so it must return a value (UBSan: "execution reached the
+// end of a value-returning function without returning a value").
+TEST_F(TfileTest, AcceptsZeroVectorCount) {
+ std::vector<char> data = MakeCountBuffer(0, 0);
+ TFile fp;
+ EXPECT_TRUE(fp.Open(&data[0], data.size()));
+ std::vector<int> vec = {1, 2, 3};
+ EXPECT_TRUE(fp.DeSerialize(vec));
+ EXPECT_TRUE(vec.empty());
+}
+
+// A count that is fully backed by the remaining bytes must still be
+// accepted, so the stream-backing check does not reject valid data.
+TEST_F(TfileTest, AcceptsFullyBackedVectorCount) {
+ TFile fpw;
+ std::vector<char> data;
+ fpw.OpenWrite(&data);
+ std::vector<int> values = {7, -12, 0};
+ EXPECT_TRUE(fpw.Serialize(values));
+ TFile fp;
+ EXPECT_TRUE(fp.Open(&data[0], data.size()));
+ std::vector<int> vec;
+ EXPECT_TRUE(fp.DeSerialize(vec));
+ ASSERT_EQ(3, vec.size());
+ EXPECT_EQ(7, vec[0]);
+ EXPECT_EQ(-12, vec[1]);
+ EXPECT_EQ(0, vec[2]);
+}
+
+// A string vector that is fully backed (each string serialized as a uint32
+// length plus its bytes) must still be accepted.
+TEST_F(TfileTest, AcceptsFullyBackedStringVectorCount) {
+ TFile fpw;
+ std::vector<char> data;
+ fpw.OpenWrite(&data);
+ std::vector<std::string> strings = {"ab", ""};
+ EXPECT_TRUE(fpw.Serialize(strings));
+ TFile fpr;
+ EXPECT_TRUE(fpr.Open(&data[0], data.size()));
+ std::vector<std::string> result;
+ EXPECT_TRUE(fpr.DeSerialize(result));
+ ASSERT_EQ(2, result.size());
+ EXPECT_EQ("ab", result[0]);
+ EXPECT_EQ("", result[1]);
+}
+
+// The exact type from GHSA-fxpc-5x59-r9px: the recoder encoder_.
+// Declaring 50,000,000 RecodedCharID (~2.2 GB) from a tiny stream must be
+// rejected in the class branch.
+TEST_F(TfileTest, RejectsUnbackedRecodedCharIDVectorCount) {
+ std::vector<char> data = MakeCountBuffer(50000000, 8);
+ TFile fp;
+ EXPECT_TRUE(fp.Open(&data[0], data.size()));
+ std::vector<RecodedCharID> vec;
+ EXPECT_FALSE(fp.DeSerialize(vec));
+ EXPECT_TRUE(vec.empty());
+}
+
+// A fully backed vector of RecodedCharID must still be accepted, even
+// though sizeof(RecodedCharID) is 44 while it serializes to only 5 to 41
+// bytes per element (a self-normalized flag, a uint32 length, and that
+// many int32 codes): a sizeof-based backing check would wrongly reject
+// this.
+TEST_F(TfileTest, AcceptsFullyBackedRecodedCharIDVectorCount) {
+ std::vector<RecodedCharID> codes(3);
+ codes[0].Set(0, 1);
+ codes[1].Set3(2, 3, 4);
+ TFile fpw;
+ std::vector<char> data;
+ fpw.OpenWrite(&data);
+ EXPECT_TRUE(fpw.Serialize(codes));
+ TFile fpr;
+ EXPECT_TRUE(fpr.Open(&data[0], data.size()));
+ std::vector<RecodedCharID> result;
+ EXPECT_TRUE(fpr.DeSerialize(result));
+ ASSERT_EQ(3, result.size());
+ EXPECT_EQ(1u, result[0].length());
+ EXPECT_EQ(1, result[0](0));
+ EXPECT_EQ(3u, result[1].length());
+ EXPECT_EQ(4, result[1](2));
+}
+
} // namespace tesseract