Commit ab566c794e for openssl.org

commit ab566c794e6564d0624a509c533b66d82b0ce99d
Author: Jakub Zelenka <jakub.zelenka@openssl.foundation>
Date:   Sat Aug 29 13:36:14 2026 +0200

    apps: cover the rsa error cases in the test recipe

    Cover the bad option failures (invalid input and output formats,
    extra arguments, unknown cipher and invalid password argument) as
    well as the failures when checking a public key, loading an invalid
    or non-RSA key file and when requesting an unsupported or impossible
    output format, verifying the error messages printed to stderr with
    the shared app_fails helper.

    Reviewed-by: Richard Levitte <levitte@openssl.org>
    Reviewed-by: Daniel Kubec <kubec@openssl.foundation>
    Merge-date: Wed Sep 16 08:47:39 2026
    Merged-from: https://github.com/openssl/openssl/pull/32591

diff --git a/test/recipes/15-test_rsa.t b/test/recipes/15-test_rsa.t
index 983549ef35..f3581484fe 100644
--- a/test/recipes/15-test_rsa.t
+++ b/test/recipes/15-test_rsa.t
@@ -12,12 +12,12 @@ use warnings;

 use File::Spec;
 use File::Compare qw/compare/;
-use OpenSSL::Test qw/:DEFAULT srctop_file/;
+use OpenSSL::Test qw/:DEFAULT srctop_file app_fails/;
 use OpenSSL::Test::Utils;

 setup("test_rsa");

-plan tests => 18;
+plan tests => 19;

 require_ok(srctop_file('test', 'recipes', 'tconversion.pl'));

@@ -28,7 +28,7 @@ run_rsa_tests("pkey");
 run_rsa_tests("rsa");

 SKIP: {
-    skip "RSA is not supported in this build", 2 if disabled("rsa");
+    skip "RSA is not supported in this build", 3 if disabled("rsa");

     subtest "rsa -modulus prints the RSA modulus" => sub {
         plan tests => 2;
@@ -97,6 +97,62 @@ SKIP: {
         ok(!grep(/privateExponent/, @pub),
            "-text does not print a private exponent for a public key");
     };
+
+    subtest "rsa error cases" => sub {
+        plan tests => 20;
+
+        my $privkey = srctop_file("test", "testrsa.pem");
+        my $pubkey = srctop_file("test", "testrsapub.pem");
+
+        app_fails('rsa', "invalid input format should fail",
+                  qr/Invalid format "BAD" for option -inform/,
+                  '-inform', 'BAD', '-in', $privkey);
+        app_fails('rsa', "invalid output format should fail",
+                  qr/Invalid format "BAD" for option -outform/,
+                  '-outform', 'BAD', '-in', $privkey);
+        app_fails('rsa', "extra positional argument should fail",
+                  qr/Extra option: "extra"/,
+                  '-in', $privkey, 'extra');
+        app_fails('rsa', "unknown cipher option should fail",
+                  qr/Unknown option or cipher: badcipher/,
+                  '-badcipher', '-in', $privkey);
+        app_fails('rsa', "invalid passin argument should fail",
+                  qr/Error getting passwords/,
+                  '-passin', 'bad:pass', '-in', $privkey);
+        app_fails('rsa', "checking a public key should fail",
+                  qr/Only private keys can be checked/,
+                  '-check', '-pubin', '-in', $pubkey);
+        app_fails('rsa', "unsupported output format should fail",
+                  qr/bad output format specified for outfile/,
+                  '-in', $privkey, '-outform', 'NSS', '-out', 'rsa-nss.out');
+        app_fails('rsa', "PVK output for public key input should fail",
+                  qr/PVK form impossible with public key input/,
+                  '-pubin', '-in', $pubkey, '-outform', 'PVK',
+                  '-out', 'rsa-pvk.out');
+
+        my $garbage = "garbage.pem";
+        open(my $fh, '>', $garbage) or die "Cannot write $garbage: $!";
+        print $fh "not a valid RSA key file\n";
+        close($fh);
+        app_fails('rsa', "loading garbage key file should fail",
+                  qr/Could not find or decode private key/,
+                  '-in', $garbage, '-noout');
+
+        SKIP: {
+            my $nonrsa = !disabled("ec")
+                ? srctop_file("test", "testec-p256.pem")
+                : !disabled("dsa")
+                    ? srctop_file("test", "testdsa.pem")
+                    : undef;
+
+            skip "No non-RSA key type available in this build", 2
+                if !defined $nonrsa;
+
+            app_fails('rsa', "loading a non-RSA key should fail",
+                      qr/Not an RSA key/,
+                      '-in', $nonrsa, '-noout');
+        }
+    };
 }

 sub run_rsa_tests {