digest: let #finish raise if the output size is unknown

In OpenSSL 3.4 or later, SHAKE digests do not have a default output
length and EVP_MD_CTX_size() returns -1. Raise
OpenSSL::Digest::DigestError if the function returns a negative number.

Fortunately, this is currently caught by a sanity check in
rb_str_new() and leads to ArgumentError with the message "negative
string size (or size too big)". Still, the caller should check it.

Also, override #inspect to rescue an exception raised in #hexdigest.

Author: rhenium

ext/openssl/ossl_digest.c

@@ -283,9 +283,20 @@ ossl_digest_finish(VALUE self)
 {
     EVP_MD_CTX *ctx;
     VALUE str;
+    int size;
 
     GetDigest(self, ctx);
-    str = rb_str_new(NULL, EVP_MD_CTX_size(ctx));
+    size = EVP_MD_CTX_size(ctx);
+    if (size <= 0) {
+#ifdef EVP_MD_FLAG_XOF /* Not in LibreSSL 4.3 (latest) */
+        if (EVP_MD_flags(EVP_MD_CTX_get0_md(ctx)) & EVP_MD_FLAG_XOF)
+            ossl_raise(eDigestError,
+                       "output length not set for XOF; " \
+                       "use OpenSSL::Digest#squeeze instead");
+#endif
+        ossl_raise(eDigestError, "EVP_MD_CTX_size");
+    }
+    str = rb_str_new(NULL, size);
     if (!EVP_DigestFinal_ex(ctx, (unsigned char *)RSTRING_PTR(str), NULL))
         ossl_raise(eDigestError, "EVP_DigestFinal_ex");
 

lib/openssl/digest.rb

@@ -52,6 +52,10 @@ def hexdigest(data)
     class Digest < Digest; end # :nodoc:
     deprecate_constant :Digest
 
+    def inspect
+      "#<#{self.class}: #{name} #{hexdigest rescue $!.inspect}>"
+    end
+
   end # Digest
 
   # Returns a Digest subclass by _name_

test/openssl/test_digest.rb

@@ -158,6 +158,15 @@ def test_xof_squeeze
     assert_raise(OpenSSL::Digest::DigestError) { digest.update("a") }
   end
 
+  def test_xof_digest
+    # SHAKE had a default output length in OpenSSL < 3.4
+    return unless openssl? && openssl?(3, 4, 0)
+
+    digest = OpenSSL::Digest.new("SHAKE128")
+    assert_raise(OpenSSL::Digest::DigestError) { digest.digest }
+    assert_match(/#<OpenSSL::Digest: SHAKE128/, digest.inspect)
+  end
+
   def test_openssl_digest
     assert_equal OpenSSL::Digest::MD5, OpenSSL::Digest("MD5")