asn1: use new ASN1_BIT_STRING accessor functions with OpenSSL 4.0

ASN1_STRING has been made opaque in OpenSSL's master branch. Use the
new accessor functions instead of accessing fields directly.

Other uses of ASN1_STRING fields were already updated in
<#978>. This patch converts the
remaining ones, which require the new functions added in OpenSSL 4.0
and were not available at that time.

Author: rhenium

ext/openssl/extconf.rb

@@ -169,6 +169,9 @@ def find_openssl_library
 # added in 3.5.0
 have_func("SSL_get0_peer_signature_name(NULL, NULL)", ssl_h)
 
+# added in 4.0.0
+have_func("ASN1_BIT_STRING_set1(NULL, NULL, 0, 0)", "openssl/asn1.h")
+
 Logging::message "=== Checking done. ===\n"
 
 # Append flags from environment variables.

ext/openssl/openssl_missing.h

@@ -29,4 +29,27 @@
 #  define EVP_PKEY_eq(a, b) EVP_PKEY_cmp(a, b)
 #endif
 
+/* added in 4.0.0 */
+#ifndef HAVE_ASN1_BIT_STRING_SET1
+static inline int
+ASN1_BIT_STRING_set1(ASN1_BIT_STRING *bitstr, const uint8_t *data,
+                     size_t length, int unused_bits)
+{
+    if (length > INT_MAX || !ASN1_STRING_set(bitstr, data, (int)length))
+        return 0;
+    bitstr->flags &= ~(ASN1_STRING_FLAG_BITS_LEFT | 0x07);
+    bitstr->flags |= ASN1_STRING_FLAG_BITS_LEFT | unused_bits;
+    return 1;
+}
+
+static inline int
+ASN1_BIT_STRING_get_length(const ASN1_BIT_STRING *bitstr, size_t *length,
+                           int *unused_bits)
+{
+    *length = bitstr->length;
+    *unused_bits = bitstr->flags & 0x07;
+    return 1;
+}
+#endif
+
 #endif /* _OSSL_OPENSSL_MISSING_H_ */

ext/openssl/ossl_asn1.c

@@ -228,19 +228,19 @@ obj_to_asn1int(VALUE obj)
 }
 
 static ASN1_BIT_STRING*
-obj_to_asn1bstr(VALUE obj, long unused_bits)
+obj_to_asn1bstr(VALUE obj, int unused_bits)
 {
     ASN1_BIT_STRING *bstr;
 
     if (unused_bits < 0 || unused_bits > 7)
         ossl_raise(eASN1Error, "unused_bits for a bitstring value must be in "\
                    "the range 0 to 7");
     StringValue(obj);
-    if(!(bstr = ASN1_BIT_STRING_new()))
-        ossl_raise(eASN1Error, NULL);
-    ASN1_BIT_STRING_set(bstr, (unsigned char *)RSTRING_PTR(obj), RSTRING_LENINT(obj));
-    bstr->flags &= ~(ASN1_STRING_FLAG_BITS_LEFT|0x07); /* clear */
-    bstr->flags |= ASN1_STRING_FLAG_BITS_LEFT | unused_bits;
+    if (!(bstr = ASN1_BIT_STRING_new()))
+        ossl_raise(eASN1Error, "ASN1_BIT_STRING_new");
+    if (!ASN1_BIT_STRING_set1(bstr, (uint8_t *)RSTRING_PTR(obj),
+                              RSTRING_LEN(obj), unused_bits))
+        ossl_raise(eASN1Error, "ASN1_BIT_STRING_set1");
 
     return bstr;
 }
@@ -364,22 +364,25 @@ decode_int(unsigned char* der, long length)
 }
 
 static VALUE
-decode_bstr(unsigned char* der, long length, long *unused_bits)
+decode_bstr(unsigned char* der, long length, int *unused_bits)
 {
     ASN1_BIT_STRING *bstr;
     const unsigned char *p;
-    long len;
+    size_t len;
     VALUE ret;
+    int state;
 
     p = der;
-    if(!(bstr = d2i_ASN1_BIT_STRING(NULL, &p, length)))
-        ossl_raise(eASN1Error, NULL);
-    len = bstr->length;
-    *unused_bits = 0;
-    if(bstr->flags & ASN1_STRING_FLAG_BITS_LEFT)
-        *unused_bits = bstr->flags & 0x07;
-    ret = rb_str_new((const char *)bstr->data, len);
+    if (!(bstr = d2i_ASN1_BIT_STRING(NULL, &p, length)))
+        ossl_raise(eASN1Error, "d2i_ASN1_BIT_STRING");
+    if (!ASN1_BIT_STRING_get_length(bstr, &len, unused_bits)) {
+        ASN1_BIT_STRING_free(bstr);
+        ossl_raise(eASN1Error, "ASN1_BIT_STRING_get_length");
+    }
+    ret = ossl_str_new((const char *)ASN1_STRING_get0_data(bstr), len, &state);
     ASN1_BIT_STRING_free(bstr);
+    if (state)
+        rb_jump_tag(state);
 
     return ret;
 }
@@ -763,7 +766,7 @@ int_ossl_asn1_decode0_prim(unsigned char **pp, long length, long hlen, int tag,
 {
     VALUE value, asn1data;
     unsigned char *p;
-    long flag = 0;
+    int flag = 0;
 
     p = *pp;
 
@@ -820,7 +823,7 @@ int_ossl_asn1_decode0_prim(unsigned char **pp, long length, long hlen, int tag,
         asn1data = rb_obj_alloc(klass);
         ossl_asn1_initialize(4, args, asn1data);
         if(tag == V_ASN1_BIT_STRING){
-            rb_ivar_set(asn1data, sivUNUSED_BITS, LONG2NUM(flag));
+            rb_ivar_set(asn1data, sivUNUSED_BITS, INT2NUM(flag));
         }
     }
     else {