Extension parsing: add new fallback code which uses the new cryptography API

changelogs/fragments/331-cryptography-extensions.yml

@@ -0,0 +1,2 @@
+bugfixes:
+- "get_certificate, openssl_csr_info, x509_certificate_info - add fallback code for extension parsing that works with cryptography 36.0.0 and newer. This code re-serializes de-serialized extensions and thus can return slightly different values if the extension in the original CSR resp. certificate was not canonicalized correctly. This code is currently used as a fallback if the existing code stops working, but we will switch it to be the main code in a future release (https://github.com/ansible-collections/community.crypto/pull/331)."

plugins/module_utils/crypto/cryptography_support.py

@@ -68,87 +68,113 @@
 
 
 def cryptography_get_extensions_from_cert(cert):
-    # Since cryptography won't give us the DER value for an extension
-    # (that is only stored for unrecognized extensions), we have to re-do
-    # the extension parsing outselves.
-    backend = default_backend()
-    try:
-        # For certain old versions of cryptography, backend is a MultiBackend object,
-        # which has no _lib attribute. In that case, revert to the old approach.
-        backend._lib
-    except AttributeError:
-        backend = cert._backend
-
     result = dict()
-    x509_obj = cert._x509
-    # With cryptography 35.0.0, we can no longer use obj2txt. Unfortunately it still does
-    # not allow to get the raw value of an extension, so we have to use this ugly hack:
-    exts = list(cert.extensions)
-
-    for i in range(backend._lib.X509_get_ext_count(x509_obj)):
-        ext = backend._lib.X509_get_ext(x509_obj, i)
-        if ext == backend._ffi.NULL:
-            continue
-        crit = backend._lib.X509_EXTENSION_get_critical(ext)
-        data = backend._lib.X509_EXTENSION_get_data(ext)
-        backend.openssl_assert(data != backend._ffi.NULL)
-        der = backend._ffi.buffer(data.data, data.length)[:]
-        entry = dict(
-            critical=(crit == 1),
-            value=base64.b64encode(der),
-        )
+    try:
+        # Since cryptography won't give us the DER value for an extension
+        # (that is only stored for unrecognized extensions), we have to re-do
+        # the extension parsing outselves.
+        backend = default_backend()
         try:
-            oid = obj2txt(backend._lib, backend._ffi, backend._lib.X509_EXTENSION_get_object(ext))
+            # For certain old versions of cryptography, backend is a MultiBackend object,
+            # which has no _lib attribute. In that case, revert to the old approach.
+            backend._lib
         except AttributeError:
-            oid = exts[i].oid.dotted_string
-        result[oid] = entry
+            backend = cert._backend
+
+        x509_obj = cert._x509
+        # With cryptography 35.0.0, we can no longer use obj2txt. Unfortunately it still does
+        # not allow to get the raw value of an extension, so we have to use this ugly hack:
+        exts = list(cert.extensions)
+
+        for i in range(backend._lib.X509_get_ext_count(x509_obj)):
+            ext = backend._lib.X509_get_ext(x509_obj, i)
+            if ext == backend._ffi.NULL:
+                continue
+            crit = backend._lib.X509_EXTENSION_get_critical(ext)
+            data = backend._lib.X509_EXTENSION_get_data(ext)
+            backend.openssl_assert(data != backend._ffi.NULL)
+            der = backend._ffi.buffer(data.data, data.length)[:]
+            entry = dict(
+                critical=(crit == 1),
+                value=base64.b64encode(der),
+            )
+            try:
+                oid = obj2txt(backend._lib, backend._ffi, backend._lib.X509_EXTENSION_get_object(ext))
+            except AttributeError:
+                oid = exts[i].oid.dotted_string
+            result[oid] = entry
+
+    except Exception:
+        # In case the above method breaks, we likely have cryptography 36.0.0 or newer.
+        # Use it's public_bytes() feature in that case. We will later switch this around
+        # so that this code will be the default, but for now this will act as a fallback
+        # since it will re-serialize de-serialized data, which can be different (if the
+        # original data was not canonicalized) from what was contained in the certificate.
+        for ext in cert.extensions:
+            result[ext.oid.dotted_string] = dict(
+                critical=ext.critical,
+                value=base64.b64encode(ext.value.public_bytes()),
+            )
 
     return result
 
 
 def cryptography_get_extensions_from_csr(csr):
-    # Since cryptography won't give us the DER value for an extension
-    # (that is only stored for unrecognized extensions), we have to re-do
-    # the extension parsing outselves.
     result = dict()
-    backend = default_backend()
     try:
-        # For certain old versions of cryptography, backend is a MultiBackend object,
-        # which has no _lib attribute. In that case, revert to the old approach.
-        backend._lib
-    except AttributeError:
-        backend = csr._backend
-
-    extensions = backend._lib.X509_REQ_get_extensions(csr._x509_req)
-    extensions = backend._ffi.gc(
-        extensions,
-        lambda ext: backend._lib.sk_X509_EXTENSION_pop_free(
-            ext,
-            backend._ffi.addressof(backend._lib._original_lib, "X509_EXTENSION_free")
-        )
-    )
-
-    # With cryptography 35.0.0, we can no longer use obj2txt. Unfortunately it still does
-    # not allow to get the raw value of an extension, so we have to use this ugly hack:
-    exts = list(csr.extensions)
-
-    for i in range(backend._lib.sk_X509_EXTENSION_num(extensions)):
-        ext = backend._lib.sk_X509_EXTENSION_value(extensions, i)
-        if ext == backend._ffi.NULL:
-            continue
-        crit = backend._lib.X509_EXTENSION_get_critical(ext)
-        data = backend._lib.X509_EXTENSION_get_data(ext)
-        backend.openssl_assert(data != backend._ffi.NULL)
-        der = backend._ffi.buffer(data.data, data.length)[:]
-        entry = dict(
-            critical=(crit == 1),
-            value=base64.b64encode(der),
-        )
+        # Since cryptography won't give us the DER value for an extension
+        # (that is only stored for unrecognized extensions), we have to re-do
+        # the extension parsing outselves.
+        backend = default_backend()
         try:
-            oid = obj2txt(backend._lib, backend._ffi, backend._lib.X509_EXTENSION_get_object(ext))
+            # For certain old versions of cryptography, backend is a MultiBackend object,
+            # which has no _lib attribute. In that case, revert to the old approach.
+            backend._lib
         except AttributeError:
-            oid = exts[i].oid.dotted_string
-        result[oid] = entry
+            backend = csr._backend
+
+        extensions = backend._lib.X509_REQ_get_extensions(csr._x509_req)
+        extensions = backend._ffi.gc(
+            extensions,
+            lambda ext: backend._lib.sk_X509_EXTENSION_pop_free(
+                ext,
+                backend._ffi.addressof(backend._lib._original_lib, "X509_EXTENSION_free")
+            )
+        )
+
+        # With cryptography 35.0.0, we can no longer use obj2txt. Unfortunately it still does
+        # not allow to get the raw value of an extension, so we have to use this ugly hack:
+        exts = list(csr.extensions)
+
+        for i in range(backend._lib.sk_X509_EXTENSION_num(extensions)):
+            ext = backend._lib.sk_X509_EXTENSION_value(extensions, i)
+            if ext == backend._ffi.NULL:
+                continue
+            crit = backend._lib.X509_EXTENSION_get_critical(ext)
+            data = backend._lib.X509_EXTENSION_get_data(ext)
+            backend.openssl_assert(data != backend._ffi.NULL)
+            der = backend._ffi.buffer(data.data, data.length)[:]
+            entry = dict(
+                critical=(crit == 1),
+                value=base64.b64encode(der),
+            )
+            try:
+                oid = obj2txt(backend._lib, backend._ffi, backend._lib.X509_EXTENSION_get_object(ext))
+            except AttributeError:
+                oid = exts[i].oid.dotted_string
+            result[oid] = entry
+
+    except Exception:
+        # In case the above method breaks, we likely have cryptography 36.0.0 or newer.
+        # Use it's public_bytes() feature in that case. We will later switch this around
+        # so that this code will be the default, but for now this will act as a fallback
+        # since it will re-serialize de-serialized data, which can be different (if the
+        # original data was not canonicalized) from what was contained in the CSR.
+        for ext in csr.extensions:
+            result[ext.oid.dotted_string] = dict(
+                critical=ext.critical,
+                value=base64.b64encode(ext.value.public_bytes()),
+            )
 
     return result
 

tests/integration/targets/openssl_csr_info/tasks/impl.yml

@@ -8,14 +8,29 @@
     select_crypto_backend: '{{ select_crypto_backend }}'
   register: result
 
-- name: "({{ select_crypto_backend }}) Check whether subject behaves as expected"
+- name: "({{ select_crypto_backend }}) Check whether subject and extensions behaves as expected"
   assert:
     that:
       - result.subject.organizationalUnitName == 'ACME Department'
       - "['organizationalUnitName', 'Crypto Department'] in result.subject_ordered"
       - "['organizationalUnitName', 'ACME Department'] in result.subject_ordered"
       - result.public_key_type == 'RSA'
       - result.public_key_data.size == default_rsa_key_size
+      # TLS Feature
+      - result.extensions_by_oid['1.3.6.1.5.5.7.1.24'].critical == false
+      - result.extensions_by_oid['1.3.6.1.5.5.7.1.24'].value == 'MAMCAQU='
+      # Key Usage
+      - result.extensions_by_oid['2.5.29.15'].critical == true
+      - result.extensions_by_oid['2.5.29.15'].value in ['AwMA/4A=', 'AwMH/4A=']
+      # Subject Alternative Names
+      - result.extensions_by_oid['2.5.29.17'].critical == false
+      - result.extensions_by_oid['2.5.29.17'].value == 'MGCCD3d3dy5hbnNpYmxlLmNvbYcEAQIDBIcQAAAAAAAAAAAAAAAAAAAAAYEQdGVzdEBleGFtcGxlLm9yZ4YjaHR0cHM6Ly9leGFtcGxlLm9yZy90ZXN0L2luZGV4Lmh0bWw='
+      # Basic Constraints
+      - result.extensions_by_oid['2.5.29.19'].critical == true
+      - result.extensions_by_oid['2.5.29.19'].value == 'MAYBAf8CARc='
+      # Extended Key Usage
+      - result.extensions_by_oid['2.5.29.37'].critical == false
+      - result.extensions_by_oid['2.5.29.37'].value == 'MHQGCCsGAQUFBwMBBggrBgEFBQcDAQYIKwYBBQUHAwIGCCsGAQUFBwMDBggrBgEFBQcDBAYIKwYBBQUHAwgGCCsGAQUFBwMJBgRVHSUABggrBgEFBQcBAwYIKwYBBQUHAwoGCCsGAQUFBwMHBggrBgEFBQcBAg=='
 
 - name: "({{ select_crypto_backend }}) Check SubjectKeyIdentifier and AuthorityKeyIdentifier"
   assert:
@@ -24,6 +39,10 @@
       - result.authority_key_identifier == "44:55:66:77"
       - result.authority_cert_issuer == expected_authority_cert_issuer
       - result.authority_cert_serial_number == 12345
+      # Subject Key Identifier
+      - result.extensions_by_oid['2.5.29.14'].critical == false
+      # Authority Key Identifier
+      - result.extensions_by_oid['2.5.29.35'].critical == false
   vars:
     expected_authority_cert_issuer:
       - "DNS:ca.example.org"

tests/integration/targets/x509_certificate_info/tasks/impl.yml

@@ -8,7 +8,7 @@
     select_crypto_backend: '{{ select_crypto_backend }}'
   register: result
 
-- name: Check whether issuer and subject behave as expected
+- name: Check whether issuer and subject and extensions behave as expected
   assert:
     that:
       - result.issuer.organizationalUnitName == 'ACME Department'
@@ -27,6 +27,26 @@
           'email:test@example.org',
           'URI:https://example.org/test/index.html'
         ]"
+      # TLS Feature
+      - result.extensions_by_oid['1.3.6.1.5.5.7.1.24'].critical == false
+      - result.extensions_by_oid['1.3.6.1.5.5.7.1.24'].value == 'MAMCAQU='
+      # Key Usage
+      - result.extensions_by_oid['2.5.29.15'].critical == true
+      - result.extensions_by_oid['2.5.29.15'].value in ['AwMA/4A=', 'AwMH/4A=']
+      # Subject Alternative Names
+      - result.extensions_by_oid['2.5.29.17'].critical == false
+      - >
+        result.extensions_by_oid['2.5.29.17'].value == (
+          'MG+CD3d3dy5hbnNpYmxlLmNvbYINeG4tLTdjYTNhLmNvbYcEAQIDBIcQAAAAAAAAAAAAAAAAAAAAAYEQdGVzdEBleGFtcGxlLm9yZ4YjaHR0cHM6Ly9leGFtcGxlLm9yZy90ZXN0L2luZGV4Lmh0bWw='
+          if cryptography_version.stdout is version('2.1', '<') else
+          'MG2CD3d3dy5hbnNpYmxlLmNvbYILeG4tLTc0aC5jb22HBAECAwSHEAAAAAAAAAAAAAAAAAAAAAGBEHRlc3RAZXhhbXBsZS5vcmeGI2h0dHBzOi8vZXhhbXBsZS5vcmcvdGVzdC9pbmRleC5odG1s'
+        )
+      # Basic Constraints
+      - result.extensions_by_oid['2.5.29.19'].critical == true
+      - result.extensions_by_oid['2.5.29.19'].value == 'MAYBAf8CARc='
+      # Extended Key Usage
+      - result.extensions_by_oid['2.5.29.37'].critical == false
+      - result.extensions_by_oid['2.5.29.37'].value == 'MHQGCCsGAQUFBwMBBggrBgEFBQcDAQYIKwYBBQUHAwIGCCsGAQUFBwMDBggrBgEFBQcDBAYIKwYBBQUHAwgGCCsGAQUFBwMJBgRVHSUABggrBgEFBQcBAwYIKwYBBQUHAwoGCCsGAQUFBwMHBggrBgEFBQcBAg=='
 
 - name: Check SubjectKeyIdentifier and AuthorityKeyIdentifier
   assert:
@@ -35,6 +55,10 @@
       - result.authority_key_identifier == "44:55:66:77"
       - result.authority_cert_issuer == expected_authority_cert_issuer
       - result.authority_cert_serial_number == 12345
+      # Subject Key Identifier
+      - result.extensions_by_oid['2.5.29.14'].critical == false
+      # Authority Key Identifier
+      - result.extensions_by_oid['2.5.29.35'].critical == false
   vars:
     expected_authority_cert_issuer:
       - "DNS:ca.example.org"
@@ -114,10 +138,39 @@
     path: '{{ remote_tmp_dir }}/packed-cert-1.pem'
     select_crypto_backend: '{{ select_crypto_backend }}'
   register: result
-- assert:
+- name: Check extensions
+  assert:
     that:
       - "'ocsp_uri' in result"
       - "result.ocsp_uri == 'http://ocsp.int-x3.letsencrypt.org'"
+      - result.extensions_by_oid | length == 9
+      # Precert Signed Certificate Timestamps
+      - result.extensions_by_oid['1.3.6.1.4.1.11129.2.4.2'].critical == false
+      - result.extensions_by_oid['1.3.6.1.4.1.11129.2.4.2'].value == 'BIHyAPAAdgDBFkrgp3LS1DktyArBB3DU8MSb3pkaSEDB+gdRZPYzYAAAAWTdAoU6AAAEAwBHMEUCIG5WpfKF536KKa9fnVlYbwcfrKh09Hi2MSRwU2kad49UAiEA4RUKjJOgw11IHFNdit+sy1RcCU3QCSOEQYrJ1/oPltAAdgApPFGWVMg5ZbqqUPxYB9S3b79Yeily3KTDDPTlRUf0eAAAAWTdAoc+AAAEAwBHMEUCIQCJjo75K4rVDSiWQe3XFLY6MiG3zcHQrKb0YhM17r1UKAIgGa8qMoN03DLp+Rm9nRJ9XLbTJz1vbuu9PyXUY741P8E='
+      # Authority Information Access
+      - result.extensions_by_oid['1.3.6.1.5.5.7.1.1'].critical == false
+      - result.extensions_by_oid['1.3.6.1.5.5.7.1.1'].value == 'MGEwLgYIKwYBBQUHMAGGImh0dHA6Ly9vY3NwLmludC14My5sZXRzZW5jcnlwdC5vcmcwLwYIKwYBBQUHMAKGI2h0dHA6Ly9jZXJ0LmludC14My5sZXRzZW5jcnlwdC5vcmcv'
+      # Subject Key Identifier
+      - result.extensions_by_oid['2.5.29.14'].critical == false
+      - result.extensions_by_oid['2.5.29.14'].value == 'BBRtcOI/yg62Ehbu5vQzxMUUdBOYMw=='
+      # Key Usage (The certificate has 'AwIFoA==', while de-serializing and re-serializing yields 'AwIAoA=='!)
+      - result.extensions_by_oid['2.5.29.15'].critical == true
+      - result.extensions_by_oid['2.5.29.15'].value in ['AwIFoA==', 'AwIAoA==']
+      # Subject Alternative Names
+      - result.extensions_by_oid['2.5.29.17'].critical == false
+      - result.extensions_by_oid['2.5.29.17'].value == 'MIIB5IIbY2VydC5pbnQteDEubGV0c2VuY3J5cHQub3JnghtjZXJ0LmludC14Mi5sZXRzZW5jcnlwdC5vcmeCG2NlcnQuaW50LXgzLmxldHNlbmNyeXB0Lm9yZ4IbY2VydC5pbnQteDQubGV0c2VuY3J5cHQub3JnghxjZXJ0LnJvb3QteDEubGV0c2VuY3J5cHQub3Jngh9jZXJ0LnN0YWdpbmcteDEubGV0c2VuY3J5cHQub3Jngh9jZXJ0LnN0Zy1pbnQteDEubGV0c2VuY3J5cHQub3JngiBjZXJ0LnN0Zy1yb290LXgxLmxldHNlbmNyeXB0Lm9yZ4ISY3AubGV0c2VuY3J5cHQub3JnghpjcC5yb290LXgxLmxldHNlbmNyeXB0Lm9yZ4ITY3BzLmxldHNlbmNyeXB0Lm9yZ4IbY3BzLnJvb3QteDEubGV0c2VuY3J5cHQub3Jnghtjcmwucm9vdC14MS5sZXRzZW5jcnlwdC5vcmeCD2xldHNlbmNyeXB0Lm9yZ4IWb3JpZ2luLmxldHNlbmNyeXB0Lm9yZ4IXb3JpZ2luMi5sZXRzZW5jcnlwdC5vcmeCFnN0YXR1cy5sZXRzZW5jcnlwdC5vcmeCE3d3dy5sZXRzZW5jcnlwdC5vcmc='
+      # Basic Constraints
+      - result.extensions_by_oid['2.5.29.19'].critical == true
+      - result.extensions_by_oid['2.5.29.19'].value == 'MAA='
+      # Certificate Policies
+      - result.extensions_by_oid['2.5.29.32'].critical == false
+      - result.extensions_by_oid['2.5.29.32'].value == 'MIHzMAgGBmeBDAECATCB5gYLKwYBBAGC3xMBAQEwgdYwJgYIKwYBBQUHAgEWGmh0dHA6Ly9jcHMubGV0c2VuY3J5cHQub3JnMIGrBggrBgEFBQcCAjCBngyBm1RoaXMgQ2VydGlmaWNhdGUgbWF5IG9ubHkgYmUgcmVsaWVkIHVwb24gYnkgUmVseWluZyBQYXJ0aWVzIGFuZCBvbmx5IGluIGFjY29yZGFuY2Ugd2l0aCB0aGUgQ2VydGlmaWNhdGUgUG9saWN5IGZvdW5kIGF0IGh0dHBzOi8vbGV0c2VuY3J5cHQub3JnL3JlcG9zaXRvcnkv'
+      # Authority Key Identifier
+      - result.extensions_by_oid['2.5.29.35'].critical == false
+      - result.extensions_by_oid['2.5.29.35'].value == 'MBaAFKhKamMEfd265tE5t6ZFZe/zqOyh'
+      # Extended Key Usage
+      - result.extensions_by_oid['2.5.29.37'].critical == false
+      - result.extensions_by_oid['2.5.29.37'].value == 'MBQGCCsGAQUFBwMBBggrBgEFBQcDAg=='
 - name: Check fingerprints
   assert:
     that: