Merge pull request pyca#1986 from reaperhulk/cffi10

cffi 1.0

.gitignore

@@ -3,7 +3,7 @@ _build/
 build/
 dist/
 htmlcov/
-src/cryptography/_Cryptography_cffi_*
+*.so
 .tox/
 .cache/
 .coverage

.travis.yml

@@ -88,10 +88,6 @@ matrix:
           os: osx
           osx_image: beta-xcode6.3
           env: TOXENV=pypy
-        - language: generic
-          os: osx
-          osx_image: beta-xcode6.3
-          env: TOXENV=pypy3
         - language: generic
           os: osx
           osx_image: beta-xcode6.3
@@ -112,10 +108,6 @@ matrix:
           os: osx
           osx_image: beta-xcode6.3
           env: TOXENV=pypy OPENSSL=0.9.8
-        - language: generic
-          os: osx
-          osx_image: beta-xcode6.3
-          env: TOXENV=pypy3 OPENSSL=0.9.8
         - language: generic
           os: osx
           osx_image: beta-xcode6.3

.travis/install.sh

@@ -35,8 +35,8 @@ if [[ "$(uname -s)" == 'Darwin' ]]; then
             ;;
         pypy)
             brew outdated pyenv || brew upgrade pyenv
-            pyenv install pypy-2.5.1
-            pyenv global pypy-2.5.1
+            pyenv install pypy-2.6.0
+            pyenv global pypy-2.6.0
             ;;
         pypy3)
             brew outdated pyenv || brew upgrade pyenv
@@ -51,6 +51,15 @@ if [[ "$(uname -s)" == 'Darwin' ]]; then
     pyenv rehash
     python -m pip install --user virtualenv
 else
+    # temporary pyenv installation to get pypy-2.6 before container infra upgrade
+    if [[ "${TOXENV}" == "pypy" ]]; then
+        git clone https://github.com/yyuu/pyenv.git ~/.pyenv
+        PYENV_ROOT="$HOME/.pyenv"
+        PATH="$PYENV_ROOT/bin:$PATH"
+        eval "$(pyenv init -)"
+        pyenv install pypy-2.6.0
+        pyenv global pypy-2.6.0
+    fi
     pip install virtualenv
 fi
 

.travis/run.sh

@@ -16,6 +16,13 @@ if [[ "$(uname -s)" == "Darwin" ]]; then
         # CommonCrypto when we test against brew OpenSSL
         export TOX_FLAGS="--backend=openssl"
     fi
+else
+    if [[ "${TOXENV}" == "pypy" ]]; then
+        PYENV_ROOT="$HOME/.pyenv"
+        PATH="$PYENV_ROOT/bin:$PATH"
+        eval "$(pyenv init -)"
+        pyenv global pypy-2.6.0
+    fi
 fi
 source ~/.venv/bin/activate
 tox -- $TOX_FLAGS

CHANGELOG.rst

@@ -6,6 +6,11 @@ Changelog
 
 .. note:: This version is not yet released and is under active development.
 
+* Switched to the new `cffi`_ ``set_source`` out-of-line API mode for
+  compilation. This results in significantly faster imports and lowered
+  memory consumption. Due to this change we no longer support PyPy releases
+  older than 2.6 nor do we support any released version of PyPy3 (until a
+  version supporting cffi 1.0 comes out).
 * Support serialization of certificate signing requests using the
   ``public_bytes`` method of
   :class:`~cryptography.x509.CertificateSigningRequest`.
@@ -442,3 +447,4 @@ Changelog
 * Initial release.
 
 .. _`master`: https://github.com/pyca/cryptography/
+.. _`cffi`: https://cffi.readthedocs.org/en/latest/

MANIFEST.in

@@ -7,8 +7,7 @@ include LICENSE.BSD
 include README.rst
 
 recursive-include docs *
-recursive-include src/cryptography/hazmat/primitives/src *.c *.h
-recursive-include src/cryptography/hazmat/bindings/openssl/src *.c *.h
+recursive-include src/_cffi_src *.py *.c *.h
 prune docs/_build
 recursive-include tests *.py
 recursive-exclude vectors *

setup.py

@@ -45,7 +45,7 @@
     requirements.append("ipaddress")
 
 if platform.python_implementation() != "PyPy":
-    requirements.append("cffi>=0.8")
+    requirements.append("cffi>=1.1.0")
 
 # If you add a new dep here you probably need to add it in the tox.ini as well
 test_requirements = [
@@ -75,52 +75,6 @@ def cc_is_available():
     )
 
 
-def get_ext_modules():
-    from cryptography.hazmat.bindings.commoncrypto.binding import (
-        Binding as CommonCryptoBinding
-    )
-    from cryptography.hazmat.bindings.openssl.binding import (
-        Binding as OpenSSLBinding
-    )
-    from cryptography.hazmat.primitives import constant_time, padding
-
-    ext_modules = [
-        OpenSSLBinding.ffi.verifier.get_extension(),
-        constant_time._ffi.verifier.get_extension(),
-        padding._ffi.verifier.get_extension()
-    ]
-    if cc_is_available():
-        ext_modules.append(CommonCryptoBinding.ffi.verifier.get_extension())
-    return ext_modules
-
-
-class CFFIBuild(build):
-    """
-    This class exists, instead of just providing ``ext_modules=[...]`` directly
-    in ``setup()`` because importing cryptography requires we have several
-    packages installed first.
-
-    By doing the imports here we ensure that packages listed in
-    ``setup_requires`` are already installed.
-    """
-
-    def finalize_options(self):
-        self.distribution.ext_modules = get_ext_modules()
-        build.finalize_options(self)
-
-
-class CFFIInstall(install):
-    """
-    As a consequence of CFFIBuild and it's late addition of ext_modules, we
-    need the equivalent for the ``install`` command to install into platlib
-    install-dir rather than purelib.
-    """
-
-    def finalize_options(self):
-        self.distribution.ext_modules = get_ext_modules()
-        install.finalize_options(self)
-
-
 class PyTest(test):
     def finalize_options(self):
         test.finalize_options(self)
@@ -234,19 +188,26 @@ def argument_without_setup_requirements(argv, i):
            for i in range(1, len(argv))):
         return {
             "cmdclass": {
-                "build": DummyCFFIBuild,
-                "install": DummyCFFIInstall,
+                "build": DummyBuild,
+                "install": DummyInstall,
                 "test": DummyPyTest,
             }
         }
     else:
+        cffi_modules = [
+            "src/_cffi_src/build_openssl.py:ffi",
+            "src/_cffi_src/build_constant_time.py:ffi",
+            "src/_cffi_src/build_padding.py:ffi",
+        ]
+        if cc_is_available():
+            cffi_modules.append("src/_cffi_src/build_commoncrypto.py:ffi")
+
         return {
             "setup_requires": requirements,
             "cmdclass": {
-                "build": CFFIBuild,
-                "install": CFFIInstall,
                 "test": PyTest,
-            }
+            },
+            "cffi_modules": cffi_modules
         }
 
 
@@ -255,7 +216,7 @@ def argument_without_setup_requirements(argv, i):
                         "free command or option.")
 
 
-class DummyCFFIBuild(build):
+class DummyBuild(build):
     """
     This class makes it very obvious when ``keywords_with_side_effects()`` has
     incorrectly interpreted the command line arguments to ``setup.py build`` as
@@ -266,7 +227,7 @@ def run(self):
         raise RuntimeError(setup_requires_error)
 
 
-class DummyCFFIInstall(install):
+class DummyInstall(install):
     """
     This class makes it very obvious when ``keywords_with_side_effects()`` has
     incorrectly interpreted the command line arguments to ``setup.py install``
@@ -327,15 +288,17 @@ def run_tests(self):
     ],
 
     package_dir={"": "src"},
-    packages=find_packages(where="src", exclude=["tests", "tests.*"]),
+    packages=find_packages(
+        where="src", exclude=["_cffi_src", "_cffi_src.*", "tests", "tests.*"]
+    ),
     include_package_data=True,
 
     install_requires=requirements,
     tests_require=test_requirements,
 
     # for cffi
     zip_safe=False,
-    ext_package="cryptography",
+    ext_package="cryptography.hazmat.bindings",
     entry_points={
         "cryptography.backends": backends,
     },

src/_cffi_src/build_commoncrypto.py

@@ -0,0 +1,29 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+from _cffi_src.utils import build_ffi_for_binding
+
+
+ffi = build_ffi_for_binding(
+    module_name="_commoncrypto",
+    module_prefix="_cffi_src.commoncrypto.",
+    modules=[
+        "cf",
+        "common_digest",
+        "common_hmac",
+        "common_key_derivation",
+        "common_cryptor",
+        "common_symmetric_key_wrap",
+        "secimport",
+        "secitem",
+        "seckey",
+        "seckeychain",
+        "sectransform",
+    ],
+    extra_link_args=[
+        "-framework", "Security", "-framework", "CoreFoundation"
+    ],
+)

src/_cffi_src/build_constant_time.py

@@ -0,0 +1,26 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+import os
+
+from _cffi_src.utils import build_ffi
+
+
+with open(os.path.join(
+    os.path.dirname(__file__), "hazmat_src/constant_time.h"
+)) as f:
+    types = f.read()
+
+with open(os.path.join(
+    os.path.dirname(__file__), "hazmat_src/constant_time.c"
+)) as f:
+    functions = f.read()
+
+ffi = build_ffi(
+    module_name="_constant_time",
+    cdef_source=types,
+    verify_source=functions
+)

src/_cffi_src/build_openssl.py

@@ -0,0 +1,98 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+import os
+import sys
+
+from _cffi_src.utils import (
+    build_ffi_for_binding
+)
+
+
+def _get_openssl_libraries(platform):
+    # OpenSSL goes by a different library name on different operating systems.
+    if platform != "win32":
+        # In some circumstances, the order in which these libs are
+        # specified on the linker command-line is significant;
+        # libssl must come before libcrypto
+        # (http://marc.info/?l=openssl-users&m=135361825921871)
+        return ["ssl", "crypto"]
+    else:
+        link_type = os.environ.get("PYCA_WINDOWS_LINK_TYPE", "static")
+        return _get_openssl_windows_libraries(link_type)
+
+
+def _get_openssl_windows_libraries(link_type):
+    if link_type == "dynamic":
+        return ["libeay32", "ssleay32", "advapi32"]
+    elif link_type == "static" or link_type == "":
+        return ["libeay32mt", "ssleay32mt", "advapi32",
+                "crypt32", "gdi32", "user32", "ws2_32"]
+    else:
+        raise ValueError(
+            "PYCA_WINDOWS_LINK_TYPE must be 'static' or 'dynamic'"
+        )
+
+
+_OSX_PRE_INCLUDE = """
+#ifdef __APPLE__
+#include <AvailabilityMacros.h>
+#define __ORIG_DEPRECATED_IN_MAC_OS_X_VERSION_10_7_AND_LATER \
+    DEPRECATED_IN_MAC_OS_X_VERSION_10_7_AND_LATER
+#undef DEPRECATED_IN_MAC_OS_X_VERSION_10_7_AND_LATER
+#define DEPRECATED_IN_MAC_OS_X_VERSION_10_7_AND_LATER
+#endif
+"""
+
+_OSX_POST_INCLUDE = """
+#ifdef __APPLE__
+#undef DEPRECATED_IN_MAC_OS_X_VERSION_10_7_AND_LATER
+#define DEPRECATED_IN_MAC_OS_X_VERSION_10_7_AND_LATER \
+    __ORIG_DEPRECATED_IN_MAC_OS_X_VERSION_10_7_AND_LATER
+#endif
+"""
+
+
+ffi = build_ffi_for_binding(
+    module_name="_openssl",
+    module_prefix="_cffi_src.openssl.",
+    modules=[
+        "aes",
+        "asn1",
+        "bignum",
+        "bio",
+        "cmac",
+        "cms",
+        "conf",
+        "crypto",
+        "dh",
+        "dsa",
+        "ec",
+        "ecdh",
+        "ecdsa",
+        "engine",
+        "err",
+        "evp",
+        "hmac",
+        "nid",
+        "objects",
+        "opensslv",
+        "osrandom_engine",
+        "pem",
+        "pkcs7",
+        "pkcs12",
+        "rand",
+        "rsa",
+        "ssl",
+        "x509",
+        "x509name",
+        "x509v3",
+        "x509_vfy"
+    ],
+    pre_include=_OSX_PRE_INCLUDE,
+    post_include=_OSX_POST_INCLUDE,
+    libraries=_get_openssl_libraries(sys.platform)
+)