Use a specific version of Freetype for testing

This should theoretically allow us to turn down the
tolerance on image comparison tests.

.gitignore

@@ -39,6 +39,7 @@ dist
 .eggs
 # tox testing tool
 .tox
+setup.cfg
 
 # OS generated files #
 ######################

.travis.yml

@@ -75,7 +75,7 @@ install:
   # version since is it basically just a .ttf file
   # The current Travis Ubuntu image is to old to search .local/share/fonts so we store fonts in .fonts
 
-  # We install ipython to use the console highlighting. From IPython 3 this depends on jsonschema and misture.
+  # We install ipython to use the console highlighting. From IPython 3 this depends on jsonschema and mistune.
   # Neihter is installed as a dependency of IPython since they are not used by the IPython console.
   - |
     if [[ $BUILD_DOCS == true ]]; then
@@ -90,6 +90,10 @@ install:
       cp Felipa-Regular.ttf ~/.fonts
       fc-cache -f -v
     fi;
+
+  # Use the special local version of freetype for testing
+  - echo '[test]\ntesting_freetype = True' > setup.cfg
+
   - python setup.py install
 
 script:

doc/devel/testing.rst

@@ -33,6 +33,22 @@ Optionally you can install:
 
   - `pep8 <http://pep8.readthedocs.org/en/latest>`_ to test coding standards
 
+Build matplotlib for image comparison tests
+-------------------------------------------
+
+matplotlib's test suite makes heavy use of image comparison tests,
+meaning the result of a plot is compared against a known good result.
+Unfortunately, different versions of freetype produce differently
+formed characters, causing these image comparisons to fail.  To make
+them reproducible, matplotlib can be built with a special local copy
+of freetype.  This is recommended for all matplotlib developers.
+
+Add the following content to a ``setup.cfg`` file at the root of the
+matplotlib source directory::
+
+  [test]
+  testing_freetype = True
+
 Running the tests
 -----------------
 
@@ -185,17 +201,6 @@ decorator:
      If some variation is expected in the image between runs, this
      value may be adjusted.
 
-Freetype version
-----------------
-
-Due to subtle differences in the font rendering under different
-version of freetype some care must be taken when generating the
-baseline images.  Currently (early 2015), almost all of the images
-were generated using ``freetype 2.5.3-21`` on Fedora 21 and only the
-fonts that ship with ``matplotlib`` (regenerated in PR #4031 / commit
-005cfde02751d274f2ab8016eddd61c3b3828446) and travis is using
-``freetype 2.4.8`` on ubuntu.
-
 Known failing tests
 -------------------
 

lib/matplotlib/__init__.py

@@ -1499,6 +1499,17 @@ def _init_tests():
     if not os.path.isdir(os.path.join(os.path.dirname(__file__), 'tests')):
         raise ImportError("matplotlib test data is not installed")
 
+    # The version of freetype to install locally for running the
+    # tests.  This must match the value in `setupext.py`
+    LOCAL_FREETYPE_VERSION = '2.5.2'
+
+    from matplotlib import ft2font
+    if (ft2font.__freetype_version__ != LOCAL_FREETYPE_VERSION or
+        ft2font.__freetype_build_type__ != 'local'):
+        raise ImportError(
+            "matplotlib is not built with the correct freetype version to run "
+            "tests.  Set local_freetype=True in setup.cfg and rebuild.")
+
     try:
         import nose
         try:

setup.cfg.template

@@ -8,6 +8,13 @@
 # This can be a single directory or a comma-delimited list of directories.
 #basedirlist = /usr
 
+[test]
+# If you plan to develop matplotlib and run or add to the test suite,
+# set this to True.  It will download and build a specific version of
+# freetype, and then use that to build the ft2font extension.  This
+# ensures that test images are exactly reproducible.
+#local_freetype = False
+
 [status]
 # To suppress display of the dependencies and their versions
 # at the top of the build log, uncomment the following line:

setup.py

@@ -9,6 +9,7 @@
 from distribute_setup import use_setuptools
 use_setuptools()
 from setuptools.command.test import test as TestCommand
+from setuptools.command.build_ext import build_ext as BuildExtCommand
 
 import sys
 
@@ -239,8 +240,19 @@ def run_tests(self):
                   argv=['nosetests'] + self.test_args,
                   exit=True)
 
+
+class BuildExtraLibraries(BuildExtCommand):
+    def run(self):
+        for package in good_packages:
+            package.do_custom_build()
+
+        return super(BuildExtraLibraries, self).run()
+
+
 cmdclass = versioneer.get_cmdclass()
 cmdclass['test'] = NoseTestCommand
+cmdclass['build_ext'] = BuildExtraLibraries
+
 
 # One doesn't normally see `if __name__ == '__main__'` blocks in a setup.py,
 # however, this is needed on Windows to avoid creating infinite subprocesses
@@ -303,8 +315,6 @@ def run_tests(self):
     # Now collect all of the information we need to build all of the
     # packages.
     for package in good_packages:
-        if isinstance(package, str):
-            continue
         packages.extend(package.get_packages())
         namespace_packages.extend(package.get_namespace_packages())
         py_modules.extend(package.get_py_modules())

setupext.py

@@ -20,6 +20,12 @@
 PY3 = (sys.version_info[0] >= 3)
 
 
+# This is the version of freetype to use when building a local version
+# of freetype.  It must match the value in
+# lib/matplotlib.__init__.py:validate_test_dependencies
+LOCAL_FREETYPE_VERSION = '2.5.2'
+
+
 if sys.platform != 'win32':
     if sys.version_info[0] < 3:
         from commands import getstatusoutput
@@ -47,22 +53,19 @@
     config = configparser.SafeConfigParser()
     config.read(setup_cfg)
 
-    try:
+    if config.has_option('status', 'suppress'):
         options['display_status'] = not config.getboolean("status", "suppress")
-    except:
-        pass
 
-    try:
+    if config.has_option('rc_options', 'backend'):
         options['backend'] = config.get("rc_options", "backend")
-    except:
-        pass
 
-    try:
+    if config.has_option('directories', 'basedirlist'):
         options['basedirlist'] = [
             x.strip() for x in
             config.get("directories", "basedirlist").split(',')]
-    except:
-        pass
+
+    if config.has_option('test', 'local_freetype'):
+        options['local_freetype'] = config.get("test", "local_freetype")
 else:
     config = None
 
@@ -448,6 +451,14 @@ def _check_for_pkg_config(self, package, include_file, min_version=None,
 
         return 'version %s' % version
 
+    def do_custom_build(self):
+        """
+        If a package needs to do extra custom things, such as building a
+        third-party library, before building an extension, it should
+        override this method.
+        """
+        pass
+
 
 class OptionalPackage(SetupPackage):
     optional = True
@@ -461,10 +472,9 @@ def get_config(cls):
         if the package is at default state ("auto"), forced by the user (True)
         or opted-out (False).
         """
-        try:
-            return config.getboolean(cls.config_category, cls.name)
-        except:
-            return "auto"
+        if config is not None and config.has_option(cls.config_category, cls.name):
+            return config.get(cls.config_category, cls.name)
+        return "auto"
 
     def check(self):
         """
@@ -872,6 +882,9 @@ class FreeType(SetupPackage):
     name = "freetype"
 
     def check(self):
+        if options.get('local_freetype'):
+            return "Using local version for testing"
+
         if sys.platform == 'win32':
             check_include_file(get_include_dirs(), 'ft2build.h', 'freetype')
             return 'Using unknown version found on system.'
@@ -917,15 +930,60 @@ def version_from_header(self):
                 return '.'.join([major, minor, patch])
 
     def add_flags(self, ext):
-        pkg_config.setup_extension(
-            ext, 'freetype2',
-            default_include_dirs=[
-                'include/freetype2', 'freetype2',
-                'lib/freetype2/include',
-                'lib/freetype2/include/freetype2'],
-            default_library_dirs=[
-                'freetype2/lib'],
-            default_libraries=['freetype', 'z'])
+        if options.get('local_freetype'):
+            src_path = os.path.join(
+                'build', 'freetype-{0}'.format(LOCAL_FREETYPE_VERSION))
+            # Statically link to the locally-built freetype.
+            # This is certainly broken on Windows.
+            ext.include_dirs.insert(0, os.path.join(src_path, 'include'))
+            ext.extra_objects.insert(
+                0, os.path.join(src_path, 'objs', '.libs', 'libfreetype.a'))
+            ext.define_macros.append(('FREETYPE_BUILD_TYPE', 'local'))
+        else:
+            pkg_config.setup_extension(
+                ext, 'freetype2',
+                default_include_dirs=[
+                    'include/freetype2', 'freetype2',
+                    'lib/freetype2/include',
+                    'lib/freetype2/include/freetype2'],
+                default_library_dirs=[
+                    'freetype2/lib'],
+                default_libraries=['freetype', 'z'])
+            ext.define_macros.append(('FREETYPE_BUILD_TYPE', 'system'))
+
+    def do_custom_build(self):
+        # We're using a system freetype
+        if not options.get('local_freetype'):
+            return
+
+        src_path = os.path.join(
+            'build', 'freetype-{0}'.format(LOCAL_FREETYPE_VERSION))
+
+        # We've already built freetype
+        if os.path.isfile(os.path.join(src_path, 'objs', '.libs', 'libfreetype.a')):
+            return
+
+        tarball = 'freetype-{0}.tar.gz'.format(LOCAL_FREETYPE_VERSION)
+        tarball_path = os.path.join('build', tarball)
+        if not os.path.isfile(tarball_path):
+            print("Downloading {0}".format(tarball))
+            if sys.version_info[0] == 2:
+                from urllib import urlretrieve
+            else:
+                from urllib.request import urlretrieve
+
+            urlretrieve(
+                'http://download.savannah.gnu.org/releases/freetype/{0}'.format(tarball),
+                tarball_path)
+
+        print("Building {0}".format(tarball))
+        subprocess.check_call(
+            ['tar zxf {0}'.format(tarball)], shell=True, cwd='build')
+        subprocess.check_call(
+            ['./configure --without-zlib --without-bzip2 --without-png'],
+            shell=True, cwd=src_path)
+        subprocess.check_call(
+            ['make'], shell=True, cwd=src_path)
 
 
 class FT2Font(SetupPackage):

src/ft2font_wrapper.cpp

@@ -7,6 +7,9 @@
 // From Python
 #include <structmember.h>
 
+#define STRINGIFY(s) XSTRINGIFY(s)
+#define XSTRINGIFY(s) #s
+
 static PyObject *convert_xys_to_array(std::vector<double> &xys)
 {
     npy_intp dims[] = {(npy_intp)xys.size() / 2, 2 };
@@ -1759,7 +1762,7 @@ PyMODINIT_FUNC initft2font(void)
     int error = FT_Init_FreeType(&_ft2Library);
 
     if (error) {
-        PyErr_SetString(PyExc_RuntimeError, "Could not find initialize the freetype2 library");
+        PyErr_SetString(PyExc_RuntimeError, "Could not initialize the freetype2 library");
         INITERROR;
     }
 
@@ -1774,6 +1777,10 @@ PyMODINIT_FUNC initft2font(void)
         }
     }
 
+    if (PyModule_AddStringConstant(m, "__freetype_build_type__", STRINGIFY(FREETYPE_BUILD_TYPE))) {
+        INITERROR;
+    }
+
     import_array();
 
 #if PY3K