Add schema version

changelog.txt

@@ -23,19 +23,20 @@ Next
 Added
 +++++
 
+ - Official support for Python 3.8 (#258).
  - Add properties ``Project.id`` and ``Job.id`` (#250).
- - Add function to initialize a sample data space for testing purposes.
- - Official support for Python 3.8.
+ - Add function to initialize a sample data space for testing purposes (#215).
+ - Add schema version to ensure compatibility and enable migrations in future package versions (#165, #253).
 
 Changed
 +++++++
 
- - Implemented ``Project.__contains__`` check in constant time.
+ - Implemented ``Project.__contains__`` check in constant time (#231).
 
 Fixed
 +++++
 
- - Attempting to create a linked view for a Project on Windows now raises an informative error message.
+ - Attempting to create a linked view for a Project on Windows now raises an informative error message (#214, #236).
  - Project configuration is initialized using ConfigObj, allowing the configuration to include commas and special characters (#251, #252).
 
 Deprecated
@@ -47,7 +48,7 @@ Deprecated
 Removed
 +++++++
 
- - Dropped support for Python 2.7.
+ - Dropped support for Python 2.7 (#232).
 
 
 [1.2.0] -- 2019-07-22

requirements.txt

@@ -1 +1,3 @@
 deprecation>=2
+filelock~=3.0
+packaging>=15.0

setup.py

@@ -5,6 +5,10 @@
 requirements = [
     # Deprecation management
     'deprecation>=2',
+    # Platform-independent file locking
+    'filelock~=3.0',
+    # Used for version parsing and comparison
+    'packaging>=15.0',
 ]
 
 description = "Simple file data management database."

signac/__main__.py

@@ -672,6 +672,31 @@ def main_update_cache(args):
         _print_err("Updated cache (size={}).".format(n))
 
 
+# UNCOMMENT THE FOLLOWING BLOCK WHEN THE FIRST MIGRATION IS INTRODUCED.
+# def main_migrate(args):
+#     "Migrate the project's schema to the current schema version."
+#     from .contrib.migration import apply_migrations
+#     project = get_project(_ignore_schema_version=True)
+#
+#     schema_version = version.parse(SCHEMA_VERSION)
+#     config_schema_version = version.parse(project.config['schema_version'])
+#
+#     if config_schema_version > schema_version:
+#         _print_err(
+#             "The schema version of the project ({}) is newer than the schema "
+#             "version supported by signac version {}: {}. Try updating signac.".format(
+#                 config_schema_version, __version__, schema_version))
+#     elif config_schema_version == schema_version:
+#         _print_err(
+#             "The schema version of the project ({}) is up to date. "
+#             "Nothing to do.".format(config_schema_version))
+#     elif args.yes or query_yes_no(
+#         "Do you want to migrate this project's schema version from '{}' to '{}'? "
+#         "WARNING: THIS PROCESS IS IRREVERSIBLE!".format(
+#             config_schema_version, schema_version), 'no'):
+#         apply_migrations(project)
+#
+#
 def verify_config(cfg, preserve_errors=True):
     verification = cfg.verify(
         preserve_errors=preserve_errors, skip_missing=True)
@@ -1716,6 +1741,13 @@ def main():
     parser_verify = config_subparsers.add_parser('verify')
     parser_verify.set_defaults(func=main_config_verify)
 
+# UNCOMMENT THE FOLLOWING BLOCK WHEN THE FIRST MIGRATION IS INTRODUCED.
+#     parser_migrate = subparsers.add_parser(
+#         'migrate',
+#         description="Irreversibly migrate this project's schema version to the "
+#                     "supported version.")
+#     parser_migrate.set_defaults(func=main_migrate)
+#
     # This is a hack, as argparse itself does not
     # allow to parse only --version without any
     # of the other required arguments.

signac/common/validate.py

@@ -10,16 +10,6 @@
 logger = logging.getLogger(__name__)
 
 
-def version(value, *args, **kwargs):
-    try:
-        if isinstance(value, str):
-            return tuple((int(v) for v in value.split(',')))
-        else:
-            return tuple((int(v) for v in value))
-    except Exception:
-        raise VdtValueError(value)
-
-
 def mongodb_uri(value, *args, **kwargs):
     if isinstance(value, list):
         value = ','.join(value)
@@ -43,7 +33,6 @@ def password(value, *args, **kwargs):
 
 def get_validator():
     return Validator({
-        'version': version,
         'mongodb_uri': mongodb_uri,
         'password': password,
     })
@@ -52,7 +41,7 @@ def get_validator():
 cfg = """
 project = string()
 workspace_dir = string(default='workspace')
-signac_version = version(default='0,1,0')
+schema_version = string(default='1')
 
 [General]
 default_host = string(default=None)

signac/contrib/errors.py

@@ -31,3 +31,8 @@ def __init__(self, job_ids):
 class StatepointParsingError(Error, RuntimeError):
     "Indicates an error that occurred while trying to identify a state point."
     pass
+
+
+class IncompatibleSchemaVersion(Error):
+    "The project's schema version is incompatible with this version of signac."
+    pass

signac/contrib/migration/__init__.py

@@ -0,0 +1,102 @@
+# Copyright (c) 2019 The Regents of the University of Michigan
+# All rights reserved.
+# This software is licensed under the BSD 3-Clause License.
+import os
+import sys
+from packaging import version
+from contextlib import contextmanager
+
+from filelock import FileLock
+
+from ...common.config import get_config
+from ...version import __version__, SCHEMA_VERSION
+
+
+from .v0_to_v1 import migrate_v0_to_v1
+
+
+FN_MIGRATION_LOCKFILE = '.SIGNAC_PROJECT_MIGRATION_LOCK'
+
+
+MIGRATIONS = {
+    ('0', '1'):    migrate_v0_to_v1,
+}
+
+
+def _reload_project_config(project):
+    project_reloaded = project.get_project(
+        root=project._rd, search=False, _ignore_schema_version=True)
+    project._config = project_reloaded._config
+
+
+def _update_project_config(project, **kwargs):
+    "Update the project configuration."
+    for fn in ('signac.rc', '.signacrc'):
+        config = get_config(project.fn(fn))
+        if 'project' in config:
+            break
+    else:
+        raise RuntimeError("Unable to determine project configuration file.")
+    config.update(kwargs)
+    config.write()
+    _reload_project_config(project)
+
+
+@contextmanager
+def _lock_for_migration(project):
+    lock = FileLock(project.fn(FN_MIGRATION_LOCKFILE))
+    try:
+        with lock:
+            yield
+    finally:
+        try:
+            os.unlink(lock.lock_file)
+        except FileNotFoundError:
+            pass
+
+
+def _collect_migrations(project):
+    schema_version = version.parse(SCHEMA_VERSION)
+
+    def config_schema_version():
+        return version.parse(project._config['schema_version'])
+
+    if config_schema_version() > schema_version:
+        # Project config schema version is newer and therefore not supported.
+        raise RuntimeError(
+            "The signac schema version used by this project is {}, but signac {} "
+            "only supports up to schema version {}. Try updating signac.".format(
+                config_schema_version, __version__, SCHEMA_VERSION))
+
+    while config_schema_version() < schema_version:
+        for (origin, destination), migration in MIGRATIONS.items():
+            if version.parse(origin) == config_schema_version():
+                yield (origin, destination), migration
+                break
+        else:
+            raise RuntimeError(
+                "The signac schema version used by this project is {}, but signac {} "
+                "uses schema version {} and does not know how to migrate.".format(
+                    config_schema_version(), __version__, schema_version))
+
+
+def apply_migrations(project):
+    with _lock_for_migration(project):
+        for (origin, destination), migrate in _collect_migrations(project):
+            try:
+                print("Applying migration for "
+                      "version {} to {}... ".format(origin, destination), end='',
+                      file=sys.stderr)
+                migrate(project)
+            except Exception as e:
+                raise RuntimeError(
+                    "Failed to apply migration {}.".format(destination)) from e
+            else:
+                _update_project_config(project, schema_version=destination)
+                print("OK", file=sys.stderr)
+                yield origin, destination
+
+
+__all__ = [
+    'apply_migrations',
+    ]

signac/contrib/migration/v0_to_v1.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2019 The Regents of the University of Michigan
+# All rights reserved.
+# This software is licensed under the BSD 3-Clause License.
+"""Migrate from schema version 0 to version 1.
+
+This migration is a null-migration that serves as a template
+for future migrations and testing purposes.
+"""
+
+
+def migrate_v0_to_v1(project):
+    "Migrate from schema version 0 to version 1."
+    pass  # nothing to do here, serves purely as example

signac/contrib/project.py

@@ -17,8 +17,9 @@
 from itertools import groupby
 from multiprocessing.pool import ThreadPool
 from tempfile import TemporaryDirectory
+from packaging import version
 
-from ..version import __version__
+from ..version import __version__, SCHEMA_VERSION
 from .. import syncutil
 from ..core import json
 from ..core.jsondict import JSONDict
@@ -35,6 +36,7 @@
 from .errors import WorkspaceError
 from .errors import DestinationExistsError
 from .errors import JobsCorruptedError
+from .errors import IncompatibleSchemaVersion
 
 logger = logging.getLogger(__name__)
 
@@ -109,7 +111,6 @@ def __setitem__(self, key, value):
                           "will be removed in version 2.0.",
                           DeprecationWarning)
 
-            from packaging import version
             assert version.parse(__version__) < version.parse("2.0")
         return super(_ProjectConfig, self).__setitem__(key, value)
 
@@ -136,7 +137,7 @@ class Project(object):
 
     _use_pandas_for_html_repr = True  # toggle use of pandas for html repr
 
-    def __init__(self, config=None):
+    def __init__(self, config=None, _ignore_schema_version=False):
         if config is None:
             config = load_config()
         self._config = _ProjectConfig(config)
@@ -148,6 +149,10 @@ def __init__(self, config=None):
                 "Please verify that '{}' is a signac project path.".format(
                     os.path.abspath(self.config.get('project_dir', os.getcwd()))))
 
+        # Ensure that the project's data schema is supported.
+        if not _ignore_schema_version:
+            self._check_schema_compatibility()
+
         # Prepare project document
         self._fn_doc = os.path.join(self._rd, self.FN_DOCUMENT)
         self._document = None
@@ -240,6 +245,32 @@ def id(self):
         except KeyError:
             return None
 
+    def _check_schema_compatibility(self):
+        """Checks whether this project's data schema is compatible with this version.
+
+        :raises RuntimeError:
+            If the schema version is incompatible.
+        """
+        schema_version = version.parse(SCHEMA_VERSION)
+        config_schema_version = version.parse(self.config['schema_version'])
+        if config_schema_version > schema_version:
+            # Project config schema version is newer and therefore not supported.
+            raise IncompatibleSchemaVersion(
+                "The signac schema version used by this project is '{}', but signac {} "
+                "only supports up to schema version '{}'. Try updating signac.".format(
+                    config_schema_version, __version__, schema_version))
+        elif config_schema_version < schema_version:
+            raise IncompatibleSchemaVersion(
+                "The signac schema version used by this project is '{}', but signac {} "
+                "requires schema version '{}'. Please use '$ signac migrate' to "
+                "irreversibly migrate this project's schema to the supported "
+                "version.".format(
+                    config_schema_version, __version__, schema_version))
+        else:   # identical and therefore compatible
+            logger.debug(
+                "The project's schema version {} is supported.".format(
+                    config_schema_version))
+
     def min_len_unique_id(self):
         "Determine the minimum length required for an id to be unique."
         job_ids = list(self._find_job_ids())
@@ -1476,6 +1507,7 @@ def init_project(cls, name, root=None, workspace=None, make_dir=True):
             config['project'] = name
             if workspace is not None:
                 config['workspace_dir'] = workspace
+            config['schema_version'] = SCHEMA_VERSION
             config.write()
             project = cls.get_project(root=root)
             assert project.id == str(name)
@@ -1494,7 +1526,7 @@ def init_project(cls, name, root=None, workspace=None, make_dir=True):
                         name, os.path.abspath(root)))
 
     @classmethod
-    def get_project(cls, root=None, search=True):
+    def get_project(cls, root=None, search=True, **kwargs):
         """Find a project configuration and return the associated project.
 
         :param root:
@@ -1516,7 +1548,8 @@ def get_project(cls, root=None, search=True):
                 (not search and os.path.realpath(config['project_dir']) != os.path.realpath(root)):
             raise LookupError(
                 "Unable to determine project id for path '{}'.".format(os.path.abspath(root)))
-        return cls(config=config)
+
+        return cls(config=config, **kwargs)
 
     @classmethod
     def get_job(cls, root=None):
@@ -1876,7 +1909,7 @@ def init_project(name, root=None, workspace=None, make_dir=True):
     return Project.init_project(name=name, root=root, workspace=workspace, make_dir=make_dir)
 
 
-def get_project(root=None, search=True):
+def get_project(root=None, search=True, **kwargs):
     """Find a project configuration and return the associated project.
 
     :param root:
@@ -1892,7 +1925,7 @@ def get_project(root=None, search=True):
     :rtype: :py:class:`~.Project`
     :raises LookupError: If no project configuration can be found.
     """
-    return Project.get_project(root=root, search=search)
+    return Project.get_project(root=root, search=search, **kwargs)
 
 
 def get_job(root=None):

signac/version.py

@@ -5,4 +5,10 @@
 
 __version__ = "1.2.0"
 
-__all__ = ['__version__']
+SCHEMA_VERSION = "1"
+
+
+__all__ = [
+    '__version__',
+    'SCHEMA_VERSION',
+]