Multimanifest prep

src/west/main.py

@@ -19,6 +19,7 @@
 import shutil
 import sys
 from subprocess import CalledProcessError
+import tempfile
 import textwrap
 import traceback
 
@@ -522,6 +523,15 @@ def get_extension_commands():
     return extensions
 
 
+def dump_traceback():
+    # Save the current exception to a file and return its path.
+    fd, name = tempfile.mkstemp(prefix='west-exc-', suffix='.txt')
+    os.close(fd)        # traceback has no use for the fd
+    with open(name, 'w') as f:
+        traceback.print_exc(file=f)
+    return name
+
+
 def main(argv=None):
     # Makes ANSI color escapes work on Windows, and strips them when
     # stdout/stderr isn't a terminal
@@ -550,34 +560,33 @@ def main(argv=None):
         argv = sys.argv[1:]
     args, unknown = parse_args(argv, extensions, topdir)
 
-    for_stack_trace = 'run as "west -v {}" for a stack trace'.format(
-        quote_sh_list(argv))
     try:
         args.handler(args, unknown)
     except KeyboardInterrupt:
         sys.exit(0)
     except CalledProcessError as cpe:
-        log.err('command exited with status {}: {}'.format(
-            cpe.args[0], quote_sh_list(cpe.args[1])))
+        log.err('command exited with status {}: {}'.
+                format(cpe.returncode, quote_sh_list(cpe.cmd)))
         if args.verbose:
             traceback.print_exc()
-        else:
-            log.inf(for_stack_trace)
         sys.exit(cpe.returncode)
     except ExtensionCommandError as ece:
-        log.err('extension command', args.command,
-                'was improperly defined and could not be run{}'.
-                format(': ' + ece.hint if ece.hint else ''))
+        msg = 'extension command "{}" could not be run{}.'.format(
+            args.command, ': ' + ece.hint if ece.hint else '')
         if args.verbose:
+            log.err(msg)
             traceback.print_exc()
         else:
-            log.inf(for_stack_trace)
+            log.err(msg, 'See {} for a traceback.'.format(dump_traceback()))
         sys.exit(ece.returncode)
     except CommandContextError as cce:
         log.err('command', args.command, 'cannot be run in this context:',
                 *cce.args)
+        log.err('see {} for a traceback.'.format(dump_traceback()))
         sys.exit(cce.returncode)
     except CommandError as ce:
+        # No need to dump_traceback() here. The command is responsible
+        # for logging its own errors.
         sys.exit(ce.returncode)
 
 

src/west/manifest-schema.yml

@@ -36,7 +36,7 @@ mapping:
   # and a fetch URL base. These URL bases are concatenated with project names
   # (separated by a /) to form complete fetch URLs.
   remotes:
-    required: true
+    required: false
     type: seq
     sequence:
       - type: map
@@ -64,6 +64,10 @@ mapping:
           remote:
             required: false
             type: str
+          # Explicit project URL (may not be given with a remote as well)
+          url:
+            required: false
+            type: str
           # Revision to check out. May be a branch, tag, or SHA.
           revision:
             required: false

src/west/manifest.py

@@ -144,7 +144,7 @@ def __init__(self, source_file=None, source_data=None):
 
         self.remotes = None
         '''Sequence of west.manifest.Remote objects representing manifest
-        remotes.'''
+        remotes. Note that not all projects have a remote.'''
 
         self.projects = None
         '''Sequence of west.manifest.Project objects representing manifest
@@ -210,6 +210,7 @@ def _load(self, data):
         # Initialize this instance's fields from values given in the
         # manifest data, which must be validated according to the schema.
         projects = []
+        project_names = set()
         project_abspaths = set()
 
         manifest = data.get('manifest')
@@ -226,7 +227,7 @@ def _load(self, data):
 
         # Map from each remote's name onto that remote's data in the manifest.
         remotes = tuple(Remote(r['name'], r['url-base']) for r in
-                        manifest['remotes'])
+                        manifest.get('remotes', []))
         remotes_dict = {r.name: r for r in remotes}
 
         # Get any defaults out of the manifest.
@@ -254,24 +255,37 @@ def _load(self, data):
             # Validate the project name.
             name = mp['name']
 
-            # Validate the project remote.
+            # Validate the project remote or URL.
             remote_name = mp.get('remote', default_remote_name)
-            if remote_name is None:
-                self._malformed('project {} does not specify a remote'.
+            url = mp.get('url')
+            if remote_name is None and url is None:
+                self._malformed('project {} does not specify a remote or URL'.
                                 format(name))
-            if remote_name not in remotes_dict:
-                self._malformed('project {} remote {} is not defined'.
-                                format(name, remote_name))
+            if remote_name:
+                if remote_name not in remotes_dict:
+                    self._malformed('project {} remote {} is not defined'.
+                                    format(name, remote_name))
+                remote = remotes_dict[remote_name]
+            else:
+                remote = None
 
             # Create the project instance for final checking.
-            project = Project(name,
-                              remotes_dict[remote_name],
-                              defaults,
-                              path=mp.get('path'),
-                              clone_depth=mp.get('clone-depth'),
-                              revision=mp.get('revision'),
-                              west_commands=mp.get('west-commands'))
-
+            try:
+                project = Project(name,
+                                  defaults,
+                                  path=mp.get('path'),
+                                  clone_depth=mp.get('clone-depth'),
+                                  revision=mp.get('revision'),
+                                  west_commands=mp.get('west-commands'),
+                                  remote=remote,
+                                  url=url)
+            except ValueError as ve:
+                self._malformed(ve.args[0])
+
+            # Project names must be unique.
+            if project.name in project_names:
+                self._malformed('project name {} is already used'.
+                                format(project.name))
             # Two projects cannot have the same path. We use absolute
             # paths to check for collisions to ensure paths are
             # normalized (e.g. for case-insensitive file systems or
@@ -281,6 +295,7 @@ def _load(self, data):
                 self._malformed('project {} path {} is already in use'.
                                 format(project.name, project.path))
 
+            project_names.add(project.name)
             project_abspaths.add(project.abspath)
             projects.append(project)
 
@@ -317,8 +332,8 @@ def __init__(self, remote=None, revision=None):
                        or None (an actual Remote object, not the name of
                        a remote as a string).
         :param revision: Default Git revision; 'master' if not given.'''
-        if remote is not None:
-            _wrn_if_not_remote(remote)
+        if remote is not None and not isinstance(remote, Remote):
+            raise ValueError('{} is not a Remote'.format(remote))
         if revision is None:
             revision = 'master'
 
@@ -389,16 +404,14 @@ class Project:
     __slots__ = ('name remote url path abspath posixpath clone_depth '
                  'revision west_commands').split()
 
-    def __init__(self, name, remote, defaults, path=None, clone_depth=None,
-                 revision=None, west_commands=None):
+    def __init__(self, name, defaults=None, path=None, clone_depth=None,
+                 revision=None, west_commands=None, remote=None, url=None):
         '''Specify a Project by name, Remote, and optional information.
 
         :param name: Project's user-defined name in the manifest.
-        :param remote: Remote instance corresponding to this Project as
-                       specified in the manifest. This is used to build
-                       the project's URL, and is also stored as an attribute.
         :param defaults: If the revision parameter is not given, the project's
-                         revision is set to defaults.revision.
+                         revision is set to defaults.revision if defaults is
+                         not None, or the west-wide default otherwise.
         :param path: Relative path to the project in the west
                      installation, if present in the manifest. If not given,
                      the project's ``name`` is used.
@@ -409,15 +422,25 @@ def __init__(self, name, remote, defaults, path=None, clone_depth=None,
         :param west_commands: path to a YAML file in the project containing
                               a description of west extension commands provided
                               by the project, if given.
+        :param remote: Remote instance corresponding to this Project as
+                       specified in the manifest. This is used to build
+                       the project's URL, and is also stored as an attribute.
+        :param url: The project's fetch URL. This cannot be given with `remote`
+                    as well: choose one.
         '''
-        _wrn_if_not_remote(remote)
+        if remote and url:
+            raise ValueError('got remote={} and url={}'.format(remote, url))
+        if not (remote or url):
+            raise ValueError('got neither a remote nor a URL')
+
+        if defaults is None:
+            defaults = _DEFAULTS
 
         self.name = name
         '''Project name as it appears in the manifest.'''
-        self.remote = remote
-        '''`Remote` instance corresponding to the project's remote.'''
-        self.url = remote.url_base + '/' + name
-        '''Complete fetch URL for the project.'''
+        self.url = url or (remote.url_base + '/' + name)
+        '''Complete fetch URL for the project, either as given by the url kwarg
+        or computed from the remote URL base and the project name.'''
         self.path = os.path.normpath(path or name)
         '''Relative path to the project in the installation.'''
         self.abspath = os.path.realpath(os.path.join(util.west_topdir(),
@@ -432,6 +455,8 @@ def __init__(self, name, remote, defaults, path=None, clone_depth=None,
         from manifest defaults, or from the default supplied by west.'''
         self.west_commands = west_commands
         '''Path to project's "west-commands", or None.'''
+        self.remote = remote
+        '''`Remote` instance corresponding to the project's remote, or None.'''
 
     def __eq__(self, other):
         return NotImplemented
@@ -650,9 +675,6 @@ def as_dict(self):
             ret['west-commands'] = self.west_commands
         return ret
 
-def _wrn_if_not_remote(remote):
-    if not isinstance(remote, Remote):
-        log.wrn('Remote', remote, 'is not a Remote instance')
-
 
 _SCHEMA_PATH = os.path.join(os.path.dirname(__file__), "manifest-schema.yml")
+_DEFAULTS = Defaults()