Multimanifest prep

src/west/main.py

@@ -19,6 +19,7 @@
 import shutil
 import sys
 from subprocess import CalledProcessError
+import tempfile
 import textwrap
 import traceback
 
@@ -522,6 +523,15 @@ def get_extension_commands():
     return extensions
 
 
+def dump_traceback():
+    # Save the current exception to a file and return its path.
+    fd, name = tempfile.mkstemp(prefix='west-exc-', suffix='.txt')
+    os.close(fd)        # traceback has no use for the fd
+    with open(name, 'w') as f:
+        traceback.print_exc(file=f)
+    return name
+
+
 def main(argv=None):
     # Makes ANSI color escapes work on Windows, and strips them when
     # stdout/stderr isn't a terminal
@@ -550,34 +560,33 @@ def main(argv=None):
         argv = sys.argv[1:]
     args, unknown = parse_args(argv, extensions, topdir)
 
-    for_stack_trace = 'run as "west -v {}" for a stack trace'.format(
-        quote_sh_list(argv))
     try:
         args.handler(args, unknown)
     except KeyboardInterrupt:
         sys.exit(0)
     except CalledProcessError as cpe:
-        log.err('command exited with status {}: {}'.format(
-            cpe.args[0], quote_sh_list(cpe.args[1])))
+        log.err('command exited with status {}: {}'.
+                format(cpe.returncode, quote_sh_list(cpe.cmd)))
         if args.verbose:
             traceback.print_exc()
-        else:
-            log.inf(for_stack_trace)
         sys.exit(cpe.returncode)
     except ExtensionCommandError as ece:
-        log.err('extension command', args.command,
-                'was improperly defined and could not be run{}'.
-                format(': ' + ece.hint if ece.hint else ''))
+        msg = 'extension command "{}" could not be run{}.'.format(
+            args.command, ': ' + ece.hint if ece.hint else '')
         if args.verbose:
+            log.err(msg)
             traceback.print_exc()
         else:
-            log.inf(for_stack_trace)
+            log.err(msg, 'See {} for a traceback.'.format(dump_traceback()))
         sys.exit(ece.returncode)
     except CommandContextError as cce:
         log.err('command', args.command, 'cannot be run in this context:',
                 *cce.args)
+        log.err('see {} for a traceback.'.format(dump_traceback()))
         sys.exit(cce.returncode)
     except CommandError as ce:
+        # No need to dump_traceback() here. The command is responsible
+        # for logging its own errors.
         sys.exit(ce.returncode)
 
 

tests/west/commands/conftest.py → tests/conftest.py

@@ -1,5 +1,4 @@
 import os
-from os.path import dirname
 import shlex
 import shutil
 import subprocess
@@ -10,101 +9,52 @@
 import pytest
 
 GIT = shutil.which('git')
-# Assumes this file is west/tests/west/commands/conftest.py, returns
-# path to toplevel 'west'
-THIS_WEST = os.path.abspath(dirname(dirname(dirname(dirname(__file__)))))
+MANIFEST_TEMPLATE = '''\
+manifest:
+  defaults:
+    remote: test-local
+
+  remotes:
+    - name: test-local
+      url-base: THE_URL_BASE
+
+  projects:
+    - name: Kconfiglib
+      revision: zephyr
+      path: subdir/Kconfiglib
+    - name: net-tools
+      west-commands: scripts/west-commands.yml
+  self:
+    path: zephyr
+'''
 
 #
 # Test fixtures
 #
 
-@pytest.fixture
-def repos_tmpdir(tmpdir):
-    '''Fixture for tmpdir with "remote" repositories, manifest, and west.
-
-    These can then be used to bootstrap an installation and run
-    project-related commands on it with predictable results.
-
-    Switches directory to, and returns, the top level tmpdir -- NOT
-    the subdirectory containing the repositories themselves.
-
-    Initializes placeholder upstream repositories in <tmpdir>/remote-repos/
-    with the following contents:
-
-    repos/
-    ├── west (branch: master)
-    │   └── (contains this west's worktree contents)
-    ├── manifest (branch: master)
-    │   └── west.yml
-    ├── Kconfiglib (branch: zephyr)
-    │   └── kconfiglib.py
-    ├── net-tools (branch: master)
-    │   └── qemu-script.sh
-    └── zephyr (branch: master)
-        ├── CODEOWNERS
-        ├── west.yml
-        ├── include
-        │   └── header.h
-        └── subsys
-            └── bluetooth
-                └── code.c
-
-    The contents of west.yml are:
-
-    west:
-      url: file://<tmpdir>/west
-    manifest:
-      defaults:
-        remote: test-local
-      remotes:
-        - name: test-local
-          url-base: file://<tmpdir>/remote-repos
-      projects:
-        - name: Kconfiglib
-          revision: zephyr
-          path: subdir/Kconfiglib
-        - name: net-tools
-          clone_depth: 1
-          west-commands: scripts/west-commands.yml
-      self:
-        path: zephyr
-    '''
-    rr = tmpdir.mkdir('repos')  # "remote" repositories
-    rp = {}                     # individual repository paths under rr
+@pytest.fixture(scope='session')
+def _session_repos():
+    '''Just a helper, do not use directly.'''
 
-    # Mirror this west tree into a "remote" west repository under rr.
-    wdst = rr.join('west')
-    mirror_west_repo(wdst)
-    rp['west'] = str(wdst)
+    # It saves time to create repositories once at session scope, then
+    # clone the results as needed in per-test fixtures.
+    session_repos = os.path.join(os.environ['TOXTEMPDIR'], 'session_repos')
+    print('initializing session repositories in', session_repos)
+    shutil.rmtree(session_repos, ignore_errors=True)
 
-    # Create the other repositories.
+    # Create the repositories.
+    rp = {}      # individual repository paths
     for repo in 'net-tools', 'Kconfiglib', 'zephyr':
-        path = str(rr.join(repo))
+        path = os.path.join(session_repos, repo)
         rp[repo] = path
         create_repo(path)
 
-    # Initialize the manifest repository.
-    add_commit(rp['zephyr'], 'test manifest',
-               files={'west.yml': textwrap.dedent('''\
-                      west:
-                        url: file://{west}
-                      manifest:
-                        defaults:
-                          remote: test-local
-
-                        remotes:
-                          - name: test-local
-                            url-base: file://{rr}
-
-                        projects:
-                          - name: Kconfiglib
-                            revision: zephyr
-                            path: subdir/Kconfiglib
-                          - name: net-tools
-                            west-commands: scripts/west-commands.yml
-                        self:
-                          path: zephyr
-                      '''.format(west=rp['west'], rr=str(rr)))})
+    # Initialize the "zephyr" repository.
+    # The caller needs to add west.yml with the right url-base.
+    add_commit(rp['zephyr'], 'base zephyr commit',
+               files={'CODEOWNERS': '',
+                      'include/header.h': '#pragma once\n',
+                      'subsys/bluetooth/code.c': 'void foo(void) {}\n'})
 
     # Initialize the Kconfiglib repository.
     subprocess.check_call([GIT, 'checkout', '-b', 'zephyr'],
@@ -139,29 +89,77 @@ def do_run(self, args, ignored):
                       '''),
                       })
 
-    # Initialize the zephyr repository.
-    add_commit(rp['zephyr'], 'test zephyr commit',
-               files={'CODEOWNERS': '',
-                      'include/header.h': '#pragma once\n',
-                      'subsys/bluetooth/code.c': 'void foo(void) {}\n'})
+    # Return the top-level temporary directory. Don't clean it up on
+    # teardown, so the contents can be inspected post-portem.
+    print('finished initializing session repositories')
+    return session_repos
 
-    # Switch to and return the top-level temporary directory.
-    #
-    # This can be used to populate a west installation alongside.
+@pytest.fixture
+def repos_tmpdir(tmpdir, _session_repos):
+    '''Fixture for tmpdir with "remote" repositories, manifest, and west.
 
-    # Switch to the top-level West installation directory
-    tmpdir.chdir()
-    return tmpdir
+    These can then be used to bootstrap an installation and run
+    project-related commands on it with predictable results.
+
+    Switches directory to, and returns, the top level tmpdir -- NOT
+    the subdirectory containing the repositories themselves.
+
+    Initializes placeholder upstream repositories in tmpdir with the
+    following contents:
+
+    repos/
+    ├── Kconfiglib (branch: zephyr)
+    │   └── kconfiglib.py
+    ├── net-tools (branch: master)
+    │   └── qemu-script.sh
+    └── zephyr (branch: master)
+        ├── CODEOWNERS
+        ├── west.yml
+        ├── include
+        │   └── header.h
+        └── subsys
+            └── bluetooth
+                └── code.c
 
+    The contents of west.yml are:
+
+    manifest:
+      defaults:
+        remote: test-local
+      remotes:
+        - name: test-local
+          url-base: <tmpdir>/repos
+      projects:
+        - name: Kconfiglib
+          revision: zephyr
+          path: subdir/Kconfiglib
+        - name: net-tools
+          clone_depth: 1
+          west-commands: scripts/west-commands.yml
+      self:
+        path: zephyr
+
+    '''
+    kconfiglib, net_tools, zephyr = [os.path.join(_session_repos, x) for x in
+                                     ['Kconfiglib', 'net-tools', 'zephyr']]
+    repos = tmpdir.mkdir('repos')
+    repos.chdir()
+    for r in [kconfiglib, net_tools, zephyr]:
+        subprocess.check_call([GIT, 'clone', r])
+
+    manifest = MANIFEST_TEMPLATE.replace('THE_URL_BASE',
+                                         str(tmpdir.join('repos')))
+    add_commit(str(repos.join('zephyr')), 'add manifest',
+               files={'west.yml': manifest})
+    return tmpdir
 
 @pytest.fixture
 def west_init_tmpdir(repos_tmpdir):
     '''Fixture for a tmpdir with 'remote' repositories and 'west init' run.
 
     Uses the remote repositories from the repos_tmpdir fixture to
     create a west installation using the system bootstrapper's init
-    command -- and thus the test environment must install the
-    bootstrapper from the current west source code tree under test.
+    command.
 
     The contents of the west installation aren't checked at all.
     This is left up to the test cases.
@@ -194,52 +192,6 @@ def check_output(*args, **kwargs):
         raise
     return out_bytes.decode(sys.getdefaultencoding())
 
-
-def mirror_west_repo(dst):
-    # Create a west repository in dst which mirrors the exact state of
-    # the current tree, except ignored files.
-    #
-    # This is done in a simple way:
-    #
-    # 1. recursively copy THIS_WEST there (except .git and ignored files)
-    # 2. init a new git repository there
-    # 3. add the entire tree, and commit
-    #
-    # (We can't just clone THIS_WEST because we want to allow
-    # developers to test their working trees without having to make a
-    # commit -- remember, 'west init' clones the remote.)
-    wut = str(dst)  # "west under test"
-
-    # Copy the west working tree, except ignored files.
-    def ignore(directory, files):
-        # Get newline separated list of ignored files, as a string.
-        try:
-            ignored = check_output([GIT, 'check-ignore'] + files,
-                                   cwd=directory)
-        except subprocess.CalledProcessError as e:
-            # From the manpage: return values 0 and 1 respectively
-            # mean that some and no argument files were ignored. These
-            # are both OK. Treat other return values as errors.
-            if e.returncode not in (0, 1):
-                raise
-            else:
-                ignored = e.output.decode(sys.getdefaultencoding())
-
-        # Convert ignored to a set of file names as strings.
-        ignored = set(ignored.splitlines())
-
-        # Also ignore the .git directory itself.
-        if '.git' in files:
-            ignored.add('.git')
-
-        return ignored
-    shutil.copytree(THIS_WEST, wut, ignore=ignore)
-
-    # Create a fresh .git and commit existing directory tree.
-    create_repo(wut)
-    subprocess.check_call([GIT, 'add', '-A'], cwd=wut)
-    add_commit(wut, 'west under test')
-
 def cmd(cmd, cwd=None, stderr=None):
     # Run a west command in a directory (cwd defaults to os.getcwd()).
     #
@@ -255,8 +207,10 @@ def cmd(cmd, cwd=None, stderr=None):
     # a python subprocess so that program-level setup and teardown
     # happen fresh.
 
+    cmdlst = shlex.split('west ' + cmd)
+    print('running:', cmdlst)
     try:
-        return check_output(shlex.split('west ' + cmd), cwd=cwd, stderr=stderr)
+        return check_output(cmdlst, cwd=cwd, stderr=stderr)
     except subprocess.CalledProcessError:
         print('cmd: west:', shutil.which('west'), file=sys.stderr)
         raise

tests/west/manifest/test_manifest.py → tests/test_manifest.py

@@ -267,7 +267,8 @@ def test_west_is_ok():
 
 # Invalid manifests should raise MalformedManifest.
 @pytest.mark.parametrize('invalid',
-                         glob(os.path.join(THIS_DIRECTORY, 'invalid_*.yml')))
+                         glob(os.path.join(THIS_DIRECTORY, 'manifests',
+                                           'invalid_*.yml')))
 @patch('west.util.west_topdir', return_value='/west_top')
 def test_invalid(topdir, invalid):
     with pytest.raises(MalformedManifest):