Finalized the new-bootstrap script

Author: hartbit

Documentation/Development.md

@@ -24,7 +24,8 @@ $ ../swift/utils/build-script -R --llbuild --swiftpm --xctest --foundation --lib
 This will build the compiler and friends in the `build/` directory. It takes about 1
 hour for the initial build process. However, it is not really required to build
 the entire compiler in order to work on the Package Manager. A faster option is
-using a [snapshot](https://swift.org/download/#releases) from swift.org.
+using a [snapshot](https://swift.org/download/#releases) from swift.org, or even faster,
+using Xcode 11.
 
 ## Using a Trunk Snapshot
 
@@ -64,7 +65,7 @@ Note: Make sure the directory for llbuild is called "llbuild" and not
 
 ```sh
 $ cd swiftpm
-$ Utilities/bootstrap
+$ Utilities/bootstrap build
 ```
 
  Note: The bootstrap script requires having [CMake](https://cmake.org/) and [Ninja](https://ninja-build.org/) installed. Please refer to the [Swift project repo](https://github.com/apple/swift/blob/master/README.md#macos) for installation instructions.
@@ -73,9 +74,6 @@ This command builds the Package Manager inside the `.build/` directory.
     Run the bootstrap script to rebuild after making a change to the source
     code.
 
-   You can also use the built binaries: `swift-build`, `swift-package`,
-    `swift-test`, `swift-run`.
-
 ### Example
 
 ```sh
@@ -87,42 +85,34 @@ $ /path/to/swiftpm/.build/x86_64-apple-macosx/debug/swift-build
 5. Test the Swift Package Manager.
 
 ```sh
-$ Utilities/bootstrap test --test-parallel
+$ Utilities/bootstrap test --parallel
 ```
 
 Use this command to run the tests. All tests must pass before a patch can be accepted.
 
 ## Self Hosting a Swift Package
 
-It is possible to build SwiftPM with itself using a special script that is
-emitted during bootstrapping. This is useful when you want to rebuild just the
+It is possible to build SwiftPM with itself using the built SwiftPM
+binaries. This is useful when you want to rebuild just the
 sources or run a single test. Make sure you run the bootstrap script first.
 
 ```sh
 $ cd swiftpm
 
 # Rebuild just the sources.
-$ .build/x86_64-apple-macosx/debug/spm build
+$ .build/x86_64-apple-macosx/debug/swift-build
 
 # Run a single test.
-$ .build/x86_64-apple-macosx/debug/spm test --filter PackageGraphTests.DependencyResolverTests/testBasics
+$ .build/x86_64-apple-macosx/debug/swift-test --filter PackageGraphTests.DependencyResolverTests/testBasics
 ```
 
-Note: If you make any changes to the `PackageDescription` runtime-related targets,
-you **will** need to rebuild using the bootstrap script.
+Note: If you make any changes to the `PackageDescription4` target, you **will** need to rebuild using the bootstrap script.
 
 ## Developing using Xcode
 
-Run the following commands to generate and open an Xcode project.
+Simply open SwiftPM's `Package.swift` manifest with the latest release of Xcode. Make sure you have run `Utilites/bootstrap` beforehand.
 
-```sh
-$ Utilities/bootstrap --generate-xcodeproj
-generated: ./SwiftPM.xcodeproj
-$ open SwiftPM.xcodeproj
-```
-
-Note: If you make any changes to the `PackageDescription` or `PackageDescription4`
-targets, you will need to regenerate the Xcode project using the above command.
+Note: If you make any changes to the `PackageDescription4` target, you will need to run `bootstrap` again.
 
 ## Using Continuous Integration
 
@@ -142,17 +132,6 @@ be used to get quick feedback.
 
 Note: Smoke tests are still required for merging pull-requests.
 
-## Running the Performance Tests
-
-Running performance tests is a little awkward right now. First, generate the
-Xcode project using this command:
-
-```sh
-$ Utilities/bootstrap --generate-xcodeproj --enable-perf-tests
-```
-
-Then, open the generated project and run the `PerformanceTest` scheme.
-
 ## Testing on Linux with Docker
 
 For contributors on macOS who need to test on Linux, install Docker and use the

Sources/TSCBasic/Process.swift

@@ -39,6 +39,9 @@ public struct ProcessResult: CustomStringConvertible {
     /// The arguments with which the process was launched.
     public let arguments: [String]
 
+    /// The environment with which the process was launched.
+    public let environment: [String: String]
+
     /// The exit status of the process.
     public let exitStatus: ExitStatus
 
@@ -55,6 +58,7 @@ public struct ProcessResult: CustomStringConvertible {
     /// See `waitpid(2)` for information on the exit status code.
     public init(
         arguments: [String],
+        environment: [String: String],
         exitStatusCode: Int32,
         output: Result<[UInt8], AnyError>,
         stderrOutput: Result<[UInt8], AnyError>
@@ -70,18 +74,20 @@ public struct ProcessResult: CustomStringConvertible {
             exitStatus = .terminated(code: WEXITSTATUS(exitStatusCode))
         }
       #endif
-        self.init(arguments: arguments, exitStatus: exitStatus, output: output,
+        self.init(arguments: arguments, environment: environment, exitStatus: exitStatus, output: output,
             stderrOutput: stderrOutput)
     }
 
     /// Create an instance using an exit status and output result.
     public init(
         arguments: [String],
+        environment: [String: String],
         exitStatus: ExitStatus,
         output: Result<[UInt8], AnyError>,
         stderrOutput: Result<[UInt8], AnyError>
     ) {
         self.arguments = arguments
+        self.environment = environment
         self.output = output
         self.stderrOutput = stderrOutput
         self.exitStatus = exitStatus
@@ -474,6 +480,7 @@ public final class Process: ObjectIdentifierProtocol {
             // Construct the result.
             let executionResult = ProcessResult(
                 arguments: arguments,
+                environment: environment,
                 exitStatusCode: exitStatusCode,
                 output: stdout.result,
                 stderrOutput: stderr.result
@@ -657,7 +664,10 @@ extension ProcessResult.Error: CustomStringConvertible {
             case .signalled(let signal):
                 stream <<< "signalled(\(signal)): "
             }
- 
+
+            stream <<< result.environment.map({ "\($0)=\($1)" }).joined(separator: " ")
+            stream <<< " "
+
             // Strip sandbox information from arguments to keep things pretty.
             var args = result.arguments
             // This seems a little fragile.

Sources/Workspace/UserToolchain.swift

@@ -234,12 +234,18 @@ public final class UserToolchain: Toolchain {
         if let pdLibDirEnvStr = ProcessEnv.vars["SWIFTPM_PD_LIBS"] {
             // We pick the first path which exists in a colon seperated list.
             let paths = pdLibDirEnvStr.split(separator: ":").map(String.init)
+            var foundPDLibDir = false
             for pathString in paths {
                 if let path = try? AbsolutePath(validating: pathString), localFileSystem.exists(path) {
                     pdLibDir = path
+                    foundPDLibDir = true
                     break
                 }
             }
+
+            if !foundPDLibDir {
+                fatalError("Couldn't find any SWIFTPM_PD_LIBS directory: \(pdLibDirEnvStr)")
+            }
         }
 
         manifestResources = UserManifestResources(

Tests/WorkspaceTests/WorkspaceTests.swift

@@ -147,7 +147,7 @@ final class WorkspaceTests: XCTestCase {
                         """
                 }
 
-                XCTAssertMatch((ws.interpreterFlags(for: foo)), [.contains("swift/pm/4")])
+                XCTAssertMatch((ws.interpreterFlags(for: foo)), [.contains("pm/4")])
                 XCTAssertMatch((ws.interpreterFlags(for: foo)), [.equal("-swift-version"), .equal("4")])
             }
 

Utilities/Docker/Dockerfile

@@ -1,44 +1,54 @@
 # This source file is part of the Swift.org open source project
 #
-# Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
+# Copyright (c) 2014 - 2019 Apple Inc. and the Swift project authors
 # Licensed under Apache License v2.0 with Runtime Library Exception
 #
 # See http://swift.org/LICENSE.txt for license information
 # See http://swift.org/CONTRIBUTORS.txt for Swift project authors
 
-FROM ubuntu:16.04
+FROM ubuntu:18.04
+
 # Install related packages.
-RUN apt-get -q update && \
+RUN export DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true && \
+    apt-get -q update && \
     apt-get -q install -y \
-    build-essential \
-    make \
-    libc6-dev \
-    clang-3.6 \
-    curl \
-    libedit-dev \
-    python2.7 \
-    python2.7-dev \
-    libicu-dev \
-    rsync \
-    libxml2 \
-    git \
-    libcurl4-openssl-dev \
-    vim \
-    libblocksruntime-dev \
-    cmake \
-    ninja-build \
-    sqlite3 \
-    libsqlite3-dev \
-    libncurses5-dev \
-    && update-alternatives --quiet --install /usr/bin/clang clang /usr/bin/clang-3.6 100 \
-    && update-alternatives --quiet --install /usr/bin/clang++ clang++ /usr/bin/clang++-3.6 100 \
-    && rm -r /var/lib/apt/lists/*
-ARG SNAPSHOT
+        make \
+        libc6-dev \
+        clang-3.9 \
+        curl \
+        libedit-dev \
+        libpython2.7 \
+        libicu-dev \
+        libssl-dev \
+        libxml2 \
+        tzdata \
+        git \
+        libcurl4-openssl-dev \
+        ninja-build \
+        sqlite3 \
+        libsqlite3-dev \
+        libncurses5-dev \
+        pkg-config \
+        apt-transport-https \
+        ca-certificates \
+        gnupg \
+        wget \
+        software-properties-common && \
+    wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | apt-key add - && \
+    apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main' && \
+    apt-get -q update && \
+    apt-get -q install -y cmake && \
+    update-alternatives --quiet --install /usr/bin/clang clang /usr/bin/clang-3.9 100 && \
+    update-alternatives --quiet --install /usr/bin/clang++ clang++ /usr/bin/clang++-3.9 100 && \
+    rm -r /var/lib/apt/lists/*
 
+ARG SNAPSHOT
 COPY $SNAPSHOT /
 RUN tar -xvzf $SNAPSHOT --directory / --strip-components=1 && \
     rm -rf swift-DEVELOPMENT-SNAPSHOT*
+
 # Set Swift Path
 ENV PATH /usr/bin:$PATH
+
 # Print Installed Swift Version
 RUN swift --version

Utilities/Docker/docker-utils

@@ -24,7 +24,7 @@ script_dir = os.path.dirname(os.path.realpath(__file__))
 
 def call(args):
     """Prints and executes a command."""
-    print " ".join(args)
+    print(" ".join(args))
     return subprocess.call(args)
 
 def docker_run(command_args):
@@ -40,24 +40,24 @@ def docker_run(command_args):
         "-v", "%s:/llbuild" % llbuild_dir,
         "-w", "/swiftpm",
         "--rm",
-        "swiftpm-docker-1604"
+        "swiftpm-docker-1804"
     ] + command_args)
 
 def build(_):
     """Builds a docker image with the latest snapshot."""
 
     # Get latest snapshot info.
-    latest_build_url = "https://swift.org/builds/development/ubuntu1604/latest-build.yml"
+    latest_build_url = "https://swift.org/builds/development/ubuntu1804/latest-build.yml"
     latest_build_data = yaml.load(urllib2.urlopen(latest_build_url).read())
     snapshot_filename = latest_build_data["download"]
     # FIXME: We shouldn"t need to do this, it should be available in the API.
-    snapshot_name = snapshot_filename.replace("-ubuntu16.04.tar.gz", "")
+    snapshot_name = snapshot_filename.replace("-ubuntu18.04.tar.gz", "")
     base_url = latest_build_url.rsplit("/", 1)[0]
     latest_snapshot_url = base_url + "/" + snapshot_name + "/" + snapshot_filename
 
     # Download latest snapshot (if necessary).
     if snapshot_filename in os.listdir(script_dir):
-        print "Not downloading: " + latest_snapshot_url
+        print("Not downloading: %s" % latest_snapshot_url)
     else:
         # FIXME: We should remove old tarballs if we have newer ones.
         result = call([
@@ -68,14 +68,14 @@ def build(_):
         ])
 
         if result != 0:
-            print "Unable to install package: " + latest_snapshot_url
+            print("Unable to install package: %s" % latest_snapshot_url)
             exit(1)
 
     # Create docker image.
     call([
         "docker",
         "build",
-        "-t", "swiftpm-docker-1604",
+        "-t", "swiftpm-docker-1804",
         "--build-arg", "SNAPSHOT=%s" % snapshot_filename,
         script_dir
     ])
@@ -90,13 +90,13 @@ def bootstrap(args):
 
 def build_run(args):
     """Runs a built swift executable in the container."""
-    docker_run([".build/x86_64-unknown-linux-gnu/debug/" + args.command] + args.arguments)
+    docker_run([".build/x86_64-unknown-linux-gnu/debug/swift-" + args.command] + args.arguments)
 
 def main():
     """Main script entry-point."""
 
     parser = argparse.ArgumentParser(
-        usage="%(prog)s [build|run|bootstrap|spm]",
+        usage="%(prog)s [build|run|bootstrap|swift]",
         description="This script simplifies all the docker operations to build "
                     "and run a container for SwiftPM development and testing "
                     "on Linux.")
@@ -125,8 +125,8 @@ def main():
 
     # spm
     parser_swift_build = subparsers.add_parser(
-        "spm",
-        help="runs the spm helper script in a container.")
+        "swift",
+        help="runs a SwiftPM executable in a container.")
     parser_swift_build.add_argument("arguments", nargs="*")
     parser_swift_build.set_defaults(func=build_run)