eden/edenapi and mononoke integration tests: add edenapi/tools to getdeps and use them in tests (#51)

Summary:
Pull Request resolved: #51

This diff extends capabilities of CargoBuilder in getdeps so that individual manifests can be build even without workspaces. Thanks to that a build for edenapi/tools can be made and its artifacts can be used in mononoke integration tests.

Reviewed By: StanislavGlebik

Differential Revision: D23574887

fbshipit-source-id: 8a974a6b5235d36a44fe082aad55cd380d84dd09

Author: lukaspiatkowski

.github/workflows/eden_scm_lib_edenapi_tools_linux.yml

@@ -0,0 +1,35 @@
+# This file was @generated by getdeps.py
+
+name: EdenSCM Lib EdenAPI Tools Linux
+
+on:
+  push:
+    branches:
+    - master
+  pull_request:
+    branches:
+    - master
+
+jobs:
+  build:
+    runs-on: ubuntu-18.04
+    steps:
+    - uses: actions/checkout@v1
+    - name: Install Rust Stable
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: stable
+        default: true
+        profile: minimal
+    - name: Install system deps
+      run: sudo python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive eden_scm_lib_edenapi_tools
+    - name: Build eden_scm_lib_edenapi_tools
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --src-dir=. eden_scm_lib_edenapi_tools  --project-install-prefix eden_scm_lib_edenapi_tools:/usr/local
+    - name: Copy artifacts
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fixup-dyn-deps --strip --src-dir=. eden_scm_lib_edenapi_tools _artifacts/linux  --project-install-prefix eden_scm_lib_edenapi_tools:/usr/local --final-install-prefix /usr/local
+    - uses: actions/upload-artifact@master
+      with:
+        name: eden_scm_lib_edenapi_tools
+        path: _artifacts
+    - name: Test eden_scm_lib_edenapi_tools
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages test --src-dir=. eden_scm_lib_edenapi_tools  --project-install-prefix eden_scm_lib_edenapi_tools:/usr/local

.github/workflows/eden_scm_lib_edenapi_tools_mac.yml

@@ -0,0 +1,37 @@
+# This file was @generated by getdeps.py
+
+name: EdenSCM Lib EdenAPI Tools Mac
+
+on:
+  push:
+    branches:
+    - master
+  pull_request:
+    branches:
+    - master
+
+jobs:
+  build:
+    runs-on: macOS-latest
+    env:
+      DEVELOPER_DIR: /Applications/Xcode_10.3.app/Contents/Developer
+    steps:
+    - uses: actions/checkout@v1
+    - name: Install Rust Stable
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: stable
+        default: true
+        profile: minimal
+    - name: Install system deps
+      run: sudo python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive eden_scm_lib_edenapi_tools
+    - name: Build eden_scm_lib_edenapi_tools
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --src-dir=. eden_scm_lib_edenapi_tools  --project-install-prefix eden_scm_lib_edenapi_tools:/usr/local
+    - name: Copy artifacts
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fixup-dyn-deps --src-dir=. eden_scm_lib_edenapi_tools _artifacts/mac  --project-install-prefix eden_scm_lib_edenapi_tools:/usr/local --final-install-prefix /usr/local
+    - uses: actions/upload-artifact@master
+      with:
+        name: eden_scm_lib_edenapi_tools
+        path: _artifacts
+    - name: Test eden_scm_lib_edenapi_tools
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages test --src-dir=. eden_scm_lib_edenapi_tools  --project-install-prefix eden_scm_lib_edenapi_tools:/usr/local

.github/workflows/mononoke-integration_linux.yml

@@ -35,7 +35,7 @@ jobs:
         python-version: '2.7'
     - name: Install Apt-get dependencies
       run: |
-        sudo apt-get install nmap
+        sudo apt-get install nmap tree
     - name: Install system deps
       run: sudo python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive eden_scm
     - name: Build eden_scm dependencies
@@ -60,6 +60,20 @@ jobs:
       run: |
         rm -rf /tmp/build/build/eden_scm/*
         df -h
+    - name: Build eden_scm_lib_edenapi_tools
+      run: >-
+        python3 build/fbcode_builder/getdeps.py build
+        --allow-system-packages
+        --scratch-path /tmp/build
+        --no-deps
+        --src-dir=.
+        eden_scm_lib_edenapi_tools
+    - name: Check space before cleanup
+      run: df -h
+    - name: Clean up eden_scm_lib_edenapi_tools build
+      run: |
+        rm -rf /tmp/build/build/eden_scm_lib_edenapi_tools/*
+        df -h
     - name: Build mononoke dependencies
       run: >-
         python3 build/fbcode_builder/getdeps.py build

.github/workflows/mononoke-integration_mac.yml

@@ -29,7 +29,7 @@ jobs:
         python-version: '2.7'
     - name: Install Brew dependencies
       run: |
-        brew install bash coreutils curl-openssl gnu-sed grep jq nmap
+        brew install bash coreutils curl-openssl gnu-sed grep jq nmap tree
     - name: Install system deps
       run: sudo python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive eden_scm
     - name: Build eden_scm dependencies
@@ -48,6 +48,14 @@ jobs:
         --no-deps
         --src-dir=.
         eden_scm
+    - name: Build eden_scm_lib_edenapi_tools
+      run: >-
+        python3 build/fbcode_builder/getdeps.py build
+        --allow-system-packages
+        --scratch-path /tmp/build
+        --no-deps
+        --src-dir=.
+        eden_scm_lib_edenapi_tools
     - name: Build mononoke dependencies
       run: >-
         python3 build/fbcode_builder/getdeps.py build

build/fbcode_builder/getdeps/builder.py

@@ -999,13 +999,15 @@ def __init__(
         inst_dir,
         build_doc,
         workspace_dir,
+        manifests_to_build,
         loader,
     ):
         super(CargoBuilder, self).__init__(
             build_opts, ctx, manifest, src_dir, build_dir, inst_dir
         )
         self.build_doc = build_doc
         self.ws_dir = workspace_dir
+        self.manifests_to_build = manifests_to_build and manifests_to_build.split(",")
         self.loader = loader
 
     def run_cargo(self, install_dirs, operation, args=None):
@@ -1026,7 +1028,10 @@ def build_source_dir(self):
         return os.path.join(self.build_dir, "source")
 
     def workspace_dir(self):
-        return os.path.join(self.build_source_dir(), self.ws_dir)
+        return os.path.join(self.build_source_dir(), self.ws_dir or "")
+
+    def manifest_dir(self, manifest):
+        return os.path.join(self.build_source_dir(), manifest)
 
     def recreate_dir(self, src, dst):
         if os.path.isdir(dst):
@@ -1058,7 +1063,8 @@ def _build(self, install_dirs, reconfigure):
                 )
             )
 
-        self._patchup_workspace()
+        if self.ws_dir is not None:
+            self._patchup_workspace()
 
         try:
             from getdeps.facebook.rust import vendored_crates
@@ -1069,11 +1075,26 @@ def _build(self, install_dirs, reconfigure):
             # so just rely on cargo downloading crates on it's own
             pass
 
-        self.run_cargo(
-            install_dirs,
-            "build",
-            ["--out-dir", os.path.join(self.inst_dir, "bin"), "-Zunstable-options"],
-        )
+        if self.manifests_to_build is None:
+            self.run_cargo(
+                install_dirs,
+                "build",
+                ["--out-dir", os.path.join(self.inst_dir, "bin"), "-Zunstable-options"],
+            )
+        else:
+            for manifest in self.manifests_to_build:
+                self.run_cargo(
+                    install_dirs,
+                    "build",
+                    [
+                        "--out-dir",
+                        os.path.join(self.inst_dir, "bin"),
+                        "-Zunstable-options",
+                        "--manifest-path",
+                        self.manifest_dir(manifest),
+                    ],
+                )
+
         self.recreate_dir(build_source_dir, os.path.join(self.inst_dir, "source"))
 
     def run_tests(
@@ -1082,11 +1103,18 @@ def run_tests(
         if test_filter:
             args = ["--", test_filter]
         else:
-            args = None
+            args = []
 
-        self.run_cargo(install_dirs, "test", args)
-        if self.build_doc:
-            self.run_cargo(install_dirs, "doc", ["--no-deps"])
+        if self.manifests_to_build is None:
+            self.run_cargo(install_dirs, "test", args)
+            if self.build_doc:
+                self.run_cargo(install_dirs, "doc", ["--no-deps"])
+        else:
+            for manifest in self.manifests_to_build:
+                margs = ["--manifest-path", self.manifest_dir(manifest)]
+                self.run_cargo(install_dirs, "test", args + margs)
+                if self.build_doc:
+                    self.run_cargo(install_dirs, "doc", ["--no-deps"] + margs)
 
     def _patchup_workspace(self):
         """

build/fbcode_builder/getdeps/manifest.py

@@ -74,7 +74,11 @@
     "msbuild": {"optional_section": True, "fields": {"project": REQUIRED}},
     "cargo": {
         "optional_section": True,
-        "fields": {"build_doc": OPTIONAL, "workspace_dir": OPTIONAL},
+        "fields": {
+            "build_doc": OPTIONAL,
+            "workspace_dir": OPTIONAL,
+            "manifests_to_build": OPTIONAL,
+        },
     },
     "cmake.defines": {"optional_section": True},
     "autoconf.args": {"optional_section": True},
@@ -489,7 +493,8 @@ def create_builder(  # noqa:C901
 
         if builder == "cargo":
             build_doc = self.get("cargo", "build_doc", False, ctx)
-            workspace_dir = self.get("cargo", "workspace_dir", "", ctx)
+            workspace_dir = self.get("cargo", "workspace_dir", None, ctx)
+            manifests_to_build = self.get("cargo", "manifests_to_build", None, ctx)
             return CargoBuilder(
                 build_options,
                 ctx,
@@ -499,6 +504,7 @@ def create_builder(  # noqa:C901
                 inst_dir,
                 build_doc,
                 workspace_dir,
+                manifests_to_build,
                 loader,
             )
 

build/fbcode_builder/manifests/eden_scm_lib_edenapi_tools

@@ -0,0 +1,36 @@
+[manifest]
+name = eden_scm_lib_edenapi_tools
+fbsource_path = fbcode/eden
+shipit_project = eden
+shipit_fbcode_builder = true
+
+[git]
+repo_url = https://github.com/facebookexperimental/eden.git
+
+[build]
+builder = cargo
+
+[cargo]
+build_doc = true
+manifests_to_build = eden/scm/lib/edenapi/tools/make_req/Cargo.toml,eden/scm/lib/edenapi/tools/read_res/Cargo.toml
+
+[shipit.pathmap]
+fbcode/eden/oss = .
+fbcode/eden = eden
+fbcode/tools/lfs = tools/lfs
+fbcode/fboss/common = common
+
+[shipit.strip]
+^fbcode/eden/fs/eden-config\.h$
+^fbcode/eden/fs/py/eden/config\.py$
+^fbcode/eden/hg/.*$
+^fbcode/eden/mononoke/(?!lfs_protocol)
+^fbcode/eden/scm/build/.*$
+^fbcode/eden/scm/lib/third-party/rust/.*/Cargo.toml$
+^fbcode/eden/.*/\.cargo/.*$
+^.*/fb/.*$
+/Cargo\.lock$
+\.pyc$
+
+[dependencies.fb=on]
+rust

build/fbcode_builder/manifests/rust-shed

@@ -12,6 +12,7 @@ builder = cargo
 
 [cargo]
 build_doc = true
+workspace_dir =
 
 [shipit.pathmap]
 fbcode/common/rust/shed = shed

eden/mononoke/tests/integration/manifest_deps

@@ -43,6 +43,14 @@ EDENSCM_BINS = {
     "BINARY_HG": "hg",
 }
 
+# The values here should correspond to both the name of binary as builded by
+# getdeps.py's eden_scm_lib_edenapi_tools project and the buck target. Those
+# names must be unique.
+EDENSCMLIBEDENAPITOOLS_BINS = {
+    "EDENAPI_MAKE_REQ": "make_req",
+    "EDENAPI_READ_RES": "read_res",
+}
+
 # The "//" in the values here corresponds to the root of repo (both GitHub and
 # fbcode repos have the same folder layout)
 OSS_DEPS = {

eden/mononoke/tests/integration/run_tests_getdeps.py

@@ -63,7 +63,7 @@ def parse_args():
 
 def prepare_manifest_deps(install_dir, build_dir, repo_root):
     exec(
-        "global OSS_DEPS; global MONONOKE_BINS; global EDENSCM_BINS; "
+        "global OSS_DEPS; global MONONOKE_BINS; global EDENSCM_BINS; global EDENSCMLIBEDENAPITOOLS_BINS; "
         + open(
             join(repo_root, "eden/mononoke/tests/integration/manifest_deps"), "r"
         ).read()
@@ -79,6 +79,8 @@ def prepare_manifest_deps(install_dir, build_dir, repo_root):
         MANIFEST_DEPS[k] = join(install_dir, "mononoke/bin", v)
     for k, v in EDENSCM_BINS.items():  # noqa: F821
         MANIFEST_DEPS[k] = join(install_dir, "eden_scm/bin", v)
+    for k, v in EDENSCMLIBEDENAPITOOLS_BINS.items():  # noqa: F821
+        MANIFEST_DEPS[k] = join(install_dir, "eden_scm_lib_edenapi_tools/bin", v)
 
     os.makedirs(build_dir, exist_ok=True)
     with open(join(build_dir, "manifest.json"), "w") as f:
@@ -120,12 +122,6 @@ def get_test_groups(repo_root):
             "test-walker-error-as-data.t",
         },
         TestGroup.BROKEN: {
-            "test-edenapi-server-commit-location-to-hash.t",  # Missing eden/scm's commands
-            "test-edenapi-server-commit-revlog-data.t",  # Missing eden/scm's commands
-            "test-edenapi-server-complete-trees.t",  # Missing eden/scm's commands
-            "test-edenapi-server-files.t",  # Missing eden/scm's commands
-            "test-edenapi-server-history.t",  # Missing eden/scm's commands
-            "test-edenapi-server-trees.t",  # Missing eden/scm's commands
             "test-fastreplay-inline-args.t",  # Returns different data in OSS
             "test-gitimport.t",  # Issue with hggit extension
             "test-hook-tailer.t",  # Issue with hggit extension