Build incremental components through build workflow with loading prev…

…ious build manifest (#4289)

Signed-off-by: Zelin Hao <zelinhao@amazon.com>

jenkins/opensearch/distribution-build.jenkinsfile

@@ -138,7 +138,7 @@ pipeline {
                             def snapshotBuild =
                                 build job: 'publish-opensearch-min-snapshots',
                                 propagate: false,
-                                wait: false,                               
+                                wait: false,
                                 parameters: [
                                     string(name: 'INPUT_MANIFEST', value: "${INPUT_MANIFEST}"),
                                 ]
@@ -849,4 +849,4 @@ def markStageUnstableIfPluginsFailedToBuild() {
     if (stageLogs.any{e -> e.contains('Failed plugins are')}) {
         unstable('Some plugins failed to build. See the ./build.sh step for logs and more details')
     }
-}
+}

src/build_workflow/README.md

@@ -1,10 +1,11 @@
 - [Building from Source](#building-from-source)
     - [OpenSearch](#opensearch)
     - [OpenSearch Dashboards](#opensearch-dashboards)
-  - [Build Paths](#build-paths)
-  - [Build.sh Options](#buildsh-options)
-  - [Custom Build Scripts](#custom-build-scripts)
-  - [Avoiding Rebuilds](#avoiding-rebuilds)
+    - [Build Paths](#build-paths)
+    - [Build.sh Options](#buildsh-options)
+    - [Custom Build Scripts](#custom-build-scripts)
+    - [Avoiding Rebuilds](#avoiding-rebuilds)
+    - [Incremental Build](#incremental-build)
 
 ## Building from Source
 
@@ -101,3 +102,15 @@ fi
 ```
 
 The [Jenkins workflows](../../jenkins) in this repository can use this mechanism to avoid rebuilding all of OpenSearch and OpenSearch Dashboards unnecessarily. 
+
+### Incremental Build
+
+This functionality augments the existing build process by introducing the `--incremental` binary parameter.
+
+Sample command: `./build.sh manifests/2.12.0/opensearch-2.12.0.yml --incremental`.
+
+The build workflow will examine the build manifest from the previous build using path `{distribution}/builds/opensearch/manifest.yml` when this command is executed.
+The build workflow will be executed in accordance with the comparison between the commits for each component in the preceding build manifest and the current input manifest.
+It will contain every modified component, and every component that relies on these revised components based on the `depends_on` entry in the input manifest.
+
+Once build is finished, new built artifacts will override the previous artifacts and a new build manifest will be generated using the previous build manifest as a reference, ensuring that all non-modified components remain unchanged.

src/build_workflow/build_args.py

@@ -51,14 +51,6 @@ def __init__(self) -> None:
             default=False,
             help="Build snapshot.",
         )
-        parser.add_argument(
-            "-c",
-            "--component",
-            dest="components",
-            nargs='*',
-            type=str,
-            help="Rebuild one or more components."
-        )
         parser.add_argument(
             "--keep",
             dest="keep",
@@ -104,7 +96,16 @@ def __init__(self) -> None:
             action="store_true",
             help="Do not fail the distribution build on any plugin component failure.",
         )
-        parser.add_argument(
+        group = parser.add_mutually_exclusive_group()
+        group.add_argument(
+            "-c",
+            "--component",
+            dest="components",
+            nargs='*',
+            type=str,
+            help="Rebuild one or more components."
+        )
+        group.add_argument(
             "-i",
             "--incremental",
             dest="incremental",

src/build_workflow/build_recorder.py

@@ -17,8 +17,8 @@
 
 
 class BuildRecorder:
-    def __init__(self, target: BuildTarget) -> None:
-        self.build_manifest = self.BuildManifestBuilder(target)
+    def __init__(self, target: BuildTarget, build_manifest: BuildManifest = None) -> None:
+        self.build_manifest = self.BuildManifestBuilder(target, build_manifest)
         self.target = target
         self.name = target.name
 
@@ -53,18 +53,24 @@ def write_manifest(self) -> None:
         logging.info(f"Created build manifest {manifest_path}")
 
     class BuildManifestBuilder:
-        def __init__(self, target: BuildTarget) -> None:
+        def __init__(self, target: BuildTarget, build_manifest: BuildManifest = None) -> None:
             self.data: Dict[str, Any] = {}
-            self.data["build"] = {}
-            self.data["build"]["id"] = target.build_id
-            self.data["build"]["name"] = target.name
-            self.data["build"]["version"] = target.opensearch_version
-            self.data["build"]["platform"] = target.platform
-            self.data["build"]["architecture"] = target.architecture
-            self.data["build"]["distribution"] = target.distribution if target.distribution else "tar"
-            self.data["schema-version"] = "1.2"
             self.components_hash: Dict[str, Dict[str, Any]] = {}
 
+            if build_manifest:
+                self.data = build_manifest.__to_dict__()
+                for component in build_manifest.components.select():
+                    self.components_hash[component.name] = component.__to_dict__()
+            else:
+                self.data["build"] = {}
+                self.data["build"]["id"] = target.build_id
+                self.data["build"]["name"] = target.name
+                self.data["build"]["version"] = target.opensearch_version
+                self.data["build"]["platform"] = target.platform
+                self.data["build"]["architecture"] = target.architecture
+                self.data["build"]["distribution"] = target.distribution if target.distribution else "tar"
+                self.data["schema-version"] = "1.2"
+
         def append_component(self, name: str, version: str, repository_url: str, ref: str, commit_id: str) -> None:
             component = {
                 "name": name,
@@ -75,6 +81,7 @@ def append_component(self, name: str, version: str, repository_url: str, ref: st
                 "version": version,
             }
             self.components_hash[name] = component
+            logging.info(f"Appended {name} component in build manifest.")
 
         def append_artifact(self, component: str, type: str, path: str) -> None:
             artifacts = self.components_hash[component]["artifacts"]

src/run_build.py

@@ -15,6 +15,7 @@
 from build_workflow.build_recorder import BuildRecorder
 from build_workflow.build_target import BuildTarget
 from build_workflow.builders import Builders
+from manifests.build_manifest import BuildManifest
 from manifests.input_manifest import InputManifest
 from paths.build_output_dir import BuildOutputDir
 from system import console
@@ -25,6 +26,8 @@ def main() -> int:
     args = BuildArgs()
     console.configure(level=args.logging_level)
     manifest = InputManifest.from_file(args.manifest)
+    build_manifest = None
+    components = args.components
     failed_plugins = []
 
     if args.ref_manifest:
@@ -45,8 +48,20 @@ def main() -> int:
     if args.incremental:
         buildIncremental = BuildIncremental(manifest, args.distribution)
         list_of_updated_plugins = buildIncremental.commits_diff(manifest)
-        logging.info(f"Plugins for incremental build: {buildIncremental.rebuild_plugins(list_of_updated_plugins, manifest)}")
-        return 0
+        components = buildIncremental.rebuild_plugins(list_of_updated_plugins, manifest)
+        if not components:
+            logging.info("No commit difference found between any components. Skipping the build")
+            return 0
+
+        logging.info(f"Plugins for incremental build: {components}")
+
+        build_manifest_path = os.path.join(args.distribution, "builds", manifest.build.filename, "manifest.yml")
+        if not os.path.exists(build_manifest_path):
+            logging.error(f"Previous build manifest missing at path: {build_manifest_path}")
+
+        logging.info(f"Build {components} incrementally.")
+
+        build_manifest = BuildManifest.from_path(build_manifest_path)
 
     with TemporaryDirectory(keep=args.keep, chdir=True) as work_dir:
         logging.info(f"Building in {work_dir.name}")
@@ -63,11 +78,11 @@ def main() -> int:
             architecture=args.architecture or manifest.build.architecture,
         )
 
-        build_recorder = BuildRecorder(target)
+        build_recorder = BuildRecorder(target, build_manifest) if args.incremental else BuildRecorder(target)
 
         logging.info(f"Building {manifest.build.name} ({target.architecture}) into {target.output_dir}")
 
-        for component in manifest.components.select(focus=args.components, platform=target.platform):
+        for component in manifest.components.select(focus=components, platform=target.platform):
             logging.info(f"Building {component.name}")
 
             builder = Builders.builder_from(component, target)