WIP: might work but needs history cleanup

.gitlab-ci.yml

@@ -69,10 +69,11 @@ generate-ostree-build-config:
     INTERNAL_NETWORK: "true"
   script:
     - sudo dnf -y install go python3 gpgme-devel s3cmd
-    - ./test/generate-ostree-build-config build-config.yml
+    - ./test/generate-ostree-build-config build-config.yml ./build-configs/
   artifacts:
     paths:
       - build-config.yml
+      - ./build-configs
 
 image-build-ostree-trigger:
   stage: test-ostree

test/generate-build-config

@@ -26,6 +26,9 @@ build/{distro}/{arch}/{image_type}/{config_name}:
 
 
 def generate_manifests(outputdir):
+    """
+    Generate all manifest using the default config map and return a dictionary mapping each manifest filename to its ID.
+    """
     arches_arg = ",".join(ARCHITECTURES)
     cmd = ["go", "run", "./cmd/gen-manifests",
            "-cache", os.path.join(testlib.TEST_CACHE_ROOT, "rpmmd"),
@@ -47,6 +50,7 @@ def generate_manifests(outputdir):
         print(line)
 
     print("Manifest generation done!\n")
+    return testlib.read_manifests(outputdir)
 
 
 def generate_configs(build_requests, pipeline_file):
@@ -82,8 +86,8 @@ def main():
     testlib.check_config_names()
 
     with TemporaryDirectory() as manifest_dir:
-        generate_manifests(manifest_dir)
-        build_requests = testlib.filter_builds(manifest_dir)
+        manifests = generate_manifests(manifest_dir)
+        build_requests = testlib.filter_builds(manifests)
 
     with open(config_path, "w") as config_file:
         if len(build_requests) == 0:

test/generate-ostree-build-config

@@ -22,6 +22,7 @@ build/{distro}/{arch}/{image_type}/{config_name}:
     RUNNER: aws/fedora-38-{arch}
     INTERNAL_NETWORK: "{internal}"
   dependencies:
+    # get config files from generator artifacts
     - gen-ostree
 """
 
@@ -55,6 +56,8 @@ def gen_dependency_manifests(config_map, outputdir):
     Generate manifests for the dependencies of the configs in config map. The function generates a temporary config map
     that maps each config that appears in a dependency to the dependency image type and runs gen-manifests with that
     config map.
+
+    Returns a dictionary mapping manifest file name (without path) to the ID of each manifest file.
     """
     print("Generating manifests for dependencies")
     arches_arg = ",".join(ARCHITECTURES)
@@ -103,6 +106,7 @@ def gen_dependency_manifests(config_map, outputdir):
         print(line)
 
     print("Manifest generation done!\n")
+    return testlib.read_manifests(outputdir)
 
 
 def gen_image_manifests(config_map, configs, outputdir):
@@ -113,7 +117,7 @@ def gen_image_manifests(config_map, configs, outputdir):
     """
     with TemporaryDirectory() as tmpdir:
         # write each config to a separate file
-        for config in configs:
+        for config in configs.values():
             with open(os.path.join(tmpdir, config["name"] + ".json"), "w", encoding="utf-8") as config_file:
                 json.dump(config, config_file)
 
@@ -144,6 +148,7 @@ def gen_image_manifests(config_map, configs, outputdir):
         print(line)
 
     print("Manifest generation done!\n")
+    return testlib.read_manifests(outputdir)
 
 
 def default_ref(distro, arch):
@@ -160,18 +165,20 @@ def default_ref(distro, arch):
 
 
 @contextmanager
-def setup_dependencies(manifest_dir, config_map):
+def setup_dependencies(manifests, config_map):
     """
     For each config in the config map, list all image configurations (distro, arch, image type) that it applies to and
-    use the manifests in the manifest dir to find the corresponding manifest IDs. Pull and run the corresponding
-    dependency container from the registry and write a new config (and config map) that replaces the ostree options with
-    the container URL and port.
+    find the corresponding manifest IDs from the provided dictionary. Pull and run the corresponding dependency
+    container from the registry and write a new config (and config map) that replaces the ostree options with the
+    container URL
+    and port.
+
     The config map and configs that function yields can be used to generate manifests with ostree commits resolved.
     The containers are stopped when the context exits.
     """
     container_ids = []
     new_config_map = {}
-    new_configs = []
+    new_configs = {}
 
     container_ports: dict[str, int] = {}
     for config_path, filters in config_map.items():
@@ -198,10 +205,7 @@ def setup_dependencies(manifest_dir, config_map):
                 continue
             image_type = image_config["image-type"]
             dep_build_name = testlib.gen_build_name(distro, arch, dep_image_type, dep_config_name)
-            manifest_path = os.path.join(manifest_dir, dep_build_name + ".json")
-            with open(manifest_path) as manifest_file:
-                manifest_data = json.load(manifest_file)
-            manifest_id = testlib.get_manifest_id(manifest_data["manifest"])
+            manifest_id = manifests[dep_build_name + ".json"]["id"]
             container = f"{testlib.REGISTRY}/{dep_build_name}:build-{manifest_id}"
 
             # start each container once on an incremental port
@@ -233,7 +237,7 @@ def setup_dependencies(manifest_dir, config_map):
                     "port": port,
                 }
             }
-            new_configs.append(new_config)
+            new_configs[config_name] = new_config
             new_config_map[config_name + ".json"] = {
                 "distros": [distro],
                 "arches": [arch],
@@ -248,8 +252,9 @@ def setup_dependencies(manifest_dir, config_map):
             testlib.runcmd(["podman", "stop", cont_id])
 
 
-def generate_configs(build_requests, pipeline_file):
+def generate_configs(build_requests, pull_configs, pipeline_file, configs_dir):
     print(f"Generating dynamic pipelines for {len(build_requests)} builds")
+    os.makedirs(configs_dir, exist_ok=True)
     for build in build_requests:
         distro = build["distro"]
         arch = build["arch"]
@@ -258,33 +263,41 @@ def generate_configs(build_requests, pipeline_file):
 
         config_name = config["name"]
 
-        # TODO: start container command
-        # TODO: write config in artifact path and insert path to config into template
+        # write the config to the artifacts directory
+        build_config_path = os.path.join(configs_dir, config_name + ".json")
+        with open(build_config_path, "w", encoding="utf-8") as build_config_file:
+            json.dump(config, build_config_file)
+
+        # generate script line to pull and start container
+        container = pull_configs[config_name]["container"]
+        container_name = container["name"]
+        container_port = container["port"]
+        container_cmd = f"podman run -d --rm -p{container_port}:8080 {container_name}"
 
-        config_path = os.path.join(testlib.CONFIGS_PATH, config_name+".json")
         pipeline_file.write(JOB_TEMPLATE.format(distro=distro, arch=arch, image_type=image_type,
-                                                config_name=config_name, config=config_path,
-                                                start_container="<FIXME>",
+                                                config_name=config_name, config=build_config_path,
+                                                start_container=container_cmd,
                                                 internal="true" if "rhel" in distro else "false"))
     print("DONE!")
 
 
 def main():
     config_path = sys.argv[1]
+    configs_dir = sys.argv[2]
 
     testlib.check_config_names()
 
     config_map = configs_with_deps(read_config_map())  # filtered config map: only configs with deps
 
     with TemporaryDirectory() as cache_root:
         dep_manifest_dir = os.path.join(cache_root, "dependencies")
-        gen_dependency_manifests(config_map, dep_manifest_dir)
+        dep_manifests = gen_dependency_manifests(config_map, dep_manifest_dir)
 
-        with setup_dependencies(dep_manifest_dir, config_map) as (config_map, configs):
+        with setup_dependencies(dep_manifests, config_map) as (pull_config_map, pull_configs):
             manifest_dir = os.path.join(cache_root, "manifests")
-            gen_image_manifests(config_map, configs, manifest_dir)
+            manifests = gen_image_manifests(pull_config_map, pull_configs, manifest_dir)
 
-        build_requests = testlib.filter_builds(manifest_dir, skip_ostree_pull=False)
+        build_requests = testlib.filter_builds(manifests, skip_ostree_pull=False)
 
     with open(config_path, "w") as config_file:
         if len(build_requests) == 0:
@@ -293,7 +306,7 @@ def main():
             return
 
         config_file.write(testlib.BASE_CONFIG)
-        generate_configs(build_requests, config_file)
+        generate_configs(build_requests, pull_configs, config_file, configs_dir)
 
 
 if __name__ == "__main__":

test/imgtestlib.py

@@ -8,8 +8,7 @@
 CONFIGS_PATH = "./test/configs"
 CONFIG_MAP = "./test/config-map.json"
 
-# S3_BUCKET = "s3://image-builder-ci-artifacts"
-S3_BUCKET = "s3://redhat.akoutsou.testing"
+S3_BUCKET = "s3://image-builder-ci-artifacts"
 S3_PREFIX = "images/builds"
 
 REGISTRY = "registry.gitlab.com/redhat/services/products/image-builder/ci/images"
@@ -139,7 +138,25 @@ def check_config_names():
         sys.exit(1)
 
 
-def filter_builds(manifest_dir, skip_ostree_pull=True):
+def read_manifests(path):
+    """
+    Calculates the IDs for all the manifests in the given path and returns a dictionary mapping each filename to its ID.
+    """
+    print(f"Reading manifests in {path}")
+    manifest_ids = {}
+    for manifest_fname in os.listdir(path):
+        manifest_path = os.path.join(path, manifest_fname)
+        with open(manifest_path) as manifest_file:
+            manifest_data = json.load(manifest_file)
+        manifest_ids[manifest_fname] = {
+            "data": manifest_data,
+            "id": get_manifest_id(manifest_data["manifest"]),
+        }
+    print("Done")
+    return manifest_ids
+
+
+def filter_builds(manifests, skip_ostree_pull=True):
     """
     Returns a list of build requests for manifests in manifest_dir that have no matching config in the test build cache.
     """
@@ -152,18 +169,11 @@ def filter_builds(manifest_dir, skip_ostree_pull=True):
 
     errors = []
 
-    for manifest_file in os.listdir(manifest_dir):
-        manifest_path = os.path.join(manifest_dir, manifest_file)
-
-        with open(manifest_path) as manifest_fp:
-            data = json.load(manifest_fp)
-
-        manifest_data = data["manifest"]
-
-        # generate manifest id based on concatenated stage IDs calculated from osbuild
-        manifest_id = get_manifest_id(manifest_data)
+    for manifest_fname, data in manifests.items():
+        manifest_id = data["id"]
         id_fname = manifest_id + ".json"
 
+        data = data.get("data")
         build_request = data["build-request"]
         distro = build_request["distro"]
         arch = build_request["arch"]
@@ -189,7 +199,7 @@ def filter_builds(manifest_dir, skip_ostree_pull=True):
                 with open(id_config_path) as dl_config_fp:
                     dl_config = json.load(dl_config_fp)
                 commit = dl_config["commit"]
-                print(f"Manifest {manifest_file} was successfully built in commit {commit}")
+                print(f"Manifest {manifest_fname} was successfully built in commit {commit}")
                 continue
             except json.JSONDecodeError as jd:
                 errors.append((