Build ostree images that depend on commits

.gitlab-ci.yml

@@ -1,14 +1,23 @@
+---
 stages:
   - init
   - gen
   - test
+  - gen-ostree
+  - test-ostree
   - finish
 
+variables:
+  # Set the parent pipeline ID as a global variable so that the child pipeline
+  # jobs can use it to retrieve artifacts.
+  PARENT_PIPELINE_ID: $CI_PIPELINE_ID
+
 .base:
   before_script:
     - mkdir -p /tmp/artifacts
     - schutzbot/ci_details.sh > /tmp/artifacts/ci-details-before-run.txt
-    - cat schutzbot/team_ssh_keys.txt | tee -a ~/.ssh/authorized_keys > /dev/null
+    - cat schutzbot/team_ssh_keys.txt |
+        tee -a ~/.ssh/authorized_keys > /dev/null
   after_script:
     - schutzbot/ci_details.sh > /tmp/artifacts/ci-details-after-run.txt || true
     - schutzbot/update_github_status.sh update || true
@@ -37,26 +46,98 @@ init:
   script:
     - schutzbot/update_github_status.sh start
 
-generate-build-config:
+generate-build-config-x86_64:
   stage: gen
   extends: .terraform
   variables:
     RUNNER: aws/fedora-38-x86_64
     INTERNAL_NETWORK: "true"
+    PYTHONUNBUFFERED: 1
+  script:
+    - sudo dnf -y install go python3 gpgme-devel s3cmd
+      osbuild osbuild-luks2 osbuild-lvm2 osbuild-ostree osbuild-selinux
+    - ./test/generate-build-config build-config.yml
+  artifacts:
+    paths:
+      - build-config.yml
+
+generate-build-config-aarch64:
+  stage: gen
+  extends: .terraform
+  variables:
+    RUNNER: aws/fedora-38-aarch64
+    INTERNAL_NETWORK: "true"
+    PYTHONUNBUFFERED: 1
   script:
     - sudo dnf -y install go python3 gpgme-devel s3cmd
       osbuild osbuild-luks2 osbuild-lvm2 osbuild-ostree osbuild-selinux
-    - ./test/cases/generate-build-config build-config.yml
+    - ./test/generate-build-config build-config.yml
   artifacts:
     paths:
       - build-config.yml
 
-image-build-trigger:
+image-build-trigger-x86_64:
   stage: test
   trigger:
     include:
       - artifact: build-config.yml
-        job: generate-build-config
+        job: generate-build-config-x86_64
+    strategy: depend
+
+image-build-trigger-aarch64:
+  stage: test
+  trigger:
+    include:
+      - artifact: build-config.yml
+        job: generate-build-config-aarch64
+    strategy: depend
+
+generate-ostree-build-config-x86_64:
+  stage: gen-ostree
+  extends: .terraform
+  variables:
+    RUNNER: aws/fedora-38-x86_64
+    INTERNAL_NETWORK: "true"
+    PYTHONUNBUFFERED: 1
+  script:
+    - sudo dnf -y install go python3 gpgme-devel s3cmd
+      osbuild osbuild-luks2 osbuild-lvm2 osbuild-ostree osbuild-selinux podman
+    - ./test/generate-ostree-build-config build-config.yml build-configs
+  artifacts:
+    paths:
+      - build-config.yml
+      - build-configs
+
+generate-ostree-build-config-aarch64:
+  stage: gen-ostree
+  extends: .terraform
+  variables:
+    RUNNER: aws/fedora-38-aarch64
+    INTERNAL_NETWORK: "true"
+    PYTHONUNBUFFERED: 1
+  script:
+    - sudo dnf -y install go python3 gpgme-devel s3cmd
+      osbuild osbuild-luks2 osbuild-lvm2 osbuild-ostree osbuild-selinux podman
+    - ./test/generate-ostree-build-config build-config.yml build-configs
+  artifacts:
+    paths:
+      - build-config.yml
+      - build-configs
+
+image-build-ostree-trigger-x86_64:
+  stage: test-ostree
+  trigger:
+    include:
+      - artifact: build-config.yml
+        job: generate-ostree-build-config-x86_64
+    strategy: depend
+
+image-build-ostree-trigger-aarch64:
+  stage: test-ostree
+  trigger:
+    include:
+      - artifact: build-config.yml
+        job: generate-ostree-build-config-aarch64
     strategy: depend
 
 finish:

cmd/gen-manifests/main.go

@@ -81,18 +81,68 @@ type buildRequest struct {
 	Arch         string       `json:"arch,omitempty"`
 	ImageType    string       `json:"image-type,omitempty"`
 	Repositories []repository `json:"repositories,omitempty"`
-	Config       *buildConfig `json:"config"`
+	Config       *BuildConfig `json:"config"`
 }
 
-type buildConfig struct {
+type BuildConfig struct {
 	Name      string         `json:"name"`
 	OSTree    *ostreeOptions `json:"ostree,omitempty"`
 	Blueprint *crBlueprint   `json:"blueprint,omitempty"`
 }
 
-type configMap map[string][]buildConfig
+type BuildDependency struct {
+	Config    string `json:"config"`
+	ImageType string `json:"image-type"`
+}
+
+// BuildConfigs is a nested map representing the configs to use for each
+// distro/arch/image-type. If any component is empty, it maps to all values.
+type BuildConfigs map[string]map[string]map[string][]BuildConfig
+
+func (bc BuildConfigs) Insert(distro, arch, imageType string, cfg BuildConfig) {
+	distroCfgs := bc[distro]
+	if distroCfgs == nil {
+		distroCfgs = make(map[string]map[string][]BuildConfig)
+	}
+
+	distroArchCfgs := distroCfgs[arch]
+	if distroArchCfgs == nil {
+		distroArchCfgs = make(map[string][]BuildConfig)
+	}
+
+	distroArchItCfgs := distroArchCfgs[imageType]
+	if distroArchItCfgs == nil {
+		distroArchItCfgs = make([]BuildConfig, 0)
+	}
+
+	distroArchItCfgs = append(distroArchItCfgs, cfg)
+	distroArchCfgs[imageType] = distroArchItCfgs
+	distroCfgs[arch] = distroArchCfgs
+	bc[distro] = distroCfgs
+}
 
-func loadConfig(path string) buildConfig {
+func (bc BuildConfigs) Get(distro, arch, imageType string) []BuildConfig {
+	configs := make([]BuildConfig, 0)
+	for distroName, distroCfgs := range bc {
+		distroGlob := glob.MustCompile(distroName)
+		if distroGlob.Match(distro) {
+			for archName, distroArchCfgs := range distroCfgs {
+				archGlob := glob.MustCompile(archName)
+				if archGlob.Match(arch) {
+					for itName, distroArchItCfgs := range distroArchCfgs {
+						itGlob := glob.MustCompile(itName)
+						if itGlob.Match(imageType) {
+							configs = append(configs, distroArchItCfgs...)
+						}
+					}
+				}
+			}
+		}
+	}
+	return configs
+}
+
+func loadConfig(path string) BuildConfig {
 	fp, err := os.Open(path)
 	if err != nil {
 		panic(fmt.Sprintf("failed to open config %q: %s", path, err.Error()))
@@ -102,14 +152,21 @@ func loadConfig(path string) buildConfig {
 	if err != nil {
 		panic(fmt.Sprintf("failed to read config %q: %s", path, err.Error()))
 	}
-	var conf buildConfig
+	var conf BuildConfig
 	if err := json.Unmarshal(data, &conf); err != nil {
 		panic(fmt.Sprintf("failed to unmarshal config %q: %s", path, err.Error()))
 	}
 	return conf
 }
 
-func loadConfigMap(configPath string) configMap {
+func loadConfigMap(configPath string) BuildConfigs {
+	type configFilters struct {
+		ImageTypes []string `json:"image-types"`
+		Distros    []string `json:"distros"`
+		Arches     []string `json:"arches"`
+	}
+	type configMap map[string]configFilters
+
 	fp, err := os.Open(configPath)
 	if err != nil {
 		panic(fmt.Sprintf("failed to open config map %q: %s", configPath, err.Error()))
@@ -119,19 +176,36 @@ func loadConfigMap(configPath string) configMap {
 	if err != nil {
 		panic(fmt.Sprintf("failed to read config map %q: %s", configPath, err.Error()))
 	}
-	configPaths := make(map[string][]string)
-	if err := json.Unmarshal(data, &configPaths); err != nil {
+	var cfgMap configMap
+	if err := json.Unmarshal(data, &cfgMap); err != nil {
 		panic(fmt.Sprintf("failed to unmarshal config map %q: %s", configPath, err.Error()))
 	}
 
+	emptyFallback := func(list []string) []string {
+		if len(list) == 0 {
+			// empty list means everything so let's add the * to catch
+			// everything with glob
+			return []string{"*"}
+		}
+		return list
+	}
+
 	// load each config from its path
-	cm := make(configMap)
-	for it, paths := range configPaths {
-		configs := make([]buildConfig, len(paths))
-		for idx, path := range paths {
-			configs[idx] = loadConfig(path)
+	cm := make(BuildConfigs)
+	for path, filters := range cfgMap {
+		// config paths can be relative to the location of the config map
+		if !filepath.IsAbs(path) {
+			cfgDir := filepath.Dir(configPath)
+			path = filepath.Join(cfgDir, path)
+		}
+		config := loadConfig(path)
+		for _, d := range emptyFallback(filters.Distros) {
+			for _, a := range emptyFallback(filters.Arches) {
+				for _, it := range emptyFallback(filters.ImageTypes) {
+					cm.Insert(d, a, it, config)
+				}
+			}
 		}
-		cm[it] = configs
 	}
 
 	return cm
@@ -142,7 +216,7 @@ type manifestJob func(chan string) error
 func makeManifestJob(
 	name string,
 	imgType distro.ImageType,
-	bc buildConfig,
+	bc BuildConfig,
 	distribution distro.Distro,
 	repos []repository,
 	archName string,
@@ -512,12 +586,13 @@ func main() {
 	// common args
 	var outputDir, cacheRoot, configPath string
 	var nWorkers int
-	var metadata bool
+	var metadata, skipNoconfig bool
 	flag.StringVar(&outputDir, "output", "test/data/manifests/", "manifest store directory")
 	flag.IntVar(&nWorkers, "workers", 16, "number of workers to run concurrently")
 	flag.StringVar(&cacheRoot, "cache", "/tmp/rpmmd", "rpm metadata cache directory")
 	flag.BoolVar(&metadata, "metadata", true, "store metadata in the file")
 	flag.StringVar(&configPath, "config", "test/config-map.json", "configuration file mapping image types to configs")
+	flag.BoolVar(&skipNoconfig, "skip-noconfig", false, "skip distro-arch-image configurations that have no config (otherwise fail)")
 
 	// content args
 	var packages, containers, commits bool
@@ -589,14 +664,12 @@ func main() {
 					continue
 				}
 
-				imgTypeConfigs := configs[imgTypeName]
-				if len(imgTypeConfigs) == 0 {
-					// No configs specified. Use default.
-					imgTypeConfigs = configs["default"]
-				}
-
+				imgTypeConfigs := configs.Get(distroName, archName, imgTypeName)
 				if len(imgTypeConfigs) == 0 {
-					panic(fmt.Sprintf("no configs found for image type %q and no default defined", imgTypeName))
+					if skipNoconfig {
+						continue
+					}
+					panic(fmt.Sprintf("no configs defined for image type %q", imgTypeName))
 				}
 
 				for _, itConfig := range imgTypeConfigs {