Skip to content
This repository has been archived by the owner on Oct 22, 2021. It is now read-only.

Commit

Permalink
Merge pull request #376 from SUSE/mm-rename-role-manifest
Browse files Browse the repository at this point in the history 
Rename roles to instance groups
  • Loading branch information
Vlad Iovanov authored Aug 8, 2018
2 parents d62c076 + c834d98 commit 711088f
Show file tree
Hide file tree
Showing 108 changed files with 1,101 additions and 1,101 deletions.
2 changes: 2 additions & 0 deletions Makefile
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
#!/usr/bin/env make

ifeq ($(GIT_ROOT),)


GIT_ROOT:=$(shell git rev-parse --show-toplevel)
endif

Expand Down
78 changes: 39 additions & 39 deletions app/fissile.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -301,7 +301,7 @@ func newPropertyInfo(maybeHash bool) *propertyInfo {
}

// Compile will compile a list of dev BOSH releases
func (f *Fissile) Compile(stemcellImageName string, targetPath, roleManifestPath, metricsPath string, roleNames, releaseNames []string, workerCount int, dockerNetworkMode string, withoutDocker, verbose bool) error {
func (f *Fissile) Compile(stemcellImageName string, targetPath, roleManifestPath, metricsPath string, instanceGroupNames, releaseNames []string, workerCount int, dockerNetworkMode string, withoutDocker, verbose bool) error {
if len(f.releases) == 0 {
return fmt.Errorf("Releases not loaded")
}
Expand Down Expand Up @@ -344,12 +344,12 @@ func (f *Fissile) Compile(stemcellImageName string, targetPath, roleManifestPath
}
}

roles, err := roleManifest.SelectRoles(roleNames)
instanceGroups, err := roleManifest.SelectInstanceGroups(instanceGroupNames)
if err != nil {
return fmt.Errorf("Error selecting packages to build: %s", err.Error())
}

if err := comp.Compile(workerCount, releases, roles, verbose); err != nil {
if err := comp.Compile(workerCount, releases, instanceGroups, verbose); err != nil {
return fmt.Errorf("Error compiling packages: %s", err.Error())
}

Expand Down Expand Up @@ -420,7 +420,7 @@ func (f *Fissile) CleanCache(targetPath string) error {

// GeneratePackagesRoleImage builds the docker image for the packages layer
// where all packages are included
func (f *Fissile) GeneratePackagesRoleImage(stemcellImageName string, roleManifest *model.RoleManifest, noBuild, force bool, roles model.Roles, packagesImageBuilder *builder.PackagesImageBuilder, labels map[string]string) error {
func (f *Fissile) GeneratePackagesRoleImage(stemcellImageName string, roleManifest *model.RoleManifest, noBuild, force bool, instanceGroups model.InstanceGroups, packagesImageBuilder *builder.PackagesImageBuilder, labels map[string]string) error {
if len(f.releases) == 0 {
return fmt.Errorf("Releases not loaded")
}
Expand All @@ -430,9 +430,9 @@ func (f *Fissile) GeneratePackagesRoleImage(stemcellImageName string, roleManife
return fmt.Errorf("Error connecting to docker: %s", err.Error())
}

packagesLayerImageName, err := packagesImageBuilder.GetPackagesLayerImageName(roleManifest, roles, f)
packagesLayerImageName, err := packagesImageBuilder.GetPackagesLayerImageName(roleManifest, instanceGroups, f)
if err != nil {
return fmt.Errorf("Error finding role's package name: %s", err.Error())
return fmt.Errorf("Error finding instance group's package name: %s", err.Error())
}
if !force {
if hasImage, err := dockerManager.HasImage(packagesLayerImageName); err == nil && hasImage {
Expand Down Expand Up @@ -460,7 +460,7 @@ func (f *Fissile) GeneratePackagesRoleImage(stemcellImageName string, roleManife
docker.ColoredBuildStringFunc(packagesLayerImageName),
)

tarPopulator := packagesImageBuilder.NewDockerPopulator(roles, labels, force)
tarPopulator := packagesImageBuilder.NewDockerPopulator(instanceGroups, labels, force)
err = dockerManager.BuildImageFromCallback(packagesLayerImageName, stdoutWriter, tarPopulator)
if err != nil {
log.WriteTo(f.UI)
Expand All @@ -473,14 +473,14 @@ func (f *Fissile) GeneratePackagesRoleImage(stemcellImageName string, roleManife

// GeneratePackagesRoleTarball builds a tarball snapshot of the build context
// for the docker image for the packages layer where all packages are included
func (f *Fissile) GeneratePackagesRoleTarball(repository string, roleManifest *model.RoleManifest, noBuild, force bool, roles model.Roles, outputDirectory string, packagesImageBuilder *builder.PackagesImageBuilder, labels map[string]string) error {
func (f *Fissile) GeneratePackagesRoleTarball(repository string, roleManifest *model.RoleManifest, noBuild, force bool, instanceGroups model.InstanceGroups, outputDirectory string, packagesImageBuilder *builder.PackagesImageBuilder, labels map[string]string) error {
if len(f.releases) == 0 {
return fmt.Errorf("Releases not loaded")
}

packagesLayerImageName, err := packagesImageBuilder.GetPackagesLayerImageName(roleManifest, roles, f)
packagesLayerImageName, err := packagesImageBuilder.GetPackagesLayerImageName(roleManifest, instanceGroups, f)
if err != nil {
return fmt.Errorf("Error finding role's package name: %v", err)
return fmt.Errorf("Error finding instance group's package name: %v", err)
}
outputPath := filepath.Join(outputDirectory, fmt.Sprintf("%s.tar", packagesLayerImageName))

Expand All @@ -507,7 +507,7 @@ func (f *Fissile) GeneratePackagesRoleTarball(repository string, roleManifest *m

// We always force build all packages here to avoid needing to talk to the
// docker daemon to figure out what we can keep
tarPopulator := packagesImageBuilder.NewDockerPopulator(roles, labels, true)
tarPopulator := packagesImageBuilder.NewDockerPopulator(instanceGroups, labels, true)
err = tarPopulator(tarWriter)
if err != nil {
return fmt.Errorf("Error writing tar file: %s", err)
Expand All @@ -522,14 +522,14 @@ func (f *Fissile) GeneratePackagesRoleTarball(repository string, roleManifest *m
}

// GenerateRoleImages generates all role images using releases
func (f *Fissile) GenerateRoleImages(targetPath, registry, organization, repository, stemcellImageName, stemcellImageID, metricsPath string, noBuild, force bool, tagExtra string, roleNames []string, workerCount int, roleManifestPath, compiledPackagesPath, lightManifestPath, darkManifestPath, outputDirectory string, labels map[string]string) error {
func (f *Fissile) GenerateRoleImages(targetPath, registry, organization, repository, stemcellImageName, stemcellImageID, metricsPath string, noBuild, force bool, tagExtra string, instanceGroupNames []string, workerCount int, roleManifestPath, compiledPackagesPath, lightManifestPath, darkManifestPath, outputDirectory string, labels map[string]string) error {
if len(f.releases) == 0 {
return fmt.Errorf("Releases not loaded")
}

if metricsPath != "" {
stampy.Stamp(metricsPath, "fissile", "create-role-images", "start")
defer stampy.Stamp(metricsPath, "fissile", "create-role-images", "done")
stampy.Stamp(metricsPath, "fissile", "create-images", "start")
defer stampy.Stamp(metricsPath, "fissile", "create-images", "done")
}

roleManifest, err := model.LoadRoleManifest(roleManifestPath, f.releases, f)
Expand Down Expand Up @@ -570,21 +570,21 @@ func (f *Fissile) GenerateRoleImages(targetPath, registry, organization, reposit
return err
}

roles, err := roleManifest.SelectRoles(roleNames)
instanceGroups, err := roleManifest.SelectInstanceGroups(instanceGroupNames)
if err != nil {
return err
}

if outputDirectory == "" {
err = f.GeneratePackagesRoleImage(stemcellImageName, roleManifest, noBuild, force, roles, packagesImageBuilder, labels)
err = f.GeneratePackagesRoleImage(stemcellImageName, roleManifest, noBuild, force, instanceGroups, packagesImageBuilder, labels)
} else {
err = f.GeneratePackagesRoleTarball(stemcellImageName, roleManifest, noBuild, force, roles, outputDirectory, packagesImageBuilder, labels)
err = f.GeneratePackagesRoleTarball(stemcellImageName, roleManifest, noBuild, force, instanceGroups, outputDirectory, packagesImageBuilder, labels)
}
if err != nil {
return err
}

packagesLayerImageName, err := packagesImageBuilder.GetPackagesLayerImageName(roleManifest, roles, f)
packagesLayerImageName, err := packagesImageBuilder.GetPackagesLayerImageName(roleManifest, instanceGroups, f)
if err != nil {
return err
}
Expand All @@ -605,7 +605,7 @@ func (f *Fissile) GenerateRoleImages(targetPath, registry, organization, reposit
return err
}

return roleBuilder.BuildRoleImages(roles, registry, organization, repository, packagesLayerImageName, outputDirectory, force, noBuild, workerCount)
return roleBuilder.BuildRoleImages(instanceGroups, registry, organization, repository, packagesLayerImageName, outputDirectory, force, noBuild, workerCount)
}

// ListRoleImages lists all dev role images
Expand Down Expand Up @@ -638,13 +638,13 @@ func (f *Fissile) ListRoleImages(registry, organization, repository, roleManifes
return fmt.Errorf("Error loading opinions: %s", err.Error())
}

for _, role := range roleManifest.Roles {
devVersion, err := role.GetRoleDevVersion(opinions, tagExtra, f.Version, f)
for _, instanceGroup := range roleManifest.InstanceGroups {
devVersion, err := instanceGroup.GetRoleDevVersion(opinions, tagExtra, f.Version, f)
if err != nil {
return fmt.Errorf("Error creating role checksum: %s", err.Error())
return fmt.Errorf("Error creating instance group checksum: %s", err.Error())
}

imageName := builder.GetRoleDevImageName(registry, organization, repository, role, devVersion)
imageName := builder.GetRoleDevImageName(registry, organization, repository, instanceGroup, devVersion)

if !existingOnDocker {
f.UI.Println(imageName)
Expand Down Expand Up @@ -1033,9 +1033,9 @@ func (f *Fissile) writeHelmNode(dirName, fileName string, node helm.Node) error
return err
}

func (f *Fissile) generateBoshTaskRole(outputFile *os.File, role *model.Role, settings kube.ExportSettings) error {
if role.HasTag(model.RoleTagStopOnFailure) {
pod, err := kube.NewPod(role, settings, f)
func (f *Fissile) generateBoshTaskRole(outputFile *os.File, instanceGroup *model.InstanceGroup, settings kube.ExportSettings) error {
if instanceGroup.HasTag(model.RoleTagStopOnFailure) {
pod, err := kube.NewPod(instanceGroup, settings, f)
if err != nil {
return err
}
Expand All @@ -1044,7 +1044,7 @@ func (f *Fissile) generateBoshTaskRole(outputFile *os.File, role *model.Role, se
return err
}
} else {
job, err := kube.NewJob(role, settings, f)
job, err := kube.NewJob(instanceGroup, settings, f)
if err != nil {
return err
}
Expand All @@ -1057,10 +1057,10 @@ func (f *Fissile) generateBoshTaskRole(outputFile *os.File, role *model.Role, se
return nil
}

// roleHasStorage returns true if a given role uses shared or
// instanceGroupHasStorage returns true if a given group uses shared or
// persistent volumes.
func (f *Fissile) roleHasStorage(role *model.Role) bool {
for _, volume := range role.Run.Volumes {
func (f *Fissile) instanceGroupHasStorage(instanceGroup *model.InstanceGroup) bool {
for _, volume := range instanceGroup.Run.Volumes {
switch volume.Type {
case model.VolumeTypePersistent, model.VolumeTypeShared:
return true
Expand All @@ -1070,15 +1070,15 @@ func (f *Fissile) roleHasStorage(role *model.Role) bool {
}

func (f *Fissile) generateKubeRoles(settings kube.ExportSettings) error {
for _, role := range settings.RoleManifest.Roles {
if role.IsColocatedContainerRole() {
for _, instanceGroup := range settings.RoleManifest.InstanceGroups {
if instanceGroup.IsColocated() {
continue
}
if settings.CreateHelmChart && role.Run.FlightStage == model.FlightStageManual {
if settings.CreateHelmChart && instanceGroup.Run.FlightStage == model.FlightStageManual {
continue
}

subDir := string(role.Type)
subDir := string(instanceGroup.Type)
if settings.CreateHelmChart {
subDir = "templates"
}
Expand All @@ -1087,11 +1087,11 @@ func (f *Fissile) generateKubeRoles(settings kube.ExportSettings) error {
if err != nil {
return err
}
outputPath := filepath.Join(roleTypeDir, fmt.Sprintf("%s.yaml", role.Name))
outputPath := filepath.Join(roleTypeDir, fmt.Sprintf("%s.yaml", instanceGroup.Name))

f.UI.Printf("Writing config %s for role %s\n",
color.CyanString(outputPath),
color.CyanString(role.Name),
color.CyanString(instanceGroup.Name),
)

outputFile, err := os.Create(outputPath)
Expand All @@ -1100,17 +1100,17 @@ func (f *Fissile) generateKubeRoles(settings kube.ExportSettings) error {
}
defer outputFile.Close()

switch role.Type {
switch instanceGroup.Type {
case model.RoleTypeBoshTask:
err := f.generateBoshTaskRole(outputFile, role, settings)
err := f.generateBoshTaskRole(outputFile, instanceGroup, settings)
if err != nil {
return err
}

case model.RoleTypeBosh:
enc := helm.NewEncoder(outputFile)

statefulSet, deps, err := kube.NewStatefulSet(role, settings, f)
statefulSet, deps, err := kube.NewStatefulSet(instanceGroup, settings, f)
if err != nil {
return err
}
Expand Down
8 changes: 4 additions & 4 deletions app/fissile_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -577,20 +577,20 @@ func TestFissileSelectRolesToBuild(t *testing.T) {
},
{
roleNames: []string{"missing_role"},
err: "Some roles are unknown: [missing_role]",
err: "Some instance groups are unknown: [missing_role]",
},
}

for _, sample := range testSamples {
t.Run(strings.Join(sample.roleNames, ","), func(t *testing.T) {
results, err := roleManifest.SelectRoles(sample.roleNames)
results, err := roleManifest.SelectInstanceGroups(sample.roleNames)
if sample.err != "" {
assert.EqualError(t, err, sample.err, "while testing %v", sample.roleNames)
} else {
assert.NoError(t, err, "while testing %v", sample.roleNames)
var actualNames []string
for _, role := range results {
actualNames = append(actualNames, role.Name)
for _, instanceGroup := range results {
actualNames = append(actualNames, instanceGroup.Name)
}
sort.Strings(actualNames)
assert.Equal(t, sample.expectedNames, actualNames, "while testing %v", sample.roleNames)
Expand Down
18 changes: 9 additions & 9 deletions app/validation.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,13 +124,13 @@ func checkParentsOfUndefined(p string, bosh propertyDefaults) bool {
}

// collectManifestProperties returns a map merging the global and
// per-role properties/templates into a single structure.
// per-instance-group properties/templates into a single structure.
func collectManifestProperties(roleManifest *model.RoleManifest) map[string]string {
properties := make(map[string]string)

// Per-role properties
for _, role := range roleManifest.Roles {
for property, template := range role.Configuration.Templates {
// Per-instance-group properties
for _, instanceGroup := range roleManifest.InstanceGroups {
for property, template := range instanceGroup.Configuration.Templates {
properties[property] = template
}
}
Expand Down Expand Up @@ -189,13 +189,13 @@ func checkForDuplicatesBetweenManifestAndLight(light map[string]string, roleMani
check[property] = struct{}{}
}

// ... then the per-role properties
for _, role := range roleManifest.Roles {
prefix := fmt.Sprintf("roles[%s].configuration.templates", role.Name)
// ... then the per-instance-group properties
for _, instanceGroup := range roleManifest.InstanceGroups {
prefix := fmt.Sprintf("instance-groups[%s].configuration.templates", instanceGroup.Name)

for property, template := range role.Configuration.Templates {
for property, template := range instanceGroup.Configuration.Templates {
// Skip over duplicates of the global
// properties in the per-role data, we already
// properties in the per-instance-group data, we already
// checked them, see above.
if _, ok := check[property]; ok {
continue
Expand Down
2 changes: 1 addition & 1 deletion app/validation_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ func TestValidation(t *testing.T) {
`properties.tor.masked_opinion: Forbidden: Dark opinion found in light opinions`,
// checkForDuplicatesBetweenManifestAndLight
`configuration.templates[properties.tor.hostname]: Forbidden: Role-manifest overrides opinion, remove opinion`,
`roles[myrole].configuration.templates[properties.tor.bogus]: Forbidden: Role-manifest duplicates opinion, remove from manifest`,
`instance-groups[myrole].configuration.templates[properties.tor.bogus]: Forbidden: Role-manifest duplicates opinion, remove from manifest`,
// checkForUndefinedBOSHProperties light, manifest - For the bogus property used above for checkOverridden
`role-manifest 'tor.bogus': Not found: "In any BOSH release"`,
`light opinion 'tor.bogus': Not found: "In any BOSH release"`,
Expand Down
22 changes: 11 additions & 11 deletions builder/packages_image.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -176,21 +176,21 @@ func (p *PackagesImageBuilder) determinePackagesLayerBaseImage(packages model.Pa
}

// NewDockerPopulator returns a function which can populate a tar stream with the docker context to build the packages layer image with
func (p *PackagesImageBuilder) NewDockerPopulator(roles model.Roles, labels map[string]string, forceBuildAll bool) func(*tar.Writer) error {
func (p *PackagesImageBuilder) NewDockerPopulator(instanceGroups model.InstanceGroups, labels map[string]string, forceBuildAll bool) func(*tar.Writer) error {
return func(tarWriter *tar.Writer) error {
var err error
if len(roles) == 0 {
return fmt.Errorf("No roles to build")
if len(instanceGroups) == 0 {
return fmt.Errorf("No instance groups to build")
}

// Collect compiled packages
foundFingerprints := make(map[string]struct{})
var packages model.Packages
for _, role := range roles {
for _, roleJob := range role.RoleJobs {
for _, pkg := range roleJob.Packages {
for _, instanceGroup := range instanceGroups {
for _, jobReference := range instanceGroup.JobReferences {
for _, pkg := range jobReference.Packages {
if _, ok := foundFingerprints[pkg.Fingerprint]; ok {
// Package has already been found (possibly due to a different role)
// Package has already been found (possibly due to a different instance group)
continue
}
packages = append(packages, pkg)
Expand Down Expand Up @@ -266,12 +266,12 @@ func (p *PackagesImageBuilder) generateDockerfile(baseImage string, packages mod
return dockerfileTemplate.Execute(outputFile, context)
}

// GetPackagesLayerImageName generates a docker image name for the amalgamation holding all packages used in the specified roles
func (p *PackagesImageBuilder) GetPackagesLayerImageName(roleManifest *model.RoleManifest, roles model.Roles, grapher util.ModelGrapher) (string, error) {
// GetPackagesLayerImageName generates a docker image name for the amalgamation holding all packages used in the specified instance group
func (p *PackagesImageBuilder) GetPackagesLayerImageName(roleManifest *model.RoleManifest, instanceGroups model.InstanceGroups, grapher util.ModelGrapher) (string, error) {
// Get the list of packages; use the fingerprint to ensure we have no repeats
pkgMap := make(map[string]*model.Package)
for _, r := range roles {
for _, j := range r.RoleJobs {
for _, r := range instanceGroups {
for _, j := range r.JobReferences {
for _, pkg := range j.Packages {
pkgMap[pkg.Fingerprint] = pkg
}
Expand Down
Loading

0 comments on commit 711088f

Please sign in to comment.