diff --git a/README.md b/README.md index 22cc364..ac84ff2 100644 --- a/README.md +++ b/README.md @@ -36,9 +36,13 @@ Whenever Aquayman synchronizes an organization, it will perform these steps: 1. Find a matching repository configuration, based on the name. This can be either an exact match, or a glob expression match. - 2. If no configuration is found, ignore the repository. + 2. If no configuration is found, delete the repository if Aquayman runs with + `-delete-repos`. Otherwise leave the repository alone. 3. Otherwise, adjust the assigned teams and individual users/robots. +4. If running with `-create-repos`, list all configured repositories from the YAML + file. Create and initialize all not yet existing repositories. + ## Usage You need an OAuth2 token to authenticate against the API. In your organization settings @@ -144,6 +148,11 @@ aquayman -config myconfig.yaml -confirm 2020/04/16 23:32:12 ✓ Permissions successfully synchronized. ``` +Note that repositories by default can freely exist without being configured in Aquayman. +This is meant as a safe default, so introducing Aquayman in an existing organiztion is +easier. To fully synchronize (delete dangling and create missing) repositories, run +Aquayman with `-create-repos` and `-delete-repos`. + ## Troubleshooting If you encounter issues [file an issue][1] or talk to us on the [#kubermatic-labs channel][12] on the [Kubermatic Slack][15]. diff --git a/config.example.yaml b/config.example.yaml index 9cdc9a4..98d785c 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -21,14 +21,23 @@ teams: # each item here is treated as a glob pattern, but during # matching an exact match is preferred. If to items match # a given repository, the longest match wins. -# In contrast to teams and robots, there does not need to -# be a rule for every repository. Repositories that are -# not matched will be ignored during synchronization. +# By default, repositories on quay.io can exist without +# being mentioned here. If Aquayman runs with -create-repos, +# it will also create every missing repo (unless it's a +# wildcard repository, i.e. contains "*"). When running with +# -delete-repos, repositories not on this list are deleted +# on quay.io. Running with both flags effetively gives you +# a full sync. repositories: # This is effectively a "fallback" that applies to # all repositories that have no more spcific configurations # (like "myapp"). - name: '*' + # The repository's visibility, one of public or private; + # this must be configured. + visibility: public + # The repository's optional description. + description: 'The best repository ever!' # A mapping of team names to their roles in this repository; # possible roles are read, write or admin. # Teams not listed here will be removed from the repository. diff --git a/main.go b/main.go index 894e3c2..b8ba736 100644 --- a/main.go +++ b/main.go @@ -23,17 +23,23 @@ var ( func main() { ctx := context.Background() - configFile := "" - showVersion := false - confirm := false - validate := false - exportMode := false + var ( + configFile = "" + showVersion = false + confirm = false + validate = false + exportMode = false + createRepositories = false + deleteRepositories = false + ) flag.StringVar(&configFile, "config", configFile, "path to the config.yaml") flag.BoolVar(&showVersion, "version", showVersion, "show the Aquayman version and exit") flag.BoolVar(&confirm, "confirm", confirm, "must be set to actually perform any changes on quay.io") flag.BoolVar(&validate, "validate", validate, "validate the given configuration and then exit") flag.BoolVar(&exportMode, "export", exportMode, "export quay.io state and update the config file (-config flag)") + flag.BoolVar(&createRepositories, "create-repos", createRepositories, "create repositories listed in the config file but not existing on quay.io yet") + flag.BoolVar(&deleteRepositories, "delete-repos", deleteRepositories, "delete repositories on quay.io that are not listed in the config file") flag.Parse() if showVersion { @@ -80,7 +86,12 @@ func main() { log.Printf("► Updating organization %s…", cfg.Organization) - err = sync.Sync(ctx, cfg, client) + options := sync.Options{ + CreateMissingRepositories: createRepositories, + DeleteDanglingRepositories: deleteRepositories, + } + + err = sync.Sync(ctx, cfg, client, options) if err != nil { log.Fatalf("⚠ Failed to sync state: %v.", err) } diff --git a/pkg/config/config.go b/pkg/config/config.go index f56731c..da2fca5 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -29,9 +29,15 @@ type TeamConfig struct { } type RepositoryConfig struct { - Name string `yaml:"name"` - Teams map[string]quay.RepositoryRole `yaml:"teams,omitempty"` - Users map[string]quay.RepositoryRole `yaml:"users,omitempty"` + Name string `yaml:"name"` + Visibility quay.RepositoryVisibility `yaml:"visibility"` + Description string `yaml:"description,omitempty"` + Teams map[string]quay.RepositoryRole `yaml:"teams,omitempty"` + Users map[string]quay.RepositoryRole `yaml:"users,omitempty"` +} + +func (c *RepositoryConfig) IsWildcard() bool { + return strings.Contains(c.Name, "*") } type RobotConfig struct { @@ -116,12 +122,20 @@ func (c *Config) Validate() error { } repoNames := []string{} + visibilities := []string{ + string(quay.Public), + string(quay.Private), + } for _, repo := range c.Repositories { if util.StringSliceContains(repoNames, repo.Name) { return fmt.Errorf("duplicate repository %q defined", repo.Name) } + if !util.StringSliceContains(visibilities, string(repo.Visibility)) { + return fmt.Errorf("invalid visibility %q for repository %q, must be one of %v", repo.Visibility, repo.Name, visibilities) + } + for teamName, roleName := range repo.Teams { if !validRepositoryRole(roleName) { return fmt.Errorf("role for team %s in repo %q is invalid (%q), must be one of %v", teamName, repo.Name, roleName, quay.AllRepositoryRoles) @@ -173,7 +187,7 @@ func (c *Config) GetRepositoryConfig(repo string) *RepositoryConfig { var result RepositoryConfig for idx, r := range c.Repositories { - if !strings.Contains(r.Name, "*") || len(r.Name) < longestMatch { + if !r.IsWildcard() || len(r.Name) < longestMatch { continue } diff --git a/pkg/export/export.go b/pkg/export/export.go index 2e9f10d..30fea68 100644 --- a/pkg/export/export.go +++ b/pkg/export/export.go @@ -59,12 +59,12 @@ func exportRepositories(ctx context.Context, client *quay.Client, cfg *config.Co } for _, repo := range repos { - visibility := "" + visibilitySuffix := "" if !repo.IsPublic { - visibility = " (private)" + visibilitySuffix = " (private)" } - log.Printf(" ⚒ %s%s", repo.Name, visibility) + log.Printf(" ⚒ %s%s", repo.Name, visibilitySuffix) teamPermissions, err := client.GetRepositoryTeamPermissions(ctx, repo.FullName()) if err != nil { @@ -88,10 +88,17 @@ func exportRepositories(ctx context.Context, client *quay.Client, cfg *config.Co users[user.Name] = user.Role } + visibility := quay.Private + if repo.IsPublic { + visibility = quay.Public + } + cfg.Repositories = append(cfg.Repositories, config.RepositoryConfig{ - Name: repo.Name, - Teams: teams, - Users: users, + Name: repo.Name, + Description: repo.Description, + Visibility: visibility, + Teams: teams, + Users: users, }) } diff --git a/pkg/quay/repository.go b/pkg/quay/repository.go index 98f9748..d96b944 100644 --- a/pkg/quay/repository.go +++ b/pkg/quay/repository.go @@ -7,6 +7,20 @@ import ( "sort" ) +type RepositoryKind string + +const ( + ImageRepository RepositoryKind = "image" + ApplicationRepository RepositoryKind = "application" +) + +type RepositoryVisibility string + +const ( + Public RepositoryVisibility = "public" + Private RepositoryVisibility = "private" +) + type Repository struct { Kind string `json:"kind"` Name string `json:"name"` @@ -62,3 +76,40 @@ func (c *Client) GetRepositories(ctx context.Context, options GetRepositoriesOpt return repositories, err } + +type CreateRepositoryOptions struct { + Kind RepositoryKind `json:"kind"` + Namespace string `json:"namespace"` + Repository string `json:"repository"` + Visibility RepositoryVisibility `json:"visibility"` + Description string `json:"description"` +} + +func (c *Client) CreateRepository(ctx context.Context, opt CreateRepositoryOptions) error { + return c.call(ctx, "POST", "/repository", nil, toBody(opt), nil) +} + +type UpdateRepositoryOptions struct { + Description string `json:"description"` +} + +func (c *Client) UpdateRepository(ctx context.Context, repo string, opt UpdateRepositoryOptions) error { + return c.call(ctx, "PUT", fmt.Sprintf("/repository/%s", repo), nil, toBody(opt), nil) +} + +type changeRepositoryVisibilityBody struct { + Visibility RepositoryVisibility `json:"visibility"` +} + +func (c *Client) ChangeRepositoryVisibility(ctx context.Context, repo string, visibility RepositoryVisibility) error { + url := fmt.Sprintf("/repository/%s/changevisibility", repo) + body := toBody(changeRepositoryVisibilityBody{ + Visibility: visibility, + }) + + return c.call(ctx, "POST", url, nil, body, nil) +} + +func (c *Client) DeleteRepository(ctx context.Context, repo string) error { + return c.call(ctx, "DELETE", fmt.Sprintf("/repository/%s", repo), nil, nil, nil) +} diff --git a/pkg/sync/sync.go b/pkg/sync/sync.go index ad69668..639c1e1 100644 --- a/pkg/sync/sync.go +++ b/pkg/sync/sync.go @@ -10,7 +10,16 @@ import ( "github.com/kubermatic-labs/aquayman/pkg/util" ) -func Sync(ctx context.Context, config *config.Config, client *quay.Client) error { +type Options struct { + CreateMissingRepositories bool + DeleteDanglingRepositories bool +} + +func DefaultOptions() Options { + return Options{} +} + +func Sync(ctx context.Context, config *config.Config, client *quay.Client, options Options) error { if err := syncRobots(ctx, config, client); err != nil { return fmt.Errorf("failed to sync robots: %v", err) } @@ -19,7 +28,7 @@ func Sync(ctx context.Context, config *config.Config, client *quay.Client) error return fmt.Errorf("failed to sync teams: %v", err) } - if err := syncRepositories(ctx, config, client); err != nil { + if err := syncRepositories(ctx, config, client, options); err != nil { return fmt.Errorf("failed to sync repositories: %v", err) } @@ -125,9 +134,9 @@ func syncTeamMembers(ctx context.Context, config *config.Config, client *quay.Cl ) if !client.Dry { - yesPlase := true + yesPlease := true getTeamOptions := quay.GetTeamMembersOptions{ - IncludePending: &yesPlase, + IncludePending: &yesPlease, } currentMembers, err = client.GetTeamMembers(ctx, config.Organization, team.Name, getTeamOptions) @@ -163,53 +172,116 @@ func syncTeamMembers(ctx context.Context, config *config.Config, client *quay.Cl return nil } -func syncRepositories(ctx context.Context, config *config.Config, client *quay.Client) error { +func syncRepositories(ctx context.Context, config *config.Config, client *quay.Client, options Options) error { log.Println("⇄ Syncing repositories…") - options := quay.GetRepositoriesOptions{ + requestOptions := quay.GetRepositoriesOptions{ Namespace: config.Organization, } - repositories, err := client.GetRepositories(ctx, options) + currentRepos, err := client.GetRepositories(ctx, requestOptions) if err != nil { return fmt.Errorf("failed to retrieve repositories: %v", err) } - for _, repo := range repositories { - if err := syncRepository(ctx, config, client, repo); err != nil { + // update/delete existing repos + currentRepoNames := []string{} + for _, repo := range currentRepos { + repoConfig := config.GetRepositoryConfig(repo.Name) + if repoConfig == nil { + if options.DeleteDanglingRepositories { + log.Printf(" - ⚒ %s", repo.Name) + if err := client.DeleteRepository(ctx, repo.FullName()); err != nil { + return err + } + } + + continue + } + + log.Printf(" ✎ ⚒ %s", repo.Name) + if err := syncRepository(ctx, client, repo, repoConfig); err != nil { return err } + + currentRepoNames = append(currentRepoNames, repo.Name) + } + + // create missing repos on quay.io + if options.CreateMissingRepositories { + for _, repoConfig := range config.Repositories { + // ignore wildcard rules + if repoConfig.IsWildcard() { + continue + } + + if !util.StringSliceContains(currentRepoNames, repoConfig.Name) { + log.Printf(" + ⚒ %s", repoConfig.Name) + + options := quay.CreateRepositoryOptions{ + Namespace: config.Organization, + Repository: repoConfig.Name, + Description: repoConfig.Description, + Visibility: repoConfig.Visibility, + } + + if err := client.CreateRepository(ctx, options); err != nil { + return err + } + + // doing it like this instead of GETing the repo after creation makes it + // safe for running in dry mode + repo := quay.Repository{ + Namespace: config.Organization, + Name: repoConfig.Name, + IsPublic: repoConfig.Visibility == quay.Public, + Description: repoConfig.Description, + } + + if err := syncRepository(ctx, client, repo, &repoConfig); err != nil { + return err + } + } + } } return nil } -func syncRepository(ctx context.Context, config *config.Config, client *quay.Client, repo quay.Repository) error { - // ignore repos for which we have no matching rule set - repoConfig := config.GetRepositoryConfig(repo.Name) - if repoConfig == nil { - return nil +func syncRepository(ctx context.Context, client *quay.Client, repo quay.Repository, repoConfig *config.RepositoryConfig) error { + // ensure repos are not public if they are configured to be private (phrase it like this + // just in case quay ever introduces a third visibility state) + if repo.IsPublic && repoConfig.Visibility != quay.Public { + log.Printf(" - set visibility to %s", repoConfig.Visibility) + if err := client.ChangeRepositoryVisibility(ctx, repo.FullName(), repoConfig.Visibility); err != nil { + return fmt.Errorf("failed to set visibility: %v", err) + } } - visibility := "" - if !repo.IsPublic { - visibility = " (private)" - } + if repo.Description != repoConfig.Description { + options := quay.UpdateRepositoryOptions{ + Description: repoConfig.Description, + } - log.Printf(" ⚒ %s%s", repo.Name, visibility) + if err := client.UpdateRepository(ctx, repo.FullName(), options); err != nil { + return fmt.Errorf("failed to update description: %v", err) + } + } - if err := syncRepositoryTeams(ctx, config, client, repo.FullName(), repoConfig); err != nil { - return err + if err := syncRepositoryTeams(ctx, client, repo.FullName(), repoConfig); err != nil { + return fmt.Errorf("failed to teams: %v", err) } - if err := syncRepositoryUsers(ctx, config, client, repo.FullName(), repoConfig); err != nil { - return err + if err := syncRepositoryUsers(ctx, client, repo.FullName(), repoConfig); err != nil { + return fmt.Errorf("failed to users: %v", err) } return nil } -func syncRepositoryTeams(ctx context.Context, config *config.Config, client *quay.Client, fullRepoName string, repo *config.RepositoryConfig) error { +func syncRepositoryTeams(ctx context.Context, client *quay.Client, fullRepoName string, repo *config.RepositoryConfig) error { + // amazingly, this API call does not fail if the repo does not exist, so we can + // perform it even in dry mode currentTeams, err := client.GetRepositoryTeamPermissions(ctx, fullRepoName) if err != nil { return fmt.Errorf("failed to get team permissions: %v", err) @@ -249,7 +321,9 @@ func syncRepositoryTeams(ctx context.Context, config *config.Config, client *qua return nil } -func syncRepositoryUsers(ctx context.Context, config *config.Config, client *quay.Client, fullRepoName string, repo *config.RepositoryConfig) error { +func syncRepositoryUsers(ctx context.Context, client *quay.Client, fullRepoName string, repo *config.RepositoryConfig) error { + // amazingly, this API call does not fail if the repo does not exist, so we can + // perform it even in dry mode currentUsers, err := client.GetRepositoryUserPermissions(ctx, fullRepoName) if err != nil { return fmt.Errorf("failed to get user permissions: %v", err)