implement full-sync for repositories

README.md

@@ -36,9 +36,13 @@ Whenever Aquayman synchronizes an organization, it will perform these steps:
 
    1. Find a matching repository configuration, based on the name. This can be
       either an exact match, or a glob expression match.
-   2. If no configuration is found, ignore the repository.
+   2. If no configuration is found, delete the repository if Aquayman runs with
+      `-delete-repos`. Otherwise leave the repository alone.
    3. Otherwise, adjust the assigned teams and individual users/robots.
 
+4. If running with `-create-repos`, list all configured repositories from the YAML
+   file. Create and initialize all not yet existing repositories.
+
 ## Usage
 
 You need an OAuth2 token to authenticate against the API. In your organization settings
@@ -144,6 +148,11 @@ aquayman -config myconfig.yaml -confirm
 2020/04/16 23:32:12 ✓ Permissions successfully synchronized.
 ```
 
+Note that repositories by default can freely exist without being configured in Aquayman.
+This is meant as a safe default, so introducing Aquayman in an existing organiztion is
+easier. To fully synchronize (delete dangling and create missing) repositories, run
+Aquayman with `-create-repos` and `-delete-repos`.
+
 ## Troubleshooting
 
 If you encounter issues [file an issue][1] or talk to us on the [#kubermatic-labs channel][12] on the [Kubermatic Slack][15].

config.example.yaml

@@ -21,14 +21,23 @@ teams:
 # each item here is treated as a glob pattern, but during
 # matching an exact match is preferred. If to items match
 # a given repository, the longest match wins.
-# In contrast to teams and robots, there does not need to
-# be a rule for every repository. Repositories that are
-# not matched will be ignored during synchronization.
+# By default, repositories on quay.io can exist without
+# being mentioned here. If Aquayman runs with -create-repos,
+# it will also create every missing repo (unless it's a
+# wildcard repository, i.e. contains "*"). When running with
+# -delete-repos, repositories not on this list are deleted
+# on quay.io. Running with both flags effetively gives you
+# a full sync.
 repositories:
     # This is effectively a "fallback" that applies to
     # all repositories that have no more spcific configurations
     # (like "myapp").
   - name: '*'
+    # The repository's visibility, one of public or private;
+    # this must be configured.
+    visibility: public
+    # The repository's optional description.
+    description: 'The best repository ever!'
     # A mapping of team names to their roles in this repository;
     # possible roles are read, write or admin.
     # Teams not listed here will be removed from the repository.

main.go

@@ -23,17 +23,23 @@ var (
 func main() {
 	ctx := context.Background()
 
-	configFile := ""
-	showVersion := false
-	confirm := false
-	validate := false
-	exportMode := false
+	var (
+		configFile         = ""
+		showVersion        = false
+		confirm            = false
+		validate           = false
+		exportMode         = false
+		createRepositories = false
+		deleteRepositories = false
+	)
 
 	flag.StringVar(&configFile, "config", configFile, "path to the config.yaml")
 	flag.BoolVar(&showVersion, "version", showVersion, "show the Aquayman version and exit")
 	flag.BoolVar(&confirm, "confirm", confirm, "must be set to actually perform any changes on quay.io")
 	flag.BoolVar(&validate, "validate", validate, "validate the given configuration and then exit")
 	flag.BoolVar(&exportMode, "export", exportMode, "export quay.io state and update the config file (-config flag)")
+	flag.BoolVar(&createRepositories, "create-repos", createRepositories, "create repositories listed in the config file but not existing on quay.io yet")
+	flag.BoolVar(&deleteRepositories, "delete-repos", deleteRepositories, "delete repositories on quay.io that are not listed in the config file")
 	flag.Parse()
 
 	if showVersion {
@@ -80,7 +86,12 @@ func main() {
 
 	log.Printf("► Updating organization %s…", cfg.Organization)
 
-	err = sync.Sync(ctx, cfg, client)
+	options := sync.Options{
+		CreateMissingRepositories:  createRepositories,
+		DeleteDanglingRepositories: deleteRepositories,
+	}
+
+	err = sync.Sync(ctx, cfg, client, options)
 	if err != nil {
 		log.Fatalf("⚠ Failed to sync state: %v.", err)
 	}

pkg/config/config.go

@@ -29,9 +29,15 @@ type TeamConfig struct {
 }
 
 type RepositoryConfig struct {
-	Name  string                         `yaml:"name"`
-	Teams map[string]quay.RepositoryRole `yaml:"teams,omitempty"`
-	Users map[string]quay.RepositoryRole `yaml:"users,omitempty"`
+	Name        string                         `yaml:"name"`
+	Visibility  quay.RepositoryVisibility      `yaml:"visibility"`
+	Description string                         `yaml:"description,omitempty"`
+	Teams       map[string]quay.RepositoryRole `yaml:"teams,omitempty"`
+	Users       map[string]quay.RepositoryRole `yaml:"users,omitempty"`
+}
+
+func (c *RepositoryConfig) IsWildcard() bool {
+	return strings.Contains(c.Name, "*")
 }
 
 type RobotConfig struct {
@@ -116,12 +122,20 @@ func (c *Config) Validate() error {
 	}
 
 	repoNames := []string{}
+	visibilities := []string{
+		string(quay.Public),
+		string(quay.Private),
+	}
 
 	for _, repo := range c.Repositories {
 		if util.StringSliceContains(repoNames, repo.Name) {
 			return fmt.Errorf("duplicate repository %q defined", repo.Name)
 		}
 
+		if !util.StringSliceContains(visibilities, string(repo.Visibility)) {
+			return fmt.Errorf("invalid visibility %q for repository %q, must be one of %v", repo.Visibility, repo.Name, visibilities)
+		}
+
 		for teamName, roleName := range repo.Teams {
 			if !validRepositoryRole(roleName) {
 				return fmt.Errorf("role for team %s in repo %q is invalid (%q), must be one of %v", teamName, repo.Name, roleName, quay.AllRepositoryRoles)
@@ -173,7 +187,7 @@ func (c *Config) GetRepositoryConfig(repo string) *RepositoryConfig {
 	var result RepositoryConfig
 
 	for idx, r := range c.Repositories {
-		if !strings.Contains(r.Name, "*") || len(r.Name) < longestMatch {
+		if !r.IsWildcard() || len(r.Name) < longestMatch {
 			continue
 		}
 

pkg/export/export.go

@@ -59,12 +59,12 @@ func exportRepositories(ctx context.Context, client *quay.Client, cfg *config.Co
 	}
 
 	for _, repo := range repos {
-		visibility := ""
+		visibilitySuffix := ""
 		if !repo.IsPublic {
-			visibility = " (private)"
+			visibilitySuffix = " (private)"
 		}
 
-		log.Printf("  ⚒ %s%s", repo.Name, visibility)
+		log.Printf("  ⚒ %s%s", repo.Name, visibilitySuffix)
 
 		teamPermissions, err := client.GetRepositoryTeamPermissions(ctx, repo.FullName())
 		if err != nil {
@@ -88,10 +88,17 @@ func exportRepositories(ctx context.Context, client *quay.Client, cfg *config.Co
 			users[user.Name] = user.Role
 		}
 
+		visibility := quay.Private
+		if repo.IsPublic {
+			visibility = quay.Public
+		}
+
 		cfg.Repositories = append(cfg.Repositories, config.RepositoryConfig{
-			Name:  repo.Name,
-			Teams: teams,
-			Users: users,
+			Name:        repo.Name,
+			Description: repo.Description,
+			Visibility:  visibility,
+			Teams:       teams,
+			Users:       users,
 		})
 	}
 

pkg/quay/repository.go

@@ -7,6 +7,20 @@ import (
 	"sort"
 )
 
+type RepositoryKind string
+
+const (
+	ImageRepository       RepositoryKind = "image"
+	ApplicationRepository RepositoryKind = "application"
+)
+
+type RepositoryVisibility string
+
+const (
+	Public  RepositoryVisibility = "public"
+	Private RepositoryVisibility = "private"
+)
+
 type Repository struct {
 	Kind        string      `json:"kind"`
 	Name        string      `json:"name"`
@@ -62,3 +76,40 @@ func (c *Client) GetRepositories(ctx context.Context, options GetRepositoriesOpt
 
 	return repositories, err
 }
+
+type CreateRepositoryOptions struct {
+	Kind        RepositoryKind       `json:"kind"`
+	Namespace   string               `json:"namespace"`
+	Repository  string               `json:"repository"`
+	Visibility  RepositoryVisibility `json:"visibility"`
+	Description string               `json:"description"`
+}
+
+func (c *Client) CreateRepository(ctx context.Context, opt CreateRepositoryOptions) error {
+	return c.call(ctx, "POST", "/repository", nil, toBody(opt), nil)
+}
+
+type UpdateRepositoryOptions struct {
+	Description string `json:"description"`
+}
+
+func (c *Client) UpdateRepository(ctx context.Context, repo string, opt UpdateRepositoryOptions) error {
+	return c.call(ctx, "PUT", fmt.Sprintf("/repository/%s", repo), nil, toBody(opt), nil)
+}
+
+type changeRepositoryVisibilityBody struct {
+	Visibility RepositoryVisibility `json:"visibility"`
+}
+
+func (c *Client) ChangeRepositoryVisibility(ctx context.Context, repo string, visibility RepositoryVisibility) error {
+	url := fmt.Sprintf("/repository/%s/changevisibility", repo)
+	body := toBody(changeRepositoryVisibilityBody{
+		Visibility: visibility,
+	})
+
+	return c.call(ctx, "POST", url, nil, body, nil)
+}
+
+func (c *Client) DeleteRepository(ctx context.Context, repo string) error {
+	return c.call(ctx, "DELETE", fmt.Sprintf("/repository/%s", repo), nil, nil, nil)
+}