DEVPROD-10294 amboy url in its own config section

Dockerfile-seeded-mongo

@@ -21,5 +21,5 @@ EOF
 
 COPY <<EOF /docker-entrypoint-initdb.d/02_replace-amboy-url.js
 db = db.getSiblingDB( "${DB_NAME}" );
-db.admin.updateOne({_id:"amboy"}, {"\$set": {"db_connection.url": "mongodb://mongo:27017"}})
+db.admin.updateOne({_id:"amboy_db"}, {"\$set": {"url": "mongodb://mongo:27017"}})
 EOF

config.go

@@ -63,6 +63,7 @@ type ConfigSection interface {
 type Settings struct {
 	Id                  string                    `bson:"_id" json:"id" yaml:"id"`
 	Amboy               AmboyConfig               `yaml:"amboy" bson:"amboy" json:"amboy" id:"amboy"`
+	AmboyDB             AmboyDBConfig             `yaml:"amboy_db" bson:"amboy_db" json:"amboy_db" id:"amboy_db"`
 	Api                 APIConfig                 `yaml:"api" bson:"api" json:"api" id:"api"`
 	AuthConfig          AuthConfig                `yaml:"auth" bson:"auth" json:"auth" id:"auth"`
 	AWSInstanceRole     string                    `yaml:"aws_instance_role" bson:"aws_instance_role" json:"aws_instance_role"`

config_amboy.go

@@ -16,7 +16,6 @@ import (
 type AmboyConfig struct {
 	Name                                  string                  `bson:"name" json:"name" yaml:"name"`
 	SingleName                            string                  `bson:"single_name" json:"single_name" yaml:"single_name"`
-	DBConnection                          AmboyDBConfig           `bson:"db_connection" json:"db_connection" yaml:"db_connection"`
 	PoolSizeLocal                         int                     `bson:"pool_size_local" json:"pool_size_local" yaml:"pool_size_local"`
 	PoolSizeRemote                        int                     `bson:"pool_size_remote" json:"pool_size_remote" yaml:"pool_size_remote"`
 	LocalStorage                          int                     `bson:"local_storage_size" json:"local_storage_size" yaml:"local_storage_size"`
@@ -35,12 +34,6 @@ type AmboyConfig struct {
 	SkipPreferredIndexes bool `bson:"skip_preferred_indexes" json:"skip_preferred_indexes" yaml:"skip_preferred_indexes"`
 }
 
-// AmboyDBConfig configures Amboy's database connection.
-type AmboyDBConfig struct {
-	URL      string `bson:"url" json:"url" yaml:"url"`
-	Database string `bson:"database" json:"database" yaml:"database"`
-}
-
 // AmboyRetryConfig represents configuration settings for Amboy's retryability
 // feature.
 type AmboyRetryConfig struct {
@@ -65,7 +58,6 @@ type AmboyNamedQueueConfig struct {
 var (
 	amboyNameKey                                  = bsonutil.MustHaveTag(AmboyConfig{}, "Name")
 	amboySingleNameKey                            = bsonutil.MustHaveTag(AmboyConfig{}, "SingleName")
-	amboyDBConnectionKey                          = bsonutil.MustHaveTag(AmboyConfig{}, "DBConnection")
 	amboyPoolSizeLocalKey                         = bsonutil.MustHaveTag(AmboyConfig{}, "PoolSizeLocal")
 	amboyPoolSizeRemoteKey                        = bsonutil.MustHaveTag(AmboyConfig{}, "PoolSizeRemote")
 	amboyLocalStorageKey                          = bsonutil.MustHaveTag(AmboyConfig{}, "LocalStorage")
@@ -90,7 +82,6 @@ func (c *AmboyConfig) Set(ctx context.Context) error {
 		"$set": bson.M{
 			amboyNameKey:                                  c.Name,
 			amboySingleNameKey:                            c.SingleName,
-			amboyDBConnectionKey:                          c.DBConnection,
 			amboyPoolSizeLocalKey:                         c.PoolSizeLocal,
 			amboyPoolSizeRemoteKey:                        c.PoolSizeRemote,
 			amboyLocalStorageKey:                          c.LocalStorage,
@@ -145,10 +136,6 @@ func (c *AmboyConfig) ValidateAndDefault() error {
 		c.SingleName = defaultSingleAmboyQueueName
 	}
 
-	if c.DBConnection.Database == "" {
-		c.DBConnection.Database = defaultAmboyDBName
-	}
-
 	if c.PoolSizeLocal == 0 {
 		c.PoolSizeLocal = defaultAmboyPoolSize
 	}

config_amboy_db.go

@@ -0,0 +1,42 @@
+package evergreen
+
+import (
+	"context"
+
+	"github.com/mongodb/anser/bsonutil"
+	"github.com/pkg/errors"
+	"go.mongodb.org/mongo-driver/bson"
+)
+
+// AmboyDBConfig configures Amboy's database connection.
+type AmboyDBConfig struct {
+	URL      string `bson:"url" json:"url" yaml:"url"`
+	Database string `bson:"database" json:"database" yaml:"database"`
+}
+
+func (c *AmboyDBConfig) SectionId() string { return "amboy_db" }
+
+var (
+	amboyDBURLKey      = bsonutil.MustHaveTag(AmboyDBConfig{}, "URL")
+	amboyDBDatabaseKey = bsonutil.MustHaveTag(AmboyDBConfig{}, "Database")
+)
+
+func (c *AmboyDBConfig) Get(ctx context.Context) error {
+	return getConfigSection(ctx, c)
+}
+
+func (c *AmboyDBConfig) Set(ctx context.Context) error {
+	return errors.Wrapf(setConfigSection(ctx, c.SectionId(), bson.M{
+		"$set": bson.M{
+			amboyDBURLKey:      c.URL,
+			amboyDBDatabaseKey: c.Database,
+		}}), "updating config section '%s'", c.SectionId(),
+	)
+}
+
+func (c *AmboyDBConfig) ValidateAndDefault() error {
+	if c.Database == "" {
+		c.Database = defaultAmboyDBName
+	}
+	return nil
+}

config_test.go

@@ -212,12 +212,8 @@ func (s *AdminSuite) TestAmboyConfig() {
 	defer cancel()
 
 	config := AmboyConfig{
-		Name:       "amboy",
-		SingleName: "single",
-		DBConnection: AmboyDBConfig{
-			URL:      "mongodb://localhost:27017",
-			Database: "db",
-		},
+		Name:                                  "amboy",
+		SingleName:                            "single",
 		PoolSizeLocal:                         10,
 		PoolSizeRemote:                        20,
 		LocalStorage:                          30,
@@ -237,6 +233,23 @@ func (s *AdminSuite) TestAmboyConfig() {
 	s.Equal(config, settings.Amboy)
 }
 
+func (s *AdminSuite) TestAdminDBConfig() {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	config := AmboyDBConfig{
+		URL:      "mongodb://localhost:27017",
+		Database: "db",
+	}
+
+	err := config.Set(ctx)
+	s.NoError(err)
+	settings, err := GetConfig(ctx)
+	s.NoError(err)
+	s.NotNil(settings)
+	s.Equal(config, settings.AmboyDB)
+}
+
 func (s *AdminSuite) TestApiConfig() {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()

config_test/evg_settings.yml

@@ -17,15 +17,16 @@ configdir: "config_test"
 
 amboy:
   name: test_queue
-  db_connection:
-    database: amboy_test
-    url: "mongodb://localhost:27017"
   pool_size_local: 2
   pool_size_remote: 2
   local_storage_size: 2048
   # Ensure that tests depending on the global environment don't depend on indexes to exist in a testing DB.
   skip_preferred_indexes: true
 
+amboy_db:
+  database: amboy_test
+  url: "mongodb://localhost:27017"
+
 providers:
   aws:
     ec2_keys:

config_test/evg_settings_with_3rd_party_defaults.yml

@@ -8,13 +8,14 @@ configdir: "config_test"
 
 amboy:
   name: test_queue
-  db_connection:
-    database: amboy_test
-    url: "mongodb://localhost:27017"
   pool_size_local: 2
   pool_size_remote: 2
   local_storage_size: 2048
 
+amboy_db:
+  database: amboy_test
+  url: "mongodb://localhost:27017"
+
 providers:
   aws:
     aws_secret: "aws secret"

environment.go

@@ -397,7 +397,7 @@ func (e *envState) createRemoteQueues(ctx context.Context, tracer trace.Tracer)
 	ctx, span := tracer.Start(ctx, "CreateRemoteQueues")
 	defer span.End()
 
-	url := e.settings.Amboy.DBConnection.URL
+	url := e.settings.AmboyDB.URL
 	if url == "" {
 		url = DefaultAmboyDatabaseURL
 	}
@@ -481,7 +481,7 @@ func (e *envState) createApplicationQueue(ctx context.Context, client *mongo.Cli
 	// queue.
 	opts := queue.DefaultMongoDBOptions()
 	opts.Client = client
-	opts.DB = e.settings.Amboy.DBConnection.Database
+	opts.DB = e.settings.AmboyDB.Database
 	opts.Collection = e.settings.Amboy.Name
 	opts.SkipQueueIndexBuilds = true
 	opts.SkipReportingIndexBuilds = true
@@ -559,7 +559,7 @@ func (e *envState) createRemoteQueueGroup(ctx context.Context, client *mongo.Cli
 func (e *envState) getRemoteQueueGroupDBOptions(client *mongo.Client) queue.MongoDBOptions {
 	opts := queue.DefaultMongoDBOptions()
 	opts.Client = client
-	opts.DB = e.settings.Amboy.DBConnection.Database
+	opts.DB = e.settings.AmboyDB.Database
 	opts.Collection = e.settings.Amboy.Name
 	opts.SkipQueueIndexBuilds = true
 	opts.SkipReportingIndexBuilds = true

rest/data/admin_test.go

@@ -118,6 +118,8 @@ func (s *AdminDataSuite) TestSetAndGetSettings() {
 	s.EqualValues(testSettings.Amboy.SampleSize, settingsFromConnector.Amboy.SampleSize)
 	s.EqualValues(testSettings.Amboy.Retry, settingsFromConnector.Amboy.Retry)
 	s.EqualValues(testSettings.Amboy.NamedQueues, settingsFromConnector.Amboy.NamedQueues)
+	s.EqualValues(testSettings.AmboyDB.URL, settingsFromConnector.AmboyDB.URL)
+	s.EqualValues(testSettings.AmboyDB.Database, settingsFromConnector.AmboyDB.Database)
 	s.EqualValues(testSettings.Api.HttpListenAddr, settingsFromConnector.Api.HttpListenAddr)
 	s.EqualValues(testSettings.Api.URL, settingsFromConnector.Api.URL)
 	s.EqualValues(testSettings.AuthConfig.PreferredType, settingsFromConnector.AuthConfig.PreferredType)
@@ -252,6 +254,8 @@ func (s *AdminDataSuite) TestSetAndGetSettings() {
 	s.EqualValues(testSettings.Amboy.SampleSize, settingsFromConnector.Amboy.SampleSize)
 	s.EqualValues(testSettings.Amboy.Retry, settingsFromConnector.Amboy.Retry)
 	s.EqualValues(testSettings.Amboy.NamedQueues, settingsFromConnector.Amboy.NamedQueues)
+	s.EqualValues(testSettings.AmboyDB.URL, settingsFromConnector.AmboyDB.URL)
+	s.EqualValues(testSettings.AmboyDB.Database, settingsFromConnector.AmboyDB.Database)
 	s.EqualValues(testSettings.Api.HttpListenAddr, settingsFromConnector.Api.HttpListenAddr)
 	s.EqualValues(testSettings.Api.URL, settingsFromConnector.Api.URL)
 	s.EqualValues(testSettings.AuthConfig.PreferredType, settingsFromConnector.AuthConfig.PreferredType)

rest/model/admin.go

@@ -13,6 +13,7 @@ import (
 func NewConfigModel() *APIAdminSettings {
 	return &APIAdminSettings{
 		Amboy:               &APIAmboyConfig{},
+		AmboyDB:             &APIAmboyDBConfig{},
 		Api:                 &APIapiConfig{},
 		AuthConfig:          &APIAuthConfig{},
 		Buckets:             &APIBucketsConfig{},
@@ -50,6 +51,7 @@ func NewConfigModel() *APIAdminSettings {
 // APIAdminSettings is the structure of a response to the admin route
 type APIAdminSettings struct {
 	Amboy               *APIAmboyConfig                   `json:"amboy,omitempty"`
+	AmboyDB             *APIAmboyDBConfig                 `json:"amboy_db,omitempty"`
 	Api                 *APIapiConfig                     `json:"api,omitempty"`
 	AWSInstanceRole     *string                           `json:"aws_instance_role,omitempty"`
 	AuthConfig          *APIAuthConfig                    `json:"auth,omitempty"`
@@ -305,7 +307,6 @@ func (a *APISESConfig) ToService() (interface{}, error) {
 type APIAmboyConfig struct {
 	Name                                  *string                    `json:"name"`
 	SingleName                            *string                    `json:"single_name"`
-	DBConnection                          APIAmboyDBConfig           `json:"db_connection"`
 	PoolSizeLocal                         int                        `json:"pool_size_local"`
 	PoolSizeRemote                        int                        `json:"pool_size_remote"`
 	LocalStorage                          int                        `json:"local_storage_size"`
@@ -324,9 +325,6 @@ func (a *APIAmboyConfig) BuildFromService(h interface{}) error {
 	case evergreen.AmboyConfig:
 		a.Name = utility.ToStringPtr(v.Name)
 		a.SingleName = utility.ToStringPtr(v.SingleName)
-		if err := a.DBConnection.BuildFromService(v.DBConnection); err != nil {
-			return errors.Wrap(err, "converting Amboy DB settings to API model")
-		}
 		a.PoolSizeLocal = v.PoolSizeLocal
 		a.PoolSizeRemote = v.PoolSizeRemote
 		a.LocalStorage = v.LocalStorage
@@ -360,23 +358,13 @@ func (a *APIAmboyConfig) ToService() (interface{}, error) {
 		return nil, errors.Errorf("programmatic error: expected Amboy retry config but got type %T", i)
 	}
 
-	i, err = a.DBConnection.ToService()
-	if err != nil {
-		return nil, errors.Wrap(err, "converting Amboy DB settings to service model")
-	}
-	db, ok := i.(evergreen.AmboyDBConfig)
-	if !ok {
-		return nil, errors.Errorf("programmatic error: expected Amboy DB config but got type %T", i)
-	}
-
 	var dbNamedQueues []evergreen.AmboyNamedQueueConfig
 	for _, apiNamedQueue := range a.NamedQueues {
 		dbNamedQueues = append(dbNamedQueues, apiNamedQueue.ToService())
 	}
 	return evergreen.AmboyConfig{
 		Name:                                  utility.FromStringPtr(a.Name),
 		SingleName:                            utility.FromStringPtr(a.SingleName),
-		DBConnection:                          db,
 		PoolSizeLocal:                         a.PoolSizeLocal,
 		PoolSizeRemote:                        a.PoolSizeRemote,
 		LocalStorage:                          a.LocalStorage,
@@ -394,8 +382,6 @@ func (a *APIAmboyConfig) ToService() (interface{}, error) {
 type APIAmboyDBConfig struct {
 	URL      *string `json:"url"`
 	Database *string `json:"database"`
-	Username *string `json:"username"`
-	Password *string `json:"password"`
 }
 
 func (a *APIAmboyDBConfig) BuildFromService(h interface{}) error {

rest/model/admin_test.go

@@ -81,8 +81,6 @@ func TestModelConversion(t *testing.T) {
 	assert.Equal(testSettings.ShutdownWaitSeconds, *apiSettings.ShutdownWaitSeconds)
 
 	assert.EqualValues(testSettings.Amboy.Name, utility.FromStringPtr(apiSettings.Amboy.Name))
-	assert.EqualValues(testSettings.Amboy.DBConnection.URL, utility.FromStringPtr(apiSettings.Amboy.DBConnection.URL))
-	assert.EqualValues(testSettings.Amboy.DBConnection.Database, utility.FromStringPtr(apiSettings.Amboy.DBConnection.Database))
 	assert.EqualValues(testSettings.Amboy.LocalStorage, apiSettings.Amboy.LocalStorage)
 	assert.EqualValues(testSettings.Amboy.GroupDefaultWorkers, apiSettings.Amboy.GroupDefaultWorkers)
 	assert.EqualValues(testSettings.Amboy.GroupBackgroundCreateFrequencyMinutes, apiSettings.Amboy.GroupBackgroundCreateFrequencyMinutes)
@@ -99,6 +97,8 @@ func TestModelConversion(t *testing.T) {
 		assert.Equal(testSettings.Amboy.NamedQueues[i].SampleSize, apiSettings.Amboy.NamedQueues[i].SampleSize)
 		assert.Equal(testSettings.Amboy.NamedQueues[i].LockTimeoutSeconds, apiSettings.Amboy.NamedQueues[i].LockTimeoutSeconds)
 	}
+	assert.EqualValues(testSettings.AmboyDB.URL, utility.FromStringPtr(apiSettings.AmboyDB.URL))
+	assert.EqualValues(testSettings.AmboyDB.Database, utility.FromStringPtr(apiSettings.AmboyDB.Database))
 	assert.EqualValues(testSettings.Api.HttpListenAddr, utility.FromStringPtr(apiSettings.Api.HttpListenAddr))
 	assert.EqualValues(testSettings.Api.URL, utility.FromStringPtr(apiSettings.Api.URL))
 	assert.EqualValues(testSettings.AuthConfig.PreferredType, utility.FromStringPtr(apiSettings.AuthConfig.PreferredType))
@@ -223,7 +223,7 @@ func TestModelConversion(t *testing.T) {
 	assert.NoError(err)
 	dbSettings := dbInterface.(evergreen.Settings)
 	assert.EqualValues(testSettings.Amboy.Name, dbSettings.Amboy.Name)
-	assert.EqualValues(testSettings.Amboy.DBConnection, dbSettings.Amboy.DBConnection)
+	assert.EqualValues(testSettings.AmboyDB, dbSettings.AmboyDB)
 	assert.EqualValues(testSettings.Amboy.LocalStorage, dbSettings.Amboy.LocalStorage)
 	assert.EqualValues(testSettings.Amboy.GroupDefaultWorkers, dbSettings.Amboy.GroupDefaultWorkers)
 	assert.EqualValues(testSettings.Amboy.GroupBackgroundCreateFrequencyMinutes, dbSettings.Amboy.GroupBackgroundCreateFrequencyMinutes)
@@ -240,6 +240,8 @@ func TestModelConversion(t *testing.T) {
 		assert.Equal(testSettings.Amboy.NamedQueues[i].SampleSize, dbSettings.Amboy.NamedQueues[i].SampleSize)
 		assert.Equal(testSettings.Amboy.NamedQueues[i].LockTimeoutSeconds, dbSettings.Amboy.NamedQueues[i].LockTimeoutSeconds)
 	}
+	assert.EqualValues(testSettings.AmboyDB.URL, dbSettings.AmboyDB.URL)
+	assert.EqualValues(testSettings.AmboyDB.Database, dbSettings.AmboyDB.Database)
 	assert.EqualValues(testSettings.Api.HttpListenAddr, dbSettings.Api.HttpListenAddr)
 	assert.EqualValues(testSettings.Api.URL, dbSettings.Api.URL)
 	assert.EqualValues(testSettings.AuthConfig.Naive.Users[0].Username, dbSettings.AuthConfig.Naive.Users[0].Username)

rest/route/admin_test.go

@@ -149,6 +149,8 @@ func (s *AdminRouteSuite) TestAdminRoute() {
 	s.EqualValues(testSettings.Amboy.SampleSize, settings.Amboy.SampleSize)
 	s.EqualValues(testSettings.Amboy.Retry, settings.Amboy.Retry)
 	s.EqualValues(testSettings.Amboy.NamedQueues, settings.Amboy.NamedQueues)
+	s.EqualValues(testSettings.AmboyDB.URL, settings.AmboyDB.URL)
+	s.EqualValues(testSettings.AmboyDB.Database, settings.AmboyDB.Database)
 	s.EqualValues(testSettings.Api.HttpListenAddr, settings.Api.HttpListenAddr)
 	s.EqualValues(testSettings.Api.URL, settings.Api.URL)
 	s.EqualValues(testSettings.AuthConfig.Okta.ClientID, settings.AuthConfig.Okta.ClientID)

service/templates/admin.html

@@ -1521,11 +1521,11 @@ <h2 class="modal-title">[[modalTitle]]</h2>
 									</md-input-container>
 									<md-input-container class="control" style="width:45%;">
 										<label>DB URL</label>
-										<input type="text" ng-model="Settings.amboy.db_connection.url">
+										<input type="text" ng-model="Settings.amboy_db.url">
 									</md-input-container>
 									<md-input-container class="control" style="width:45%;">
 										<label>DB name</label>
-										<input type="text" ng-model="Settings.amboy.db_connection.database">
+										<input type="text" ng-model="Settings.amboy_db.database">
 									</md-input-container>
 									<md-input-container class="control" style="width:45%;">
 										<label>Local pool size</label>

smoke/internal/testdata/admin_settings.yml

@@ -12,12 +12,13 @@ database:
     wmode: majority
 
 amboy:
-  db_connection:
-    url: mongodb://localhost:27017
-    database: amboy_smoke
   name: evg.service
   skip_preferred_indexes: true
 
+amboy_db:
+  database: amboy_test
+  url: "mongodb://localhost:27017"
+
 buckets:
   log_bucket:
     name: "logs"

smoke/internal/testdata/db/admin.json

@@ -1,3 +1,5 @@
 { "_id" : "service_flags", "github_status_api_disabled" : false, "alerts_disabled" : false, "repotracker_disabled" : false, "scheduler_disabled" : false, "check_blocked_tasks_disabled": false, "github_pr_testing_disabled" : false, "repotracker_push_event_disabled" : false, "cli_updates_disabled" : false, "task_dispatch_disabled" : false, "s3_binary_downloads_disabled": true, "large_parser_projects_disabled": true, "monitor_disabled" : false, "notifications_disabled" : false, "taskrunner_disabled" : false, "background_stats_disabled" : true, "event_processing_disabled" : false, "webhook_notifications_disabled" : false, "jira_notifications_disabled" : false, "slack_notifications_disabled" : false, "email_notifications_disabled" : false, "task_logging_disabled" : false, "cache_stats_job_disabled" : false, "agent_start_disabled" : false, "host_init_disabled" : false, "pod_init_disabled": true, "commit_queue_disabled" : false, "cache_stats_endpoint_disabled" : false, "host_allocator_disabled" : false, "pod_allocator_disabled": false, "task_reliability_disabled" : false, "background_reauth_disabled": true, "background_cleanup_disabled": false, "cloud_cleanup_disabled": true, "unrecognized_pod_cleanup_disabled": true, "sleep_schedule_disabled": true}
+{ "_id": "global", "api_url": "http://localhost:9090" }
+{ "_id": "amboy", "name": "evg.service", "skip_preferred_indexes": true }
+{ "_id": "amboy_db", "url": "mongodb://localhost:27017", "database": "amboy_smoke" }
 { "_id": "api", "url": "http://localhost:9090" }
-{ "_id": "amboy", "name": "evg.service", "db_connection": {"url": "mongodb://localhost:27017", "database": "amboy_smoke"}, "skip_preferred_indexes": true }