Skip to content

Commit 65b37d4

Browse files
InfernoJJInfernoJJ
authored
Merge pull request #302 from moov-io/goembed
Go:Embed
2 parents 9ab6386 + cc60a79 commit 65b37d4

File tree

10 files changed

+306
-56
lines changed

10 files changed

+306
-56
lines changed

‎config/config.go‎

Lines changed: 42 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
package config
22

33
import (
4+
"fmt"
5+
"io"
6+
"io/fs"
47
"os"
58
"strings"
69

@@ -29,6 +32,19 @@ func (s *Service) Load(config interface{}) error {
2932
return err
3033
}
3134

35+
return s.MergeEnvironments(config)
36+
}
37+
38+
func (s *Service) LoadFromFS(config interface{}, fs fs.FS) error {
39+
if err := s.LoadEmbeddedFile("configs/config.default.yml", config, fs); err != nil {
40+
return err
41+
}
42+
43+
return s.MergeEnvironments(config)
44+
}
45+
46+
func (s *Service) MergeEnvironments(config interface{}) error {
47+
3248
if err := LoadEnvironmentFile(s.logger, APP_CONFIG, config); err != nil {
3349
return err
3450
}
@@ -49,14 +65,38 @@ func (s *Service) LoadFile(file string, config interface{}) error {
4965
return logger.LogErrorf("pkger unable to load %s: %w", file, err).Err()
5066
}
5167

68+
if err := configFromReader(config, f); err != nil {
69+
return logger.LogError(err).Err()
70+
}
71+
72+
return nil
73+
}
74+
75+
func (s *Service) LoadEmbeddedFile(file string, config interface{}, fs fs.FS) error {
76+
logger := s.logger.Set("file", log.String(file))
77+
logger.Info().Logf("loading config file")
78+
79+
f, err := fs.Open(file)
80+
if err != nil {
81+
return logger.LogErrorf("go:embed FS unable to load %s: %w", file, err).Err()
82+
}
83+
84+
if err := configFromReader(config, f); err != nil {
85+
return logger.LogError(err).Err()
86+
}
87+
88+
return nil
89+
}
90+
91+
func configFromReader(config interface{}, f io.Reader) error {
5292
deflt := viper.New()
5393
deflt.SetConfigType("yaml")
5494
if err := deflt.ReadConfig(f); err != nil {
55-
return logger.LogErrorf("unable to load the defaults: %w", err).Err()
95+
return fmt.Errorf("unable to load the defaults: %w", err)
5696
}
5797

5898
if err := deflt.UnmarshalExact(config, overwriteConfig); err != nil {
59-
return logger.LogErrorf("unable to unmarshal the defaults: %w", err).Err()
99+
return fmt.Errorf("unable to unmarshal the defaults: %w", err)
60100
}
61101

62102
return nil

‎config/config_test.go‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"path/filepath"
66
"testing"
77

8+
"github.com/moov-io/base"
89
"github.com/moov-io/base/config"
910
"github.com/moov-io/base/log"
1011
"github.com/stretchr/testify/require"
@@ -60,3 +61,42 @@ func Test_Load(t *testing.T) {
6061
require.NotNil(t, err)
6162
require.Contains(t, err.Error(), `'Config' has invalid keys: extra`)
6263
}
64+
65+
func Test_Embedded_Load(t *testing.T) {
66+
os.Setenv(config.APP_CONFIG, filepath.Join("..", "configs", "config.app.yml"))
67+
os.Setenv(config.APP_CONFIG_SECRETS, filepath.Join("..", "configs", "config.secrets.yml"))
68+
t.Cleanup(func() {
69+
os.Unsetenv(config.APP_CONFIG)
70+
os.Unsetenv(config.APP_CONFIG_SECRETS)
71+
})
72+
73+
cfg := &GlobalConfigModel{}
74+
75+
service := config.NewService(log.NewDefaultLogger())
76+
err := service.LoadFromFS(cfg, base.ConfigDefaults)
77+
require.Nil(t, err)
78+
79+
require.Equal(t, "default", cfg.Config.Default)
80+
require.Equal(t, "app", cfg.Config.App)
81+
require.Equal(t, "keep secret!", cfg.Config.Secret)
82+
83+
// This test documents some unexpected behavior where slices are merged and not overwritten.
84+
// Slices in secrets should overwrite the slice in app and default.
85+
require.Len(t, cfg.Config.Values, 1)
86+
require.Equal(t, "secret", cfg.Config.Values[0])
87+
88+
require.Equal(t, "", cfg.Config.Zero)
89+
90+
// Verify attempting to load from our default file errors on extra fields
91+
cfg = &GlobalConfigModel{}
92+
err = service.LoadFile("/configs/config.extra.yml", &cfg)
93+
require.NotNil(t, err)
94+
require.Contains(t, err.Error(), `'Config' has invalid keys: extra`)
95+
96+
// Verify attempting to load additional fields via env vars errors out
97+
os.Setenv(config.APP_CONFIG, filepath.Join("..", "configs", "config.extra.yml"))
98+
cfg = &GlobalConfigModel{}
99+
err = service.Load(cfg)
100+
require.NotNil(t, err)
101+
require.Contains(t, err.Error(), `'Config' has invalid keys: extra`)
102+
}

‎database/database.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ func New(ctx context.Context, logger log.Logger, config DatabaseConfig) (*sql.DB
3535
return nil, fmt.Errorf("database config not defined")
3636
}
3737

38-
func NewAndMigrate(ctx context.Context, logger log.Logger, config DatabaseConfig) (*sql.DB, error) {
38+
func NewAndMigrate(ctx context.Context, logger log.Logger, config DatabaseConfig, opts ...MigrateOption) (*sql.DB, error) {
3939
if logger == nil {
4040
logger = log.NewNopLogger()
4141
}
@@ -45,7 +45,7 @@ func NewAndMigrate(ctx context.Context, logger log.Logger, config DatabaseConfig
4545
}
4646

4747
// run migrations first
48-
if err := RunMigrations(logger, config); err != nil {
48+
if err := RunMigrations(logger, config, opts...); err != nil {
4949
return nil, err
5050
}
5151

‎database/database_test.go‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,18 @@
11
// Copyright 2020 The Moov Authors
22
// Use of this source code is governed by an Apache License
33
// license that can be found in the LICENSE file.
4-
package database
4+
package database_test
55

66
import (
77
"errors"
88
"testing"
9+
10+
"github.com/moov-io/base/database"
911
)
1012

1113
func TestUniqueViolation(t *testing.T) {
1214
err := errors.New(`problem upserting depository="282f6ffcd9ba5b029afbf2b739ee826e22d9df3b", userId="f25f48968da47ef1adb5b6531a1c2197295678ce": Error 1062: Duplicate entry '282f6ffcd9ba5b029afbf2b739ee826e22d9df3b' for key 'PRIMARY'`)
13-
if !UniqueViolation(err) {
15+
if !database.UniqueViolation(err) {
1416
t.Error("should have matched unique violation")
1517
}
1618
}

‎database/migrator.go‎

Lines changed: 96 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -7,31 +7,41 @@ import (
77
"context"
88
"database/sql"
99
"fmt"
10+
"io/fs"
1011
"sync"
12+
"time"
1113

1214
"github.com/golang-migrate/migrate/v4"
1315
"github.com/golang-migrate/migrate/v4/database"
1416
migmysql "github.com/golang-migrate/migrate/v4/database/mysql"
1517
"github.com/golang-migrate/migrate/v4/source"
18+
"github.com/golang-migrate/migrate/v4/source/iofs"
1619

1720
"github.com/moov-io/base/log"
1821
)
1922

2023
var migrationMutex sync.Mutex
2124

22-
func RunMigrations(logger log.Logger, config DatabaseConfig) error {
25+
func RunMigrations(logger log.Logger, config DatabaseConfig, opts ...MigrateOption) error {
2326
logger.Info().Log("Running Migrations")
2427

25-
source, driver, err := GetDriver(logger, config)
28+
// apply all of our optional arguments
29+
o := &migrateOptions{}
30+
for _, opt := range opts {
31+
if err := opt(o); err != nil {
32+
return err
33+
}
34+
}
35+
36+
source, driver, err := getDriver(logger, config, o)
2637
if err != nil {
2738
return err
2839
}
29-
3040
defer driver.Close()
3141

3242
migrationMutex.Lock()
3343
m, err := migrate.NewWithInstance(
34-
"filtering-pkger",
44+
source.name,
3545
source,
3646
config.DatabaseName,
3747
driver,
@@ -40,6 +50,10 @@ func RunMigrations(logger log.Logger, config DatabaseConfig) error {
4050
return logger.Fatal().LogErrorf("Error running migration: %w", err).Err()
4151
}
4252

53+
if o.timeout != nil {
54+
m.LockTimeout = *o.timeout
55+
}
56+
4357
err = m.Up()
4458
migrationMutex.Unlock()
4559

@@ -56,41 +70,63 @@ func RunMigrations(logger log.Logger, config DatabaseConfig) error {
5670
return nil
5771
}
5872

73+
// Deprecated: Here to not break compatibility since it was once public.
5974
func GetDriver(logger log.Logger, config DatabaseConfig) (source.Driver, database.Driver, error) {
75+
return getDriver(logger, config, &migrateOptions{})
76+
}
77+
78+
func getDriver(logger log.Logger, config DatabaseConfig, opts *migrateOptions) (*SourceDriver, database.Driver, error) {
79+
var err error
80+
6081
if config.MySQL != nil {
61-
src, err := NewPkgerSource("mysql", true)
62-
if err != nil {
63-
return nil, nil, err
82+
if opts.source == nil {
83+
src, err := NewPkgerSource("mysql", true)
84+
if err != nil {
85+
return nil, nil, err
86+
}
87+
opts.source = &SourceDriver{
88+
name: "pkger-mysql",
89+
Driver: src,
90+
}
6491
}
6592

66-
db, err := New(context.Background(), logger, config)
67-
if err != nil {
68-
return nil, nil, err
69-
}
70-
defer db.Close()
93+
if opts.driver == nil {
94+
db, err := New(context.Background(), logger, config)
95+
if err != nil {
96+
return nil, nil, err
97+
}
7198

72-
drv, err := MySQLDriver(db)
73-
if err != nil {
74-
return nil, nil, err
99+
opts.driver, err = MySQLDriver(db)
100+
if err != nil {
101+
return nil, nil, err
102+
}
75103
}
76104

77-
return src, drv, nil
78-
79105
} else if config.Spanner != nil {
80-
src, err := NewPkgerSource("spanner", false)
81-
if err != nil {
82-
return nil, nil, err
106+
if opts.source == nil {
107+
src, err := NewPkgerSource("spanner", false)
108+
if err != nil {
109+
return nil, nil, err
110+
}
111+
opts.source = &SourceDriver{
112+
name: "pkger-spanner",
113+
Driver: src,
114+
}
83115
}
84116

85-
drv, err := SpannerDriver(config)
86-
if err != nil {
87-
return nil, nil, err
117+
if opts.driver == nil {
118+
opts.driver, err = SpannerDriver(config)
119+
if err != nil {
120+
return nil, nil, err
121+
}
88122
}
123+
}
89124

90-
return src, drv, nil
125+
if opts.source == nil || opts.driver == nil {
126+
return nil, nil, fmt.Errorf("database config not defined")
91127
}
92128

93-
return nil, nil, fmt.Errorf("database config not defined")
129+
return opts.source, opts.driver, nil
94130
}
95131

96132
func MySQLDriver(db *sql.DB) (database.Driver, error) {
@@ -100,3 +136,38 @@ func MySQLDriver(db *sql.DB) (database.Driver, error) {
100136
func SpannerDriver(config DatabaseConfig) (database.Driver, error) {
101137
return SpannerMigrationDriver(*config.Spanner, config.DatabaseName)
102138
}
139+
140+
type MigrateOption func(o *migrateOptions) error
141+
142+
type SourceDriver struct {
143+
name string
144+
source.Driver
145+
}
146+
147+
type migrateOptions struct {
148+
source *SourceDriver
149+
driver database.Driver
150+
151+
timeout *time.Duration
152+
}
153+
154+
func WithEmbeddedMigrations(f fs.FS) MigrateOption {
155+
return func(o *migrateOptions) error {
156+
src, err := iofs.New(f, "migrations")
157+
if err != nil {
158+
return err
159+
}
160+
o.source = &SourceDriver{
161+
name: "embedded",
162+
Driver: src,
163+
}
164+
return nil
165+
}
166+
}
167+
168+
func WithTimeout(dur time.Duration) MigrateOption {
169+
return func(o *migrateOptions) error {
170+
o.timeout = &dur
171+
return nil
172+
}
173+
}

‎database/model_config_test.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,23 @@
11
// Copyright 2020 The Moov Authors
22
// Use of this source code is governed by an Apache License
33
// license that can be found in the LICENSE file.
4-
package database
4+
package database_test
55

66
import (
77
"bytes"
88
"encoding/json"
99
"testing"
1010

11+
"github.com/moov-io/base/database"
1112
"github.com/stretchr/testify/require"
1213
)
1314

1415
func TestMySQLConfig(t *testing.T) {
15-
cfg := &MySQLConfig{
16+
cfg := &database.MySQLConfig{
1617
Address: "tcp(localhost:3306)",
1718
User: "app",
1819
Password: "secret",
19-
Connections: ConnectionsConfig{
20+
Connections: database.ConnectionsConfig{
2021
MaxOpen: 100,
2122
},
2223
}

0 commit comments

Comments
 (0)