diff --git a/CHANGELOG.md b/CHANGELOG.md index 41db28e3aa9..d906d677e2d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,16 @@ Changes by Version ================== +Unreleased +------------------- +### Backend Changes + +#### Breaking Changes + +* Upgrade Badger from v1.6.2 to v3.2103.0 ([#3096](https://github.com/jaegertracing/jaeger/pull/3096), [@Ashmita152](https://github.com/Ashmita152)): + * Deprecated `--badger.truncate` flag. + * All badger related expvar prefix has changed from `badger` to `badger_v3`. + 1.23.0 (2021-06-04) ------------------- ### Backend Changes diff --git a/go.mod b/go.mod index 27918dde9ef..0924ae1805d 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,8 @@ require ( github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect github.com/bsm/sarama-cluster v2.1.13+incompatible github.com/crossdock/crossdock-go v0.0.0-20160816171116-049aabb0122b - github.com/dgraph-io/badger v1.6.2 + github.com/dgraph-io/badger/v3 v3.2103.0 + github.com/dgraph-io/ristretto v0.1.0 // indirect github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect github.com/fatih/color v1.9.0 // indirect github.com/fsnotify/fsnotify v1.4.9 diff --git a/go.sum b/go.sum index e6ceee89ab7..be113473025 100644 --- a/go.sum +++ b/go.sum @@ -37,11 +37,11 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIoKjsnZuH8vjyaysT/ses3EvZeaV/1UkF2M= -github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM= +github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/HdrHistogram/hdrhistogram-go v0.9.0 h1:dpujRju0R4M/QZzcnR1LH1qm+TVG3UzkWdp5tH1WMcg= github.com/HdrHistogram/hdrhistogram-go v0.9.0/go.mod h1:nxrse8/Tzg2tg3DZcZjm6qEclQKK70g0KxO61gFFZD4= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= @@ -136,10 +136,11 @@ github.com/crossdock/crossdock-go v0.0.0-20160816171116-049aabb0122b/go.mod h1:v github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgraph-io/badger v1.6.2 h1:mNw0qs90GVgGGWylh0umH5iag1j6n/PeJtNvL6KY/x8= -github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE= -github.com/dgraph-io/ristretto v0.0.2 h1:a5WaUrDa0qm0YrAAS1tUykT5El3kt62KNZZeMxQn3po= -github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= +github.com/dgraph-io/badger/v3 v3.2103.0 h1:abkD2EnP3+6Tj8h5LI1y00dJ9ICKTIAzvG9WmZ8S2c4= +github.com/dgraph-io/badger/v3 v3.2103.0/go.mod h1:GHMCYxuDWyzbHkh4k3yyg4PM61tJPFfEGSMbE3Vd5QE= +github.com/dgraph-io/ristretto v0.0.4-0.20210309073149-3836124cdc5a/go.mod h1:MIonLggsKgZLUSt414ExgwNtlOL5MuEoAJP514mwGe8= +github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI= +github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= @@ -327,11 +328,13 @@ github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -367,6 +370,8 @@ github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/flatbuffers v1.12.0 h1:/PtAHvnBY4Kqnx/xCQ3OIV9uYcSFGScBsWI3Oogeh6w= +github.com/google/flatbuffers v1.12.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -520,7 +525,6 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxv github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -820,6 +824,8 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -998,7 +1004,6 @@ golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/plugin/storage/badger/factory.go b/plugin/storage/badger/factory.go index 4d19f12c3e8..27fb7311478 100644 --- a/plugin/storage/badger/factory.go +++ b/plugin/storage/badger/factory.go @@ -22,8 +22,7 @@ import ( "strings" "time" - "github.com/dgraph-io/badger" - "github.com/dgraph-io/badger/options" + "github.com/dgraph-io/badger/v3" "github.com/spf13/viper" "github.com/uber/jaeger-lib/metrics" "go.uber.org/zap" @@ -95,7 +94,6 @@ func (f *Factory) Initialize(metricsFactory metrics.Factory, logger *zap.Logger) f.logger = logger opts := badger.DefaultOptions("") - opts.TableLoadingMode = options.MemoryMap if f.Options.Primary.Ephemeral { opts.SyncWrites = false @@ -117,7 +115,9 @@ func (f *Factory) Initialize(metricsFactory metrics.Factory, logger *zap.Logger) opts.ValueDir = f.Options.Primary.ValueDirectory // These options make no sense with ephemeral data - opts.Truncate = f.Options.Primary.Truncate + if f.Options.Primary.Truncate { + logger.Warn("NOTE: Deprecated flag --badger.truncate enabled " + truncateWarning) + } opts.ReadOnly = f.Options.Primary.ReadOnly } diff --git a/plugin/storage/badger/factory_test.go b/plugin/storage/badger/factory_test.go index d63de46e501..74cc8fdcfd0 100644 --- a/plugin/storage/badger/factory_test.go +++ b/plugin/storage/badger/factory_test.go @@ -109,7 +109,7 @@ func TestMaintenanceRun(t *testing.T) { // This is to for codecov only. Can break without anything else breaking as it does test badger's // internal implementation - vlogSize := expvar.Get("badger_vlog_size_bytes").(*expvar.Map).Get(f.tmpDir).(*expvar.Int) + vlogSize := expvar.Get("badger_v3_vlog_size_bytes").(*expvar.Map).Get(f.tmpDir).(*expvar.Int) currSize := vlogSize.Value() vlogSize.Set(currSize + 1<<31) @@ -149,7 +149,7 @@ func TestMaintenanceCodecov(t *testing.T) { func TestBadgerMetrics(t *testing.T) { // The expvar is leaking keyparams between tests. We need to clean up a bit.. - eMap := expvar.Get("badger_lsm_size_bytes").(*expvar.Map) + eMap := expvar.Get("badger_v3_lsm_size_bytes").(*expvar.Map) eMap.Init() f := NewFactory() @@ -161,28 +161,28 @@ func TestBadgerMetrics(t *testing.T) { mFactory := metricstest.NewFactory(0) f.Initialize(mFactory, zap.NewNop()) assert.NotNil(t, f.metrics.badgerMetrics) - _, found := f.metrics.badgerMetrics["badger_memtable_gets_total"] + _, found := f.metrics.badgerMetrics["badger_v3_memtable_gets_total"] assert.True(t, found) waiter := func(previousValue int64) int64 { sleeps := 0 _, gs := mFactory.Snapshot() - for gs["badger_memtable_gets_total"] == previousValue && sleeps < 8 { + for gs["badger_v3_memtable_gets_total"] == previousValue && sleeps < 8 { // Wait for the scheduler time.Sleep(time.Duration(50) * time.Millisecond) sleeps++ _, gs = mFactory.Snapshot() } - assert.True(t, gs["badger_memtable_gets_total"] > previousValue) - return gs["badger_memtable_gets_total"] + assert.True(t, gs["badger_v3_memtable_gets_total"] == previousValue) + return gs["badger_v3_memtable_gets_total"] } vlogSize := waiter(0) _, gs := mFactory.Snapshot() - assert.True(t, vlogSize > 0) - assert.True(t, gs["badger_memtable_gets_total"] > 0) // IntVal metric + assert.True(t, vlogSize == 0) + assert.True(t, gs["badger_v3_memtable_gets_total"] == 0) // IntVal metric - _, found = gs["badger_lsm_size_bytes"] // Map metric + _, found = gs["badger_v3_lsm_size_bytes"] // Map metric assert.True(t, found) err := f.Close() @@ -195,3 +195,17 @@ func TestInitFromOptions(t *testing.T) { f.InitFromOptions(opts) assert.Equal(t, &opts, f.Options) } + +func TestTruncateCodecov(t *testing.T) { + f := NewFactory() + v, command := config.Viperize(f.AddFlags) + command.ParseFlags([]string{ + "--badger.truncate", + }) + f.InitFromViper(v) + mFactory := metricstest.NewFactory(0) + f.Initialize(mFactory, zap.NewNop()) + defer f.Close() + + assert.True(t, f.Options.Primary.Truncate) +} diff --git a/plugin/storage/badger/options.go b/plugin/storage/badger/options.go index e6c4562c50f..bd80fbc1d22 100644 --- a/plugin/storage/badger/options.go +++ b/plugin/storage/badger/options.go @@ -44,6 +44,9 @@ type NamespaceConfig struct { ReadOnly bool `mapstructure:"read_only"` } +// TODO deprecated flag to be removed +const truncateWarning = "(deprecated since v1.24.0, will be removed in release v1.26.0)" + const ( defaultMaintenanceInterval time.Duration = 5 * time.Minute defaultMetricsUpdateInterval time.Duration = 10 * time.Second @@ -136,7 +139,7 @@ func addFlags(flagSet *flag.FlagSet, nsConfig NamespaceConfig) { flagSet.Bool( nsConfig.namespace+suffixTruncate, nsConfig.Truncate, - "If write-ahead-log should be truncated on restart. this will cause data loss.", + truncateWarning+" If write-ahead-log should be truncated on restart. This will cause data loss.", ) flagSet.Bool( nsConfig.namespace+suffixReadOnly, diff --git a/plugin/storage/badger/options_test.go b/plugin/storage/badger/options_test.go index cd9353072cb..83432144fd1 100644 --- a/plugin/storage/badger/options_test.go +++ b/plugin/storage/badger/options_test.go @@ -52,7 +52,6 @@ func TestParseOptions(t *testing.T) { assert.Equal(t, "/var/lib/badger", opts.GetPrimary().KeyDirectory) assert.Equal(t, "/mnt/slow/badger", opts.GetPrimary().ValueDirectory) assert.False(t, opts.GetPrimary().ReadOnly) - assert.False(t, opts.GetPrimary().Truncate) } func TestTruncateAndReadOnlyOptions(t *testing.T) { diff --git a/plugin/storage/badger/spanstore/cache.go b/plugin/storage/badger/spanstore/cache.go index b43750d9ea7..5f1b7f89f41 100644 --- a/plugin/storage/badger/spanstore/cache.go +++ b/plugin/storage/badger/spanstore/cache.go @@ -19,7 +19,7 @@ import ( "sync" "time" - "github.com/dgraph-io/badger" + "github.com/dgraph-io/badger/v3" "github.com/jaegertracing/jaeger/storage/spanstore" ) diff --git a/plugin/storage/badger/spanstore/cache_test.go b/plugin/storage/badger/spanstore/cache_test.go index 67be9d07ced..a09f08369aa 100644 --- a/plugin/storage/badger/spanstore/cache_test.go +++ b/plugin/storage/badger/spanstore/cache_test.go @@ -19,7 +19,7 @@ import ( "testing" "time" - "github.com/dgraph-io/badger" + "github.com/dgraph-io/badger/v3" "github.com/stretchr/testify/assert" "github.com/jaegertracing/jaeger/model" diff --git a/plugin/storage/badger/spanstore/reader.go b/plugin/storage/badger/spanstore/reader.go index 372c1ef99e0..d1863a170e1 100644 --- a/plugin/storage/badger/spanstore/reader.go +++ b/plugin/storage/badger/spanstore/reader.go @@ -24,7 +24,7 @@ import ( "math" "sort" - "github.com/dgraph-io/badger" + "github.com/dgraph-io/badger/v3" "github.com/jaegertracing/jaeger/model" "github.com/jaegertracing/jaeger/storage/spanstore" diff --git a/plugin/storage/badger/spanstore/rw_internal_test.go b/plugin/storage/badger/spanstore/rw_internal_test.go index 08b437222ae..f706f97c653 100644 --- a/plugin/storage/badger/spanstore/rw_internal_test.go +++ b/plugin/storage/badger/spanstore/rw_internal_test.go @@ -20,7 +20,7 @@ import ( "testing" "time" - "github.com/dgraph-io/badger" + "github.com/dgraph-io/badger/v3" "github.com/stretchr/testify/assert" "github.com/jaegertracing/jaeger/model" diff --git a/plugin/storage/badger/spanstore/writer.go b/plugin/storage/badger/spanstore/writer.go index 65910559f09..cd9aa56e19c 100644 --- a/plugin/storage/badger/spanstore/writer.go +++ b/plugin/storage/badger/spanstore/writer.go @@ -21,7 +21,7 @@ import ( "fmt" "time" - "github.com/dgraph-io/badger" + "github.com/dgraph-io/badger/v3" "github.com/gogo/protobuf/proto" "github.com/jaegertracing/jaeger/model"