Merge pull request #17900 from influxdata/feat/use-algo-w

chore: promote algorithm W to master
influxdata · May 27, 2020 · d22380d · d22380d
2 parents 97dc4d1 + de1b1b3
commit d22380d
Show file tree

Hide file tree

Showing 93 changed files with 3,395 additions and 1,846 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -257,7 +257,7 @@ jobs:
             # filter internal/promqltests because it has special go.mod
             TESTFILES=$(go list ./... | grep -v internal/promqltests | circleci tests split --split-by=timings)
             echo $TESTFILES
-            GOTRACEBACK=all GO111MODULE=on FLUX_PARSER_TYPE=rust CGO_LDFLAGS="$(cat .cgo_ldflags)" gotestsum --format standard-quiet --junitfile /tmp/test-results/gotestsum.xml -- -race -count=1 -tags 'libflux' $TESTFILES
+            GOTRACEBACK=all GO111MODULE=on ./env gotestsum --format standard-quiet --junitfile /tmp/test-results/gotestsum.xml -- -race -count=1 $TESTFILES
       - save_cache:
           name: Saving GOCACHE
           key: influxdb-gocache-{{ .Branch }}-{{ .Revision }}
@@ -352,7 +352,7 @@ jobs:
       - run: make checktidy
       - run: GO111MODULE=on go mod vendor # staticcheck looks in vendor for dependencies.
       - run: GO111MODULE=on go install honnef.co/go/tools/cmd/staticcheck # Install staticcheck from the version we specify in go.mod.
-      - run: GO111MODULE=on staticcheck ./...
+      - run: GO111MODULE=on ./env staticcheck ./...
       - save_cache:
           name: Saving GOCACHE
           key: influxdb-gocache-{{ .Branch }}-{{ .Revision }}
@@ -509,6 +509,7 @@ workflows:
             branches:
               only:
                 - master
+                - feat/use-algo-w
     jobs:
       - gotest
       - golint
@@ -522,7 +523,9 @@ workflows:
             - jslint
           filters:
             branches:
-              only: master
+              only:
+                - master
+                - feat/use-algo-w
       - litmus_nightly:
           requires:
             - deploy-nightly

diff --git a/Makefile b/Makefile
@@ -14,8 +14,7 @@
 # SUBDIRS are directories that have their own Makefile.
 # It is required that all SUBDIRS have the `all` and `clean` targets.
 SUBDIRS := http ui chronograf query storage
-# The 'libflux' tag is required for instructing the flux to be compiled with the Rust parser
-GO_TAGS=libflux
+GO_TAGS=
 GO_ARGS=-tags '$(GO_TAGS)'
 ifeq ($(OS), Windows_NT)
 	VERSION := $(shell git describe --exact-match --tags 2>nil)
@@ -31,13 +30,14 @@ endif
 
 
 # Test vars can be used by all recursive Makefiles
+export PKG_CONFIG:=$(PWD)/scripts/pkg-config.sh
 export GOOS=$(shell go env GOOS)
-export GO_BUILD=env GO111MODULE=on CGO_LDFLAGS="$$(cat .cgo_ldflags)" go build $(GO_ARGS) -ldflags "$(LDFLAGS)"
-export GO_INSTALL=env GO111MODULE=on CGO_LDFLAGS="$$(cat .cgo_ldflags)" go install $(GO_ARGS) -ldflags "$(LDFLAGS)"
-export GO_TEST=env FLUX_PARSER_TYPE=rust GOTRACEBACK=all GO111MODULE=on CGO_LDFLAGS="$$(cat .cgo_ldflags)" go test $(GO_ARGS)
+export GO_BUILD=env GO111MODULE=on go build $(GO_ARGS) -ldflags "$(LDFLAGS)"
+export GO_INSTALL=env GO111MODULE=on go install $(GO_ARGS) -ldflags "$(LDFLAGS)"
+export GO_TEST=env GOTRACEBACK=all GO111MODULE=on go test $(GO_ARGS)
 # Do not add GO111MODULE=on to the call to go generate so it doesn't pollute the environment.
 export GO_GENERATE=go generate $(GO_ARGS)
-export GO_VET=env GO111MODULE=on CGO_LDFLAGS="$$(cat .cgo_ldflags)" go vet $(GO_ARGS)
+export GO_VET=env GO111MODULE=on go vet $(GO_ARGS)
 export GO_RUN=env GO111MODULE=on go run $(GO_ARGS)
 export PATH := $(PWD)/bin/$(GOOS):$(PATH)
 
@@ -78,7 +78,7 @@ $(SUBDIRS):
 #
 # Define targets for commands
 #
-$(CMDS): $(SOURCES) libflux
+$(CMDS): $(SOURCES)
 	$(GO_BUILD) -o $@ ./cmd/$(shell basename "$@")
 
 # Ease of use build for just the go binary
@@ -110,12 +110,6 @@ ui_client:
 # Define action only targets
 #
 
-libflux: .cgo_ldflags
-
-.cgo_ldflags: go.mod
-	$(GO_RUN) github.com/influxdata/flux/internal/cmd/flux-config --libs --verbose > .cgo_ldflags.tmp
-	mv .cgo_ldflags.tmp .cgo_ldflags
-
 fmt: $(SOURCES_NO_VENDOR)
 	gofmt -w -s $^
 
@@ -140,7 +134,7 @@ generate: $(SUBDIRS)
 test-js: node_modules
 	make -C ui test
 
-test-go: libflux
+test-go:
 	$(GO_TEST) ./...
 
 test-promql-e2e:
@@ -152,13 +146,13 @@ test-integration:
 
 test: test-go test-js
 
-test-go-race: libflux
+test-go-race:
 	$(GO_TEST) -v -race -count=1 ./...
 
-vet: libflux
+vet:
 	$(GO_VET) -v ./...
 
-bench: libflux
+bench:
 	$(GO_TEST) -bench=. -run=^$$ ./...
 
 build: all
@@ -181,7 +175,6 @@ clean:
 	@for d in $(SUBDIRS); do $(MAKE) -C $$d clean; done
 	$(RM) -r bin
 	$(RM) -r dist
-	$(RM) .cgo_ldflags
 
 define CHRONOGIRAFFE
              ._ o o

diff --git a/check.go b/check.go
@@ -13,14 +13,14 @@ const (
 
 // Check represents the information required to generate a periodic check task.
 type Check interface {
-	Valid() error
+	Valid(lang FluxLanguageService) error
 	Type() string
 	ClearPrivateData()
 	SetTaskID(ID)
 	GetTaskID() ID
 	GetOwnerID() ID
 	SetOwnerID(ID)
-	GenerateFlux() (string, error)
+	GenerateFlux(lang FluxLanguageService) (string, error)
 	json.Marshaler
 
 	CRUDLogSetter

diff --git a/cmd/influx/query.go b/cmd/influx/query.go
@@ -6,8 +6,10 @@ import (
 	"os"
 	"strings"
 
-	"github.com/influxdata/flux"
+	"github.com/influxdata/flux/plan"
+	"github.com/influxdata/flux/runtime"
 	_ "github.com/influxdata/flux/stdlib"
+	"github.com/influxdata/flux/stdlib/influxdata/influxdb"
 	_ "github.com/influxdata/influxdb/v2/query/stdlib"
 	"github.com/spf13/cobra"
 )
@@ -76,19 +78,19 @@ func fluxQueryF(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("failed to load query: %v", err)
 	}
 
-	orgSvc, err := newOrganizationService()
-	if err != nil {
-		return fmt.Errorf("failed to initialized organization service client: %v", err)
-	}
-
-	orgID, err := queryFlags.org.getID(orgSvc)
-	if err != nil {
-		return err
-	}
-
-	flux.FinalizeBuiltIns()
-
-	r, err := getFluxREPL(flags.Host, flags.Token, flags.skipVerify, orgID)
+	plan.RegisterLogicalRules(
+		influxdb.DefaultFromAttributes{
+			Org: &influxdb.NameOrID{
+				ID:   queryFlags.org.id,
+				Name: queryFlags.org.name,
+			},
+			Host:  &flags.Host,
+			Token: &flags.Token,
+		},
+	)
+	runtime.FinalizeBuiltIns()
+
+	r, err := getFluxREPL(flags.skipVerify)
 	if err != nil {
 		return fmt.Errorf("failed to get the flux REPL: %v", err)
 	}

diff --git a/cmd/influx/repl.go b/cmd/influx/repl.go
@@ -2,14 +2,17 @@ package main
 
 import (
 	"context"
+	"crypto/tls"
 	"fmt"
+	"net/http"
 
 	"github.com/influxdata/flux"
+	"github.com/influxdata/flux/dependencies/filesystem"
+	"github.com/influxdata/flux/plan"
 	"github.com/influxdata/flux/repl"
+	"github.com/influxdata/flux/runtime"
 	_ "github.com/influxdata/flux/stdlib"
-	platform "github.com/influxdata/influxdb/v2"
-	"github.com/influxdata/influxdb/v2/http"
-	"github.com/influxdata/influxdb/v2/query"
+	"github.com/influxdata/flux/stdlib/influxdata/influxdb"
 	_ "github.com/influxdata/influxdb/v2/query/stdlib"
 	"github.com/spf13/cobra"
 )
@@ -37,19 +40,19 @@ func replF(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	orgSVC, err := newOrganizationService()
-	if err != nil {
-		return err
-	}
+	plan.RegisterLogicalRules(
+		influxdb.DefaultFromAttributes{
+			Org: &influxdb.NameOrID{
+				ID:   replFlags.org.id,
+				Name: replFlags.org.name,
+			},
+			Host:  &flags.Host,
+			Token: &flags.Token,
+		},
+	)
+	runtime.FinalizeBuiltIns()
 
-	orgID, err := replFlags.org.getID(orgSVC)
-	if err != nil {
-		return err
-	}
-
-	flux.FinalizeBuiltIns()
-
-	r, err := getFluxREPL(flags.Host, flags.Token, flags.skipVerify, orgID)
+	r, err := getFluxREPL(flags.skipVerify)
 	if err != nil {
 		return err
 	}
@@ -58,17 +61,18 @@ func replF(cmd *cobra.Command, args []string) error {
 	return nil
 }
 
-func getFluxREPL(addr, token string, skipVerify bool, orgID platform.ID) (*repl.REPL, error) {
-	qs := &http.FluxQueryService{
-		Addr:               addr,
-		Token:              token,
-		InsecureSkipVerify: skipVerify,
-	}
-	q := &query.REPLQuerier{
-		OrganizationID: orgID,
-		QueryService:   qs,
+func getFluxREPL(skipVerify bool) (*repl.REPL, error) {
+	deps := flux.NewDefaultDependencies()
+	deps.Deps.FilesystemService = filesystem.SystemFS
+	if skipVerify {
+		deps.Deps.HTTPClient = &http.Client{
+			Transport: &http.Transport{
+				TLSClientConfig: &tls.Config{
+					InsecureSkipVerify: true,
+				},
+			},
+		}
 	}
-	// background context is OK here, and DefaultDependencies are noop deps.  Also safe
-	// since we send all queries to the server side.
-	return repl.New(context.Background(), flux.NewDefaultDependencies(), q), nil
+	ctx := deps.Inject(context.Background())
+	return repl.New(ctx, deps), nil
 }
diff --git a/cmd/influxd/launcher/launcher.go b/cmd/influxd/launcher/launcher.go
@@ -43,6 +43,7 @@ import (
 	infprom "github.com/influxdata/influxdb/v2/prometheus"
 	"github.com/influxdata/influxdb/v2/query"
 	"github.com/influxdata/influxdb/v2/query/control"
+	"github.com/influxdata/influxdb/v2/query/fluxlang"
 	"github.com/influxdata/influxdb/v2/query/stdlib/influxdata/influxdb"
 	"github.com/influxdata/influxdb/v2/session"
 	"github.com/influxdata/influxdb/v2/snowflake"
@@ -541,7 +542,8 @@ func (m *Launcher) run(ctx context.Context) (err error) {
 	}
 
 	serviceConfig := kv.ServiceConfig{
-		SessionLength: time.Duration(m.sessionLength) * time.Minute,
+		SessionLength:       time.Duration(m.sessionLength) * time.Minute,
+		FluxLanguageService: fluxlang.DefaultService,
 	}
 
 	flushers := flushers{}
@@ -898,6 +900,7 @@ func (m *Launcher) run(ctx context.Context) (err error) {
 		PasswordsService:                passwdsSvc,
 		InfluxQLService:                 storageQueryService,
 		FluxService:                     storageQueryService,
+		FluxLanguageService:             fluxlang.DefaultService,
 		TaskService:                     taskSvc,
 		TelegrafService:                 telegrafSvc,
 		NotificationRuleStore:           notificationRuleSvc,

diff --git a/cmd/influxd/launcher/query_test.go b/cmd/influxd/launcher/query_test.go
@@ -18,6 +18,7 @@ import (
 	"github.com/influxdata/flux/execute"
 	"github.com/influxdata/flux/execute/executetest"
 	"github.com/influxdata/flux/lang"
+	"github.com/influxdata/flux/runtime"
 	"github.com/influxdata/flux/values"
 	"github.com/influxdata/influxdb/v2"
 	"github.com/influxdata/influxdb/v2/cmd/influxd/launcher"
@@ -215,7 +216,7 @@ func queryPoints(ctx context.Context, t *testing.T, l *launcher.TestLauncher, op
 	if d.verbose {
 		t.Logf("query:\n%s", qs)
 	}
-	pkg, err := flux.Parse(qs)
+	pkg, err := runtime.ParseToJSON(qs)
 	if err != nil {
 		t.Fatal(err)
 	}

diff --git a/env b/env
@@ -2,6 +2,9 @@
 
 set -e
 
+# https://stackoverflow.com/a/246128
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
 verbose_flag=""
 while getopts vx arg; do
   case "$arg" in
@@ -11,8 +14,7 @@ while getopts vx arg; do
 done
 shift $((OPTIND-1))
 
-CGO_LDFLAGS=$(go run github.com/influxdata/flux/internal/cmd/flux-config --libs $verbose_flag)
-export CGO_LDFLAGS
+export PKG_CONFIG="${DIR}/scripts/pkg-config.sh"
 
 # If this script is being executed, it will be executed under bash
 # so the bash source variable should be present. If the variable

diff --git a/flags.yml b/flags.yml
@@ -27,6 +27,18 @@
   contact: Gavin Cabbage
   expose: true
 
+- name: Push Down Window Aggregate Count
+  description: Enable Count variant of PushDownWindowAggregateRule and PushDownBareAggregateRule
+  key: pushDownWindowAggregateCount
+  default: false
+  contact: Query Team
+
+- name: Push Down Window Aggregate Rest
+  description: Enable non-Count variants of PushDownWindowAggregateRule and PushDownWindowAggregateRule (stage 2)
+  key: pushDownWindowAggregateRest
+  default: false
+  contact: Query Team
+
 - name: New Auth Package
   description: Enables the refactored authorization api
   key: newAuth
@@ -40,3 +52,9 @@
   default: false
   contact: Lyon Hill
   expose: true
+
+- name: Push Down Group Aggregate Count
+  description: Enable the count variant of PushDownGroupAggregate planner rule
+  key: pushDownGroupAggregateCount
+  default: false
+  contact: Query Team