Integrate WAL into engine

cmd/influxd/main.go

@@ -3,7 +3,6 @@ package main
 import (
 	"context"
 	"fmt"
-	"io"
 	nethttp "net/http"
 	_ "net/http/pprof"
 	"os"
@@ -57,7 +56,7 @@ var (
 	httpBindAddress   string
 	authorizationPath string
 	boltPath          string
-	walPath           string
+	natsPath           string
 	developerMode     bool
 	enginePath        string
 )
@@ -115,10 +114,11 @@ func init() {
 		os.Exit(1)
 	}
 
-	platformCmd.Flags().StringVar(&walPath, "wal-path", filepath.Join(dir, "wal"), "path to persistent WAL files")
-	viper.BindEnv("WAL_PATH")
-	if h := viper.GetString("WAL_PATH"); h != "" {
-		walPath = h
+	// TODO(edd): do we need NATS for anything?
+	platformCmd.Flags().StringVar(&natsPath, "nats-path", filepath.Join(dir, "nats"), "path to persistent NATS files")
+	viper.BindEnv("NATS_PATH")
+	if h := viper.GetString("NATS_PATH"); h != "" {
+		natsPath = h
 	}
 
 	platformCmd.Flags().StringVar(&enginePath, "engine-path", filepath.Join(dir, "engine"), "path to persistent engine files")
@@ -211,14 +211,11 @@ func platformF(cmd *cobra.Command, args []string) {
 
 	// TODO(jeff): this block is hacky support for a storage engine. it is not intended to
 	// be a long term solution.
-	var (
-		natsHandler         nats.Handler
-		storageQueryService query.ProxyQueryService
-	)
+	var	storageQueryService query.ProxyQueryService
+	var pointsWriter storage.PointsWriter
 	{
 		config := storage.NewConfig()
-		config.CacheSnapshotMemorySize = 0
-		config.TraceLoggingEnabled = true
+		config.EngineOptions.WALEnabled = true // Enable a disk-based WAL.
 		config.EngineOptions.Config = config.Config
 
 		engine := storage.NewEngine(enginePath, config)
@@ -229,12 +226,7 @@ func platformF(cmd *cobra.Command, args []string) {
 			os.Exit(1)
 		}
 
-		engineHandler := nats.NewEngineHandler()
-		engineHandler.Logger = logger.With(zap.String("handler", "engine"))
-		engineHandler.Engine = engine
-
-		natsHandler = engineHandler
-
+		pointsWriter = engine
 		storageQueryService = query.ProxyQueryServiceBridge{
 			QueryService: query.QueryServiceBridge{
 				AsyncQueryService: &queryAdapter{
@@ -297,7 +289,7 @@ func platformF(cmd *cobra.Command, args []string) {
 	signal.Notify(sigs, syscall.SIGTERM, os.Interrupt)
 
 	// NATS streaming server
-	natsServer := nats.NewServer(nats.Config{FilestoreDir: walPath})
+	natsServer := nats.NewServer(nats.Config{FilestoreDir: natsPath})
 	if err := natsServer.Open(); err != nil {
 		logger.Error("failed to start nats streaming server", zap.Error(err))
 		os.Exit(1)
@@ -316,11 +308,6 @@ func platformF(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 
-	if err := subscriber.Subscribe(IngressSubject, IngressGroup, natsHandler); err != nil {
-		logger.Error("failed to create nats subscriber", zap.Error(err))
-		os.Exit(1)
-	}
-
 	scraperScheduler, err := gather.NewScheduler(10, logger, scraperTargetSvc, publisher, subscriber, 0, 0)
 	if err != nil {
 		logger.Error("failed to create scraper subscriber", zap.Error(err))
@@ -338,9 +325,7 @@ func platformF(cmd *cobra.Command, args []string) {
 		Logger:           logger,
 		NewBucketService: source.NewBucketService,
 		NewQueryService:  source.NewQueryService,
-		PublisherFn: func(r io.Reader) error {
-			return publisher.Publish(IngressSubject, r)
-		},
+		PointsWriter: pointsWriter,
 		AuthorizationService:       authSvc,
 		BucketService:              bucketSvc,
 		SessionService:             sessionSvc,

http/api_handler.go

@@ -1,13 +1,13 @@
 package http
 
 import (
-	"io"
 	http "net/http"
 	"strings"
 
 	"github.com/influxdata/platform"
 	"github.com/influxdata/platform/chronograf/server"
 	"github.com/influxdata/platform/query"
+	"github.com/influxdata/platform/storage"
 	"go.uber.org/zap"
 )
 
@@ -39,8 +39,8 @@ type APIBackend struct {
 	NewBucketService func(*platform.Source) (platform.BucketService, error)
 	NewQueryService  func(*platform.Source) (query.ProxyQueryService, error)
 
-	PublisherFn func(r io.Reader) error
-
+
+	PointsWriter storage.PointsWriter
 	AuthorizationService       platform.AuthorizationService
 	BucketService              platform.BucketService
 	SessionService             platform.SessionService
@@ -105,7 +105,7 @@ func NewAPIHandler(b *APIBackend) *APIHandler {
 	h.TaskHandler = NewTaskHandler(b.Logger)
 	h.TaskHandler.TaskService = b.TaskService
 
-	h.WriteHandler = NewWriteHandler(b.PublisherFn)
+	h.WriteHandler = NewWriteHandler(b.PointsWriter)
 	h.WriteHandler.AuthorizationService = b.AuthorizationService
 	h.WriteHandler.OrganizationService = b.OrganizationService
 	h.WriteHandler.BucketService = b.BucketService

http/write_handler.go

@@ -1,18 +1,17 @@
 package http
 
 import (
-	"bytes"
 	"compress/gzip"
 	"context"
 	"fmt"
-	"io"
 	"io/ioutil"
 	"net/http"
 
 	"github.com/influxdata/platform"
 	pcontext "github.com/influxdata/platform/context"
 	"github.com/influxdata/platform/kit/errors"
 	"github.com/influxdata/platform/models"
+	"github.com/influxdata/platform/storage"
 	"github.com/influxdata/platform/tsdb"
 	"github.com/julienschmidt/httprouter"
 	"go.uber.org/zap"
@@ -28,15 +27,15 @@ type WriteHandler struct {
 	BucketService        platform.BucketService
 	OrganizationService  platform.OrganizationService
 
-	Publish func(io.Reader) error
+	PointsWriter storage.PointsWriter
 }
 
 // NewWriteHandler creates a new handler at /api/v2/write to receive line protocol.
-func NewWriteHandler(publishFn func(io.Reader) error) *WriteHandler {
+func NewWriteHandler(writer storage.PointsWriter) *WriteHandler {
 	h := &WriteHandler{
-		Router:  httprouter.New(),
-		Logger:  zap.NewNop(),
-		Publish: publishFn,
+		Router: httprouter.New(),
+		Logger: zap.NewNop(),
+		PointsWriter: writer,
 	}
 
 	h.HandlerFunc("POST", "/api/v2/write", h.handleWrite)
@@ -137,7 +136,6 @@ func (h *WriteHandler) handleWrite(w http.ResponseWriter, r *http.Request) {
 	// TODO(jeff): we should be publishing with the org and bucket instead of
 	// parsing, rewriting, and publishing, but the interface isn't quite there yet.
 	// be sure to remove this when it is there!
-	{
 		data, err := ioutil.ReadAll(in)
 		if err != nil {
 			logger.Info("Error reading body", zap.Error(err))
@@ -159,19 +157,11 @@ func (h *WriteHandler) handleWrite(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 
-		var buf []byte
-		for _, pt := range exploded {
-			buf = pt.AppendString(buf)
-			buf = append(buf, '\n')
+		if err := h.PointsWriter.WritePoints(exploded); err != nil {
+			EncodeError(ctx, errors.BadRequestError(err.Error()), w)
+			return
 		}
 
-		in = ioutil.NopCloser(bytes.NewReader(buf))
-	}
-
-	if err := h.Publish(in); err != nil {
-		EncodeError(ctx, errors.BadRequestError(err.Error()), w)
-		return
-	}
 
 	w.WriteHeader(http.StatusNoContent)
 }