[chore][pkg/stanza] Improve tests for internal splitter package (#26534)

open-telemetry · Sep 12, 2023 · bba0e58 · bba0e58
1 parent 129bc75
commit bba0e58
Show file tree

Hide file tree

Showing 2 changed files with 150 additions and 70 deletions.
diff --git a/pkg/stanza/fileconsumer/internal/splitter/custom_test.go b/pkg/stanza/fileconsumer/internal/splitter/custom_test.go
@@ -11,33 +11,58 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func TestCustomFactory(t *testing.T) {
-	tests := []struct {
-		name        string
-		splitter    bufio.SplitFunc
-		flushPeriod time.Duration
-		wantErr     bool
-	}{
-		{
-			name: "default configuration",
-			splitter: func(data []byte, atEOF bool) (advance int, token []byte, err error) {
-				return len(data), data, nil
-			},
-			flushPeriod: 100 * time.Millisecond,
-			wantErr:     false,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			factory := NewCustomFactory(tt.splitter, tt.flushPeriod)
-			got, err := factory.SplitFunc()
-			if (err != nil) != tt.wantErr {
-				t.Errorf("Build() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-			if err == nil {
-				assert.NotNil(t, got)
-			}
-		})
-	}
+func TestCustom(t *testing.T) {
+	factory := NewCustomFactory(bufio.ScanLines, 0)
+	splitFunc, err := factory.SplitFunc()
+	assert.NoError(t, err)
+	assert.NotNil(t, splitFunc)
+
+	input := []byte(" hello \n world \n extra ")
+
+	advance, token, err := splitFunc(input, false)
+	assert.NoError(t, err)
+	assert.Equal(t, 8, advance)
+	assert.Equal(t, []byte(" hello "), token)
+
+	advance, token, err = splitFunc(input[8:], false)
+	assert.NoError(t, err)
+	assert.Equal(t, 8, advance)
+	assert.Equal(t, []byte(" world "), token)
+
+	advance, token, err = splitFunc(input[16:], false)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, advance)
+	assert.Nil(t, token)
+}
+
+func TestCustomWithFlush(t *testing.T) {
+	flushPeriod := 100 * time.Millisecond
+	factory := NewCustomFactory(bufio.ScanLines, flushPeriod)
+	splitFunc, err := factory.SplitFunc()
+	assert.NoError(t, err)
+	assert.NotNil(t, splitFunc)
+
+	input := []byte(" hello \n world \n extra ")
+
+	advance, token, err := splitFunc(input, false)
+	assert.NoError(t, err)
+	assert.Equal(t, 8, advance)
+	assert.Equal(t, []byte(" hello "), token)
+
+	advance, token, err = splitFunc(input[8:], false)
+	assert.NoError(t, err)
+	assert.Equal(t, 8, advance)
+	assert.Equal(t, []byte(" world "), token)
+
+	advance, token, err = splitFunc(input[16:], false)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, advance)
+	assert.Nil(t, token)
+
+	time.Sleep(2 * flushPeriod)
+
+	advance, token, err = splitFunc(input[16:], false)
+	assert.NoError(t, err)
+	assert.Equal(t, 7, advance)
+	assert.Equal(t, []byte(" extra "), token)
 }
diff --git a/pkg/stanza/fileconsumer/internal/splitter/multiline_test.go b/pkg/stanza/fileconsumer/internal/splitter/multiline_test.go
@@ -8,52 +8,107 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/assert"
-	"golang.org/x/text/encoding"
 	"golang.org/x/text/encoding/unicode"
 
 	"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/split"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/trim"
 )
 
-func TestSplitFuncFactory(t *testing.T) {
-	tests := []struct {
-		name        string
-		splitConfig split.Config
-		encoding    encoding.Encoding
-		maxLogSize  int
-		flushPeriod time.Duration
-		wantErr     bool
-	}{
-		{
-			name:        "default configuration",
-			encoding:    unicode.UTF8,
-			maxLogSize:  1024,
-			flushPeriod: 100 * time.Millisecond,
-			wantErr:     false,
-		},
-		{
-			name: "split config  error",
-			splitConfig: split.Config{
-				LineStartPattern: "START",
-				LineEndPattern:   "END",
-			},
-			flushPeriod: 100 * time.Millisecond,
-			encoding:    unicode.UTF8,
-			maxLogSize:  1024,
-			wantErr:     true,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			factory := NewSplitFuncFactory(tt.splitConfig, tt.encoding, tt.maxLogSize, trim.Nop, tt.flushPeriod)
-			got, err := factory.SplitFunc()
-			if (err != nil) != tt.wantErr {
-				t.Errorf("SplitFunc() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-			if err == nil {
-				assert.NotNil(t, got)
-			}
-		})
+func TestSplitFuncError(t *testing.T) {
+	sCfg := split.Config{
+		LineStartPattern: "START",
+		LineEndPattern:   "END",
 	}
+	factory := NewSplitFuncFactory(sCfg, unicode.UTF8, 1024, trim.Nop, 0)
+	splitFunc, err := factory.SplitFunc()
+	assert.Error(t, err)
+	assert.Nil(t, splitFunc)
+}
+
+func TestSplitFunc(t *testing.T) {
+	factory := NewSplitFuncFactory(split.Config{}, unicode.UTF8, 1024, trim.Nop, 0)
+	splitFunc, err := factory.SplitFunc()
+	assert.NoError(t, err)
+	assert.NotNil(t, splitFunc)
+
+	input := []byte(" hello \n world \n extra ")
+
+	advance, token, err := splitFunc(input, false)
+	assert.NoError(t, err)
+	assert.Equal(t, 8, advance)
+	assert.Equal(t, []byte(" hello "), token)
+
+	advance, token, err = splitFunc(input[8:], false)
+	assert.NoError(t, err)
+	assert.Equal(t, 8, advance)
+	assert.Equal(t, []byte(" world "), token)
+
+	advance, token, err = splitFunc(input[16:], false)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, advance)
+	assert.Nil(t, token)
+}
+
+func TestSplitFuncWithFlush(t *testing.T) {
+	flushPeriod := 100 * time.Millisecond
+	factory := NewSplitFuncFactory(split.Config{}, unicode.UTF8, 1024, trim.Nop, flushPeriod)
+	splitFunc, err := factory.SplitFunc()
+	assert.NoError(t, err)
+	assert.NotNil(t, splitFunc)
+
+	input := []byte(" hello \n world \n extra ")
+
+	advance, token, err := splitFunc(input, false)
+	assert.NoError(t, err)
+	assert.Equal(t, 8, advance)
+	assert.Equal(t, []byte(" hello "), token)
+
+	advance, token, err = splitFunc(input[8:], false)
+	assert.NoError(t, err)
+	assert.Equal(t, 8, advance)
+	assert.Equal(t, []byte(" world "), token)
+
+	advance, token, err = splitFunc(input[16:], false)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, advance)
+	assert.Nil(t, token)
+
+	time.Sleep(2 * flushPeriod)
+
+	advance, token, err = splitFunc(input[16:], false)
+	assert.NoError(t, err)
+	assert.Equal(t, 7, advance)
+	assert.Equal(t, []byte(" extra "), token)
+}
+
+func TestSplitFuncWithTrim(t *testing.T) {
+	flushPeriod := 100 * time.Millisecond
+	factory := NewSplitFuncFactory(split.Config{}, unicode.UTF8, 1024, trim.Whitespace, flushPeriod)
+	splitFunc, err := factory.SplitFunc()
+	assert.NoError(t, err)
+	assert.NotNil(t, splitFunc)
+
+	input := []byte(" hello \n world \n extra ")
+
+	advance, token, err := splitFunc(input, false)
+	assert.NoError(t, err)
+	assert.Equal(t, 8, advance)
+	assert.Equal(t, []byte("hello"), token)
+
+	advance, token, err = splitFunc(input[8:], false)
+	assert.NoError(t, err)
+	assert.Equal(t, 8, advance)
+	assert.Equal(t, []byte("world"), token)
+
+	advance, token, err = splitFunc(input[16:], false)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, advance)
+	assert.Nil(t, token)
+
+	time.Sleep(2 * flushPeriod)
+
+	advance, token, err = splitFunc(input[16:], false)
+	assert.NoError(t, err)
+	assert.Equal(t, 7, advance)
+	assert.Equal(t, []byte("extra"), token) // Ensure trim applies to flushed token
 }