Add optional lexer interfaces for fast-pathing strings and byte slices.

lexer/lexer.go

@@ -25,6 +25,18 @@ type Definition interface {
 	Lex(filename string, r io.Reader) (Lexer, error)
 }
 
+// StringDefinition is an optional interface lexer Definition's can implement
+// to offer a fast path for lexing strings.
+type StringDefinition interface {
+	LexString(filename string, input string) (Lexer, error)
+}
+
+// BytesDefinition is an optional interface lexer Definition's can implement
+// to offer a fast path for lexing byte slices.
+type BytesDefinition interface {
+	LexBytes(filename string, input []byte) (Lexer, error)
+}
+
 // A Lexer returns tokens from a source.
 type Lexer interface {
 	// Next consumes and returns the next token.

lexer/stateful/codegen/codegen_test.go

@@ -41,23 +41,23 @@ func TestGenerate(t *testing.T) {
 	err := codegen.Generate(w, "codegen_test", exprLexer)
 	require.NoError(t, err)
 	source := w.String()
-	cmd := exec.Command("pbcopy")
-	cmd.Stdin = strings.NewReader(source)
-	err = cmd.Run()
-	require.NoError(t, err)
+	// cmd := exec.Command("pbcopy")
+	// cmd.Stdin = strings.NewReader(source)
+	// err = cmd.Run()
+	// require.NoError(t, err)
 
 	formatted := &bytes.Buffer{}
-	cmd = exec.Command("goimports")
+	cmd := exec.Command("gofmt", "-s")
 	cmd.Stdin = strings.NewReader(source)
 	cmd.Stdout = formatted
 	cmd.Stderr = os.Stderr
 	err = cmd.Run()
 	require.NoError(t, err, source)
 
-	cmd = exec.Command("pbcopy")
-	cmd.Stdin = formatted
-	err = cmd.Run()
-	require.NoError(t, err)
+	// cmd = exec.Command("pbcopy")
+	// cmd.Stdin = formatted
+	// err = cmd.Run()
+	// require.NoError(t, err)
 }
 
 func BenchmarkStatefulGenerated(b *testing.B) {

parser.go

@@ -152,10 +152,18 @@ func (p *Parser) ParseFromLexer(lex *lexer.PeekingLexer, v interface{}, options
 	return p.parseOne(ctx, rv)
 }
 
+func (p *Parser) parse(lex lexer.Lexer, v interface{}, options ...ParseOption) (err error) {
+	peeker, err := lexer.Upgrade(lex, p.getElidedTypes()...)
+	if err != nil {
+		return err
+	}
+	return p.ParseFromLexer(peeker, v, options...)
+}
+
 // Parse from r into grammar v which must be of the same type as the grammar passed to
 // Build().
 //
-// This may return a Error.
+// This may return an Error.
 func (p *Parser) Parse(filename string, r io.Reader, v interface{}, options ...ParseOption) (err error) {
 	if filename == "" {
 		filename = lexer.NameOfReader(r)
@@ -164,27 +172,35 @@ func (p *Parser) Parse(filename string, r io.Reader, v interface{}, options ...P
 	if err != nil {
 		return err
 	}
-	peeker, err := lexer.Upgrade(lex, p.getElidedTypes()...)
-	if err != nil {
-		return err
-	}
-	return p.ParseFromLexer(peeker, v, options...)
+	return p.parse(lex, v, options...)
 }
 
 // ParseString from s into grammar v which must be of the same type as the grammar passed to
 // Build().
 //
-// This may return a Error.
-func (p *Parser) ParseString(filename string, s string, v interface{}, options ...ParseOption) error {
-	return p.Parse(filename, strings.NewReader(s), v, options...)
+// This may return an Error.
+func (p *Parser) ParseString(filename string, s string, v interface{}, options ...ParseOption) (err error) {
+	var lex lexer.Lexer
+	if sl, ok := p.lex.(lexer.StringDefinition); ok {
+		lex, err = sl.LexString(filename, s)
+	} else {
+		lex, err = p.lex.Lex(filename, strings.NewReader(s))
+	}
+	return p.parse(lex, v, options...)
 }
 
 // ParseBytes from b into grammar v which must be of the same type as the grammar passed to
 // Build().
 //
-// This may return a Error.
-func (p *Parser) ParseBytes(filename string, b []byte, v interface{}, options ...ParseOption) error {
-	return p.Parse(filename, bytes.NewReader(b), v, options...)
+// This may return an Error.
+func (p *Parser) ParseBytes(filename string, b []byte, v interface{}, options ...ParseOption) (err error) {
+	var lex lexer.Lexer
+	if sl, ok := p.lex.(lexer.BytesDefinition); ok {
+		lex, err = sl.LexBytes(filename, b)
+	} else {
+		lex, err = p.lex.Lex(filename, bytes.NewReader(b))
+	}
+	return p.parse(lex, v, options...)
 }
 
 func (p *Parser) parseStreaming(ctx *parseContext, rv reflect.Value) error {