From 47d6dcedde5c8c8623758044eb247c172ba4e4bf Mon Sep 17 00:00:00 2001 From: Roger Peppe Date: Thu, 23 Nov 2023 15:55:50 +0000 Subject: [PATCH] internal/cueimports: export API and fix for CUE syntax MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The original code still needed some adapting from the original Go version. Specifically: - it looked for semicolons not commas - it recognized the Go backquoted string syntax - it recognized the Go /*...*/ comment syntax Also, make it a little less general, as currently we don't seem to need the import-saving functionality (we can use `parser.ParseFile` with `CommentsOnly` which will keep us honest, even at a little bit of runtime cost), and because of that, we don't need to try to detect syntax errors. We can optimize at a later date if need be. Signed-off-by: Roger Peppe Change-Id: I276087961c73d04222a115b462e01e130b4cd4e3 Reviewed-on: https://review.gerrithub.io/c/cue-lang/cue/+/1172690 TryBot-Result: CUEcueckoo Unity-Result: CUE porcuepine Reviewed-by: Daniel Martí --- internal/cueimports/read.go | 64 +++++++-------------- internal/cueimports/read_test.go | 96 ++++++++++++-------------------- 2 files changed, 56 insertions(+), 104 deletions(-) diff --git a/internal/cueimports/read.go b/internal/cueimports/read.go index 285237806f6..af0ed94567d 100644 --- a/internal/cueimports/read.go +++ b/internal/cueimports/read.go @@ -1,6 +1,4 @@ -//go:build ignore - -// Copyright 2018 The CUE Authors +// Copyright 2023 The CUE Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,7 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -package load +// Package cueimports provides support for reading the import +// section of a CUE file without needing to read the rest of it. +package cueimports import ( "bufio" @@ -76,7 +76,7 @@ func (r *importReader) readByte() byte { func (r *importReader) peekByte(skipSpace bool) byte { if r.err != nil { if r.nerr++; r.nerr > 10000 { - panic("go/build: import reader looping") + panic("import reader looping") } return 0 } @@ -90,10 +90,10 @@ func (r *importReader) peekByte(skipSpace bool) byte { } for r.err == nil && !r.eof { if skipSpace { - // For the purposes of this reader, semicolons are never necessary to + // For the purposes of this reader, commas are never necessary to // understand the input and are treated as spaces. switch c { - case ' ', '\f', '\t', '\r', '\n', ';': + case ' ', '\f', '\t', '\r', '\n', ',': c = r.readByte() continue @@ -103,14 +103,6 @@ func (r *importReader) peekByte(skipSpace bool) byte { for c != '\n' && r.err == nil && !r.eof { c = r.readByte() } - } else if c == '*' { - var c1 byte - for (c != '*' || c1 != '/') && r.err == nil { - if r.eof { - r.syntaxError() - } - c, c1 = c1, r.readByte() - } } else { r.syntaxError() } @@ -161,29 +153,15 @@ func (r *importReader) readIdent() { // readString reads a quoted string literal from the input. // If an identifier is not present, readString records a syntax error. -func (r *importReader) readString(save *[]string) { +func (r *importReader) readString() { switch r.nextByte(true) { - case '`': - start := len(r.buf) - 1 - for r.err == nil { - if r.nextByte(false) == '`' { - if save != nil { - *save = append(*save, string(r.buf[start:])) - } - break - } - if r.eof { - r.syntaxError() - } - } case '"': - start := len(r.buf) - 1 + // Note: although the syntax in the specification only allows + // a simple string literal here, the cue/parser package also + // allows #"..."# and """ literals, so there's some impedance-mismatch here. for r.err == nil { c := r.nextByte(false) if c == '"' { - if save != nil { - *save = append(*save, string(r.buf[start:])) - } break } if r.eof || c == '\n' { @@ -200,19 +178,17 @@ func (r *importReader) readString(save *[]string) { // readImport reads an import clause - optional identifier followed by quoted string - // from the input. -func (r *importReader) readImport(imports *[]string) { +func (r *importReader) readImport() { c := r.peekByte(true) - if c == '.' { - r.peek = 0 - } else if isIdent(c) { + if isIdent(c) { r.readIdent() } - r.readString(imports) + r.readString() } -// readImports is like io.ReadAll, except that it expects a CUE file as +// Read is like io.ReadAll, except that it expects a CUE file as // input and stops reading the input once the imports have completed. -func readImports(f io.Reader, reportSyntaxError bool, imports *[]string) ([]byte, errors.Error) { +func Read(f io.Reader) ([]byte, errors.Error) { r := &importReader{b: bufio.NewReader(f)} r.readKeyword("package") @@ -222,11 +198,11 @@ func readImports(f io.Reader, reportSyntaxError bool, imports *[]string) ([]byte if r.peekByte(true) == '(' { r.nextByte(false) for r.peekByte(true) != ')' && r.err == nil { - r.readImport(imports) + r.readImport() } r.nextByte(false) } else { - r.readImport(imports) + r.readImport() } } @@ -237,8 +213,8 @@ func readImports(f io.Reader, reportSyntaxError bool, imports *[]string) ([]byte } // If we stopped for a syntax error, consume the whole file so that - // we are sure we don't change the errors that go/parser returns. - if r.err == errSyntax && !reportSyntaxError { + // we are sure we don't change the errors that the cue/parser returns. + if r.err == errSyntax { r.err = nil for r.err == nil && !r.eof { r.readByte() diff --git a/internal/cueimports/read_test.go b/internal/cueimports/read_test.go index 9fa8c33bc19..f4e9b17f0cd 100644 --- a/internal/cueimports/read_test.go +++ b/internal/cueimports/read_test.go @@ -1,5 +1,3 @@ -//go:build ignore - // Copyright 2018 The CUE Authors // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package load +package cueimports import ( "io" @@ -24,8 +22,6 @@ import ( "cuelang.org/go/cue/errors" ) -const quote = "`" - type readTest struct { // Test input contains ℙ where readImports should stop. in string @@ -38,15 +34,15 @@ var readImportsTests = []readTest{ "", }, { - `package p; import "x"`, + `package p, import "x"`, "", }, { - `package p; import . "x"`, + `package p, import . "x"`, "", }, { - `package p; import "x";ℙvar x = 1`, + `package p, import "x",ℙvar x = 1`, "", }, { @@ -62,15 +58,12 @@ var readImportsTests = []readTest{ "x" _ "x" a "x" // comment - ` + quote + `x` + quote + ` - _ ` + quote + `x` + quote + ` - a ` + quote + `x` + quote + ` ) import ( ) import () import()import()import() - import();import();import() + import(),import(),import() ℙvar x = 1 `, @@ -116,97 +109,80 @@ func testRead(t *testing.T, tests []readTest, read func(io.Reader) ([]byte, erro func TestReadImports(t *testing.T) { testRead(t, readImportsTests, func(r io.Reader) ([]byte, errors.Error) { - return readImports(r, true, nil) + return Read(r) }) } var readFailuresTests = []readTest{ { `package`, - "syntax error", + "", }, { "package p\n\x00\nimport `math`\n", "unexpected NUL in input", }, { - `package p; import`, - "syntax error", + `package p, import`, + "", }, { - `package p; import "`, - "syntax error", + `package p, import "`, + "", }, { - "package p; import ` \n\n", - "syntax error", + "package p, import ` \n\n", + "", }, { - `package p; import "x`, - "syntax error", + `package p, import "x`, + "", }, { - `package p; import _`, - "syntax error", + `package p, import _`, + "", }, { - `package p; import _ "`, - "syntax error", + `package p, import _ "`, + "", }, { - `package p; import _ "x`, - "syntax error", + `package p, import _ "x`, + "", }, { - `package p; import .`, - "syntax error", + `package p, import .`, + "", }, { - `package p; import . "`, - "syntax error", + `package p, import . "`, + "", }, { - `package p; import . "x`, - "syntax error", + `package p, import . "x`, + "", }, { - `package p; import (`, - "syntax error", + `package p, import (`, + "", }, { - `package p; import ("`, - "syntax error", + `package p, import ("`, + "", }, { - `package p; import ("x`, - "syntax error", + `package p, import ("x`, + "", }, { - `package p; import ("x"`, - "syntax error", + `package p, import ("x"`, + "", }, } func TestReadFailures(t *testing.T) { // Errors should be reported (true arg to readImports). testRead(t, readFailuresTests, func(r io.Reader) ([]byte, errors.Error) { - return readImports(r, true, nil) - }) -} - -func TestReadFailuresIgnored(t *testing.T) { - // Syntax errors should not be reported (false arg to readImports). - // Instead, entire file should be the output and no error. - // Convert tests not to return syntax errors. - tests := make([]readTest, len(readFailuresTests)) - copy(tests, readFailuresTests) - for i := range tests { - tt := &tests[i] - if !strings.Contains(tt.err, "NUL") { - tt.err = "" - } - } - testRead(t, tests, func(r io.Reader) ([]byte, errors.Error) { - return readImports(r, false, nil) + return Read(r) }) }