Skip to content

Commit

Permalink
internal/cueimports: export API and fix for CUE syntax
Browse files Browse the repository at this point in the history 
The original code still needed some adapting from the original
Go version. Specifically:
- it looked for semicolons not commas
- it recognized the Go backquoted string syntax
- it recognized the Go /*...*/ comment syntax

Also, make it a little less general, as currently we don't
seem to need the import-saving functionality (we can use
`parser.ParseFile` with `CommentsOnly` which will keep us
honest, even at a little bit of runtime cost), and because
of that, we don't need to try to detect syntax errors.
We can optimize at a later date if need be.

Signed-off-by: Roger Peppe <rogpeppe@gmail.com>
Change-Id: I276087961c73d04222a115b462e01e130b4cd4e3
Reviewed-on: https://review.gerrithub.io/c/cue-lang/cue/+/1172690
TryBot-Result: CUEcueckoo <cueckoo@cuelang.org>
Unity-Result: CUE porcuepine <cue.porcuepine@gmail.com>
Reviewed-by: Daniel Martí <mvdan@mvdan.cc>
  • Loading branch information
@rogpeppe
rogpeppe committed Nov 24, 2023
1 parent 9e55783 commit 47d6dce
Show file tree
Hide file tree
Showing 2 changed files with 56 additions and 104 deletions.
64 changes: 20 additions & 44 deletions internal/cueimports/read.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,4 @@
//go:build ignore

// Copyright 2018 The CUE Authors
// Copyright 2023 The CUE Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand All @@ -14,7 +12,9 @@
// See the License for the specific language governing permissions and
// limitations under the License.

package load
// Package cueimports provides support for reading the import
// section of a CUE file without needing to read the rest of it.
package cueimports

import (
"bufio"
Expand Down Expand Up @@ -76,7 +76,7 @@ func (r *importReader) readByte() byte {
func (r *importReader) peekByte(skipSpace bool) byte {
if r.err != nil {
if r.nerr++; r.nerr > 10000 {
panic("go/build: import reader looping")
panic("import reader looping")
}
return 0
}
Expand All @@ -90,10 +90,10 @@ func (r *importReader) peekByte(skipSpace bool) byte {
}
for r.err == nil && !r.eof {
if skipSpace {
// For the purposes of this reader, semicolons are never necessary to
// For the purposes of this reader, commas are never necessary to
// understand the input and are treated as spaces.
switch c {
case ' ', '\f', '\t', '\r', '\n', ';':
case ' ', '\f', '\t', '\r', '\n', ',':
c = r.readByte()
continue

Expand All @@ -103,14 +103,6 @@ func (r *importReader) peekByte(skipSpace bool) byte {
for c != '\n' && r.err == nil && !r.eof {
c = r.readByte()
}
} else if c == '*' {
var c1 byte
for (c != '*' || c1 != '/') && r.err == nil {
if r.eof {
r.syntaxError()
}
c, c1 = c1, r.readByte()
}
} else {
r.syntaxError()
}
Expand Down Expand Up @@ -161,29 +153,15 @@ func (r *importReader) readIdent() {

// readString reads a quoted string literal from the input.
// If an identifier is not present, readString records a syntax error.
func (r *importReader) readString(save *[]string) {
func (r *importReader) readString() {
switch r.nextByte(true) {
case '`':
start := len(r.buf) - 1
for r.err == nil {
if r.nextByte(false) == '`' {
if save != nil {
*save = append(*save, string(r.buf[start:]))
}
break
}
if r.eof {
r.syntaxError()
}
}
case '"':
start := len(r.buf) - 1
// Note: although the syntax in the specification only allows
// a simple string literal here, the cue/parser package also
// allows #"..."# and """ literals, so there's some impedance-mismatch here.
for r.err == nil {
c := r.nextByte(false)
if c == '"' {
if save != nil {
*save = append(*save, string(r.buf[start:]))
}
break
}
if r.eof || c == '\n' {
Expand All @@ -200,19 +178,17 @@ func (r *importReader) readString(save *[]string) {

// readImport reads an import clause - optional identifier followed by quoted string -
// from the input.
func (r *importReader) readImport(imports *[]string) {
func (r *importReader) readImport() {
c := r.peekByte(true)
if c == '.' {
r.peek = 0
} else if isIdent(c) {
if isIdent(c) {
r.readIdent()
}
r.readString(imports)
r.readString()
}

// readImports is like io.ReadAll, except that it expects a CUE file as
// Read is like io.ReadAll, except that it expects a CUE file as
// input and stops reading the input once the imports have completed.
func readImports(f io.Reader, reportSyntaxError bool, imports *[]string) ([]byte, errors.Error) {
func Read(f io.Reader) ([]byte, errors.Error) {
r := &importReader{b: bufio.NewReader(f)}

r.readKeyword("package")
Expand All @@ -222,11 +198,11 @@ func readImports(f io.Reader, reportSyntaxError bool, imports *[]string) ([]byte
if r.peekByte(true) == '(' {
r.nextByte(false)
for r.peekByte(true) != ')' && r.err == nil {
r.readImport(imports)
r.readImport()
}
r.nextByte(false)
} else {
r.readImport(imports)
r.readImport()
}
}

Expand All @@ -237,8 +213,8 @@ func readImports(f io.Reader, reportSyntaxError bool, imports *[]string) ([]byte
}

// If we stopped for a syntax error, consume the whole file so that
// we are sure we don't change the errors that go/parser returns.
if r.err == errSyntax && !reportSyntaxError {
// we are sure we don't change the errors that the cue/parser returns.
if r.err == errSyntax {
r.err = nil
for r.err == nil && !r.eof {
r.readByte()
Expand Down
96 changes: 36 additions & 60 deletions internal/cueimports/read_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
//go:build ignore

// Copyright 2018 The CUE Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
Expand All @@ -14,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.

package load
package cueimports

import (
"io"
Expand All @@ -24,8 +22,6 @@ import (
"cuelang.org/go/cue/errors"
)

const quote = "`"

type readTest struct {
// Test input contains ℙ where readImports should stop.
in string
Expand All @@ -38,15 +34,15 @@ var readImportsTests = []readTest{
"",
},
{
`package p; import "x"`,
`package p, import "x"`,
"",
},
{
`package p; import . "x"`,
`package p, import . "x"`,
"",
},
{
`package p; import "x";ℙvar x = 1`,
`package p, import "x",ℙvar x = 1`,
"",
},
{
Expand All @@ -62,15 +58,12 @@ var readImportsTests = []readTest{
"x"
_ "x"
a "x" // comment
` + quote + `x` + quote + `
_ ` + quote + `x` + quote + `
a ` + quote + `x` + quote + `
)
import (
)
import ()
import()import()import()
import();import();import()
import(),import(),import()
ℙvar x = 1
`,
Expand Down Expand Up @@ -116,97 +109,80 @@ func testRead(t *testing.T, tests []readTest, read func(io.Reader) ([]byte, erro

func TestReadImports(t *testing.T) {
testRead(t, readImportsTests, func(r io.Reader) ([]byte, errors.Error) {
return readImports(r, true, nil)
return Read(r)
})
}

var readFailuresTests = []readTest{
{
`package`,
"syntax error",
"",
},
{
"package p\n\x00\nimport `math`\n",
"unexpected NUL in input",
},
{
`package p; import`,
"syntax error",
`package p, import`,
"",
},
{
`package p; import "`,
"syntax error",
`package p, import "`,
"",
},
{
"package p; import ` \n\n",
"syntax error",
"package p, import ` \n\n",
"",
},
{
`package p; import "x`,
"syntax error",
`package p, import "x`,
"",
},
{
`package p; import _`,
"syntax error",
`package p, import _`,
"",
},
{
`package p; import _ "`,
"syntax error",
`package p, import _ "`,
"",
},
{
`package p; import _ "x`,
"syntax error",
`package p, import _ "x`,
"",
},
{
`package p; import .`,
"syntax error",
`package p, import .`,
"",
},
{
`package p; import . "`,
"syntax error",
`package p, import . "`,
"",
},
{
`package p; import . "x`,
"syntax error",
`package p, import . "x`,
"",
},
{
`package p; import (`,
"syntax error",
`package p, import (`,
"",
},
{
`package p; import ("`,
"syntax error",
`package p, import ("`,
"",
},
{
`package p; import ("x`,
"syntax error",
`package p, import ("x`,
"",
},
{
`package p; import ("x"`,
"syntax error",
`package p, import ("x"`,
"",
},
}

func TestReadFailures(t *testing.T) {
// Errors should be reported (true arg to readImports).
testRead(t, readFailuresTests, func(r io.Reader) ([]byte, errors.Error) {
return readImports(r, true, nil)
})
}

func TestReadFailuresIgnored(t *testing.T) {
// Syntax errors should not be reported (false arg to readImports).
// Instead, entire file should be the output and no error.
// Convert tests not to return syntax errors.
tests := make([]readTest, len(readFailuresTests))
copy(tests, readFailuresTests)
for i := range tests {
tt := &tests[i]
if !strings.Contains(tt.err, "NUL") {
tt.err = ""
}
}
testRead(t, tests, func(r io.Reader) ([]byte, errors.Error) {
return readImports(r, false, nil)
return Read(r)
})
}

0 comments on commit 47d6dce

Please sign in to comment.