internal/core/adt: support interpolation of bytes and bool

Also fixes bytes interpolation, which was still WIP.

For bool: use JSON representation

For bytes: assume bytes are UTF-8 and replace illegal characters according to the recommendations of the Unicode consortium and W3C requirement for character encodings. (So not the Go standard for replacement.)

Details clarified in spec.

Fixes #475.

Change-Id: I94c068f8a73a3948194b179a33e556c37692c05f
Reviewed-on: https://cue-review.googlesource.com/c/cue/+/6951
Reviewed-by: Marcel van Lohuizen <mpvl@golang.org>
Reviewed-by: CUE cueckoo <cueckoo@gmail.com>

cue/testdata/interpolation/041_interpolation.txtar

@@ -33,7 +33,7 @@ e: _|_ // expression in interpolation must evaluate to a number kind or string (
 }
 -- out/eval --
 Errors:
-e: invalid interpolation: cannot use [] (type list) as type (string|number):
+e: invalid interpolation: cannot use [] (type list) as type (bool|string|bytes|number):
     ./in.cue:7:4
 
 Result:
@@ -52,7 +52,7 @@ Result:
   }
   r: (_){ _ }
   e: (_|_){
-    // [eval] e: invalid interpolation: cannot use [] (type list) as type (string|number):
+    // [eval] e: invalid interpolation: cannot use [] (type list) as type (bool|string|bytes|number):
     //     ./in.cue:7:4
   }
 }

cue/testdata/interpolation/scalars.txtar

@@ -0,0 +1,48 @@
+-- in.cue --
+bool1: "1+1=2:  \(true)"
+bool1: "1+1=2:  \(true)"
+bool2: "1+1=1:  \(false)"
+
+// one replacement character
+b1: 'a\xED\x95a'
+bytes1s: "\(b1)"
+bytes1b: '\(b1)'
+
+// two replacement characters
+b2: 'a\x80\x95a'
+bytes2s: "\(b2)"
+bytes2b: '\(b2)'
+
+// preserve precision
+n1: "\(1) \(2.00)"
+
+// but normalize representation
+n2: "\(1e2)"
+-- out/eval --
+(struct){
+  bool1: (string){ "1+1=2:  true" }
+  bool2: (string){ "1+1=1:  false" }
+  b1: (bytes){ 'a\xed\x95a' }
+  bytes1s: (string){ "a�a" }
+  bytes1b: (bytes){ 'a\xed\x95a' }
+  b2: (bytes){ 'a\x80\x95a' }
+  bytes2s: (string){ "a��a" }
+  bytes2b: (bytes){ 'a\x80\x95a' }
+  n1: (string){ "1 2.00" }
+  n2: (string){ "1E+2" }
+}
+-- out/compile --
+--- in.cue
+{
+  bool1: "1+1=2:  \(true)"
+  bool1: "1+1=2:  \(true)"
+  bool2: "1+1=1:  \(false)"
+  b1: 'a\xed\x95a'
+  bytes1s: "\(〈0;b1〉)"
+  bytes1b: '\(〈0;b1〉)'
+  b2: 'a\x80\x95a'
+  bytes2s: "\(〈0;b2〉)"
+  bytes2b: '\(〈0;b2〉)'
+  n1: "\(1) \(2.00)"
+  n2: "\(1E+2)"
+}

doc/ref/spec.md

@@ -2609,8 +2609,21 @@ expressions with their string representation.
 String interpolation may be used in single- and double-quoted strings, as well
 as their multiline equivalent.
 
-A placeholder consists of "\(" followed by an expression and a ")". The
-expression is evaluated within the scope within which the string is defined.
+A placeholder consists of "\(" followed by an expression and a ")".
+The expression is evaluated in the scope within which the string is defined.
+
+The result of the expression is substituted as follows:
+- string: as is
+- bool: the JSON representation of the bool
+- number: a JSON representation of the number that preserves the
+precision of the underlying binary coded decimal
+- bytes: as if substituted within single quotes or
+converted to valid UTF-8 replacing the
+maximal subpart of ill-formed subsequences with a single
+replacement character (W3C encoding standard) otherwise
+- list: illegal
+- struct: illegal
+
 
 ```
 a: "World"

internal/core/adt/context.go

@@ -20,7 +20,7 @@ import (
 	"regexp"
 
 	"github.com/cockroachdb/apd/v2"
-	"golang.org/x/text/runes"
+	"golang.org/x/text/encoding/unicode"
 
 	"cuelang.org/go/cue/ast"
 	"cuelang.org/go/cue/errors"
@@ -738,9 +738,17 @@ func (c *OpContext) StringValue(v Value) string {
 	return c.stringValue(v, nil)
 }
 
-// ToString returns the string value of a numeric or string value.
+// ToBytes returns the bytes value of a scalar value.
+func (c *OpContext) ToBytes(v Value) []byte {
+	if x, ok := v.(*Bytes); ok {
+		return x.B
+	}
+	return []byte(c.ToString(v))
+}
+
+// ToString returns the string value of a scalar value.
 func (c *OpContext) ToString(v Value) string {
-	return c.toStringValue(v, StringKind|NumKind, nil)
+	return c.toStringValue(v, StringKind|NumKind|BytesKind|BoolKind, nil)
 
 }
 
@@ -766,18 +774,29 @@ func (c *OpContext) toStringValue(v Value, k Kind, as interface{}) string {
 		return x.Str
 
 	case *Bytes:
-		return string(runes.ReplaceIllFormed().Bytes(x.B))
+		return bytesToString(x.B)
 
 	case *Num:
 		return x.X.String()
 
+	case *Bool:
+		if x.B {
+			return "true"
+		}
+		return "false"
+
 	default:
 		c.addErrf(IncompleteError, c.pos(),
 			"non-concrete value %s (type %s)", c.Str(v), v.Kind())
 	}
 	return ""
 }
 
+func bytesToString(b []byte) string {
+	b, _ = unicode.UTF8.NewDecoder().Bytes(b)
+	return string(b)
+}
+
 func (c *OpContext) bytesValue(v Value, as interface{}) []byte {
 	v = Unwrap(v)
 	if isError(v) {

internal/core/adt/expr.go

@@ -717,8 +717,11 @@ func (x *Interpolation) evaluate(c *OpContext) Value {
 	buf := bytes.Buffer{}
 	for _, e := range x.Parts {
 		v := c.value(e)
-		s := c.ToString(v)
-		buf.WriteString(s)
+		if x.K == BytesKind {
+			buf.Write(c.ToBytes(v))
+		} else {
+			buf.WriteString(c.ToString(v))
+		}
 	}
 	if err := c.Err(); err != nil {
 		err = &Bottom{
@@ -729,9 +732,9 @@ func (x *Interpolation) evaluate(c *OpContext) Value {
 		// return nil
 		return err
 	}
-	// if k == bytesKind {
-	// 	return &BytesLit{x.source, buf.String(), nil}
-	// }
+	if x.K == BytesKind {
+		return &Bytes{x.Src, buf.Bytes(), nil}
+	}
 	return &String{x.Src, buf.String(), nil}
 }
 

internal/core/compile/compile.go

@@ -799,11 +799,16 @@ func (c *compiler) expr(expr ast.Expr) adt.Expr {
 		if len(n.Elts) == 1 {
 			return c.expr(n.Elts[0])
 		}
-		lit := &adt.Interpolation{Src: n, K: adt.StringKind}
+		lit := &adt.Interpolation{Src: n}
 		info, prefixLen, _, err := literal.ParseQuotes(first.Value, last.Value)
 		if err != nil {
 			return c.errf(n, "invalid interpolation: %v", err)
 		}
+		if info.IsDouble() {
+			lit.K = adt.StringKind
+		} else {
+			lit.K = adt.BytesKind
+		}
 		prefix := ""
 		for i := 0; i < len(n.Elts); i += 2 {
 			l, ok := n.Elts[i].(*ast.BasicLit)

internal/core/debug/compact.go

@@ -226,20 +226,7 @@ func (w *compactPrinter) node(n adt.Node) {
 		w.string("]")
 
 	case *adt.Interpolation:
-		w.string(`"`)
-		for i := 0; i < len(x.Parts); i += 2 {
-			if s, ok := x.Parts[i].(*adt.String); ok {
-				w.string(s.Str)
-			} else {
-				w.string("<bad string>")
-			}
-			if i+1 < len(x.Parts) {
-				w.string(`\(`)
-				w.node(x.Parts[i+1])
-				w.string(`)`)
-			}
-		}
-		w.string(`"`)
+		w.interpolation(x)
 
 	case *adt.UnaryExpr:
 		fmt.Fprint(w, x.Op)

internal/core/debug/debug.go

@@ -106,6 +106,36 @@ func (w *printer) shortError(errs errors.Error) {
 	}
 }
 
+func (w *printer) interpolation(x *adt.Interpolation) {
+	quote := `"`
+	if x.K == adt.BytesKind {
+		quote = `'`
+	}
+	w.string(quote)
+	for i := 0; i < len(x.Parts); i += 2 {
+		switch x.K {
+		case adt.StringKind:
+			if s, ok := x.Parts[i].(*adt.String); ok {
+				w.string(s.Str)
+			} else {
+				w.string("<bad string>")
+			}
+		case adt.BytesKind:
+			if s, ok := x.Parts[i].(*adt.Bytes); ok {
+				_, _ = w.Write(s.B)
+			} else {
+				w.string("<bad bytes>")
+			}
+		}
+		if i+1 < len(x.Parts) {
+			w.string(`\(`)
+			w.node(x.Parts[i+1])
+			w.string(`)`)
+		}
+	}
+	w.string(quote)
+}
+
 func (w *printer) node(n adt.Node) {
 	switch x := n.(type) {
 	case *adt.Vertex:
@@ -375,20 +405,7 @@ func (w *printer) node(n adt.Node) {
 		w.string("]")
 
 	case *adt.Interpolation:
-		w.string(`"`)
-		for i := 0; i < len(x.Parts); i += 2 {
-			if s, ok := x.Parts[i].(*adt.String); ok {
-				w.string(s.Str)
-			} else {
-				w.string("<bad string>")
-			}
-			if i+1 < len(x.Parts) {
-				w.string(`\(`)
-				w.node(x.Parts[i+1])
-				w.string(`)`)
-			}
-		}
-		w.string(`"`)
+		w.interpolation(x)
 
 	case *adt.UnaryExpr:
 		fmt.Fprint(w, x.Op)