reflow/syntax: implement full comprehension support (cross products, …

…filters)

Summary:
This change adds full comprehension support into Reflow. Before this
change, Reflow supported only a limited form of comprehensions:
mapping over a single list or map. This change introduces support for
cross products and predicates, which can be combined arbitrarily.

Each comprehension takes a number of clauses, each of which is either
an "enumeration" or a "filter". Enumerations are the familiar

	pat <- e1

form, while filters are of the form

	if e1

The list of clauses are separated by commas. After this change,
all of the following work:

	val TestCompr = {
		l := [1,2,3,4,5,6,7,8,9]
		x2 := [x*2 | x <- l]
		test.All([x*2 == y | (x, y) <- zip(l, x2)])
	}

	val TestCompr2 = {
		strings := make("$/strings")
		x := ["a", "b", "c"]
		y := [1, 2]
		xy := [x + strings.FromInt(y) | x <- x, y <- y]
		test.All([x == y | (x, y) <- zip(xy, ["a1", "a2", "b1", "b2", "c1", "c2"])])
	}

	val TestCompr3 = {
		xs := [1,2,3,4]
		xy := [x*y | x <- xs, if x%2 == 0, y <- xs]
		expect := [2, 4, 6, 8, 4, 8, 12, 16]
		test.All([x == y | (x, y) <- zip(xy, expect)])
	}

	val TestCompr4 = {
		xss := [["a", "b", "c"], ["e"], ["f", "g", "h"]]
		xy := [x | xs <- xss, x <- xs, if x != "a"]
		expect := ["b", "c", "e", "f", "g", "h"]
		test.All([x == y | (x, y) <- zip(xy, expect)])
	}

Full comprehensions can replace a lot of other ad-hoc functionality.
For example, flatten(x) can be expressed as [x | xxs <- xs, x <- xs].
There is other existing Reflow code which could benefit from filtering
(e.g., filter a directory of files).

Reviewers: pgopal, escott

Reviewed By: pgopal

Subscribers: sbagaria, escott

Differential Revision: https://phabricator.grailbio.com/D8862

fbshipit-source-id: 091a8a9

LANGUAGE.md

@@ -181,6 +181,23 @@ val integers = [(1, 2), (-4, 3), (1, 1)]
 val multiplied = [absMul(x, y) | (x, y) <- integers]
 </pre>
 Each pair of integers are multiplied together, producing <code>[2, 12, 1]</code>.
+<p>
+Comprehensions can range over multiple lists (or maps),
+computing the cross product of these. For example
+<pre>
+val integers = [1,2,3,4]
+val chars = ["a", "b", "c"]
+val cross = [(i, c) | i <- integers, c <- chars]
+</pre>
+computes the cross product <code>[(1, "a"), (1, "b"), (1, "c"), (2, "a"), (2, "b"), (2, "c"), (3, "a"), (3, "b"), (3, "c"), (4, "a"), (4, "b"), (4, "c")]</code>.
+<p>
+Finally, comprehensions can apply filters.
+As an example, the following computes the
+cross product only for even integers:
+<pre>
+val evenCross = [(i, c) | i <- integers, if i%2 == 0, c <- chars]
+</pre>
+resulting in <code>[(2, "a"), (2, "b"), (2, "c"), (4, "a"), (4, "b"), (4, "c")]</code>.
   </dd>
 </dl>
 

syntax/digest.go

@@ -29,7 +29,7 @@ func init() {
 }
 
 func digestN(i int) digest.Digest {
-	if i < len(nDigest) {
+	if i >= 0 && i < len(nDigest) {
 		return nDigest[i]
 	}
 	return reflow.Digester.FromBytes([]byte{byte(i)})
@@ -161,9 +161,16 @@ func (e *Expr) digest(w io.Writer, env *values.Env) {
 		e.Right.digest(w, env)
 	case ExprCompr:
 		env = env.Push()
-		d := e.Left.Digest(env)
-		for _, id := range e.Pat.Idents(nil) {
-			env.Bind(id, d)
+		for _, clause := range e.ComprClauses {
+			switch clause.Kind {
+			case ComprEnum:
+				d := clause.Expr.Digest(env)
+				for _, id := range clause.Pat.Idents(nil) {
+					env.Bind(id, d)
+				}
+			case ComprFilter:
+				clause.Expr.digest(w, env)
+			}
 		}
 		e.ComprExpr.digest(w, env)
 	case ExprThunk:

syntax/digest_test.go

@@ -10,7 +10,7 @@ import (
 	"github.com/grailbio/reflow"
 )
 
-func TestStableDigest(t *testing.T) {
+func TestDigestExec(t *testing.T) {
 	for _, expr := range []string{
 		`exec(image := "ubuntu") (out file) {" cp {{file("s3://blah")}} {{out}} "}`,
 		`exec(image := "ubuntu") (out file) {" cp {{    file("s3://blah")  }} {{  out}} "}`,
@@ -27,3 +27,35 @@ func TestStableDigest(t *testing.T) {
 		}
 	}
 }
+
+func TestDigestDelay(t *testing.T) {
+	for _, expr := range []string{
+		`{x := 1; delay(x)}`,
+		`{y := 1; delay(y)}`,
+	} {
+		v, _, _, err := eval(expr)
+		if err != nil {
+			t.Fatalf("%s: %v", expr, err)
+		}
+		f := v.(*reflow.Flow)
+		if got, want := f.Digest().String(), "sha256:df14f3294bd1c14c9fd6423b6078f4699ae85d44bdf5c44bf838cbc6e9c99db1"; got != want {
+			t.Errorf("got %v, want %v", got, want)
+		}
+	}
+}
+
+func TestDigestCompr(t *testing.T) {
+	for _, expr := range []string{
+		`[x*x | x <- delay([1,2,3])]`,
+		`[y*y | y <- delay([1,2,3])]`,
+	} {
+		v, _, _, err := eval(expr)
+		if err != nil {
+			t.Fatalf("%s: %v", expr, err)
+		}
+		f := v.(*reflow.Flow)
+		if got, want := f.Digest().String(), "sha256:8310ad9a33309b3b9b37c1bed2ec7898757cad3b3b5929aee538797bfee8fba0"; got != want {
+			t.Errorf("got %v, want %v", got, want)
+		}
+	}
+}

syntax/doc.go

@@ -67,11 +67,15 @@ expressions, d1, d2, .. are declarations; t1, t2, .. are types):
 	make(strlit, d1, ..., dn)          // builtin make primitive. identifiers are valid declarations in
 	                                   // this context; they are deparsed as id := id.
 	panic(e1)                          // terminate the program with error e1
-	[e1 | p1 <- e2]                    // list comprehension: bind pattern p1 using expression e2,
-	                                   // evaluate e1 with this environment.
+	[e1 | c1, c2,..., cn]              // list comprehension: evaluate e1 in the environment provided by
+	                                   // the given clauses (see below)
 	e1 ~> e2                           // force evaluation of e1, ignore its result, then evaluate e2.
 	flatten(e1)                        // flatten a list of lists into a single list
 
+A comprehension clause is one of the following:
+
+	pat <- e1                          // bind a pattern to each of the list or map e1
+	if e1                              // filter entries that do not meet the predicate e1
 
 A Reflow declaration is one of the following: