feat: support annotations in SQL file

pom.xml

@@ -220,6 +220,7 @@
                   src/main/jjtree-sources/ddl_keywords.jjt \
                   src/main/jjtree-sources/ddl_string_bytes_tokens.jjt \
                   src/main/jjtree-sources/ddl_expression.jjt \
+                  src/main/jjtree-sources/ddl_annotation.jjt \
                   src/main/jjtree-sources/ddl_parser.jjt \
                   &gt; ${project.build.directory}/generated-sources/jjtree-src/DdlParser.jjt</argument>
               </arguments>

src/main/java/com/google/cloud/solutions/spannerddl/diff/DdlDiff.java

@@ -707,6 +707,11 @@ private static String getDatabaseNameFromAlterDatabase(List<ASTddl_statement> st
    * @throws DdlDiffException if there is an error in parsing the DDL
    */
   public static List<ASTddl_statement> parseDdl(String original) throws DdlDiffException {
+    // the annotations are prefixed with "--" so that SQL file remains valid.
+    // strip the comment prefix before so that annotations can be parsed.
+    // otherwise they will be ignored as comment lines
+    original = original.replaceAll("-- *@", "@");
+
     // Remove "--" comments and split by ";"
     List<String> statements = Splitter.on(';').splitToList(original.replaceAll("--.*(\n|$)", ""));
     ArrayList<ASTddl_statement> ddlStatements = new ArrayList<>(statements.size());

src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTannotation.java

@@ -0,0 +1,52 @@
+/*
+ * Copyright 2019 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.solutions.spannerddl.parser;
+
+import static com.google.cloud.solutions.spannerddl.diff.AstTreeUtils.getChildByType;
+
+import com.google.cloud.solutions.spannerddl.diff.AstTreeUtils;
+import java.util.ArrayList;
+import java.util.List;
+
+/** Abstract Syntax Tree parser object for "annotation" token */
+public class ASTannotation extends SimpleNode {
+  public ASTannotation(int id) {
+    super(id);
+  }
+
+  public ASTannotation(DdlParser p, int id) {
+    super(p, id);
+  }
+
+  public String getName() {
+    return AstTreeUtils.tokensToString(getChildByType(children, ASTname.class));
+  }
+
+  public List<ASTannotation_param> getParams() {
+    List<ASTannotation_param> params = new ArrayList<>();
+    for (Node child : children) {
+      if (child instanceof ASTannotation_param) {
+        params.add((ASTannotation_param) child);
+      }
+    }
+    return params;
+  }
+
+  public String getAnnotation() {
+    return AstTreeUtils.tokensToString(this, false).replaceAll(" ", "");
+  }
+}

src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTannotation_param.java

@@ -0,0 +1,43 @@
+/*
+ * Copyright 2019 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.solutions.spannerddl.parser;
+
+import com.google.cloud.solutions.spannerddl.diff.AstTreeUtils;
+
+public class ASTannotation_param extends SimpleNode {
+  public ASTannotation_param(int id) {
+    super(id);
+  }
+
+  public ASTannotation_param(DdlParser p, int id) {
+    super(p, id);
+  }
+
+  public String getKey() {
+    ASTparam_key key = AstTreeUtils.getChildByType(this, ASTparam_key.class);
+    return AstTreeUtils.tokensToString(key);
+  }
+
+  public String getValue() {
+    ASTparam_val val = AstTreeUtils.getOptionalChildByType(this, ASTparam_val.class);
+    if (val != null) {
+      return AstTreeUtils.tokensToString(val);
+    } else {
+      return null;
+    }
+  }
+}

src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTcreate_table_statement.java

@@ -140,7 +140,8 @@ private void validateChildren() {
             ASTcheck_constraint.class,
             ASTprimary_key.class,
             ASTtable_interleave_clause.class,
-            ASTrow_deletion_policy_clause.class));
+            ASTrow_deletion_policy_clause.class,
+            ASTannotation.class));
   }
 
   @Override

src/main/jjtree-sources/ddl_annotation.jjt

@@ -0,0 +1,24 @@
+TOKEN:
+{
+  <ANNOTATION: "@ANNOTATION">
+}
+
+void column_annotation() #void: {}
+{
+  <ANNOTATION> annotation()
+}
+
+void annotation(): {}
+{
+  qualified_identifier() #name [ annotation_params() ]
+}
+
+void annotation_params() #void: {}
+{
+  "(" annotation_param() ( "," annotation_param() )* ")"
+}
+
+void annotation_param(): {}
+{
+  identifier() #param_key [ "=" identifier() #param_val ]
+}

src/main/jjtree-sources/ddl_parser.jjt

@@ -212,6 +212,7 @@ void table_element() #void :
     LOOKAHEAD(3) foreign_key()
   | LOOKAHEAD(3) check_constraint()
   | LOOKAHEAD(3) synonym_clause()
+  | column_annotation()
   | column_def()
 }
 

src/test/java/com/google/cloud/solutions/spannerddl/parser/DDLAnnotationTest.java

@@ -0,0 +1,59 @@
+package com.google.cloud.solutions.spannerddl.parser;
+
+import static com.google.common.truth.Truth.assertWithMessage;
+import static org.junit.Assert.fail;
+
+import com.google.cloud.solutions.spannerddl.testUtils.ReadTestDatafile;
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import org.junit.Test;
+
+public class DDLAnnotationTest {
+
+  @Test
+  public void validateAnnotations() throws IOException {
+    Map<String, String> tests = ReadTestDatafile.readDdlSegmentsFromFile("annotations.txt");
+    Map<String, String> expects =
+        ReadTestDatafile.readDdlSegmentsFromFile("expectedAnnotations.txt");
+
+    Iterator<Entry<String, String>> testIt = tests.entrySet().iterator();
+    Iterator<Entry<String, String>> expectedIt = expects.entrySet().iterator();
+
+    while (testIt.hasNext() && expectedIt.hasNext()) {
+      Entry<String, String> test = testIt.next();
+      String expected = expectedIt.next().getValue();
+      String segmentName = test.getKey();
+
+      try {
+        DdlParser parser = new DdlParser(new StringReader(test.getValue()));
+        parser.ddl_statement();
+        Node tableStatement = parser.jjtree.rootNode().jjtGetChild(0);
+
+        // get all annotations
+        List<String> annotations = new ArrayList<>();
+        for (int i = 0, count = tableStatement.jjtGetNumChildren(); i < count; i++) {
+          Node child = tableStatement.jjtGetChild(i);
+          if (child instanceof ASTannotation) {
+            annotations.add(((ASTannotation) child).getAnnotation());
+          }
+        }
+
+        List<String> expectedList =
+            expected != null ? Arrays.asList(expected.split("\n")) : Collections.emptyList();
+
+        assertWithMessage("Mismatch for section " + segmentName)
+            .that(annotations)
+            .isEqualTo(expectedList);
+      } catch (ParseException e) {
+        fail("Failed to parse section: '" + segmentName + "': " + e);
+      }
+    }
+  }
+}

src/test/java/com/google/cloud/solutions/spannerddl/testUtils/ReadTestDatafile.java

@@ -32,16 +32,26 @@ public abstract class ReadTestDatafile {
    * @return LinkedHashMap of segment name => contents
    */
   public static Map<String, String> readDdlSegmentsFromFile(String filename) throws IOException {
+    return readDdlSegmentsFromFile(filename, false);
+  }
+
+  /**
+   * Reads the test data file, parsing out the test titles and data from the file.
+   *
+   * @return LinkedHashMap of segment name => contents
+   */
+  public static Map<String, String> readDdlSegmentsFromFile(String filename, boolean preserveSpace)
+      throws IOException {
     File file = new File("src/test/resources/" + filename).getAbsoluteFile();
     LinkedHashMap<String, String> output = new LinkedHashMap<>();
 
     try (BufferedReader in = Files.newBufferedReader(file.toPath(), UTF_8)) {
 
       String sectionName = null;
       StringBuilder section = new StringBuilder();
-      String line;
-      while (null != (line = in.readLine())) {
-        line = line.replaceAll("#.*", "").trim();
+      String rawLine;
+      while (null != (rawLine = in.readLine())) {
+        String line = preserveSpace ? rawLine : rawLine.replaceAll("#.*", "").trim();
         if (line.isEmpty()) {
           continue;
         }

src/test/resources/annotations.txt

@@ -0,0 +1,12 @@
+== Test 1 all column annotations
+
+CREATE TABLE test1 (
+  @ANNOTATION DEPRECATED,
+  @ANNOTATION PII,
+  @ANNOTATION TAG.business(internal),
+  @ANNOTATION TAG.business(key1,key2),
+  @ANNOTATION TAG.business(key1=val1,key2=value),
+  id STRING(36),
+) PRIMARY KEY (id)
+
+==

src/test/resources/expectedAnnotations.txt

@@ -0,0 +1,9 @@
+== Test 1 all column annotations
+
+DEPRECATED
+PII
+TAG.business(internal)
+TAG.business(key1,key2)
+TAG.business(key1=val1,key2=value)
+
+==

src/test/resources/expectedDdlDiff.txt

@@ -325,6 +325,14 @@ ALTER SEARCH INDEX AlbumsIndex ADD STORED COLUMN scol1
 
 ALTER SEARCH INDEX AlbumsIndex DROP STORED COLUMN scol1
 
-==
+== TEST 61 Add annotations to columns should not generate a diff
+
+== TEST 62 Remove annotations from columns should not generate a diff
+
+== TEST 63 Recorder annotations should not generate a diff
 
+== TEST 64 Adding annotation as well as column should only generate the column diff
 
+ALTER TABLE AlbumsIndex ADD COLUMN new_col STRING(255)
+
+==

src/test/resources/newDdl.txt

@@ -511,4 +511,33 @@ STORING (scol1);
 CREATE SEARCH INDEX AlbumsIndex
 ON Albums (col1, col2)
 
+== TEST 61 Add annotations to columns should not generate a diff
+
+CREATE TABLE AlbumsIndex (
+  -- @ANNOTATION DEPRECATED,
+  id STRING(36),
+) PRIMARY KEY (id)
+
+== TEST 62 Remove annotations from columns should not generate a diff
+
+CREATE TABLE AlbumsIndex (
+  id STRING(36),
+) PRIMARY KEY (id)
+
+== TEST 63 Recorder annotations should not generate a diff
+
+CREATE TABLE AlbumsIndex (
+  -- @ANNOTATION PII,
+  -- @ANNOTATION DEPRECATED,
+  id STRING(36),
+) PRIMARY KEY (id)
+
+== TEST 64 Adding annotation as well as column should only generate the column diff
+
+CREATE TABLE AlbumsIndex (
+  -- @ANNOTATION PII,
+  id STRING(36),
+  new_col STRING(255),
+) PRIMARY KEY (id)
+
 ==

src/test/resources/originalDdl.txt

@@ -509,4 +509,31 @@ CREATE SEARCH INDEX AlbumsIndex
 ON Albums (col1, col2)
 STORING (scol1)
 
+== TEST 61 Add annotations to columns should not generate a diff
+
+CREATE TABLE AlbumsIndex (
+  id STRING(36),
+) PRIMARY KEY (id)
+
+== TEST 62 Remove annotations from columns should not generate a diff
+
+CREATE TABLE AlbumsIndex (
+  -- @ANNOTATION DEPRECATED,
+  id STRING(36),
+) PRIMARY KEY (id)
+
+== TEST 63 Recorder annotations should not generate a diff
+
+CREATE TABLE AlbumsIndex (
+  -- @ANNOTATION DEPRECATED,
+  -- @ANNOTATION PII,
+  id STRING(36),
+) PRIMARY KEY (id)
+
+== TEST 64 Adding annotation as well as column should only generate the column diff
+
+CREATE TABLE AlbumsIndex (
+  id STRING(36),
+) PRIMARY KEY (id)
+
 ==