Output bare label status in scan tests and example

README.md

@@ -111,7 +111,7 @@ int main() {
     scan_token = scan_result->tokens[j];
     token_kind = protobuf_c_enum_descriptor_get_value(&pg_query__token__descriptor, scan_token->token);
     keyword_kind = protobuf_c_enum_descriptor_get_value(&pg_query__keyword_kind__descriptor, scan_token->keyword_kind);
-    printf("  \"%.*s\" = [ %d, %d, %s, %s ]\n", scan_token->end - scan_token->start, &(input[scan_token->start]), scan_token->start, scan_token->end, token_kind->name, keyword_kind->name);
+    printf("  \"%.*s\" = [ %d, %d, %s, %s, %s ]\n", scan_token->end - scan_token->start, &(tests[i][scan_token->start]), scan_token->start, scan_token->end, token_kind->name, keyword_kind->name, scan_token->bare_label ? "BARE_LABEL" : "AS_LABEL");
   }
 
   pg_query__scan_result__free_unpacked(scan_result, NULL);
@@ -125,13 +125,13 @@ This will output the following:
 
 ```
   version: 130008, tokens: 7, size: 77
-  "SELECT" = [ 0, 6, SELECT, RESERVED_KEYWORD ]
-  "update" = [ 7, 13, UPDATE, UNRESERVED_KEYWORD ]
-  "AS" = [ 14, 16, AS, RESERVED_KEYWORD ]
-  "left" = [ 17, 21, LEFT, TYPE_FUNC_NAME_KEYWORD ]
-  "/* comment */" = [ 22, 35, C_COMMENT, NO_KEYWORD ]
-  "FROM" = [ 36, 40, FROM, RESERVED_KEYWORD ]
-  "between" = [ 41, 48, BETWEEN, COL_NAME_KEYWORD ]
+  "SELECT" = [ 0, 6, SELECT, RESERVED_KEYWORD, BARE_LABEL ]
+  "update" = [ 7, 13, UPDATE, UNRESERVED_KEYWORD, BARE_LABEL ]
+  "AS" = [ 14, 16, AS, RESERVED_KEYWORD, AS_LABEL ]
+  "left" = [ 17, 21, LEFT, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL ]
+  "/* comment */" = [ 22, 35, C_COMMENT, NO_KEYWORD, UNKNOWN_LABEL ]
+  "FROM" = [ 36, 40, FROM, RESERVED_KEYWORD, AS_LABEL ]
+  "between" = [ 41, 48, BETWEEN, COL_NAME_KEYWORD, BARE_LABEL ]
 ```
 
 Where the each element in the token list has the following fields:
@@ -145,6 +145,10 @@ Where the each element in the token list has the following fields:
   `COL_NAME_KEYWORD`: Unreserved keyword (can be unescaped column/table/etc names, cannot be unescaped function or type name)
   `TYPE_FUNC_NAME_KEYWORD`: Reserved keyword (can be unescaped function or type name, cannot be unescaped column/table/etc names)
   `RESERVED_KEYWORD`: Reserved keyword (cannot be unescaped column/table/variable/type/function names)
+5. Label type, if the keyword is not `NO_KEYWORD`, indicates whether it requires a preceding `AS` to parse, possible values:
+  `AS_LABEL`: Requires `AS`
+  `BARE_LABEL`: Does not require `AS`
+  `UNKNOWN_LABEL`: Token is not a keyword
 
 Note that whitespace does not show as tokens.
 

examples/scan.c

@@ -44,7 +44,16 @@ int main() {
         scan_token = scan_result->tokens[j];
         token_kind = protobuf_c_enum_descriptor_get_value(&pg_query__token__descriptor, scan_token->token);
         keyword_kind = protobuf_c_enum_descriptor_get_value(&pg_query__keyword_kind__descriptor, scan_token->keyword_kind);
-        printf("  \"%.*s\" = [ %d, %d, %s, %s ]\n", scan_token->end - scan_token->start, &(tests[i][scan_token->start]), scan_token->start, scan_token->end, token_kind->name, keyword_kind->name);
+        printf(
+          "  \"%.*s\" = [ %d, %d, %s, %s, %s ]\n",
+          scan_token->end - scan_token->start,
+          &(tests[i][scan_token->start]),
+          scan_token->start,
+          scan_token->end,
+          token_kind->name,
+          keyword_kind->name,
+          scan_token->bare_label ? "BARE_LABEL" : "AS_LABEL",
+        );
       }
 
       pg_query__scan_result__free_unpacked(scan_result, NULL);

protobuf/pg_query.pb-c.c

@@ -35341,22 +35341,22 @@ static const ProtobufCFieldDescriptor pg_query__scan_token__field_descriptors[5]
     0,NULL,NULL    /* reserved1,reserved2, etc */
   },
   {
-    "bare_label",
+    "label_kind",
     6,
     PROTOBUF_C_LABEL_NONE,
-    PROTOBUF_C_TYPE_BOOL,
+    PROTOBUF_C_TYPE_ENUM,
     0,   /* quantifier_offset */
-    offsetof(PgQuery__ScanToken, bare_label),
-    NULL,
+    offsetof(PgQuery__ScanToken, label_kind),
+    &pg_query__label_kind__descriptor,
     NULL,
     0,             /* flags */
     0,NULL,NULL    /* reserved1,reserved2, etc */
   },
 };
 static const unsigned pg_query__scan_token__field_indices_by_name[] = {
-  4,   /* field[4] = bare_label */
   1,   /* field[1] = end */
   3,   /* field[3] = keyword_kind */
+  4,   /* field[4] = label_kind */
   0,   /* field[0] = start */
   2,   /* field[2] = token */
 };
@@ -37621,6 +37621,36 @@ const ProtobufCEnumDescriptor pg_query__keyword_kind__descriptor =
   pg_query__keyword_kind__value_ranges,
   NULL,NULL,NULL,NULL   /* reserved[1234] */
 };
+static const ProtobufCEnumValue pg_query__label_kind__enum_values_by_number[3] =
+{
+  { "UNKNOWN_LABEL", "PG_QUERY__LABEL_KIND__UNKNOWN_LABEL", 0 },
+  { "BARE_LABEL", "PG_QUERY__LABEL_KIND__BARE_LABEL", 1 },
+  { "AS_LABEL", "PG_QUERY__LABEL_KIND__AS_LABEL", 2 },
+};
+static const ProtobufCIntRange pg_query__label_kind__value_ranges[] = {
+{0, 0},{0, 3}
+};
+static const ProtobufCEnumValueIndex pg_query__label_kind__enum_values_by_name[3] =
+{
+  { "AS_LABEL", 2 },
+  { "BARE_LABEL", 1 },
+  { "UNKNOWN_LABEL", 0 },
+};
+const ProtobufCEnumDescriptor pg_query__label_kind__descriptor =
+{
+  PROTOBUF_C__ENUM_DESCRIPTOR_MAGIC,
+  "pg_query.LabelKind",
+  "LabelKind",
+  "PgQuery__LabelKind",
+  "pg_query",
+  3,
+  pg_query__label_kind__enum_values_by_number,
+  3,
+  pg_query__label_kind__enum_values_by_name,
+  1,
+  pg_query__label_kind__value_ranges,
+  NULL,NULL,NULL,NULL   /* reserved[1234] */
+};
 static const ProtobufCEnumValue pg_query__token__enum_values_by_number[505] =
 {
   { "NUL", "PG_QUERY__TOKEN__NUL", 0 },

protobuf/pg_query.proto

@@ -3058,7 +3058,7 @@ message ScanToken {
   int32 end = 2;
   Token token = 4;
   KeywordKind keyword_kind = 5;
-  bool bare_label = 6;
+  LabelKind label_kind = 6;
 }
 
 enum KeywordKind {
@@ -3069,6 +3069,12 @@ enum KeywordKind {
   RESERVED_KEYWORD = 4;
 }
 
+enum LabelKind {
+  UNKNOWN_LABEL = 0;
+  BARE_LABEL = 1;
+  AS_LABEL = 2;
+}
+
 enum Token {
   NUL = 0;
   // Single-character tokens that are returned 1:1 (identical with "self" list in scan.l)

scripts/generate_protobuf_and_funcs.rb

@@ -374,7 +374,7 @@ def generate!
   int32 end = 2;
   Token token = 4;
   KeywordKind keyword_kind = 5;
-  bool bare_label = 6;
+  LabelKind label_kind = 6;
 }
 
 enum KeywordKind {
@@ -385,6 +385,12 @@ def generate!
   RESERVED_KEYWORD = 4;
 }
 
+enum LabelKind {
+  UNKNOWN_LABEL = 0;
+  BARE_LABEL = 1;
+  AS_LABEL = 2;
+}
+
 enum Token {
   NUL = 0;
   // Single-character tokens that are returned 1:1 (identical with \"self\" list in scan.l)

src/pg_query_scan.c

@@ -101,11 +101,17 @@ PgQueryScanResult pg_query_scan(const char* input)
       output_tokens[i]->token = tok;
 
       switch (tok) {
-      #define BARE_LABEL true
-      #define AS_LABEL false
-      #define PG_KEYWORD(a,b,c,d) case b: output_tokens[i]->keyword_kind = c + 1; output_tokens[i]->bare_label = d; break;
-      #include "parser/kwlist.h"
-      default: output_tokens[i]->keyword_kind = 0;
+        #define BARE_LABEL 1
+        #define AS_LABEL 2
+        #define PG_KEYWORD(a,b,c,d) case b: output_tokens[i]->keyword_kind = c + 1; output_tokens[i]->label_kind = d; break;
+        #include "parser/kwlist.h"
+        #undef PG_KEYWORD
+        #undef AS_LABEL
+        #undef BARE_LABEL
+        default: {
+          output_tokens[i]->keyword_kind = 0;
+          output_tokens[i]->label_kind = 0;
+        }
       }
     }
 

test/scan.c

@@ -17,6 +17,7 @@ int main() {
   PgQuery__ScanToken *scan_token;
   const ProtobufCEnumValue *token_kind;
   const ProtobufCEnumValue *keyword_kind;
+  const ProtobufCEnumValue *label_kind;
   PgQueryScanResult result;
 
   for (i = 0; i < testsCount * 2; i += 2) {
@@ -36,7 +37,8 @@ int main() {
         scan_token = scan_result->tokens[j];
         token_kind = protobuf_c_enum_descriptor_get_value(&pg_query__token__descriptor, scan_token->token);
         keyword_kind = protobuf_c_enum_descriptor_get_value(&pg_query__keyword_kind__descriptor, scan_token->keyword_kind);
-        sprintf(buffer2, "%.*s = %s, %s\n", scan_token->end - scan_token->start, &(tests[i][scan_token->start]), token_kind->name, keyword_kind->name);
+        label_kind = protobuf_c_enum_descriptor_get_value(&pg_query__label_kind__descriptor, scan_token->label_kind);
+        sprintf(buffer2, "%.*s = %s, %s, %s\n", scan_token->end - scan_token->start, &(tests[i][scan_token->start]), token_kind->name, keyword_kind->name, label_kind->name);
         strcat(buffer, buffer2);
       }