Project import generated by Copybara. (#122)

GitOrigin-RevId: db1aeceab3849acc274df148925bc6b33016c470

Co-authored-by: A. Googler <cloud-spanner-emulator-bot@google.com>

README.md

@@ -51,7 +51,7 @@ images are also tagged with version numbers, so you can run a specific version
 with:
 
 ```shell
-VERSION=1.5.2
+VERSION=1.5.6
 docker run -p 9010:9010 -p 9020:9020 gcr.io/cloud-spanner-emulator/emulator:$VERSION
 ```
 Works on x86 and arm64 architectures.
@@ -62,10 +62,12 @@ The emulator is also distributed as a standalone linux binary. Note that this
 binary is not fully static, but has been tested on Ubuntu 18.04+, CentOS
 7+, RHEL 8+ and Debian 10+.
 
+Set `ARCHITECTURE` to `arm64` in following command if you are working on arm machine.
 ```shell
-VERSION=1.5.2
-wget https://storage.googleapis.com/cloud-spanner-emulator/releases/${VERSION}/cloud-spanner-emulator_linux_amd64-${VERSION}.tar.gz
-tar zxvf cloud-spanner-emulator_linux_amd64-${VERSION}.tar.gz
+VERSION=1.5.6
+ARCHITECTURE=amd64
+wget https://storage.googleapis.com/cloud-spanner-emulator/releases/${VERSION}/cloud-spanner-emulator_linux_${ARCHITECTURE}-${VERSION}.tar.gz
+tar xvf cloud-spanner-emulator_linux_${ARCHITECTURE}-${VERSION}.tar.gz
 chmod u+x gateway_main emulator_main
 ```
 
@@ -166,10 +168,6 @@ Notable supported features:
 
 - Dataflow templates
 
-Features not currently available:
-
-- Views
-
 Notable limitations:
 
 - The gRPC and REST endpoints run on separate ports and serve unencrypted

WORKSPACE

@@ -78,13 +78,13 @@ http_archive(
 
 load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies")
 
-go_register_toolchains(version = "1.20.4")
+go_register_toolchains(version = "1.20.5")
 
-_bazel_gazelle_version = "0.30.0"
+_bazel_gazelle_version = "0.31.1"
 
 http_archive(
     name = "bazel_gazelle",
-    sha256 = "727f3e4edd96ea20c29e8c2ca9e8d2af724d8c7778e7923a854b2c80952bc405",
+    sha256 = "b8b6d75de6e4bf7c41b7737b183523085f56283f6db929b86c5e7e1f09cf59c9",
     urls = [
         "https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v{0}/bazel-gazelle-v{0}.tar.gz".format(_bazel_gazelle_version),
         "https://github.com/bazelbuild/bazel-gazelle/releases/download/v{0}/bazel-gazelle-v{0}.tar.gz".format(_bazel_gazelle_version),

backend/actions/column_value_test.cc

@@ -91,7 +91,6 @@ class ColumnValueTest : public test::ActionsTest {
  public:
   ColumnValueTest() {
     EmulatorFeatureFlags::Flags flags;
-    flags.enable_stored_generated_columns = false;
     emulator::test::ScopedEmulatorFeatureFlagsSetter setter(flags);
 
     schema_ = emulator::test::CreateSchemaFromDDL(

backend/query/OWNERS

@@ -0,0 +1,2 @@
+adirastogi # KIR
+arawind #BLR

backend/query/catalog.cc

@@ -178,7 +178,6 @@ zetasql::Catalog* Catalog::GetNetFunctionsCatalog() const {
   }
   return net_catalog_.get();
 }
-
 }  // namespace backend
 }  // namespace emulator
 }  // namespace spanner

backend/query/catalog.h

@@ -72,7 +72,6 @@ class Catalog : public zetasql::EnumerableCatalog {
   absl::Status GetFunction(const std::string& name,
                            const zetasql::Function** function,
                            const FindOptions& options) final;
-
   // Implementation of the zetasql::EnumerableCatalog interface.
   absl::Status GetCatalogs(
       absl::flat_hash_set<const zetasql::Catalog*>* output) const final;

backend/query/catalog_test.cc

@@ -83,7 +83,7 @@ TEST_F(AnalyzeStatementTest, SelectNonexistentColumnReturnsError) {
               StatusIs(absl::StatusCode::kInvalidArgument));
 }
 
-TEST_F(AnalyzeStatementTest, SelectNestedCatalogFunctions) {
+TEST_F(AnalyzeStatementTest, SelectNestedCatalogNetFunctions) {
   ZETASQL_EXPECT_OK(
       AnalyzeStatement("SELECT "
                        "NET.IPV4_TO_INT64(b\"\\x00\\x00\\x00\\x00\")"));

backend/query/query_engine.cc

@@ -36,6 +36,7 @@
 #include "zetasql/public/value.h"
 #include "zetasql/resolved_ast/resolved_ast.h"
 #include "zetasql/resolved_ast/resolved_node_kind.pb.h"
+#include "absl/container/flat_hash_map.h"
 #include "absl/container/flat_hash_set.h"
 #include "absl/memory/memory.h"
 #include "absl/status/status.h"

backend/query/query_engine.h

@@ -18,6 +18,7 @@
 #define THIRD_PARTY_CLOUD_SPANNER_EMULATOR_BACKEND_QUERY_QUERY_ENGINE_H_
 
 #include <memory>
+#include <optional>
 #include <string>
 
 #include "google/protobuf/struct.pb.h"

backend/query/queryable_column.h

@@ -58,6 +58,8 @@ class QueryableColumn : public zetasql::Column {
     return !wrapped_column_->is_generated();
   }
 
+  bool IsPseudoColumn() const override { return wrapped_column_->hidden(); }
+
   bool HasDefaultValue() const override {
     return wrapped_column_->has_default_value();
   }

backend/schema/OWNERS

@@ -0,0 +1,2 @@
+jinjj # CAM
+hengfeng # SYD

backend/schema/backfills/change_stream_backfill.cc

@@ -0,0 +1,149 @@
+//
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#include "backend/schema/backfills/change_stream_backfill.h"
+
+#include <cstdint>
+#include <cstdlib>
+#include <string>
+#include <vector>
+
+#include "zetasql/public/value.h"
+#include "absl/time/time.h"
+#include "backend/common/ids.h"
+#include "backend/common/rows.h"
+#include "backend/schema/catalog/column.h"
+#include "common/errors.h"
+#include "common/limits.h"
+#include "zetasql/base/status_macros.h"
+
+namespace google {
+namespace spanner {
+namespace emulator {
+namespace backend {
+
+absl::StatusOr<Key> ComputeChangeStreamPartitionTableKey(
+    std::string partition_token_str, const ChangeStream* change_stream) {
+  // Columns must be added to the key for each column in change stream partition
+  // table primary key.
+  Key key;
+  key.AddColumn(zetasql::Value::String(partition_token_str),
+                change_stream->change_stream_partition_table()
+                    ->FindKeyColumn("partition_token")
+                    ->is_descending());
+  const int64_t key_size = key.LogicalSizeInBytes();
+  if (key_size > limits::kMaxKeySizeBytes) {
+    return error::KeyTooLarge(
+        change_stream->change_stream_partition_table()->Name(), key_size,
+        limits::kMaxKeySizeBytes);
+  }
+  return key;
+}
+
+std::vector<zetasql::Value> CreateInitialBackfillPartitions(
+    std::vector<zetasql::Value> row_values, std::string partition_token_str,
+    absl::Time start_time, absl::Time end_time) {
+  // specify partition_token
+  row_values.push_back(zetasql::Value::String(partition_token_str));
+  // Specify start_time
+  row_values.push_back(zetasql::Value::Timestamp(start_time));
+  // Specify end_time
+  row_values.push_back(zetasql::Value::Timestamp(end_time));
+  // Specify parents
+  std::vector<zetasql::Value> parents_values;
+  parents_values.push_back(zetasql::Value::NullString());
+  row_values.push_back(zetasql::Value::Array(
+      zetasql::types::StringArrayType(), parents_values));
+  // Specify children
+  std::vector<zetasql::Value> children_values;
+  children_values.push_back(zetasql::Value::NullString());
+  row_values.push_back(zetasql::Value::Array(
+      zetasql::types::StringArrayType(), children_values));
+  return row_values;
+}
+
+std::string gen_random(const int len) {
+  static const char alphanum[] =
+      "0123456789"
+      "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+      "abcdefghijklmnopqrstuvwxyz";
+  std::string tmp_s, token_string;
+  tmp_s.reserve(len);
+
+  for (int i = 0; i < len; ++i) {
+    tmp_s += alphanum[rand() % (sizeof(alphanum) - 1)];
+  }
+  absl::WebSafeBase64Escape(tmp_s, &token_string);
+  return token_string;
+}
+
+std::string CreatePartitionTokenString() {
+  const int64_t min_partition_token_len = 130;
+  const int64_t max_partition_token_len = 170;
+  const int64_t partition_token_len =
+      min_partition_token_len +
+      rand() % (max_partition_token_len - min_partition_token_len) + 1;
+  return gen_random(partition_token_len);
+}
+
+absl::Status BackfillChangeStreamPartition(
+    const SchemaValidationContext* context, const ChangeStream* change_stream,
+    absl::Span<const Column* const> change_stream_partition_table_columns,
+    std::vector<ColumnID> change_stream_partition_table_column_ids) {
+  std::string partition_token_str = CreatePartitionTokenString();
+  // Populate values for the row representing the first partition
+  std::vector<zetasql::Value> initial_row_values;
+  initial_row_values.reserve(change_stream_partition_table_columns.size());
+  initial_row_values = CreateInitialBackfillPartitions(
+      initial_row_values, partition_token_str, absl::UnixEpoch(),
+      absl::FromUnixMicros(zetasql::types::kTimestampMax));
+  // Create Key for the change stream partition table
+  ZETASQL_ASSIGN_OR_RETURN(
+      Key change_stream_partition_table_key,
+      ComputeChangeStreamPartitionTableKey(partition_token_str, change_stream),
+      _.SetErrorCode(absl::StatusCode::kFailedPrecondition));
+  // Insert the second new row in the change_stream_partition_table.
+  ZETASQL_RETURN_IF_ERROR(context->storage()->Write(
+      context->pending_commit_timestamp(),
+      change_stream->change_stream_partition_table()->id(),
+      change_stream_partition_table_key,
+      change_stream_partition_table_column_ids, initial_row_values));
+  return absl::OkStatus();
+}
+
+absl::Status BackfillChangeStream(const ChangeStream* change_stream,
+                                  const SchemaValidationContext* context) {
+  const Table* change_stream_partition_table =
+      change_stream->change_stream_partition_table();
+  absl::Span<const Column* const> change_stream_partition_table_columns =
+      change_stream_partition_table->columns();
+  std::vector<ColumnID> change_stream_partition_table_column_ids =
+      GetColumnIDs(change_stream_partition_table_columns);
+  // Backfill the first partition
+  // Generate partition token
+  const int64_t num_initial_partitions = 2;
+  for (int i = 0; i < num_initial_partitions; ++i) {
+    ZETASQL_RETURN_IF_ERROR(BackfillChangeStreamPartition(
+        context, change_stream, change_stream_partition_table_columns,
+        change_stream_partition_table_column_ids));
+  }
+  return absl::OkStatus();
+}
+
+}  // namespace backend
+}  // namespace emulator
+}  // namespace spanner
+}  // namespace google

backend/schema/backfills/change_stream_backfill.h

@@ -0,0 +1,38 @@
+//
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#ifndef THIRD_PARTY_CLOUD_SPANNER_EMULATOR_BACKEND_SCHEMA_BACKFILLS_CHANGE_STREAM_BACKFILL_H_
+#define THIRD_PARTY_CLOUD_SPANNER_EMULATOR_BACKEND_SCHEMA_BACKFILLS_CHANGE_STREAM_BACKFILL_H_
+
+#include "backend/schema/catalog/change_stream.h"
+#include "backend/schema/updater/schema_validation_context.h"
+
+namespace google {
+namespace spanner {
+namespace emulator {
+namespace backend {
+
+// Handles backfilling of a newly created change stream partition table and
+// change stream data table.
+absl::Status BackfillChangeStream(const ChangeStream* change_stream,
+                                  const SchemaValidationContext* context);
+
+}  // namespace backend
+}  // namespace emulator
+}  // namespace spanner
+}  // namespace google
+
+#endif  // THIRD_PARTY_CLOUD_SPANNER_EMULATOR_BACKEND_SCHEMA_BACKFILLS_CHANGE_STREAM_BACKFILL_H_