Merge remote-tracking branch 'dev/penberg/keyspace-merging/v5' from s…

…eastar-dev.git

From Pekka:

"This patch series converts LegacySchemaTables keyspace merging code to
C++. After this series, keyspaces are actually created as demonstrated
by the newly added test in cql_query_test.cc."

bytes_ostream.hh

@@ -211,6 +211,10 @@ public:
         return _size;
     }
 
+    bool empty() const {
+        return _size == 0;
+    }
+
     void reserve(size_t size) {
         // FIXME: implement
     }

config/ks_meta_data.hh

@@ -45,35 +45,41 @@ public:
 
     const ::shared_ptr<ut_meta_data> user_types;
 
-#if 0
-    public KSMetaData(String name,
-                      Class<? extends AbstractReplicationStrategy> strategyClass,
-                      Map<String, String> strategyOptions,
-                      boolean durableWrites)
-    {
-        this(name, strategyClass, strategyOptions, durableWrites, Collections.<CFMetaData>emptyList(), new UTMetaData());
-    }
+    ks_meta_data(sstring name_,
+                 sstring strategy_name_,
+                 std::unordered_map<sstring, sstring> strategy_options_,
+                 bool durable_writes_)
+        : ks_meta_data{std::move(name_),
+                       std::move(strategy_name_),
+                       std::move(strategy_options_),
+                       durable_writes_,
+                       {}, ::make_shared<ut_meta_data>()}
+    { }
+
+    ks_meta_data(sstring name_,
+                 sstring strategy_name_,
+                 std::unordered_map<sstring, sstring> strategy_options_,
+                 bool durable_writes_,
+                 std::vector<schema_ptr> cf_defs)
+            : ks_meta_data{std::move(name_),
+                           std::move(strategy_name_),
+                           std::move(strategy_options_),
+                           durable_writes_,
+                           std::move(cf_defs),
+                           ::make_shared<ut_meta_data>()}
+    { }
 
-    public KSMetaData(String name,
-                      Class<? extends AbstractReplicationStrategy> strategyClass,
-                      Map<String, String> strategyOptions,
-                      boolean durableWrites,
-                      Iterable<CFMetaData> cfDefs)
-    {
-        this(name, strategyClass, strategyOptions, durableWrites, cfDefs, new UTMetaData());
-    }
-#endif
     ks_meta_data(sstring name_,
                        sstring strategy_name_,
                        std::unordered_map<sstring, sstring> strategy_options_,
                        bool durable_writes_,
                        std::vector<schema_ptr> cf_defs,
                        shared_ptr<ut_meta_data> user_types_)
-        : name{name_}
+        : name{std::move(name_)}
         , strategy_name{strategy_name_.empty() ? "NetworkTopologyStrategy" : strategy_name_}
         , strategy_options{std::move(strategy_options_)}
         , durable_writes{durable_writes_}
-        , user_types{user_types_}
+        , user_types{std::move(user_types_)}
     {
         for (auto&& s : cf_defs) {
             _cf_meta_data.emplace(s->cf_name(), s);

configure.py

@@ -408,6 +408,7 @@ def have_xen():
                  'dht/murmur3_partitioner.cc',
                  'unimplemented.cc',
                  'query.cc',
+                 'query-result-set.cc',
                  'locator/abstract_replication_strategy.cc',
                  'locator/simple_strategy.cc',
                  'locator/token_metadata.cc',

database.cc

@@ -303,6 +303,14 @@ keyspace& database::add_keyspace(sstring name, keyspace k) {
     return _keyspaces.emplace(std::move(name), std::move(k)).first->second;
 }
 
+void database::update_keyspace(const sstring& name) {
+    throw std::runtime_error("not implemented");
+}
+
+void database::drop_keyspace(const sstring& name) {
+    throw std::runtime_error("not implemented");
+}
+
 void database::add_column_family(const utils::UUID& uuid, column_family&& cf) {
     if (_keyspaces.count(cf._schema->ks_name()) == 0) {
         throw std::invalid_argument("Keyspace " + cf._schema->ks_name() + " not defined");

database.hh

@@ -138,6 +138,8 @@ public:
     keyspace& find_keyspace(const sstring& name) throw (no_such_keyspace);
     const keyspace& find_keyspace(const sstring& name) const throw (no_such_keyspace);
     bool has_keyspace(const sstring& name) const;
+    void update_keyspace(const sstring& name);
+    void drop_keyspace(const sstring& name);
     column_family& find_column_family(const sstring& ks, const sstring& name) throw (no_such_column_family);
     const column_family& find_column_family(const sstring& ks, const sstring& name) const throw (no_such_column_family);
     column_family& find_column_family(const utils::UUID&) throw (no_such_column_family);

db/legacy_schema_tables.hh

@@ -29,11 +29,18 @@
 #include "schema.hh"
 
 #include <vector>
+#include <map>
+
+namespace query {
+class result_set;
+}
 
 /** system.schema_* tables used to store keyspace/table/type attributes prior to C* 3.0 */
 namespace db {
 namespace legacy_schema_tables {
 
+using schema_result = std::map<dht::decorated_key, foreign_ptr<lw_shared_ptr<query::result_set>>>;
+
 static constexpr auto KEYSPACES = "schema_keyspaces";
 static constexpr auto COLUMNFAMILIES = "schema_columnfamilies";
 static constexpr auto COLUMNS = "schema_columns";
@@ -46,13 +53,23 @@ extern std::vector<const char*> ALL;
 
 std::vector<schema_ptr> all_tables();
 
+future<std::pair<dht::decorated_key, foreign_ptr<lw_shared_ptr<query::result_set>>>>
+read_schema_partition_for_keyspace(service::storage_proxy& proxy, const sstring& schema_table_name, const sstring& keyspace_name);
+
+future<std::pair<dht::decorated_key, foreign_ptr<lw_shared_ptr<query::result_set>>>>
+read_schema_partition_for_keyspace(service::storage_proxy& proxy, const sstring& schema_table_name, const dht::decorated_key& keyspace_key);
+
 future<> merge_schema(service::storage_proxy& proxy, std::vector<mutation> mutations);
 
 future<> merge_schema(service::storage_proxy& proxy, std::vector<mutation> mutations, bool do_flush);
 
-mutation make_create_keyspace_mutation(lw_shared_ptr<config::ks_meta_data> keyspace, api::timestamp_type timestamp, bool with_tables_and_types_and_functions = true);
+future<std::set<sstring>> merge_keyspaces(service::storage_proxy& proxy, schema_result&& before, schema_result&& after);
+
+std::vector<mutation> make_create_keyspace_mutations(lw_shared_ptr<config::ks_meta_data> keyspace, api::timestamp_type timestamp, bool with_tables_and_types_and_functions = true);
+
+lw_shared_ptr<config::ks_meta_data> create_keyspace_from_schema_partition(const std::pair<dht::decorated_key, foreign_ptr<lw_shared_ptr<query::result_set>>>& partition);
 
-void add_table_to_schema_mutation(schema_ptr table, api::timestamp_type timestamp, bool with_columns_and_triggers, mutation& m);
+void add_table_to_schema_mutation(schema_ptr table, api::timestamp_type timestamp, bool with_columns_and_triggers, const partition_key& pkey, std::vector<mutation>& mutations);
 
 } // namespace legacy_schema_tables
 } // namespace db

keys.hh

@@ -274,6 +274,12 @@ public:
 public:
     using compound = lw_shared_ptr<compound_type<allow_prefixes::no>>;
 
+    bytes_view get_component(const schema& s, size_t idx) const {
+      auto it = begin(s);
+      std::advance(it, idx);
+      return *it;
+    }
+
     static partition_key from_bytes(bytes b) {
         return partition_key(std::move(b));
     }

mutation.cc

@@ -23,6 +23,15 @@ void mutation::set_clustered_cell(const exploded_clustering_prefix& prefix, cons
     update_column(row, def, std::move(value));
 }
 
+void mutation::set_clustered_cell(const clustering_key& key, const bytes& name, const boost::any& value,
+        api::timestamp_type timestamp, ttl_opt ttl) {
+    auto column_def = _schema->get_column_definition(name);
+    if (!column_def) {
+        throw std::runtime_error(sprint("no column definition found for '%s'", name));
+    }
+    return set_clustered_cell(key, *column_def, atomic_cell::make_live(timestamp, ttl, column_def->type->decompose(value)));
+}
+
 void mutation::set_clustered_cell(const clustering_key& key, const column_definition& def, atomic_cell_or_collection value) {
     auto& row = _p.clustered_row(key).cells;
     update_column(row, def, std::move(value));

mutation.hh

@@ -23,6 +23,7 @@ public:
     mutation(const mutation&) = default;
     void set_static_cell(const column_definition& def, atomic_cell_or_collection value);
     void set_clustered_cell(const exploded_clustering_prefix& prefix, const column_definition& def, atomic_cell_or_collection value);
+    void set_clustered_cell(const clustering_key& key, const bytes& name, const boost::any& value, api::timestamp_type timestamp, ttl_opt ttl = {});
     void set_clustered_cell(const clustering_key& key, const column_definition& def, atomic_cell_or_collection value);
     void set_cell(const exploded_clustering_prefix& prefix, const bytes& name, const boost::any& value, api::timestamp_type timestamp, ttl_opt ttl = {});
     void set_cell(const exploded_clustering_prefix& prefix, const column_definition& def, atomic_cell_or_collection value);

query-result-set.cc

@@ -0,0 +1,106 @@
+/*
+ * Copyright 2015 Cloudius Systems
+ */
+
+#include "query-result-set.hh"
+
+namespace query {
+
+result_set_builder::result_set_builder(schema_ptr schema)
+    : _schema{schema}
+{ }
+
+lw_shared_ptr<result_set> result_set_builder::build() const {
+    return make_lw_shared<result_set>(_rows);
+}
+
+void result_set_builder::accept_new_partition(const partition_key& key, uint32_t row_count)
+{
+    _pkey_cells = deserialize(key);
+}
+
+void result_set_builder::accept_new_partition(uint32_t row_count)
+{
+}
+
+void result_set_builder::accept_new_row(const clustering_key& key, const result_row_view& static_row, const result_row_view& row)
+{
+    auto ckey_cells = deserialize(key);
+    auto static_cells = deserialize(static_row, true);
+    auto regular_cells = deserialize(row, false);
+
+    std::unordered_map<sstring, boost::any> cells;
+    cells.insert(_pkey_cells.begin(), _pkey_cells.end());
+    cells.insert(ckey_cells.begin(), ckey_cells.end());
+    cells.insert(static_cells.begin(), static_cells.end());
+    cells.insert(regular_cells.begin(), regular_cells.end());
+    _rows.emplace_back(_schema, std::move(cells));
+}
+
+void result_set_builder::accept_new_row(const query::result_row_view &static_row, const query::result_row_view &row)
+{
+    auto static_cells = deserialize(static_row, true);
+    auto regular_cells = deserialize(row, false);
+
+    std::unordered_map<sstring, boost::any> cells;
+    cells.insert(_pkey_cells.begin(), _pkey_cells.end());
+    cells.insert(static_cells.begin(), static_cells.end());
+    cells.insert(regular_cells.begin(), regular_cells.end());
+    _rows.emplace_back(_schema, std::move(cells));
+}
+
+void result_set_builder::accept_partition_end(const result_row_view& static_row)
+{
+    _pkey_cells.clear();
+}
+
+std::unordered_map<sstring, boost::any>
+result_set_builder::deserialize(const partition_key& key)
+{
+    std::unordered_map<sstring, boost::any> cells;
+    auto i = key.begin(*_schema);
+    for (auto&& col : _schema->partition_key_columns()) {
+        cells.emplace(col.name_as_text(), col.type->deserialize(*i));
+        ++i;
+    }
+    return cells;
+}
+
+std::unordered_map<sstring, boost::any>
+result_set_builder::deserialize(const clustering_key& key)
+{
+    std::unordered_map<sstring, boost::any> cells;
+    auto i = key.begin(*_schema);
+    for (auto&& col : _schema->clustering_key_columns()) {
+        cells.emplace(col.name_as_text(), col.type->deserialize(*i));
+        ++i;
+    }
+    return cells;
+}
+
+std::unordered_map<sstring, boost::any>
+result_set_builder::deserialize(const result_row_view& row, bool is_static)
+{
+    std::unordered_map<sstring, boost::any> cells;
+    auto i = row.iterator();
+    auto columns = is_static ? _schema->static_columns() : _schema->regular_columns();
+    for (auto &&col : columns) {
+        if (col.is_atomic()) {
+            auto cell = i.next_atomic_cell();
+            if (cell) {
+                auto view = cell.value();
+                cells.emplace(col.name_as_text(), col.type->deserialize(view.value()));
+            }
+        } else {
+            auto cell = i.next_collection_cell();
+            if (cell) {
+                auto ctype = static_pointer_cast<collection_type_impl>(col.type);
+                auto view = cell.value();
+                cells.emplace(col.name_as_text(), ctype->deserialize(view.data, serialization_format::internal()));
+            }
+        }
+    }
+    return cells;
+}
+
+}

query-result-set.hh

@@ -0,0 +1,102 @@
+/*
+ * Copyright 2015 Cloudius Systems
+ */
+
+#pragma once
+
+#include "query-result-reader.hh"
+
+#include "core/shared_ptr.hh"
+
+#include <experimental/optional>
+#include <stdexcept>
+
+#include <boost/any.hpp>
+
+namespace query {
+
+class no_such_column : public std::runtime_error {
+public:
+    using runtime_error::runtime_error;
+};
+
+class null_column_value : public std::runtime_error {
+public:
+    using runtime_error::runtime_error;
+};
+
+// Result set row is a set of cells that are associated with a row
+// including regular column cells, partition keys, as well as static values.
+class result_set_row {
+    schema_ptr _schema;
+    std::unordered_map<sstring, boost::any> _cells;
+public:
+    result_set_row(schema_ptr schema, std::unordered_map<sstring, boost::any>&& cells)
+        : _schema{schema}
+        , _cells{std::move(cells)}
+    { }
+    // Look up a deserialized row cell value by column name.
+    template<typename T>
+    std::experimental::optional<T>
+    get(const sstring& column_name) const throw (no_such_column) {
+        auto it = _cells.find(column_name);
+        if (it == _cells.end()) {
+            throw no_such_column(column_name);
+        }
+        if (it->second.empty()) {
+            return std::experimental::nullopt;
+        }
+        return std::experimental::optional<T>{boost::any_cast<T>(it->second)};
+    }
+    template<typename T>
+    T get_nonnull(const sstring& column_name) const throw (no_such_column, null_column_value) {
+        auto v = get<T>(column_name);
+        if (v) {
+            return *v;
+        }
+        throw null_column_value(column_name);
+    }
+};
+
+// Result set is an in-memory representation of query results in
+// deserialized format. To obtain a result set, use the result_set_builder
+// class as a visitor to query_result::consume() function.
+class result_set {
+    std::vector<result_set_row> _rows;
+public:
+    result_set(const std::vector<result_set_row>& rows)
+        : _rows{std::move(rows)}
+    { }
+    bool empty() const {
+        return _rows.empty();
+    }
+    const result_set_row& row(size_t idx) const throw (std::out_of_range) {
+        if (idx >= _rows.size()) {
+            throw std::out_of_range("no such row in result set: " + std::to_string(idx));
+        }
+        return _rows[idx];
+    }
+};
+
+// Result set builder is passed as a visitor to query_result::consume()
+// function. You can call the build() method to obtain a result set that
+// contains cells from the visited results.
+class result_set_builder {
+    schema_ptr _schema;
+    std::vector<result_set_row> _rows;
+    std::unordered_map<sstring, boost::any> _pkey_cells;
+public:
+    result_set_builder(schema_ptr schema);
+    lw_shared_ptr<result_set> build() const;
+    void accept_new_partition(const partition_key& key, uint32_t row_count);
+    void accept_new_partition(uint32_t row_count);
+    void accept_new_row(const clustering_key& key, const result_row_view& static_row, const result_row_view& row);
+    void accept_new_row(const result_row_view &static_row, const result_row_view &row);
+    void accept_partition_end(const result_row_view& static_row);
+private:
+    std::unordered_map<sstring, boost::any> deserialize(const partition_key& key);
+    std::unordered_map<sstring, boost::any> deserialize(const clustering_key& key);
+    std::unordered_map<sstring, boost::any> deserialize(const result_row_view& row, bool is_static);
+};
+
+}