SPARQL Update Part 2: DeltaTriples

src/index/CMakeLists.txt

@@ -5,5 +5,6 @@ add_library(index
         LocatedTriples.cpp Permutation.cpp TextMetaData.cpp
         DocsDB.cpp FTSAlgorithms.cpp
         PrefixHeuristic.cpp CompressedRelation.cpp
-        PatternCreator.cpp)
+        PatternCreator.cpp
+        DeltaTriples.cpp)
 qlever_target_link_libraries(index util parser vocabulary compilationInfo ${STXXL_LIBRARIES})

src/index/DeltaTriples.cpp

@@ -0,0 +1,189 @@
+// Copyright 2023 - 2024, University of Freiburg
+//  Chair of Algorithms and Data Structures.
+//  Authors:
+//    2023 Hannah Bast <bast@cs.uni-freiburg.de>
+//    2024 Julian Mundhahs <mundhahj@tf.uni-freiburg.de>
+
+#include "index/DeltaTriples.h"
+
+#include "absl/strings/str_cat.h"
+#include "index/Index.h"
+#include "index/IndexImpl.h"
+#include "index/LocatedTriples.h"
+#include "parser/TurtleParser.h"
+
+// ____________________________________________________________________________
+LocatedTriples::iterator& DeltaTriples::LocatedTripleHandles::forPermutation(
+    Permutation::Enum permutation) {
+  switch (permutation) {
+    case Permutation::PSO:
+      return forPSO_;
+    case Permutation::POS:
+      return forPOS_;
+    case Permutation::SPO:
+      return forSPO_;
+    case Permutation::SOP:
+      return forSOP_;
+    case Permutation::OSP:
+      return forOSP_;
+    case Permutation::OPS:
+      return forOPS_;
+    default:
+      AD_FAIL();
+  }
+}
+
+// ____________________________________________________________________________
+void DeltaTriples::clear() {
+  triplesInserted_.clear();
+  triplesDeleted_.clear();
+  locatedTriplesPerBlockInPSO_.clear();
+  locatedTriplesPerBlockInPOS_.clear();
+  locatedTriplesPerBlockInSPO_.clear();
+  locatedTriplesPerBlockInSOP_.clear();
+  locatedTriplesPerBlockInOSP_.clear();
+  locatedTriplesPerBlockInOPS_.clear();
+}
+
+// ____________________________________________________________________________
+std::vector<DeltaTriples::LocatedTripleHandles>
+DeltaTriples::locateAndAddTriples(
+    ad_utility::SharedCancellationHandle cancellationHandle,
+    std::span<const IdTriple<0>> idTriples, bool shouldExist) {
+  ad_utility::HashMap<Permutation::Enum, std::vector<LocatedTriples::iterator>>
+      intermediateHandles;
+  for (auto permutation : Permutation::ALL) {
+    auto& perm = index_.getImpl().getPermutation(permutation);
+    auto locatedTriples = LocatedTriple::locateTriplesInPermutation(
+        // TODO<qup42>: replace with the method for update block metadata once
+        // integration is done
+        idTriples, perm.metaData().blockData(), perm.keyOrder(), shouldExist,
+        cancellationHandle);
+    cancellationHandle->throwIfCancelled();
+    intermediateHandles[permutation] =
+        getLocatedTriplesPerBlock(permutation).add(locatedTriples);
+    cancellationHandle->throwIfCancelled();
+  }
+  std::vector<DeltaTriples::LocatedTripleHandles> handles{idTriples.size()};
+  for (auto permutation : Permutation::ALL) {
+    for (size_t i = 0; i < idTriples.size(); i++) {
+      handles[i].forPermutation(permutation) =
+          intermediateHandles[permutation][i];
+    }
+  }
+  return handles;
+}
+
+// ____________________________________________________________________________
+void DeltaTriples::eraseTripleInAllPermutations(
+    DeltaTriples::LocatedTripleHandles& handles) {
+  // Erase for all permutations.
+  for (auto permutation : Permutation::ALL) {
+    auto ltIter = handles.forPermutation(permutation);
+    getLocatedTriplesPerBlock(permutation).erase(ltIter->blockIndex_, ltIter);
+  }
+}
+
+// ____________________________________________________________________________
+void DeltaTriples::insertTriples(
+    ad_utility::SharedCancellationHandle cancellationHandle,
+    std::vector<IdTriple<0>> triples) {
+  LOG(DEBUG) << "Inserting " << triples.size()
+             << " triples (including idempotent triples)." << std::endl;
+  std::ranges::sort(triples);
+  // Unique moves all duplicate items to the end and returns iterators for that
+  // subrange.
+  auto [first, last] = std::ranges::unique(triples);
+  triples.erase(first, last);
+  std::erase_if(triples, [this](const IdTriple<0>& triple) {
+    return triplesInserted_.contains(triple);
+  });
+  std::ranges::for_each(triples, [this](const IdTriple<0>& triple) {
+    auto handle = triplesDeleted_.find(triple);
+    if (handle != triplesDeleted_.end()) {
+      eraseTripleInAllPermutations(handle->second);
+      triplesDeleted_.erase(triple);
+    }
+  });
+
+  std::vector<LocatedTripleHandles> handles =
+      locateAndAddTriples(std::move(cancellationHandle), triples, true);
+
+  AD_CORRECTNESS_CHECK(triples.size() == handles.size());
+  // TODO<qup42>: replace with std::views::zip in C++23
+  for (size_t i = 0; i < triples.size(); i++) {
+    triplesInserted_.insert({triples[i], handles[i]});
+  }
+}
+
+// ____________________________________________________________________________
+void DeltaTriples::deleteTriples(
+    ad_utility::SharedCancellationHandle cancellationHandle,
+    std::vector<IdTriple<0>> triples) {
+  LOG(DEBUG) << "Deleting " << triples.size()
+             << " triples (including idempotent triples)." << std::endl;
+  std::ranges::sort(triples);
+  auto [first, last] = std::ranges::unique(triples);
+  triples.erase(first, last);
+  std::erase_if(triples, [this](const IdTriple<0>& triple) {
+    return triplesDeleted_.contains(triple);
+  });
+  std::ranges::for_each(triples, [this](const IdTriple<0>& triple) {
+    auto handle = triplesInserted_.find(triple);
+    if (handle != triplesInserted_.end()) {
+      eraseTripleInAllPermutations(handle->second);
+      triplesInserted_.erase(triple);
+    }
+  });
+
+  std::vector<LocatedTripleHandles> handles =
+      locateAndAddTriples(std::move(cancellationHandle), triples, false);
+
+  AD_CORRECTNESS_CHECK(triples.size() == handles.size());
+  // TODO<qup42>: replace with std::views::zip in C++23
+  for (size_t i = 0; i < triples.size(); i++) {
+    triplesDeleted_.insert({triples[i], handles[i]});
+  }
+}
+
+// ____________________________________________________________________________
+const LocatedTriplesPerBlock& DeltaTriples::getLocatedTriplesPerBlock(
+    Permutation::Enum permutation) const {
+  switch (permutation) {
+    case Permutation::PSO:
+      return locatedTriplesPerBlockInPSO_;
+    case Permutation::POS:
+      return locatedTriplesPerBlockInPOS_;
+    case Permutation::SPO:
+      return locatedTriplesPerBlockInSPO_;
+    case Permutation::SOP:
+      return locatedTriplesPerBlockInSOP_;
+    case Permutation::OSP:
+      return locatedTriplesPerBlockInOSP_;
+    case Permutation::OPS:
+      return locatedTriplesPerBlockInOPS_;
+    default:
+      AD_FAIL();
+  }
+}
+
+// ____________________________________________________________________________
+LocatedTriplesPerBlock& DeltaTriples::getLocatedTriplesPerBlock(
+    Permutation::Enum permutation) {
+  switch (permutation) {
+    case Permutation::PSO:
+      return locatedTriplesPerBlockInPSO_;
+    case Permutation::POS:
+      return locatedTriplesPerBlockInPOS_;
+    case Permutation::SPO:
+      return locatedTriplesPerBlockInSPO_;
+    case Permutation::SOP:
+      return locatedTriplesPerBlockInSOP_;
+    case Permutation::OSP:
+      return locatedTriplesPerBlockInOSP_;
+    case Permutation::OPS:
+      return locatedTriplesPerBlockInOPS_;
+    default:
+      AD_FAIL();
+  }
+}

src/index/DeltaTriples.h

@@ -0,0 +1,134 @@
+// Copyright 2023 - 2024, University of Freiburg
+//  Chair of Algorithms and Data Structures.
+//  Authors:
+//    2023 Hannah Bast <bast@cs.uni-freiburg.de>
+//    2024 Julian Mundhahs <mundhahj@tf.uni-freiburg.de>
+
+#pragma once
+
+#include "engine/LocalVocab.h"
+#include "global/IdTriple.h"
+#include "index/Index.h"
+#include "index/IndexBuilderTypes.h"
+#include "index/LocatedTriples.h"
+#include "parser/TurtleParser.h"
+#include "util/HashSet.h"
+
+// A class for maintaining triples that are inserted or deleted after index
+// building, we call these delta triples. How it works in principle:
+//
+// 1. For each delta triple, find the block index in each permutation (see
+// `LocatedTriple` in `index/LocatedTriples.h`).
+//
+// 2. For each permutation and each block, store a sorted list of the positions
+// of the delta triples within that block (see `LocatedTriplesPerBlock` in
+// `index/LocatedTriples.h`).
+//
+// 3. In the call of `PermutationImpl::scan`, use the respective lists to merge
+// the relevant delta triples into the index scan result.
+//
+// NOTE: The delta triples currently do not go well together with CACHING. See
+// the discussion at the end of this file.
+class DeltaTriples {
+ private:
+  // The index to which these triples are added.
+  const Index& index_;
+
+  // The local vocabulary of the delta triples (they may have components,
+  // which are not contained in the vocabulary of the original index).
+  LocalVocab localVocab_;
+
+  // The positions of the delta triples in each of the six permutations.
+  LocatedTriplesPerBlock locatedTriplesPerBlockInPSO_;
+  LocatedTriplesPerBlock locatedTriplesPerBlockInPOS_;
+  LocatedTriplesPerBlock locatedTriplesPerBlockInSPO_;
+  LocatedTriplesPerBlock locatedTriplesPerBlockInSOP_;
+  LocatedTriplesPerBlock locatedTriplesPerBlockInOSP_;
+  LocatedTriplesPerBlock locatedTriplesPerBlockInOPS_;
+
+  FRIEND_TEST(DeltaTriplesTest, insertTriplesAndDeleteTriples);
+
+  // Each delta triple needs to know where it is stored in each of the six
+  // `LocatedTriplesPerBlock` above.
+  struct LocatedTripleHandles {
+    LocatedTriples::iterator forPSO_;
+    LocatedTriples::iterator forPOS_;
+    LocatedTriples::iterator forSPO_;
+    LocatedTriples::iterator forSOP_;
+    LocatedTriples::iterator forOPS_;
+    LocatedTriples::iterator forOSP_;
+
+    LocatedTriples::iterator& forPermutation(Permutation::Enum permutation);
+  };
+
+  // The sets of triples added to and subtracted from the original index. In
+  // particular, no triple can be in both of these sets.
+  ad_utility::HashMap<IdTriple<0>, LocatedTripleHandles> triplesInserted_;
+  ad_utility::HashMap<IdTriple<0>, LocatedTripleHandles> triplesDeleted_;
+
+ public:
+  // Construct for given index.
+  explicit DeltaTriples(const Index& index) : index_(index) {}
+
+  // Get the common `LocalVocab` of the delta triples.
+  LocalVocab& localVocab() { return localVocab_; }
+  const LocalVocab& localVocab() const { return localVocab_; }
+
+  // Clear `_triplesAdded` and `_triplesSubtracted` and all associated data
+  // structures.
+  void clear();
+
+  // The number of delta triples added and subtracted.
+  size_t numInserted() const { return triplesInserted_.size(); }
+  size_t numDeleted() const { return triplesDeleted_.size(); }
+
+  // Insert triples.
+  void insertTriples(ad_utility::SharedCancellationHandle cancellationHandle,
+                     std::vector<IdTriple<0>> triples);
+
+  // Delete triples.
+  void deleteTriples(ad_utility::SharedCancellationHandle cancellationHandle,
+                     std::vector<IdTriple<0>> triples);
+
+  // Get `TripleWithPosition` objects for given permutation.
+  LocatedTriplesPerBlock& getLocatedTriplesPerBlock(
+      Permutation::Enum permutation);
+  const LocatedTriplesPerBlock& getLocatedTriplesPerBlock(
+      Permutation::Enum permutation) const;
+
+ private:
+  // Find the position of the given triple in the given permutation and add it
+  // to each of the six `LocatedTriplesPerBlock` maps (one per permutation).
+  // Return the iterators of where it was added (so that we can easily delete it
+  // again from these maps later).
+  std::vector<LocatedTripleHandles> locateAndAddTriples(
+      ad_utility::SharedCancellationHandle cancellationHandle,
+      std::span<const IdTriple<0>> idTriples, bool shouldExist);
+
+  // Erase `LocatedTriple` object from each `LocatedTriplesPerBlock` list. The
+  // argument are iterators for each list, as returned by the method
+  // `locateTripleInAllPermutations` above.
+  //
+  // NOTE: The iterators are invalid afterward. That is OK, as long as we also
+  // delete the respective entry in `triplesInserted_` or `triplesDeleted_`,
+  // which stores these iterators.
+  void eraseTripleInAllPermutations(LocatedTripleHandles& handles);
+};
+
+// DELTA TRIPLES AND THE CACHE
+//
+// For now, our approach only works when the results of index scans are not
+// cached (unless there are no relevant delta triples for a particular scan).
+// There are two ways how this can play out in the future:
+//
+// Either we generally do not cache the results of index scans anymore. This
+// would have various advantages, in particular, joining with something like
+// `rdf:type` would then be possible without storing the whole relation in
+// RAM. However, we need a faster decompression then and maybe a smaller block
+// size (currently 8 MB).
+//
+// Or we add the delta triples when iterating over the cached (uncompressed)
+// result from the index scan. In that case, we would need to (in Step 1 above)
+// store and maintain the positions in those uncompressed index scans. However,
+// this would only work for the results of index scans. For the results of more
+// complex subqueries, it's hard to figure out which delta triples are relevant.

src/index/Permutation.h

@@ -31,6 +31,8 @@ class Permutation {
   static constexpr auto SOP = Enum::SOP;
   static constexpr auto OPS = Enum::OPS;
   static constexpr auto OSP = Enum::OSP;
+  static constexpr auto ALL = {Enum::PSO, Enum::POS, Enum::SPO,
+                               Enum::SOP, Enum::OPS, Enum::OSP};
 
   using MetaData = IndexMetaDataMmapView;
   using Allocator = ad_utility::AllocatorWithLimit<Id>;
@@ -128,6 +130,9 @@ class Permutation {
   // _______________________________________________________
   const bool& isLoaded() const { return isLoaded_; }
 
+  // _______________________________________________________
+  const MetaData& metaData() const { return meta_; }
+
  private:
   // for Log output, e.g. "POS"
   std::string readableName_;
@@ -137,7 +142,6 @@ class Permutation {
   // sorted, for example {1, 0, 2} for PSO.
   array<size_t, 3> keyOrder_;
 
-  const MetaData& metaData() const { return meta_; }
   MetaData meta_;
 
   // This member is `optional` because we initialize it in a deferred way in the

test/CMakeLists.txt

@@ -118,6 +118,8 @@ addLinkAndDiscoverTest(LocatedTriplesTest index)
 
 addLinkAndDiscoverTestSerial(IdTripleTest index)
 
+addLinkAndDiscoverTestSerial(DeltaTriplesTest index)
+
 addLinkAndDiscoverTest(EngineTest engine)
 
 addLinkAndDiscoverTest(JoinTest engine)