VER: Release 0.36.0

.gitignore

@@ -2,4 +2,5 @@ build
 CMakeSettings.json
 vcpkg_installed
 .vs
+.cache
 !*.zst

CHANGELOG.md

@@ -1,5 +1,41 @@
 # Changelog
 
+## 0.36.0 - 2025-05-27
+
+This version marks the release of DBN version 3 (DBNv3), which is the new default.
+API methods and `DbnDecoder` support decoding all versions of DBN, but now default to
+upgrading data to version 3.
+
+### Enhancements
+- Added `UpgradeToV3` variant to `VersionUpgradePolicy` enum
+- Version 1 and 2 structs can be converted to version 3 structs with templated `Upgrade`
+  method
+- Metadata will now always be encoded with a length divisible by 8 bytes for better
+  alignment
+- Added `is_last` field to live subscription requests which will be used to improve the
+  handling of split subscription requests
+
+### Breaking changes
+- Release of DBN version 3:
+  - Definition schema:
+    - Updated `InstrumentDefMsg` with new `leg_` fields to support multi-leg strategy
+      definitions.
+    - Expanded `raw_instrument_id` to 64 bits to support more venues. Like other 64-bit
+      integer fields, its value will now be quoted in JSON
+    - Removed `trading_reference_date`, `trading_reference_price`, and
+      `settl_price_type` fields which will be normalized in the statistics schema
+    - Removed `md_security_trading_status` better served by the status schema
+    - Updated `asset` to 11 bytes and `kAssetCstrLen` to match
+  - Statistics schema:
+    - Updated `StatMsg` has an expanded 64-bit `quantity` field. `kUndefStatQuantity`
+      has been updated to match
+    - The previous `StatMsg` has been moved to `v2::StatMsg` or `StatMsgV2`
+
+### Bug fixes
+- Fixed "Zstd error decompressing: Operation made no progress over multiple calls, due
+  to output buffer being full" error with `TimeseriesGetRange`
+- Fixed missing implementation of `HistoricalBuilder::SetLogReceiver`
+
 ## 0.35.1 - 2025-05-20
 
 ### Bug fixes

CMakeLists.txt

@@ -6,7 +6,7 @@ cmake_minimum_required(VERSION 3.24..4.0)
 
 project(
   databento
-  VERSION 0.35.1
+  VERSION 0.36.0
   LANGUAGES CXX
   DESCRIPTION "Official Databento client library"
 )

cmake/SourcesAndHeaders.cmake

@@ -71,5 +71,5 @@ set(sources
   src/symbol_map.cpp
   src/symbology.cpp
   src/v1.cpp
-  src/v3.cpp
+  src/v2.cpp
 )

examples/historical/timeseries_get_range.cpp

@@ -1,34 +1,15 @@
-#include <chrono>
-#include <ctime>
-#include <iomanip>
 #include <iostream>  // setw
-#ifdef _WIN32
-// _mkgmtime is equivalent to timegm
-#define timegm _mkgmtime
-#endif
 
 #include "databento/constants.hpp"
-#include "databento/datetime.hpp"
 #include "databento/enums.hpp"
 #include "databento/historical.hpp"
 
-// Converts a date to Unix Epoch nanoseconds
-databento::UnixNanos DateToUnixNanos(int year, int month, int day) {
-  std::tm time{};
-  time.tm_year = year - 1900;
-  // January is 0
-  time.tm_mon = month - 1;
-  time.tm_mday = day;
-  return databento::UnixNanos{std::chrono::seconds{::timegm(&time)}};
-}
-
 int main() {
   auto client = databento::HistoricalBuilder{}.SetKeyFromEnv().Build();
-  const databento::UnixNanos start = DateToUnixNanos(2022, 10, 3);
-  const databento::UnixNanos end = DateToUnixNanos(2022, 10, 4);
   const auto limit = 1000;
   client.TimeseriesGetRange(
-      databento::dataset::kGlbxMdp3, {start, end}, {"ESZ2"},
+      databento::dataset::kGlbxMdp3,
+      databento::DateTimeRange<std::string>{"2022-10-03"}, {"ESZ2"},
       databento::Schema::Trades, databento::SType::RawSymbol,
       databento::SType::InstrumentId, limit,
       [](databento::Metadata&& metadata) { std::cout << metadata << '\n'; },

include/databento/constants.hpp

@@ -16,16 +16,16 @@ static constexpr auto kUndefOrderSize =
     std::numeric_limits<std::uint32_t>::max();
 // The sentinel value for an unset statistic quantity.
 static constexpr auto kUndefStatQuantity =
-    std::numeric_limits<std::int32_t>::max();
+    std::numeric_limits<std::int64_t>::max();
 // The sentinel value for an unset or null timestamp.
 static constexpr auto kUndefTimestamp =
     std::numeric_limits<std::uint64_t>::max();
 // The current version of the DBN encoding.
-static constexpr auto kDbnVersion = 2;
+static constexpr auto kDbnVersion = 3;
 // The length of fixed-length symbol strings.
 static constexpr auto kSymbolCstrLen = 71;
 // The length of fixed-length asset string.
-static constexpr auto kAssetCstrLen = 7;
+static constexpr auto kAssetCstrLen = 11;
 // The multiplier for converting the `length` field in `RecordHeader` to bytes.
 static constexpr std::size_t kRecordHeaderLengthMultiplier = 4;
 

include/databento/dbn_decoder.hpp

@@ -15,7 +15,8 @@
 
 namespace databento {
 // DBN decoder. Set upgrade_policy to control how DBN version 1 data should be
-// handled. Defaults to upgrading DBNv1 data to version 2 (the current version).
+// handled. Defaults to upgrading DBN versions 1 and 2 to version 3 (the current
+// version).
 class DbnDecoder {
  public:
   DbnDecoder(ILogReceiver* log_receiver, InFileStream file_stream);

include/databento/dbn_encoder.hpp

@@ -32,7 +32,8 @@ class DbnEncoder {
   void EncodeRecord(const Record& record);
 
  private:
-  static std::uint32_t CalcLength(const Metadata& metadata);
+  static std::pair<std::uint32_t, std::uint32_t> CalcLength(
+      const Metadata& metadata);
 
   IWritable* output_;
 };

include/databento/detail/buffer.hpp

@@ -3,6 +3,7 @@
 #include <cstddef>
 #include <memory>
 #include <new>
+#include <ostream>
 
 #include "databento/ireadable.hpp"
 #include "databento/iwritable.hpp"
@@ -22,22 +23,32 @@ class Buffer : public IReadable, public IWritable {
   void WriteAll(const char* data, std::size_t length);
   void WriteAll(const std::byte* data, std::size_t length) override;
 
-  std::byte*& WriteBegin() { return write_pos_; }
+  std::byte* WriteBegin() { return write_pos_; }
   std::byte* WriteEnd() { return end_; }
   const std::byte* WriteBegin() const { return write_pos_; }
   const std::byte* WriteEnd() const { return end_; }
+  // Indicate how many bytes were written
+  void Fill(std::size_t length) { write_pos_ += length; }
   std::size_t WriteCapacity() const {
     return static_cast<std::size_t>(end_ - write_pos_);
   }
 
-  /// Will throw if `length > ReadCapacity()`.
+  // Will throw if `length > ReadCapacity()`.
   void ReadExact(std::byte* buffer, std::size_t length) override;
   std::size_t ReadSome(std::byte* buffer, std::size_t max_length) override;
 
-  std::byte*& ReadBegin() { return read_pos_; }
+  std::byte* ReadBegin() { return read_pos_; }
   std::byte* ReadEnd() { return write_pos_; }
   const std::byte* ReadBegin() const { return read_pos_; }
   const std::byte* ReadEnd() const { return write_pos_; }
+  // Indicate how mnay bytes were read
+  void Consume(std::size_t length) {
+    read_pos_ += length;
+    if (static_cast<std::size_t>(read_pos_ - buf_.get()) > (Capacity() / 2)) {
+      Shift();
+    }
+  }
+  void ConsumeNoShift(std::size_t length) { read_pos_ += length; }
   std::size_t ReadCapacity() const {
     return static_cast<std::size_t>(write_pos_ - read_pos_);
   }
@@ -52,6 +63,8 @@ class Buffer : public IReadable, public IWritable {
   void Reserve(std::size_t capacity);
   void Shift();
 
+  friend std::ostream& operator<<(std::ostream& stream, const Buffer& buffer);
+
  private:
   static constexpr std::align_val_t kAlignment{8};
 

include/databento/enums.hpp

@@ -303,6 +303,7 @@ enum class StatUpdateAction : std::uint8_t {
 enum class VersionUpgradePolicy : std::uint8_t {
   AsIs = 0,
   UpgradeToV2 = 1,
+  UpgradeToV3 = 2,
 };
 
 namespace status_action {

include/databento/live.hpp

@@ -52,7 +52,7 @@ class LiveBuilder {
   std::string key_;
   std::string dataset_;
   bool send_ts_out_{false};
-  VersionUpgradePolicy upgrade_policy_{VersionUpgradePolicy::UpgradeToV2};
+  VersionUpgradePolicy upgrade_policy_{VersionUpgradePolicy::UpgradeToV3};
   std::chrono::seconds heartbeat_interval_{};
 };
 }  // namespace databento

include/databento/record.hpp

@@ -17,12 +17,6 @@
 #include "databento/publishers.hpp"  // Publisher
 
 namespace databento {
-// Forward declare
-namespace v3 {
-struct InstrumentDefMsg;
-struct StatMsg;
-}  // namespace v3
-
 // Common data for all Databento Records.
 struct RecordHeader {
   static constexpr std::size_t kLengthMultiplier =
@@ -379,7 +373,6 @@ static_assert(alignof(StatusMsg) == 8, "Must have 8-byte alignment");
 struct InstrumentDefMsg {
   static bool HasRType(RType rtype) { return rtype == RType::InstrumentDef; }
 
-  v3::InstrumentDefMsg ToV3() const;
   UnixNanos IndexTs() const { return ts_recv; }
   const char* Currency() const { return currency.data(); }
   const char* SettlCurrency() const { return settl_currency.data(); }
@@ -395,6 +388,7 @@ struct InstrumentDefMsg {
   const char* StrikePriceCurrency() const {
     return strike_price_currency.data();
   }
+  const char* LegRawSymbol() const { return leg_raw_symbol.data(); }
 
   RecordHeader hd;
   UnixNanos ts_recv;
@@ -405,14 +399,15 @@ struct InstrumentDefMsg {
   std::int64_t high_limit_price;
   std::int64_t low_limit_price;
   std::int64_t max_price_variation;
-  std::int64_t trading_reference_price;
   std::int64_t unit_of_measure_qty;
   std::int64_t min_price_increment_amount;
   std::int64_t price_ratio;
   std::int64_t strike_price;
+  std::uint64_t raw_instrument_id;
+  std::int64_t leg_price;
+  std::int64_t leg_delta;
   std::int32_t inst_attrib_value;
   std::uint32_t underlying_id;
-  std::uint32_t raw_instrument_id;
   std::int32_t market_depth_implied;
   std::int32_t market_depth;
   std::uint32_t market_segment_id;
@@ -424,11 +419,18 @@ struct InstrumentDefMsg {
   std::int32_t contract_multiplier;
   std::int32_t decay_quantity;
   std::int32_t original_contract_size;
-  std::uint16_t trading_reference_date;
+  std::uint32_t leg_instrument_id;
+  std::int32_t leg_ratio_price_numerator;
+  std::int32_t leg_ratio_price_denominator;
+  std::int32_t leg_ratio_qty_numerator;
+  std::int32_t leg_ratio_qty_denominator;
+  std::uint32_t leg_underlying_id;
   std::int16_t appl_id;
   std::uint16_t maturity_year;
   std::uint16_t decay_start_date;
   std::uint16_t channel_id;
+  std::uint16_t leg_count;
+  std::uint16_t leg_index;
   std::array<char, 4> currency;
   std::array<char, 4> settl_currency;
   std::array<char, 6> secsubtype;
@@ -441,12 +443,11 @@ struct InstrumentDefMsg {
   std::array<char, 31> unit_of_measure;
   std::array<char, 21> underlying;
   std::array<char, 4> strike_price_currency;
+  std::array<char, kSymbolCstrLen> leg_raw_symbol;
   InstrumentClass instrument_class;
   MatchAlgorithm match_algorithm;
-  std::uint8_t md_security_trading_status;
   std::uint8_t main_fraction;
   std::uint8_t price_display_format;
-  std::uint8_t settl_price_type;
   std::uint8_t sub_fraction;
   std::uint8_t underlying_product;
   SecurityUpdateAction security_update_action;
@@ -457,9 +458,12 @@ struct InstrumentDefMsg {
   std::int8_t contract_multiplier_unit;
   std::int8_t flow_schedule_type;
   std::uint8_t tick_rule;
-  std::array<char, 10> reserved;
+  InstrumentClass leg_instrument_class;
+  Side leg_side;
+  // padding for alignment
+  std::array<char, 17> reserved;
 };
-static_assert(sizeof(InstrumentDefMsg) == 400,
+static_assert(sizeof(InstrumentDefMsg) == 520,
               "InstrumentDefMsg size must match Rust");
 static_assert(alignof(InstrumentDefMsg) == 8, "Must have 8-byte alignment");
 
@@ -502,23 +506,22 @@ static_assert(alignof(ImbalanceMsg) == 8, "Must have 8-byte alignment");
 struct StatMsg {
   static bool HasRType(RType rtype) { return rtype == RType::Statistics; }
 
-  v3::StatMsg ToV3() const;
   UnixNanos IndexTs() const { return ts_recv; }
 
   RecordHeader hd;
   UnixNanos ts_recv;
   UnixNanos ts_ref;
   std::int64_t price;
-  std::int32_t quantity;
+  std::int64_t quantity;
   std::uint32_t sequence;
   TimeDeltaNanos ts_in_delta;
   StatType stat_type;
   std::uint16_t channel_id;
   StatUpdateAction update_action;
   std::uint8_t stat_flags;
-  std::array<char, 6> reserved;
+  std::array<char, 18> reserved;
 };
-static_assert(sizeof(StatMsg) == 64, "StatMsg size must match Rust");
+static_assert(sizeof(StatMsg) == 80, "StatMsg size must match Rust");
 static_assert(alignof(StatMsg) == 8, "Must have 8-byte alignment");
 
 // An error message from the Live Subscription Gateway (LSG). This will never
@@ -798,4 +801,6 @@ std::ostream& operator<<(std::ostream& stream,
 
 // The length in bytes of the largest record type.
 static constexpr std::size_t kMaxRecordLen = 520 + 8;
+static_assert(kMaxRecordLen == sizeof(InstrumentDefMsg) + sizeof(UnixNanos),
+              "v3 definition with ts_out should be the largest record");
 }  // namespace databento