VER: Release 0.37.0

CHANGELOG.md

@@ -1,5 +1,21 @@
 # Changelog
 
+## 0.37.0 - 2025-06-03
+
+### Breaking changes
+- Changed the type of the `output_dir` parameter in ` HistoricalClient::BatchDownload()`
+  to a `std::filesystem::path` and the return type to `std::filesystem::path`
+- Changed the type of the `file_path` parameter in
+  `HistoricalClient::TimeseriesGetRangeToFile()` to a `std::filesystem::path`
+- Changed the type of the `file_path` parameter in the `DbnFileStore`, `InFileStream`,
+  and `OutFileStream` constructors to a `std::filesystem::path`
+- Made `stype_in` and `schema` optionals in `Metadata` to align with how
+  `Metadata` is represented in other languages and removed the boolean
+  `has_mixed_stype_in` and `has_mixed_schema` fields
+
+### Bug fixes
+- Fixed behavior where encoding metadata could lower the `version`
+
 ## 0.36.0 - 2025-05-27
 
 This version marks the release of DBN version 3 (DBNv3), which is the new default.

CMakeLists.txt

@@ -6,7 +6,7 @@ cmake_minimum_required(VERSION 3.24..4.0)
 
 project(
   databento
-  VERSION 0.36.0
+  VERSION 0.37.0
   LANGUAGES CXX
   DESCRIPTION "Official Databento client library"
 )

README.md

@@ -95,16 +95,17 @@ Here is a simple program that fetches 10 seconds of trades for all ES mini futur
 #include <databento/live.hpp>
 #include <databento/symbol_map.hpp>
 #include <iostream>
-#include <string>
 #include <thread>
 
 using namespace databento;
 
 int main() {
   PitSymbolMap symbol_mappings;
 
-  auto client =
-      LiveBuilder{}.SetKeyFromEnv().SetDataset("GLBX.MDP3").BuildThreaded();
+  auto client = LiveBuilder{}
+                    .SetKeyFromEnv()
+                    .SetDataset(Dataset::GlbxMdp3)
+                    .BuildThreaded();
 
   auto handler = [&symbol_mappings](const Record& rec) {
     symbol_mappings.OnRecord(rec);

examples/live/readme.cpp

@@ -5,16 +5,17 @@
 #include <databento/live.hpp>
 #include <databento/symbol_map.hpp>
 #include <iostream>
-#include <string>
 #include <thread>
 
 using namespace databento;
 
 int main() {
   PitSymbolMap symbol_mappings;
 
-  auto client =
-      LiveBuilder{}.SetKeyFromEnv().SetDataset("GLBX.MDP3").BuildThreaded();
+  auto client = LiveBuilder{}
+                    .SetKeyFromEnv()
+                    .SetDataset(Dataset::GlbxMdp3)
+                    .BuildThreaded();
 
   auto handler = [&symbol_mappings](const Record& rec) {
     symbol_mappings.OnRecord(rec);

examples/live/simple.cpp

@@ -23,13 +23,13 @@ int main() {
                     .SetLogReceiver(log_receiver.get())
                     .SetSendTsOut(true)
                     .SetKeyFromEnv()
-                    .SetDataset(databento::dataset::kGlbxMdp3)
+                    .SetDataset(databento::Dataset::GlbxMdp3)
                     .BuildThreaded();
 
   // Set up signal handler for Ctrl+C
   std::signal(SIGINT, [](int signal) { gSignal = signal; });
 
-  std::vector<std::string> symbols{"ESZ4", "ESZ4 C4200", "ESZ4 P4100"};
+  std::vector<std::string> symbols{"ESZ5", "ESZ5 C6200", "ESZ5 P5500"};
   client.Subscribe(symbols, databento::Schema::Definition,
                    databento::SType::RawSymbol);
   client.Subscribe(symbols, databento::Schema::Mbo,

include/databento/dbn.hpp

@@ -3,6 +3,7 @@
 #include <date/date.h>
 
 #include <cstdint>
+#include <optional>
 #include <ostream>
 #include <string>
 #include <vector>
@@ -37,12 +38,10 @@ struct Metadata {
   std::uint8_t version;
   // The dataset code.
   std::string dataset;
-  // Indicates the DBN stream may contain multiple record types and data
-  // schemas. If true, the schema field should be ignored.
-  bool has_mixed_schema;
-  // The data record schema which affects the type record of present. If
-  // has_mixed_schema is true, this field should be ignored.
-  Schema schema;
+  // The data record schema which affects the type record of present. Will be
+  // nullopt for live where there can be a mix of `schema`s across
+  // subscriptions.
+  std::optional<Schema> schema;
   // The UNIX timestamp of the query start, or the first record if the file was
   // split.
   UnixNanos start;
@@ -51,12 +50,9 @@ struct Metadata {
   UnixNanos end;
   // The maximum number of records for the query.
   std::uint64_t limit;
-  // Indicates the DBN stream may been create with a mix of stype_in. This
-  // will always be the case for live data where stype_in is specified per
-  // subscription.
-  bool has_mixed_stype_in;
-  // The input symbology type. Should be ignored if has_mixed_schemas is true.
-  SType stype_in;
+  // The input symbology type. Will be nullopt for live data where there can be
+  // a mix of `stype_in`s across subscriptions.
+  std::optional<SType> stype_in;
   // The output symbology type.
   SType stype_out;
   // Whether the records contain an appended send timestamp.
@@ -105,13 +101,10 @@ inline bool operator!=(const SymbolMapping& lhs, const SymbolMapping& rhs) {
 
 inline bool operator==(const Metadata& lhs, const Metadata& rhs) {
   return lhs.version == rhs.version && lhs.dataset == rhs.dataset &&
-         (lhs.has_mixed_schema ? rhs.has_mixed_schema
-                               : lhs.schema == rhs.schema) &&
-         lhs.start == rhs.start && lhs.end == rhs.end &&
-         lhs.limit == rhs.limit &&
-         (lhs.has_mixed_stype_in ? rhs.has_mixed_stype_in
-                                 : lhs.stype_in == rhs.stype_in) &&
-         lhs.stype_out == rhs.stype_out && lhs.ts_out == rhs.ts_out &&
+         lhs.schema == rhs.schema && lhs.start == rhs.start &&
+         lhs.end == rhs.end && lhs.limit == rhs.limit &&
+         lhs.stype_in == rhs.stype_in && lhs.stype_out == rhs.stype_out &&
+         lhs.ts_out == rhs.ts_out &&
          lhs.symbol_cstr_len == rhs.symbol_cstr_len &&
          lhs.symbols == rhs.symbols && lhs.partial == rhs.partial &&
          lhs.not_found == rhs.not_found && lhs.mappings == rhs.mappings;

include/databento/dbn_file_store.hpp

@@ -1,6 +1,6 @@
 #pragma once
 
-#include <string>
+#include <filesystem>  // path
 
 #include "databento/dbn.hpp"          // DecodeMetadata
 #include "databento/dbn_decoder.hpp"  // DbnDecoder
@@ -16,8 +16,9 @@ namespace databento {
 // used on a given instance.
 class DbnFileStore {
  public:
-  explicit DbnFileStore(const std::string& file_path);
-  DbnFileStore(ILogReceiver* log_receiver, const std::string& file_path,
+  explicit DbnFileStore(const std::filesystem::path& file_path);
+  DbnFileStore(ILogReceiver* log_receiver,
+               const std::filesystem::path& file_path,
                VersionUpgradePolicy upgrade_policy);
 
   // Callback API: calling Replay consumes the input.

include/databento/file_stream.hpp

@@ -1,16 +1,16 @@
 #pragma once
 
-#include <cstddef>  // byte, size_t
-#include <fstream>  // ifstream, ofstream
-#include <string>
+#include <cstddef>     // byte, size_t
+#include <filesystem>  // path
+#include <fstream>     // ifstream, ofstream
 
 #include "databento/ireadable.hpp"
 #include "databento/iwritable.hpp"
 
 namespace databento {
 class InFileStream : public IReadable {
  public:
-  explicit InFileStream(const std::string& file_path);
+  explicit InFileStream(const std::filesystem::path& file_path);
 
   // Read exactly `length` bytes into `buffer`.
   void ReadExact(std::byte* buffer, std::size_t length) override;
@@ -24,7 +24,7 @@ class InFileStream : public IReadable {
 
 class OutFileStream : public IWritable {
  public:
-  explicit OutFileStream(const std::string& file_path);
+  explicit OutFileStream(const std::filesystem::path& file_path);
 
   void WriteAll(const std::byte* buffer, std::size_t length) override;
 

include/databento/historical.hpp

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <cstdint>
+#include <filesystem>
 #include <map>  // multimap
 #include <string>
 #include <vector>
@@ -75,12 +76,12 @@ class Historical {
   // Lists all files associated with a batch job.
   std::vector<BatchFileDesc> BatchListFiles(const std::string& job_id);
   // Returns the paths of the downloaded files.
-  std::vector<std::string> BatchDownload(const std::string& output_dir,
-                                         const std::string& job_id);
+  std::vector<std::filesystem::path> BatchDownload(
+      const std::filesystem::path& output_dir, const std::string& job_id);
   // Returns the path of the downloaded file.
-  std::string BatchDownload(const std::string& output_dir,
-                            const std::string& job_id,
-                            const std::string& filename_to_download);
+  std::filesystem::path BatchDownload(const std::filesystem::path& output_dir,
+                                      const std::string& job_id,
+                                      const std::string& filename_to_download);
 
   /*
    * Metadata API
@@ -206,30 +207,33 @@ class Historical {
       const std::string& dataset,
       const DateTimeRange<UnixNanos>& datetime_range,
       const std::vector<std::string>& symbols, Schema schema,
-      const std::string& file_path);
+      const std::filesystem::path& file_path);
   DbnFileStore TimeseriesGetRangeToFile(
       const std::string& dataset,
       const DateTimeRange<std::string>& datetime_range,
       const std::vector<std::string>& symbols, Schema schema,
-      const std::string& file_path);
+      const std::filesystem::path& file_path);
   DbnFileStore TimeseriesGetRangeToFile(
       const std::string& dataset,
       const DateTimeRange<UnixNanos>& datetime_range,
       const std::vector<std::string>& symbols, Schema schema, SType stype_in,
-      SType stype_out, std::uint64_t limit, const std::string& file_path);
+      SType stype_out, std::uint64_t limit,
+      const std::filesystem::path& file_path);
   DbnFileStore TimeseriesGetRangeToFile(
       const std::string& dataset,
       const DateTimeRange<std::string>& datetime_range,
       const std::vector<std::string>& symbols, Schema schema, SType stype_in,
-      SType stype_out, std::uint64_t limit, const std::string& file_path);
+      SType stype_out, std::uint64_t limit,
+      const std::filesystem::path& file_path);
 
  private:
   using HttplibParams = std::multimap<std::string, std::string>;
 
   BatchJob BatchSubmitJob(const HttplibParams& params);
   void StreamToFile(const std::string& url_path, const HttplibParams& params,
-                    const std::string& file_path);
-  void DownloadFile(const std::string& url, const std::string& output_path);
+                    const std::filesystem::path& file_path);
+  void DownloadFile(const std::string& url,
+                    const std::filesystem::path& output_path);
   std::vector<BatchJob> BatchListJobs(const HttplibParams& params);
   std::vector<DatasetConditionDetail> MetadataGetDatasetCondition(
       const HttplibParams& params);
@@ -240,7 +244,7 @@ class Historical {
                           const MetadataCallback& metadata_callback,
                           const RecordCallback& record_callback);
   DbnFileStore TimeseriesGetRangeToFile(const HttplibParams& params,
-                                        const std::string& file_path);
+                                        const std::filesystem::path& file_path);
 
   ILogReceiver* log_receiver_;
   const std::string key_;

pkg/PKGBUILD

@@ -1,7 +1,7 @@
 # Maintainer: Databento <support@databento.com>
 _pkgname=databento-cpp
 pkgname=databento-cpp-git
-pkgver=0.36.0
+pkgver=0.37.0
 pkgrel=1
 pkgdesc="Official C++ client for Databento"
 arch=('any')

src/dbn.cpp

@@ -34,12 +34,10 @@ std::ostream& operator<<(std::ostream& stream, const Metadata& metadata) {
                     .Build()
                     .AddField("version", metadata.version)
                     .AddField("dataset", metadata.dataset)
-                    .AddField("has_mixed_schema", metadata.has_mixed_schema)
                     .AddField("schema", metadata.schema)
                     .AddField("start", metadata.start)
                     .AddField("end", metadata.end)
                     .AddField("limit", metadata.limit)
-                    .AddField("has_mixed_stype_in", metadata.has_mixed_stype_in)
                     .AddField("stype_in", metadata.stype_in)
                     .AddField("stype_out", metadata.stype_out)
                     .AddField("ts_out", metadata.ts_out)

src/dbn_decoder.cpp

@@ -4,6 +4,7 @@
 
 #include <algorithm>  // copy
 #include <cstring>    // strncmp
+#include <optional>
 #include <vector>
 
 #include "databento/compat.hpp"
@@ -121,12 +122,9 @@ databento::Metadata DbnDecoder::DecodeMetadataFields(
   res.dataset = Consume(buffer, kDatasetCstrLen, "dataset");
   const auto raw_schema = Consume<std::uint16_t>(buffer);
   if (raw_schema == kNullSchema) {
-    res.has_mixed_schema = true;
-    // must initialize
-    res.schema = Schema::Mbo;
+    res.schema = std::nullopt;
   } else {
-    res.has_mixed_schema = false;
-    res.schema = static_cast<Schema>(raw_schema);
+    res.schema = {static_cast<Schema>(raw_schema)};
   }
   res.start =
       UnixNanos{std::chrono::nanoseconds{Consume<std::uint64_t>(buffer)}};
@@ -138,12 +136,9 @@ databento::Metadata DbnDecoder::DecodeMetadataFields(
   }
   const auto raw_stype_in = Consume<std::uint8_t>(buffer);
   if (raw_stype_in == kNullSType) {
-    res.has_mixed_stype_in = true;
-    // must initialize
-    res.stype_in = SType::InstrumentId;
+    res.stype_in = std::nullopt;
   } else {
-    res.has_mixed_stype_in = false;
-    res.stype_in = static_cast<SType>(raw_stype_in);
+    res.stype_in = {static_cast<SType>(raw_stype_in)};
   }
   res.stype_out = static_cast<SType>(Consume<std::uint8_t>(buffer));
   res.ts_out = static_cast<bool>(Consume<std::uint8_t>(buffer));

src/dbn_encoder.cpp

@@ -87,17 +87,24 @@ DbnEncoder::DbnEncoder(const Metadata& metadata, IWritable* output)
 }
 
 void DbnEncoder::EncodeMetadata(const Metadata& metadata, IWritable* output) {
-  const auto version = std::min<std::uint8_t>(
-      std::max<std::uint8_t>(1, metadata.version), kDbnVersion);
+  if (metadata.version > kDbnVersion) {
+    throw databento::InvalidArgumentError{
+        "EncodeMetadata", "metadata",
+        "Can't encode Metadata with version " +
+            std::to_string(+metadata.version) +
+            " which is greater than the maximum supported version " +
+            std::to_string(+kDbnVersion)};
+  }
+  const auto version = std::max<std::uint8_t>(1, metadata.version);
   EncodeChars(kDbnPrefix, kMagicSize - 1, output);
   EncodeAsBytes(version, output);
   const auto [length, end_padding] = CalcLength(metadata);
   EncodeAsBytes(length, output);
   EncodeFixedLenCStr(kDatasetCstrLen, metadata.dataset, output);
-  if (metadata.has_mixed_schema) {
-    EncodeAsBytes(kNullSchema, output);
+  if (metadata.schema.has_value()) {
+    EncodeAsBytes(*metadata.schema, output);
   } else {
-    EncodeAsBytes(metadata.schema, output);
+    EncodeAsBytes(kNullSchema, output);
   }
   EncodeAsBytes(metadata.start, output);
   EncodeAsBytes(metadata.end, output);
@@ -106,10 +113,10 @@ void DbnEncoder::EncodeMetadata(const Metadata& metadata, IWritable* output) {
     // backwards compatibility for record_count
     EncodeAsBytes(kNullRecordCount, output);
   }
-  if (metadata.has_mixed_stype_in) {
-    EncodeAsBytes(kNullSType, output);
+  if (metadata.stype_in.has_value()) {
+    EncodeAsBytes(*metadata.stype_in, output);
   } else {
-    EncodeAsBytes(metadata.stype_in, output);
+    EncodeAsBytes(kNullSType, output);
   }
   EncodeAsBytes(metadata.stype_out, output);
   EncodeAsBytes(static_cast<std::uint8_t>(metadata.ts_out), output);

src/dbn_file_store.cpp

@@ -8,11 +8,11 @@
 
 using databento::DbnFileStore;
 
-DbnFileStore::DbnFileStore(const std::string& file_path)
+DbnFileStore::DbnFileStore(const std::filesystem::path& file_path)
     : decoder_{ILogReceiver::Default(), InFileStream{file_path}} {}
 
 DbnFileStore::DbnFileStore(ILogReceiver* log_receiver,
-                           const std::string& file_path,
+                           const std::filesystem::path& file_path,
                            VersionUpgradePolicy upgrade_policy)
     : decoder_{log_receiver, std::make_unique<InFileStream>(file_path),
                upgrade_policy} {}