diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 94c2aa10..48d019cb 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.1.0 +current_version = 4.0.0 commit = True tag = False @@ -26,3 +26,7 @@ replace = questdb-rs/{new_version}/ [bumpversion:file:questdb-rs-ffi/Cargo.toml] search = version = "{current_version}" replace = version = "{new_version}" + +[bumpversion:file:include/questdb/ingress/line_sender.hpp] +search = questdb/c++/{current_version} +replace = questdb/c++/{new_version} diff --git a/CMakeLists.txt b/CMakeLists.txt index 40b604f0..79875941 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 3.15.0) -project(c-questdb-client VERSION 3.1.0) +project(c-questdb-client VERSION 4.0.0) set(CPACK_PROJECT_NAME ${PROJECT_NAME}) set(CPACK_PROJECT_VERSION ${PROJECT_VERSION}) @@ -90,16 +90,30 @@ endfunction() if (QUESTDB_TESTS_AND_EXAMPLES) compile_example( line_sender_c_example + examples/concat.c examples/line_sender_c_example.c) compile_example( line_sender_c_example_auth + examples/concat.c examples/line_sender_c_example_auth.c) compile_example( line_sender_c_example_tls_ca + examples/concat.c examples/line_sender_c_example_tls_ca.c) compile_example( line_sender_c_example_auth_tls + examples/concat.c examples/line_sender_c_example_auth_tls.c) + compile_example( + line_sender_c_example_http + examples/concat.c + examples/line_sender_c_example_http.c) + compile_example( + line_sender_c_example_from_conf + examples/line_sender_c_example_from_conf.c) + compile_example( + line_sender_c_example_from_env + examples/line_sender_c_example_from_env.c) compile_example( line_sender_cpp_example examples/line_sender_cpp_example.cpp) @@ -112,6 +126,15 @@ if (QUESTDB_TESTS_AND_EXAMPLES) compile_example( line_sender_cpp_example_auth_tls examples/line_sender_cpp_example_auth_tls.cpp) + compile_example( + line_sender_cpp_example_http + examples/line_sender_cpp_example_http.cpp) + compile_example( + line_sender_cpp_example_from_conf + examples/line_sender_cpp_example_from_conf.cpp) + compile_example( + line_sender_cpp_example_from_env + examples/line_sender_cpp_example_from_env.cpp) # Include Rust tests as part of the tests run add_test( diff --git a/README.md b/README.md index 1906dd86..c43ee20c 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ This library makes it easy to insert data into [QuestDB](https://questdb.io/). This client library implements the [InfluxDB Line Protocol]( -https://questdb.io/docs/reference/api/ilp/overview/) (ILP) over TCP. +https://questdb.io/docs/reference/api/ilp/overview/) (ILP) over TCP or HTTP. * Implementation is in Rust, with no additional [run-time or link-time dependencies](doc/BUILD.md#pre-requisites-and-dependencies) @@ -16,21 +16,21 @@ https://questdb.io/docs/reference/api/ilp/overview/) (ILP) over TCP. ## Insertion Protocols Overview -Inserting data into QuestDB can be done via one of three protocols. +Inserting data into QuestDB can be done in several ways. + +This library supports ILP/HTTP (default-recommended) and ILP/TCP (specific +streaming use cases). | Protocol | Record Insertion Reporting | Data Insertion Performance | | -------- | -------------------------- | -------------------------- | -| [ILP](https://questdb.io/docs/reference/api/ilp/overview/)| Errors in logs; Disconnect on error | **Best** | +| **[ILP/HTTP](https://questdb.io/docs/reference/api/ilp/overview/)** | Transaction-level (on flush) | **Excellent** | +| [ILP/TCP](https://questdb.io/docs/reference/api/ilp/overview/)| Errors in logs; Disconnect on error | **Best** (tolerates higher latency networks) | | [CSV Upload via HTTP](https://questdb.io/docs/reference/api/rest/#imp---import-data) | Configurable | Very Good | | [PostgreSQL](https://questdb.io/docs/reference/api/postgres/) | Transaction-level | Good | -This library implements the **ILP protocol** and mitigates the lack of confirmation -and error reporting by validating data ahead of time before any data is sent -to the database instance. - -For example, the client library will report that a supplied string isn't encoded -in UTF-8. Some issues unfortunately can't be caught by the library and require -some [care and diligence to avoid data problems](doc/CONSIDERATIONS.md). +Server errors are only reported back to the client for ILP/HTTP. +See the [flush troubleshooting](doc/CONSIDERATIONS.md) docs for more details on +how to debug ILP/TCP. For an overview and code examples, see the [ILP page of the developer docs](https://questdb.io/docs/develop/insert-data/#influxdb-line-protocol). diff --git a/cbindgen.toml b/cbindgen.toml index bdb770c5..3d94ffdb 100644 --- a/cbindgen.toml +++ b/cbindgen.toml @@ -8,7 +8,7 @@ header = """/******************************************************************* * \\__\\_\\\\__,_|\\___||___/\\__|____/|____/ * * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2023 QuestDB + * Copyright (c) 2019-2024 QuestDB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/ci/compile.yaml b/ci/compile.yaml new file mode 100644 index 00000000..e761d75c --- /dev/null +++ b/ci/compile.yaml @@ -0,0 +1,14 @@ +steps: + - script: | + rustup update $(toolchain) + rustup default $(toolchain) + condition: ne(variables['toolchain'], '') + displayName: "Update and set Rust toolchain" + - script: cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DQUESTDB_TESTS_AND_EXAMPLES=ON + env: + JAVA_HOME: $(JAVA_HOME_11_X64) + displayName: "Build Makefile with CMake" + - script: cmake --build build + env: + JAVA_HOME: $(JAVA_HOME_11_X64) + displayName: "Make" diff --git a/ci/run_all_tests.py b/ci/run_all_tests.py index 533207a6..3a976c10 100644 --- a/ci/run_all_tests.py +++ b/ci/run_all_tests.py @@ -38,8 +38,9 @@ def main(): test_line_sender_path = next(iter( build_dir.glob(f'**/test_line_sender{exe_suffix}'))) system_test_path = pathlib.Path('system_test') / 'test.py' - qdb_v = '7.3.2' # The version of QuestDB we'll test against. + qdb_v = '7.3.10' # The version of QuestDB we'll test against. + run_cmd('cargo', 'test', '--', '--nocapture', cwd='questdb-rs') run_cmd('cargo', 'test', '--all-features', '--', '--nocapture', cwd='questdb-rs') run_cmd(str(test_line_sender_path)) run_cmd('python3', str(system_test_path), 'run', '--versions', qdb_v, '-v') diff --git a/ci/run_tests_pipeline.yaml b/ci/run_tests_pipeline.yaml index 1087a073..78c25296 100644 --- a/ci/run_tests_pipeline.yaml +++ b/ci/run_tests_pipeline.yaml @@ -19,38 +19,27 @@ stages: linux: imageName: "ubuntu-latest" poolName: "Azure Pipelines" - BUILD_WITH_MINGW: "0" linux-stable: imageName: "ubuntu-latest" poolName: "Azure Pipelines" - BUILD_WITH_MINGW: "0" toolchain: "stable" linux-beta: imageName: "ubuntu-latest" poolName: "Azure Pipelines" - BUILD_WITH_MINGW: "0" toolchain: "beta" linux-nightly: imageName: "ubuntu-latest" poolName: "Azure Pipelines" - BUILD_WITH_MINGW: "0" toolchain: "nightly" mac: imageName: "macos-latest" poolName: "Azure Pipelines" - BUILD_WITH_MINGW: "0" windows-msvc-2022: imageName: "windows-2022" poolName: "Azure Pipelines" - BUILD_WITH_MINGW: "0" windows-msvc-2019: imageName: "windows-2019" poolName: "Azure Pipelines" - BUILD_WITH_MINGW: "0" - # windows-mingw: - # imageName: "windows-2022" - # poolName: "Azure Pipelines" - # BUILD_WITH_MINGW: "1" pool: name: $(poolName) vmImage: $(imageName) @@ -60,34 +49,11 @@ stages: fetchDepth: 1 lfs: false submodules: false - - script: | - rustup update $(toolchain) - rustup default $(toolchain) - condition: ne(variables['toolchain'], '') - displayName: "Update and set Rust toolchain" + - template: compile.yaml - script: | cd questdb-rs - cargo build --example basic --features=chrono_timestamp - cargo build --example auth --features=chrono_timestamp - cargo build --example auth_tls --features=chrono_timestamp - displayName: Build Rust examples. - - script: cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DQUESTDB_TESTS_AND_EXAMPLES=ON - env: - JAVA_HOME: $(JAVA_HOME_11_X64) - displayName: "Build Makefile with CMake" - condition: eq(variables['BUILD_WITH_MINGW'], '0') - # - script: | - # choco upgrade mingw -y - # rustup target add x86_64-pc-windows-gnu - # cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -G "MinGW Makefiles" - # env: - # JAVA_HOME: $(JAVA_HOME_11_X64) - # displayName: "Build Makefile with CMake" - # condition: eq(variables['BUILD_WITH_MINGW'], '1') - - script: cmake --build build - env: - JAVA_HOME: $(JAVA_HOME_11_X64) - displayName: "Make" + cargo build --examples --all-features + displayName: "Build Rust examples" - script: python3 ci/run_all_tests.py env: JAVA_HOME: $(JAVA_HOME_11_X64) @@ -96,7 +62,6 @@ stages: inputs: pathToPublish: ./build displayName: "Publish build directory" - condition: eq(variables['BUILD_WITH_MINGW'], '1') - job: CargoFmtAndClippy displayName: "cargo fmt and clippy" pool: @@ -134,3 +99,26 @@ stages: cd tls_proxy cargo clippy --all-targets --all-features -- -D warnings displayName: "tls_proxy: clippy" + - job: TestVsQuestDBMaster + displayName: "Vs QuestDB 'master'" + pool: + vmImage: 'ubuntu-latest' + timeoutInMinutes: 60 + steps: + - checkout: self + fetchDepth: 1 + lfs: false + submodules: false + - template: compile.yaml + - script: | + git clone --depth 1 https://github.com/questdb/questdb.git + displayName: git clone questdb + - task: Maven@3 + displayName: "Compile QuestDB" + inputs: + mavenPOMFile: 'questdb/pom.xml' + jdkVersionOption: '1.11' + options: "-DskipTests -Pbuild-web-console" + - script: | + python3 system_test/test.py run --repo ./questdb -v + displayName: "integration test" diff --git a/cpp_test/build_env.h b/cpp_test/build_env.h index a4de13b1..23a88486 100644 --- a/cpp_test/build_env.h +++ b/cpp_test/build_env.h @@ -6,7 +6,7 @@ * \__\_\\__,_|\___||___/\__|____/|____/ * * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2023 QuestDB + * Copyright (c) 2019-2024 QuestDB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/cpp_test/mock_server.cpp b/cpp_test/mock_server.cpp index 562105dd..71eb3654 100644 --- a/cpp_test/mock_server.cpp +++ b/cpp_test/mock_server.cpp @@ -6,7 +6,7 @@ * \__\_\\__,_|\___||___/\__|____/|____/ * * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2023 QuestDB + * Copyright (c) 2019-2024 QuestDB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/cpp_test/mock_server.hpp b/cpp_test/mock_server.hpp index 13df198f..9993e66c 100644 --- a/cpp_test/mock_server.hpp +++ b/cpp_test/mock_server.hpp @@ -6,7 +6,7 @@ * \__\_\\__,_|\___||___/\__|____/|____/ * * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2023 QuestDB + * Copyright (c) 2019-2024 QuestDB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/cpp_test/test_line_sender.cpp b/cpp_test/test_line_sender.cpp index 5f284064..21e9f1d2 100644 --- a/cpp_test/test_line_sender.cpp +++ b/cpp_test/test_line_sender.cpp @@ -6,7 +6,7 @@ * \__\_\\__,_|\___||___/\__|____/|____/ * * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2023 QuestDB + * Copyright (c) 2019-2024 QuestDB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -58,9 +58,12 @@ TEST_CASE("line_sender c api basics") }}; ::line_sender_utf8 host = {0, nullptr}; CHECK(::line_sender_utf8_init(&host, 9, "localhost", &err)); - ::line_sender_opts* opts = ::line_sender_opts_new(host, server.port()); + ::line_sender_opts* opts = ::line_sender_opts_new( + ::line_sender_protocol_tcp, + host, + server.port()); CHECK_NE(opts, nullptr); - ::line_sender* sender = ::line_sender_connect(opts, &err); + ::line_sender* sender = ::line_sender_build(opts, &err); line_sender_opts_free(opts); CHECK_NE(sender, nullptr); CHECK_FALSE(::line_sender_must_close(sender)); @@ -96,10 +99,38 @@ TEST_CASE("line_sender c api basics") CHECK(server.msgs().front() == "test,t1=v1 f1=0.5 10000000\n"); } +TEST_CASE("Opts service API tests") { + // We just check these compile and link. + + line_sender_utf8 host = QDB_UTF8_LITERAL("localhost"); + line_sender_utf8 port = QDB_UTF8_LITERAL("9009"); + + ::line_sender_protocol protocols[] = { + ::line_sender_protocol_tcp, + ::line_sender_protocol_tcps, + ::line_sender_protocol_http, + ::line_sender_protocol_https + }; + + for (size_t index = 0; index < sizeof(protocols) / sizeof(::line_sender_protocol); ++index) + { + auto proto = protocols[index]; + ::line_sender_opts* opts1 = ::line_sender_opts_new_service( + proto, + host, + port); + ::line_sender_opts_free(opts1); + } + +} + TEST_CASE("line_sender c++ connect disconnect") { questdb::ingress::test::mock_server server; - questdb::ingress::line_sender sender{"localhost", server.port()}; + questdb::ingress::line_sender sender{ + questdb::ingress::protocol::tcp, + "localhost", + server.port()}; CHECK_FALSE(sender.must_close()); server.accept(); CHECK(server.recv() == 0); @@ -109,6 +140,7 @@ TEST_CASE("line_sender c++ api basics") { questdb::ingress::test::mock_server server; questdb::ingress::line_sender sender{ + questdb::ingress::protocol::tcp, std::string("localhost"), std::to_string(server.port())}; CHECK_FALSE(sender.must_close()); @@ -133,9 +165,11 @@ TEST_CASE("line_sender c++ api basics") TEST_CASE("test multiple lines") { questdb::ingress::test::mock_server server; - questdb::ingress::line_sender sender{ - "localhost", - server.port()}; + std::string conf_str = + "tcp::addr=localhost:" + + std::to_string(server.port()) + ";"; + questdb::ingress::line_sender sender = + questdb::ingress::line_sender::from_conf(conf_str); CHECK_FALSE(sender.must_close()); server.accept(); CHECK(server.recv() == 0); @@ -175,6 +209,7 @@ TEST_CASE("State machine testing -- flush without data.") { questdb::ingress::test::mock_server server; questdb::ingress::line_sender sender{ + questdb::ingress::protocol::tcp, std::string_view{"localhost"}, std::to_string(server.port())}; @@ -192,6 +227,7 @@ TEST_CASE("One symbol only - flush before server accept") { questdb::ingress::test::mock_server server; questdb::ingress::line_sender sender{ + questdb::ingress::protocol::tcp, std::string{"localhost"}, server.port()}; @@ -214,6 +250,7 @@ TEST_CASE("One column only - server.accept() after flush, before close") { questdb::ingress::test::mock_server server; questdb::ingress::line_sender sender{ + questdb::ingress::protocol::tcp, "localhost", server.port()}; @@ -235,6 +272,7 @@ TEST_CASE("Symbol after column") { questdb::ingress::test::mock_server server; questdb::ingress::line_sender sender{ + questdb::ingress::protocol::tcp, "localhost", server.port()}; @@ -346,17 +384,17 @@ TEST_CASE("Validation of bad chars in key names.") TEST_CASE("Buffer move and copy ctor testing") { - const size_t init_capacity = 128; + const size_t init_buf_size = 128; - questdb::ingress::line_sender_buffer buffer1{init_capacity}; + questdb::ingress::line_sender_buffer buffer1{init_buf_size}; buffer1.table("buffer1"); CHECK(buffer1.peek() == "buffer1"); - questdb::ingress::line_sender_buffer buffer2{2 * init_capacity}; + questdb::ingress::line_sender_buffer buffer2{2 * init_buf_size}; buffer2.table("buffer2"); CHECK(buffer2.peek() == "buffer2"); - questdb::ingress::line_sender_buffer buffer3{3 * init_capacity}; + questdb::ingress::line_sender_buffer buffer3{3 * init_buf_size}; buffer3.table("buffer3"); CHECK(buffer3.peek() == "buffer3"); @@ -395,6 +433,7 @@ TEST_CASE("Sender move testing.") const questdb::ingress::utf8_view& host_ref = host; questdb::ingress::line_sender sender1{ + questdb::ingress::protocol::tcp, host_ref, server1.port()}; @@ -422,6 +461,7 @@ TEST_CASE("Sender move testing.") CHECK(sender2.must_close()); questdb::ingress::line_sender sender3{ + questdb::ingress::protocol::tcp, "localhost", server2.port()}; CHECK_FALSE(sender3.must_close()); @@ -434,7 +474,10 @@ TEST_CASE("Bad hostname") { try { - questdb::ingress::line_sender sender{"dummy_hostname", "9009"}; + questdb::ingress::line_sender sender{ + questdb::ingress::protocol::tcp, + "dummy_hostname", + "9009"}; CHECK_MESSAGE(false, "Expected exception"); } catch (const questdb::ingress::line_sender_error& se) @@ -454,8 +497,11 @@ TEST_CASE("Bad interface") { try { - questdb::ingress::opts opts{"localhost", "9009"}; - opts.net_interface("dummy_hostname"); + questdb::ingress::opts opts{ + questdb::ingress::protocol::tcp, + "localhost", + "9009"}; + opts.bind_interface("dummy_hostname"); questdb::ingress::line_sender sender{opts}; CHECK_MESSAGE(false, "Expected exception"); } @@ -479,6 +525,7 @@ TEST_CASE("Bad port") try { questdb::ingress::line_sender sender{ + questdb::ingress::protocol::tcp, "localhost", bad_port}; CHECK_MESSAGE(false, "Expected exception"); @@ -509,6 +556,7 @@ TEST_CASE("Bad connect") // Port 1 is generally the tcpmux service which one would // very much expect to never be running. questdb::ingress::line_sender sender{ + questdb::ingress::protocol::tcp, "localhost", 1}; CHECK_MESSAGE(false, "Expected exception"); @@ -531,15 +579,22 @@ TEST_CASE("Bad CA path") try { questdb::ingress::test::mock_server server; - questdb::ingress::opts opts{"localhost", server.port()}; - opts.tls("/an/invalid/path/to/ca.pem"); + questdb::ingress::opts opts{ + questdb::ingress::protocol::tcps, + "localhost", + server.port()}; + + opts.auth_timeout(1000); + + opts.tls_ca(questdb::ingress::ca::pem_file); + opts.tls_roots("/an/invalid/path/to/ca.pem"); questdb::ingress::line_sender sender{opts}; } catch(const questdb::ingress::line_sender_error& se) { std::string msg{se.what()}; CHECK_MESSAGE( - msg.rfind("Could not open certificate authority file", 0) == 0, + msg.rfind("Could not open root certificate file from path", 0) == 0, msg); } catch (...) @@ -552,32 +607,69 @@ TEST_CASE("os certs") { // We're just checking these APIs don't throw. questdb::ingress::test::mock_server server; - questdb::ingress::opts opts{"localhost", server.port()}; - opts.tls_os_roots(); - opts.tls_webpki_and_os_roots(); + { + questdb::ingress::opts opts{ + questdb::ingress::protocol::tcps, + "localhost", + server.port()}; + opts.tls_ca(questdb::ingress::ca::webpki_roots); + } + + { + questdb::ingress::opts opts{ + questdb::ingress::protocol::https, + "localhost", + server.port()}; + opts.tls_ca(questdb::ingress::ca::os_roots); + } + + { + questdb::ingress::opts opts{ + questdb::ingress::protocol::https, + "localhost", + server.port()}; + opts.tls_ca(questdb::ingress::ca::webpki_and_os_roots); + } } TEST_CASE("Opts copy ctor, assignment and move testing.") { { - questdb::ingress::opts opts1{"localhost", "9009"}; + questdb::ingress::opts opts1{ + questdb::ingress::protocol::tcp, + "localhost", + "9009"}; questdb::ingress::opts opts2{std::move(opts1)}; } { - questdb::ingress::opts opts1{"localhost", "9009"}; + questdb::ingress::opts opts1{ + questdb::ingress::protocol::tcps, + "localhost", + "9009"}; questdb::ingress::opts opts2{opts1}; } { - questdb::ingress::opts opts1{"localhost", "9009"}; - questdb::ingress::opts opts2{"altavista.digital.com", "9009"}; + questdb::ingress::opts opts1{questdb::ingress::protocol::tcp, + "localhost", + "9009"}; + questdb::ingress::opts opts2{ + questdb::ingress::protocol::tcp, + "altavista.digital.com", + "9009"}; opts1 = std::move(opts2); } { - questdb::ingress::opts opts1{"localhost", "9009"}; - questdb::ingress::opts opts2{"altavista.digital.com", "9009"}; + questdb::ingress::opts opts1{ + questdb::ingress::protocol::https, + "localhost", + "9009"}; + questdb::ingress::opts opts2{ + questdb::ingress::protocol::https, + "altavista.digital.com", + "9009"}; opts1 = opts2; } } @@ -585,7 +677,10 @@ TEST_CASE("Opts copy ctor, assignment and move testing.") TEST_CASE("Test timestamp column.") { questdb::ingress::test::mock_server server; - questdb::ingress::line_sender sender{"localhost", server.port()}; + questdb::ingress::line_sender sender{ + questdb::ingress::protocol::tcp, + "localhost", + server.port()}; const auto now = std::chrono::system_clock::now(); const auto now_micros = @@ -706,7 +801,7 @@ TEST_CASE("Empty Buffer") { CHECK(b5.size() == 9); questdb::ingress::test::mock_server server; - questdb::ingress::line_sender sender{"localhost", server.port()}; + questdb::ingress::line_sender sender{questdb::ingress::protocol::tcp, "localhost", server.port()}; CHECK_THROWS_WITH_AS( sender.flush(b1), "State error: Bad call to `flush`, should have called `table` instead.", @@ -715,4 +810,52 @@ TEST_CASE("Empty Buffer") { sender.flush_and_keep(b1), "State error: Bad call to `flush`, should have called `table` instead.", questdb::ingress::line_sender_error); +} + +TEST_CASE("Opts from conf") { + questdb::ingress::opts opts1 = questdb::ingress::opts::from_conf("tcp::addr=localhost:9009;"); + questdb::ingress::opts opts2 = questdb::ingress::opts::from_conf("tcps::addr=localhost:9009;"); + questdb::ingress::opts opts3 = questdb::ingress::opts::from_conf("https::addr=localhost:9009;"); + questdb::ingress::opts opts4 = questdb::ingress::opts::from_conf("https::addr=localhost:9009;"); +} + +TEST_CASE("HTTP basics") { + questdb::ingress::opts opts1{ + questdb::ingress::protocol::http, + "localhost", + 1}; + questdb::ingress::opts opts1conf = questdb::ingress::opts::from_conf( + "http::addr=localhost:1;username=user;password=pass;request_timeout=5000;retry_timeout=5;"); + questdb::ingress::opts opts2{ + questdb::ingress::protocol::https, + "localhost", + "1"}; + questdb::ingress::opts opts2conf = questdb::ingress::opts::from_conf( + "http::addr=localhost:1;token=token;request_min_throughput=1000;retry_timeout=0;"); + opts1 + .username("user") + .password("pass") + .max_buf_size(1000000) + .request_timeout(5000) + .retry_timeout(5); + opts2 + .token("token") + .request_min_throughput(1000) + .retry_timeout(0); + questdb::ingress::line_sender sender1{opts1}; + questdb::ingress::line_sender sender1conf{opts1conf}; + questdb::ingress::line_sender sender2{opts2}; + questdb::ingress::line_sender sender2conf{opts2conf}; + + questdb::ingress::line_sender_buffer b1; + b1.table("test").symbol("a", "b").at_now(); + + CHECK_THROWS_AS(sender1.flush(b1), questdb::ingress::line_sender_error); + CHECK_THROWS_AS(sender1conf.flush(b1), questdb::ingress::line_sender_error); + CHECK_THROWS_AS(sender2.flush(b1), questdb::ingress::line_sender_error); + CHECK_THROWS_AS(sender2conf.flush(b1), questdb::ingress::line_sender_error); + + CHECK_THROWS_AS( + questdb::ingress::opts::from_conf("http::addr=localhost:1;bind_interface=0.0.0.0;"), + questdb::ingress::line_sender_error); } \ No newline at end of file diff --git a/doc/C.md b/doc/C.md index 5673cb17..5968e0ee 100644 --- a/doc/C.md +++ b/doc/C.md @@ -27,27 +27,21 @@ ... -line_sender_error* err = NULL; -line_sender_opts* opts = NULL; -line_sender* sender = NULL; - -line_sender_utf8 host = QDB_UTF8_LITERAL("localhost"); -opts = line_sender_opts_new(host, 9009, &err); -if (!opts) { - /* ... handle error ... */ -} +line_sender_utf8 conf = QDB_UTF8_LITERAL( + "http::addr=localhost:9000;"); -sender = line_sender_connect(opts, &err); -line_sender_opts_free(opts); -opts = NULL; +line_sender_error* err = NULL; +line_sender* sender = sender = line_sender_from_conf(&err); if (!sender) { /* ... handle error ... */ } ``` -The `opts` object can additionally take parameters for the outbound interface, -authentication, full-connection encryption via TLS and more. +See the main [client libraries](https://questdb.io/docs/reference/clients/overview/) +documentation for the full config string params, including authentication, tls, etc. + +You can also connect programmatically using `line_sender_opts_new`. ### Building Messages diff --git a/doc/CONSIDERATIONS.md b/doc/CONSIDERATIONS.md index 2f8c9667..872d05ed 100644 --- a/doc/CONSIDERATIONS.md +++ b/doc/CONSIDERATIONS.md @@ -98,12 +98,22 @@ To determine the buffer size, call: ### Disconnections, Data Errors and troubleshooting -A failure when flushing data gerally indicates that the network connection was -dropped. +#### ILP/HTTP -The ILP protocol does not send errors back to the client. Instead, by design, -it will disconnect a client if it encounters any insertion errors. This is to -avoid errors going unnoticed. +When using ILP/HTTP, server errors are reported back to the client from +`flush` calls. + +#### ILP/TCP + +When using ILP/TCP, errors which cause disconnects can be found +in the QuestDB server logs. + +When using TCP, failure when flushing data gerally indicates that the network +connection was dropped. + +The ILP over TCP protocol does not send errors back to the client. +Instead, by design, it will disconnect a client if it encounters any insertion +errors. This is to avoid errors going unnoticed. As an example, if a client were to insert a `STRING` value into a `BOOLEAN` column, the QuestDB server would disconnect the client. diff --git a/doc/CPP.md b/doc/CPP.md index fdb544df..99a9d442 100644 --- a/doc/CPP.md +++ b/doc/CPP.md @@ -27,17 +27,15 @@ ... -// Automatically connects on object construction. -questdb::ingress::line_sender sender{ - "localhost", // QuestDB hostname - 9009}; // QuestDB port +auto sender = questdb::ingress::line_sender::from_conf( + "http::addr=localhost:9000;"); ``` -For more advanced use cases, such as those requiring authentication or -full-connection encryption via TLS, first, create an `opts` object then call its -methods to populate its options and then pass the `opts` object to the -`line_sender` constructor. +See the main [client libraries](https://questdb.io/docs/reference/clients/overview/) +documentation for the full config string params, including authentication, tls, etc. + +You can also connect programmatically using the `questdb::ingress::opts` object. ### Building Messages diff --git a/doc/SECURITY.md b/doc/SECURITY.md index 56ae4812..e2d4bd2e 100644 --- a/doc/SECURITY.md +++ b/doc/SECURITY.md @@ -35,7 +35,7 @@ A few important technical details on TLS: are managed centrally. For API usage: -* Rust: `SenderBuilder`'s [`auth`](https://docs.rs/questdb-rs/3.1.0/questdb/ingress/struct.SenderBuilder.html#method.auth) - and [`tls`](https://docs.rs/questdb-rs/3.1.0/questdb/ingress/struct.SenderBuilder.html#method.tls) methods. +* Rust: `SenderBuilder`'s [`auth`](https://docs.rs/questdb-rs/4.0.0/questdb/ingress/struct.SenderBuilder.html#method.auth) + and [`tls`](https://docs.rs/questdb-rs/4.0.0/questdb/ingress/struct.SenderBuilder.html#method.tls) methods. * C: [examples/line_sender_c_example_auth.c](../examples/line_sender_c_example_auth.c) * C++: [examples/line_sender_cpp_example_auth.cpp](../examples/line_sender_cpp_example_auth.cpp) diff --git a/examples.manifest.yaml b/examples.manifest.yaml index 2db64d42..0e5a0beb 100644 --- a/examples.manifest.yaml +++ b/examples.manifest.yaml @@ -102,3 +102,24 @@ addr: host: localhost port: 9009 + +- name: ilp-from-conf + lang: c + path: examples/line_sender_c_example_from_conf.c + header: |- + [C client library docs](https://github.com/questdb/c-questdb-client/blob/main/doc/C.md) + conf: tcp::addr=localhost:9009; + +- name: ilp-from-conf + lang: cpp + path: examples/line_sender_cpp_example_from_conf.cpp + header: |- + [C client library docs](https://github.com/questdb/c-questdb-client/blob/main/doc/CPP.md) + conf: tcp::addr=localhost:9009; + +- name: ilp-from-conf + lang: rust + path: questdb-rs/examples/from_conf.rs + header: |- + [Rust client library docs](https://docs.rs/crate/questdb-rs/latest) + conf: tcp::addr=localhost:9009; diff --git a/examples/concat.c b/examples/concat.c new file mode 100644 index 00000000..5ea5de8f --- /dev/null +++ b/examples/concat.c @@ -0,0 +1,19 @@ +#include "concat.h" + +/** Concatenate a list of nul-terminated strings. Pass an extra NULL arg to terminate. */ +char* concat_(const char* first, ...) { + va_list args; + size_t total_len = strlen(first) + 1; + va_start(args, first); + const char* str; + while((str = va_arg(args, char*)) != NULL) total_len += strlen(str); + va_end(args); + char* result = calloc(total_len, sizeof(char)); + if(!result) return NULL; + strcpy(result, first); + va_start(args, first); + while((str = va_arg(args, char*)) != NULL) strcat(result, str); + va_end(args); + return result; +} + diff --git a/examples/concat.h b/examples/concat.h new file mode 100644 index 00000000..af18d88d --- /dev/null +++ b/examples/concat.h @@ -0,0 +1,11 @@ +#pragma once + +#include +#include +#include +#include + +char* concat_(const char* first, ...); + +// A macro that passes the list of arguments to concat_ and adds a NULL terminator. +#define concat(...) concat_(__VA_ARGS__, NULL) diff --git a/examples/line_sender_c_example.c b/examples/line_sender_c_example.c index 265a563d..3e0b5786 100644 --- a/examples/line_sender_c_example.c +++ b/examples/line_sender_c_example.c @@ -2,29 +2,28 @@ #include #include #include +#include "concat.h" static bool example(const char* host, const char* port) { line_sender_error* err = NULL; - line_sender_opts* opts = NULL; line_sender* sender = NULL; line_sender_buffer* buffer = NULL; - - line_sender_utf8 host_utf8 = { 0, NULL }; - if (!line_sender_utf8_init(&host_utf8, strlen(host), host, &err)) - goto on_error; - - line_sender_utf8 port_utf8 = { 0, NULL }; - if (!line_sender_utf8_init(&port_utf8, strlen(port), port, &err)) + char* conf_str = concat("tcp::addr=", host, ":", port, ";"); + if (!conf_str) { + fprintf(stderr, "Could not concatenate configuration string.\n"); + return false; + } + line_sender_utf8 conf_str_utf8 = { 0, NULL }; + if (!line_sender_utf8_init(&conf_str_utf8, strlen(conf_str), conf_str, &err)) goto on_error; - // Call `line_sender_opts_new` if instead you have an integer port. - opts = line_sender_opts_new_service(host_utf8, port_utf8); - sender = line_sender_connect(opts, &err); - line_sender_opts_free(opts); - opts = NULL; + sender = line_sender_from_conf(conf_str_utf8, &err); if (!sender) - goto on_error; + goto on_error; + + free(conf_str); + conf_str = NULL; buffer = line_sender_buffer_new(); line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB buffer initial size. @@ -85,10 +84,10 @@ static bool example(const char* host, const char* port) return true; on_error: ; - line_sender_opts_free(opts); size_t err_len = 0; const char* err_msg = line_sender_error_msg(err, &err_len); fprintf(stderr, "Error running example: %.*s\n", (int)err_len, err_msg); + free(conf_str); line_sender_error_free(err); line_sender_buffer_free(buffer); line_sender_close(sender); diff --git a/examples/line_sender_c_example_auth.c b/examples/line_sender_c_example_auth.c index 4dc4a94b..bb7cf543 100644 --- a/examples/line_sender_c_example_auth.c +++ b/examples/line_sender_c_example_auth.c @@ -2,45 +2,33 @@ #include #include #include +#include "concat.h" static bool example(const char* host, const char* port) { line_sender_error* err = NULL; - line_sender_opts* opts = NULL; line_sender* sender = NULL; line_sender_buffer* buffer = NULL; - - line_sender_utf8 host_utf8 = { 0, NULL }; - if (!line_sender_utf8_init(&host_utf8, strlen(host), host, &err)) - goto on_error; - - line_sender_utf8 port_utf8 = { 0, NULL }; - if (!line_sender_utf8_init(&port_utf8, strlen(port), port, &err)) + char* conf_str = concat( + "tcp::addr=", host, ":", port, ";" + "username=testUser1;" + "token=5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48;" + "token_x=fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU;" + "token_y=Dt5tbS1dEDMSYfym3fgMv0B99szno-dFc1rYF9t0aac;"); + if (!conf_str) { + fprintf(stderr, "Could not concatenate configuration string.\n"); + return false; + } + line_sender_utf8 conf_str_utf8 = { 0, NULL }; + if (!line_sender_utf8_init(&conf_str_utf8, strlen(conf_str), conf_str, &err)) goto on_error; - // Call `line_sender_opts_new` if instead you have an integer port. - opts = line_sender_opts_new_service(host_utf8, port_utf8); - - // Use `QDB_UTF_8_FROM_STR_OR` to init from `const char*`. - line_sender_utf8 key_id = QDB_UTF8_LITERAL("testUser1"); - line_sender_utf8 priv_key = QDB_UTF8_LITERAL( - "5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48"); - line_sender_utf8 pub_key_x = QDB_UTF8_LITERAL( - "fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU"); - line_sender_utf8 pub_key_y = QDB_UTF8_LITERAL( - "Dt5tbS1dEDMSYfym3fgMv0B99szno-dFc1rYF9t0aac"); - - line_sender_opts_auth( - opts, - key_id, // kid - priv_key, // d - pub_key_x, // x - pub_key_y); // y - sender = line_sender_connect(opts, &err); - line_sender_opts_free(opts); - opts = NULL; + sender = line_sender_from_conf(conf_str_utf8, &err); if (!sender) - goto on_error; + goto on_error; + + free(conf_str); + conf_str = NULL; buffer = line_sender_buffer_new(); line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB buffer initial size. @@ -101,10 +89,10 @@ static bool example(const char* host, const char* port) return true; on_error: ; - line_sender_opts_free(opts); size_t err_len = 0; const char* err_msg = line_sender_error_msg(err, &err_len); fprintf(stderr, "Error running example: %.*s\n", (int)err_len, err_msg); + free(conf_str); line_sender_error_free(err); line_sender_buffer_free(buffer); line_sender_close(sender); diff --git a/examples/line_sender_c_example_auth_tls.c b/examples/line_sender_c_example_auth_tls.c index dd12b89c..f727e5c5 100644 --- a/examples/line_sender_c_example_auth_tls.c +++ b/examples/line_sender_c_example_auth_tls.c @@ -2,51 +2,33 @@ #include #include #include +#include "concat.h" static bool example(const char* host, const char* port) { line_sender_error* err = NULL; - line_sender_opts* opts = NULL; line_sender* sender = NULL; line_sender_buffer* buffer = NULL; - - line_sender_utf8 host_utf8 = { 0, NULL }; - if (!line_sender_utf8_init(&host_utf8, strlen(host), host, &err)) - goto on_error; - - line_sender_utf8 port_utf8 = { 0, NULL }; - if (!line_sender_utf8_init(&port_utf8, strlen(port), port, &err)) + char* conf_str = concat( + "tcps::addr=", host, ":", port, ";" + "username=testUser1;" + "token=5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48;" + "token_x=fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU;" + "token_y=Dt5tbS1dEDMSYfym3fgMv0B99szno-dFc1rYF9t0aac;"); + if (!conf_str) { + fprintf(stderr, "Could not concatenate configuration string.\n"); + return false; + } + line_sender_utf8 conf_str_utf8 = { 0, NULL }; + if (!line_sender_utf8_init(&conf_str_utf8, strlen(conf_str), conf_str, &err)) goto on_error; - // Call `line_sender_opts_new` if instead you have an integer port. - opts = line_sender_opts_new_service(host_utf8, port_utf8); - - // Enable TLS to accept connections using common trusted CAs. - line_sender_opts_tls(opts); - - //// Alternatively, to use the OS-provided root certificates: - // line_sender_opts_tls_os_roots(opts); - - // Use `QDB_UTF_8_FROM_STR_OR` to init from `const char*`. - line_sender_utf8 key_id = QDB_UTF8_LITERAL("testUser1"); - line_sender_utf8 priv_key = QDB_UTF8_LITERAL( - "5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48"); - line_sender_utf8 pub_key_x = QDB_UTF8_LITERAL( - "fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU"); - line_sender_utf8 pub_key_y = QDB_UTF8_LITERAL( - "Dt5tbS1dEDMSYfym3fgMv0B99szno-dFc1rYF9t0aac"); - - line_sender_opts_auth( - opts, - key_id, // kid - priv_key, // d - pub_key_x, // x - pub_key_y); // y - sender = line_sender_connect(opts, &err); - line_sender_opts_free(opts); - opts = NULL; + sender = line_sender_from_conf(conf_str_utf8, &err); if (!sender) - goto on_error; + goto on_error; + + free(conf_str); + conf_str = NULL; buffer = line_sender_buffer_new(); line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB buffer initial size. @@ -54,7 +36,7 @@ static bool example(const char* host, const char* port) // We prepare all our table names and column names in advance. // If we're inserting multiple rows, this allows us to avoid // re-validating the same strings over and over again. - line_sender_table_name table_name = QDB_TABLE_NAME_LITERAL("c_cars_tls"); + line_sender_table_name table_name = QDB_TABLE_NAME_LITERAL("c_cars_auth_tls"); line_sender_column_name id_name = QDB_COLUMN_NAME_LITERAL("id"); line_sender_column_name x_name = QDB_COLUMN_NAME_LITERAL("x"); line_sender_column_name y_name = QDB_COLUMN_NAME_LITERAL("y"); @@ -107,10 +89,10 @@ static bool example(const char* host, const char* port) return true; on_error: ; - line_sender_opts_free(opts); size_t err_len = 0; const char* err_msg = line_sender_error_msg(err, &err_len); fprintf(stderr, "Error running example: %.*s\n", (int)err_len, err_msg); + free(conf_str); line_sender_error_free(err); line_sender_buffer_free(buffer); line_sender_close(sender); diff --git a/examples/line_sender_c_example_from_conf.c b/examples/line_sender_c_example_from_conf.c new file mode 100644 index 00000000..808ab6b8 --- /dev/null +++ b/examples/line_sender_c_example_from_conf.c @@ -0,0 +1,83 @@ +#include +#include +#include +#include + +int main(int argc, const char* argv[]) +{ + line_sender_error* err = NULL; + line_sender_buffer* buffer = NULL; + + line_sender_utf8 conf = QDB_UTF8_LITERAL( + "tcp::addr=localhost:9009;"); + line_sender* sender = line_sender_from_conf(conf, &err); + if (!sender) + goto on_error; + + buffer = line_sender_buffer_new(); + line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB buffer initial size. + + // We prepare all our table names and column names in advance. + // If we're inserting multiple rows, this allows us to avoid + // re-validating the same strings over and over again. + line_sender_table_name table_name = QDB_TABLE_NAME_LITERAL("c_cars_from_conf"); + line_sender_column_name id_name = QDB_COLUMN_NAME_LITERAL("id"); + line_sender_column_name x_name = QDB_COLUMN_NAME_LITERAL("x"); + line_sender_column_name y_name = QDB_COLUMN_NAME_LITERAL("y"); + line_sender_column_name booked_name = QDB_COLUMN_NAME_LITERAL("booked"); + line_sender_column_name passengers_name = QDB_COLUMN_NAME_LITERAL( + "passengers"); + line_sender_column_name driver_name = QDB_COLUMN_NAME_LITERAL("driver"); + + if (!line_sender_buffer_table(buffer, table_name, &err)) + goto on_error; + + line_sender_utf8 id_value = QDB_UTF8_LITERAL( + "d6e5fe92-d19f-482a-a97a-c105f547f721"); + if (!line_sender_buffer_symbol(buffer, id_name, id_value, &err)) + goto on_error; + + if (!line_sender_buffer_column_f64(buffer, x_name, 30.5, &err)) + goto on_error; + + if (!line_sender_buffer_column_f64(buffer, y_name, -150.25, &err)) + goto on_error; + + if (!line_sender_buffer_column_bool(buffer, booked_name, true, &err)) + goto on_error; + + if (!line_sender_buffer_column_i64(buffer, passengers_name, 3, &err)) + goto on_error; + + line_sender_utf8 driver_value = QDB_UTF8_LITERAL("John Doe"); + if (!line_sender_buffer_column_str(buffer, driver_name, driver_value, &err)) + goto on_error; + + // 1997-07-04 04:56:55 UTC + int64_t designated_timestamp = 867992215000000000; + if (!line_sender_buffer_at_nanos(buffer, designated_timestamp, &err)) + goto on_error; + + //// If you want to get the current system timestamp as nanos, call: + // if (!line_sender_buffer_at_nanos(buffer, line_sender_now_nanos(), &err)) + // goto on_error; + + // To insert more records, call `line_sender_buffer_table(..)...` again. + // It's recommended to keep a timer and/or maximum buffer size to flush + // the buffer periodically with any accumulated records. + if (!line_sender_flush(sender, buffer, &err)) + goto on_error; + + line_sender_close(sender); + + return 0; + +on_error: ; + size_t err_len = 0; + const char* err_msg = line_sender_error_msg(err, &err_len); + fprintf(stderr, "Error running example: %.*s\n", (int)err_len, err_msg); + line_sender_error_free(err); + line_sender_buffer_free(buffer); + line_sender_close(sender); + return 1; +} diff --git a/examples/line_sender_c_example_from_env.c b/examples/line_sender_c_example_from_env.c new file mode 100644 index 00000000..5c636b00 --- /dev/null +++ b/examples/line_sender_c_example_from_env.c @@ -0,0 +1,82 @@ +#include +#include +#include +#include + +int main(int argc, const char* argv[]) +{ + line_sender_error* err = NULL; + line_sender_buffer* buffer = NULL; + + // Construct a sender from the `QDB_CLIENT_CONF` environment variable. + line_sender* sender = line_sender_from_env(&err); + if (!sender) + goto on_error; + + buffer = line_sender_buffer_new(); + line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB buffer initial size. + + // We prepare all our table names and column names in advance. + // If we're inserting multiple rows, this allows us to avoid + // re-validating the same strings over and over again. + line_sender_table_name table_name = QDB_TABLE_NAME_LITERAL("c_cars_from_env"); + line_sender_column_name id_name = QDB_COLUMN_NAME_LITERAL("id"); + line_sender_column_name x_name = QDB_COLUMN_NAME_LITERAL("x"); + line_sender_column_name y_name = QDB_COLUMN_NAME_LITERAL("y"); + line_sender_column_name booked_name = QDB_COLUMN_NAME_LITERAL("booked"); + line_sender_column_name passengers_name = QDB_COLUMN_NAME_LITERAL( + "passengers"); + line_sender_column_name driver_name = QDB_COLUMN_NAME_LITERAL("driver"); + + if (!line_sender_buffer_table(buffer, table_name, &err)) + goto on_error; + + line_sender_utf8 id_value = QDB_UTF8_LITERAL( + "d6e5fe92-d19f-482a-a97a-c105f547f721"); + if (!line_sender_buffer_symbol(buffer, id_name, id_value, &err)) + goto on_error; + + if (!line_sender_buffer_column_f64(buffer, x_name, 30.5, &err)) + goto on_error; + + if (!line_sender_buffer_column_f64(buffer, y_name, -150.25, &err)) + goto on_error; + + if (!line_sender_buffer_column_bool(buffer, booked_name, true, &err)) + goto on_error; + + if (!line_sender_buffer_column_i64(buffer, passengers_name, 3, &err)) + goto on_error; + + line_sender_utf8 driver_value = QDB_UTF8_LITERAL("John Doe"); + if (!line_sender_buffer_column_str(buffer, driver_name, driver_value, &err)) + goto on_error; + + // 1997-07-04 04:56:55 UTC + int64_t designated_timestamp = 867992215000000000; + if (!line_sender_buffer_at_nanos(buffer, designated_timestamp, &err)) + goto on_error; + + //// If you want to get the current system timestamp as nanos, call: + // if (!line_sender_buffer_at_nanos(buffer, line_sender_now_nanos(), &err)) + // goto on_error; + + // To insert more records, call `line_sender_buffer_table(..)...` again. + // It's recommended to keep a timer and/or maximum buffer size to flush + // the buffer periodically with any accumulated records. + if (!line_sender_flush(sender, buffer, &err)) + goto on_error; + + line_sender_close(sender); + + return 0; + +on_error: ; + size_t err_len = 0; + const char* err_msg = line_sender_error_msg(err, &err_len); + fprintf(stderr, "Error running example: %.*s\n", (int)err_len, err_msg); + line_sender_error_free(err); + line_sender_buffer_free(buffer); + line_sender_close(sender); + return 1; +} diff --git a/examples/line_sender_c_example_http.c b/examples/line_sender_c_example_http.c new file mode 100644 index 00000000..28134b2f --- /dev/null +++ b/examples/line_sender_c_example_http.c @@ -0,0 +1,129 @@ +#include +#include +#include +#include +#include "concat.h" + +static bool example(const char* host, const char* port) +{ + line_sender_error* err = NULL; + line_sender* sender = NULL; + line_sender_buffer* buffer = NULL; + // Use `https` to enable TLS. + // Use `username=...;password=...;` or `token=...` for authentication. + char* conf_str = concat("http::addr=", host, ":", port, ";"); + if (!conf_str) { + fprintf(stderr, "Could not concatenate configuration string.\n"); + return false; + } + line_sender_utf8 conf_str_utf8 = { 0, NULL }; + if (!line_sender_utf8_init(&conf_str_utf8, strlen(conf_str), conf_str, &err)) + goto on_error; + + sender = line_sender_from_conf(conf_str_utf8, &err); + if (!sender) + goto on_error; + + free(conf_str); + conf_str = NULL; + + buffer = line_sender_buffer_new(); + line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB buffer initial size. + + // We prepare all our table names and column names in advance. + // If we're inserting multiple rows, this allows us to avoid + // re-validating the same strings over and over again. + line_sender_table_name table_name = QDB_TABLE_NAME_LITERAL("c_cars_http"); + line_sender_column_name id_name = QDB_COLUMN_NAME_LITERAL("id"); + line_sender_column_name x_name = QDB_COLUMN_NAME_LITERAL("x"); + line_sender_column_name y_name = QDB_COLUMN_NAME_LITERAL("y"); + line_sender_column_name booked_name = QDB_COLUMN_NAME_LITERAL("booked"); + line_sender_column_name passengers_name = QDB_COLUMN_NAME_LITERAL( + "passengers"); + line_sender_column_name driver_name = QDB_COLUMN_NAME_LITERAL("driver"); + + if (!line_sender_buffer_table(buffer, table_name, &err)) + goto on_error; + + line_sender_utf8 id_value = QDB_UTF8_LITERAL( + "d6e5fe92-d19f-482a-a97a-c105f547f721"); + if (!line_sender_buffer_symbol(buffer, id_name, id_value, &err)) + goto on_error; + + if (!line_sender_buffer_column_f64(buffer, x_name, 30.5, &err)) + goto on_error; + + if (!line_sender_buffer_column_f64(buffer, y_name, -150.25, &err)) + goto on_error; + + if (!line_sender_buffer_column_bool(buffer, booked_name, true, &err)) + goto on_error; + + if (!line_sender_buffer_column_i64(buffer, passengers_name, 3, &err)) + goto on_error; + + line_sender_utf8 driver_value = QDB_UTF8_LITERAL("John Doe"); + if (!line_sender_buffer_column_str(buffer, driver_name, driver_value, &err)) + goto on_error; + + // 1997-07-04 04:56:55 UTC + int64_t designated_timestamp = 867992215000000000; + if (!line_sender_buffer_at_nanos(buffer, designated_timestamp, &err)) + goto on_error; + + //// If you want to get the current system timestamp as nanos, call: + // if (!line_sender_buffer_at_nanos(buffer, line_sender_now_nanos(), &err)) + // goto on_error; + + // To insert more records, call `line_sender_buffer_table(..)...` again. + // It's recommended to keep a timer and/or maximum buffer size to flush + // the buffer periodically with any accumulated records. + if (!line_sender_flush(sender, buffer, &err)) + goto on_error; + + line_sender_close(sender); + + return true; + +on_error: ; + size_t err_len = 0; + const char* err_msg = line_sender_error_msg(err, &err_len); + fprintf(stderr, "Error running example: %.*s\n", (int)err_len, err_msg); + free(conf_str); + line_sender_error_free(err); + line_sender_buffer_free(buffer); + line_sender_close(sender); + return false; +} + +static bool displayed_help(int argc, const char* argv[]) +{ + for (int index = 1; index < argc; ++index) + { + const char* arg = argv[index]; + if ((strncmp(arg, "-h", 2) == 0) || (strncmp(arg, "--help", 6) == 0)) + { + fprintf(stderr, "Usage:\n"); + fprintf(stderr, "line_sender_c_example: [HOST [PORT]]\n"); + fprintf(stderr, " HOST: ILP host (defaults to \"localhost\").\n"); + fprintf(stderr, " PORT: HTTP port (defaults to \"9000\").\n"); + return true; + } + } + return false; +} + +int main(int argc, const char* argv[]) +{ + if (displayed_help(argc, argv)) + return 0; + + const char* host = "localhost"; + if (argc >= 2) + host = argv[1]; + const char* port = "9000"; + if (argc >= 3) + port = argv[2]; + + return !example(host, port); +} diff --git a/examples/line_sender_c_example_tls_ca.c b/examples/line_sender_c_example_tls_ca.c index e3e6b5b0..4636d396 100644 --- a/examples/line_sender_c_example_tls_ca.c +++ b/examples/line_sender_c_example_tls_ca.c @@ -2,53 +2,34 @@ #include #include #include +#include "concat.h" static bool example(const char* ca_path, const char* host, const char* port) { line_sender_error* err = NULL; - line_sender_opts* opts = NULL; line_sender* sender = NULL; line_sender_buffer* buffer = NULL; - - line_sender_utf8 host_utf8 = { 0, NULL }; - if (!line_sender_utf8_init(&host_utf8, strlen(host), host, &err)) - goto on_error; - - line_sender_utf8 port_utf8 = { 0, NULL }; - if (!line_sender_utf8_init(&port_utf8, strlen(port), port, &err)) + char* conf_str = concat( + "tcps::addr=", host, ":", port, ";", + "tls_roots=", ca_path, ";", + "username=testUser1;" + "token=5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48;" + "token_x=fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU;" + "token_y=Dt5tbS1dEDMSYfym3fgMv0B99szno-dFc1rYF9t0aac;"); + if (!conf_str) { + fprintf(stderr, "Could not concatenate configuration string.\n"); + return false; + } + line_sender_utf8 conf_str_utf8 = { 0, NULL }; + if (!line_sender_utf8_init(&conf_str_utf8, strlen(conf_str), conf_str, &err)) goto on_error; - // Call `line_sender_opts_new` if instead you have an integer port. - opts = line_sender_opts_new_service(host_utf8, port_utf8); - - // This example uses a custom certificate authority file. - // You can use the default certificate authority by instead calling - // `line_sender_opts_tls` which takes no arguments. - line_sender_utf8 ca_path_utf8 = { 0, NULL }; - if (!line_sender_utf8_init(&ca_path_utf8, strlen(ca_path), ca_path, &err)) - goto on_error; - line_sender_opts_tls_ca(opts, ca_path_utf8); - - // Use `QDB_UTF_8_FROM_STR_OR` to init from `const char*`. - line_sender_utf8 key_id = QDB_UTF8_LITERAL("testUser1"); - line_sender_utf8 priv_key = QDB_UTF8_LITERAL( - "5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48"); - line_sender_utf8 pub_key_x = QDB_UTF8_LITERAL( - "fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU"); - line_sender_utf8 pub_key_y = QDB_UTF8_LITERAL( - "Dt5tbS1dEDMSYfym3fgMv0B99szno-dFc1rYF9t0aac"); - - line_sender_opts_auth( - opts, - key_id, // kid - priv_key, // d - pub_key_x, // x - pub_key_y); // y - sender = line_sender_connect(opts, &err); - line_sender_opts_free(opts); - opts = NULL; + sender = line_sender_from_conf(conf_str_utf8, &err); if (!sender) - goto on_error; + goto on_error; + + free(conf_str); + conf_str = NULL; buffer = line_sender_buffer_new(); line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB buffer initial size. @@ -56,7 +37,7 @@ static bool example(const char* ca_path, const char* host, const char* port) // We prepare all our table names and column names in advance. // If we're inserting multiple rows, this allows us to avoid // re-validating the same strings over and over again. - line_sender_table_name table_name = QDB_TABLE_NAME_LITERAL("c_cars_tls"); + line_sender_table_name table_name = QDB_TABLE_NAME_LITERAL("c_cars_tls_ca"); line_sender_column_name id_name = QDB_COLUMN_NAME_LITERAL("id"); line_sender_column_name x_name = QDB_COLUMN_NAME_LITERAL("x"); line_sender_column_name y_name = QDB_COLUMN_NAME_LITERAL("y"); @@ -109,10 +90,10 @@ static bool example(const char* ca_path, const char* host, const char* port) return true; on_error: ; - line_sender_opts_free(opts); size_t err_len = 0; const char* err_msg = line_sender_error_msg(err, &err_len); fprintf(stderr, "Error running example: %.*s\n", (int)err_len, err_msg); + free(conf_str); line_sender_error_free(err); line_sender_buffer_free(buffer); line_sender_close(sender); diff --git a/examples/line_sender_cpp_example.cpp b/examples/line_sender_cpp_example.cpp index 9b77112c..7e7dddca 100644 --- a/examples/line_sender_cpp_example.cpp +++ b/examples/line_sender_cpp_example.cpp @@ -8,8 +8,8 @@ static bool example(std::string_view host, std::string_view port) { try { - // Connect. - questdb::ingress::line_sender sender{host, port}; + auto sender = questdb::ingress::line_sender::from_conf( + "tcp::addr=" + std::string{host} + ":" + std::string{port} + ";"); // We prepare all our table names and column names in advance. // If we're inserting multiple rows, this allows us to avoid @@ -23,8 +23,6 @@ static bool example(std::string_view host, std::string_view port) const auto driver_name = "driver"_cn; questdb::ingress::line_sender_buffer buffer; - // 1997-07-04 04:56:55 UTC - questdb::ingress::timestamp_nanos designated_timestamp{867992215000000000}; buffer .table(table_name) .symbol(id_name, "d6e5fe92-d19f-482a-a97a-c105f547f721"_utf8) @@ -33,9 +31,7 @@ static bool example(std::string_view host, std::string_view port) .column(booked_name, true) .column(passengers_name, int64_t{3}) .column(driver_name, "John Doe"_utf8) - .at(designated_timestamp); - - // Call `.at(timestamp_nanos::now())` to use the current system time. + .at(questdb::ingress::timestamp_nanos::now()); // To insert more records, call `buffer.table(..)...` again. diff --git a/examples/line_sender_cpp_example_auth.cpp b/examples/line_sender_cpp_example_auth.cpp index bd33e161..f25c2534 100644 --- a/examples/line_sender_cpp_example_auth.cpp +++ b/examples/line_sender_cpp_example_auth.cpp @@ -8,17 +8,12 @@ static bool example(std::string_view host, std::string_view port) { try { - // Follow our authentication documentation to generate your own keys: - // https://questdb.io/docs/reference/api/ilp/authenticate - questdb::ingress::opts opts{host, port}; - opts.auth( - "testUser1", // key_id - "5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48", // priv_key - "fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU", // pub_key_x - "Dt5tbS1dEDMSYfym3fgMv0B99szno-dFc1rYF9t0aac"); // pub_key_y - - // Connect. - questdb::ingress::line_sender sender{opts}; + auto sender = questdb::ingress::line_sender::from_conf( + "tcp::addr=" + std::string{host} + ":" + std::string{port} + ";" + "username=testUser1;" + "token=5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48;" + "token_x=fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU;" + "token_y=Dt5tbS1dEDMSYfym3fgMv0B99szno-dFc1rYF9t0aac;"); // We prepare all our table names and column names in advance. // If we're inserting multiple rows, this allows us to avoid @@ -32,8 +27,6 @@ static bool example(std::string_view host, std::string_view port) const auto driver_name = "driver"_cn; questdb::ingress::line_sender_buffer buffer; - // 1997-07-04 04:56:55 UTC - questdb::ingress::timestamp_nanos designated_timestamp{867992215000000000}; buffer .table(table_name) .symbol(id_name, "d6e5fe92-d19f-482a-a97a-c105f547f721"_utf8) @@ -42,9 +35,7 @@ static bool example(std::string_view host, std::string_view port) .column(booked_name, true) .column(passengers_name, int64_t{3}) .column(driver_name, "John Doe"_utf8) - .at(designated_timestamp); - - // Call `.at(timestamp_nanos::now())` to use the current system time. + .at(questdb::ingress::timestamp_nanos::now()); // To insert more records, call `buffer.table(..)...` again. diff --git a/examples/line_sender_cpp_example_auth_tls.cpp b/examples/line_sender_cpp_example_auth_tls.cpp index 4185b512..7299c718 100644 --- a/examples/line_sender_cpp_example_auth_tls.cpp +++ b/examples/line_sender_cpp_example_auth_tls.cpp @@ -10,29 +10,17 @@ static bool example( { try { - questdb::ingress::opts opts{host, port}; - - // Enable TLS to accept connections using common trusted CAs. - opts.tls(); - - //// Alternatively, to use the OS-provided root certificates: - // opts.tls_os_roots(opts); - - // Follow our authentication documentation to generate your own keys: - // https://questdb.io/docs/reference/api/ilp/authenticate - opts.auth( - "testUser1", // key_id - "5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48", // priv_key - "fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU", // pub_key_x - "Dt5tbS1dEDMSYfym3fgMv0B99szno-dFc1rYF9t0aac"); // pub_key_y - - // Connect. - questdb::ingress::line_sender sender{opts}; + auto sender = questdb::ingress::line_sender::from_conf( + "tcps::addr=" + std::string{host} + ":" + std::string{port} + ";" + "username=testUser1;" + "token=5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48;" + "token_x=fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU;" + "token_y=Dt5tbS1dEDMSYfym3fgMv0B99szno-dFc1rYF9t0aac;"); // We prepare all our table names and column names in advance. // If we're inserting multiple rows, this allows us to avoid // re-validating the same strings over and over again. - const auto table_name = "cpp_cars_tls"_tn; + const auto table_name = "cpp_cars_auth_tls"_tn; const auto id_name = "id"_cn; const auto x_name = "x"_cn; const auto y_name = "y"_cn; @@ -41,8 +29,6 @@ static bool example( const auto driver_name = "driver"_cn; questdb::ingress::line_sender_buffer buffer; - // 1997-07-04 04:56:55 UTC - questdb::ingress::timestamp_nanos designated_timestamp{867992215000000000}; buffer .table(table_name) .symbol(id_name, "d6e5fe92-d19f-482a-a97a-c105f547f721"_utf8) @@ -51,9 +37,7 @@ static bool example( .column(booked_name, true) .column(passengers_name, int64_t{3}) .column(driver_name, "John Doe"_utf8) - .at(designated_timestamp); - - // Call `.at(timestamp_nanos::now())` to use the current system time. + .at(questdb::ingress::timestamp_nanos::now()); // To insert more records, call `buffer.table(..)...` again. diff --git a/examples/line_sender_cpp_example_from_conf.cpp b/examples/line_sender_cpp_example_from_conf.cpp new file mode 100644 index 00000000..1d3bb29a --- /dev/null +++ b/examples/line_sender_cpp_example_from_conf.cpp @@ -0,0 +1,54 @@ +#include +#include + +using namespace std::literals::string_view_literals; +using namespace questdb::ingress::literals; + +int main(int argc, const char* argv[]) +{ + try + { + auto sender = questdb::ingress::line_sender::from_conf( + "tcp::addr=localhost:9009;"); + + // We prepare all our table names and column names in advance. + // If we're inserting multiple rows, this allows us to avoid + // re-validating the same strings over and over again. + const auto table_name = "cpp_cars_from_conf"_tn; + const auto id_name = "id"_cn; + const auto x_name = "x"_cn; + const auto y_name = "y"_cn; + const auto booked_name = "booked"_cn; + const auto passengers_name = "passengers"_cn; + const auto driver_name = "driver"_cn; + + questdb::ingress::line_sender_buffer buffer; + buffer + .table(table_name) + .symbol(id_name, "d6e5fe92-d19f-482a-a97a-c105f547f721"_utf8) + .column(x_name, 30.5) + .column(y_name, -150.25) + .column(booked_name, true) + .column(passengers_name, int64_t{3}) + .column(driver_name, "John Doe"_utf8) + .at(questdb::ingress::timestamp_nanos::now()); + + // To insert more records, call `buffer.table(..)...` again. + + sender.flush(buffer); + + // It's recommended to keep a timer and/or maximum buffer size to flush + // the buffer periodically with any accumulated records. + + return 0; + } + catch (const questdb::ingress::line_sender_error& err) + { + std::cerr + << "Error running example: " + << err.what() + << std::endl; + + return 1; + } +} diff --git a/examples/line_sender_cpp_example_from_env.cpp b/examples/line_sender_cpp_example_from_env.cpp new file mode 100644 index 00000000..7526fb88 --- /dev/null +++ b/examples/line_sender_cpp_example_from_env.cpp @@ -0,0 +1,53 @@ +#include +#include + +using namespace std::literals::string_view_literals; +using namespace questdb::ingress::literals; + +int main(int argc, const char* argv[]) +{ + try + { + auto sender = questdb::ingress::line_sender::from_env(); + + // We prepare all our table names and column names in advance. + // If we're inserting multiple rows, this allows us to avoid + // re-validating the same strings over and over again. + const auto table_name = "cpp_cars_from_env"_tn; + const auto id_name = "id"_cn; + const auto x_name = "x"_cn; + const auto y_name = "y"_cn; + const auto booked_name = "booked"_cn; + const auto passengers_name = "passengers"_cn; + const auto driver_name = "driver"_cn; + + questdb::ingress::line_sender_buffer buffer; + buffer + .table(table_name) + .symbol(id_name, "d6e5fe92-d19f-482a-a97a-c105f547f721"_utf8) + .column(x_name, 30.5) + .column(y_name, -150.25) + .column(booked_name, true) + .column(passengers_name, int64_t{3}) + .column(driver_name, "John Doe"_utf8) + .at(questdb::ingress::timestamp_nanos::now()); + + // To insert more records, call `buffer.table(..)...` again. + + sender.flush(buffer); + + // It's recommended to keep a timer and/or maximum buffer size to flush + // the buffer periodically with any accumulated records. + + return 0; + } + catch (const questdb::ingress::line_sender_error& err) + { + std::cerr + << "Error running example: " + << err.what() + << std::endl; + + return 1; + } +} diff --git a/examples/line_sender_cpp_example_http.cpp b/examples/line_sender_cpp_example_http.cpp new file mode 100644 index 00000000..979657f7 --- /dev/null +++ b/examples/line_sender_cpp_example_http.cpp @@ -0,0 +1,88 @@ +#include +#include + +using namespace std::literals::string_view_literals; +using namespace questdb::ingress::literals; + +static bool example(std::string_view host, std::string_view port) +{ + try + { + auto sender = questdb::ingress::line_sender::from_conf( + "http::addr=" + std::string{host} + ":" + std::string{port} + ";"); + + // We prepare all our table names and column names in advance. + // If we're inserting multiple rows, this allows us to avoid + // re-validating the same strings over and over again. + const auto table_name = "cpp_cars_http"_tn; + const auto id_name = "id"_cn; + const auto x_name = "x"_cn; + const auto y_name = "y"_cn; + const auto booked_name = "booked"_cn; + const auto passengers_name = "passengers"_cn; + const auto driver_name = "driver"_cn; + + questdb::ingress::line_sender_buffer buffer; + buffer + .table(table_name) + .symbol(id_name, "d6e5fe92-d19f-482a-a97a-c105f547f721"_utf8) + .column(x_name, 30.5) + .column(y_name, -150.25) + .column(booked_name, true) + .column(passengers_name, int64_t{3}) + .column(driver_name, "John Doe"_utf8) + .at(questdb::ingress::timestamp_nanos::now()); + + // To insert more records, call `buffer.table(..)...` again. + + sender.flush(buffer); + + // It's recommended to keep a timer and/or maximum buffer size to flush + // the buffer periodically with any accumulated records. + + return true; + } + catch (const questdb::ingress::line_sender_error& err) + { + std::cerr + << "Error running example: " + << err.what() + << std::endl; + + return false; + } +} + +static bool displayed_help(int argc, const char* argv[]) +{ + for (int index = 1; index < argc; ++index) + { + const std::string_view arg{argv[index]}; + if ((arg == "-h"sv) || (arg == "--help"sv)) + { + std::cerr + << "Usage:\n" + << "line_sender_c_example: [HOST [PORT]]\n" + << " HOST: ILP host (defaults to \"localhost\").\n" + << " PORT: ILP port (defaults to \"9009\")." + << std::endl; + return true; + } + } + return false; +} + +int main(int argc, const char* argv[]) +{ + if (displayed_help(argc, argv)) + return 0; + + auto host = "localhost"sv; + if (argc >= 2) + host = std::string_view{argv[1]}; + auto port = "9009"sv; + if (argc >= 3) + port = std::string_view{argv[2]}; + + return !example(host, port); +} diff --git a/examples/line_sender_cpp_example_tls_ca.cpp b/examples/line_sender_cpp_example_tls_ca.cpp index 42e297f1..64231f0d 100644 --- a/examples/line_sender_cpp_example_tls_ca.cpp +++ b/examples/line_sender_cpp_example_tls_ca.cpp @@ -11,28 +11,18 @@ static bool example( { try { - questdb::ingress::opts opts{host, port}; - - // This example uses a custom certificate authority file. - // You can use the default certificate authority by calling the `.tls()` - // overload that takes no arguments. - opts.tls(ca_path); - - // Follow our authentication documentation to generate your own keys: - // https://questdb.io/docs/reference/api/ilp/authenticate - opts.auth( - "testUser1", // key_id - "5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48", // priv_key - "fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU", // pub_key_x - "Dt5tbS1dEDMSYfym3fgMv0B99szno-dFc1rYF9t0aac"); // pub_key_y - - // Connect. - questdb::ingress::line_sender sender{opts}; + auto sender = questdb::ingress::line_sender::from_conf( + "tcps::addr=" + std::string{host} + ":" + std::string{port} + ";" + "username=testUser1;" + "token=5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48;" + "token_x=fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU;" + "token_y=Dt5tbS1dEDMSYfym3fgMv0B99szno-dFc1rYF9t0aac;" + "tls_roots=" + std::string{ca_path} + ";"); // path to custom `.pem` file. // We prepare all our table names and column names in advance. // If we're inserting multiple rows, this allows us to avoid // re-validating the same strings over and over again. - const auto table_name = "cpp_cars_tls"_tn; + const auto table_name = "cpp_cars_tls_ca"_tn; const auto id_name = "id"_cn; const auto x_name = "x"_cn; const auto y_name = "y"_cn; @@ -41,8 +31,6 @@ static bool example( const auto driver_name = "driver"_cn; questdb::ingress::line_sender_buffer buffer; - // 1997-07-04 04:56:55 UTC - questdb::ingress::timestamp_nanos designated_timestamp{867992215000000000}; buffer .table(table_name) .symbol(id_name, "d6e5fe92-d19f-482a-a97a-c105f547f721"_utf8) @@ -51,9 +39,7 @@ static bool example( .column(booked_name, true) .column(passengers_name, int64_t{3}) .column(driver_name, "John Doe"_utf8) - .at(designated_timestamp); - - // Call `.at(timestamp_nanos::now())` to use the current system time. + .at(questdb::ingress::timestamp_nanos::now()); // To insert more records, call `buffer.table(..)...` again. diff --git a/include/questdb/ingress/line_sender.h b/include/questdb/ingress/line_sender.h index 2a4e014b..a51b06d9 100644 --- a/include/questdb/ingress/line_sender.h +++ b/include/questdb/ingress/line_sender.h @@ -6,7 +6,7 @@ * \__\_\\__,_|\___||___/\__|____/|____/ * * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2023 QuestDB + * Copyright (c) 2019-2024 QuestDB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -69,8 +69,48 @@ typedef enum line_sender_error_code /** Error during TLS handshake. */ line_sender_error_tls_error, + + /** The server does not support ILP over HTTP. */ + line_sender_error_http_not_supported, + + /** Error sent back from the server during flush. */ + line_sender_error_server_flush_error, + + /** Bad configuration. */ + line_sender_error_config_error, } line_sender_error_code; +/** The protocol used to connect with. */ +typedef enum line_sender_protocol +{ + /** InfluxDB Line Protocol over TCP. */ + line_sender_protocol_tcp, + + /** InfluxDB Line Protocol over TCP with TLS. */ + line_sender_protocol_tcps, + + /** InfluxDB Line Protocol over HTTP. */ + line_sender_protocol_http, + + /** InfluxDB Line Protocol over HTTP with TLS. */ + line_sender_protocol_https, +} line_sender_protocol; + +/* Possible sources of the root certificates used to validate the server's TLS certificate. */ +typedef enum line_sender_ca { + /** Use the set of root certificates provided by the `webpki` crate. */ + line_sender_ca_webpki_roots, + + /** Use the set of root certificates provided by the operating system. */ + line_sender_ca_os_roots, + + /** Use the set of root certificates provided by both the `webpki` crate and the operating system. */ + line_sender_ca_webpki_and_os_roots, + + /** Use a custom root certificate `.pem` file. */ + line_sender_ca_pem_file, +} line_sender_ca; + /** Error code categorizing the error. */ LINESENDER_API line_sender_error_code line_sender_error_get_code(const line_sender_error*); @@ -238,7 +278,7 @@ line_sender_buffer* line_sender_buffer_new(); LINESENDER_API line_sender_buffer* line_sender_buffer_with_max_name_len(size_t max_name_len); -/** Release the buffer object. */ +/** Release the `line_sender_buffer` object. */ LINESENDER_API void line_sender_buffer_free(line_sender_buffer* buffer); @@ -298,6 +338,17 @@ void line_sender_buffer_clear(line_sender_buffer* buffer); LINESENDER_API size_t line_sender_buffer_size(const line_sender_buffer* buffer); +/** The number of rows accumulated in the buffer. */ +LINESENDER_API +size_t line_sender_buffer_row_count(const line_sender_buffer* buffer); + +/** + * The buffer is transactional if sent over HTTP. + * A buffer stops being transactional if it contains rows for multiple tables. + */ +LINESENDER_API +bool line_sender_buffer_transactional(const line_sender_buffer* buffer); + /** * Peek into the accumulated buffer that is to be sent out at the next `flush`. * @@ -510,86 +561,112 @@ typedef struct line_sender line_sender; */ typedef struct line_sender_opts line_sender_opts; +/** + * Create a new `line_sender_opts` instance from configuration string. + * The format of the string is: "tcp::addr=host:port;key=value;...;" + * Alongside "tcp" you can also specify "tcps", "http", and "https". + * The accepted set of keys and values is the same as for the opt's API. + * E.g. "https::addr=host:port;username=alice;password=secret;tls_ca=os_roots;" + */ +LINESENDER_API +line_sender_opts* line_sender_opts_from_conf( + line_sender_utf8 config, + line_sender_error** err_out); + +/** + * Create a new `line_sender_opts` instance from configuration string + * read from the `QDB_CLIENT_CONF` environment variable. + */ +LINESENDER_API +line_sender_opts* line_sender_opts_from_env( + line_sender_error** err_out); + /** * A new set of options for a line sender connection. + * @param[in] protocol The protocol to use. * @param[in] host The QuestDB database host. - * @param[in] port The QuestDB database port. + * @param[in] port The QuestDB ILP TCP port. */ LINESENDER_API line_sender_opts* line_sender_opts_new( + line_sender_protocol protocol, line_sender_utf8 host, uint16_t port); /** - * A new set of options for a line sender connection. - * @param[in] host The QuestDB database host. - * @param[in] port The QuestDB database port as service name. + * Variant of line_sender_opts_new that takes a service name for port. */ LINESENDER_API line_sender_opts* line_sender_opts_new_service( + line_sender_protocol protocol, line_sender_utf8 host, line_sender_utf8 port); -/** Select local outbound interface. */ -LINESENDER_API -void line_sender_opts_net_interface( - line_sender_opts* opts, - line_sender_utf8 net_interface); - /** - * Authentication Parameters. - * @param[in] key_id Key id. AKA "kid" - * @param[in] priv_key Private key. AKA "d". - * @param[in] pub_key_x Public key X coordinate. AKA "x". - * @param[in] pub_key_y Public key Y coordinate. AKA "y". + * Select local outbound network "bind" interface. + * + * This may be relevant if your machine has multiple network interfaces. + * + * The default is `0.0.0.0`. */ LINESENDER_API -void line_sender_opts_auth( +bool line_sender_opts_bind_interface( line_sender_opts* opts, - line_sender_utf8 key_id, - line_sender_utf8 priv_key, - line_sender_utf8 pub_key_x, - line_sender_utf8 pub_key_y); + line_sender_utf8 bind_interface, + line_sender_error** err_out); /** - * Enable full connection encryption via TLS. - * The connection will accept certificates by well-known certificate - * authorities as per the "webpki-roots" Rust crate. + * Set the username for authentication. + * + * For TCP this is the `kid` part of the ECDSA key set. + * The other fields are `token` `token_x` and `token_y`. + * + * For HTTP this is part of basic authentication. + * Also see `password`. */ LINESENDER_API -void line_sender_opts_tls(line_sender_opts* opts); +bool line_sender_opts_username( + line_sender_opts* opts, + line_sender_utf8 username, + line_sender_error** err_out); /** - * Enable full connection encryption via TLS, using OS-provided certificate roots. + * Set the password for basic HTTP authentication. + * Also see `username`. */ LINESENDER_API -void line_sender_opts_tls_os_roots(line_sender_opts* opts); +bool line_sender_opts_password( + line_sender_opts* opts, + line_sender_utf8 password, + line_sender_error** err_out); -/* - * Enable full connection encryption via TLS, accepting certificates signed by either - * the OS-provided certificate roots or well-known certificate authorities as per - * the "webpki-roots" Rust crate. +/** + * Token (Bearer) Authentication Parameters for ILP over HTTP, + * or the ECDSA private key for ILP over TCP authentication. */ LINESENDER_API -void line_sender_opts_tls_webpki_and_os_roots(line_sender_opts* opts); +bool line_sender_opts_token( + line_sender_opts* opts, + line_sender_utf8 token, + line_sender_error** err_out); /** - * Enable full connection encryption via TLS. - * The connection will accept certificates by the specified certificate - * authority file. + * The ECDSA public key X for ILP over TCP authentication. */ LINESENDER_API -void line_sender_opts_tls_ca( +bool line_sender_opts_token_x( line_sender_opts* opts, - line_sender_utf8 ca_path); + line_sender_utf8 token_x, + line_sender_error** err_out); /** - * Enable TLS whilst dangerously accepting any certificate as valid. - * This should only be used for debugging. - * Consider using calling "tls_ca" instead. + * The ECDSA public key Y for ILP over TCP authentication. */ LINESENDER_API -void line_sender_opts_tls_insecure_skip_verify(line_sender_opts* opts); +bool line_sender_opts_token_y( + line_sender_opts* opts, + line_sender_utf8 token_y, + line_sender_error** err_out); /** * Configure how long to wait for messages from the QuestDB server during @@ -597,33 +674,155 @@ void line_sender_opts_tls_insecure_skip_verify(line_sender_opts* opts); * The default is 15 seconds. */ LINESENDER_API -void line_sender_opts_read_timeout( +bool line_sender_opts_auth_timeout( line_sender_opts* opts, - uint64_t timeout_millis); + uint64_t millis, + line_sender_error** err_out); /** - * Duplicate the opts object. + * Set to `false` to disable TLS certificate verification. + * This should only be used for debugging purposes as it reduces security. + * + * For testing consider specifying a path to a `.pem` file instead via + * the `tls_roots` setting. + */ +LINESENDER_API +bool line_sender_opts_tls_verify( + line_sender_opts* opts, + bool verify, + line_sender_error** err_out); + +/** + * Specify where to find the root certificates used to validate the + * server's TLS certificate. + */ +LINESENDER_API +bool line_sender_opts_tls_ca( + line_sender_opts* opts, + line_sender_ca ca, + line_sender_error** err_out); + +/** + * Set the path to a custom root certificate `.pem` file. + * This is used to validate the server's certificate during the TLS handshake. + * + * See notes on how to test with self-signed certificates: + * https://github.com/questdb/c-questdb-client/tree/main/tls_certs. + */ +LINESENDER_API +bool line_sender_opts_tls_roots( + line_sender_opts* opts, + line_sender_utf8 path, + line_sender_error** err_out); + +/** + * The maximum buffer size that the client will flush to the server. + * The default is 100 MiB. + */ +LINESENDER_API +bool line_sender_opts_max_buf_size( + line_sender_opts* opts, + size_t max_buf_size, + line_sender_error** err_out); + +/** + * Cumulative duration spent in retries. + * The default is 10 seconds. + */ +LINESENDER_API +bool line_sender_opts_retry_timeout( + line_sender_opts* opts, + uint64_t millis, + line_sender_error** err_out); + +/** + * Minimum expected throughput in bytes per second for HTTP requests. + * If the throughput is lower than this value, the connection will time out. + * The default is 100 KiB/s. + * The value is expressed as a number of bytes per second. + */ +LINESENDER_API +bool line_sender_opts_request_min_throughput( + line_sender_opts* opts, + uint64_t bytes_per_sec, + line_sender_error** err_out); + +/** + * Grace request timeout before relying on the minimum throughput logic. + * The default is 10 seconds. + */ +LINESENDER_API +bool line_sender_opts_request_timeout( + line_sender_opts* opts, + uint64_t millis, + line_sender_error** err_out); + +// Do not call: Private API for the C++ and Python bindings. +bool line_sender_opts_user_agent( + line_sender_opts* opts, + line_sender_utf8 user_agent, + line_sender_error** err_out); + +/** + * Duplicate the `line_sender_opts` object. * Both old and new objects will have to be freed. */ LINESENDER_API line_sender_opts* line_sender_opts_clone( line_sender_opts* opts); -/** Release the opts object. */ +/** Release the `line_sender_opts` object. */ LINESENDER_API void line_sender_opts_free(line_sender_opts* opts); /** - * Synchronously connect to the QuestDB database. - * The connection should be accessed by only a single thread a time. + * Create the client. + * The client should be accessed by only a single thread a time. * @param[in] opts Options for the connection. * @note The opts object is freed. */ LINESENDER_API -line_sender *line_sender_connect( +line_sender* line_sender_build( const line_sender_opts* opts, line_sender_error** err_out); +/** + * Create a new `line_sender` instance from configuration string. + * The format of the string is: "tcp::addr=host:port;key=value;...;" + * Alongside "tcp" you can also specify "tcps", "http", and "https". + * The accepted set of keys and values is the same as for the opt's API. + * E.g. "https::addr=host:port;username=alice;password=secret;tls_ca=os_roots;" + * + * For the full list of keys, search this header for `bool line_sender_opts_`. + */ +LINESENDER_API +line_sender* line_sender_from_conf( + line_sender_utf8 config, + line_sender_error** err_out); + +/** + * Create a new `line_sender` instance from configuration string read from the + * `QDB_CLIENT_CONF` environment variable. + */ +LINESENDER_API +line_sender* line_sender_from_env( + line_sender_error** err_out); + +/** + * Check if an error occurred previously and the sender must be closed. + * @param[in] sender Line sender object. + * @return true if an error occurred with a sender and it must be closed. + */ +LINESENDER_API +bool line_sender_must_close(const line_sender* sender); + +/** + * Close the connection. Does not flush. Non-idempotent. + * @param[in] sender Line sender object. + */ +LINESENDER_API +void line_sender_close(line_sender* sender); + /** * Send buffer of rows to the QuestDB server. * @@ -656,20 +855,22 @@ bool line_sender_flush_and_keep( line_sender_error** err_out); /** - * Check if an error occurred previously and the sender must be closed. - * @param[in] sender Line sender object. - * @return true if an error occurred with a sender and it must be closed. - */ -LINESENDER_API -bool line_sender_must_close(const line_sender* sender); - -/** - * Close the connection. Does not flush. Non-idempotent. - * @param[in] sender Line sender object. + * Variant of `.flush()` that does not clear the buffer and allows for + * transactional flushes. + * + * A transactional flush is simply a flush that ensures that all rows in + * the ILP buffer refer to the same table, thus allowing the server to + * treat the flush request as a single transaction. + * + * This is because QuestDB does not support transactions spanning multiple + * tables. */ LINESENDER_API -void line_sender_close(line_sender* sender); - +bool line_sender_flush_and_keep_with_flags( + line_sender* sender, + line_sender_buffer* buffer, + bool transactional, + line_sender_error** err_out); /////////// Getting the current timestamp. diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index 6649e3b2..79115da8 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -6,7 +6,7 @@ * \__\_\\__,_|\___||___/\__|____/|____/ * * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2023 QuestDB + * Copyright (c) 2019-2024 QuestDB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -67,7 +67,47 @@ namespace questdb::ingress auth_error, /** Error during TLS handshake. */ - tls_error + tls_error, + + /** The server does not support ILP over HTTP. */ + http_not_supported, + + /** Error sent back from the server during flush. */ + server_flush_error, + + /** Bad configuration. */ + config_error, + }; + + /** The protocol used to connect with. */ + enum class protocol + { + /** InfluxDB Line Protocol over TCP. */ + tcp, + + /** InfluxDB Line Protocol over TCP with TLS. */ + tcps, + + /** InfluxDB Line Protocol over HTTP. */ + http, + + /** InfluxDB Line Protocol over HTTP with TLS. */ + https, + }; + + /* Possible sources of the root certificates used to validate the server's TLS certificate. */ + enum class ca { + /** Use the set of root certificates provided by the `webpki` crate. */ + webpki_roots, + + /** Use the set of root certificates provided by the operating system. */ + os_roots, + + /** Use the set of root certificates provided by both the `webpki` crate and the operating system. */ + webpki_and_os_roots, + + /** Use a custom root certificate `.pem` file. */ + pem_file, }; /** @@ -275,28 +315,28 @@ namespace questdb::ingress class line_sender_buffer { public: - explicit line_sender_buffer(size_t init_capacity = 64 * 1024) noexcept - : line_sender_buffer{init_capacity, 127} + explicit line_sender_buffer(size_t init_buf_size = 64 * 1024) noexcept + : line_sender_buffer{init_buf_size, 127} {} line_sender_buffer( - size_t init_capacity, + size_t init_buf_size, size_t max_name_len) noexcept : _impl{nullptr} - , _init_capacity{init_capacity} + , _init_buf_size{init_buf_size} , _max_name_len{max_name_len} { } line_sender_buffer(const line_sender_buffer& other) noexcept : _impl{::line_sender_buffer_clone(other._impl)} - , _init_capacity{other._init_capacity} + , _init_buf_size{other._init_buf_size} , _max_name_len{other._max_name_len} {} line_sender_buffer(line_sender_buffer&& other) noexcept : _impl{other._impl} - , _init_capacity{other._init_capacity} + , _init_buf_size{other._init_buf_size} , _max_name_len{other._max_name_len} { other._impl = nullptr; @@ -311,7 +351,7 @@ namespace questdb::ingress _impl = ::line_sender_buffer_clone(other._impl); else _impl = nullptr; - _init_capacity = other._init_capacity; + _init_buf_size = other._init_buf_size; _max_name_len = other._max_name_len; } return *this; @@ -323,7 +363,7 @@ namespace questdb::ingress { ::line_sender_buffer_free(_impl); _impl = other._impl; - _init_capacity = other._init_capacity; + _init_buf_size = other._init_buf_size; _max_name_len = other._max_name_len; other._impl = nullptr; } @@ -343,6 +383,7 @@ namespace questdb::ingress ::line_sender_buffer_reserve(_impl, additional); } + /** Get the current capacity of the buffer. */ size_t capacity() const noexcept { if (_impl) @@ -351,6 +392,7 @@ namespace questdb::ingress return 0; } + /** Number of bytes in the accumulated buffer. */ size_t size() const noexcept { if (_impl) @@ -359,6 +401,31 @@ namespace questdb::ingress return 0; } + /** The number of rows accumulated in the buffer. */ + size_t row_count() const noexcept + { + if (_impl) + return ::line_sender_buffer_row_count(_impl); + else + return 0; + } + + /** + * The buffer is transactional if sent over HTTP. + * A buffer stops being transactional if it contains rows for multiple tables. + */ + bool transactional() const noexcept + { + if (_impl) + return ::line_sender_buffer_transactional(_impl); + else + return 0; + } + + /** + * Get a view of the accumulated buffer. + * This is useful for debugging. + */ std::string_view peek() const noexcept { if (_impl) @@ -373,6 +440,13 @@ namespace questdb::ingress } } + /** + * Mark a rewind point. + * This allows undoing accumulated changes to the buffer for one or more + * rows by calling `rewind_to_marker`. + * Any previous marker will be discarded. + * Once the marker is no longer needed, call `clear_marker`. + */ void set_marker() { may_init(); @@ -380,6 +454,10 @@ namespace questdb::ingress ::line_sender_buffer_set_marker, _impl); } + /** + * Undo all changes since the last `set_marker` call. + * As a side-effect, this also clears the marker. + */ void rewind_to_marker() { may_init(); @@ -387,12 +465,17 @@ namespace questdb::ingress ::line_sender_buffer_rewind_to_marker, _impl); } + /** Discard the marker. */ void clear_marker() noexcept { if (_impl) ::line_sender_buffer_clear_marker(_impl); } + /** + * Remove all accumulated data and prepare the buffer for new lines. + * This does not affect the buffer's capacity. + */ void clear() noexcept { if (_impl) @@ -634,41 +717,109 @@ namespace questdb::ingress if (!_impl) { _impl = ::line_sender_buffer_with_max_name_len(_max_name_len); - ::line_sender_buffer_reserve(_impl, _init_capacity); + ::line_sender_buffer_reserve(_impl, _init_buf_size); } } ::line_sender_buffer* _impl; - size_t _init_capacity; + size_t _init_buf_size; size_t _max_name_len; friend class line_sender; }; + class _user_agent + { + private: + static inline ::line_sender_utf8 name() + { + // Maintained by .bumpversion.cfg + static const char user_agent[] = "questdb/c++/4.0.0"; + ::line_sender_utf8 utf8 = ::line_sender_utf8_assert( + sizeof(user_agent) - 1, + user_agent); + return utf8; + } + + friend class opts; + }; + class opts { public: + /** + * Create a new `opts` instance from configuration string. + * The format of the string is: "tcp::addr=host:port;key=value;...;" + * Alongside "tcp" you can also specify "tcps", "http", and "https". + * The accepted set of keys and values is the same as for the opt's API. + * E.g. "https::addr=host:port;username=alice;password=secret;tls_ca=os_roots;" + */ + static inline opts from_conf(utf8_view conf) + { + return {line_sender_error::wrapped_call( + ::line_sender_opts_from_conf, + conf._impl)}; + } + + /** + * Create a new `opts` instance from configuration string read from the + * `QDB_CLIENT_CONF` environment variable. + */ + static inline opts from_env() + { + opts impl{line_sender_error::wrapped_call( + ::line_sender_opts_from_env)}; + line_sender_error::wrapped_call( + ::line_sender_opts_user_agent, + impl._impl, + _user_agent::name()); + return impl; + } + /** * A new set of options for a line sender connection. + * @param[in] protocol The protocol to use. * @param[in] host The QuestDB database host. - * @param[in] port The QuestDB database port. + * @param[in] port The QuestDB tcp or http port. */ opts( + protocol protocol, utf8_view host, uint16_t port) noexcept - : _impl{::line_sender_opts_new(host._impl, port)} - {} + : _impl{ + ::line_sender_opts_new( + static_cast<::line_sender_protocol>(protocol), + host._impl, + port) + } + { + line_sender_error::wrapped_call( + ::line_sender_opts_user_agent, + _impl, + _user_agent::name()); + } /** * A new set of options for a line sender connection. + * @param[in] protocol The protocol to use. * @param[in] host The QuestDB database host. - * @param[in] port The QuestDB database port as service name. + * @param[in] port The QuestDB tcp or http port as service name. */ opts( + protocol protocol, utf8_view host, utf8_view port) noexcept - : _impl{::line_sender_opts_new_service(host._impl, port._impl)} - {} + : _impl{ + ::line_sender_opts_new_service( + static_cast<::line_sender_protocol>(protocol), + host._impl, port._impl) + } + { + line_sender_error::wrapped_call( + ::line_sender_opts_user_agent, + _impl, + _user_agent::name()); + } opts(const opts& other) noexcept : _impl{::line_sender_opts_clone(other._impl)} @@ -701,98 +852,201 @@ namespace questdb::ingress return *this; } - /** Select local outbound interface. */ - opts& net_interface(utf8_view net_interface) noexcept + /** + * Select local outbound network "bind" interface. + * + * This may be relevant if your machine has multiple network interfaces. + * + * The default is `0.0.0.0`. + */ + opts& bind_interface(utf8_view bind_interface) { - ::line_sender_opts_net_interface( + line_sender_error::wrapped_call( + ::line_sender_opts_bind_interface, _impl, - net_interface._impl); + bind_interface._impl); return *this; } /** - * Authentication Parameters. - * @param[in] key_id Key id. AKA "kid" - * @param[in] priv_key Private key. AKA "d". - * @param[in] pub_key_x Public key X coordinate. AKA "x". - * @param[in] pub_key_y Public key Y coordinate. AKA "y". + * Set the username for basic HTTP authentication. + * + * For TCP this is the `kid` part of the ECDSA key set. + * The other fields are `token` `token_x` and `token_y`. + * + * For HTTP this is part of basic authentication. + * Also see `password`. */ - opts& auth( - utf8_view key_id, - utf8_view priv_key, - utf8_view pub_key_x, - utf8_view pub_key_y) noexcept + opts& username(utf8_view username) { - ::line_sender_opts_auth( + line_sender_error::wrapped_call( + ::line_sender_opts_username, _impl, - key_id._impl, - priv_key._impl, - pub_key_x._impl, - pub_key_y._impl); + username._impl); return *this; } /** - * Enable full connection encryption via TLS. - * The connection will accept certificates by well-known certificate - * authorities as per the "webpki-roots" Rust crate. + * Set the password for basic HTTP authentication. + * Also see `username`. */ - opts& tls() noexcept + opts& password(utf8_view password) { - ::line_sender_opts_tls(_impl); + line_sender_error::wrapped_call( + ::line_sender_opts_password, + _impl, + password._impl); return *this; } - /* - * Enable full connection encryption via TLS, using OS-provided certificate roots. + /** + * Token (Bearer) Authentication Parameters for ILP over HTTP, + * or the ECDSA private key for ILP over TCP authentication. */ - opts& tls_os_roots() noexcept + opts& token(utf8_view token) { - ::line_sender_opts_tls_os_roots(_impl); + line_sender_error::wrapped_call( + ::line_sender_opts_token, + _impl, + token._impl); return *this; } - /* - * Enable full connection encryption via TLS, accepting certificates signed by either - * the OS-provided certificate roots or well-known certificate authorities as per - * the "webpki-roots" Rust crate. + /** + * The ECDSA public key X for ILP over TCP authentication. */ - opts& tls_webpki_and_os_roots() noexcept + opts& token_x(utf8_view token_x) { - ::line_sender_opts_tls_webpki_and_os_roots(_impl); + line_sender_error::wrapped_call( + ::line_sender_opts_token_x, + _impl, + token_x._impl); return *this; } /** - * Enable full connection encryption via TLS. - * The connection will accept certificates by the specified certificate - * authority file. + * The ECDSA public key Y for ILP over TCP authentication. */ - opts& tls(utf8_view ca_file) noexcept + opts& token_y(utf8_view token_y) { - ::line_sender_opts_tls_ca(_impl, ca_file._impl); + line_sender_error::wrapped_call( + ::line_sender_opts_token_y, + _impl, + token_y._impl); return *this; } /** - * Enable TLS whilst dangerously accepting any certificate as valid. - * This should only be used for debugging. - * Consider using calling "tls_ca" instead. + * Configure how long to wait for messages from the QuestDB server during + * the TLS handshake and authentication process. + * The default is 15 seconds. */ - opts& tls_insecure_skip_verify() noexcept + opts& auth_timeout(uint64_t millis) { - ::line_sender_opts_tls_insecure_skip_verify(_impl); + line_sender_error::wrapped_call( + ::line_sender_opts_auth_timeout, + _impl, + millis); return *this; } /** - * Configure how long to wait for messages from the QuestDB server - * during the TLS handshake and authentication process. - * The default is 15 seconds. + * Set to `false` to disable TLS certificate verification. + * This should only be used for debugging purposes as it reduces security. + * + * For testing consider specifying a path to a `.pem` file instead via + * the `tls_roots` setting. + */ + opts& tls_verify(bool verify) + { + line_sender_error::wrapped_call( + ::line_sender_opts_tls_verify, + _impl, + verify); + return *this; + } + + /** + * Specify where to find the certificate authority used to validate the + * server's TLS certificate. */ - opts& read_timeout(uint64_t timeout_millis) noexcept + opts& tls_ca(ca ca) { - ::line_sender_opts_read_timeout(_impl, timeout_millis); + ::line_sender_ca ca_impl = static_cast<::line_sender_ca>(ca); + line_sender_error::wrapped_call( + ::line_sender_opts_tls_ca, + _impl, + ca_impl); + return *this; + } + + /** + * Set the path to a custom root certificate `.pem` file. + * This is used to validate the server's certificate during the TLS handshake. + * + * See notes on how to test with self-signed certificates: + * https://github.com/questdb/c-questdb-client/tree/main/tls_certs. + */ + opts& tls_roots(utf8_view path) + { + line_sender_error::wrapped_call( + ::line_sender_opts_tls_roots, + _impl, + path._impl); + return *this; + } + + /** + * The maximum buffer size that the client will flush to the server. + * The default is 100 MiB. + */ + opts& max_buf_size(size_t max_buf_size) + { + line_sender_error::wrapped_call( + ::line_sender_opts_max_buf_size, + _impl, + max_buf_size); + return *this; + } + + /** + * Cumulative duration spent in retries. + * The default is 10 seconds. + */ + opts& retry_timeout(uint64_t millis) + { + line_sender_error::wrapped_call( + ::line_sender_opts_retry_timeout, + _impl, + millis); + return *this; + } + + /** + * Minimum expected throughput in bytes per second for HTTP requests. + * If the throughput is lower than this value, the connection will time out. + * The default is 100 KiB/s. + * The value is expressed as a number of bytes per second. + */ + opts& request_min_throughput(uint64_t bytes_per_sec) + { + line_sender_error::wrapped_call( + ::line_sender_opts_request_min_throughput, + _impl, + bytes_per_sec); + return *this; + } + + /** + * Grace request timeout before relying on the minimum throughput logic. + * The default is 10 seconds. + */ + opts& request_timeout(uint64_t millis) + { + line_sender_error::wrapped_call( + ::line_sender_opts_request_timeout, + _impl, + millis); return *this; } @@ -801,6 +1055,9 @@ namespace questdb::ingress reset(); } private: + opts(::line_sender_opts* impl) : _impl{impl} + {} + void reset() noexcept { if (_impl) @@ -826,17 +1083,38 @@ namespace questdb::ingress class line_sender { public: - line_sender(utf8_view host, uint16_t port) - : line_sender{opts{host, port}} + /** + * Create a new `line_sender` instance from configuration string. + * The format of the string is: "tcp::addr=host:port;key=value;...;" + * Alongside "tcp" you can also specify "tcps", "http", and "https". + * The accepted set of keys and values is the same as for the opt's API. + * E.g. "https::addr=host:port;username=alice;password=secret;tls_ca=os_roots;" + */ + static inline line_sender from_conf(utf8_view conf) + { + return {opts::from_conf(conf)}; + } + + /** + * Create a new `line_sender` instance from configuration string read from the + * `QDB_CLIENT_CONF` environment variable. + */ + static inline line_sender from_env() + { + return {opts::from_env()}; + } + + line_sender(protocol protocol, utf8_view host, uint16_t port) + : line_sender{opts{protocol, host, port}} {} - line_sender(utf8_view host, utf8_view port) - : line_sender{opts{host, port}} + line_sender(protocol protocol, utf8_view host, utf8_view port) + : line_sender{opts{protocol, host, port}} {} line_sender(const opts& opts) : _impl{line_sender_error::wrapped_call( - ::line_sender_connect, opts._impl)} + ::line_sender_build, opts._impl)} {} line_sender(const line_sender&) = delete; diff --git a/questdb-rs-ffi/Cargo.lock b/questdb-rs-ffi/Cargo.lock index c058bd5f..795e4ef9 100644 --- a/questdb-rs-ffi/Cargo.lock +++ b/questdb-rs-ffi/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + [[package]] name = "autocfg" version = "1.1.0" @@ -10,9 +16,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "base64" -version = "0.21.4" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" [[package]] name = "base64ct" @@ -28,15 +34,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" - -[[package]] -name = "bumpalo" -version = "3.14.0" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" +checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" [[package]] name = "cbindgen" @@ -73,9 +73,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "core-foundation" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" dependencies = [ "core-foundation-sys", "libc", @@ -83,54 +83,71 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.4" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" + +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if", +] [[package]] name = "dns-lookup" -version = "2.0.3" +version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d0fa3cd8dc96ada974e126a940d37d4079bbbe6a24aca15b1113c2f362441c5" +checksum = "e5766087c2235fec47fafa4cfecc81e494ee679d0fd4a59887ea0919bfb0e4fc" dependencies = [ "cfg-if", "libc", "socket2", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] name = "errno" -version = "0.3.3" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "136526188508e25c6fef639d7927dfb3e0e3084488bf202267829cf7fc23dbdd" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" dependencies = [ - "errno-dragonfly", "libc", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] -name = "errno-dragonfly" -version = "0.1.2" +name = "fastrand" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" + +[[package]] +name = "flate2" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" dependencies = [ - "cc", - "libc", + "crc32fast", + "miniz_oxide", ] [[package]] -name = "fastrand" -version = "2.0.0" +name = "form_urlencoded" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] [[package]] name = "getrandom" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" +checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" dependencies = [ "cfg-if", "libc", @@ -149,6 +166,45 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +[[package]] +name = "hoot" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df22a4d90f1b0e65fe3e0d6ee6a4608cc4d81f4b2eb3e670f44bb6bde711e452" +dependencies = [ + "httparse", + "log", +] + +[[package]] +name = "hootbin" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "354e60868e49ea1a39c44b9562ad207c4259dc6eabf9863bf3b0f058c55cfdb2" +dependencies = [ + "fastrand", + "hoot", + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "idna" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "indexmap" version = "1.9.3" @@ -167,30 +223,21 @@ checksum = "1e186cfbae8084e513daff4240b4797e342f988cecda4fb6c939150f96315fd8" [[package]] name = "itoa" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" - -[[package]] -name = "js-sys" -version = "0.3.64" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" -dependencies = [ - "wasm-bindgen", -] +checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" [[package]] name = "libc" -version = "0.2.150" +version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" [[package]] name = "linux-raw-sys" -version = "0.4.7" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a9bad9f94746442c783ca431b22403b519cd7fbeed0533fdd6328b2f2212128" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" [[package]] name = "log" @@ -198,11 +245,20 @@ version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +[[package]] +name = "miniz_oxide" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" +dependencies = [ + "adler", +] + [[package]] name = "once_cell" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "openssl-probe" @@ -210,172 +266,211 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + [[package]] name = "proc-macro2" -version = "1.0.67" +version = "1.0.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328" +checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" dependencies = [ "unicode-ident", ] +[[package]] +name = "questdb-confstr" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a85ae0d477308986d89496bce6227833e11490cd27b98b809e4704b105000fd8" + +[[package]] +name = "questdb-confstr-ffi" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a745c819f8ef71ca240dff7119bad4db368e0b11489c6e5e07b8e5cf2c775561" +dependencies = [ + "questdb-confstr", +] + [[package]] name = "questdb-rs" -version = "3.1.0" +version = "4.0.0" dependencies = [ "base64ct", "dns-lookup", "indoc", "itoa", "libc", - "ring 0.17.5", + "questdb-confstr", + "rand", + "ring", "rustls", "rustls-native-certs", "rustls-pemfile", + "rustls-pki-types", "ryu", "serde", "serde_json", "slugify", "socket2", + "ureq", "webpki-roots", "winapi", ] [[package]] name = "questdb-rs-ffi" -version = "3.1.0" +version = "4.0.0" dependencies = [ "cbindgen", "libc", + "questdb-confstr-ffi", "questdb-rs", ] [[package]] name = "quote" -version = "1.0.33" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" dependencies = [ "proc-macro2", ] [[package]] -name = "redox_syscall" -version = "0.3.5" +name = "rand" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ - "bitflags 1.3.2", + "libc", + "rand_chacha", + "rand_core", ] [[package]] -name = "ring" -version = "0.16.20" +name = "rand_chacha" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ - "cc", - "libc", - "once_cell", - "spin 0.5.2", - "untrusted 0.7.1", - "web-sys", - "winapi", + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", ] [[package]] name = "ring" -version = "0.17.5" +version = "0.17.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb0205304757e5d899b9c2e448b867ffd03ae7f988002e47cd24954391394d0b" +checksum = "688c63d65483050968b2a8937f7995f443e27041a0f7700aa59b0822aedebb74" dependencies = [ "cc", "getrandom", "libc", - "spin 0.9.8", - "untrusted 0.9.0", - "windows-sys", + "spin", + "untrusted", + "windows-sys 0.48.0", ] [[package]] name = "rustix" -version = "0.38.14" +version = "0.38.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "747c788e9ce8e92b12cd485c49ddf90723550b654b32508f979b71a7b1ecda4f" +checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.4.2", "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] name = "rustls" -version = "0.21.7" +version = "0.22.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8" +checksum = "e87c9956bd9807afa1f77e0f7594af32566e830e088a5576d27c5b6f30f49d41" dependencies = [ "log", - "ring 0.16.20", + "ring", + "rustls-pki-types", "rustls-webpki", - "sct", + "subtle", + "zeroize", ] [[package]] name = "rustls-native-certs" -version = "0.6.3" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +checksum = "8f1fb85efa936c42c6d5fc28d2629bb51e4b2f4b8a5211e297d599cc5a093792" dependencies = [ "openssl-probe", "rustls-pemfile", + "rustls-pki-types", "schannel", "security-framework", ] [[package]] name = "rustls-pemfile" -version = "1.0.3" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" +checksum = "35e4980fa29e4c4b212ffb3db068a564cbf560e51d3944b7c88bd8bf5bec64f4" dependencies = [ "base64", + "rustls-pki-types", ] +[[package]] +name = "rustls-pki-types" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a716eb65e3158e90e17cd93d855216e27bde02745ab842f2cab4a39dba1bacf" + [[package]] name = "rustls-webpki" -version = "0.101.6" +version = "0.102.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c7d5dece342910d9ba34d259310cae3e0154b873b35408b787b59bce53d34fe" +checksum = "faaa0a62740bedb9b2ef5afa303da42764c012f743917351dc9a237ea1663610" dependencies = [ - "ring 0.16.20", - "untrusted 0.7.1", + "ring", + "rustls-pki-types", + "untrusted", ] [[package]] name = "ryu" -version = "1.0.15" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" +checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" [[package]] name = "schannel" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" dependencies = [ - "windows-sys", -] - -[[package]] -name = "sct" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" -dependencies = [ - "ring 0.16.20", - "untrusted 0.7.1", + "windows-sys 0.52.0", ] [[package]] @@ -403,29 +498,29 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.188" +version = "1.0.196" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" +checksum = "870026e60fa08c69f064aa766c10f10b1d62db9ccd4d0abb206472bee0ce3b32" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.188" +version = "1.0.196" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" +checksum = "33c85360c95e7d137454dc81d9a4ed2b8efd8fbe19cee57357b32b9771fccb67" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.48", ] [[package]] name = "serde_json" -version = "1.0.107" +version = "1.0.113" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" +checksum = "69801b70b1c3dac963ecb03a364ba0ceda9cf60c71cfe475e99864759c8b8a79" dependencies = [ "itoa", "ryu", @@ -443,25 +538,25 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.4" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" +checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] name = "spin" -version = "0.5.2" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" [[package]] -name = "spin" -version = "0.9.8" +name = "subtle" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "syn" @@ -476,9 +571,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.37" +version = "2.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7303ef2c05cd654186cb250d29049a24840ca25d2747c25c0381c8d9e2f582e8" +checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" dependencies = [ "proc-macro2", "quote", @@ -487,17 +582,51 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.8.0" +version = "3.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" +checksum = "a365e8cd18e44762ef95d87f284f4b5cd04107fec2ff3052bd6a3e6069669e67" dependencies = [ "cfg-if", "fastrand", - "redox_syscall", "rustix", - "windows-sys", + "windows-sys 0.52.0", ] +[[package]] +name = "thiserror" +version = "1.0.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "toml" version = "0.5.11" @@ -507,6 +636,12 @@ dependencies = [ "serde", ] +[[package]] +name = "unicode-bidi" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" + [[package]] name = "unicode-ident" version = "1.0.12" @@ -514,16 +649,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] -name = "unidecode" -version = "0.3.0" +name = "unicode-normalization" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "402bb19d8e03f1d1a7450e2bd613980869438e0666331be3e073089124aa1adc" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] [[package]] -name = "untrusted" -version = "0.7.1" +name = "unidecode" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +checksum = "402bb19d8e03f1d1a7450e2bd613980869438e0666331be3e073089124aa1adc" [[package]] name = "untrusted" @@ -532,81 +670,49 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] -name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" - -[[package]] -name = "wasm-bindgen" -version = "0.2.87" +name = "ureq" +version = "2.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" +checksum = "0b52731d03d6bb2fd18289d4028aee361d6c28d44977846793b994b13cdcc64d" dependencies = [ - "cfg-if", - "wasm-bindgen-macro", -] - -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.87" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" -dependencies = [ - "bumpalo", + "base64", + "flate2", + "hootbin", "log", "once_cell", - "proc-macro2", - "quote", - "syn 2.0.37", - "wasm-bindgen-shared", + "rustls", + "rustls-pki-types", + "rustls-webpki", + "url", + "webpki-roots", ] [[package]] -name = "wasm-bindgen-macro" -version = "0.2.87" +name = "url" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" dependencies = [ - "quote", - "wasm-bindgen-macro-support", + "form_urlencoded", + "idna", + "percent-encoding", ] [[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.87" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.37", - "wasm-bindgen-backend", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.87" +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] -name = "web-sys" -version = "0.3.64" +name = "webpki-roots" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" +checksum = "b3de34ae270483955a94f4b21bdaaeb83d508bb84a01435f393818edb0012009" dependencies = [ - "js-sys", - "wasm-bindgen", + "rustls-pki-types", ] -[[package]] -name = "webpki-roots" -version = "0.25.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc" - [[package]] name = "winapi" version = "0.3.9" @@ -635,7 +741,16 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets", + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.0", ] [[package]] @@ -644,13 +759,28 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +dependencies = [ + "windows_aarch64_gnullvm 0.52.0", + "windows_aarch64_msvc 0.52.0", + "windows_i686_gnu 0.52.0", + "windows_i686_msvc 0.52.0", + "windows_x86_64_gnu 0.52.0", + "windows_x86_64_gnullvm 0.52.0", + "windows_x86_64_msvc 0.52.0", ] [[package]] @@ -659,38 +789,86 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" + [[package]] name = "windows_i686_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" +[[package]] +name = "windows_i686_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" + [[package]] name = "windows_i686_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" +[[package]] +name = "windows_i686_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" + +[[package]] +name = "zeroize" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" diff --git a/questdb-rs-ffi/Cargo.toml b/questdb-rs-ffi/Cargo.toml index bfbec857..8af545a6 100644 --- a/questdb-rs-ffi/Cargo.toml +++ b/questdb-rs-ffi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "questdb-rs-ffi" -version = "3.1.0" +version = "4.0.0" edition = "2021" publish = false @@ -9,13 +9,20 @@ name = "questdb_client" crate-type = ["cdylib", "staticlib"] [dependencies] -questdb-rs = { path = "../questdb-rs", features = ["insecure-skip-verify", "tls-native-certs"] } +questdb-rs = { path = "../questdb-rs", features = [ + "insecure-skip-verify", "tls-native-certs", "ilp-over-http"] } libc = "0.2" +questdb-confstr-ffi = { version = "0.1.0", optional = true } [build-dependencies] cbindgen = { version = "0.26.0", optional = true, default-features = false } [features] +# Expose the config parsing C API. +# This used by `py-questdb-client` to parse the config file. +# It is exposed here to avoid having multiple copies of the `questdb-confstr` +# crate in the final binary. +confstr-ffi = ["dep:questdb-confstr-ffi"] # Auto-generate the header. This is for dev-debugging-diffing only. # A hand-crafted header is easier on the eyes. diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index a674ccfc..dd34a80d 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -6,7 +6,7 @@ * \__\_\\__,_|\___||___/\__|____/|____/ * * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2023 QuestDB + * Copyright (c) 2019-2024 QuestDB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -35,8 +35,8 @@ use std::str; use questdb::{ ingress::{ - Buffer, CertificateAuthority, ColumnName, Sender, SenderBuilder, TableName, - TimestampMicros, TimestampNanos, Tls, + Buffer, CertificateAuthority, ColumnName, Protocol, Sender, SenderBuilder, TableName, + TimestampMicros, TimestampNanos, }, Error, ErrorCode, }; @@ -60,10 +60,39 @@ macro_rules! bubble_err_to_c { /// Update the Rust builder inside the C opts object /// after calling a method that takes ownership of the builder. macro_rules! upd_opts { - ($opts:expr, $func:ident, $($args:expr),*) => { - ptr::write( - &mut (*$opts).0, - ptr::read(&(*$opts).0).$func($($args),*)); + // This is square-peg-round-hole code. + // The C API is not designed to handle Rust's move semantics. + // So we're going to do some very unsafe things here. + // We need to extract a `T` from a `*mut T` and then replace it with + // another `T` in situ. + ($opts:expr, $err_out:expr, $func:ident $(, $($args:expr),*)?) => { + { + let builder_ref: *mut SenderBuilder = &mut (*$opts).0; + let forced_builder = ptr::read(&(*$opts).0); + let new_builder_or_err = forced_builder.$func($($($args),*)?); + let new_builder = match new_builder_or_err { + Ok(builder) => builder, + Err(err) => { + *$err_out = Box::into_raw(Box::new(line_sender_error(err))); + // We're really messing with the borrow-checker here. + // We've moved ownership of `forced_builder` (which is actually + // just an alias of the real `SenderBuilder` owned by the caller + // via a pointer - but the Rust compiler doesn't know that) + // into this function. + // This leaves the original caller holding a pointer to an + // already cleaned up object. + // To avoid double-freeing, we need to construct a valid "dummy" + // object on top of the memory that is still owned by the caller. + let dummy = SenderBuilder::new(Protocol::Tcp, "localhost", 1); + ptr::write(builder_ref, dummy); + return false; + } + }; + + // Overwrite the original builder with the new one. + ptr::write(builder_ref, new_builder); + true + } }; } @@ -97,6 +126,15 @@ pub enum line_sender_error_code { /// Error during TLS handshake. line_sender_error_tls_error, + + /// The server does not support ILP over HTTP. + line_sender_error_http_not_supported, + + /// Error sent back from the server during flush. + line_sender_error_server_flush_error, + + /// Bad configuration. + line_sender_error_config_error, } impl From for line_sender_error_code { @@ -114,6 +152,95 @@ impl From for line_sender_error_code { } ErrorCode::AuthError => line_sender_error_code::line_sender_error_auth_error, ErrorCode::TlsError => line_sender_error_code::line_sender_error_tls_error, + ErrorCode::HttpNotSupported => { + line_sender_error_code::line_sender_error_http_not_supported + } + ErrorCode::ServerFlushError => { + line_sender_error_code::line_sender_error_server_flush_error + } + ErrorCode::ConfigError => line_sender_error_code::line_sender_error_config_error, + } + } +} + +/// The protocol used to connect with. +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub enum line_sender_protocol { + /// InfluxDB Line Protocol over TCP. + line_sender_protocol_tcp, + + /// InfluxDB Line Protocol over TCP with TLS. + line_sender_protocol_tcps, + + /// InfluxDB Line Protocol over HTTP. + line_sender_protocol_http, + + /// InfluxDB Line Protocol over HTTP with TLS. + line_sender_protocol_https, +} + +impl From for line_sender_protocol { + fn from(protocol: Protocol) -> Self { + match protocol { + Protocol::Tcp => line_sender_protocol::line_sender_protocol_tcp, + Protocol::Tcps => line_sender_protocol::line_sender_protocol_tcps, + Protocol::Http => line_sender_protocol::line_sender_protocol_http, + Protocol::Https => line_sender_protocol::line_sender_protocol_https, + } + } +} + +impl From for Protocol { + fn from(protocol: line_sender_protocol) -> Self { + match protocol { + line_sender_protocol::line_sender_protocol_tcp => Protocol::Tcp, + line_sender_protocol::line_sender_protocol_tcps => Protocol::Tcps, + line_sender_protocol::line_sender_protocol_http => Protocol::Http, + line_sender_protocol::line_sender_protocol_https => Protocol::Https, + } + } +} + +/// Possible sources of the root certificates used to validate the server's TLS certificate. +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub enum line_sender_ca { + /// Use the set of root certificates provided by the `webpki` crate. + line_sender_ca_webpki_roots, + + /// Use the set of root certificates provided by the operating system. + line_sender_ca_os_roots, + + /// Use the set of root certificates provided by both the `webpki` crate and the operating system. + line_sender_ca_webpki_and_os_roots, + + /// Use a custom root certificate `.pem` file. + line_sender_ca_pem_file, +} + +impl From for line_sender_ca { + fn from(ca: CertificateAuthority) -> Self { + match ca { + CertificateAuthority::WebpkiRoots => line_sender_ca::line_sender_ca_webpki_roots, + CertificateAuthority::OsRoots => line_sender_ca::line_sender_ca_os_roots, + CertificateAuthority::WebpkiAndOsRoots => { + line_sender_ca::line_sender_ca_webpki_and_os_roots + } + CertificateAuthority::PemFile => line_sender_ca::line_sender_ca_pem_file, + } + } +} + +impl From for CertificateAuthority { + fn from(ca: line_sender_ca) -> Self { + match ca { + line_sender_ca::line_sender_ca_webpki_roots => CertificateAuthority::WebpkiRoots, + line_sender_ca::line_sender_ca_os_roots => CertificateAuthority::OsRoots, + line_sender_ca::line_sender_ca_webpki_and_os_roots => { + CertificateAuthority::WebpkiAndOsRoots + } + line_sender_ca::line_sender_ca_pem_file => CertificateAuthority::PemFile, } } } @@ -409,142 +536,6 @@ pub unsafe extern "C" fn line_sender_column_name_assert( } } -/// Accumulates parameters for creating a line sender connection. -pub struct line_sender_opts(SenderBuilder); - -/// A new set of options for a line sender connection. -/// @param[in] host The QuestDB database host. -/// @param[in] port The QuestDB database port. -#[no_mangle] -pub unsafe extern "C" fn line_sender_opts_new( - host: line_sender_utf8, - port: u16, -) -> *mut line_sender_opts { - let builder = SenderBuilder::new(host.as_str(), port); - Box::into_raw(Box::new(line_sender_opts(builder))) -} - -/// A new set of options for a line sender connection. -/// @param[in] host The QuestDB database host. -/// @param[in] port The QuestDB database port as service name. -#[no_mangle] -pub unsafe extern "C" fn line_sender_opts_new_service( - host: line_sender_utf8, - port: line_sender_utf8, -) -> *mut line_sender_opts { - let builder = SenderBuilder::new(host.as_str(), port.as_str()); - Box::into_raw(Box::new(line_sender_opts(builder))) -} - -/// Select local outbound interface. -#[no_mangle] -pub unsafe extern "C" fn line_sender_opts_net_interface( - opts: *mut line_sender_opts, - net_interface: line_sender_utf8, -) { - upd_opts!(opts, net_interface, net_interface.as_str()); -} - -/// Authentication Parameters. -/// @param[in] key_id Key id. AKA "kid" -/// @param[in] priv_key Private key. AKA "d". -/// @param[in] pub_key_x Public key X coordinate. AKA "x". -/// @param[in] pub_key_y Public key Y coordinate. AKA "y". -#[no_mangle] -pub unsafe extern "C" fn line_sender_opts_auth( - opts: *mut line_sender_opts, - key_id: line_sender_utf8, - priv_key: line_sender_utf8, - pub_key_x: line_sender_utf8, - pub_key_y: line_sender_utf8, -) { - upd_opts!( - opts, - auth, - key_id.as_str(), - priv_key.as_str(), - pub_key_x.as_str(), - pub_key_y.as_str() - ); -} - -/// Enable full connection encryption via TLS. -/// The connection will accept certificates by well-known certificate -/// authorities as per the "webpki-roots" Rust crate. -#[no_mangle] -pub unsafe extern "C" fn line_sender_opts_tls(opts: *mut line_sender_opts) { - upd_opts!(opts, tls, Tls::Enabled(CertificateAuthority::WebpkiRoots)); -} - -/// Enable full connection encryption via TLS, using OS-provided certificate roots. -#[no_mangle] -pub unsafe extern "C" fn line_sender_opts_tls_os_roots(opts: *mut line_sender_opts) { - upd_opts!(opts, tls, Tls::Enabled(CertificateAuthority::OsRoots)); -} - -/// Enable full connection encryption via TLS, accepting certificates signed by either -/// the OS-provided certificate roots or well-known certificate authorities as per -/// the "webpki-roots" Rust crate. -#[no_mangle] -pub unsafe extern "C" fn line_sender_opts_tls_webpki_and_os_roots(opts: *mut line_sender_opts) { - upd_opts!( - opts, - tls, - Tls::Enabled(CertificateAuthority::WebpkiAndOsRoots) - ); -} - -/// Enable full connection encryption via TLS. -/// The connection will accept certificates by the specified certificate -/// authority file. -#[no_mangle] -pub unsafe extern "C" fn line_sender_opts_tls_ca( - opts: *mut line_sender_opts, - ca_path: line_sender_utf8, -) { - let ca_path = PathBuf::from(ca_path.as_str()); - upd_opts!(opts, tls, Tls::Enabled(CertificateAuthority::File(ca_path))); -} - -/// Enable TLS whilst dangerously accepting any certificate as valid. -/// This should only be used for debugging. -/// Consider using calling "tls_ca" instead. -#[no_mangle] -pub unsafe extern "C" fn line_sender_opts_tls_insecure_skip_verify(opts: *mut line_sender_opts) { - upd_opts!(opts, tls, Tls::InsecureSkipVerify); -} - -/// Configure how long to wait for messages from the QuestDB server during -/// the TLS handshake and authentication process. -/// The default is 15 seconds. -#[no_mangle] -pub unsafe extern "C" fn line_sender_opts_read_timeout( - opts: *mut line_sender_opts, - timeout_millis: u64, -) { - let timeout = std::time::Duration::from_millis(timeout_millis); - upd_opts!(opts, read_timeout, timeout); -} - -/// Duplicate the opts object. -/// Both old and new objects will have to be freed. -#[no_mangle] -pub unsafe extern "C" fn line_sender_opts_clone( - opts: *const line_sender_opts, -) -> *mut line_sender_opts { - let builder = &(*opts).0; - let new_builder = builder.clone(); - Box::into_raw(Box::new(line_sender_opts(new_builder))) -} - -/// Release the opts object. -#[no_mangle] -pub unsafe extern "C" fn line_sender_opts_free(opts: *mut line_sender_opts) { - if !opts.is_null() { - drop(Box::from_raw(opts)); - } -} - /// Prepare rows for sending via the line sender's `flush` function. /// Buffer objects are re-usable and cleared automatically when flushing. pub struct line_sender_buffer(Buffer); @@ -565,7 +556,7 @@ pub unsafe extern "C" fn line_sender_buffer_with_max_name_len( Box::into_raw(Box::new(line_sender_buffer(buffer))) } -/// Release the buffer object. +/// Release the `line_sender_buffer` object. #[no_mangle] pub unsafe extern "C" fn line_sender_buffer_free(buffer: *mut line_sender_buffer) { if !buffer.is_null() { @@ -657,6 +648,23 @@ pub unsafe extern "C" fn line_sender_buffer_size(buffer: *const line_sender_buff buffer.len() } +/// The number of rows accumulated in the buffer. +#[no_mangle] +pub unsafe extern "C" fn line_sender_buffer_row_count(buffer: *const line_sender_buffer) -> size_t { + let buffer = unwrap_buffer(buffer); + buffer.row_count() +} + +/// The buffer is transactional if sent over HTTP. +/// A buffer stops being transactional if it contains rows for multiple tables. +#[no_mangle] +pub unsafe extern "C" fn line_sender_buffer_transactional( + buffer: *const line_sender_buffer, +) -> bool { + let buffer = unwrap_buffer(buffer); + buffer.transactional() +} + /// Peek into the accumulated buffer that is to be sent out at the next `flush`. /// /// @param[in] buffer Line buffer object. @@ -883,21 +891,325 @@ pub unsafe extern "C" fn line_sender_buffer_at_now( true } +/// Accumulates parameters for creating a line sender connection. +pub struct line_sender_opts(SenderBuilder); + +/// Create a new `line_sender_opts` instance from configuration string. +/// The format of the string is: "tcp::addr=host:port;key=value;...;" +/// Alongside "tcp" you can also specify "tcps", "http", and "https". +/// The accepted set of keys and values is the same as for the opt's API. +/// E.g. "https::addr=host:port;username=alice;password=secret;tls_ca=os_roots;" +#[no_mangle] +pub unsafe extern "C" fn line_sender_opts_from_conf( + config: line_sender_utf8, + err_out: *mut *mut line_sender_error, +) -> *mut line_sender_opts { + let config = config.as_str(); + let builder = bubble_err_to_c!(err_out, SenderBuilder::from_conf(config), ptr::null_mut()); + Box::into_raw(Box::new(line_sender_opts(builder))) +} + +/// Create a new `line_sender_opts` instance from configuration string read +/// from the `QDB_CLIENT_CONF` environment variable. +#[no_mangle] +pub unsafe extern "C" fn line_sender_opts_from_env( + err_out: *mut *mut line_sender_error, +) -> *mut line_sender_opts { + let builder = bubble_err_to_c!(err_out, SenderBuilder::from_env(), ptr::null_mut()); + Box::into_raw(Box::new(line_sender_opts(builder))) +} + +/// A new set of options for a line sender connection for ILP/TCP. +/// @param[in] protocol The protocol to use. +/// @param[in] host The QuestDB database host. +/// @param[in] port The QuestDB ILP TCP port. +#[no_mangle] +pub unsafe extern "C" fn line_sender_opts_new( + protocol: line_sender_protocol, + host: line_sender_utf8, + port: u16, +) -> *mut line_sender_opts { + let builder = SenderBuilder::new(protocol.into(), host.as_str(), port); + let builder = builder + .user_agent(concat!("questdb/c/", env!("CARGO_PKG_VERSION"))) + .expect("user_agent set"); + Box::into_raw(Box::new(line_sender_opts(builder))) +} + +/// Variant of line_sender_opts_new that takes a service name for port. +#[no_mangle] +pub unsafe extern "C" fn line_sender_opts_new_service( + protocol: line_sender_protocol, + host: line_sender_utf8, + port: line_sender_utf8, +) -> *mut line_sender_opts { + let builder = SenderBuilder::new(protocol.into(), host.as_str(), port.as_str()); + let builder = builder + .user_agent(concat!("questdb/c/", env!("CARGO_PKG_VERSION"))) + .expect("user_agent set"); + Box::into_raw(Box::new(line_sender_opts(builder))) +} + +/// Select local outbound network "bind" interface. +/// +/// This may be relevant if your machine has multiple network interfaces. +/// +/// The default is `0.0.0.0`. +#[no_mangle] +pub unsafe extern "C" fn line_sender_opts_bind_interface( + opts: *mut line_sender_opts, + bind_interface: line_sender_utf8, + err_out: *mut *mut line_sender_error, +) -> bool { + upd_opts!(opts, err_out, bind_interface, bind_interface.as_str()) +} + +/// Set the username for authentication. +/// +/// For TCP this is the `kid` part of the ECDSA key set. +/// The other fields are `token` `token_x` and `token_y`. +/// +/// For HTTP this is part of basic authentication. +/// Also see `password`. +#[no_mangle] +pub unsafe extern "C" fn line_sender_opts_username( + opts: *mut line_sender_opts, + username: line_sender_utf8, + err_out: *mut *mut line_sender_error, +) -> bool { + upd_opts!(opts, err_out, username, username.as_str()) +} + +/// Set the password for basic HTTP authentication. +/// Also see `username`. +#[no_mangle] +pub unsafe extern "C" fn line_sender_opts_password( + opts: *mut line_sender_opts, + password: line_sender_utf8, + err_out: *mut *mut line_sender_error, +) -> bool { + upd_opts!(opts, err_out, password, password.as_str()) +} + +/// Token (Bearer) Authentication Parameters for ILP over HTTP, +/// or the ECDSA private key for ILP over TCP authentication. +#[no_mangle] +pub unsafe extern "C" fn line_sender_opts_token( + opts: *mut line_sender_opts, + token: line_sender_utf8, + err_out: *mut *mut line_sender_error, +) -> bool { + upd_opts!(opts, err_out, token, token.as_str()) +} + +/// The ECDSA public key X for ILP over TCP authentication. +#[no_mangle] +pub unsafe extern "C" fn line_sender_opts_token_x( + opts: *mut line_sender_opts, + token_x: line_sender_utf8, + err_out: *mut *mut line_sender_error, +) -> bool { + upd_opts!(opts, err_out, token_x, token_x.as_str()) +} + +/// The ECDSA public key Y for ILP over TCP authentication. +#[no_mangle] +pub unsafe extern "C" fn line_sender_opts_token_y( + opts: *mut line_sender_opts, + token_y: line_sender_utf8, + err_out: *mut *mut line_sender_error, +) -> bool { + upd_opts!(opts, err_out, token_y, token_y.as_str()) +} + +/// Configure how long to wait for messages from the QuestDB server during +/// the TLS handshake and authentication process. +/// The default is 15000 milliseconds. +#[no_mangle] +pub unsafe extern "C" fn line_sender_opts_auth_timeout( + opts: *mut line_sender_opts, + timeout_millis: u64, + err_out: *mut *mut line_sender_error, +) -> bool { + let timeout = std::time::Duration::from_millis(timeout_millis); + upd_opts!(opts, err_out, auth_timeout, timeout) +} + +/// Set to `false` to disable TLS certificate verification. +/// This should only be used for debugging purposes as it reduces security. +/// +/// For testing consider specifying a path to a `.pem` file instead via +/// the `tls_roots` setting. +#[no_mangle] +pub unsafe extern "C" fn line_sender_opts_tls_verify( + opts: *mut line_sender_opts, + verify: bool, + err_out: *mut *mut line_sender_error, +) -> bool { + upd_opts!(opts, err_out, tls_verify, verify) +} + +/// Specify where to find the certificate authority used to validate +/// the validate the server's TLS certificate. +#[no_mangle] +pub unsafe extern "C" fn line_sender_opts_tls_ca( + opts: *mut line_sender_opts, + ca: line_sender_ca, + err_out: *mut *mut line_sender_error, +) -> bool { + let ca: CertificateAuthority = ca.into(); + upd_opts!(opts, err_out, tls_ca, ca) +} + +/// Set the path to a custom root certificate `.pem` file. +/// This is used to validate the server's certificate during the TLS handshake. +/// +/// See notes on how to test with [self-signed certificates](https://github.com/questdb/c-questdb-client/tree/main/tls_certs). +#[no_mangle] +pub unsafe extern "C" fn line_sender_opts_tls_roots( + opts: *mut line_sender_opts, + path: line_sender_utf8, + err_out: *mut *mut line_sender_error, +) -> bool { + let path = PathBuf::from(path.as_str()); + upd_opts!(opts, err_out, tls_roots, path) +} + +/// The maximum buffer size that the client will flush to the server. +/// The default is 100 MiB. +#[no_mangle] +pub unsafe extern "C" fn line_sender_opts_max_buf_size( + opts: *mut line_sender_opts, + max_buf_size: size_t, + err_out: *mut *mut line_sender_error, +) -> bool { + upd_opts!(opts, err_out, max_buf_size, max_buf_size) +} + +/// Cumulative duration spent in retries. +/// The default is 10 seconds. +#[no_mangle] +pub unsafe extern "C" fn line_sender_opts_retry_timeout( + opts: *mut line_sender_opts, + millis: u64, + err_out: *mut *mut line_sender_error, +) -> bool { + let retry_timeout = std::time::Duration::from_millis(millis); + upd_opts!(opts, err_out, retry_timeout, retry_timeout) +} + +/// Minimum expected throughput in bytes per second for HTTP requests. +/// If the throughput is lower than this value, the connection will time out. +/// The default is 100 KiB/s. +/// The value is expressed as a number of bytes per second. +#[no_mangle] +pub unsafe extern "C" fn line_sender_opts_request_min_throughput( + opts: *mut line_sender_opts, + bytes_per_sec: u64, + err_out: *mut *mut line_sender_error, +) -> bool { + upd_opts!(opts, err_out, request_min_throughput, bytes_per_sec) +} + +/// Grace request timeout before relying on the minimum throughput logic. +/// The default is 5 seconds. +#[no_mangle] +pub unsafe extern "C" fn line_sender_opts_request_timeout( + opts: *mut line_sender_opts, + millis: u64, + err_out: *mut *mut line_sender_error, +) -> bool { + let request_timeout = std::time::Duration::from_millis(millis); + upd_opts!(opts, err_out, request_timeout, request_timeout) +} + +/// Set the HTTP user agent. Internal API. Do not use. +#[doc(hidden)] +#[no_mangle] +pub unsafe extern "C" fn line_sender_opts_user_agent( + opts: *mut line_sender_opts, + user_agent: line_sender_utf8, + err_out: *mut *mut line_sender_error, +) -> bool { + upd_opts!(opts, err_out, user_agent, user_agent.as_str()) +} + +/// Duplicate the `line_sender_opts` object. +/// Both old and new objects will have to be freed. +#[no_mangle] +pub unsafe extern "C" fn line_sender_opts_clone( + opts: *const line_sender_opts, +) -> *mut line_sender_opts { + let builder = &(*opts).0; + let new_builder = builder.clone(); + Box::into_raw(Box::new(line_sender_opts(new_builder))) +} + +/// Release the `line_sender_opts` object. +#[no_mangle] +pub unsafe extern "C" fn line_sender_opts_free(opts: *mut line_sender_opts) { + if !opts.is_null() { + drop(Box::from_raw(opts)); + } +} + /// Insert data into QuestDB via the InfluxDB Line Protocol. /// /// Batch up rows in `buffer` objects, then call `flush` to send them. pub struct line_sender(Sender); -/// Synchronously connect to the QuestDB database. +/// Build the line sender. +/// +/// In case of TCP, this synchronously establishes the TCP connection, and +/// returns once the connection is fully established. If the connection +/// requires authentication or TLS, these will also be completed before +/// returning. +/// /// The connection should be accessed by only a single thread a time. +/// /// @param[in] opts Options for the connection. #[no_mangle] -pub unsafe extern "C" fn line_sender_connect( +pub unsafe extern "C" fn line_sender_build( opts: *const line_sender_opts, err_out: *mut *mut line_sender_error, ) -> *mut line_sender { let builder = &(*opts).0; - let sender = bubble_err_to_c!(err_out, builder.connect(), ptr::null_mut()); + let sender = bubble_err_to_c!(err_out, builder.build(), ptr::null_mut()); + Box::into_raw(Box::new(line_sender(sender))) +} + +/// Create a new `line_sender` instance from configuration string. +/// The format of the string is: "tcp::addr=host:port;key=value;...;" +/// Alongside "tcp" you can also specify "tcps", "http", and "https". +/// The accepted set of keys and values is the same as for the opt's API. +/// E.g. "https::addr=host:port;username=alice;password=secret;tls_ca=os_roots;" +/// +/// For the full list of keys, search this file for `fn line_sender_opts_`. +#[no_mangle] +pub unsafe extern "C" fn line_sender_from_conf( + config: line_sender_utf8, + err_out: *mut *mut line_sender_error, +) -> *mut line_sender { + let config = config.as_str(); + let builder = bubble_err_to_c!(err_out, SenderBuilder::from_conf(config), ptr::null_mut()); + let builder = builder + .user_agent(concat!("questdb/c/", env!("CARGO_PKG_VERSION"))) + .expect("user_agent set"); + let sender = bubble_err_to_c!(err_out, builder.build(), ptr::null_mut()); + Box::into_raw(Box::new(line_sender(sender))) +} + +/// Create a new `line_sender` instance from configuration string read from the +/// `QDB_CLIENT_CONF` environment variable. +#[no_mangle] +pub unsafe extern "C" fn line_sender_from_env( + err_out: *mut *mut line_sender_error, +) -> *mut line_sender { + let builder = bubble_err_to_c!(err_out, SenderBuilder::from_env(), ptr::null_mut()); + let builder = builder + .user_agent(concat!("questdb/c/", env!("CARGO_PKG_VERSION"))) + .expect("user_agent set"); + let sender = bubble_err_to_c!(err_out, builder.build(), ptr::null_mut()); Box::into_raw(Box::new(line_sender(sender))) } @@ -965,6 +1277,31 @@ pub unsafe extern "C" fn line_sender_flush_and_keep( true } +/// Variant of `.flush()` that does not clear the buffer and allows for +/// transactional flushes. +/// +/// A transactional flush is simply a flush that ensures that all rows in +/// the ILP buffer refer to the same table, thus allowing the server to +/// treat the flush request as a single transaction. +/// +/// This is because QuestDB does not support transactions spanning multiple +/// tables. +#[no_mangle] +pub unsafe extern "C" fn line_sender_flush_and_keep_with_flags( + sender: *mut line_sender, + buffer: *const line_sender_buffer, + transactional: bool, + err_out: *mut *mut line_sender_error, +) -> bool { + let sender = unwrap_sender_mut(sender); + let buffer = unwrap_buffer(buffer); + bubble_err_to_c!( + err_out, + sender.flush_and_keep_with_flags(buffer, transactional) + ); + true +} + /// Get the current time in nanoseconds since the unix epoch (UTC). #[no_mangle] pub unsafe extern "C" fn line_sender_now_nanos() -> i64 { @@ -976,3 +1313,16 @@ pub unsafe extern "C" fn line_sender_now_nanos() -> i64 { pub unsafe extern "C" fn line_sender_now_micros() -> i64 { TimestampMicros::now().as_i64() } + +#[cfg(feature = "confstr-ffi")] +use questdb_confstr_ffi::questdb_conf_str_parse_err; + +#[cfg(feature = "confstr-ffi")] +/// A build system hack. +/// Without this, the `questdb-confstr-ffi` crate dependency is not +/// included in the final binary. +/// This is because otherwise `cargo` will optimise out the dependency. +pub unsafe fn _build_system_hack(err: *mut questdb_conf_str_parse_err) { + use questdb_confstr_ffi::questdb_conf_str_parse_err_free; + questdb_conf_str_parse_err_free(err); +} diff --git a/questdb-rs/Cargo.toml b/questdb-rs/Cargo.toml index 4114121c..fdb9dec4 100644 --- a/questdb-rs/Cargo.toml +++ b/questdb-rs/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "questdb-rs" -version = "3.1.0" +version = "4.0.0" edition = "2021" license = "Apache-2.0" description = "QuestDB Client Library for Rust" @@ -10,40 +10,52 @@ keywords = ["questdb", "ilp", "client-library"] categories = ["database"] authors = ["Adam Cimarosti "] +[package.metadata.docs.rs] +all-features = true + [lib] name = "questdb" crate-type = ["lib"] [dependencies] libc = "0.2" -socket2 = "0.5.4" -dns-lookup = "2.0.2" +socket2 = "0.5.5" +dns-lookup = "2.0.4" base64ct = { version = "1.6.0", features = ["alloc"] } -ring = "0.17.5" -rustls = "0.21.7" -rustls-pemfile = "1.0.3" -webpki-roots = { version = "0.25.2", optional = true } +rustls-pemfile = "2.0.0" ryu = "1.0.15" itoa = "1.0.9" +ring = "0.17.5" +rustls-pki-types = "1.0.1" +rustls = "0.22.0" +rustls-native-certs = { version = "0.7.0", optional = true } +webpki-roots = { version = "0.26.0", optional = true } chrono = { version = "0.4.30", optional = true } -rustls-native-certs = { version = "0.6.3", optional = true } +ureq = { version = "2.9.4", optional = true } +serde_json = { version = "1.0.108", optional = true } +questdb-confstr = "0.1.0" +rand = { version = "0.8.5", optional = true } [target.'cfg(windows)'.dependencies] winapi = { version = "0.3.9", features = ["ws2def"] } [build-dependencies] -serde_json = { version = "1.0.107" } -serde = { version = "1.0.188", features = ["derive"] } +serde_json = { version = "1.0.108" } +serde = { version = "1.0.193", features = ["derive"] } slugify = "0.1.0" -indoc = "2.0.3" +indoc = "2.0.4" [dev-dependencies] -mio = { version = "0.8.8", features = ["os-poll", "net"] } -chrono = "0.4.30" +mio = { version = "0.8.10", features = ["os-poll", "net"] } +chrono = "0.4.31" +tempfile = "3.2.0" [features] default = ["tls-webpki-certs"] +# Include support for ILP over HTTP. +ilp-over-http = ["dep:ureq", "dep:serde_json", "dep:rand"] + # Allow use OS-provided root TLS certificates tls-native-certs = ["dep:rustls-native-certs"] @@ -51,7 +63,7 @@ tls-native-certs = ["dep:rustls-native-certs"] tls-webpki-certs = ["dep:webpki-roots"] # Allow skipping verification of insecure certificates. -insecure-skip-verify = ["rustls/dangerous_configuration"] +insecure-skip-verify = [] # Enable code-generation in `build.rs` for additional tests. json_tests = [] @@ -70,3 +82,7 @@ required-features = ["chrono_timestamp"] [[example]] name = "auth_tls" required-features = ["chrono_timestamp"] + +[[example]] +name = "http" +required-features = ["ilp-over-http"] diff --git a/questdb-rs/README.md b/questdb-rs/README.md index e8f9ad3c..290a59cb 100644 --- a/questdb-rs/README.md +++ b/questdb-rs/README.md @@ -1,17 +1,24 @@ # QuestDB Client Library for Rust +Official Rust client for [QuestDB](https://questdb.io/), an open-source SQL database designed to process time-series data, faster. + +The client library is designed for fast ingestion of data into QuestDB via the InfluxDB Line Protocol (ILP). + +* [QuestDB Database docs](https://questdb.io/docs/) +* [ILP docs](https://questdb.io/docs/reference/api/ilp/overview/) + ## Getting Started To start using `questdb-rs` add it to your `Cargo.toml`: ```toml [dependencies] -questdb-rs = "3.1.0" +questdb-rs = "4.0.0" ``` ## Docs -See documentation for the [`ingress`](https://docs.rs/questdb-rs/3.1.0/questdb/ingress/) module to insert data into QuestDB via the ILP protocol. +See documentation for the [`ingress`](https://docs.rs/questdb-rs/4.0.0/questdb/ingress/) module to insert data into QuestDB via the ILP protocol. * Latest API docs: [https://docs.rs/questdb-rs/latest/](https://docs.rs/questdb-rs/latest/) @@ -23,11 +30,10 @@ use questdb::{ ingress::{ Sender, Buffer, - SenderBuilder, TimestampNanos}}; fn main() -> Result<()> { - let mut sender = SenderBuilder::new("localhost", 9009).connect()?; + let mut sender = Sender::from_conf("http::addr=localhost:9000;")?; let mut buffer = Buffer::new(); buffer .table("sensors")? @@ -40,6 +46,31 @@ fn main() -> Result<()> { } ``` +## Crate features + +This Rust crate supports a number of optional features. + +For example, if you want to work with ILP/HTTP and work with Chrono timestamps, +use: + +```bash +cargo add questdb-rs --features ilp-over-http chrono +``` + +### Default-enabled features + +* `tls-webpki-certs`: Use the `webpki-roots` crate for TLS cert verification. + +### Optional features + +These features are opt-in as they bring in additional downstream dependencies. + +* `ilp-over-http`: Enables ILP/HTTP support via the `ureq` crate. +* `tls-native-certs`: Supports validating TLS certificates against the OS's + certificates store. +* `insecure-skip-verify`: Allows skipping TLS validation. +* `chrono_timestamp`: Allows specifying timestamps as `chrono::Datetime` objects. + ## C, C++ and Python APIs This crate is also exposed as a C and C++ API and in turn exposed to Python. diff --git a/questdb-rs/examples/auth.rs b/questdb-rs/examples/auth.rs index 175d6e5a..642d2ffd 100644 --- a/questdb-rs/examples/auth.rs +++ b/questdb-rs/examples/auth.rs @@ -1,24 +1,22 @@ use chrono::{TimeZone, Utc}; use questdb::{ - ingress::{Buffer, SenderBuilder, TimestampNanos}, + ingress::{Buffer, Sender, TimestampNanos}, Result, }; fn main() -> Result<()> { let host: String = std::env::args().nth(1).unwrap_or("localhost".to_string()); - let port: u16 = std::env::args() - .nth(2) - .unwrap_or("9009".to_string()) - .parse() - .unwrap(); - let mut sender = SenderBuilder::new(host, port) - .auth( - "testUser1", // kid - "5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48", // d - "fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU", // x - "Dt5tbS1dEDMSYfym3fgMv0B99szno-dFc1rYF9t0aac", // y - ) - .connect()?; + let port: &str = &std::env::args().nth(2).unwrap_or("9009".to_string()); + let mut sender = Sender::from_conf(format!( + concat!( + "tcp::addr={}:{};", + "username=testUser1;", + "token=5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48;", + "token_x=fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU;", + "token_y=Dt5tbS1dEDMSYfym3fgMv0B99szno-dFc1rYF9t0aac;" + ), + host, port + ))?; let mut buffer = Buffer::new(); let designated_timestamp = TimestampNanos::from_datetime(Utc.with_ymd_and_hms(1997, 7, 4, 4, 56, 55).unwrap())?; diff --git a/questdb-rs/examples/auth_tls.rs b/questdb-rs/examples/auth_tls.rs index d30f9e7b..cb2132eb 100644 --- a/questdb-rs/examples/auth_tls.rs +++ b/questdb-rs/examples/auth_tls.rs @@ -1,26 +1,22 @@ use chrono::{TimeZone, Utc}; use questdb::{ - ingress::{Buffer, CertificateAuthority, SenderBuilder, TimestampNanos, Tls}, + ingress::{Buffer, Sender, TimestampNanos}, Result, }; fn main() -> Result<()> { let host: String = std::env::args().nth(1).unwrap_or("localhost".to_string()); - let port: u16 = std::env::args() - .nth(2) - .unwrap_or("9009".to_string()) - .parse() - .unwrap(); - let mut sender = SenderBuilder::new(host, port) - .auth( - "testUser1", // kid - "5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48", // d - "fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU", // x - "Dt5tbS1dEDMSYfym3fgMv0B99szno-dFc1rYF9t0aac", // y - ) - .tls(Tls::Enabled(CertificateAuthority::WebpkiRoots)) - // Alternatively: .tls(Tls::Enabled(CertificateAuthority::OsRoots)) - .connect()?; + let port: &str = &std::env::args().nth(2).unwrap_or("9009".to_string()); + let mut sender = Sender::from_conf(format!( + concat!( + "tcps::addr={}:{};", + "username=testUser1;", + "token=5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48;", + "token_x=fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU;", + "token_y=Dt5tbS1dEDMSYfym3fgMv0B99szno-dFc1rYF9t0aac;", + ), + host, port + ))?; let mut buffer = Buffer::new(); let designated_timestamp = TimestampNanos::from_datetime(Utc.with_ymd_and_hms(1997, 7, 4, 4, 56, 55).unwrap())?; diff --git a/questdb-rs/examples/basic.rs b/questdb-rs/examples/basic.rs index 1dd25ae0..336fbf2c 100644 --- a/questdb-rs/examples/basic.rs +++ b/questdb-rs/examples/basic.rs @@ -1,17 +1,13 @@ use chrono::{TimeZone, Utc}; use questdb::{ - ingress::{Buffer, SenderBuilder, TimestampNanos}, + ingress::{Buffer, Sender, TimestampNanos}, Result, }; fn main() -> Result<()> { let host: String = std::env::args().nth(1).unwrap_or("localhost".to_string()); - let port: u16 = std::env::args() - .nth(2) - .unwrap_or("9009".to_string()) - .parse() - .unwrap(); - let mut sender = SenderBuilder::new(host, port).connect()?; + let port: &str = &std::env::args().nth(2).unwrap_or("9009".to_string()); + let mut sender = Sender::from_conf(format!("tcp::addr={host}:{port};"))?; let mut buffer = Buffer::new(); let designated_timestamp = TimestampNanos::from_datetime(Utc.with_ymd_and_hms(1997, 7, 4, 4, 56, 55).unwrap())?; diff --git a/questdb-rs/examples/from_conf.rs b/questdb-rs/examples/from_conf.rs new file mode 100644 index 00000000..524da8be --- /dev/null +++ b/questdb-rs/examples/from_conf.rs @@ -0,0 +1,17 @@ +use questdb::{ + ingress::{Buffer, Sender, TimestampNanos}, + Result, +}; + +fn main() -> Result<()> { + let mut sender = Sender::from_conf("tcp::addr=localhost:9009;")?; + let mut buffer = Buffer::new(); + buffer + .table("sensors")? + .symbol("id", "toronto1")? + .column_f64("temperature", 20.0)? + .column_i64("humidity", 50)? + .at(TimestampNanos::now())?; + sender.flush(&mut buffer)?; + Ok(()) +} diff --git a/questdb-rs/examples/from_env.rs b/questdb-rs/examples/from_env.rs new file mode 100644 index 00000000..a3dbf768 --- /dev/null +++ b/questdb-rs/examples/from_env.rs @@ -0,0 +1,18 @@ +use questdb::{ + ingress::{Buffer, Sender, TimestampNanos}, + Result, +}; + +fn main() -> Result<()> { + // Read configuration string from the `QDB_CLIENT_CONF` environment variable. + let mut sender = Sender::from_env()?; + let mut buffer = Buffer::new(); + buffer + .table("sensors")? + .symbol("id", "toronto1")? + .column_f64("temperature", 20.0)? + .column_i64("humidity", 50)? + .at(TimestampNanos::now())?; + sender.flush(&mut buffer)?; + Ok(()) +} diff --git a/questdb-rs/examples/http.rs b/questdb-rs/examples/http.rs new file mode 100644 index 00000000..3eb72d6f --- /dev/null +++ b/questdb-rs/examples/http.rs @@ -0,0 +1,17 @@ +use questdb::{ + ingress::{Buffer, Sender, TimestampNanos}, + Result, +}; + +fn main() -> Result<()> { + let mut sender = Sender::from_conf("https::addr=localhost:9000;username=foo;password=bar;")?; + let mut buffer = Buffer::new(); + buffer + .table("sensors")? + .symbol("id", "toronto1")? + .column_f64("temperature", 20.0)? + .column_i64("humidity", 50)? + .at(TimestampNanos::now())?; + sender.flush(&mut buffer)?; + Ok(()) +} diff --git a/questdb-rs/src/error.rs b/questdb-rs/src/error.rs index d7e2cb48..6848280a 100644 --- a/questdb-rs/src/error.rs +++ b/questdb-rs/src/error.rs @@ -39,6 +39,15 @@ pub enum ErrorCode { /// Error during TLS handshake. TlsError, + + /// The server does not support ILP-over-HTTP. + HttpNotSupported, + + /// Error sent back from the server during flush. + ServerFlushError, + + /// Bad configuration. + ConfigError, } /// An error that occurred when using QuestDB client library. diff --git a/questdb-rs/src/gai.rs b/questdb-rs/src/gai.rs index b7161849..0270b69e 100644 --- a/questdb-rs/src/gai.rs +++ b/questdb-rs/src/gai.rs @@ -6,7 +6,7 @@ * \__\_\\__,_|\___||___/\__|____/|____/ * * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2023 QuestDB + * Copyright (c) 2019-2024 QuestDB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/questdb-rs/src/ingress/conf.rs b/questdb-rs/src/ingress/conf.rs new file mode 100644 index 00000000..011d8620 --- /dev/null +++ b/questdb-rs/src/ingress/conf.rs @@ -0,0 +1,75 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2024 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +use std::ops::Deref; + +use crate::{Error, ErrorCode, Result}; + +/// Wraps a SenderBuilder config setting with the intent of tracking +/// whether the value was user-specified or defaulted. +/// This helps the builder API ensure that a user-specified value can't +/// be changed once set. +#[derive(Debug, Clone)] +pub(crate) enum ConfigSetting { + Defaulted(T), + Specified(T), +} + +impl ConfigSetting { + pub(crate) fn new_default(value: T) -> Self { + ConfigSetting::Defaulted(value) + } + + pub(crate) fn new_specified(value: T) -> Self { + ConfigSetting::Specified(value) + } + + /// Set the user-defined value. + /// Note that it can't be changed once set. + /// If the value is already specified, returns an error. + pub(crate) fn set_specified(&mut self, setting_name: &str, value: T) -> Result<()> { + match self { + ConfigSetting::Defaulted(_) => { + *self = ConfigSetting::Specified(value); + Ok(()) + } + ConfigSetting::Specified(curr_value) if *curr_value == value => Ok(()), + _ => Err(Error::new( + ErrorCode::ConfigError, + format!("{setting_name:?} is already specified"), + )), + } + } +} + +impl Deref for ConfigSetting { + type Target = T; + + fn deref(&self) -> &Self::Target { + match self { + ConfigSetting::Defaulted(v) => v, + ConfigSetting::Specified(v) => v, + } + } +} diff --git a/questdb-rs/src/ingress/http.rs b/questdb-rs/src/ingress/http.rs new file mode 100644 index 00000000..177c87ed --- /dev/null +++ b/questdb-rs/src/ingress/http.rs @@ -0,0 +1,235 @@ +use crate::{error, Error}; +use base64ct::Base64; +use base64ct::Encoding; +use rand::Rng; +use std::fmt::Write; +use std::thread::sleep; +use std::time::Duration; + +use super::conf::ConfigSetting; + +#[derive(PartialEq, Debug, Clone)] +pub(super) struct BasicAuthParams { + pub(super) username: String, + pub(super) password: String, +} + +impl BasicAuthParams { + pub(super) fn to_header_string(&self) -> String { + let pair = format!("{}:{}", self.username, self.password); + let encoded = Base64::encode_string(pair.as_bytes()); + format!("Basic {encoded}") + } +} + +#[derive(PartialEq, Debug, Clone)] +pub(super) struct TokenAuthParams { + pub(super) token: String, +} + +impl TokenAuthParams { + pub(super) fn to_header_string(&self) -> crate::Result { + if self.token.contains('\n') { + return Err(error::fmt!( + AuthError, + "Bad auth token: Should not contain new-line char." + )); + } + Ok(format!("Bearer {}", self.token)) + } +} + +#[derive(Debug, Clone)] +pub(super) struct HttpConfig { + pub(super) request_min_throughput: ConfigSetting, + pub(super) user_agent: String, + pub(super) retry_timeout: ConfigSetting, + pub(super) request_timeout: ConfigSetting, +} + +impl Default for HttpConfig { + fn default() -> Self { + Self { + request_min_throughput: ConfigSetting::new_default(102400), // 100 KiB/s + user_agent: concat!("questdb/rust/", env!("CARGO_PKG_VERSION")).to_string(), + retry_timeout: ConfigSetting::new_default(Duration::from_secs(10)), + request_timeout: ConfigSetting::new_default(Duration::from_secs(10)), + } + } +} + +pub(super) struct HttpHandlerState { + /// Maintains a pool of open HTTP connections to the endpoint. + pub(super) agent: ureq::Agent, + + /// The URL of the HTTP endpoint. + pub(super) url: String, + + /// The content of the `Authorization` HTTP header. + pub(super) auth: Option, + + /// HTTP params configured via the `SenderBuilder`. + pub(super) config: HttpConfig, +} + +pub(super) fn parse_json_error(json: &serde_json::Value, msg: &str) -> Error { + let mut description = msg.to_string(); + error::fmt!(ServerFlushError, "Could not flush buffer: {}", msg); + + let error_id = json.get("errorId").and_then(|v| v.as_str()); + let code = json.get("code").and_then(|v| v.as_str()); + let line = json.get("line").and_then(|v| v.as_i64()); + + let mut printed_detail = false; + if error_id.is_some() || code.is_some() || line.is_some() { + description.push_str(" ["); + + if let Some(error_id) = error_id { + description.push_str("id: "); + description.push_str(error_id); + printed_detail = true; + } + + if let Some(code) = code { + if printed_detail { + description.push_str(", "); + } + description.push_str("code: "); + description.push_str(code); + printed_detail = true; + } + + if let Some(line) = line { + if printed_detail { + description.push_str(", "); + } + description.push_str("line: "); + write!(description, "{}", line).unwrap(); + } + + description.push(']'); + } + + error::fmt!(ServerFlushError, "Could not flush buffer: {}", description) +} + +pub(super) fn parse_http_error(http_status_code: u16, response: ureq::Response) -> Error { + if http_status_code == 404 { + return error::fmt!( + HttpNotSupported, + "Could not flush buffer: HTTP endpoint does not support ILP." + ); + } else if [401, 403].contains(&http_status_code) { + let description = match response.into_string() { + Ok(msg) if !msg.is_empty() => format!(": {}", msg), + _ => "".to_string(), + }; + return error::fmt!( + AuthError, + "Could not flush buffer: HTTP endpoint authentication error{} [code: {}]", + description, + http_status_code + ); + } + + let is_json = response + .content_type() + .eq_ignore_ascii_case("application/json"); + match response.into_string() { + Ok(msg) => { + let string_err = || error::fmt!(ServerFlushError, "Could not flush buffer: {}", msg); + + if !is_json { + return string_err(); + } + + let json: serde_json::Value = match serde_json::from_str(&msg) { + Ok(json) => json, + Err(_) => { + return string_err(); + } + }; + + return if let Some(serde_json::Value::String(ref msg)) = json.get("message") { + parse_json_error(&json, msg) + } else { + string_err() + }; + } + Err(err) => { + error::fmt!(SocketError, "Could not flush buffer: {}", err) + } + } +} + +pub(super) fn is_retriable_error(err: &ureq::Error) -> bool { + use ureq::Error::*; + match err { + Transport(_) => true, + + // Official HTTP codes + Status(500, _) | // Internal Server Error + Status(503, _) | // Service Unavailable + Status(504, _) | // Gateway Timeout + + // Unofficial extensions + Status(507, _) | // Insufficient Storage + Status(509, _) | // Bandwidth Limit Exceeded + Status(523, _) | // Origin is Unreachable + Status(524, _) | // A Timeout Occurred + Status(529, _) | // Site is overloaded + Status(599, _) => { // Network Connect Timeout Error + true + } + _ => false + } +} + +#[allow(clippy::result_large_err)] // `ureq::Error` is large enough to cause this warning. +fn retry_http_send( + request: ureq::Request, + buf: &[u8], + retry_timeout: Duration, + mut last_err: ureq::Error, +) -> Result { + let mut rng = rand::thread_rng(); + let retry_end = std::time::Instant::now() + retry_timeout; + let mut retry_interval_ms = 10; + loop { + let jitter_ms = rng.gen_range(-5i32..5); + let to_sleep_ms = retry_interval_ms + jitter_ms; + let to_sleep = Duration::from_millis(to_sleep_ms as u64); + if (std::time::Instant::now() + to_sleep) > retry_end { + return Err(last_err); + } + sleep(to_sleep); + last_err = match request.clone().send_bytes(buf) { + Ok(res) => return Ok(res), + Err(err) => { + if !is_retriable_error(&err) { + return Err(err); + } + err + } + }; + retry_interval_ms = (retry_interval_ms * 2).min(1000); + } +} + +#[allow(clippy::result_large_err)] // `ureq::Error` is large enough to cause this warning. +pub(super) fn http_send_with_retries( + request: ureq::Request, + buf: &[u8], + retry_timeout: Duration, +) -> Result { + let last_err = match request.clone().send_bytes(buf) { + Ok(res) => return Ok(res), + Err(err) => err, + }; + + if retry_timeout.is_zero() || !is_retriable_error(&last_err) { + return Err(last_err); + } + + retry_http_send(request, buf, retry_timeout, last_err) +} diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index c08620e5..1b40e4ca 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -6,7 +6,7 @@ * \__\_\\__,_|\___||___/\__|____/|____/ * * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2023 QuestDB + * Copyright (c) 2019-2024 QuestDB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -29,21 +29,20 @@ //! (ILP) over TCP. //! //! To get started: -//! * Connect to QuestDB by creating a [`Sender`] object. +//! * Connect to QuestDB by creating a [`Sender`] object, usually via [`Sender::from_conf`]. //! * Populate a [`Buffer`] with one or more rows of data. //! * Send the buffer via the Sender's [`flush`](Sender::flush) method. //! -//! ```no_run +//! ```rust no_run //! use questdb::{ //! Result, //! ingress::{ //! Sender, //! Buffer, -//! SenderBuilder, //! TimestampNanos}}; //! //! fn main() -> Result<()> { -//! let mut sender = SenderBuilder::new("localhost", 9009).connect()?; +//! let mut sender = Sender::from_conf("http::addr=localhost:9000;")?; //! let mut buffer = Buffer::new(); //! buffer //! .table("sensors")? @@ -78,31 +77,31 @@ //! # Connection Security Options //! //! To establish an [authenticated](https://questdb.io/docs/reference/api/ilp/authenticate) -//! and TLS-encrypted connection, call the SenderBuilder's -//! [`auth`](SenderBuilder::auth) and [`tls`](SenderBuilder::tls) methods. +//! and TLS-encrypted connection, call the SenderBuilder's authentication and tls methods. //! -//! Here's an example that uses full security: +//! Here's an example that uses full security with TCP: //! //! ```no_run //! # use questdb::Result; -//! use questdb::ingress::{SenderBuilder, Tls, CertificateAuthority}; +//! use questdb::ingress::{Protocol, SenderBuilder}; //! //! # fn main() -> Result<()> { //! // See: https://questdb.io/docs/reference/api/ilp/authenticate -//! let mut sender = SenderBuilder::new("localhost", 9009) -//! .auth( -//! "testUser1", // kid -//! "5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48", // d -//! "fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU", // x -//! "Dt5tbS1dEDMSYfym3fgMv0B99szno-dFc1rYF9t0aac") // y -//! .tls(Tls::Enabled(CertificateAuthority::WebpkiRoots)) -//! .connect()?; +//! let mut sender = SenderBuilder::new(Protocol::Tcps, "localhost", 9009) +//! .username("testUser1")? // kid +//! .token("5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48")? // d +//! .token_x("fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU")? // x +//! .token_y("Dt5tbS1dEDMSYfym3fgMv0B99szno-dFc1rYF9t0aac")? // y +//! .build()?; //! # Ok(()) //! # } //! ``` //! -//! Note that as of writing QuestDB does not natively support TLS encryption. -//! To use TLS use a TLS proxy such as [HAProxy](http://www.haproxy.org/). +//! Note that Open Source QuestDB does not natively support TLS +//! encryption (this is a QuestDB enterprise feature). +//! +//! To use TLS with QuestDB open source, use a TLS proxy such as +//! [HAProxy](http://www.haproxy.org/). //! //! For testing, you can use a self-signed certificate and key. //! @@ -114,13 +113,12 @@ //! ```no_run //! # use questdb::Result; //! use std::path::PathBuf; -//! use questdb::ingress::{SenderBuilder, Tls, CertificateAuthority}; +//! use questdb::ingress::{SenderBuilder, Protocol, CertificateAuthority}; //! //! # fn main() -> Result<()> { -//! let mut sender = SenderBuilder::new("localhost", 9009) -//! .tls(Tls::Enabled(CertificateAuthority::File( -//! PathBuf::from("/path/to/server_rootCA.pem")))) -//! .connect()?; +//! let mut sender = SenderBuilder::new(Protocol::Tcps, "localhost", 9009) +//! .tls_roots("/path/to/server_rootCA.pem")? +//! .build()?; //! # Ok(()) //! # } //! ``` @@ -186,19 +184,23 @@ pub use self::timestamp::*; use crate::error::{self, Error, Result}; use crate::gai; +use crate::ingress::conf::ConfigSetting; use core::time::Duration; -use itoa; -use std::convert::{Infallible, TryFrom, TryInto}; -use std::fmt::{Formatter, Write}; +use std::collections::HashMap; +use std::convert::Infallible; +use std::fmt::{Debug, Display, Formatter, Write}; use std::io::{self, BufRead, BufReader, ErrorKind, Write as IoWrite}; +use std::ops::Deref; use std::path::PathBuf; +use std::str::FromStr; use std::sync::Arc; use base64ct::{Base64, Base64UrlUnpadded, Encoding}; use ring::rand::SystemRandom; use ring::signature::{EcdsaKeyPair, ECDSA_P256_SHA256_FIXED_SIGNING}; -use rustls::{ClientConnection, OwnedTrustAnchor, RootCertStore, ServerName, StreamOwned}; -use socket2::{Domain, Protocol, SockAddr, Socket, Type}; +use rustls::{ClientConnection, RootCertStore, StreamOwned}; +use rustls_pki_types::ServerName; +use socket2::{Domain, Protocol as SockProtocol, SockAddr, Socket, Type}; #[derive(Debug, Copy, Clone)] enum Op { @@ -467,6 +469,77 @@ enum Connection { Tls(Box>), } +impl Connection { + fn send_key_id(&mut self, key_id: &str) -> Result<()> { + writeln!(self, "{}", key_id) + .map_err(|io_err| map_io_to_socket_err("Failed to send key_id: ", io_err))?; + Ok(()) + } + + fn read_challenge(&mut self) -> Result> { + let mut buf = Vec::new(); + let mut reader = BufReader::new(self); + reader.read_until(b'\n', &mut buf).map_err(|io_err| { + map_io_to_socket_err( + "Failed to read authentication challenge (timed out?): ", + io_err, + ) + })?; + if buf.last().copied().unwrap_or(b'\0') != b'\n' { + return Err(if buf.is_empty() { + error::fmt!( + AuthError, + concat!( + "Did not receive auth challenge. ", + "Is the database configured to require ", + "authentication?" + ) + ) + } else { + error::fmt!(AuthError, "Received incomplete auth challenge: {:?}", buf) + }); + } + buf.pop(); // b'\n' + Ok(buf) + } + + fn authenticate(&mut self, auth: &EcdsaAuthParams) -> Result<()> { + if auth.key_id.contains('\n') { + return Err(error::fmt!( + AuthError, + "Bad key id {:?}: Should not contain new-line char.", + auth.key_id + )); + } + let key_pair = parse_key_pair(auth)?; + self.send_key_id(auth.key_id.as_str())?; + let challenge = self.read_challenge()?; + let rng = SystemRandom::new(); + let signature = key_pair + .sign(&rng, &challenge[..]) + .map_err(|unspecified_err| { + error::fmt!(AuthError, "Failed to sign challenge: {}", unspecified_err) + })?; + let mut encoded_sig = Base64::encode_string(signature.as_ref()); + encoded_sig.push('\n'); + let buf = encoded_sig.as_bytes(); + if let Err(io_err) = self.write_all(buf) { + return Err(map_io_to_socket_err( + "Could not send signed challenge: ", + io_err, + )); + } + Ok(()) + } +} + +enum ProtocolHandler { + Socket(Connection), + + #[cfg(feature = "ilp-over-http")] + Http(HttpHandlerState), +} + impl io::Read for Connection { fn read(&mut self, buf: &mut [u8]) -> io::Result { match self { @@ -493,7 +566,7 @@ impl io::Write for Connection { } #[derive(Debug, Copy, Clone, PartialEq)] -enum State { +enum OpCase { Init = Op::Table as isize, TableWritten = Op::Symbol as isize | Op::Column as isize, SymbolWritten = Op::Symbol as isize | Op::Column as isize | Op::At as isize, @@ -501,16 +574,42 @@ enum State { MayFlushOrTable = Op::Flush as isize | Op::Table as isize, } -impl State { +impl OpCase { fn next_op_descr(self) -> &'static str { match self { - State::Init => "should have called `table` instead", - State::TableWritten => "should have called `symbol` or `column` instead", - State::SymbolWritten => "should have called `symbol`, `column` or `at` instead", - State::ColumnWritten => "should have called `column` or `at` instead", - State::MayFlushOrTable => "should have called `flush` or `table` instead", + OpCase::Init => "should have called `table` instead", + OpCase::TableWritten => "should have called `symbol` or `column` instead", + OpCase::SymbolWritten => "should have called `symbol`, `column` or `at` instead", + OpCase::ColumnWritten => "should have called `column` or `at` instead", + OpCase::MayFlushOrTable => "should have called `flush` or `table` instead", + } + } +} + +#[derive(Debug, Clone)] +struct BufferState { + op_case: OpCase, + row_count: usize, + first_table: Option, + transactional: bool, +} + +impl BufferState { + fn new() -> Self { + Self { + op_case: OpCase::Init, + row_count: 0, + first_table: None, + transactional: true, } } + + fn clear(&mut self) { + self.op_case = OpCase::Init; + self.row_count = 0; + self.first_table = None; + self.transactional = true; + } } /// A reusable buffer to prepare ILP messages. @@ -598,11 +697,11 @@ impl State { /// [`rewind_to_marker`](Buffer::rewind_to_marker) method to go back to the /// marked last known good state. /// -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone)] pub struct Buffer { - state: State, output: String, - marker: Option<(usize, State)>, + state: BufferState, + marker: Option<(usize, BufferState)>, max_name_len: usize, } @@ -611,8 +710,8 @@ impl Buffer { /// which is the same as the QuestDB default. pub fn new() -> Self { Self { - state: State::Init, output: String::new(), + state: BufferState::new(), marker: None, max_name_len: 127, } @@ -644,6 +743,17 @@ impl Buffer { self.output.len() } + /// The number of rows accumulated in the buffer. + pub fn row_count(&self) -> usize { + self.state.row_count + } + + /// The buffer is transactional if sent over HTTP. + /// A buffer stops being transactional if it contains rows for multiple tables. + pub fn transactional(&self) -> bool { + self.state.transactional + } + pub fn is_empty(&self) -> bool { self.output.is_empty() } @@ -666,7 +776,7 @@ impl Buffer { /// Once the marker is no longer needed, call /// [`clear_marker`](Buffer::clear_marker). pub fn set_marker(&mut self) -> Result<()> { - if (self.state as isize & Op::Table as isize) == 0 { + if (self.state.op_case as isize & Op::Table as isize) == 0 { return Err(error::fmt!( InvalidApiCall, concat!( @@ -676,7 +786,7 @@ impl Buffer { ) )); } - self.marker = Some((self.output.len(), self.state)); + self.marker = Some((self.output.len(), self.state.clone())); Ok(()) } @@ -685,10 +795,9 @@ impl Buffer { /// /// As a side-effect, this also clears the marker. pub fn rewind_to_marker(&mut self) -> Result<()> { - if let Some((position, state)) = self.marker { + if let Some((position, state)) = self.marker.take() { self.output.truncate(position); self.state = state; - self.marker = None; Ok(()) } else { Err(error::fmt!( @@ -701,7 +810,7 @@ impl Buffer { /// Discard any marker as may have been set by /// [`set_marker`](Buffer::set_marker). /// - /// Idempodent. + /// Idempotent. pub fn clear_marker(&mut self) { self.marker = None; } @@ -710,22 +819,23 @@ impl Buffer { /// [`capacity`](Buffer::capacity). pub fn clear(&mut self) { self.output.clear(); + self.state.clear(); self.marker = None; - self.state = State::Init; } + /// Check if the next API operation is allowed as per the OP case state machine. #[inline(always)] - fn check_state(&self, op: Op) -> Result<()> { - if (self.state as isize & op as isize) > 0 { - return Ok(()); + fn check_op(&self, op: Op) -> Result<()> { + if (self.state.op_case as isize & op as isize) > 0 { + Ok(()) + } else { + Err(error::fmt!( + InvalidApiCall, + "State error: Bad call to `{}`, {}.", + op.descr(), + self.state.op_case.next_op_descr() + )) } - let error = error::fmt!( - InvalidApiCall, - "State error: Bad call to `{}`, {}.", - op.descr(), - self.state.next_op_descr() - ); - Err(error) } #[inline(always)] @@ -774,9 +884,18 @@ impl Buffer { { let name: TableName<'a> = name.try_into()?; self.validate_max_name_len(name.name)?; - self.check_state(Op::Table)?; + self.check_op(Op::Table)?; write_escaped_unquoted(&mut self.output, name.name); - self.state = State::TableWritten; + self.state.op_case = OpCase::TableWritten; + + // A buffer stops being transactional if it targets multiple tables. + if let Some(first_table) = &self.state.first_table { + if first_table != name.name { + self.state.transactional = false; + } + } else { + self.state.first_table = Some(name.name.to_owned()); + } Ok(self) } @@ -831,12 +950,12 @@ impl Buffer { { let name: ColumnName<'a> = name.try_into()?; self.validate_max_name_len(name.name)?; - self.check_state(Op::Symbol)?; + self.check_op(Op::Symbol)?; self.output.push(','); write_escaped_unquoted(&mut self.output, name.name); self.output.push('='); write_escaped_unquoted(&mut self.output, value.as_ref()); - self.state = State::SymbolWritten; + self.state.op_case = OpCase::SymbolWritten; Ok(self) } @@ -847,16 +966,16 @@ impl Buffer { { let name: ColumnName<'a> = name.try_into()?; self.validate_max_name_len(name.name)?; - self.check_state(Op::Column)?; + self.check_op(Op::Column)?; self.output - .push(if (self.state as isize & Op::Symbol as isize) > 0 { + .push(if (self.state.op_case as isize & Op::Symbol as isize) > 0 { ' ' } else { ',' }); write_escaped_unquoted(&mut self.output, name.name); self.output.push('='); - self.state = State::ColumnWritten; + self.state.op_case = OpCase::ColumnWritten; Ok(self) } @@ -1139,7 +1258,7 @@ impl Buffer { T: TryInto, Error: From, { - self.check_state(Op::At)?; + self.check_op(Op::At)?; let timestamp: Timestamp = timestamp.try_into()?; // https://github.com/rust-lang/rust/issues/115880 @@ -1159,7 +1278,8 @@ impl Buffer { self.output.push(' '); self.output.push_str(printed); self.output.push('\n'); - self.state = State::MayFlushOrTable; + self.state.op_case = OpCase::MayFlushOrTable; + self.state.row_count += 1; Ok(()) } @@ -1190,9 +1310,10 @@ impl Buffer { /// assigned by the server may drift significantly from when the data /// was recorded in the buffer. pub fn at_now(&mut self) -> Result<()> { - self.check_state(Op::At)?; + self.check_op(Op::At)?; self.output.push('\n'); - self.state = State::MayFlushOrTable; + self.state.op_case = OpCase::MayFlushOrTable; + self.state.row_count += 1; Ok(()) } } @@ -1205,13 +1326,14 @@ impl Default for Buffer { /// Connects to a QuestDB instance and inserts data via the ILP protocol. /// -/// * To construct an instance, use the [`SenderBuilder`]. +/// * To construct an instance, use [`Sender::from_conf`] or the [`SenderBuilder`]. /// * To prepare messages, use [`Buffer`] objects. /// * To send messages, call the [`flush`](Sender::flush) method. pub struct Sender { descr: String, - conn: Connection, + handler: ProtocolHandler, connected: bool, + max_buf_size: usize, } impl std::fmt::Debug for Sender { @@ -1220,18 +1342,27 @@ impl std::fmt::Debug for Sender { } } -#[derive(Debug, Clone)] -struct AuthParams { +#[derive(PartialEq, Debug, Clone)] +struct EcdsaAuthParams { key_id: String, priv_key: String, pub_key_x: String, pub_key_y: String, } -/// Root used to determine how to validate the server's TLS certificate. -/// -/// Used when configuring the [`tls`](SenderBuilder::tls) option. -#[derive(Debug, Clone)] +#[derive(PartialEq, Debug, Clone)] +enum AuthParams { + Ecdsa(EcdsaAuthParams), + + #[cfg(feature = "ilp-over-http")] + Basic(BasicAuthParams), + + #[cfg(feature = "ilp-over-http")] + Token(TokenAuthParams), +} + +/// Possible sources of the root certificates used to validate the server's TLS certificate. +#[derive(PartialEq, Debug, Clone, Copy)] pub enum CertificateAuthority { /// Use the root certificates provided by the /// [`webpki-roots`](https://crates.io/crates/webpki-roots) crate. @@ -1247,105 +1378,100 @@ pub enum CertificateAuthority { WebpkiAndOsRoots, /// Use the root certificates provided by a PEM-encoded file. - File(PathBuf), -} - -/// Options for full-connection encryption via TLS. -#[derive(Debug, Clone)] -pub enum Tls { - /// No TLS encryption. - Disabled, - - /// Use TLS encryption, verifying the server's certificate. - Enabled(CertificateAuthority), - - /// Use TLS encryption, whilst dangerously ignoring the server's certificate. - /// This should only be used for deubgging purposes. - /// For testing consider specifying a [`CertificateAuthority::File`] instead. - /// - /// *This option requires the `insecure-skip-verify` feature.* - #[cfg(feature = "insecure-skip-verify")] - InsecureSkipVerify, -} - -impl Tls { - /// Returns true if TLS is enabled. - pub fn is_enabled(&self) -> bool { - !matches!(self, Tls::Disabled) - } + PemFile, } /// A `u16` port number or `String` port service name as is registered with /// `/etc/services` or equivalent. /// /// ``` -/// use questdb::ingress::Service; +/// use questdb::ingress::Port; /// use std::convert::Into; /// -/// let service: Service = 9009.into(); +/// let service: Port = 9009.into(); /// ``` /// /// or /// /// ``` -/// use questdb::ingress::Service; +/// use questdb::ingress::Port; /// use std::convert::Into; /// /// // Assuming the service name is registered. -/// let service: Service = "qdb_ilp".into(); // or with a String too. +/// let service: Port = "qdb_ilp".into(); // or with a String too. /// ``` -pub struct Service(String); +pub struct Port(String); -impl From for Service { +impl From for Port { fn from(s: String) -> Self { - Service(s) + Port(s) } } -impl From<&str> for Service { +impl From<&str> for Port { fn from(s: &str) -> Self { - Service(s.to_owned()) + Port(s.to_owned()) } } -impl From for Service { +impl From for Port { fn from(p: u16) -> Self { - Service(p.to_string()) + Port(p.to_string()) } } #[cfg(feature = "insecure-skip-verify")] mod danger { + use rustls::client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier}; + use rustls::{DigitallySignedStruct, Error, SignatureScheme}; + use rustls_pki_types::{CertificateDer, ServerName, UnixTime}; + + #[derive(Debug)] pub struct NoCertificateVerification {} - impl rustls::client::ServerCertVerifier for NoCertificateVerification { + impl ServerCertVerifier for NoCertificateVerification { fn verify_server_cert( &self, - _end_entity: &rustls::Certificate, - _intermediates: &[rustls::Certificate], - _server_name: &rustls::ServerName, - _scts: &mut dyn Iterator, - _ocsp: &[u8], - _now: std::time::SystemTime, - ) -> Result { - Ok(rustls::client::ServerCertVerified::assertion()) + _end_entity: &CertificateDer<'_>, + _intermediates: &[CertificateDer<'_>], + _server_name: &ServerName<'_>, + _ocsp_response: &[u8], + _now: UnixTime, + ) -> Result { + Ok(ServerCertVerified::assertion()) + } + + fn verify_tls12_signature( + &self, + _message: &[u8], + _cert: &CertificateDer<'_>, + _dss: &DigitallySignedStruct, + ) -> Result { + Ok(HandshakeSignatureValid::assertion()) + } + + fn verify_tls13_signature( + &self, + _message: &[u8], + _cert: &CertificateDer<'_>, + _dss: &DigitallySignedStruct, + ) -> Result { + Ok(HandshakeSignatureValid::assertion()) } - } -} -fn map_rustls_err(descr: &str, err: rustls::Error) -> Error { - error::fmt!(TlsError, "{}: {}", descr, err) + fn supported_verify_schemes(&self) -> Vec { + rustls::crypto::ring::default_provider() + .signature_verification_algorithms + .supported_schemes() + } + } } #[cfg(feature = "tls-webpki-certs")] fn add_webpki_roots(root_store: &mut RootCertStore) { - root_store.add_trust_anchors(webpki_roots::TLS_SERVER_ROOTS.iter().map(|ta| { - OwnedTrustAnchor::from_subject_spki_name_constraints( - ta.subject, - ta.spki, - ta.name_constraints, - ) - })); + root_store + .roots + .extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned()) } #[cfg(feature = "tls-native-certs")] @@ -1358,8 +1484,7 @@ fn add_os_roots(root_store: &mut RootCertStore) -> Result<()> { ) })?; - let os_certs: Vec> = os_certs.into_iter().map(|cert| cert.0).collect(); - let (valid_count, invalid_count) = root_store.add_parsable_certificates(&os_certs[..]); + let (valid_count, invalid_count) = root_store.add_parsable_certificates(os_certs); if valid_count == 0 && invalid_count > 0 { return Err(error::fmt!( TlsError, @@ -1370,34 +1495,56 @@ fn add_os_roots(root_store: &mut RootCertStore) -> Result<()> { Ok(()) } -fn configure_tls(tls: &Tls) -> Result>> { - if !tls.is_enabled() { +fn configure_tls( + tls_enabled: bool, + tls_verify: bool, + tls_ca: CertificateAuthority, + tls_roots: &Option, +) -> Result>> { + if !tls_enabled { return Ok(None); } let mut root_store = RootCertStore::empty(); - - if let Tls::Enabled(ca) = tls { - match ca { + if tls_verify { + match (tls_ca, tls_roots) { #[cfg(feature = "tls-webpki-certs")] - CertificateAuthority::WebpkiRoots => { + (CertificateAuthority::WebpkiRoots, None) => { add_webpki_roots(&mut root_store); } + + #[cfg(feature = "tls-webpki-certs")] + (CertificateAuthority::WebpkiRoots, Some(_)) => { + return Err(error::fmt!(ConfigError, "Config parameter \"tls_roots\" must be unset when \"tls_ca\" is set to \"webpki_roots\".")); + } + #[cfg(feature = "tls-native-certs")] - CertificateAuthority::OsRoots => { + (CertificateAuthority::OsRoots, None) => { add_os_roots(&mut root_store)?; } + + #[cfg(feature = "tls-native-certs")] + (CertificateAuthority::OsRoots, Some(_)) => { + return Err(error::fmt!(ConfigError, "Config parameter \"tls_roots\" must be unset when \"tls_ca\" is set to \"os_roots\".")); + } + #[cfg(all(feature = "tls-webpki-certs", feature = "tls-native-certs"))] - CertificateAuthority::WebpkiAndOsRoots => { + (CertificateAuthority::WebpkiAndOsRoots, None) => { add_webpki_roots(&mut root_store); add_os_roots(&mut root_store)?; } - CertificateAuthority::File(ca_file) => { + + #[cfg(all(feature = "tls-webpki-certs", feature = "tls-native-certs"))] + (CertificateAuthority::WebpkiAndOsRoots, Some(_)) => { + return Err(error::fmt!(ConfigError, "Config parameter \"tls_roots\" must be unset when \"tls_ca\" is set to \"webpki_and_os_roots\".")); + } + + (CertificateAuthority::PemFile, Some(ca_file)) => { let certfile = std::fs::File::open(ca_file).map_err(|io_err| { error::fmt!( TlsError, concat!( - "Could not open certificate authority ", + "Could not open tls_roots certificate authority ", "file from path {:?}: {}" ), ca_file, @@ -1405,31 +1552,29 @@ fn configure_tls(tls: &Tls) -> Result>> { ) })?; let mut reader = BufReader::new(certfile); - let der_certs = &rustls_pemfile::certs(&mut reader).map_err(|io_err| { - error::fmt!( - TlsError, - concat!( - "Could not read certificate authority ", - "file from path {:?}: {}" - ), - ca_file, - io_err - ) - })?; + let der_certs = rustls_pemfile::certs(&mut reader) + .collect::, _>>() + .map_err(|io_err| { + error::fmt!( + TlsError, + concat!( + "Could not read certificate authority ", + "file from path {:?}: {}" + ), + ca_file, + io_err + ) + })?; root_store.add_parsable_certificates(der_certs); } + + (CertificateAuthority::PemFile, None) => { + return Err(error::fmt!(ConfigError, "Config parameter \"tls_roots\" is required when \"tls_ca\" is set to \"pem_file\".")); + } } } - // else if let Tls::InsecureSkipVerify { - // We don't need to set up any certificates. - // An empty root is fine if we're going to ignore validity anyways. - // } let mut config = rustls::ClientConfig::builder() - .with_safe_default_cipher_suites() - .with_safe_default_kx_groups() - .with_safe_default_protocol_versions() - .map_err(|rustls_err| map_rustls_err("Bad protocol version selection", rustls_err))? .with_root_certificates(root_store) .with_no_client_auth(); @@ -1438,7 +1583,7 @@ fn configure_tls(tls: &Tls) -> Result>> { config.key_log = Arc::new(rustls::KeyLogFile::new()); #[cfg(feature = "insecure-skip-verify")] - if let Tls::InsecureSkipVerify = tls { + if !tls_verify { config .dangerous() .set_certificate_verifier(Arc::new(danger::NoCertificateVerification {})); @@ -1447,220 +1592,619 @@ fn configure_tls(tls: &Tls) -> Result>> { Ok(Some(Arc::new(config))) } +fn validate_auto_flush_params(params: &HashMap) -> Result<()> { + if let Some(auto_flush) = params.get("auto_flush") { + if auto_flush.as_str() != "off" { + return Err(error::fmt!( + ConfigError, + "Invalid auto_flush value '{auto_flush}'. This client does not \ + support auto-flush, so the only accepted value is 'off'" + )); + } + } + + for ¶m in ["auto_flush_rows", "auto_flush_bytes"].iter() { + if params.contains_key(param) { + return Err(error::fmt!( + ConfigError, + "Invalid configuration parameter {:?}. This client does not support auto-flush", + param + )); + } + } + Ok(()) +} + +/// Protocol used to communicate with the QuestDB server. +#[derive(PartialEq, Debug, Clone, Copy)] +pub enum Protocol { + /// ILP over TCP (streaming). + Tcp, + + /// TCP + TLS + Tcps, + + #[cfg(feature = "ilp-over-http")] + /// ILP over HTTP (request-response, InfluxDB-compatible). + Http, + + #[cfg(feature = "ilp-over-http")] + /// HTTP + TLS + Https, +} + +impl Display for Protocol { + fn fmt(&self, f: &mut Formatter<'_>) -> std::result::Result<(), std::fmt::Error> { + f.write_str(self.schema()) + } +} + +impl Protocol { + fn default_port(&self) -> &str { + match self { + Protocol::Tcp | Protocol::Tcps => "9009", + #[cfg(feature = "ilp-over-http")] + Protocol::Http | Protocol::Https => "9000", + } + } + + fn tls_enabled(&self) -> bool { + match self { + Protocol::Tcp => false, + Protocol::Tcps => true, + #[cfg(feature = "ilp-over-http")] + Protocol::Http => false, + #[cfg(feature = "ilp-over-http")] + Protocol::Https => true, + } + } + + fn is_tcpx(&self) -> bool { + match self { + Protocol::Tcp => true, + Protocol::Tcps => true, + #[cfg(feature = "ilp-over-http")] + Protocol::Http => false, + #[cfg(feature = "ilp-over-http")] + Protocol::Https => false, + } + } + + #[cfg(feature = "ilp-over-http")] + fn is_httpx(&self) -> bool { + match self { + Protocol::Tcp => false, + Protocol::Tcps => false, + Protocol::Http => true, + Protocol::Https => true, + } + } + + fn schema(&self) -> &str { + match self { + Protocol::Tcp => "tcp", + Protocol::Tcps => "tcps", + #[cfg(feature = "ilp-over-http")] + Protocol::Http => "http", + #[cfg(feature = "ilp-over-http")] + Protocol::Https => "https", + } + } + + fn from_schema(schema: &str) -> Result { + match schema { + "tcp" => Ok(Protocol::Tcp), + "tcps" => Ok(Protocol::Tcps), + #[cfg(feature = "ilp-over-http")] + "http" => Ok(Protocol::Http), + #[cfg(feature = "ilp-over-http")] + "https" => Ok(Protocol::Https), + _ => Err(error::fmt!(ConfigError, "Unsupported protocol: {}", schema)), + } + } +} + /// Accumulate parameters for a new `Sender` instance. /// +/// The `SenderBuilder` can be created either for ILP/TCP or ILP/HTTP (with the "ilp-over-http" +/// feature enabled). +/// +/// It can also be created from a config string or the `QDB_CLIENT_CONF` environment variable. +/// +#[cfg_attr( + feature = "ilp-over-http", + doc = r##" +```no_run +# use questdb::Result; +use questdb::ingress::{Protocol, SenderBuilder}; +# fn main() -> Result<()> { +let mut sender = SenderBuilder::new(Protocol::Http, "localhost", 9009).build()?; +# Ok(()) +# } +``` +"## +)] +/// +/// ```no_run +/// # use questdb::Result; +/// use questdb::ingress::{Protocol, SenderBuilder}; +/// +/// # fn main() -> Result<()> { +/// let mut sender = SenderBuilder::new(Protocol::Tcp, "localhost", 9009).build()?; +/// # Ok(()) +/// # } +/// ``` +/// /// ```no_run /// # use questdb::Result; /// use questdb::ingress::SenderBuilder; /// /// # fn main() -> Result<()> { -/// let mut sender = SenderBuilder::new("localhost", 9009).connect()?; +/// let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; /// # Ok(()) /// # } /// ``` /// -/// Additional options for: -/// * Binding a specific [outbound network address](SenderBuilder::net_interface). -/// * Connection security -/// ([authentication](SenderBuilder::auth), [encryption](SenderBuilder::tls)). -/// * Authentication [timeouts](SenderBuilder::read_timeout). +/// ```no_run +/// # use questdb::Result; +/// use questdb::ingress::SenderBuilder; +/// +/// # fn main() -> Result<()> { +/// // export QDB_CLIENT_CONF="https::addr=localhost:9000;" +/// let mut sender = SenderBuilder::from_env()?.build()?; +/// # Ok(()) +/// # } +/// ``` /// #[derive(Debug, Clone)] pub struct SenderBuilder { - read_timeout: Duration, - host: String, - port: String, - net_interface: Option, - auth: Option, - tls: Tls, + protocol: Protocol, + host: ConfigSetting, + port: ConfigSetting, + net_interface: ConfigSetting>, + max_buf_size: ConfigSetting, + auth_timeout: ConfigSetting, + username: ConfigSetting>, + password: ConfigSetting>, + token: ConfigSetting>, + token_x: ConfigSetting>, + token_y: ConfigSetting>, + + #[cfg(feature = "insecure-skip-verify")] + tls_verify: ConfigSetting, + + tls_ca: ConfigSetting, + tls_roots: ConfigSetting>, + + #[cfg(feature = "ilp-over-http")] + http: Option, } impl SenderBuilder { - /// QuestDB server and port. - /// - /// ```no_run - /// # use questdb::Result; - /// use questdb::ingress::SenderBuilder; - /// - /// # fn main() -> Result<()> { - /// let mut sender = SenderBuilder::new("localhost", 9009).connect()?; - /// # Ok(()) - /// # } - /// ``` - pub fn new, P: Into>(host: H, port: P) -> Self { - let service: Service = port.into(); - Self { - read_timeout: Duration::from_secs(15), - host: host.into(), - port: service.0, - net_interface: None, - auth: None, - tls: Tls::Disabled, - } - } - - /// Select local outbound interface. - /// - /// This may be relevant if your machine has multiple network interfaces. + /// Create a new `SenderBuilder` instance from configuration string. /// - /// If unspecified, the default is to use any available interface and is - /// equivalent to calling: + /// The format of the string is: `"http::addr=host:port;key=value;...;"`. /// - /// ```no_run - /// # use questdb::Result; - /// # use questdb::ingress::SenderBuilder; - /// # fn main() -> Result<()> { - /// let mut sender = SenderBuilder::new("localhost", 9009) - /// .net_interface("0.0.0.0") - /// .connect()?; - /// # Ok(()) - /// # } - /// ``` - pub fn net_interface>(mut self, addr: I) -> Self { - self.net_interface = Some(addr.into()); - self - } - - /// Authentication Parameters. + /// Alongside `"http"` you can also specify `"https"`, `"tcp"`, and `"tcps"`. /// - /// If not called, authentication is disabled. + /// HTTP is recommended in most cases as is provides better error feedback + /// allows controlling transactions. TCP can sometimes be faster in higher-latency + /// networks, but misses out on a number of features. /// - /// # Arguments - /// * `key_id` - Key identifier, AKA "kid" in JWT. This is sometimes - /// referred to as the username. - /// * `priv_key` - Private key, AKA "d" in JWT. - /// * `pub_key_x` - X coordinate of the public key, AKA "x" in JWT. - /// * `pub_key_y` - Y coordinate of the public key, AKA "y" in JWT. + /// The accepted set of keys and values is the same as for the `SenderBuilder`'s API. /// - /// # Example + /// E.g. `"https::addr=host:port;username=alice;password=secret;tls_ca=os_roots;"`. /// - /// ```no_run - /// # use questdb::Result; - /// # use questdb::ingress::SenderBuilder; - /// # fn main() -> Result<()> { - /// let mut sender = SenderBuilder::new("localhost", 9009) - /// .auth( - /// "testUser1", // kid - /// "5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48", // d - /// "fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU", // x - /// "Dt5tbS1dEDMSYfym3fgMv0B99szno-dFc1rYF9t0aac") // y - /// .connect()?; - /// # Ok(()) - /// # } - /// ``` + /// If you prefer, you can also load the configuration from an environment variable. + /// See [`SenderBuilder::from_env`]. /// - /// Follow the QuestDB [authentication - /// documentation](https://questdb.io/docs/reference/api/ilp/authenticate) - /// for instructions on generating keys. - pub fn auth(mut self, key_id: A, priv_key: B, pub_key_x: C, pub_key_y: D) -> Self - where - A: Into, - B: Into, - C: Into, - D: Into, - { - self.auth = Some(AuthParams { - key_id: key_id.into(), - priv_key: priv_key.into(), - pub_key_x: pub_key_x.into(), - pub_key_y: pub_key_y.into(), - }); - self - } + /// Once a `SenderBuilder` is created from a string (or from the environment variable) + /// it can be further customized before calling [`SenderBuilder::build`]. + pub fn from_conf>(conf: T) -> Result { + let conf = conf.as_ref(); + let conf = questdb_confstr::parse_conf_str(conf) + .map_err(|e| error::fmt!(ConfigError, "Config parse error: {}", e))?; + let service = conf.service(); + let params = conf.params(); - /// Configure TLS handshake. - /// - /// The default is [`Tls::Disabled`]. - /// - /// ```no_run - /// # use questdb::Result; - /// # use questdb::ingress::SenderBuilder; - /// # use questdb::ingress::Tls; - /// # fn main() -> Result<()> { - /// let mut sender = SenderBuilder::new("localhost", 9009) - /// .tls(Tls::Disabled) - /// .connect()?; - /// # Ok(()) - /// # } - /// ``` - /// - /// To enable with commonly accepted certificates, use: - /// - /// ```no_run - /// # use questdb::Result; - /// # use questdb::ingress::SenderBuilder; - /// # use questdb::ingress::Tls; - /// use questdb::ingress::CertificateAuthority; - /// - /// # fn main() -> Result<()> { - /// let mut sender = SenderBuilder::new("localhost", 9009) - /// .tls(Tls::Enabled(CertificateAuthority::WebpkiRoots)) - /// .connect()?; - /// # Ok(()) - /// # } - /// ``` + let protocol = Protocol::from_schema(service)?; + + let Some(addr) = params.get("addr") else { + return Err(error::fmt!( + ConfigError, + "Missing \"addr\" parameter in config string" + )); + }; + let (host, port) = match addr.split_once(':') { + Some((h, p)) => (h, p), + None => (addr.as_str(), protocol.default_port()), + }; + let mut builder = SenderBuilder::new(protocol, host, port); + + validate_auto_flush_params(params)?; + + for (key, val) in params.iter().map(|(k, v)| (k.as_str(), v.as_str())) { + builder = match key { + "username" => builder.username(val)?, + "password" => builder.password(val)?, + "token" => builder.token(val)?, + "token_x" => builder.token_x(val)?, + "token_y" => builder.token_y(val)?, + "bind_interface" => builder.bind_interface(val)?, + + "init_buf_size" => { + return Err(error::fmt!( + ConfigError, + "\"init_buf_size\" is not supported in config string" + )) + } + + "max_buf_size" => builder.max_buf_size(parse_conf_value(key, val)?)?, + + "auth_timeout" => { + builder.auth_timeout(Duration::from_millis(parse_conf_value(key, val)?))? + } + + "tls_verify" => { + let verify = match val { + "on" => true, + "unsafe_off" => false, + _ => { + return Err(error::fmt!( + ConfigError, + r##"Config parameter "tls_verify" must be either "on" or "unsafe_off".'"##, + )) + } + }; + + #[cfg(not(feature = "insecure-skip-verify"))] + { + if !verify { + return Err(error::fmt!( + ConfigError, + r##"The "insecure-skip-verify" feature is not enabled, so "tls_verify=unsafe_off" is not supported"##, + )); + } + builder + } + + #[cfg(feature = "insecure-skip-verify")] + builder.tls_verify(verify)? + } + + "tls_ca" => { + let ca = match val { + #[cfg(feature = "tls-webpki-certs")] + "webpki_roots" => CertificateAuthority::WebpkiRoots, + + #[cfg(not(feature = "tls-webpki-certs"))] + "webpki_roots" => return Err(error::fmt!(ConfigError, "Config parameter \"tls_ca=webpki_roots\" requires the \"tls-webpki-certs\" feature")), + + #[cfg(feature = "tls-native-certs")] + "os_roots" => CertificateAuthority::OsRoots, + + #[cfg(not(feature = "tls-native-certs"))] + "os_roots" => return Err(error::fmt!(ConfigError, "Config parameter \"tls_ca=os_roots\" requires the \"tls-native-certs\" feature")), + + #[cfg(all(feature = "tls-webpki-certs", feature = "tls-native-certs"))] + "webpki_and_os_roots" => CertificateAuthority::WebpkiAndOsRoots, + + #[cfg(not(all(feature = "tls-webpki-certs", feature = "tls-native-certs")))] + "webpki_and_os_roots" => return Err(error::fmt!(ConfigError, "Config parameter \"tls_ca=webpki_and_os_roots\" requires both the \"tls-webpki-certs\" and \"tls-native-certs\" features")), + + _ => return Err(error::fmt!(ConfigError, "Invalid value {val:?} for \"tls_ca\"")), + }; + builder.tls_ca(ca)? + } + + "tls_roots" => { + let path = PathBuf::from_str(val).map_err(|e| { + error::fmt!( + ConfigError, + "Invalid path {:?} for \"tls_roots\": {}", + val, + e + ) + })?; + builder.tls_roots(path)? + } + + "tls_roots_password" => { + return Err(error::fmt!( + ConfigError, + "\"tls_roots_password\" is not supported." + )) + } + + #[cfg(feature = "ilp-over-http")] + "request_min_throughput" => { + builder.request_min_throughput(parse_conf_value(key, val)?)? + } + + #[cfg(feature = "ilp-over-http")] + "request_timeout" => { + builder.request_timeout(Duration::from_millis(parse_conf_value(key, val)?))? + } + + #[cfg(feature = "ilp-over-http")] + "retry_timeout" => { + builder.retry_timeout(Duration::from_millis(parse_conf_value(key, val)?))? + } + // Ignore other parameters. + // We don't want to fail on unknown keys as this would require releasing different + // library implementations in lock step as soon as a new parameter is added to any of them, + // even if it's not used. + _ => builder, + }; + } + + Ok(builder) + } + + /// Create a new `SenderBuilder` instance from configuration string read from the + /// `QDB_CLIENT_CONF` environment variable. /// - /// To use [self-signed certificates](https://github.com/questdb/c-questdb-client/tree/main/tls_certs): + /// The format of the string is the same as for [`SenderBuilder::from_conf`]. + pub fn from_env() -> Result { + let conf = std::env::var("QDB_CLIENT_CONF").map_err(|_| { + error::fmt!(ConfigError, "Environment variable QDB_CLIENT_CONF not set.") + })?; + Self::from_conf(conf) + } + + /// Create a new `SenderBuilder` instance from the provided QuestDB + /// server and port using ILP over the specified protocol. /// /// ```no_run /// # use questdb::Result; - /// # use questdb::ingress::SenderBuilder; - /// # use questdb::ingress::Tls; - /// use questdb::ingress::CertificateAuthority; - /// use std::path::PathBuf; + /// use questdb::ingress::{Protocol, SenderBuilder}; /// /// # fn main() -> Result<()> { - /// let mut sender = SenderBuilder::new("localhost", 9009) - /// .tls(Tls::Enabled(CertificateAuthority::File( - /// PathBuf::from("/path/to/server_rootCA.pem")))) - /// .connect()?; + /// let mut sender = SenderBuilder::new( + /// Protocol::Tcp, "localhost", 9009).build()?; /// # Ok(()) /// # } /// ``` + pub fn new, P: Into>(protocol: Protocol, host: H, port: P) -> Self { + let host = host.into(); + let port: Port = port.into(); + let port = port.0; + + #[cfg(feature = "tls-webpki-certs")] + let tls_ca = CertificateAuthority::WebpkiRoots; + + #[cfg(all(not(feature = "tls-webpki-certs"), feature = "tls-native-certs"))] + let tls_ca = CertificateAuthority::OsRoots; + + #[cfg(not(any(feature = "tls-webpki-certs", feature = "tls-native-certs")))] + let tls_ca = CertificateAuthority::PemFile; + + Self { + protocol, + host: ConfigSetting::new_specified(host), + port: ConfigSetting::new_specified(port), + net_interface: ConfigSetting::new_default(None), + max_buf_size: ConfigSetting::new_default(100 * 1024 * 1024), + auth_timeout: ConfigSetting::new_default(Duration::from_secs(15)), + username: ConfigSetting::new_default(None), + password: ConfigSetting::new_default(None), + token: ConfigSetting::new_default(None), + token_x: ConfigSetting::new_default(None), + token_y: ConfigSetting::new_default(None), + + #[cfg(feature = "insecure-skip-verify")] + tls_verify: ConfigSetting::new_default(true), + + tls_ca: ConfigSetting::new_default(tls_ca), + tls_roots: ConfigSetting::new_default(None), + + #[cfg(feature = "ilp-over-http")] + http: if protocol.is_httpx() { + Some(HttpConfig::default()) + } else { + None + }, + } + } + + /// Select local outbound interface. /// - /// If you're still struggling you may temporarily enable the dangerous - /// `insecure-skip-verify` feature to skip the certificate verification: + /// This may be relevant if your machine has multiple network interfaces. /// - /// ```ignore - /// let mut sender = SenderBuilder::new("localhost", 9009) - /// .tls(Tls::InsecureSkipVerify) - /// .connect()?; - /// ``` - pub fn tls(mut self, tls: Tls) -> Self { - self.tls = tls; - self + /// The default is `"0.0.0.0"`. + pub fn bind_interface>(mut self, addr: I) -> Result { + self.ensure_is_tcpx("bind_interface")?; + self.net_interface + .set_specified("bind_interface", Some(validate_value(addr.into())?))?; + Ok(self) + } + + /// Set the username for authentication. + /// + /// For TCP this is the `kid` part of the ECDSA key set. + /// The other fields are [`token`](SenderBuilder::token), [`token_x`](SenderBuilder::token_x), + /// and [`token_y`](SenderBuilder::token_y). + /// + /// For HTTP this is part of basic authentication. + /// Also see [`password`](SenderBuilder::password). + pub fn username(mut self, username: &str) -> Result { + self.username + .set_specified("username", Some(validate_value(username.to_string())?))?; + Ok(self) + } + + /// Set the password for basic HTTP authentication. + /// Also see [`username`](SenderBuilder::username). + pub fn password(mut self, password: &str) -> Result { + self.password + .set_specified("password", Some(validate_value(password.to_string())?))?; + Ok(self) + } + + /// Token (Bearer) Authentication Parameters for ILP over HTTP, + /// or the ECDSA private key for ILP over TCP authentication. + pub fn token(mut self, token: &str) -> Result { + self.token + .set_specified("token", Some(validate_value(token.to_string())?))?; + Ok(self) + } + + /// The ECDSA public key X for ILP over TCP authentication. + pub fn token_x(mut self, token_x: &str) -> Result { + self.token_x + .set_specified("token_x", Some(validate_value(token_x.to_string())?))?; + Ok(self) + } + + /// The ECDSA public key Y for ILP over TCP authentication. + pub fn token_y(mut self, token_y: &str) -> Result { + self.token_y + .set_specified("token_y", Some(validate_value(token_y.to_string())?))?; + Ok(self) } /// Configure how long to wait for messages from the QuestDB server during /// the TLS handshake and authentication process. /// The default is 15 seconds. - /// - /// ```no_run - /// # use questdb::Result; - /// # use questdb::ingress::SenderBuilder; - /// use std::time::Duration; - /// - /// # fn main() -> Result<()> { - /// let mut sender = SenderBuilder::new("localhost", 9009) - /// .read_timeout(Duration::from_secs(15)) - /// .connect()?; - /// # Ok(()) - /// # } - /// ``` - pub fn read_timeout(mut self, value: Duration) -> Self { - self.read_timeout = value; - self + pub fn auth_timeout(mut self, value: Duration) -> Result { + self.auth_timeout.set_specified("auth_timeout", value)?; + Ok(self) + } + + /// Ensure that TLS is enabled for the protocol. + pub fn ensure_tls_enabled(&self, property: &str) -> Result<()> { + if !self.protocol.tls_enabled() { + return Err(error::fmt!( + ConfigError, + "Cannot set {property:?}: TLS is not supported for protocol {}", + self.protocol + )); + } + Ok(()) } - /// Connect synchronously. + /// Set to `false` to disable TLS certificate verification. + /// This should only be used for debugging purposes as it reduces security. /// - /// Will return once the connection is fully established: - /// If the connection requires authentication or TLS, these will also be - /// completed before returning. - pub fn connect(&self) -> Result { - let mut descr = format!("Sender[host={:?},port={:?},", self.host, self.port); + /// For testing consider specifying a path to a `.pem` file instead via + /// the [`tls_roots`](SenderBuilder::tls_roots) method. + #[cfg(feature = "insecure-skip-verify")] + pub fn tls_verify(mut self, verify: bool) -> Result { + self.ensure_tls_enabled("tls_verify")?; + self.tls_verify.set_specified("tls_verify", verify)?; + Ok(self) + } + + /// Specify where to find the root certificate used to validate the + /// server's TLS certificate. + pub fn tls_ca(mut self, ca: CertificateAuthority) -> Result { + self.ensure_tls_enabled("tls_ca")?; + self.tls_ca.set_specified("tls_ca", ca)?; + Ok(self) + } + + /// Set the path to a custom root certificate `.pem` file. + /// This is used to validate the server's certificate during the TLS handshake. + /// + /// See notes on how to test with [self-signed certificates](https://github.com/questdb/c-questdb-client/tree/main/tls_certs). + pub fn tls_roots>(self, path: P) -> Result { + let mut builder = self.tls_ca(CertificateAuthority::PemFile)?; + let path = path.into(); + // Attempt to read the file here to catch any issues early. + let _file = std::fs::File::open(&path).map_err(|io_err| { + error::fmt!( + ConfigError, + "Could not open root certificate file from path {:?}: {}", + path, + io_err + ) + })?; + builder.tls_roots.set_specified("tls_roots", Some(path))?; + Ok(builder) + } + + /// The maximum buffer size that the client will flush to the server. + /// The default is 100 MiB. + pub fn max_buf_size(mut self, value: usize) -> Result { + let min = 1024; + if value < min { + return Err(error::fmt!( + ConfigError, + "max_buf_size\" must be at least {min} bytes." + )); + } + self.max_buf_size.set_specified("max_buf_size", value)?; + Ok(self) + } + + #[cfg(feature = "ilp-over-http")] + /// Cumulative duration spent in retries. + /// The default is 10 seconds. + pub fn retry_timeout(mut self, value: Duration) -> Result { + if let Some(http) = &mut self.http { + http.retry_timeout.set_specified("retry_timeout", value)?; + } else { + return Err(error::fmt!( + ConfigError, + "retry_timeout is supported only in ILP over HTTP." + )); + } + Ok(self) + } + + #[cfg(feature = "ilp-over-http")] + /// Minimum expected throughput in bytes per second for HTTP requests. + /// If the throughput is lower than this value, the connection will time out. + /// The default is 100 KiB/s. + /// The value is expressed as a number of bytes per second. + /// This is used to calculate additional request timeout, on top of + /// the [`request_timeout`](SenderBuilder::request_timeout). + pub fn request_min_throughput(mut self, value: u64) -> Result { + if let Some(http) = &mut self.http { + http.request_min_throughput + .set_specified("request_min_throughput", value)?; + } else { + return Err(error::fmt!( + ConfigError, + "\"request_min_throughput\" is supported only in ILP over HTTP." + )); + } + Ok(self) + } + + #[cfg(feature = "ilp-over-http")] + /// Grace request timeout before relying on the minimum throughput logic. + /// The default is 10 seconds. + /// See [`request_min_throughput`](SenderBuilder::request_min_throughput) for more details. + pub fn request_timeout(mut self, value: Duration) -> Result { + if let Some(http) = &mut self.http { + http.request_timeout + .set_specified("request_timeout", value)?; + } else { + return Err(error::fmt!( + ConfigError, + "\"request_timeout\" is supported only in ILP over HTTP." + )); + } + Ok(self) + } + + #[cfg(feature = "ilp-over-http")] + /// Internal API, do not use. + /// This is exposed exclusively for the Python client. + /// We (QuestDB) use this to help us debug which client is being used if we encounter issues. + #[doc(hidden)] + pub fn user_agent(mut self, value: &str) -> Result { + let value = validate_value(value)?; + if let Some(http) = &mut self.http { + http.user_agent = value.to_string(); + } + Ok(self) + } + + fn connect_tcp(&self, auth: &Option) -> Result { let addr: SockAddr = gai::resolve_host_port(self.host.as_str(), self.port.as_str())?; - let mut sock = Socket::new(Domain::IPV4, Type::STREAM, Some(Protocol::TCP)) + let mut sock = Socket::new(Domain::IPV4, Type::STREAM, Some(SockProtocol::TCP)) .map_err(|io_err| map_io_to_socket_err("Could not open TCP socket: ", io_err))?; // See: https://idea.popcount.org/2014-04-03-bind-before-connect/ @@ -1675,7 +2219,7 @@ impl SenderBuilder { .map_err(|io_err| map_io_to_socket_err("Could not set SO_KEEPALIVE: ", io_err))?; sock.set_nodelay(true) .map_err(|io_err| map_io_to_socket_err("Could not set TCP_NODELAY: ", io_err))?; - if let Some(ref host) = self.net_interface { + if let Some(ref host) = self.net_interface.deref() { let bind_addr = gai::resolve_host(host.as_str())?; sock.bind(&bind_addr).map_err(|io_err| { map_io_to_socket_err( @@ -1685,7 +2229,7 @@ impl SenderBuilder { })?; } sock.connect(&addr).map_err(|io_err| { - let host_port = format!("{}:{}", self.host, self.port); + let host_port = format!("{}:{}", self.host.deref(), *self.port); let prefix = format!("Could not connect to {:?}: ", host_port); map_io_to_socket_err(&prefix, io_err) })?; @@ -1694,25 +2238,27 @@ impl SenderBuilder { // We set up a read timeout to prevent the client from "hanging" // should we be connecting to a server configured in a different way // from the client. - sock.set_read_timeout(Some(self.read_timeout)) + sock.set_read_timeout(Some(*self.auth_timeout)) .map_err(|io_err| { map_io_to_socket_err("Failed to set read timeout on socket: ", io_err) })?; - match self.tls { - Tls::Disabled => write!(descr, "tls=enabled,").unwrap(), - Tls::Enabled(_) => write!(descr, "tls=enabled,").unwrap(), + #[cfg(feature = "insecure-skip-verify")] + let tls_verify = *self.tls_verify; - #[cfg(feature = "insecure-skip-verify")] - Tls::InsecureSkipVerify => write!(descr, "tls=insecure_skip_verify,").unwrap(), - } + #[cfg(not(feature = "insecure-skip-verify"))] + let tls_verify = true; - let conn = match configure_tls(&self.tls)? { + let mut conn = match configure_tls( + self.protocol.tls_enabled(), + tls_verify, + *self.tls_ca, + self.tls_roots.deref(), + )? { Some(tls_config) => { - let server_name: ServerName = - self.host.as_str().try_into().map_err(|inv_dns_err| { - error::fmt!(TlsError, "Bad host: {}", inv_dns_err) - })?; + let server_name: ServerName = ServerName::try_from(self.host.as_str()) + .map_err(|inv_dns_err| error::fmt!(TlsError, "Bad host: {}", inv_dns_err))? + .to_owned(); let mut tls_conn = ClientConnection::new(tls_config, server_name).map_err(|rustls_err| { error::fmt!(TlsError, "Could not create TLS client: {}", rustls_err) @@ -1729,7 +2275,7 @@ impl SenderBuilder { " Timed out waiting for server ", "response after {:?}." ), - self.read_timeout + *self.auth_timeout ) } else { error::fmt!(TlsError, "Failed to complete TLS handshake: {}", io_err) @@ -1740,26 +2286,253 @@ impl SenderBuilder { } None => Connection::Direct(sock), }; - if self.auth.is_some() { + + if let Some(AuthParams::Ecdsa(auth)) = auth { + conn.authenticate(auth)?; + } + + Ok(ProtocolHandler::Socket(conn)) + } + + fn build_auth(&self) -> Result> { + match ( + self.protocol, + self.username.deref(), + self.password.deref(), + self.token.deref(), + self.token_x.deref(), + self.token_y.deref(), + ) { + (_, None, None, None, None, None) => Ok(None), + ( + protocol, + Some(username), + None, + Some(token), + Some(token_x), + Some(token_y), + ) if protocol.is_tcpx() => Ok(Some(AuthParams::Ecdsa(EcdsaAuthParams { + key_id: username.to_string(), + priv_key: token.to_string(), + pub_key_x: token_x.to_string(), + pub_key_y: token_y.to_string(), + }))), + (protocol, Some(_username), Some(_password), None, None, None) + if protocol.is_tcpx() => { + Err(error::fmt!(ConfigError, + r##"The "basic_auth" setting can only be used with the ILP/HTTP protocol."##, + )) + } + (protocol, None, None, Some(_token), None, None) + if protocol.is_tcpx() => { + Err(error::fmt!(ConfigError, "Token authentication only be used with the ILP/HTTP protocol.")) + } + (protocol, _username, None, _token, _token_x, _token_y) + if protocol.is_tcpx() => { + Err(error::fmt!(ConfigError, + r##"Incomplete ECDSA authentication parameters. Specify either all or none of: "username", "token", "token_x", "token_y"."##, + )) + } + #[cfg(feature = "ilp-over-http")] + (protocol, Some(username), Some(password), None, None, None) + if protocol.is_httpx() => { + Ok(Some(AuthParams::Basic(BasicAuthParams { + username: username.to_string(), + password: password.to_string(), + }))) + } + #[cfg(feature = "ilp-over-http")] + (protocol, Some(_username), None, None, None, None) + if protocol.is_httpx() => { + Err(error::fmt!(ConfigError, + r##"Basic authentication parameter "username" is present, but "password" is missing."##, + )) + } + #[cfg(feature = "ilp-over-http")] + (protocol, None, Some(_password), None, None, None) + if protocol.is_httpx() => { + Err(error::fmt!(ConfigError, + r##"Basic authentication parameter "password" is present, but "username" is missing."##, + )) + } + #[cfg(feature = "ilp-over-http")] + (protocol, None, None, Some(token), None, None) + if protocol.is_httpx() => { + Ok(Some(AuthParams::Token(TokenAuthParams { + token: token.to_string(), + }))) + } + #[cfg(feature = "ilp-over-http")] + ( + protocol, + Some(_username), + None, + Some(_token), + Some(_token_x), + Some(_token_y), + ) if protocol.is_httpx() => { + Err(error::fmt!(ConfigError, "ECDSA authentication is only available with ILP/TCP and not available with ILP/HTTP.")) + } + #[cfg(feature = "ilp-over-http")] + (protocol, _username, _password, _token, None, None) + if protocol.is_httpx() => { + Err(error::fmt!(ConfigError, + r##"Inconsistent HTTP authentication parameters. Specify either "username" and "password", or just "token"."##, + )) + } + _ => { + Err(error::fmt!(ConfigError, + r##"Incomplete authentication parameters. Check "username", "password", "token", "token_x" and "token_y" parameters are set correctly."##, + )) + } + } + } + + /// Build the sender. + /// + /// In case of TCP, this synchronously establishes the TCP connection, and + /// returns once the connection is fully established. If the connection + /// requires authentication or TLS, these will also be completed before + /// returning. + pub fn build(&self) -> Result { + let mut descr = format!("Sender[host={:?},port={:?},", self.host, self.port); + + if self.protocol.tls_enabled() { + write!(descr, "tls=enabled,").unwrap(); + } else { + write!(descr, "tls=disabled,").unwrap(); + } + + let auth = self.build_auth()?; + + let handler = match self.protocol { + Protocol::Tcp | Protocol::Tcps => self.connect_tcp(&auth)?, + #[cfg(feature = "ilp-over-http")] + Protocol::Http | Protocol::Https => { + if self.net_interface.is_some() { + // See: https://github.com/algesten/ureq/issues/692 + return Err(error::fmt!( + InvalidApiCall, + "net_interface is not supported for ILP over HTTP." + )); + } + + let user_agent = self.http.as_ref().unwrap().user_agent.as_str(); + let agent_builder = ureq::AgentBuilder::new() + .user_agent(user_agent) + .no_delay(true); + + #[cfg(feature = "insecure-skip-verify")] + let tls_verify = *self.tls_verify; + + #[cfg(not(feature = "insecure-skip-verify"))] + let tls_verify = true; + + let agent_builder = match configure_tls( + self.protocol.tls_enabled(), + tls_verify, + *self.tls_ca, + self.tls_roots.deref(), + )? { + Some(tls_config) => agent_builder.tls_config(tls_config), + None => agent_builder, + }; + let auth = match auth { + Some(AuthParams::Basic(ref auth)) => Some(auth.to_header_string()), + Some(AuthParams::Token(ref auth)) => Some(auth.to_header_string()?), + Some(AuthParams::Ecdsa(_)) => { + return Err(error::fmt!( + AuthError, + "ECDSA authentication is not supported for ILP over HTTP. \ + Please use basic or token authentication instead." + )); + } + None => None, + }; + let agent = agent_builder.build(); + let proto = self.protocol.schema(); + let url = format!( + "{}://{}:{}/write", + proto, + self.host.deref(), + self.port.deref() + ); + ProtocolHandler::Http(HttpHandlerState { + agent, + url, + auth, + + config: self.http.as_ref().unwrap().clone(), + }) + } + }; + + if auth.is_some() { descr.push_str("auth=on]"); } else { descr.push_str("auth=off]"); } - let mut sender = Sender { + + let sender = Sender { descr, - conn, + handler, connected: true, + max_buf_size: *self.max_buf_size, }; - if let Some(auth) = self.auth.as_ref() { - sender.authenticate(auth)?; - } + Ok(sender) } + + fn ensure_is_tcpx(&mut self, param_name: &str) -> Result<()> { + if self.protocol.is_tcpx() { + Ok(()) + } else { + Err(error::fmt!( + ConfigError, + "The {param_name:?} setting can only be used with the TCP protocol." + )) + } + } +} + +/// When parsing from config, we exclude certain characters. +/// Here we repeat the same validation logic for consistency. +fn validate_value>(value: T) -> Result { + let str_ref = value.as_ref(); + for (p, c) in str_ref.chars().enumerate() { + if matches!(c, '\u{0}'..='\u{1f}' | '\u{7f}'..='\u{9f}') { + return Err(error::fmt!( + ConfigError, + "Invalid character {c:?} at position {p}" + )); + } + } + Ok(value) +} + +fn parse_conf_value(param_name: &str, str_value: &str) -> Result +where + T: FromStr, + T::Err: std::fmt::Debug, +{ + str_value.parse().map_err(|e| { + error::fmt!( + ConfigError, + "Could not parse {param_name:?} to number: {e:?}" + ) + }) } fn b64_decode(descr: &'static str, buf: &str) -> Result> { - Base64UrlUnpadded::decode_vec(buf) - .map_err(|b64_err| error::fmt!(AuthError, "Could not decode {}: {}", descr, b64_err)) + Base64UrlUnpadded::decode_vec(buf).map_err(|b64_err| { + error::fmt!( + AuthError, + "Misconfigured ILP authentication keys. Could not decode {}: {}. \ + Hint: Check the keys for a possible typo.", + descr, + b64_err + ) + }) } fn parse_public_key(pub_key_x: &str, pub_key_y: &str) -> Result> { @@ -1769,14 +2542,30 @@ fn parse_public_key(pub_key_x: &str, pub_key_y: &str) -> Result> { // SEC 1 Uncompressed Octet-String-to-Elliptic-Curve-Point Encoding let mut encoded = Vec::new(); encoded.push(4u8); // 0x04 magic byte that identifies this as uncompressed. - encoded.resize((32 - pub_key_x.len()) + 1, 0u8); + let pub_key_x_ken = pub_key_x.len(); + if pub_key_x_ken > 32 { + return Err(error::fmt!( + AuthError, + "Misconfigured ILP authentication keys. Public key x is too long. \ + Hint: Check the keys for a possible typo." + )); + } + let pub_key_y_len = pub_key_y.len(); + if pub_key_y_len > 32 { + return Err(error::fmt!( + AuthError, + "Misconfigured ILP authentication keys. Public key y is too long. \ + Hint: Check the keys for a possible typo." + )); + } + encoded.resize((32 - pub_key_x_ken) + 1, 0u8); encoded.append(&mut pub_key_x); - encoded.resize((32 - pub_key_y.len()) + 1 + 32, 0u8); + encoded.resize((32 - pub_key_y_len) + 1 + 32, 0u8); encoded.append(&mut pub_key_y); Ok(encoded) } -fn parse_key_pair(auth: &AuthParams) -> Result { +fn parse_key_pair(auth: &EcdsaAuthParams) -> Result { let private_key = b64_decode("private authentication key", auth.priv_key.as_str())?; let public_key = parse_public_key(auth.pub_key_x.as_str(), auth.pub_key_y.as_str())?; let system_random = SystemRandom::new(); @@ -1786,7 +2575,13 @@ fn parse_key_pair(auth: &AuthParams) -> Result { &public_key[..], &system_random, ) - .map_err(|key_rejected| error::fmt!(AuthError, "Bad private key: {}", key_rejected)) + .map_err(|key_rejected| { + error::fmt!( + AuthError, + "Misconfigured ILP authentication keys: {}. Hint: Check the keys for a possible typo.", + key_rejected + ) + }) } pub(crate) struct F64Serializer { @@ -1828,97 +2623,160 @@ impl F64Serializer { } impl Sender { - fn send_key_id(&mut self, key_id: &str) -> Result<()> { - writeln!(&mut self.conn, "{}", key_id) - .map_err(|io_err| map_io_to_socket_err("Failed to send key_id: ", io_err))?; - Ok(()) + /// Create a new `Sender` instance from configuration string. + /// + /// The format of the string is: `"http::addr=host:port;key=value;...;"`. + /// + /// Alongside `"http"` you can also specify `"https"`, `"tcp"`, and `"tcps"`. + /// + /// HTTP is recommended in most cases as is provides better error feedback + /// allows controlling transactions. TCP can sometimes be faster in higher-latency + /// networks, but misses out on a number of features. + /// + /// The accepted set of keys and values is the same as for the opt's API. + /// + /// E.g. `"https::addr=host:port;username=alice;password=secret;tls_ca=os_roots;"`. + /// + /// For full list of keys and values, see the [`SenderBuilder`] documentation: + /// The builder API and the configuration string API are equivalent. + /// + /// If you prefer, you can also load the configuration from an environment variable. + /// See [`Sender::from_env`]. + pub fn from_conf>(conf: T) -> Result { + SenderBuilder::from_conf(conf)?.build() } - fn read_challenge(&mut self) -> Result> { - let mut buf = Vec::new(); - let mut reader = BufReader::new(&mut self.conn); - reader.read_until(b'\n', &mut buf).map_err(|io_err| { - map_io_to_socket_err( - "Failed to read authentication challenge (timed out?): ", - io_err, - ) - })?; - if buf.last().copied().unwrap_or(b'\0') != b'\n' { - return Err(if buf.is_empty() { - error::fmt!( - AuthError, - concat!( - "Did not receive auth challenge. ", - "Is the database configured to require ", - "authentication?" - ) - ) - } else { - error::fmt!(AuthError, "Received incomplete auth challenge: {:?}", buf) - }); - } - buf.pop(); // b'\n' - Ok(buf) + /// Create a new `Sender` from the `QDB_CLIENT_CONF` environment variable. + /// The format is the same as that taken by [`Sender::from_conf`]. + pub fn from_env() -> Result { + SenderBuilder::from_env()?.build() } - fn authenticate(&mut self, auth: &AuthParams) -> Result<()> { - if auth.key_id.contains('\n') { + #[allow(unused_variables)] + fn flush_impl(&mut self, buf: &Buffer, transactional: bool) -> Result<()> { + if !self.connected { return Err(error::fmt!( - AuthError, - "Bad key id {:?}: Should not contain new-line char.", - auth.key_id - )); - } - let key_pair = parse_key_pair(auth)?; - self.send_key_id(auth.key_id.as_str())?; - let challenge = self.read_challenge()?; - let rng = ring::rand::SystemRandom::new(); - let signature = key_pair - .sign(&rng, &challenge[..]) - .map_err(|unspecified_err| { - error::fmt!(AuthError, "Failed to sign challenge: {}", unspecified_err) - })?; - let mut encoded_sig = Base64::encode_string(signature.as_ref()); - encoded_sig.push('\n'); - let buf = encoded_sig.as_bytes(); - if let Err(io_err) = self.conn.write_all(buf) { - return Err(map_io_to_socket_err( - "Could not send signed challenge: ", - io_err, + SocketError, + "Could not flush buffer: not connected to database." )); } - Ok(()) - } + buf.check_op(Op::Flush)?; - /// Send buffer to the QuestDB server, without clearing the - /// buffer. - /// - /// This will block until the buffer is flushed to the network socket. - /// This does not guarantee that the buffer will be sent to the server - /// or that the server has received it. - pub fn flush_and_keep(&mut self, buf: &Buffer) -> Result<()> { - if !self.connected { + if buf.len() > self.max_buf_size { return Err(error::fmt!( - SocketError, - "Could not flush buffer: not connected to database." + InvalidApiCall, + "Could not flush buffer: Buffer size of {} exceeds maximum configured allowed size of {} bytes.", + buf.len(), + self.max_buf_size )); } - buf.check_state(Op::Flush)?; + let bytes = buf.as_str().as_bytes(); - if let Err(io_err) = self.conn.write_all(bytes) { - self.connected = false; - return Err(map_io_to_socket_err("Could not flush buffer: ", io_err)); + if bytes.is_empty() { + return Ok(()); + } + match self.handler { + ProtocolHandler::Socket(ref mut conn) => { + if transactional { + return Err(error::fmt!( + InvalidApiCall, + "Transactional flushes are not supported for ILP over TCP." + )); + } + conn.write_all(bytes).map_err(|io_err| { + self.connected = false; + map_io_to_socket_err("Could not flush buffer: ", io_err) + })?; + } + #[cfg(feature = "ilp-over-http")] + ProtocolHandler::Http(ref state) => { + if transactional && !buf.transactional() { + return Err(error::fmt!( + InvalidApiCall, + "Buffer contains lines for multiple tables. \ + Transactional flushes are only supported for buffers containing lines for a single table." + )); + } + let request_min_throughput = *state.config.request_min_throughput; + let extra_time = if request_min_throughput > 0 { + (bytes.len() as f64) / (request_min_throughput as f64) + } else { + 0.0f64 + }; + let timeout = *state.config.request_timeout + Duration::from_secs_f64(extra_time); + let request = state + .agent + .post(&state.url) + .query_pairs([("precision", "n")]) + .timeout(timeout) + .set("Content-Type", "text/plain; charset=utf-8"); + let request = match state.auth.as_ref() { + Some(auth) => request.set("Authorization", auth), + None => request, + }; + let response_or_err = + http_send_with_retries(request, bytes, *state.config.retry_timeout); + match response_or_err { + Ok(_response) => { + // on success, there's no information in the response. + } + Err(ureq::Error::Status(http_status_code, response)) => { + return Err(parse_http_error(http_status_code, response)); + } + Err(ureq::Error::Transport(transport)) => { + return Err(error::fmt!( + SocketError, + "Could not flush buffer: {}", + transport + )); + } + } + } } Ok(()) } + /// Variant of `.flush()` that does not clear the buffer and allows for + /// transactional flushes. + /// + /// A transactional flush is simply a flush that ensures that all rows in + /// the ILP buffer refer to the same table, thus allowing the server to + /// treat the flush request as a single transaction. + /// + /// This is because QuestDB does not support transactions spanning multiple + /// tables. + /// + /// Note that transactional flushes are only supported for ILP over HTTP. + #[cfg(feature = "ilp-over-http")] + pub fn flush_and_keep_with_flags(&mut self, buf: &Buffer, transactional: bool) -> Result<()> { + self.flush_impl(buf, transactional) + } + + /// Variant of `.flush()` that does not clear the buffer. + pub fn flush_and_keep(&mut self, buf: &Buffer) -> Result<()> { + self.flush_impl(buf, false) + } + /// Send buffer to the QuestDB server, clearing the buffer. /// - /// This will block until the buffer is flushed to the network socket. - /// This does not guarantee that the buffer will be sent to the server - /// or that the server has received it. + /// If sending over HTTP, flushing will send an HTTP request and wait + /// for the response. If the server responds with an error, this function + /// will return a descriptive error. In case of network errors, + /// this function will retry. + /// + /// If sending over TCP, this will block until the buffer is flushed to the + /// network socket. Note that this does not guarantee that the buffer will + /// be sent to the server or that the server has received it. + /// In case of errors the server will disconnect: consult the server logs. + /// + /// Prefer HTTP in most cases, but use TCP if you need to continuously + /// data to the server at a high rate. + /// + /// To improve HTTP performance, send larger buffers (with more rows), + /// and consider parallelizing writes using multiple senders from multiple + /// threads. pub fn flush(&mut self, buf: &mut Buffer) -> Result<()> { - self.flush_and_keep(buf)?; + self.flush_impl(buf, false)?; buf.clear(); Ok(()) } @@ -1926,9 +2784,21 @@ impl Sender { /// The sender is no longer usable and must be dropped. /// /// This is caused if there was an earlier failure. + /// + /// This method is specific to ILP/TCP and is not relevant for ILP/HTTP. pub fn must_close(&self) -> bool { !self.connected } } +mod conf; mod timestamp; + +#[cfg(feature = "ilp-over-http")] +mod http; + +#[cfg(feature = "ilp-over-http")] +use http::*; + +#[cfg(test)] +mod tests; diff --git a/questdb-rs/src/ingress/tests.rs b/questdb-rs/src/ingress/tests.rs new file mode 100644 index 00000000..d5c1c1c6 --- /dev/null +++ b/questdb-rs/src/ingress/tests.rs @@ -0,0 +1,461 @@ +use super::*; +use crate::ErrorCode; +use tempfile::TempDir; + +#[cfg(feature = "ilp-over-http")] +#[test] +fn http_simple() { + let builder = SenderBuilder::from_conf("http::addr=localhost;").unwrap(); + assert_eq!(builder.protocol, Protocol::Http); + assert_specified_eq(&builder.host, "localhost"); + assert_specified_eq(&builder.port, Protocol::Http.default_port()); + assert!(!builder.protocol.tls_enabled()); +} + +#[cfg(feature = "ilp-over-http")] +#[test] +fn https_simple() { + let builder = SenderBuilder::from_conf("https::addr=localhost;").unwrap(); + assert_eq!(builder.protocol, Protocol::Https); + assert_specified_eq(&builder.host, "localhost"); + assert_specified_eq(&builder.port, Protocol::Https.default_port()); + assert!(builder.protocol.tls_enabled()); + assert_defaulted_eq(&builder.tls_ca, CertificateAuthority::WebpkiRoots); +} + +#[test] +fn tcp_simple() { + let builder = SenderBuilder::from_conf("tcp::addr=localhost;").unwrap(); + assert_eq!(builder.protocol, Protocol::Tcp); + assert_specified_eq(&builder.port, Protocol::Tcp.default_port()); + assert_specified_eq(&builder.host, "localhost"); + assert!(!builder.protocol.tls_enabled()); +} + +#[test] +fn tcps_simple() { + let builder = SenderBuilder::from_conf("tcps::addr=localhost;").unwrap(); + assert_eq!(builder.protocol, Protocol::Tcps); + assert_specified_eq(&builder.host, "localhost"); + assert_specified_eq(&builder.port, Protocol::Tcps.default_port()); + assert!(builder.protocol.tls_enabled()); + assert_defaulted_eq(&builder.tls_ca, CertificateAuthority::WebpkiRoots); +} + +#[test] +fn invalid_value() { + assert_conf_err( + SenderBuilder::from_conf("tcp::addr=localhost\n;"), + "Config parse error: invalid char '\\n' in value at position 19", + ); +} + +#[test] +fn specified_cant_change() { + let mut builder = SenderBuilder::from_conf("tcp::addr=localhost;").unwrap(); + builder = builder.bind_interface("1.1.1.1").unwrap(); + assert_conf_err( + builder.bind_interface("1.1.1.2"), + "\"bind_interface\" is already specified", + ); +} + +#[test] +fn missing_addr() { + assert_conf_err( + SenderBuilder::from_conf("tcp::"), + "Missing \"addr\" parameter in config string", + ); +} + +#[test] +fn unsupported_service() { + assert_conf_err( + SenderBuilder::from_conf("xaxa::addr=localhost;"), + "Unsupported protocol: xaxa", + ); +} + +#[cfg(feature = "ilp-over-http")] +#[test] +fn http_basic_auth() { + let builder = + SenderBuilder::from_conf("http::addr=localhost;username=user123;password=pass321;") + .unwrap(); + let auth = builder.build_auth().unwrap(); + match auth.unwrap() { + AuthParams::Basic(BasicAuthParams { username, password }) => { + assert_eq!(username, "user123"); + assert_eq!(password, "pass321"); + } + _ => { + panic!("Expected AuthParams::Basic"); + } + } +} + +#[cfg(feature = "ilp-over-http")] +#[test] +fn http_token_auth() { + let builder = SenderBuilder::from_conf("http::addr=localhost:9000;token=token123;").unwrap(); + let auth = builder.build_auth().unwrap(); + match auth.unwrap() { + AuthParams::Token(TokenAuthParams { token }) => { + assert_eq!(token, "token123"); + } + _ => { + panic!("Expected AuthParams::Token"); + } + } +} + +#[cfg(feature = "ilp-over-http")] +#[test] +fn incomplete_basic_auth() { + assert_conf_err( + SenderBuilder::from_conf("http::addr=localhost;username=user123;") + .unwrap() + .build(), + "Basic authentication parameter \"username\" is present, but \"password\" is missing.", + ); + assert_conf_err( + SenderBuilder::from_conf("http::addr=localhost;password=pass321;") + .unwrap() + .build(), + "Basic authentication parameter \"password\" is present, but \"username\" is missing.", + ); +} + +#[cfg(feature = "ilp-over-http")] +#[test] +fn misspelled_basic_auth() { + assert_conf_err( + Sender::from_conf("http::addr=localhost;username=user123;pass=pass321;"), + r##"Basic authentication parameter "username" is present, but "password" is missing."##, + ); + assert_conf_err( + Sender::from_conf("http::addr=localhost;user=user123;password=pass321;"), + r##"Basic authentication parameter "password" is present, but "username" is missing."##, + ); +} + +#[cfg(feature = "ilp-over-http")] +#[test] +fn inconsistent_http_auth() { + let expected_err_msg = r##"Inconsistent HTTP authentication parameters. Specify either "username" and "password", or just "token"."##; + assert_conf_err( + Sender::from_conf("http::addr=localhost;username=user123;token=token123;"), + expected_err_msg, + ); + assert_conf_err( + Sender::from_conf("http::addr=localhost;password=pass321;token=token123;"), + expected_err_msg, + ); +} + +#[cfg(feature = "ilp-over-http")] +#[test] +fn cant_use_basic_auth_with_tcp() { + let builder = SenderBuilder::new(Protocol::Tcp, "localhost", 9000) + .username("user123") + .unwrap() + .password("pass321") + .unwrap(); + assert_conf_err( + builder.build_auth(), + "The \"basic_auth\" setting can only be used with the ILP/HTTP protocol.", + ); +} + +#[cfg(feature = "ilp-over-http")] +#[test] +fn cant_use_token_auth_with_tcp() { + let builder = SenderBuilder::new(Protocol::Tcp, "localhost", 9000) + .token("token123") + .unwrap(); + assert_conf_err( + builder.build_auth(), + "Token authentication only be used with the ILP/HTTP protocol.", + ); +} + +#[cfg(feature = "ilp-over-http")] +#[test] +fn cant_use_ecdsa_auth_with_http() { + let builder = SenderBuilder::from_conf("http::addr=localhost;") + .unwrap() + .username("key_id123") + .unwrap() + .token("priv_key123") + .unwrap() + .token_x("pub_key1") + .unwrap() + .token_y("pub_key2") + .unwrap(); + assert_conf_err( + builder.build_auth(), + "ECDSA authentication is only available with ILP/TCP and not available with ILP/HTTP.", + ); +} + +#[test] +fn set_auth_specifies_tcp() { + let mut builder = SenderBuilder::new(Protocol::Tcp, "localhost", 9000); + assert_eq!(builder.protocol, Protocol::Tcp); + builder = builder + .username("key_id123") + .unwrap() + .token("priv_key123") + .unwrap() + .token_x("pub_key1") + .unwrap() + .token_y("pub_key2") + .unwrap(); + assert_eq!(builder.protocol, Protocol::Tcp); +} + +#[test] +fn set_net_interface_specifies_tcp() { + let builder = SenderBuilder::new(Protocol::Tcp, "localhost", 9000); + assert_eq!(builder.protocol, Protocol::Tcp); + builder.bind_interface("55.88.0.4").unwrap(); +} + +#[test] +fn tcp_ecdsa_auth() { + let builder = SenderBuilder::from_conf( + "tcp::addr=localhost:9000;username=user123;token=token123;token_x=xtok123;token_y=ytok123;", + ) + .unwrap(); + let auth = builder.build_auth().unwrap(); + match auth.unwrap() { + AuthParams::Ecdsa(EcdsaAuthParams { + key_id, + priv_key, + pub_key_x, + pub_key_y, + }) => { + assert_eq!(key_id, "user123"); + assert_eq!(priv_key, "token123"); + assert_eq!(pub_key_x, "xtok123"); + assert_eq!(pub_key_y, "ytok123"); + } + #[cfg(feature = "ilp-over-http")] + _ => { + panic!("Expected AuthParams::Ecdsa"); + } + } +} + +#[test] +fn incomplete_tcp_ecdsa_auth() { + let expected_err_msg = r##"Incomplete ECDSA authentication parameters. Specify either all or none of: "username", "token", "token_x", "token_y"."##; + assert_conf_err( + SenderBuilder::from_conf("tcp::addr=localhost;username=user123;") + .unwrap() + .build(), + expected_err_msg, + ); + assert_conf_err( + SenderBuilder::from_conf("tcp::addr=localhost;username=user123;token=token123;") + .unwrap() + .build(), + expected_err_msg, + ); + assert_conf_err( + SenderBuilder::from_conf( + "tcp::addr=localhost;username=user123;token=token123;token_x=123;", + ) + .unwrap() + .build(), + expected_err_msg, + ); +} + +#[test] +fn misspelled_tcp_ecdsa_auth() { + assert_conf_err( + Sender::from_conf("tcp::addr=localhost;username=user123;tokenx=123;"), + "Incomplete ECDSA authentication parameters. Specify either all or none of: \"username\", \"token\", \"token_x\", \"token_y\"." + ); +} + +#[test] +fn tcps_tls_verify_on() { + let builder = SenderBuilder::from_conf("tcps::addr=localhost;tls_verify=on;").unwrap(); + assert!(builder.protocol.tls_enabled()); + assert_defaulted_eq(&builder.tls_ca, CertificateAuthority::WebpkiRoots); +} + +#[cfg(feature = "insecure-skip-verify")] +#[test] +fn tcps_tls_verify_unsafe_off() { + let builder = SenderBuilder::from_conf("tcps::addr=localhost;tls_verify=unsafe_off;").unwrap(); + assert!(builder.protocol.tls_enabled()); + assert_defaulted_eq(&builder.tls_ca, CertificateAuthority::WebpkiRoots); + assert_specified_eq(&builder.tls_verify, false); +} + +#[test] +fn tcps_tls_verify_invalid() { + assert_conf_err( + SenderBuilder::from_conf("tcps::addr=localhost;tls_verify=off;"), + r##"Config parameter "tls_verify" must be either "on" or "unsafe_off".'"##, + ); +} + +#[test] +fn tcps_tls_roots_webpki() { + let builder = SenderBuilder::from_conf("tcps::addr=localhost;tls_ca=webpki_roots;").unwrap(); + assert!(builder.protocol.tls_enabled()); + assert_specified_eq(&builder.tls_ca, CertificateAuthority::WebpkiRoots); + assert_defaulted_eq(&builder.tls_roots, None); +} + +#[cfg(feature = "tls-native-certs")] +#[test] +fn tcps_tls_roots_os() { + let builder = SenderBuilder::from_conf("tcps::addr=localhost;tls_ca=os_roots;").unwrap(); + assert!(builder.protocol.tls_enabled()); + assert_specified_eq(&builder.tls_ca, CertificateAuthority::OsRoots); + assert_defaulted_eq(&builder.tls_roots, None); +} + +#[test] +fn tcps_tls_roots_file() { + // Write a dummy file to test the file path + let tmp_dir = TempDir::new().unwrap(); + let path = tmp_dir.path().join("cacerts.pem"); + let mut file = std::fs::File::create(&path).unwrap(); + file.write_all(b"dummy").unwrap(); + let builder = SenderBuilder::from_conf(format!( + "tcps::addr=localhost;tls_roots={};", + path.to_str().unwrap() + )) + .unwrap(); + assert_specified_eq(&builder.tls_ca, CertificateAuthority::PemFile); + assert_specified_eq(&builder.tls_roots, path); +} + +#[test] +fn tcps_tls_roots_file_missing() { + let err = + SenderBuilder::from_conf("tcps::addr=localhost;tls_roots=/some/invalid/path/cacerts.pem;") + .unwrap_err(); + assert_eq!(err.code(), ErrorCode::ConfigError); + assert!(err + .msg() + .contains("Could not open root certificate file from path")); +} + +#[test] +fn tcps_tls_roots_file_with_password() { + let tmp_dir = TempDir::new().unwrap(); + let path = tmp_dir.path().join("cacerts.pem"); + let mut file = std::fs::File::create(&path).unwrap(); + file.write_all(b"dummy").unwrap(); + let builder_or_err = SenderBuilder::from_conf(format!( + "tcps::addr=localhost;tls_roots={};tls_roots_password=extremely_secure;", + path.to_str().unwrap() + )); + assert_conf_err(builder_or_err, "\"tls_roots_password\" is not supported."); +} + +#[cfg(feature = "ilp-over-http")] +#[test] +fn http_request_min_throughput() { + let builder = + SenderBuilder::from_conf("http::addr=localhost;request_min_throughput=100;").unwrap(); + let Some(http_config) = builder.http else { + panic!("Expected Some(HttpConfig)"); + }; + assert_specified_eq(&http_config.request_min_throughput, 100u64); + assert_defaulted_eq(&http_config.request_timeout, Duration::from_millis(10000)); + assert_defaulted_eq(&http_config.retry_timeout, Duration::from_millis(10000)); +} + +#[cfg(feature = "ilp-over-http")] +#[test] +fn http_request_timeout() { + let builder = SenderBuilder::from_conf("http::addr=localhost;request_timeout=100;").unwrap(); + let Some(http_config) = builder.http else { + panic!("Expected Some(HttpConfig)"); + }; + assert_defaulted_eq(&http_config.request_min_throughput, 102400u64); + assert_specified_eq(&http_config.request_timeout, Duration::from_millis(100)); + assert_defaulted_eq(&http_config.retry_timeout, Duration::from_millis(10000)); +} + +#[cfg(feature = "ilp-over-http")] +#[test] +fn http_retry_timeout() { + let builder = SenderBuilder::from_conf("http::addr=localhost;retry_timeout=100;").unwrap(); + let Some(http_config) = builder.http else { + panic!("Expected Some(HttpConfig)"); + }; + assert_defaulted_eq(&http_config.request_min_throughput, 102400u64); + assert_defaulted_eq(&http_config.request_timeout, Duration::from_millis(10000)); + assert_specified_eq(&http_config.retry_timeout, Duration::from_millis(100)); +} + +#[test] +fn auto_flush_off() { + SenderBuilder::from_conf("tcps::addr=localhost;auto_flush=off;").unwrap(); +} + +#[test] +fn auto_flush_unsupported() { + assert_conf_err( + SenderBuilder::from_conf("tcps::addr=localhost;auto_flush=on;"), + "Invalid auto_flush value 'on'. This client does not support \ + auto-flush, so the only accepted value is 'off'", + ); +} + +#[test] +fn auto_flush_rows_unsupported() { + assert_conf_err( + SenderBuilder::from_conf("tcps::addr=localhost;auto_flush_rows=100;"), + "Invalid configuration parameter \"auto_flush_rows\". This client does not support auto-flush", + ); +} + +#[test] +fn auto_flush_bytes_unsupported() { + assert_conf_err( + SenderBuilder::from_conf("tcps::addr=localhost;auto_flush_bytes=100;"), + "Invalid configuration parameter \"auto_flush_bytes\". This client does not support auto-flush", + ); +} + +fn assert_specified_eq>( + actual: &ConfigSetting, + expected: IntoV, +) { + let expected = expected.into(); + if let ConfigSetting::Specified(actual_value) = actual { + assert_eq!(actual_value, &expected); + } else { + panic!("Expected Specified({:?}), but got {:?}", expected, actual); + } +} + +fn assert_defaulted_eq>( + actual: &ConfigSetting, + expected: IntoV, +) { + let expected = expected.into(); + if let ConfigSetting::Defaulted(actual_value) = actual { + assert_eq!(actual_value, &expected); + } else { + panic!("Expected Defaulted({:?}), but got {:?}", expected, actual); + } +} + +fn assert_conf_err>(result: Result, expect_msg: M) { + let Err(err) = result else { + panic!("Got Ok, expected ConfigError: {}", expect_msg.as_ref()); + }; + assert_eq!(err.code(), ErrorCode::ConfigError); + assert_eq!(err.msg(), expect_msg.as_ref()); +} diff --git a/questdb-rs/src/lib.rs b/questdb-rs/src/lib.rs index 444ea770..e72b88d2 100644 --- a/questdb-rs/src/lib.rs +++ b/questdb-rs/src/lib.rs @@ -6,7 +6,7 @@ * \__\_\\__,_|\___||___/\__|____/|____/ * * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2023 QuestDB + * Copyright (c) 2019-2024 QuestDB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/questdb-rs/src/tests/f64_serializer.rs b/questdb-rs/src/tests/f64_serializer.rs index 5f166865..3ba023e3 100644 --- a/questdb-rs/src/tests/f64_serializer.rs +++ b/questdb-rs/src/tests/f64_serializer.rs @@ -6,7 +6,7 @@ * \__\_\\__,_|\___||___/\__|____/|____/ * * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2023 QuestDB + * Copyright (c) 2019-2024 QuestDB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/questdb-rs/src/tests/http.rs b/questdb-rs/src/tests/http.rs new file mode 100644 index 00000000..7f40ca5c --- /dev/null +++ b/questdb-rs/src/tests/http.rs @@ -0,0 +1,709 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2024 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +use crate::ingress::{Buffer, Protocol, SenderBuilder, TimestampNanos}; +use crate::tests::mock::{certs_dir, HttpResponse, MockServer}; +use crate::ErrorCode; +use std::io; +use std::io::ErrorKind; +use std::time::Duration; + +use crate::tests::TestResult; + +#[test] +fn test_two_lines() -> TestResult { + let mut buffer = Buffer::new(); + buffer + .table("test")? + .symbol("sym", "bol")? + .column_f64("x", 1.0)? + .at_now()?; + buffer + .table("test")? + .symbol("sym", "bol")? + .column_f64("x", 2.0)? + .at_now()?; + let buffer2 = buffer.clone(); + + let mut server = MockServer::new()?; + let mut sender = server.lsb_http().build()?; + + let server_thread = std::thread::spawn(move || -> io::Result<()> { + server.accept()?; + + let req = server.recv_http_q()?; + assert_eq!(req.method(), "POST"); + assert_eq!(req.path(), "/write?precision=n"); + assert_eq!( + req.header("user-agent"), + Some(concat!("questdb/rust/", env!("CARGO_PKG_VERSION"))) + ); + assert_eq!(req.body_str().unwrap(), buffer2.as_str()); + + server.send_http_response_q(HttpResponse::empty())?; + + Ok(()) + }); + + let res = sender.flush(&mut buffer); + + server_thread.join().unwrap()?; + + res?; + + assert!(buffer.is_empty()); + + Ok(()) +} + +#[test] +fn test_text_plain_error() -> TestResult { + let mut buffer = Buffer::new(); + buffer + .table("test")? + .symbol("sym", "bol")? + .column_f64("x", 1.0)? + .at_now()?; + buffer.table("test")?.column_f64("sym", 2.0)?.at_now()?; + + let mut server = MockServer::new()?; + let mut sender = server.lsb_http().build()?; + + let buffer2 = buffer.clone(); + let server_thread = std::thread::spawn(move || -> io::Result<()> { + server.accept()?; + + let req = server.recv_http_q()?; + assert_eq!(req.method(), "POST"); + assert_eq!(req.path(), "/write?precision=n"); + assert_eq!(req.body_str().unwrap(), buffer2.as_str()); + + server.send_http_response_q( + HttpResponse::empty() + .with_status(400, "Bad Request") + .with_header("content-type", "text/plain") + .with_body_str("bad wombat"), + )?; + + Ok(()) + }); + + let res = sender.flush(&mut buffer); + + server_thread.join().unwrap()?; + + assert!(res.is_err()); + let err = res.unwrap_err(); + assert_eq!(err.code(), ErrorCode::ServerFlushError); + assert_eq!(err.msg(), "Could not flush buffer: bad wombat"); + + assert!(!buffer.is_empty()); + + Ok(()) +} + +#[test] +fn test_bad_json_error() -> TestResult { + let mut buffer = Buffer::new(); + buffer + .table("test")? + .symbol("sym", "bol")? + .column_f64("x", 1.0)? + .at_now()?; + buffer.table("test")?.column_f64("sym", 2.0)?.at_now()?; + + let mut server = MockServer::new()?; + let mut sender = server.lsb_http().build()?; + + let buffer2 = buffer.clone(); + let server_thread = std::thread::spawn(move || -> io::Result<()> { + server.accept()?; + + let req = server.recv_http_q()?; + assert_eq!(req.method(), "POST"); + assert_eq!(req.path(), "/write?precision=n"); + assert_eq!(req.body_str().unwrap(), buffer2.as_str()); + + server.send_http_response_q( + HttpResponse::empty() + .with_status(400, "Bad Request") + .with_body_json(&serde_json::json!({ + "error": "bad wombat", + })), + )?; + + Ok(()) + }); + + let res = sender.flush_and_keep(&buffer); + + server_thread.join().unwrap()?; + + assert!(res.is_err()); + let err = res.unwrap_err(); + assert_eq!(err.code(), ErrorCode::ServerFlushError); + assert_eq!( + err.msg(), + "Could not flush buffer: {\"error\":\"bad wombat\"}" + ); + + Ok(()) +} + +#[test] +fn test_json_error() -> TestResult { + let mut buffer = Buffer::new(); + buffer + .table("test")? + .symbol("sym", "bol")? + .column_f64("x", 1.0)? + .at_now()?; + buffer.table("test")?.column_f64("sym", 2.0)?.at_now()?; + + let mut server = MockServer::new()?; + let mut sender = server.lsb_http().build()?; + + let buffer2 = buffer.clone(); + let server_thread = std::thread::spawn(move || -> io::Result<()> { + server.accept()?; + + let req = server.recv_http_q()?; + assert_eq!(req.method(), "POST"); + assert_eq!(req.path(), "/write?precision=n"); + assert_eq!(req.body_str().unwrap(), buffer2.as_str()); + + server.send_http_response_q( + HttpResponse::empty() + .with_status(400, "Bad Request") + .with_body_json(&serde_json::json!({ + "code": "invalid", + "message": "failed to parse line protocol: invalid field format", + "errorId": "ABC-2", + "line": 2, + })), + )?; + + Ok(()) + }); + + let res = sender.flush_and_keep(&buffer); + + server_thread.join().unwrap()?; + + assert!(res.is_err()); + let err = res.unwrap_err(); + assert_eq!(err.code(), ErrorCode::ServerFlushError); + assert_eq!( + err.msg(), + "Could not flush buffer: failed to parse line protocol: invalid field format [id: ABC-2, code: invalid, line: 2]" + ); + + Ok(()) +} + +#[test] +fn test_no_connection() -> TestResult { + let mut buffer = Buffer::new(); + buffer + .table("test")? + .symbol("sym", "bol")? + .column_f64("x", 1.0)? + .at_now()?; + + let mut sender = SenderBuilder::new(Protocol::Http, "127.0.0.1", 1).build()?; + let res = sender.flush_and_keep(&buffer); + assert!(res.is_err()); + let err = res.unwrap_err(); + assert_eq!(err.code(), ErrorCode::SocketError); + assert!(err.msg().starts_with( + "Could not flush buffer: http://127.0.0.1:1/write?precision=n: Connection Failed" + )); + Ok(()) +} + +#[test] +fn test_old_server_without_ilp_http_support() -> TestResult { + let mut buffer = Buffer::new(); + buffer + .table("test")? + .symbol("sym", "bol")? + .column_f64("x", 1.0)? + .at_now()?; + + let mut server = MockServer::new()?; + let mut sender = server.lsb_http().build()?; + + let buffer2 = buffer.clone(); + let server_thread = std::thread::spawn(move || -> io::Result<()> { + server.accept()?; + + let req = server.recv_http_q()?; + assert_eq!(req.method(), "POST"); + assert_eq!(req.path(), "/write?precision=n"); + assert_eq!(req.body_str().unwrap(), buffer2.as_str()); + + server.send_http_response_q( + HttpResponse::empty() + .with_status(404, "Not Found") + .with_header("content-type", "text/plain") + .with_body_str("Not Found"), + )?; + + Ok(()) + }); + + let res = sender.flush_and_keep(&buffer); + + server_thread.join().unwrap()?; + + assert!(res.is_err()); + let err = res.unwrap_err(); + assert_eq!(err.code(), ErrorCode::HttpNotSupported); + assert_eq!( + err.msg(), + "Could not flush buffer: HTTP endpoint does not support ILP." + ); + + Ok(()) +} + +#[test] +fn test_http_basic_auth() -> TestResult { + let mut buffer = Buffer::new(); + buffer + .table("test")? + .symbol("sym", "bol")? + .column_f64("x", 1.0)? + .at_now()?; + + let mut server = MockServer::new()?; + let mut sender = server + .lsb_http() + .username("Aladdin")? + .password("OpenSesame")? + .build()?; + + let buffer2 = buffer.clone(); + let server_thread = std::thread::spawn(move || -> io::Result<()> { + server.accept()?; + + let req = server.recv_http_q()?; + + assert_eq!(req.method(), "POST"); + assert_eq!(req.path(), "/write?precision=n"); + assert_eq!( + req.header("authorization"), + Some("Basic QWxhZGRpbjpPcGVuU2VzYW1l") + ); + assert_eq!(req.body_str().unwrap(), buffer2.as_str()); + + server.send_http_response_q(HttpResponse::empty())?; + + Ok(()) + }); + + let res = sender.flush(&mut buffer); + + server_thread.join().unwrap()?; + + res?; + + assert!(buffer.is_empty()); + + Ok(()) +} + +#[test] +fn test_unauthenticated() -> TestResult { + let mut buffer = Buffer::new(); + buffer + .table("test")? + .symbol("sym", "bol")? + .column_f64("x", 1.0)? + .at_now()?; + + let mut server = MockServer::new()?; + let mut sender = server.lsb_http().build()?; + + let buffer2 = buffer.clone(); + let server_thread = std::thread::spawn(move || -> io::Result<()> { + server.accept()?; + + let req = server.recv_http_q()?; + assert_eq!(req.method(), "POST"); + assert_eq!(req.path(), "/write?precision=n"); + assert_eq!(req.body_str().unwrap(), buffer2.as_str()); + + server.send_http_response_q( + HttpResponse::empty() + .with_status(401, "Unauthorized") + .with_body_str("Unauthorized") + .with_header("WWW-Authenticate", "Basic realm=\"Our Site\""), + )?; + + Ok(()) + }); + + let res = sender.flush(&mut buffer); + + server_thread.join().unwrap()?; + + assert!(res.is_err()); + let err = res.unwrap_err(); + assert_eq!(err.code(), ErrorCode::AuthError); + assert_eq!( + err.msg(), + "Could not flush buffer: HTTP endpoint authentication error: Unauthorized [code: 401]" + ); + + assert!(!buffer.is_empty()); + + Ok(()) +} + +#[test] +fn test_token_auth() -> TestResult { + let mut buffer = Buffer::new(); + buffer + .table("test")? + .symbol("sym", "bol")? + .column_f64("x", 1.0)? + .at_now()?; + + let mut server = MockServer::new()?; + let mut sender = server.lsb_http().token("0123456789")?.build()?; + + let buffer2 = buffer.clone(); + let server_thread = std::thread::spawn(move || -> io::Result<()> { + server.accept()?; + + let req = server.recv_http_q()?; + assert_eq!(req.method(), "POST"); + assert_eq!(req.path(), "/write?precision=n"); + assert_eq!(req.header("authorization"), Some("Bearer 0123456789")); + assert_eq!(req.body_str().unwrap(), buffer2.as_str()); + + server.send_http_response_q(HttpResponse::empty())?; + + Ok(()) + }); + + let res = sender.flush(&mut buffer); + + server_thread.join().unwrap()?; + + res?; + + Ok(()) +} + +#[test] +fn test_request_timeout() -> TestResult { + let mut buffer = Buffer::new(); + buffer + .table("test")? + .symbol("sym", "bol")? + .column_f64("x", 1.0)? + .at_now()?; + + // Here we use a mock (tcp) server instead and don't send a response back. + let server = MockServer::new()?; + + let request_timeout = Duration::from_millis(50); + let time_start = std::time::Instant::now(); + let mut sender = server + .lsb_http() + .request_timeout(request_timeout)? + .build()?; + let res = sender.flush_and_keep(&buffer); + let time_elapsed = time_start.elapsed(); + assert!(res.is_err()); + let err = res.unwrap_err(); + assert_eq!(err.code(), ErrorCode::SocketError); + assert!(err.msg().contains("timed out reading response")); + assert!(time_elapsed >= request_timeout); + Ok(()) +} + +#[test] +fn test_tls() -> TestResult { + let mut ca_path = certs_dir(); + ca_path.push("server_rootCA.pem"); + + let mut buffer = Buffer::new(); + buffer + .table("test")? + .symbol("t1", "v1")? + .column_f64("f1", 0.5)? + .at(TimestampNanos::new(10000000))?; + let buffer2 = buffer.clone(); + + let mut server = MockServer::new()?; + let mut sender = server.lsb_https().tls_roots(ca_path)?.build()?; + + let server_thread = std::thread::spawn(move || -> io::Result<()> { + server.accept_tls_sync()?; + let req = server.recv_http_q()?; + assert_eq!(req.method(), "POST"); + assert_eq!(req.path(), "/write?precision=n"); + assert_eq!(req.body_str().unwrap(), buffer2.as_str()); + + server.send_http_response_q(HttpResponse::empty())?; + + Ok(()) + }); + + let res = sender.flush_and_keep(&buffer); + + server_thread.join().unwrap()?; + + // Unpacking the error here allows server errors to bubble first. + res?; + + Ok(()) +} + +#[test] +fn test_user_agent() -> TestResult { + let mut buffer = Buffer::new(); + buffer + .table("test")? + .symbol("t1", "v1")? + .column_f64("f1", 0.5)? + .at(TimestampNanos::new(10000000))?; + let buffer2 = buffer.clone(); + + let mut server = MockServer::new()?; + let mut sender = server.lsb_http().user_agent("wallabies/1.2.99")?.build()?; + + let server_thread = std::thread::spawn(move || -> io::Result<()> { + server.accept()?; + + let req = server.recv_http_q()?; + assert_eq!(req.header("user-agent"), Some("wallabies/1.2.99")); + assert_eq!(req.body_str().unwrap(), buffer2.as_str()); + + server.send_http_response_q(HttpResponse::empty())?; + + Ok(()) + }); + + let res = sender.flush_and_keep(&buffer); + + server_thread.join().unwrap()?; + + // Unpacking the error here allows server errors to bubble first. + res?; + + Ok(()) +} + +#[test] +fn test_two_retries() -> TestResult { + // Note: This also tests that the _same_ connection is being reused, i.e. tests keepalive. + + let mut buffer = Buffer::new(); + buffer + .table("test")? + .symbol("t1", "v1")? + .column_f64("f1", 0.5)? + .at(TimestampNanos::new(10000000))?; + let buffer2 = buffer.clone(); + + let mut server = MockServer::new()?; + let mut sender = server + .lsb_http() + .retry_timeout(Duration::from_secs(30))? + .build()?; + + let server_thread = std::thread::spawn(move || -> io::Result<()> { + server.accept()?; + + let req = server.recv_http_q()?; + assert_eq!(req.body_str().unwrap(), buffer2.as_str()); + + server.send_http_response_q( + HttpResponse::empty() + .with_status(500, "Internal Server Error") + .with_body_str("client should retry"), + )?; + + let start_time = std::time::Instant::now(); + + let req = server.recv_http_q()?; + assert_eq!(req.body_str().unwrap(), buffer2.as_str()); + let elapsed = std::time::Instant::now().duration_since(start_time); + assert!(elapsed > Duration::from_millis(5)); + + server.send_http_response_q( + HttpResponse::empty() + .with_status(500, "Internal Server Error") + .with_body_str("client should retry"), + )?; + + let start_time = std::time::Instant::now(); + + let req = server.recv_http_q()?; + assert_eq!(req.body_str().unwrap(), buffer2.as_str()); + let elapsed = std::time::Instant::now().duration_since(start_time); + assert!(elapsed > Duration::from_millis(15)); + + server.send_http_response_q(HttpResponse::empty())?; + + Ok(()) + }); + + let res = sender.flush_and_keep(&buffer); + + server_thread.join().unwrap()?; + + // Unpacking the error here allows server errors to bubble first. + res?; + + Ok(()) +} + +#[test] +fn test_one_retry() -> TestResult { + let mut buffer = Buffer::new(); + buffer + .table("test")? + .symbol("t1", "v1")? + .column_f64("f1", 0.5)? + .at(TimestampNanos::new(10000000))?; + let buffer2 = buffer.clone(); + + let mut server = MockServer::new()?; + let mut sender = server + .lsb_http() + .retry_timeout(Duration::from_millis(19))? + .build()?; + + let server_thread = std::thread::spawn(move || -> io::Result<()> { + server.accept()?; + + let req = server.recv_http_q()?; + assert_eq!(req.body_str().unwrap(), buffer2.as_str()); + + server.send_http_response_q( + HttpResponse::empty() + .with_status(500, "Internal Server Error") + .with_body_str("error 1"), + )?; + + let req = server.recv_http_q()?; + assert_eq!(req.body_str().unwrap(), buffer2.as_str()); + + server.send_http_response_q( + HttpResponse::empty() + .with_status(500, "Internal Server Error") + .with_body_str("error 2"), + )?; + + let req = server.recv_http(2.0); + + let err = match req { + Ok(_) => { + return Err(io::Error::new( + ErrorKind::InvalidInput, + "unexpected retry response", + )) + } + Err(err) => err, + }; + assert_eq!(err.kind(), ErrorKind::TimedOut); + + Ok(()) + }); + + let res = sender.flush_and_keep(&buffer); + + server_thread.join().unwrap()?; + + let err = res.unwrap_err(); + assert_eq!(err.code(), ErrorCode::ServerFlushError); + assert_eq!(err.msg(), "Could not flush buffer: error 2"); + + Ok(()) +} + +#[test] +fn test_transactional() -> TestResult { + // A buffer with a two tables. + let mut buffer1 = Buffer::new(); + buffer1 + .table("tab1")? + .symbol("t1", "v1")? + .column_f64("f1", 0.5)? + .at(TimestampNanos::new(10000001))?; + buffer1 + .table("tab2")? + .symbol("t1", "v1")? + .column_f64("f1", 0.6)? + .at(TimestampNanos::new(10000002))?; + assert!(!buffer1.transactional()); + + // A buffer with a single table. + let mut buffer2 = Buffer::new(); + buffer2 + .table("test")? + .symbol("t1", "v1")? + .column_f64("f1", 0.5)? + .at(TimestampNanos::new(10000000))?; + let buffer3 = buffer2.clone(); + assert!(buffer2.transactional()); + + let mut server = MockServer::new()?; + let mut sender = server.lsb_http().build()?; + + let server_thread = std::thread::spawn(move || -> io::Result<()> { + server.accept()?; + + let req = server.recv_http_q()?; + assert_eq!(req.body_str().unwrap(), buffer3.as_str()); + + server.send_http_response_q(HttpResponse::empty())?; + + Ok(()) + }); + + let res = sender.flush_and_keep_with_flags(&buffer1, true); + assert!(res.is_err()); + let err = res.unwrap_err(); + assert_eq!(err.code(), ErrorCode::InvalidApiCall); + assert_eq!( + err.msg(), + "Buffer contains lines for multiple tables. \ + Transactional flushes are only supported for buffers containing lines for a single table." + ); + + let res = sender.flush_and_keep_with_flags(&buffer2, true); + + server_thread.join().unwrap()?; + + // Unpacking the error here allows server errors to bubble first. + res?; + + Ok(()) +} diff --git a/questdb-rs/src/tests/mock.rs b/questdb-rs/src/tests/mock.rs index d200bea8..5d2dc38e 100644 --- a/questdb-rs/src/tests/mock.rs +++ b/questdb-rs/src/tests/mock.rs @@ -6,7 +6,7 @@ * \__\_\\__,_|\___||___/\__|____/|____/ * * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2023 QuestDB + * Copyright (c) 2019-2024 QuestDB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,20 +22,22 @@ * ******************************************************************************/ -use crate::ingress::SenderBuilder; +use crate::ingress::{Protocol, SenderBuilder}; use core::time::Duration; +use mio::event::Event; use mio::net::TcpStream; use mio::{Events, Interest, Poll, Token}; -use rustls::{ - server::{NoClientAuth, ServerConnection}, - Certificate, ServerConfig, Stream, -}; -use socket2::{Domain, Protocol, Socket, Type}; +use rustls::{server::ServerConnection, ServerConfig, Stream}; +use socket2::{Domain, Protocol as SockProtocol, Socket, Type}; +use std::fs::File; use std::io::{self, BufReader, Read}; use std::net::SocketAddr; -use std::path::Path; use std::sync::Arc; +use std::time::Instant; + +#[cfg(feature = "ilp-over-http")] +use std::io::Write; const CLIENT: Token = Token(0); @@ -51,36 +53,6 @@ pub struct MockServer { pub msgs: Vec, } -fn load_certs(filename: &Path) -> Vec { - let certfile = std::fs::File::open(filename).expect("cannot open certificate file"); - let mut reader = BufReader::new(certfile); - rustls_pemfile::certs(&mut reader) - .unwrap() - .iter() - .map(|v| Certificate(v.clone())) - .collect() -} - -fn load_private_key(filename: &Path) -> rustls::PrivateKey { - let keyfile = std::fs::File::open(filename).expect("cannot open private key file"); - let mut reader = BufReader::new(keyfile); - - loop { - match rustls_pemfile::read_one(&mut reader).expect("cannot parse private key .pem file") { - Some(rustls_pemfile::Item::RSAKey(key)) => return rustls::PrivateKey(key), - Some(rustls_pemfile::Item::PKCS8Key(key)) => return rustls::PrivateKey(key), - Some(rustls_pemfile::Item::ECKey(key)) => return rustls::PrivateKey(key), - None => break, - _ => {} - } - } - - panic!( - "no keys found in {:?} (encrypted keys not supported)", - filename - ); -} - pub fn certs_dir() -> std::path::PathBuf { let mut certs_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")); certs_dir.pop(); @@ -90,22 +62,141 @@ pub fn certs_dir() -> std::path::PathBuf { fn tls_config() -> Arc { let certs_dir = certs_dir(); - let cert_chain = load_certs(&certs_dir.join("server.crt")); - let key_der = load_private_key(&certs_dir.join("server.key")); + let mut cert_file = + File::open(certs_dir.join("server.crt")).expect("cannot open certificate file"); + let mut private_key_file = + File::open(certs_dir.join("server.key")).expect("cannot open private key file"); + let certs = rustls_pemfile::certs(&mut BufReader::new(&mut cert_file)) + .collect::, _>>() + .expect("cannot read certificate file"); + let private_key = rustls_pemfile::private_key(&mut BufReader::new(&mut private_key_file)) + .expect("cannot read private key file") + .expect("no private key found"); let config = ServerConfig::builder() - .with_safe_default_cipher_suites() - .with_safe_default_kx_groups() - .with_safe_default_protocol_versions() - .unwrap() - .with_client_cert_verifier(NoClientAuth::boxed()) - .with_single_cert(cert_chain, key_der) + .with_no_client_auth() + .with_single_cert(certs, private_key) .unwrap(); Arc::new(config) } +#[cfg(feature = "ilp-over-http")] +pub struct HttpRequest { + method: String, + path: String, + headers: std::collections::HashMap, + body: Vec, +} + +#[cfg(feature = "ilp-over-http")] +impl HttpRequest { + pub fn method(&self) -> &str { + &self.method + } + + pub fn path(&self) -> &str { + &self.path + } + + pub fn header(&self, key: &str) -> Option<&str> { + self.headers.get(key).map(|s| s.as_str()) + } + + pub fn body(&self) -> &[u8] { + &self.body + } + + pub fn body_str(&self) -> Result<&str, std::str::Utf8Error> { + std::str::from_utf8(self.body()) + } +} + +#[cfg(feature = "ilp-over-http")] +pub struct HttpResponse { + status_code: u16, + status_text: String, + headers: std::collections::HashMap, + body: Vec, +} + +#[cfg(feature = "ilp-over-http")] +impl HttpResponse { + pub fn empty() -> Self { + HttpResponse { + status_code: 204, + status_text: "No Content".to_string(), + headers: std::collections::HashMap::new(), + body: Vec::new(), + } + } + + pub fn with_status(mut self, code: u16, text: &str) -> Self { + self.status_code = code; + self.status_text = text.to_string(); + self + } + + pub fn with_header(mut self, key: &str, value: &str) -> Self { + self.headers.insert(key.to_string(), value.to_string()); + self + } + + pub fn with_body(mut self, body: &[u8]) -> Self { + self.body = body.to_vec(); + if self.status_code == 204 { + self.status_code = 200; + self.status_text = "OK".to_string(); + } + if !self.headers.contains_key("content-length") { + self.headers + .insert("content-length".to_string(), self.body.len().to_string()); + } + self + } + + pub fn with_body_str(mut self, body: &str) -> Self { + if !self.headers.contains_key("content-type") { + self.headers + .insert("content-type".to_string(), "text/plain".to_string()); + } + self.with_body(body.as_bytes()) + } + + pub fn with_body_json(mut self, body: &serde_json::Value) -> Self { + if !self.headers.contains_key("content-type") { + self.headers + .insert("content-type".to_string(), "application/json".to_string()); + } + self.with_body_str(&body.to_string()) + } + + pub fn as_string(&self) -> String { + let mut s = format!("HTTP/1.1 {} {}\r\n", self.status_code, self.status_text); + for (key, value) in &self.headers { + s.push_str(&format!("{}: {}\r\n", key, value)); + } + s.push_str("\r\n"); + s.push_str(std::str::from_utf8(&self.body).unwrap()); + s + } +} + +#[cfg(feature = "ilp-over-http")] +fn contains(haystack: &[u8], needle: &[u8]) -> bool { + haystack + .windows(needle.len()) + .any(|window| window == needle) +} + +#[cfg(feature = "ilp-over-http")] +fn position(haystack: &[u8], needle: &[u8]) -> Option { + haystack + .windows(needle.len()) + .position(|window| window == needle) +} + impl MockServer { pub fn new() -> io::Result { - let listener = Socket::new(Domain::IPV4, Type::STREAM, Some(Protocol::TCP))?; + let listener = Socket::new(Domain::IPV4, Type::STREAM, Some(SockProtocol::TCP))?; let address: SocketAddr = "127.0.0.1:0".parse().unwrap(); listener.bind(&address.into())?; listener.listen(128)?; @@ -127,58 +218,82 @@ impl MockServer { client.set_nonblocking(true)?; let client: std::net::TcpStream = client.into(); let mut client = TcpStream::from_std(client); - self.poll - .registry() - .register(&mut client, CLIENT, Interest::READABLE)?; + self.poll.registry().register( + &mut client, + CLIENT, + Interest::READABLE | Interest::WRITABLE, + )?; self.client = Some(client); Ok(()) } - pub fn accept_tls(mut self) -> std::thread::JoinHandle> { - std::thread::spawn(|| { - self.accept()?; - let client = self.client.as_mut().unwrap(); - self.poll.registry().reregister( - client, - CLIENT, - Interest::READABLE | Interest::WRITABLE, - )?; - let mut tls_conn = ServerConnection::new(tls_config()).unwrap(); - let mut stream = Stream::new(&mut tls_conn, client); - let begin = std::time::Instant::now(); - while stream.conn.is_handshaking() { - match stream.conn.complete_io(&mut stream.sock) { - Ok(_) => (), - Err(err) => { - if err.kind() == io::ErrorKind::WouldBlock { - let now = std::time::Instant::now(); - let elapsed = now.duration_since(begin); - if elapsed > Duration::from_secs(2) { - return Err(err); - } - self.poll - .poll(&mut self.events, Some(Duration::from_millis(200)))?; - } else { + pub fn accept_tls_sync(&mut self) -> io::Result<()> { + self.accept()?; + let client = self.client.as_mut().unwrap(); + let mut tls_conn = ServerConnection::new(tls_config()).unwrap(); + let mut stream = Stream::new(&mut tls_conn, client); + let begin = Instant::now(); + while stream.conn.is_handshaking() { + match stream.conn.complete_io(&mut stream.sock) { + Ok(_) => (), + Err(err) => { + if err.kind() == io::ErrorKind::WouldBlock { + let now = Instant::now(); + let elapsed = now.duration_since(begin); + if elapsed > Duration::from_secs(2) { return Err(err); } + self.poll + .poll(&mut self.events, Some(Duration::from_millis(200)))?; + } else { + return Err(err); } } } - self.poll - .registry() - .reregister(client, CLIENT, Interest::READABLE)?; - self.tls_conn = Some(tls_conn); + } + self.tls_conn = Some(tls_conn); + Ok(()) + } + + pub fn accept_tls(mut self) -> std::thread::JoinHandle> { + std::thread::spawn(|| { + self.accept_tls_sync()?; Ok(self) }) } - pub fn wait_for_data(&mut self, wait_timeout_sec: Option) -> io::Result { + fn wait_for

(&mut self, timeout: Option, event_predicate: P) -> io::Result + where + P: Fn(&Event) -> bool, + { // To ensure a clean death if accept wasn't called. self.client.as_ref().unwrap(); - let timeout = wait_timeout_sec.map(|sec| Duration::from_micros((sec * 1000000.0) as u64)); - self.poll.poll(&mut self.events, timeout)?; - let ready_for_read = !self.events.is_empty(); - Ok(ready_for_read) + let deadline = timeout.map(|d| Instant::now() + d); + loop { + let timeout = match deadline { + Some(deadline) => { + let timeout = deadline.checked_duration_since(Instant::now()); + if timeout.is_none() { + return Ok(false); // timed out + } + timeout + } + None => None, + }; + self.poll.poll(&mut self.events, timeout)?; + if self.events.iter().any(&event_predicate) { + return Ok(true); // evt matched + } + } + } + + pub fn wait_for_recv(&mut self, timeout: Option) -> io::Result { + self.wait_for(timeout, |event| event.is_readable()) + } + + #[cfg(feature = "ilp-over-http")] + pub fn wait_for_send(&mut self, duration: Option) -> io::Result { + self.wait_for(duration, |event| event.is_writable()) } fn do_read(&mut self, buf: &mut [u8]) -> io::Result { @@ -191,10 +306,195 @@ impl MockServer { } } - pub fn recv(&mut self, wait_timeout_sec: f64) -> io::Result { - if !self.wait_for_data(Some(wait_timeout_sec))? { - return Ok(0); + #[cfg(feature = "ilp-over-http")] + fn do_write(&mut self, buf: &[u8]) -> io::Result { + let client = self.client.as_mut().unwrap(); + if let Some(tls_conn) = self.tls_conn.as_mut() { + let mut stream = Stream::new(tls_conn, client); + stream.write(buf) + } else { + client.write(buf) + } + } + + #[cfg(feature = "ilp-over-http")] + fn do_write_all(&mut self, buf: &[u8], timeout_sec: Option) -> io::Result<()> { + let deadline = timeout_sec.map(|sec| Instant::now() + Duration::from_secs_f64(sec)); + let mut pos = 0usize; + loop { + // `self.poll` is edge-triggered, so we need to write first + // until we get an EAGAIN, then wait for the socket to become writable again. + match self.do_write(&buf[pos..]) { + Ok(count) => pos += count, + Err(ref err) if err.kind() == io::ErrorKind::WouldBlock => (), + Err(err) => return Err(err), + } + + if pos == buf.len() { + break; + } + + let timeout = match deadline { + Some(deadline) => Some( + deadline + .checked_duration_since(Instant::now()) + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::TimedOut, + "Timed out while waiting for send", + ) + })?, + ), + None => None, + }; + let _ = !self.wait_for_send(timeout)?; } + Ok(()) + } + + #[cfg(feature = "ilp-over-http")] + fn read_more(&mut self, accum: &mut Vec, deadline: Instant, stage: &str) -> io::Result<()> { + let mut chunk = [0u8; 1024]; + let count = match self.do_read(&mut chunk[..]) { + Ok(count) => count, + Err(ref err) if err.kind() == io::ErrorKind::WouldBlock => { + let timeout = match deadline.checked_duration_since(Instant::now()) { + Some(timeout) => timeout, + None => { + let mut so_far = String::new(); + for &b in accum.iter() { + let part: Vec = std::ascii::escape_default(b).collect(); + so_far.push_str(std::str::from_utf8(&part).unwrap()); + } + return Err(io::Error::new( + io::ErrorKind::TimedOut, + format!( + "{} timed out while waiting for data. Received so far: {}", + stage, so_far + ), + )); + } + }; + if self.wait_for_recv(Some(timeout))? { + // After blocking on poll, we've received a READABLE event. + // So we try again. + self.do_read(&mut chunk[..])? + } else { + return Ok(()); // No more data + } + } + Err(err) => return Err(err), + }; + accum.extend(&chunk[..count]); + + Ok(()) + } + + #[cfg(feature = "ilp-over-http")] + fn recv_http_method( + &mut self, + accum: &mut Vec, + deadline: Instant, + ) -> io::Result<(usize, String, String)> { + let end_of_line_separator = b"\r\n"; + while !contains(&accum[..], end_of_line_separator) { + self.read_more(accum, deadline, "Reading HTTP method line")?; + } + let end_of_line = position(&accum[..], b"\r\n").unwrap(); + let line = std::str::from_utf8(&accum[..end_of_line]).unwrap(); + let mut parts = line.splitn(3, ' '); + let mut method = parts + .next() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "Missing HTTP method"))? + .to_string(); + method.make_ascii_uppercase(); // case-insensitive method names + let path = parts + .next() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "Missing HTTP path"))? + .to_string(); + let _http_version = parts + .next() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "Missing HTTP version"))?; + let body_start = end_of_line + end_of_line_separator.len(); + Ok((body_start, method, path)) + } + + #[cfg(feature = "ilp-over-http")] + fn recv_http_headers( + &mut self, + pos: usize, + accum: &mut Vec, + deadline: Instant, + ) -> io::Result<(usize, std::collections::HashMap)> { + let mut headers = std::collections::HashMap::::new(); + + let header_section_sep = b"\r\n\r\n"; + while !contains(&accum[pos..], header_section_sep) { + self.read_more(accum, deadline, "Reading HTTP headers")?; + } + + // The parseable headers are all the lines up to the first double newline + let end_of_headers_pos = pos + position(&accum[pos..], header_section_sep).unwrap(); + let parseable = std::str::from_utf8(&accum[pos..end_of_headers_pos]).unwrap(); + for line in parseable.lines() { + let mut parts = line.splitn(2, ": "); + let mut key = parts.next().unwrap().to_string(); + key.make_ascii_lowercase(); // case-insensitive header keys + let value = parts.next().unwrap().trim().to_string(); + headers.insert(key, value); + } + + let body_start = end_of_headers_pos + header_section_sep.len(); + Ok((body_start, headers)) + } + + #[cfg(feature = "ilp-over-http")] + pub fn send_http_response( + &mut self, + response: HttpResponse, + timeout_sec: Option, + ) -> io::Result<()> { + self.do_write_all(response.as_string().as_bytes(), timeout_sec)?; + Ok(()) + } + + #[cfg(feature = "ilp-over-http")] + pub fn send_http_response_q(&mut self, response: HttpResponse) -> io::Result<()> { + self.send_http_response(response, Some(5.0)) + } + + #[cfg(feature = "ilp-over-http")] + pub fn recv_http(&mut self, wait_timeout_sec: f64) -> io::Result { + let mut accum = Vec::::new(); + let deadline = Instant::now() + Duration::from_secs_f64(wait_timeout_sec); + let (pos, method, path) = self.recv_http_method(&mut accum, deadline)?; + let (pos, headers) = self.recv_http_headers(pos, &mut accum, deadline)?; + let content_length = headers + .get("content-length") + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "Missing Content-Length"))? + .parse::() + .map_err(|_| { + io::Error::new(io::ErrorKind::InvalidData, "Invalid Content-Length header") + })?; + while accum.len() < pos + content_length { + self.read_more(&mut accum, deadline, "Reading HTTP body")?; + } + let body = accum[pos..(pos + content_length)].to_vec(); + Ok(HttpRequest { + method, + path, + headers, + body, + }) + } + + #[cfg(feature = "ilp-over-http")] + pub fn recv_http_q(&mut self) -> io::Result { + self.recv_http(5.0) + } + + pub fn recv(&mut self, wait_timeout_sec: f64) -> io::Result { + let deadline = Instant::now() + Duration::from_secs_f64(wait_timeout_sec); let mut accum = Vec::::new(); let mut chunk = [0u8; 1024]; @@ -202,8 +502,14 @@ impl MockServer { let count = match self.do_read(&mut chunk[..]) { Ok(count) => count, Err(ref err) if err.kind() == io::ErrorKind::WouldBlock => { - let poll_timeout = Some(Duration::from_millis(200)); - self.poll.poll(&mut self.events, poll_timeout)?; + let poll_timeout = match deadline.checked_duration_since(Instant::now()) { + Some(remain) => remain, + None => break, + }; + if !self.wait_for_recv(Some(poll_timeout))? { + // Timed out waiting for data. + break; + } continue; } Err(err) => return Err(err), @@ -237,7 +543,21 @@ impl MockServer { self.recv(0.1) } - pub fn lsb(&self) -> SenderBuilder { - SenderBuilder::new(self.host, self.port) + pub fn lsb_tcp(&self) -> SenderBuilder { + SenderBuilder::new(Protocol::Tcp, self.host, self.port) + } + + pub fn lsb_tcps(&self) -> SenderBuilder { + SenderBuilder::new(Protocol::Tcps, self.host, self.port) + } + + #[cfg(feature = "ilp-over-http")] + pub fn lsb_http(&self) -> SenderBuilder { + SenderBuilder::new(Protocol::Http, self.host, self.port) + } + + #[cfg(feature = "ilp-over-http")] + pub fn lsb_https(&self) -> SenderBuilder { + SenderBuilder::new(Protocol::Https, self.host, self.port) } } diff --git a/questdb-rs/src/tests/mod.rs b/questdb-rs/src/tests/mod.rs index 0c1c78df..50266006 100644 --- a/questdb-rs/src/tests/mod.rs +++ b/questdb-rs/src/tests/mod.rs @@ -6,7 +6,7 @@ * \__\_\\__,_|\___||___/\__|____/|____/ * * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2023 QuestDB + * Copyright (c) 2019-2024 QuestDB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,6 +22,10 @@ * ******************************************************************************/ mod f64_serializer; + +#[cfg(feature = "ilp-over-http")] +mod http; + mod mock; mod sender; diff --git a/questdb-rs/src/tests/sender.rs b/questdb-rs/src/tests/sender.rs index 526a3a93..4c2fa850 100644 --- a/questdb-rs/src/tests/sender.rs +++ b/questdb-rs/src/tests/sender.rs @@ -6,7 +6,7 @@ * \__\_\\__,_|\___||___/\__|____/|____/ * * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2023 QuestDB + * Copyright (c) 2019-2024 QuestDB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,8 +24,7 @@ use crate::{ ingress::{ - Buffer, CertificateAuthority, Sender, TableName, Timestamp, TimestampMicros, - TimestampNanos, Tls, + Buffer, CertificateAuthority, Sender, TableName, Timestamp, TimestampMicros, TimestampNanos, }, Error, ErrorCode, }; @@ -41,19 +40,15 @@ use std::{io, time::SystemTime}; #[test] fn test_basics() -> TestResult { let mut server = MockServer::new()?; - let mut sender = server.lsb().connect()?; + let mut sender = server.lsb_tcp().build()?; assert!(!sender.must_close()); server.accept()?; assert_eq!(server.recv_q()?, 0); - let ts = std::time::SystemTime::now(); - let ts_micros_num = ts - .duration_since(std::time::SystemTime::UNIX_EPOCH)? - .as_micros() as i64; - let ts_nanos_num = ts - .duration_since(std::time::SystemTime::UNIX_EPOCH)? - .as_nanos() as i64; + let ts = SystemTime::now(); + let ts_micros_num = ts.duration_since(SystemTime::UNIX_EPOCH)?.as_micros() as i64; + let ts_nanos_num = ts.duration_since(SystemTime::UNIX_EPOCH)?.as_nanos() as i64; let ts_micros = TimestampMicros::from_systemtime(ts)?; assert_eq!(ts_micros.as_i64(), ts_micros_num); let ts_nanos = TimestampNanos::from_systemtime(ts)?; @@ -86,6 +81,33 @@ fn test_basics() -> TestResult { Ok(()) } +#[test] +fn test_max_buf_size() -> TestResult { + let max = 1024; + let mut server = MockServer::new()?; + let mut sender = server.lsb_tcp().max_buf_size(max)?.build()?; + assert!(!sender.must_close()); + server.accept()?; + + let mut buffer = Buffer::new(); + + while buffer.len() < max { + buffer + .table("test")? + .symbol("t1", "v1")? + .column_f64("f1", 0.5)? + .at_now()?; + } + + let err = sender.flush(&mut buffer).unwrap_err(); + assert_eq!(err.code(), ErrorCode::InvalidApiCall); + assert_eq!( + err.msg(), + "Could not flush buffer: Buffer size of 1026 exceeds maximum configured allowed size of 1024 bytes." + ); + Ok(()) +} + #[test] fn test_table_name_too_long() -> TestResult { let mut buffer = Buffer::with_max_name_len(4); @@ -99,6 +121,138 @@ fn test_table_name_too_long() -> TestResult { Ok(()) } +#[test] +fn test_row_count() -> TestResult { + let mut buffer = Buffer::new(); + assert_eq!(buffer.row_count(), 0); + + buffer.table("x")?.symbol("y", "z1")?.at_now()?; + buffer + .table("x")? + .symbol("y", "z2")? + .at(TimestampNanos::now())?; + assert_eq!(buffer.row_count(), 2); + + buffer.set_marker()?; + + buffer.table("x")?.symbol("y", "z3")?.at_now()?; + buffer + .table("x")? + .symbol("y", "z4")? + .at(TimestampNanos::now())?; + buffer.table("x")?.symbol("y", "z5")?.at_now()?; + assert_eq!(buffer.row_count(), 5); + + buffer.rewind_to_marker()?; + assert_eq!(buffer.row_count(), 2); + + buffer.clear(); + assert_eq!(buffer.row_count(), 0); + Ok(()) +} + +#[test] +fn test_auth_inconsistent_keys() -> TestResult { + test_bad_key("fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU", // d + "fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU", // x + "Dt5tbS1dEDMSYfym3fgMv0B99szno-dFc1rYF9t0aac", + "Misconfigured ILP authentication keys: InconsistentComponents. Hint: Check the keys for a possible typo." + ) +} + +#[test] +fn test_auth_bad_base64_private_key() -> TestResult { + test_bad_key( + "bad key", // d + "fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU", // x + "Dt5tbS1dEDMSYfym3fgMv0B99szno-dFc1rYF9t0aac", // y + "Misconfigured ILP authentication keys. Could not decode private authentication key: invalid Base64 encoding. Hint: Check the keys for a possible typo." + ) +} + +#[test] +fn test_auth_private_key_too_long() -> TestResult { + test_bad_key( + "ZkxLWUVhb0ViOWxybjNua3dMREEtTV94bnVGT2RTdDl5MFo3X3ZXU0hMVWZMS1lFYW9FYjlscm4zbmt3TERBLU1feG51Rk9kU3Q5eTBaN192V1NITFU", + "fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU", // x + "Dt5tbS1dEDMSYfym3fgMv0B99szno-dFc1rYF9t0aac", // y + "Misconfigured ILP authentication keys: InvalidComponent. Hint: Check the keys for a possible typo." + ) +} + +#[test] +fn test_auth_public_key_x_too_long() -> TestResult { + test_bad_key( + "fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU", + "ZkxLWUVhb0ViOWxybjNua3dMREEtTV94bnVGT2RTdDl5MFo3X3ZXU0hMVWZMS1lFYW9FYjlscm4zbmt3TERBLU1feG51Rk9kU3Q5eTBaN192V1NITFU", // x + "Dt5tbS1dEDMSYfym3fgMv0B99szno-dFc1rYF9t0aac", // y + "Misconfigured ILP authentication keys. Public key x is too long. Hint: Check the keys for a possible typo." + ) +} + +#[test] +fn test_auth_public_key_y_too_long() -> TestResult { + test_bad_key( + "fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU", + "Dt5tbS1dEDMSYfym3fgMv0B99szno-dFc1rYF9t0aac", // x + "ZkxLWUVhb0ViOWxybjNua3dMREEtTV94bnVGT2RTdDl5MFo3X3ZXU0hMVWZMS1lFYW9FYjlscm4zbmt3TERBLU1feG51Rk9kU3Q5eTBaN192V1NITFU", // y + "Misconfigured ILP authentication keys. Public key y is too long. Hint: Check the keys for a possible typo." + ) +} + +#[test] +fn test_auth_bad_base64_public_key_x() -> TestResult { + test_bad_key( + "fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU", // d + "bad base64 encoding", // x + "Dt5tbS1dEDMSYfym3fgMv0B99szno-dFc1rYF9t0aac", // y + "Misconfigured ILP authentication keys. Could not decode public key x: invalid Base64 encoding. Hint: Check the keys for a possible typo." + ) +} + +#[test] +fn test_auth_bad_base64_public_key_y() -> TestResult { + test_bad_key( + "fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU", // d + "Dt5tbS1dEDMSYfym3fgMv0B99szno-dFc1rYF9t0aac", // x + "bad base64 encoding", // y + "Misconfigured ILP authentication keys. Could not decode public key y: invalid Base64 encoding. Hint: Check the keys for a possible typo." + ) +} + +fn test_bad_key( + priv_key: &str, + pub_key_x: &str, + pub_key_y: &str, + expected_error_msg: &str, +) -> TestResult { + let server = MockServer::new()?; + let lsb = server + .lsb_tcp() + .username("testUser1")? + .token(priv_key)? + .token_x(pub_key_x)? + .token_y(pub_key_y)?; + let sender = lsb.build(); + + match sender { + Ok(_) => panic!("Expected an error due to bad key, but connect succeeded."), + Err(err) => { + assert_eq!( + err.code(), + ErrorCode::AuthError, + "Expected an ErrorCode::AuthError" + ); + assert_eq!( + err.msg(), + expected_error_msg, + "Error message did not match expected message." + ); + } + } + Ok(()) +} + #[test] fn test_timestamp_overloads() -> TestResult { let tbl_name = TableName::new("tbl_name")?; @@ -201,11 +355,9 @@ fn test_tls_with_file_ca() -> TestResult { ca_path.push("server_rootCA.pem"); let server = MockServer::new()?; - let lsb = server - .lsb() - .tls(Tls::Enabled(CertificateAuthority::File(ca_path))); + let lsb = server.lsb_tcps().tls_roots(ca_path)?; let server_jh = server.accept_tls(); - let mut sender = lsb.connect()?; + let mut sender = lsb.build()?; let mut server: MockServer = server_jh.join().unwrap()?; let mut buffer = Buffer::new(); @@ -232,14 +384,15 @@ fn test_tls_to_plain_server() -> TestResult { let mut server = MockServer::new()?; let lsb = server - .lsb() - .read_timeout(Duration::from_millis(500)) - .tls(Tls::Enabled(CertificateAuthority::File(ca_path))); + .lsb_tcps() + .auth_timeout(Duration::from_millis(500))? + .tls_ca(CertificateAuthority::PemFile)? + .tls_roots(ca_path)?; let server_jh = std::thread::spawn(move || -> io::Result { server.accept()?; Ok(server) }); - let maybe_sender = lsb.connect(); + let maybe_sender = lsb.build(); let _server: MockServer = server_jh.join().unwrap()?; let err = maybe_sender.unwrap_err(); assert_eq!( @@ -272,12 +425,9 @@ fn expect_eventual_disconnect(sender: &mut Sender) { #[test] fn test_plain_to_tls_server() -> TestResult { let server = MockServer::new()?; - let lsb = server - .lsb() - .read_timeout(Duration::from_millis(500)) - .tls(Tls::Disabled); + let lsb = server.lsb_tcp().auth_timeout(Duration::from_millis(500))?; let server_jh = server.accept_tls(); - let maybe_sender = lsb.connect(); + let maybe_sender = lsb.build(); let server_err = server_jh.join().unwrap().unwrap_err(); // The server failed to handshake, so disconnected the client. @@ -298,9 +448,9 @@ fn test_plain_to_tls_server() -> TestResult { #[test] fn test_tls_insecure_skip_verify() -> TestResult { let server = MockServer::new()?; - let lsb = server.lsb().tls(Tls::InsecureSkipVerify); + let lsb = server.lsb_tcps().tls_verify(false)?; let server_jh = server.accept_tls(); - let mut sender = lsb.connect()?; + let mut sender = lsb.build()?; let mut server: MockServer = server_jh.join().unwrap()?; let mut buffer = Buffer::new(); @@ -319,3 +469,22 @@ fn test_tls_insecure_skip_verify() -> TestResult { assert_eq!(server.msgs[0].as_str(), exp); Ok(()) } + +#[test] +fn bad_uppercase_protocol() { + let res = Sender::from_conf("TCP::addr=localhost:9009;"); + assert!(res.is_err()); + let err = res.unwrap_err(); + assert!(err.code() == ErrorCode::ConfigError); + assert!(err.msg() == "Unsupported protocol: TCP"); +} + +#[test] +fn bad_uppercase_addr() { + let res = Sender::from_conf("tcp::ADDR=localhost:9009;"); + assert!(res.is_err()); + let err = res.unwrap_err(); + eprint!("err: {:?}", err); + assert!(err.code() == ErrorCode::ConfigError); + assert!(err.msg() == "Missing \"addr\" parameter in config string"); +} diff --git a/system_test/fixture.py b/system_test/fixture.py index 6e77a732..88e802dc 100644 --- a/system_test/fixture.py +++ b/system_test/fixture.py @@ -6,7 +6,7 @@ ## \__\_\\__,_|\___||___/\__|____/|____/ ## ## Copyright (c) 2014-2019 Appsicle -## Copyright (c) 2019-2023 QuestDB +## Copyright (c) 2019-2024 QuestDB ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -26,6 +26,7 @@ sys.dont_write_bytecode = True import os +import re import pathlib import textwrap import json @@ -47,11 +48,11 @@ # Valid keys as registered with the QuestDB fixture. -AUTH = ( - "testUser1", - "5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48", - "fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU", - "Dt5tbS1dEDMSYfym3fgMv0B99szno-dFc1rYF9t0aac") +AUTH = dict( + username="testUser1", + token="5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48", + token_x="fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU", + token_y="Dt5tbS1dEDMSYfym3fgMv0B99szno-dFc1rYF9t0aac") CA_PATH = (pathlib.Path(__file__).parent.parent / @@ -64,7 +65,7 @@ def retry( every=0.05, msg='Timed out retrying', backoff_till=5.0, - lead_sleep=0.1): + lead_sleep=0.001): """ Repeat task every `interval` until it returns a truthy value or times out. """ @@ -172,6 +173,35 @@ def install_questdb(vers: str, download_url: str): return version_dir +def install_questdb_from_repo(repo: pathlib.Path): + repo = repo.absolute() + target_dir = repo / 'core' / 'target' + try: + repo_jar = next(target_dir.glob("**/questdb*-SNAPSHOT.jar")) + except StopIteration: + raise RuntimeError( + f'Could not find QuestDB jar in repo {repo}. ' + + 'Check path and ensure you built correctly.') + print(f'Starting QuestDB from jar {repo_jar}') + proj = Project() + vers = 'repo' + questdb_dir = proj.questdb_dir / vers + if questdb_dir.exists(): + shutil.rmtree(questdb_dir) + (questdb_dir / 'data' / 'log').mkdir(parents=True) + bin_dir = questdb_dir / 'bin' + bin_dir.mkdir(parents=True) + conf_dir = questdb_dir / 'conf' + conf_dir.mkdir(parents=True) + data_conf_dir = questdb_dir / 'data' / 'conf' + data_conf_dir.mkdir(parents=True) + shutil.copy(repo_jar, bin_dir / 'questdb.jar') + repo_conf_dir = target_dir / 'classes' / 'io' / 'questdb' / 'site' / 'conf' + shutil.copy(repo_conf_dir / 'server.conf', conf_dir / 'server.conf') + shutil.copy(repo_conf_dir / 'mime.types', data_conf_dir / 'mime.types') + return questdb_dir + + def _parse_version(vers_str): def try_int(vers_part): try: @@ -203,7 +233,7 @@ class QueryError(Exception): class QuestDbFixture: - def __init__(self, root_dir: pathlib.Path, auth=False, wrap_tls=False): + def __init__(self, root_dir: pathlib.Path, auth=False, wrap_tls=False, http=False): self._root_dir = root_dir self.version = _parse_version(self._root_dir.name) self._data_dir = self._root_dir / 'data' @@ -227,6 +257,7 @@ def __init__(self, root_dir: pathlib.Path, auth=False, wrap_tls=False): auth_txt_path = self._conf_dir / 'auth.txt' with open(auth_txt_path, 'w', encoding='utf-8') as auth_file: auth_file.write(AUTH_TXT) + self.http = http def print_log(self): with open(self._log_path, 'r', encoding='utf-8') as log_file: @@ -238,6 +269,7 @@ def start(self): ports = discover_avail_ports(3) self.http_server_port, self.line_tcp_port, self.pg_port = ports auth_config = 'line.tcp.auth.db.path=conf/auth.txt' if self.auth else '' + ilp_over_http_config = 'line.http.enabled=true' if self.http else '' with open(self._conf_path, 'w', encoding='utf-8') as conf_file: conf_file.write(textwrap.dedent(rf''' http.bind.to=0.0.0.0:{self.http_server_port} @@ -249,8 +281,9 @@ def start(self): line.tcp.min.idle.ms.before.writer.release=300 telemetry.enabled=false cairo.commit.lag=100 - lne.tcp.commit.interval.fraction=0.1 + line.tcp.commit.interval.fraction=0.1 {auth_config} + {ilp_over_http_config} ''').lstrip('\n')) java = _find_java() @@ -258,16 +291,15 @@ def start(self): java, '-DQuestDB-Runtime-0', '-ea', - #'-Dnoebug', - '-Debug', + '-Dnoebug', + # '-Debug', '-XX:+UnlockExperimentalVMOptions', '-XX:+AlwaysPreTouch', - '-XX:+UseParallelOldGC', '-p', str(self._root_dir / 'bin' / 'questdb.jar'), '-m', 'io.questdb/io.questdb.ServerMain', '-d', str(self._data_dir)] sys.stderr.write( - f'Starting QuestDB: {launch_args!r} (auth: {self.auth})\n') + f'Starting QuestDB: {launch_args!r} (auth: {self.auth}, http: {self.http})\n') self._log = open(self._log_path, 'ab') try: self._proc = subprocess.Popen( @@ -283,7 +315,7 @@ def check_http_up(): raise RuntimeError('QuestDB died during startup.') req = urllib.request.Request( f'http://localhost:{self.http_server_port}', - method='HEAD') + method='GET') try: resp = urllib.request.urlopen(req, timeout=1) if resp.status == 200: @@ -297,7 +329,7 @@ def check_http_up(): sys.stderr.write('Waiting until HTTP service is up.\n') retry( check_http_up, - timeout_sec=60, + timeout_sec=300, msg='Timed out waiting for HTTP service to come up.') except: sys.stderr.write(f'QuestDB log at `{self._log_path}`:\n') @@ -307,6 +339,11 @@ def check_http_up(): atexit.register(self.stop) sys.stderr.write('QuestDB fixture instance is ready.\n') + # Read the actual version from the running process. + # This is to support a version like `7.3.2-SNAPSHOT` + # from an externally started QuestDB instance. + self.version = self.query_version() + if self.wrap_tls: self._tls_proxy = TlsProxyFixture(self.line_tcp_port) self._tls_proxy.start() @@ -333,13 +370,29 @@ def http_sql_query(self, sql_query): if 'error' in data: raise QueryError(data['error']) return data + + def query_version(self): + try: + res = self.http_sql_query('select build') + except QueryError as qe: + # For old versions that don't support `build` yet, parse from path. + return self.version + + vers = res['dataset'][0][0] + print(vers) + + # This returns a string like: + # 'Build Information: QuestDB 7.3.2, JDK 11.0.8, Commit Hash 19059deec7b0fd19c53182b297a5d59774a51892' + # We want the '7.3.2' part. + vers = re.compile(r'.*QuestDB ([0-9.]+).*').search(vers).group(1) + return _parse_version(vers) def retry_check_table( self, table_name, *, min_rows=1, - timeout_sec=30, + timeout_sec=300, log=True, log_ctx=None): sql_query = f"select * from '{table_name}'" @@ -406,6 +459,19 @@ def start(self): env = dict(os.environ) env['CARGO_TARGET_DIR'] = str(self._target_dir) self._log_file = open(self._log_path, 'wb') + + # Compile before running `cargo run`. + # Note that errors and output are purpously suppressed. + # This is just to exclude the build time from the start-up time. + # If there are build errors, they'll be reported later in the `run` + # call below. + subprocess.call( + ['cargo', 'build'], + cwd=self._code_dir, + env=env, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + self._proc = subprocess.Popen( ['cargo', 'run', str(self.qdb_ilp_port)], cwd=self._code_dir, @@ -449,3 +515,7 @@ def stop(self): self._proc.terminate() self._proc.wait() self._proc = None + if self._log_file: + self._log_file.close() + self._log_file = None + diff --git a/system_test/questdb_line_sender.py b/system_test/questdb_line_sender.py index f019a3ad..f7471258 100644 --- a/system_test/questdb_line_sender.py +++ b/system_test/questdb_line_sender.py @@ -6,7 +6,7 @@ ## \__\_\\__,_|\___||___/\__|____/|____/ ## ## Copyright (c) 2014-2019 Appsicle -## Copyright (c) 2019-2023 QuestDB +## Copyright (c) 2019-2024 QuestDB ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -46,6 +46,7 @@ import ctypes import os from datetime import datetime +from enum import Enum from ctypes import ( c_bool, @@ -68,6 +69,22 @@ class c_line_sender(ctypes.Structure): class c_line_sender_buffer(ctypes.Structure): pass +c_line_sender_protocol = ctypes.c_int + +class Protocol(Enum): + TCP = (c_line_sender_protocol(0), 'tcp') + TCPS = (c_line_sender_protocol(1), 'tcps') + HTTP = (c_line_sender_protocol(2), 'http') + HTTPS = (c_line_sender_protocol(3), 'https') + +c_line_sender_ca = ctypes.c_int + +class CertificateAuthority(Enum): + WEBPKI_ROOTS = (c_line_sender_ca(0), 'webpki_roots') + OS_ROOTS = (c_line_sender_ca(1), 'os_roots') + WEBPKI_AND_OS_ROOTS = (c_line_sender_ca(2), 'webpki_and_os_roots') + PEM_FILE = (c_line_sender_ca(3), 'pem_file') + class c_line_sender_opts(ctypes.Structure): pass @@ -250,52 +267,99 @@ def set_sig(fn, restype, *argtypes): set_sig( dll.line_sender_opts_new, c_line_sender_opts_p, + c_line_sender_protocol, c_line_sender_utf8, c_uint16) set_sig( dll.line_sender_opts_new_service, c_line_sender_opts_p, + c_line_sender_protocol, c_line_sender_utf8, c_line_sender_utf8) set_sig( - dll.line_sender_opts_net_interface, - None, + dll.line_sender_opts_bind_interface, + c_bool, c_line_sender_opts_p, - c_line_sender_utf8) + c_line_sender_utf8, + c_line_sender_error_p_p) set_sig( - dll.line_sender_opts_auth, - None, + dll.line_sender_opts_username, + c_bool, c_line_sender_opts_p, c_line_sender_utf8, + c_line_sender_error_p_p) + set_sig( + dll.line_sender_opts_password, + c_bool, + c_line_sender_opts_p, c_line_sender_utf8, + c_line_sender_error_p_p) + set_sig( + dll.line_sender_opts_token, + c_bool, + c_line_sender_opts_p, c_line_sender_utf8, - c_line_sender_utf8) + c_line_sender_error_p_p) set_sig( - dll.line_sender_opts_tls, - None, - c_line_sender_opts_p) + dll.line_sender_opts_token_x, + c_bool, + c_line_sender_opts_p, + c_line_sender_utf8, + c_line_sender_error_p_p) set_sig( - dll.line_sender_opts_tls_os_roots, - None, - c_line_sender_opts_p) + dll.line_sender_opts_token_y, + c_bool, + c_line_sender_opts_p, + c_line_sender_utf8, + c_line_sender_error_p_p) set_sig( - dll.line_sender_opts_tls_webpki_and_os_roots, - None, - c_line_sender_opts_p) + dll.line_sender_opts_auth_timeout, + c_bool, + c_line_sender_opts_p, + c_uint64, + c_line_sender_error_p_p) + set_sig( + dll.line_sender_opts_tls_verify, + c_bool, + c_line_sender_opts_p, + c_bool, + c_line_sender_error_p_p) set_sig( dll.line_sender_opts_tls_ca, - None, + c_bool, c_line_sender_opts_p, - c_line_sender_utf8) + c_line_sender_ca, + c_line_sender_error_p_p) set_sig( - dll.line_sender_opts_tls_insecure_skip_verify, - None, - c_line_sender_opts_p) + dll.line_sender_opts_tls_roots, + c_bool, + c_line_sender_opts_p, + c_line_sender_utf8, + c_line_sender_error_p_p) set_sig( - dll.line_sender_opts_read_timeout, - None, + dll.line_sender_opts_max_buf_size, + c_bool, + c_line_sender_opts_p, + c_size_t, + c_line_sender_error_p_p) + set_sig( + dll.line_sender_opts_retry_timeout, + c_bool, c_line_sender_opts_p, - c_uint64) + c_uint64, + c_line_sender_error_p_p) + set_sig( + dll.line_sender_opts_request_min_throughput, + c_bool, + c_line_sender_opts_p, + c_uint64, + c_line_sender_error_p_p) + set_sig( + dll.line_sender_opts_request_timeout, + c_bool, + c_line_sender_opts_p, + c_uint64, + c_line_sender_error_p_p) set_sig( dll.line_sender_opts_clone, c_line_sender_opts_p, @@ -305,10 +369,19 @@ def set_sig(fn, restype, *argtypes): None, c_line_sender_opts_p) set_sig( - dll.line_sender_connect, + dll.line_sender_build, c_line_sender_p, c_line_sender_opts_p, c_line_sender_error_p_p) + set_sig( + dll.line_sender_from_conf, + c_line_sender_p, + c_line_sender_utf8, + c_line_sender_error_p_p) + set_sig( + dll.line_sender_from_env, + c_line_sender_p, + c_line_sender_error_p_p) set_sig( dll.line_sender_must_close, None, @@ -329,6 +402,13 @@ def set_sig(fn, restype, *argtypes): c_line_sender_p, c_line_sender_buffer_p, c_line_sender_error_p_p) + set_sig( + dll.line_sender_flush_and_keep_with_flags, + c_bool, + c_line_sender_p, + c_line_sender_buffer_p, + c_bool, + c_line_sender_error_p_p) return dll @@ -420,9 +500,10 @@ def _fully_qual_name(obj): class _Opts: - def __init__(self, host, port): + def __init__(self, host, port, protocol=Protocol.TCP): self.impl = _error_wrapped_call( _DLL.line_sender_opts_new_service, + protocol.value[0], _utf8(str(host)), _utf8(str(port))) @@ -448,10 +529,10 @@ def __init__(self, micros: int): class Buffer: - def __init__(self, init_capacity=65536, max_name_len=127): + def __init__(self, init_buf_size=65536, max_name_len=127): self._impl = _DLL.line_sender_buffer_with_max_name_len( c_size_t(max_name_len)) - _DLL.line_sender_buffer_reserve(self._impl, c_size_t(init_capacity)) + _DLL.line_sender_buffer_reserve(self._impl, c_size_t(init_buf_size)) def __len__(self): return _DLL.line_sender_buffer_size(self._impl) @@ -558,42 +639,55 @@ def __del__(self): _DLL.line_sender_buffer_free(self._impl) +class BuildMode(Enum): + API = 1 + CONF = 2 + ENV = 3 + + +def _map_value(key, value): + """ + Return a pair of option C object and string value. + """ + if isinstance(value, bool): + if key == 'tls_verify': + return (value, 'on' if value else 'unsafe_off') + else: + return (value, 'on' if value else 'off') + elif isinstance(value, CertificateAuthority): + return value.value # a tuple of `(c_line_sender_ca, str)` + else: + return (value, f'{value}') + + class Sender: def __init__( self, + build_mode: BuildMode, + protocol, host: str, port: Union[str, int], - *, - interface: Optional[str] = None, - auth: Optional[Tuple[str, str, str, str]] = None, - tls: Union[bool, str] = False, - read_timeout: Optional[int] = None): - - opts = _Opts(host, port) - if interface: - opts.net_interface(interface) - - if auth: - opts.auth(*auth) - - if tls: - if tls is True: - opts.tls() - elif tls == 'os_roots': - opts.tls_os_roots() - elif tls == 'webpki_and_os_roots': - opts.tls_webpki_and_os_roots() - elif tls == 'insecure_skip_verify': - opts.tls_insecure_skip_verify() - else: - opts.tls_ca(str(tls)) + **kwargs): + + self._build_mode = build_mode + self._impl = None + self._conf = [ + protocol.value[1], + '::', + f'addr={host}:{port};'] + self._opts = None + self._buffer = Buffer() + opts = _Opts(host, port, protocol) + for key, value in kwargs.items(): + # Build the config string param pair. + c_value, conf_value = _map_value(key, value) + self._conf.append(f'{key}={conf_value};') - if read_timeout is not None: - opts.read_timeout(read_timeout) + # Set the option in the C object. + getattr(opts, key)(c_value) - self._buffer = Buffer() + self._conf = ''.join(self._conf) self._opts = opts - self._impl = None @property def buffer(self): @@ -602,9 +696,20 @@ def buffer(self): def connect(self): if self._impl: raise SenderError('Already connected') - self._impl = _error_wrapped_call( - _DLL.line_sender_connect, - self._opts.impl) + if self._build_mode == BuildMode.CONF: + self._impl = _error_wrapped_call( + _DLL.line_sender_from_conf, + _utf8(self._conf)) + elif self._build_mode == BuildMode.ENV: + env_var = 'QDB_CLIENT_CONF' + os.environ[env_var] = self._conf + self._impl = _error_wrapped_call( + _DLL.line_sender_from_env) + del os.environ[env_var] + else: + self._impl = _error_wrapped_call( + _DLL.line_sender_build, + self._opts.impl) def __enter__(self): self.connect() @@ -634,7 +739,7 @@ def at_now(self): def at(self, timestamp: int): self._buffer.at(timestamp) - def flush(self, buffer: Optional[Buffer]=None, clear=True): + def flush(self, buffer: Optional[Buffer]=None, clear=True, transactional=None): if (buffer is None) and not clear: raise ValueError( 'Clear flag must be True when using internal buffer') @@ -643,16 +748,27 @@ def flush(self, buffer: Optional[Buffer]=None, clear=True): if len(buffer) == 0: return try: - if clear: + if transactional is not None: + if not isinstance(transactional, bool): + raise ValueError('Transactional flag must be a boolean') _error_wrapped_call( - _DLL.line_sender_flush, + _DLL.line_sender_flush_and_keep_with_flags, self._impl, - buffer._impl) + buffer._impl, + transactional) + if clear: + buffer.clear() else: - _error_wrapped_call( - _DLL.line_sender_flush_and_keep, - self._impl, - buffer._impl) + if clear: + _error_wrapped_call( + _DLL.line_sender_flush, + self._impl, + buffer._impl) + else: + _error_wrapped_call( + _DLL.line_sender_flush_and_keep, + self._impl, + buffer._impl) except: # Prevent `.close()` from erroring if it was called # after a flush exception was raised, trapped and discarded. diff --git a/system_test/test.py b/system_test/test.py index 515829be..9d90bac6 100755 --- a/system_test/test.py +++ b/system_test/test.py @@ -8,7 +8,7 @@ ## \__\_\\__,_|\___||___/\__|____/|____/ ## ## Copyright (c) 2014-2019 Appsicle -## Copyright (c) 2019-2023 QuestDB +## Copyright (c) 2019-2024 QuestDB ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. @@ -27,8 +27,8 @@ import sys sys.dont_write_bytecode = True import os -import textwrap +import pathlib import math import datetime import argparse @@ -41,6 +41,7 @@ QuestDbFixture, TlsProxyFixture, install_questdb, + install_questdb_from_repo, list_questdb_releases, AUTH) import subprocess @@ -49,6 +50,7 @@ QDB_FIXTURE: QuestDbFixture = None TLS_PROXY_FIXTURE: TlsProxyFixture = None +BUILD_MODE = None def retry_check_table(*args, **kwargs): @@ -59,40 +61,52 @@ def ns_to_qdb_date(at_ts_ns): # We first need to match QuestDB's internal microsecond resolution. at_ts_us = int(at_ts_ns / 1000.0) at_ts_sec = at_ts_us / 1000000.0 - at_td = datetime.datetime.fromtimestamp(at_ts_sec) + at_td = datetime.datetime.utcfromtimestamp(at_ts_sec) return at_td.isoformat() + 'Z' # Valid keys, but not registered with the QuestDB fixture. -AUTH_UNRECOGNIZED = ( - "testUser2", - "xiecEl-2zbg6aYCFbxDMVWaly9BlCTaEChvcxCH5BCk", - "-nSHz3evuPl-rGLIlbIZjwOJeWao0rbk53Cll6XEgak", - "9iYksF4L5mfmArupv0CMoyVAWjQ4gNIoupdg6N5noG8") +AUTH_UNRECOGNIZED = dict( + username="testUser2", + token="xiecEl-2zbg6aYCFbxDMVWaly9BlCTaEChvcxCH5BCk", + token_x="-nSHz3evuPl-rGLIlbIZjwOJeWao0rbk53Cll6XEgak", + token_y="9iYksF4L5mfmArupv0CMoyVAWjQ4gNIoupdg6N5noG8") # Bad malformed key -AUTH_MALFORMED1 = ( - "testUser3", - "xiecEl-zzbg6aYCFbxDMVWaly9BlCTaEChvcxCH5BCk", - "-nSHz3evuPl-rGLIlbIZjwOJeWao0rbk53Cll6XEgak", - "9iYksF4L6mfmArupv0CMoyVAWjQ4gNIoupdg6N5noG8") +AUTH_MALFORMED1 = dict( + username="testUser3", + token="xiecEl-zzbg6aYCFbxDMVWaly9BlCTaEChvcxCH5BCk", + token_x="-nSHz3evuPl-rGLIlbIZjwOJeWao0rbk53Cll6XEgak", + token_y="9iYksF4L6mfmArupv0CMoyVAWjQ4gNIoupdg6N5noG8") # Another malformed key where the keys invalid base 64. -AUTH_MALFORMED2 = ( - "testUser4", - "xiecEl-zzbg6aYCFbxDMVWaly9BlCTaECH5BCk", - "-nSHz3evuPl-rGLIlbIZjwOJeWao0rbk5XEgak", - "9iYksF4L6mfmArupv0CMoyVAWjQ4gNIou5noG8") +AUTH_MALFORMED2 = dict( + username="testUser4", + token="xiecEl-zzbg6aYCFbxDMVWaly9BlCTaECH5BCk", + token_x="-nSHz3evuPl-rGLIlbIZjwOJeWao0rbk5XEgak", + token_y="9iYksF4L6mfmArupv0CMoyVAWjQ4gNIou5noG8") + + +# All the keys are valid, but the username is wrong. +AUTH_MALFORMED3 = dict( + username="wrongUser", + token=AUTH['token'], + token_x=AUTH['token_x'], + token_y=AUTH['token_y']) class TestSender(unittest.TestCase): def _mk_linesender(self): + # N.B.: We never connect with TLS here. + auth = AUTH if QDB_FIXTURE.auth else {} return qls.Sender( + BUILD_MODE, + qls.Protocol.HTTP if QDB_FIXTURE.http else qls.Protocol.TCP, QDB_FIXTURE.host, - QDB_FIXTURE.line_tcp_port, - auth=AUTH if QDB_FIXTURE.auth else None) + QDB_FIXTURE.http_server_port if QDB_FIXTURE.http else QDB_FIXTURE.line_tcp_port, + **auth) def _expect_eventual_disconnect(self, sender): with self.assertRaisesRegex( @@ -250,20 +264,36 @@ def test_mismatched_types_across_rows(self): pending = sender.buffer.peek() - # We only ever get the first row back. - resp = retry_check_table(table_name, log_ctx=pending) - exp_columns = [ - {'name': 'a', 'type': 'STRING'}, - {'name': 'timestamp', 'type': 'TIMESTAMP'}] - self.assertEqual(resp['columns'], exp_columns) + try: + sender.flush() + except qls.SenderError as e: + if not QDB_FIXTURE.http: + raise e + self.assertIn('Could not flush buffer', str(e)) + self.assertIn('cast error from', str(e)) + self.assertIn('STRING', str(e)) + self.assertIn('code: invalid, line: 2', str(e)) + + if QDB_FIXTURE.http: + # If HTTP, the error should cause the whole batch to be ignored. + # We assert that the table is empty. + with self.assertRaises(TimeoutError): + retry_check_table(table_name, timeout_sec=1, log=False) + else: + # We only ever get the first row back. + resp = retry_check_table(table_name, log_ctx=pending) + exp_columns = [ + {'name': 'a', 'type': 'STRING'}, + {'name': 'timestamp', 'type': 'TIMESTAMP'}] + self.assertEqual(resp['columns'], exp_columns) - exp_dataset = [['A']] # Comparison excludes timestamp column. - scrubbed_dataset = [row[:-1] for row in resp['dataset']] - self.assertEqual(scrubbed_dataset, exp_dataset) + exp_dataset = [['A']] # Comparison excludes timestamp column. + scrubbed_dataset = [row[:-1] for row in resp['dataset']] + self.assertEqual(scrubbed_dataset, exp_dataset) - # The second one is dropped and will not appear in results. - with self.assertRaises(TimeoutError): - retry_check_table(table_name, min_rows=2, timeout_sec=1, log=False) + # The second one is dropped and will not appear in results. + with self.assertRaises(TimeoutError): + retry_check_table(table_name, min_rows=2, timeout_sec=1, log=False) def test_at(self): if QDB_FIXTURE.version <= (6, 0, 7, 1): @@ -278,7 +308,6 @@ def test_at(self): .symbol('a', 'A') .at(at_ts_ns)) pending = sender.buffer.peek() - resp = retry_check_table(table_name, log_ctx=pending) exp_dataset = [['A', ns_to_qdb_date(at_ts_ns)]] self.assertEqual(resp['dataset'], exp_dataset) @@ -455,13 +484,18 @@ def test_timestamp_column(self): self.assertEqual(scrubbed_dataset, exp_dataset) def _test_example(self, bin_name, table_name, tls=False): + if BUILD_MODE != qls.BuildMode.API: + self.skipTest('BuildMode.API-only test') if tls and not QDB_FIXTURE.auth: self.skipTest('No auth') # Call the example program. proj = Project() ext = '.exe' if sys.platform == 'win32' else '' - bin_path = next(proj.build_dir.glob(f'**/{bin_name}{ext}')) - port = QDB_FIXTURE.line_tcp_port + try: + bin_path = next(proj.build_dir.glob(f'**/{bin_name}{ext}')) + except StopIteration: + raise RuntimeError(f'Could not find {bin_name}{ext} in {proj.build_dir}') + port = QDB_FIXTURE.http_server_port if QDB_FIXTURE.http else QDB_FIXTURE.line_tcp_port args = [str(bin_path)] if tls: ca_path = proj.tls_certs_dir / 'server_rootCA.pem' @@ -494,12 +528,14 @@ def _test_example(self, bin_name, table_name, tls=False): def test_c_example(self): suffix = '_auth' if QDB_FIXTURE.auth else '' + suffix += '_http' if QDB_FIXTURE.http else '' self._test_example( f'line_sender_c_example{suffix}', f'c_cars{suffix}') def test_cpp_example(self): suffix = '_auth' if QDB_FIXTURE.auth else '' + suffix += '_http' if QDB_FIXTURE.http else '' self._test_example( f'line_sender_cpp_example{suffix}', f'cpp_cars{suffix}') @@ -507,13 +543,13 @@ def test_cpp_example(self): def test_c_tls_example(self): self._test_example( 'line_sender_c_example_tls_ca', - 'c_cars_tls', + 'c_cars_tls_ca', tls=True) def test_cpp_tls_example(self): self._test_example( 'line_sender_cpp_example_tls_ca', - 'cpp_cars_tls', + 'cpp_cars_tls_ca', tls=True) def test_opposite_auth(self): @@ -522,12 +558,16 @@ def test_opposite_auth(self): * An authenticating client to a non-authenticating DB instance. * Or a non-authenticating client to an authenticating DB instance. """ - client_auth = None if QDB_FIXTURE.auth else AUTH + if QDB_FIXTURE.http: + self.skipTest('TCP-only test') + auth = {} if QDB_FIXTURE.auth else AUTH sender = qls.Sender( + BUILD_MODE, + qls.Protocol.TCP, QDB_FIXTURE.host, QDB_FIXTURE.line_tcp_port, - auth=client_auth) - if client_auth: + **auth) + if auth: with self.assertRaisesRegex( qls.SenderError, r'.*not receive auth challenge.*'): @@ -546,71 +586,115 @@ def test_opposite_auth(self): self._expect_eventual_disconnect(sender) def test_unrecognized_auth(self): + if QDB_FIXTURE.http: + self.skipTest('TCP-only test') + if not QDB_FIXTURE.auth: self.skipTest('No auth') sender = qls.Sender( + BUILD_MODE, + qls.Protocol.TCP, QDB_FIXTURE.host, QDB_FIXTURE.line_tcp_port, - auth=AUTH_UNRECOGNIZED) + **AUTH_UNRECOGNIZED) with sender: self._expect_eventual_disconnect(sender) def test_malformed_auth1(self): + if QDB_FIXTURE.http: + self.skipTest('TCP-only test') + if not QDB_FIXTURE.auth: self.skipTest('No auth') sender = qls.Sender( + BUILD_MODE, + qls.Protocol.TCP, QDB_FIXTURE.host, QDB_FIXTURE.line_tcp_port, - auth=AUTH_MALFORMED1) + **AUTH_MALFORMED1) with self.assertRaisesRegex( qls.SenderError, - r'.*Bad private key.*'): + r'Misconfigured ILP authentication keys: InconsistentComponents. Hint: Check the keys for a possible typo.'): sender.connect() def test_malformed_auth2(self): + if QDB_FIXTURE.http: + self.skipTest('TCP-only test') + if not QDB_FIXTURE.auth: self.skipTest('No auth') sender = qls.Sender( + BUILD_MODE, + qls.Protocol.TCP, QDB_FIXTURE.host, QDB_FIXTURE.line_tcp_port, - auth=AUTH_MALFORMED2) + **AUTH_MALFORMED2) with self.assertRaisesRegex( qls.SenderError, r'.*invalid Base64.*'): sender.connect() + def test_malformed_auth3(self): + if QDB_FIXTURE.http: + self.skipTest('TCP-only test') + + if not QDB_FIXTURE.auth: + self.skipTest('No auth') + + sender = qls.Sender( + BUILD_MODE, + qls.Protocol.TCP, + QDB_FIXTURE.host, + QDB_FIXTURE.line_tcp_port, + **AUTH_MALFORMED3) + + with sender: + self._expect_eventual_disconnect(sender) + def test_tls_insecure_skip_verify(self): + protocol = qls.Protocol.HTTPS if QDB_FIXTURE.http else qls.Protocol.TCPS + auth = AUTH if QDB_FIXTURE.auth else {} sender = qls.Sender( + BUILD_MODE, + protocol, QDB_FIXTURE.host, TLS_PROXY_FIXTURE.listen_port, - auth=AUTH if QDB_FIXTURE.auth else None, - tls='insecure_skip_verify') + tls_verify=False, + **auth) self._test_single_symbol_impl(sender) - def test_tls_ca(self): + def test_tls_roots(self): + protocol = qls.Protocol.HTTPS if QDB_FIXTURE.http else qls.Protocol.TCPS + auth = auth=AUTH if QDB_FIXTURE.auth else {} sender = qls.Sender( + BUILD_MODE, + protocol, QDB_FIXTURE.host, TLS_PROXY_FIXTURE.listen_port, - auth=AUTH if QDB_FIXTURE.auth else None, - tls=Project().tls_certs_dir / 'server_rootCA.pem') + **auth, + tls_roots=str(Project().tls_certs_dir / 'server_rootCA.pem')) self._test_single_symbol_impl(sender) - def _test_tls_special(self, tls_mode): + def _test_tls_ca(self, tls_ca): + protocol = qls.Protocol.HTTPS if QDB_FIXTURE.http else qls.Protocol.TCPS prev_ssl_cert_file = os.environ.get('SSL_CERT_FILE') try: os.environ['SSL_CERT_FILE'] = str( Project().tls_certs_dir / 'server_rootCA.pem') + auth = auth=AUTH if QDB_FIXTURE.auth else {} sender = qls.Sender( + BUILD_MODE, + protocol, QDB_FIXTURE.host, TLS_PROXY_FIXTURE.listen_port, - auth=AUTH if QDB_FIXTURE.auth else None, - tls=tls_mode) + tls_ca=tls_ca, + **auth) self._test_single_symbol_impl(sender) finally: if prev_ssl_cert_file: @@ -618,11 +702,67 @@ def _test_tls_special(self, tls_mode): else: del os.environ['SSL_CERT_FILE'] - def test_tls_os_roots(self): - self._test_tls_special('os_roots') + def test_tls_ca_os_roots(self): + self._test_tls_ca(qls.CertificateAuthority.OS_ROOTS) + + def test_tls_ca_webpki_and_os_roots(self): + self._test_tls_ca(qls.CertificateAuthority.WEBPKI_AND_OS_ROOTS) + + def test_http_transactions(self): + if not QDB_FIXTURE.http: + self.skipTest('HTTP-only test') + if QDB_FIXTURE.version <= (7, 3, 7): + self.skipTest('No ILP/HTTP support') + table_name = uuid.uuid4().hex + with self._mk_linesender() as sender: + sender.table(table_name).column('col1', 'v1').at(time.time_ns()) + sender.table(table_name).column('col1', 'v2').at(time.time_ns()) + sender.table(table_name).column('col1', 42.5).at(time.time_ns()) + + try: + sender.flush(transactional=True) + except qls.SenderError as e: + if not QDB_FIXTURE.http: + raise e + self.assertIn('Could not flush buffer', str(e)) + self.assertIn('cast error from', str(e)) + self.assertIn('STRING', str(e)) + self.assertIn('code: invalid, line: 3', str(e)) + + with self.assertRaises(TimeoutError): + retry_check_table(table_name, timeout_sec=1, log=False) + + def test_tcp_transactions(self): + if QDB_FIXTURE.http: + self.skipTest('TCP-only test') + if QDB_FIXTURE.version <= (7, 3, 7): + self.skipTest('No ILP/HTTP support') + buf = qls.Buffer() + buf.table('t1').column('c1', 'v1').at(time.time_ns()) + with self.assertRaisesRegex(qls.SenderError, r'.*Transactional .* not supported.*'): + with self._mk_linesender() as sender: + sender.flush(buf, transactional=True) + + def test_bad_env_var(self): + if not BUILD_MODE == qls.BuildMode.API: + self.skipTest('BuildMode.API-only test') + env_var = 'QDB_CLIENT_CONF' + if env_var in os.environ: + del os.environ[env_var] + with self.assertRaisesRegex(qls.SenderError, r'.*Environment variable QDB_CLIENT_CONF not set.*'): + qls._error_wrapped_call(qls._DLL.line_sender_from_env) + + def test_manifest_yaml(self): + # Check the manifest file can be read as yaml. + try: + import yaml + except ImportError: + self.skipTest('Python version does not support yaml') + proj = Project() + manifest_path = proj.root_dir / 'examples.manifest.yaml' + with open(manifest_path, 'r') as f: + yaml.safe_load(f) - def test_tls_webpki_and_os_roots(self): - self._test_tls_special('webpki_and_os_roots') def parse_args(): parser = argparse.ArgumentParser('Run system tests.') @@ -648,13 +788,20 @@ def parse_args(): metavar='HOST:ILP_PORT:HTTP_PORT', help=('Test against existing running instance. ' + 'e.g. `localhost:9009:9000`')) + version_g.add_argument( + '--repo', + type=str, + metavar='PATH', + help=('Test against existing jar from a ' + + '`mvn install -DskipTests -P build-web-console`' + + '-ed questdb repo such as `~/questdb/repos/questdb/`')) list_p = sub_p.add_parser('list', help='List latest -n releases.') list_p.set_defaults(command='list') list_p.add_argument('-n', type=int, default=30, help='number of releases') return parser.parse_known_args() -def list(args): +def list_releases(args): print('List of releases:') for vers, _ in list_questdb_releases(args.n or 1): print(f' {vers}') @@ -664,19 +811,29 @@ def run_with_existing(args): global QDB_FIXTURE MockFixture = namedtuple( 'MockFixture', - ('host', 'line_tcp_port', 'http_server_port', 'version')) + ('host', 'line_tcp_port', 'http_server_port', 'version', 'http')) host, line_tcp_port, http_server_port = args.existing.split(':') QDB_FIXTURE = MockFixture( host, int(line_tcp_port), int(http_server_port), - (999, 999, 999)) + (999, 999, 999), + True) unittest.main() -def run_with_fixtures(args): - global QDB_FIXTURE - global TLS_PROXY_FIXTURE +def iter_versions(args): + """ + Iterate target versions. + Returns a generator of prepared questdb directories. + Ensure that the DB is stopped after each use. + """ + if getattr(args, 'repo', None): + # A specific repo path was provided. + repo = pathlib.Path(args.repo) + yield install_questdb_from_repo(repo) + return + versions = None versions_args = getattr(args, 'versions', None) if versions_args: @@ -696,21 +853,45 @@ def run_with_fixtures(args): in list_questdb_releases(last_n)} for version, download_url in versions.items(): questdb_dir = install_questdb(version, download_url) - for auth in (False, True): - QDB_FIXTURE = QuestDbFixture(questdb_dir, auth=auth) - TLS_PROXY_FIXTURE = None - try: - QDB_FIXTURE.start() - TLS_PROXY_FIXTURE = TlsProxyFixture(QDB_FIXTURE.line_tcp_port) - TLS_PROXY_FIXTURE.start() + yield questdb_dir - test_prog = unittest.TestProgram(exit=False) - if not test_prog.result.wasSuccessful(): - sys.exit(1) - finally: - if TLS_PROXY_FIXTURE: - TLS_PROXY_FIXTURE.stop() - QDB_FIXTURE.stop() + +def run_with_fixtures(args): + global QDB_FIXTURE + global TLS_PROXY_FIXTURE + global BUILD_MODE + last_version = None + for questdb_dir in iter_versions(args): + for auth in (False, True): + for http in (False, True): + for build_mode in list(qls.BuildMode): + print(f'Running tests [questdb_dir={questdb_dir}, auth={auth}, http={http}, build_mode={build_mode}]') + if http and last_version <= (7, 3, 7): + print('Skipping ILP/HTTP tests for versions <= 7.3.7') + continue + if http and auth: + print('Skipping auth for ILP/HTTP tests') + continue + QDB_FIXTURE = QuestDbFixture(questdb_dir, auth=auth, http=http) + TLS_PROXY_FIXTURE = None + BUILD_MODE = build_mode + try: + QDB_FIXTURE.start() + # Read the version _after_ a first start so it can rely + # on the live one from the `select build` query. + last_version = QDB_FIXTURE.version + port_to_proxy = QDB_FIXTURE.http_server_port \ + if http else QDB_FIXTURE.line_tcp_port + TLS_PROXY_FIXTURE = TlsProxyFixture(port_to_proxy) + TLS_PROXY_FIXTURE.start() + + test_prog = unittest.TestProgram(exit=False) + if not test_prog.result.wasSuccessful(): + sys.exit(1) + finally: + if TLS_PROXY_FIXTURE: + TLS_PROXY_FIXTURE.stop() + QDB_FIXTURE.stop() def run(args, show_help=False): @@ -729,7 +910,7 @@ def run(args, show_help=False): def main(): args, extra_args = parse_args() if args.command == 'list': - list(args) + list_releases(args) else: # Repackage args for unittest's own arg parser. sys.argv[:] = sys.argv[:1] + extra_args diff --git a/system_test/tls_proxy/Cargo.lock b/system_test/tls_proxy/Cargo.lock index 9f0e9110..2ca51146 100644 --- a/system_test/tls_proxy/Cargo.lock +++ b/system_test/tls_proxy/Cargo.lock @@ -17,6 +17,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "anyhow" +version = "1.0.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" + [[package]] name = "argh" version = "0.1.12" @@ -36,7 +42,7 @@ dependencies = [ "argh_shared", "proc-macro2", "quote", - "syn 2.0.33", + "syn", ] [[package]] @@ -81,12 +87,6 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" -[[package]] -name = "bumpalo" -version = "3.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37ccbd214614c6783386c1af30caf03192f17891059cecc394b4fb119e363de3" - [[package]] name = "bytes" version = "1.1.0" @@ -95,9 +95,12 @@ checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" [[package]] name = "cc" -version = "1.0.73" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +dependencies = [ + "libc", +] [[package]] name = "cfg-if" @@ -106,34 +109,119 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] -name = "gimli" -version = "0.28.0" +name = "futures" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" +checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] [[package]] -name = "hermit-abi" -version = "0.1.19" +name = "futures-channel" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" dependencies = [ - "libc", + "futures-core", + "futures-sink", ] [[package]] -name = "js-sys" -version = "0.3.58" +name = "futures-core" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3fac17f7123a73ca62df411b1bf727ccc805daa070338fda671c86dac1bdc27" +checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" + +[[package]] +name = "futures-executor" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc" dependencies = [ - "wasm-bindgen", + "futures-core", + "futures-task", + "futures-util", ] [[package]] -name = "lazy_static" -version = "1.4.0" +name = "futures-io" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" + +[[package]] +name = "futures-macro" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" + +[[package]] +name = "futures-task" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" + +[[package]] +name = "futures-util" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "gimli" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] [[package]] name = "libc" @@ -205,12 +293,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "once_cell" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18a6dbe30758c9f83eb00cbea4ac95966305f5a7772f3f42ebfc7fc7eddbd8e1" - [[package]] name = "parking_lot" version = "0.12.1" @@ -240,6 +322,12 @@ version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + [[package]] name = "proc-macro2" version = "1.0.67" @@ -269,17 +357,16 @@ dependencies = [ [[package]] name = "ring" -version = "0.16.20" +version = "0.17.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +checksum = "688c63d65483050968b2a8937f7995f443e27041a0f7700aa59b0822aedebb74" dependencies = [ "cc", + "getrandom", "libc", - "once_cell", "spin", "untrusted", - "web-sys", - "winapi", + "windows-sys 0.48.0", ] [[package]] @@ -290,32 +377,42 @@ checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" [[package]] name = "rustls" -version = "0.21.7" +version = "0.22.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8" +checksum = "e87c9956bd9807afa1f77e0f7594af32566e830e088a5576d27c5b6f30f49d41" dependencies = [ "log", "ring", + "rustls-pki-types", "rustls-webpki", - "sct", + "subtle", + "zeroize", ] [[package]] name = "rustls-pemfile" -version = "1.0.3" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" +checksum = "35e4980fa29e4c4b212ffb3db068a564cbf560e51d3944b7c88bd8bf5bec64f4" dependencies = [ "base64", + "rustls-pki-types", ] +[[package]] +name = "rustls-pki-types" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a716eb65e3158e90e17cd93d855216e27bde02745ab842f2cab4a39dba1bacf" + [[package]] name = "rustls-webpki" -version = "0.101.5" +version = "0.102.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45a27e3b59326c16e23d30aeb7a36a24cc0d29e71d68ff611cdfb4a01d013bed" +checksum = "faaa0a62740bedb9b2ef5afa303da42764c012f743917351dc9a237ea1663610" dependencies = [ "ring", + "rustls-pki-types", "untrusted", ] @@ -325,16 +422,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" -[[package]] -name = "sct" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "serde" version = "1.0.188" @@ -352,7 +439,7 @@ checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.33", + "syn", ] [[package]] @@ -364,6 +451,15 @@ dependencies = [ "libc", ] +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + [[package]] name = "smallvec" version = "1.9.0" @@ -382,20 +478,15 @@ dependencies = [ [[package]] name = "spin" -version = "0.5.2" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" [[package]] -name = "syn" -version = "1.0.98" +name = "subtle" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c50aef8a904de4c23c788f104b7dddc7d6f79c647c7c8ce4cc8f73eb0ca773dd" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "syn" @@ -412,7 +503,9 @@ dependencies = [ name = "tls_proxy" version = "0.1.0" dependencies = [ + "anyhow", "argh", + "futures", "rustls", "rustls-pemfile", "tokio", @@ -446,16 +539,17 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.33", + "syn", ] [[package]] name = "tokio-rustls" -version = "0.24.1" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" dependencies = [ "rustls", + "rustls-pki-types", "tokio", ] @@ -467,9 +561,9 @@ checksum = "5bd2fe26506023ed7b5e1e315add59d6f584c621d037f9368fea9cfb988f368c" [[package]] name = "untrusted" -version = "0.7.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "wasi" @@ -477,92 +571,6 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" -[[package]] -name = "wasm-bindgen" -version = "0.2.81" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c53b543413a17a202f4be280a7e5c62a1c69345f5de525ee64f8cfdbc954994" -dependencies = [ - "cfg-if", - "wasm-bindgen-macro", -] - -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.81" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5491a68ab4500fa6b4d726bd67408630c3dbe9c4fe7bda16d5c82a1fd8c7340a" -dependencies = [ - "bumpalo", - "lazy_static", - "log", - "proc-macro2", - "quote", - "syn 1.0.98", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.81" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c441e177922bc58f1e12c022624b6216378e5febc2f0533e41ba443d505b80aa" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.81" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d94ac45fcf608c1f45ef53e748d35660f168490c10b23704c7779ab8f5c3048" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.98", - "wasm-bindgen-backend", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.81" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a89911bd99e5f3659ec4acf9c4d93b0a90fe4a2a11f15328472058edc5261be" - -[[package]] -name = "web-sys" -version = "0.3.58" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fed94beee57daf8dd7d51f2b15dc2bcde92d7a72304cdf662a4371008b71b90" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - [[package]] name = "windows-sys" version = "0.36.1" @@ -671,3 +679,9 @@ name = "windows_x86_64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "zeroize" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" diff --git a/system_test/tls_proxy/Cargo.toml b/system_test/tls_proxy/Cargo.toml index 3b76fbbb..73d72564 100644 --- a/system_test/tls_proxy/Cargo.toml +++ b/system_test/tls_proxy/Cargo.toml @@ -7,7 +7,9 @@ edition = "2021" [dependencies] tokio = { version = "1.32.0", features = ["full"] } -tokio-rustls = "0.24.1" -rustls = "0.21.7" -rustls-pemfile = "1.0.3" +tokio-rustls = "0.25.0" +rustls = "0.22.2" +rustls-pemfile = "2.0.0" argh = "0.1.12" +anyhow = "1.0.75" +futures = "0.3.29" \ No newline at end of file diff --git a/system_test/tls_proxy/src/lib.rs b/system_test/tls_proxy/src/lib.rs new file mode 100644 index 00000000..19e3a3b8 --- /dev/null +++ b/system_test/tls_proxy/src/lib.rs @@ -0,0 +1,181 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2024 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +// See: https://zmedley.com/tcp-proxy.html +// and: https://github.com/tokio-rs/tls/blob/master/tokio-rustls/examples/server/src/main.rs + +use std::fs::File; +use std::io::BufReader; +use std::sync::Arc; + +use tokio::io as tio; +use tokio::net::TcpListener; +use tokio::net::TcpStream; +use tokio::select; +use tokio_rustls::rustls::ServerConfig; +use tokio_rustls::TlsAcceptor; + +fn certs_dir() -> std::path::PathBuf { + let mut certs_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")); + certs_dir.push(".."); + certs_dir.push(".."); + certs_dir.push("tls_certs"); + certs_dir +} + +pub fn tls_config() -> Arc { + let certs_dir = certs_dir(); + let mut cert_file = + File::open(certs_dir.join("server.crt")).expect("cannot open certificate file"); + let mut private_key_file = + File::open(certs_dir.join("server.key")).expect("cannot open private key file"); + let certs = rustls_pemfile::certs(&mut BufReader::new(&mut cert_file)) + .collect::, _>>() + .expect("cannot read certificate file"); + let private_key = rustls_pemfile::private_key(&mut BufReader::new(&mut private_key_file)) + .expect("cannot read private key file") + .expect("no private key found"); + let config = ServerConfig::builder() + .with_no_client_auth() + .with_single_cert(certs, private_key) + .unwrap(); + Arc::new(config) +} + +async fn handle_conn( + listener: &TcpListener, + acceptor: &TlsAcceptor, + dest_addr: &str, +) -> Result<(), Box> { + eprintln!("Waiting for a connection."); + let (inbound_conn, _) = listener.accept().await?; + eprintln!("Accepted a client connection."); + let acceptor = acceptor.clone(); + let inbound_conn = acceptor.accept(inbound_conn).await?; + eprintln!("Completed TLS handshake with client connection."); + let outbound_conn = TcpStream::connect(dest_addr).await?; + eprintln!("Established outbound connection to {}.", dest_addr); + + let (mut in_read, mut in_write) = tio::split(inbound_conn); + let (mut out_read, mut out_write) = outbound_conn.into_split(); + + let in_to_out = tokio::spawn(async move { tio::copy(&mut in_read, &mut out_write).await }); + let out_to_in = tokio::spawn(async move { tio::copy(&mut out_read, &mut in_write).await }); + + select! { + _ = in_to_out => eprintln!("in_to_out shut down."), + _ = out_to_in => eprintln!("out_to_in shut down."), + } + + Ok(()) +} + +async fn loop_server( + dest_port: u16, + listen_port_sender: tokio::sync::oneshot::Sender, +) -> anyhow::Result<()> { + let dest_addr = format!("localhost:{}", dest_port); + eprintln!("Destination address is {}.", &dest_addr); + + let config = tls_config(); + let acceptor = TlsAcceptor::from(config); + + let listener = TcpListener::bind("0.0.0.0:0").await?; + let listen_port = listener.local_addr()?.port(); + eprintln!("TLS Proxy is listening on localhost:{}.", listen_port); + listen_port_sender.send(listen_port).unwrap(); + + loop { + if let Err(err) = handle_conn(&listener, &acceptor, &dest_addr).await { + eprintln!("Error handling connection: {}", err); + } + } +} + +fn recv_port(port_receiver: &mut tokio::sync::oneshot::Receiver) -> anyhow::Result { + loop { + match port_receiver.try_recv() { + Ok(port) => return Ok(port), + Err(tokio::sync::oneshot::error::TryRecvError::Empty) => { + std::thread::sleep(std::time::Duration::from_millis(100)) + } + Err(tokio::sync::oneshot::error::TryRecvError::Closed) => { + return Err(anyhow::anyhow!("Could not obtain listening port")) + } + } + } +} + +pub struct TlsProxy { + _runtime: tokio::runtime::Runtime, + loop_handle: Option>>, + dest_port: u16, + listen_port: u16, +} + +impl TlsProxy { + pub fn new(dest_port: u16) -> anyhow::Result { + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .unwrap(); + let (port_sender, mut port_receiver) = tokio::sync::oneshot::channel(); + let loop_handle = + Some(runtime.spawn(async move { loop_server(dest_port, port_sender).await })); + let listen_port = recv_port(&mut port_receiver)?; + Ok(Self { + _runtime: runtime, + loop_handle, + dest_port, + listen_port, + }) + } + + pub fn run_indefinitely(mut self) -> anyhow::Result<()> { + if self.loop_handle.is_none() { + return Err(anyhow::anyhow!("TlsProxy already stopped")); + } + let loop_handle = self.loop_handle.take().unwrap(); + futures::executor::block_on(async { loop_handle.await? }) + } + + pub fn dest_port(&self) -> u16 { + self.dest_port + } + + pub fn listen_port(&mut self) -> u16 { + self.listen_port + } +} + +impl Drop for TlsProxy { + fn drop(&mut self) { + if self.loop_handle.is_none() { + return; + } + futures::executor::block_on(async { + self.loop_handle.take().unwrap().abort(); + }); + } +} diff --git a/system_test/tls_proxy/src/main.rs b/system_test/tls_proxy/src/main.rs index 32d761b4..3836e3c0 100644 --- a/system_test/tls_proxy/src/main.rs +++ b/system_test/tls_proxy/src/main.rs @@ -6,7 +6,7 @@ * \__\_\\__,_|\___||___/\__|____/|____/ * * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2023 QuestDB + * Copyright (c) 2019-2024 QuestDB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,17 +25,7 @@ // See: https://zmedley.com/tcp-proxy.html // and: https://github.com/tokio-rs/tls/blob/master/tokio-rustls/examples/server/src/main.rs -use std::io::BufReader; -use std::path::Path; -use std::sync::Arc; - use argh::FromArgs; -use tokio::io as tio; -use tokio::net::TcpListener; -use tokio::net::TcpStream; -use tokio::select; -use tokio_rustls::rustls::{self, server::NoClientAuth, Certificate, ServerConfig}; -use tokio_rustls::TlsAcceptor; /// Options for TLS localhost proxy #[derive(FromArgs)] @@ -45,103 +35,9 @@ struct Options { port: u16, } -fn load_certs(filename: &Path) -> Vec { - let certfile = std::fs::File::open(filename).expect("cannot open certificate file"); - let mut reader = BufReader::new(certfile); - rustls_pemfile::certs(&mut reader) - .unwrap() - .iter() - .map(|v| Certificate(v.clone())) - .collect() -} - -fn load_private_key(filename: &Path) -> rustls::PrivateKey { - let keyfile = std::fs::File::open(filename).expect("cannot open private key file"); - let mut reader = BufReader::new(keyfile); - - loop { - match rustls_pemfile::read_one(&mut reader).expect("cannot parse private key .pem file") { - Some(rustls_pemfile::Item::RSAKey(key)) => return rustls::PrivateKey(key), - Some(rustls_pemfile::Item::PKCS8Key(key)) => return rustls::PrivateKey(key), - Some(rustls_pemfile::Item::ECKey(key)) => return rustls::PrivateKey(key), - None => break, - _ => {} - } - } - - panic!( - "no keys found in {:?} (encrypted keys not supported)", - filename - ); -} - -fn certs_dir() -> std::path::PathBuf { - let mut certs_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")); - certs_dir.push(".."); - certs_dir.push(".."); - certs_dir.push("tls_certs"); - certs_dir -} - -fn tls_config() -> Arc { - let certs_dir = certs_dir(); - let cert_chain = load_certs(&certs_dir.join("server.crt")); - let key_der = load_private_key(&certs_dir.join("server.key")); - let config = ServerConfig::builder() - .with_safe_default_cipher_suites() - .with_safe_default_kx_groups() - .with_safe_default_protocol_versions() - .unwrap() - .with_client_cert_verifier(NoClientAuth::boxed()) - .with_single_cert(cert_chain, key_der) - .unwrap(); - Arc::new(config) -} - -async fn handle_conn( - listener: &TcpListener, - acceptor: &TlsAcceptor, - dest_addr: &str, -) -> Result<(), Box> { - eprintln!("Waiting for a connection."); - let (inbound_conn, _) = listener.accept().await?; - eprintln!("Accepted a client connection."); - let acceptor = acceptor.clone(); - let inbound_conn = acceptor.accept(inbound_conn).await?; - eprintln!("Completed TLS handshake with client connection."); - let outbound_conn = TcpStream::connect(dest_addr).await?; - eprintln!("Established outbound connection to {}.", dest_addr); - - let (mut in_read, mut in_write) = tio::split(inbound_conn); - let (mut out_read, mut out_write) = outbound_conn.into_split(); - - let in_to_out = tokio::spawn(async move { tio::copy(&mut in_read, &mut out_write).await }); - let out_to_in = tokio::spawn(async move { tio::copy(&mut out_read, &mut in_write).await }); - - select! { - _ = in_to_out => eprintln!("in_to_out shut down."), - _ = out_to_in => eprintln!("out_to_in shut down."), - } - - Ok(()) -} - #[tokio::main] -async fn main() -> Result<(), Box> { +async fn main() -> anyhow::Result<()> { let options: Options = argh::from_env(); - let dest_addr = format!("localhost:{}", options.port); - eprintln!("Destination address is {}.", &dest_addr); - - let config = tls_config(); - let acceptor = TlsAcceptor::from(config); - - let listener = TcpListener::bind("0.0.0.0:0").await?; - let listen_port = listener.local_addr()?.port(); - eprintln!("TLS Proxy is listening on localhost:{}.", listen_port); - - loop { - if let Err(err) = handle_conn(&listener, &acceptor, &dest_addr).await { - eprintln!("Error handling connection: {}", err); - } - } + let server = tls_proxy::TlsProxy::new(options.port)?; + server.run_indefinitely() }