feat(c/driver/sqlite): add Python SQLite driver bindings (apache#201)

.github/workflows/cpp.yml

@@ -195,6 +195,20 @@ jobs:
           ctest --output-on-failure --no-tests=error
           if %errorlevel% neq 0 then exit /b %errorlevel%
           cd ..\..
+      - name: Build/Test SQLite Driver
+        shell: cmd /C call {0}
+        run: |
+          mkdir build\driver_sqlite
+          cd build\driver_sqlite
+          cmake -GNinja ..\..\c\driver\sqlite -DADBC_BUILD_TESTS=ON -DCMAKE_BUILD_TYPE=Debug -DCMAKE_PREFIX_PATH=%CONDA_PREFIX% -DADBC_BUILD_SHARED=ON -DADBC_BUILD_STATIC=OFF
+          if %errorlevel% neq 0 then exit /b %errorlevel%
+          cmake --build .
+          dir
+          if %errorlevel% neq 0 then exit /b %errorlevel%
+          set PATH=%PATH%;${{ github.workspace }}\build\driver_sqlite
+          ctest --output-on-failure --no-tests=error
+          if %errorlevel% neq 0 then exit /b %errorlevel%
+          cd ..\..
       - name: Build/Test Python Driver Manager
         shell: cmd /C call {0}
         run: |
@@ -203,3 +217,11 @@ jobs:
           set PATH=%PATH%;${{ github.workspace }}\build\driver_sqlite
           python -m pytest -vv
           cd ..\..
+      - name: Build/Test Python Driver SQLite
+        shell: cmd /C call {0}
+        run: |
+          cd python\adbc_driver_sqlite
+          set ADBC_SQLITE_LIBRARY=${{ github.workspace }}\build\driver_sqlite\adbc_driver_sqlite.dll
+          pip install -e .
+          python -m pytest -vv
+          cd ..\..

c/driver/sqlite/CMakeLists.txt

@@ -25,9 +25,16 @@ project(adbc_driver_sqlite
         VERSION "${ADBC_BASE_VERSION}"
         LANGUAGES CXX)
 include(CTest)
-find_package(PkgConfig)
 
-find_package(SQLite3 REQUIRED)
+if(WIN32)
+  # XXX: for now, assume vcpkg
+  find_package(unofficial-sqlite3 CONFIG REQUIRED)
+  set(SQLite3_LINK_LIBRARIES unofficial::sqlite3::sqlite3)
+  set(SQLite3_INCLUDE_DIRS)
+else()
+  find_package(SQLite3 REQUIRED)
+  set(SQLite3_LINK_LIBRARIES SQLite::SQLite3)
+endif()
 
 add_arrow_lib(adbc_driver_sqlite
               SOURCES
@@ -39,10 +46,10 @@ add_arrow_lib(adbc_driver_sqlite
               SHARED_LINK_FLAGS
               ${ADBC_LINK_FLAGS}
               SHARED_LINK_LIBS
-              SQLite::SQLite3
+              ${SQLite3_LINK_LIBRARIES}
               nanoarrow
               STATIC_LINK_LIBS
-              SQLite::SQLite3
+              ${SQLite3_LINK_LIBRARIES}
               nanoarrow
               ${LIBPQ_STATIC_LIBRARIES})
 include_directories(SYSTEM ${REPOSITORY_ROOT})

c/driver/sqlite/sqlite.c

@@ -1783,6 +1783,7 @@ AdbcStatusCode AdbcStatementExecutePartitions(struct AdbcStatement* statement,
 }  // NOLINT(whitespace/indent)
 // due to https://github.com/cpplint/cpplint/pull/189
 
+ADBC_EXPORT
 AdbcStatusCode AdbcDriverInit(int version, void* driver, struct AdbcError* error) {
   return SqliteDriverInit(version, driver, error);
 }

c/driver/sqlite/utils.h

@@ -21,9 +21,16 @@
 
 #include <adbc.h>
 
+#if defined(__GNUC__)
+#define SET_ERROR_ATTRIBUTE __attribute__((format(printf, 2, 3)))
+#else
+#define SET_ERROR_ATTRIBUTE
+#endif
+
 /// Set error details using a format string.
-void SetError(struct AdbcError* error, const char* format, ...)
-    __attribute__((format(printf, 2, 3)));
+void SetError(struct AdbcError* error, const char* format, ...) SET_ERROR_ATTRIBUTE;
+
+#undef SET_ERROR_ATTRIBUTE
 
 /// Wrap a single batch as a stream.
 AdbcStatusCode BatchToArrayStream(struct ArrowArray* values, struct ArrowSchema* schema,

c/driver_manager/adbc_driver_manager.cc

@@ -665,7 +665,9 @@ AdbcStatusCode AdbcLoadDriver(const char* driver_name, const char* entrypoint,
   void* load_handle = GetProcAddress(handle, entrypoint);
   init_func = reinterpret_cast<AdbcDriverInitFunc>(load_handle);
   if (!init_func) {
-    std::string message = "GetProcAddress() failed: ";
+    std::string message = "GetProcAddress(";
+    message += entrypoint;
+    message += ") failed: ";
     GetWinError(&message);
     if (!FreeLibrary(handle)) {
       message += "\nFreeLibrary() failed: ";
@@ -722,7 +724,9 @@ AdbcStatusCode AdbcLoadDriver(const char* driver_name, const char* entrypoint,
 
   void* load_handle = dlsym(handle, entrypoint);
   if (!load_handle) {
-    std::string message = "dlsym() failed: ";
+    std::string message = "dlsym(";
+    message += entrypoint;
+    message += ") failed: ";
     message += dlerror();
     SetError(error, message);
     return ADBC_STATUS_INTERNAL;

ci/scripts/python_build.sh

@@ -21,6 +21,7 @@ set -e
 : ${BUILD_ALL:=1}
 : ${BUILD_DRIVER_MANAGER:=${BUILD_ALL}}
 : ${BUILD_DRIVER_POSTGRES:=${BUILD_ALL}}
+: ${BUILD_DRIVER_SQLITE:=${BUILD_ALL}}
 
 if [[ $(uname) = "Darwin" ]]; then
     ADBC_LIBRARY_SUFFIX="dylib"
@@ -33,10 +34,13 @@ build_subproject() {
     local -r install_dir="${2}"
     local -r subproject="${3}"
 
-    if [[ "${subproject}" -eq "adbc_driver_postgres" ]]; then
+    if [[ "${subproject}" = "adbc_driver_postgres" ]]; then
         export ADBC_POSTGRES_LIBRARY="${install_dir}/lib/libadbc_driver_postgres.${ADBC_LIBRARY_SUFFIX}"
+    elif [[ "${subproject}" = "adbc_driver_sqlite" ]]; then
+        export ADBC_SQLITE_LIBRARY="${install_dir}/lib/libadbc_driver_sqlite.${ADBC_LIBRARY_SUFFIX}"
     fi
 
+    echo foo $subproject $ADBC_SQLITE_LIBRARY
     python -m pip install -e "${source_dir}/python/${subproject}"
 }
 
@@ -56,6 +60,10 @@ main() {
     if [[ "${BUILD_DRIVER_POSTGRES}" -gt 0 ]]; then
         build_subproject "${source_dir}" "${install_dir}" adbc_driver_postgres
     fi
+
+    if [[ "${BUILD_DRIVER_SQLITE}" -gt 0 ]]; then
+        build_subproject "${source_dir}" "${install_dir}" adbc_driver_sqlite
+    fi
 }
 
 main "$@"

ci/scripts/python_sdist_build.sh

@@ -20,6 +20,9 @@
 set -ex
 
 source_dir=${1}
+script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+source "${script_dir}/python_util.sh"
 
 echo "=== (${PYTHON_VERSION}) Building ADBC sdists ==="
 
@@ -30,7 +33,7 @@ pip install --upgrade pip setuptools
 # For drivers, which bundle shared libraries, defer that to install time
 export _ADBC_IS_SDIST=1
 
-for component in adbc_driver_manager adbc_driver_postgres; do
+for component in ${COMPONENTS}; do
     pushd ${source_dir}/python/$component
 
     echo "=== (${PYTHON_VERSION}) Building $component sdist ==="

ci/scripts/python_test.sh

@@ -21,6 +21,7 @@ set -e
 : ${BUILD_ALL:=1}
 : ${BUILD_DRIVER_MANAGER:=${BUILD_ALL}}
 : ${BUILD_DRIVER_POSTGRES:=${BUILD_ALL}}
+: ${BUILD_DRIVER_SQLITE:=${BUILD_ALL}}
 
 test_subproject() {
     local -r source_dir=${1}
@@ -53,6 +54,10 @@ main() {
     if [[ "${BUILD_DRIVER_POSTGRES}" -gt 0 ]]; then
         test_subproject "${source_dir}" "${install_dir}" adbc_driver_postgres
     fi
+
+    if [[ "${BUILD_DRIVER_SQLITE}" -gt 0 ]]; then
+        test_subproject "${source_dir}" "${install_dir}" adbc_driver_sqlite
+    fi
 }
 
 main "$@"

ci/scripts/python_util.sh

@@ -17,7 +17,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-COMPONENTS="adbc_driver_manager adbc_driver_postgres"
+COMPONENTS="adbc_driver_manager adbc_driver_postgres adbc_driver_sqlite"
 
 function build_drivers {
     local -r source_dir="$1"
@@ -30,21 +30,29 @@ function build_drivers {
     # Enable manifest mode
     : ${VCPKG_FEATURE_FLAGS:=manifests}
     # Add our custom triplets
-    : ${VCPKG_OVERLAY_TRIPLETS:="${source_dir}/ci/vcpkg/triplets/"}
+    export VCPKG_OVERLAY_TRIPLETS="${source_dir}/ci/vcpkg/triplets/"
 
     if [[ $(uname) == "Linux" ]]; then
         export ADBC_POSTGRES_LIBRARY=${build_dir}/lib/libadbc_driver_postgres.so
+        export ADBC_SQLITE_LIBRARY=${build_dir}/lib/libadbc_driver_sqlite.so
         export VCPKG_DEFAULT_TRIPLET="x64-linux-static-release"
+
+        # XXX: Patch the portfile
+        sed -i "s|include/postgresql/server/pg_config.h|include/server/pg_config.h|" \
+            "${VCPKG_ROOT}/ports/libpq/portfile.cmake"
     else # macOS
         export ADBC_POSTGRES_LIBRARY=${build_dir}/lib/libadbc_driver_postgres.dylib
+        export ADBC_SQLITE_LIBRARY=${build_dir}/lib/libadbc_driver_sqlite.dylib
         export VCPKG_DEFAULT_TRIPLET="x64-osx-static-release"
-    fi
 
-    echo ${VCPKG_DEFAULT_TRIPLET}
-
-    mkdir -p ${build_dir}
-    pushd ${build_dir}
+        # XXX: Patch the portfile
+        sed -i '.bak' "s|include/postgresql/server/pg_config.h|include/server/pg_config.h|" \
+            "${VCPKG_ROOT}/ports/libpq/portfile.cmake"
+    fi
 
+    echo "=== Building driver/postgres ==="
+    mkdir -p ${build_dir}/driver/postgres
+    pushd ${build_dir}/driver/postgres
     cmake \
         -G ${CMAKE_GENERATOR} \
         -DADBC_BUILD_SHARED=ON \
@@ -53,23 +61,44 @@ function build_drivers {
         -DCMAKE_INSTALL_PREFIX=${build_dir} \
         -DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake \
         -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \
+        -DVCPKG_OVERLAY_TRIPLETS="${VCPKG_OVERLAY_TRIPLETS}" \
+        -DVCPKG_TARGET_TRIPLET="${VCPKG_DEFAULT_TRIPLET}" \
         ${source_dir}/c/driver/postgres
     cmake --build . --target install -j
     popd
+
+    echo "=== Building driver/sqlite ==="
+    mkdir -p ${build_dir}/driver/sqlite
+    pushd ${build_dir}/driver/sqlite
+    cmake \
+        -G ${CMAKE_GENERATOR} \
+        -DADBC_BUILD_SHARED=ON \
+        -DADBC_BUILD_STATIC=OFF \
+        -DCMAKE_INSTALL_LIBDIR=lib \
+        -DCMAKE_INSTALL_PREFIX=${build_dir} \
+        -DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake \
+        -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \
+        -DVCPKG_OVERLAY_TRIPLETS="${VCPKG_OVERLAY_TRIPLETS}" \
+        -DVCPKG_TARGET_TRIPLET="${VCPKG_DEFAULT_TRIPLET}" \
+        ${source_dir}/c/driver/sqlite
+    cmake --build . --target install -j
+    popd
 }
 
 function test_packages {
-    python -c "
-import adbc_driver_manager
-import adbc_driver_manager.dbapi
-import adbc_driver_postgres
-import adbc_driver_postgres.dbapi
+    for component in ${COMPONENTS}; do
+        echo "=== Testing $component ==="
+
+        python -c "
+import $component
+import $component.dbapi
 "
 
-    # Will only run some smoke tests
-    # --import-mode required, else tries to import from the source dir instead of installed package
-    echo "=== Testing adbc_driver_manager ==="
-    python -m pytest -vvx --import-mode append -k "not sqlite" ${source_dir}/python/adbc_driver_manager/tests
-    echo "=== Testing adbc_driver_postgres ==="
-    python -m pytest -vvx --import-mode append ${source_dir}/python/adbc_driver_postgres/tests
+        # --import-mode required, else tries to import from the source dir instead of installed package
+        if [[ "${component}" = "adbc_driver_manager" ]]; then
+            python -m pytest -vvx --import-mode append -k "not sqlite" ${source_dir}/python/$component/tests
+        else
+            python -m pytest -vvx --import-mode append ${source_dir}/python/$component/tests
+        fi
+    done
 }

ci/scripts/python_wheel_unix_build.sh

@@ -62,18 +62,19 @@ function check_wheels {
     fi
 }
 
-echo "=== (${PYTHON_VERSION}) Building ADBC libpq driver ==="
-# Sets ADBC_POSTGRES_LIBRARY
+echo "=== (${PYTHON_VERSION}) Building C/C++ driver components ==="
+# Sets ADBC_POSTGRES_LIBRARY, ADBC_SQLITE_LIBRARY
 build_drivers "${source_dir}" "${build_dir}"
 
 # Check that we don't expose any unwanted symbols
 check_visibility $ADBC_POSTGRES_LIBRARY
+check_visibility $ADBC_SQLITE_LIBRARY
 
 # https://github.com/pypa/pip/issues/7555
 # Get the latest pip so we have in-tree-build by default
-pip install --upgrade pip
+pip install --upgrade pip auditwheel
 
-for component in adbc_driver_manager adbc_driver_postgres; do
+for component in $COMPONENTS; do
     pushd ${source_dir}/python/$component
 
     echo "=== (${PYTHON_VERSION}) Clean build artifacts==="

ci/scripts/python_wheel_windows_build.bat

@@ -30,8 +30,10 @@ set VCPKG_TARGET_TRIPLET=x64-windows-static
 
 IF NOT DEFINED VCPKG_ROOT (echo "Must set VCPKG_ROOT" && exit /B 1)
 
-mkdir %build_dir%
-pushd %build_dir%
+%VCPKG_ROOT%\vcpkg install --triplet=%VCPKG_TARGET_TRIPLET% libpq sqlite3
+
+mkdir %build_dir%\postgres
+pushd %build_dir%\postgres
 
 cmake ^
       -G "%CMAKE_GENERATOR%" ^
@@ -50,9 +52,29 @@ set ADBC_POSTGRES_LIBRARY=%build_dir%\bin\adbc_driver_postgres.dll
 
 popd
 
+mkdir %build_dir%\sqlite
+pushd %build_dir%\sqlite
+
+cmake ^
+      -G "%CMAKE_GENERATOR%" ^
+      -DADBC_BUILD_SHARED=ON ^
+      -DADBC_BUILD_STATIC=OFF ^
+      -DCMAKE_BUILD_TYPE=%CMAKE_BUILD_TYPE% ^
+      -DCMAKE_INSTALL_PREFIX=%build_dir% ^
+      -DCMAKE_TOOLCHAIN_FILE=%VCPKG_ROOT%/scripts/buildsystems/vcpkg.cmake ^
+      -DCMAKE_UNITY_BUILD=%CMAKE_UNITY_BUILD% ^
+      -DVCPKG_TARGET_TRIPLET=%VCPKG_TARGET_TRIPLET% ^
+      %source_dir%\c\driver\sqlite || exit /B 1
+cmake --build . --config %CMAKE_BUILD_TYPE% --target install -j || exit /B 1
+
+@REM XXX: CMake installs it to bin instead of lib for some reason
+set ADBC_SQLITE_LIBRARY=%build_dir%\bin\adbc_driver_sqlite.dll
+
+popd
+
 python -m pip install --upgrade pip delvewheel
 
-FOR %%c IN (adbc_driver_manager adbc_driver_postgres) DO (
+FOR %%c IN (adbc_driver_manager adbc_driver_postgres adbc_driver_sqlite) DO (
     pushd %source_dir%\python\%%c
 
     echo "=== (%PYTHON_VERSION%) Building %%c wheel ==="

ci/scripts/python_wheel_windows_test.bat

@@ -21,7 +21,7 @@ set source_dir=%1
 
 echo "=== (%PYTHON_VERSION%) Installing wheels ==="
 
-FOR %%c IN (adbc_driver_manager adbc_driver_postgres) DO (
+FOR %%c IN (adbc_driver_manager adbc_driver_postgres adbc_driver_sqlite) DO (
     FOR %%w IN (%source_dir%\python\%%c\dist\*.whl) DO (
         pip install --force-reinstall %%w || exit /B 1
     )
@@ -31,7 +31,7 @@ pip install pytest pyarrow pandas
 
 echo "=== (%PYTHON_VERSION%) Testing wheels ==="
 
-FOR %%c IN (adbc_driver_manager adbc_driver_postgres) DO (
+FOR %%c IN (adbc_driver_manager adbc_driver_postgres adbc_driver_sqlite) DO (
     echo "=== Testing %%c ==="
     python -c "import %%c" || exit /B 1
     python -c "import %%c.dbapi" || exit /B 1

docs/source/cpp/driver/index.rst

@@ -27,3 +27,4 @@ protocols/databases.  More may be available from third parties.
 
    flight_sql
    postgres
+   sqlite

docs/source/cpp/driver/postgres.rst

@@ -19,8 +19,8 @@
 libpq-based Driver
 ==================
 
-The Flight SQL Driver provides access to any database that supports
-the Postgres wire format.
+The Postgres driver provides access to any database that supports the
+Postgres wire format.
 
 Installation
 ============