Merge branch 'branch-24.08' of github.com:rapidsai/cudf into pylibcud…

…f-io-writers

python/cudf/cudf/_lib/pylibcudf/io/CMakeLists.txt

@@ -12,7 +12,7 @@
 # the License.
 # =============================================================================
 
-set(cython_sources avro.pyx json.pyx types.pyx)
+set(cython_sources avro.pyx datasource.pyx json.pyx types.pyx)
 
 set(linked_libraries cudf::cudf)
 rapids_cython_create_modules(
@@ -21,5 +21,7 @@ rapids_cython_create_modules(
   LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX pylibcudf_io_ ASSOCIATED_TARGETS cudf
 )
 
-set(targets_using_arrow_headers pylibcudf_io_avro pylibcudf_io_json pylibcudf_io_types)
+set(targets_using_arrow_headers pylibcudf_io_avro pylibcudf_io_datasource pylibcudf_io_json
+                                pylibcudf_io_types
+)
 link_to_pyarrow_headers("${targets_using_arrow_headers}")

python/cudf/cudf/_lib/pylibcudf/io/__init__.pxd

@@ -1,4 +1,4 @@
 # Copyright (c) 2024, NVIDIA CORPORATION.
 
-from . cimport avro, json, types
-from .types cimport SinkInfo, SourceInfo, TableWithMetadata
+from . cimport avro, datasource, json, types
+from .types cimport SourceInfo, TableWithMetadata

python/cudf/cudf/_lib/pylibcudf/io/__init__.py

@@ -1,4 +1,4 @@
 # Copyright (c) 2024, NVIDIA CORPORATION.
 
-from . import avro, json, types
+from . import avro, datasource, json, types
 from .types import SinkInfo, SourceInfo, TableWithMetadata

python/cudf/cudf/_lib/pylibcudf/io/types.pyx

@@ -7,7 +7,9 @@ from libcpp.string cimport string
 from libcpp.utility cimport move
 from libcpp.vector cimport vector
 
+from cudf._lib.pylibcudf.io.datasource cimport Datasource
 from cudf._lib.pylibcudf.libcudf.io.data_sink cimport data_sink
+from cudf._lib.pylibcudf.libcudf.io.datasource cimport datasource
 from cudf._lib.pylibcudf.libcudf.io.types cimport (
     column_name_info,
     host_buffer,

python/cudf/cudf/pylibcudf_tests/io/test_source_sink_info.py

@@ -14,6 +14,16 @@ def io_class(request):
     return request.param
 
 
+def _skip_invalid_sinks(io_class, sink):
+    """
+    Skip invalid sinks for SinkInfo
+    """
+    if io_class is plc.io.SinkInfo and isinstance(
+        sink, (bytes, NativeFileDatasource)
+    ):
+        pytest.skip(f"{sink} is not a valid input for SinkInfo")
+
+
 @pytest.mark.parametrize(
     "source",
     [
@@ -29,8 +39,7 @@ def test_source_info_ctor(io_class, source, tmp_path):
         file.write_bytes("hello world".encode("utf-8"))
         source = str(file)
 
-    if io_class is plc.io.SinkInfo and isinstance(source, bytes):
-        pytest.skip("bytes is not a valid input for SinkInfo")
+    _skip_invalid_sinks(source)
 
     io_class([source])
 
@@ -54,8 +63,8 @@ def test_source_info_ctor_multiple(io_class, sources, tmp_path):
             file = tmp_path / source
             file.write_bytes("hello world".encode("utf-8"))
             sources[i] = str(file)
-        elif io_class is plc.io.SinkInfo and isinstance(source, bytes):
-            pytest.skip("bytes is not a valid input for SinkInfo")
+
+        _skip_invalid_sinks(source)
 
     io_class(sources)
 
@@ -86,7 +95,11 @@ def test_source_info_ctor_mixing_invalid(io_class, sources, tmp_path):
             file = tmp_path / source
             file.write_bytes("hello world".encode("utf-8"))
             sources[i] = str(file)
-        elif io_class is plc.io.SinkInfo and isinstance(source, bytes):
-            pytest.skip("bytes is not a valid input for SinkInfo")
+        _skip_invalid_sinks(source)
     with pytest.raises(ValueError):
         io_class(sources)
+
+
+def test_source_info_invalid():
+    with pytest.raises(ValueError):
+        plc.io.SourceInfo([123])