From 4e6a04680e20b7c11d3ddcf6d5919e9418dbc41b Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Tue, 4 Aug 2026 21:45:48 +0000 Subject: [PATCH 01/18] Add pylibcudf bindings for fetch_byte_ranges_to_device --- .../io/parquet/io_utils/parquet_io_utils.cpp | 8 +- .../experimental/hybrid_scan_filters_test.cpp | 25 +++ .../source/pylibcudf/api_docs/io/index.rst | 2 + .../api_docs/io/parquet_io_utils.rst | 6 + python/pylibcudf/pylibcudf/io/CMakeLists.txt | 4 +- python/pylibcudf/pylibcudf/io/__init__.py | 2 + .../pylibcudf/io/parquet_io_utils.pxd | 18 +++ .../pylibcudf/io/parquet_io_utils.pyi | 22 +++ .../pylibcudf/io/parquet_io_utils.pyx | 151 ++++++++++++++++++ .../pylibcudf/io/parquet_metadata.pyx | 2 +- .../pylibcudf/libcudf/io/parquet_io_utils.pxd | 55 +++++++ 11 files changed, 291 insertions(+), 4 deletions(-) create mode 100644 docs/cudf/source/pylibcudf/api_docs/io/parquet_io_utils.rst create mode 100644 python/pylibcudf/pylibcudf/io/parquet_io_utils.pxd create mode 100644 python/pylibcudf/pylibcudf/io/parquet_io_utils.pyi create mode 100644 python/pylibcudf/pylibcudf/io/parquet_io_utils.pyx create mode 100644 python/pylibcudf/pylibcudf/libcudf/io/parquet_io_utils.pxd diff --git a/cpp/src/io/parquet/io_utils/parquet_io_utils.cpp b/cpp/src/io/parquet/io_utils/parquet_io_utils.cpp index b6f0413c57c1..8eab6ea3d933 100644 --- a/cpp/src/io/parquet/io_utils/parquet_io_utils.cpp +++ b/cpp/src/io/parquet/io_utils/parquet_io_utils.cpp @@ -323,8 +323,14 @@ fetch_byte_ranges_to_device_async_impl( auto const& byte_ranges = byte_ranges_per_source[source_idx]; // Total buffer size required for column chunks of this source + auto const source_size = datasources[source_idx].get().size(); auto const buffer_size = std::accumulate( - byte_ranges.begin(), byte_ranges.end(), std::size_t{0}, [](auto acc, auto const& range) { + byte_ranges.begin(), byte_ranges.end(), std::size_t{0}, [&](auto acc, auto const& range) { + CUDF_EXPECTS(range.offset() >= 0 and range.size() >= 0, + "Byte range offset and size must be non-negative"); + CUDF_EXPECTS( + static_cast(range.offset()) + static_cast(range.size()) <= source_size, + "Byte range exceeds datasource size"); return acc + range.size(); }); diff --git a/cpp/tests/io/experimental/hybrid_scan_filters_test.cpp b/cpp/tests/io/experimental/hybrid_scan_filters_test.cpp index d0d4f570adbc..b5b666258430 100644 --- a/cpp/tests/io/experimental/hybrid_scan_filters_test.cpp +++ b/cpp/tests/io/experimental/hybrid_scan_filters_test.cpp @@ -1768,3 +1768,28 @@ TEST_F(HybridScanFiltersTest, RowGroupPasses) }); } } + +TEST_F(HybridScanFiltersTest, FetchByteRangesInvalidRanges) +{ + std::vector data(1024); + auto const datasource = + cudf::io::datasource::create(cudf::host_span(data.data(), data.size())); + auto const stream = cudf::get_default_stream(); + auto const mr = cudf::get_current_device_resource_ref(); + + EXPECT_THROW( + cudf::io::parquet::fetch_byte_ranges_to_device_async( + *datasource, + std::vector{cudf::io::text::byte_range_info{-1, 16}}, + stream, + mr), + cudf::logic_error); + + EXPECT_THROW( + cudf::io::parquet::fetch_byte_ranges_to_device_async( + *datasource, + std::vector{cudf::io::text::byte_range_info{512, 1024}}, + stream, + mr), + cudf::logic_error); +} diff --git a/docs/cudf/source/pylibcudf/api_docs/io/index.rst b/docs/cudf/source/pylibcudf/api_docs/io/index.rst index 15a87175325f..45b4def70051 100644 --- a/docs/cudf/source/pylibcudf/api_docs/io/index.rst +++ b/docs/cudf/source/pylibcudf/api_docs/io/index.rst @@ -17,9 +17,11 @@ I/O Functions avro csv + experimental json orc parquet + parquet_io_utils parquet_metadata text timezone diff --git a/docs/cudf/source/pylibcudf/api_docs/io/parquet_io_utils.rst b/docs/cudf/source/pylibcudf/api_docs/io/parquet_io_utils.rst new file mode 100644 index 000000000000..3f01b0493fd4 --- /dev/null +++ b/docs/cudf/source/pylibcudf/api_docs/io/parquet_io_utils.rst @@ -0,0 +1,6 @@ +================ +Parquet IO Utils +================ + +.. automodule:: pylibcudf.io.parquet_io_utils + :members: diff --git a/python/pylibcudf/pylibcudf/io/CMakeLists.txt b/python/pylibcudf/pylibcudf/io/CMakeLists.txt index 089ea8d0e8d9..65bb31d908ff 100644 --- a/python/pylibcudf/pylibcudf/io/CMakeLists.txt +++ b/python/pylibcudf/pylibcudf/io/CMakeLists.txt @@ -1,12 +1,12 @@ # ============================================================================= # cmake-format: off -# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # cmake-format: on # ============================================================================= set(cython_sources avro.pyx csv.pyx datasource.pyx json.pyx orc.pyx parquet.pyx - parquet_metadata.pyx text.pyx timezone.pyx types.pyx + parquet_io_utils.pyx parquet_metadata.pyx text.pyx timezone.pyx types.pyx ) set(linked_libraries cudf::cudf) diff --git a/python/pylibcudf/pylibcudf/io/__init__.py b/python/pylibcudf/pylibcudf/io/__init__.py index a6a0ebad3a1e..1f0a0a218199 100644 --- a/python/pylibcudf/pylibcudf/io/__init__.py +++ b/python/pylibcudf/pylibcudf/io/__init__.py @@ -9,6 +9,7 @@ json, orc, parquet, + parquet_io_utils, parquet_metadata, text, timezone, @@ -30,6 +31,7 @@ "json", "orc", "parquet", + "parquet_io_utils", "parquet_metadata", "text", "timezone", diff --git a/python/pylibcudf/pylibcudf/io/parquet_io_utils.pxd b/python/pylibcudf/pylibcudf/io/parquet_io_utils.pxd new file mode 100644 index 000000000000..e8d872e08f61 --- /dev/null +++ b/python/pylibcudf/pylibcudf/io/parquet_io_utils.pxd @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from pylibcudf.io.text cimport ByteRangeInfo +from pylibcudf.io.types cimport SourceInfo +from rmm.pylibrmm.memory_resource cimport DeviceMemoryResource + +cpdef list fetch_byte_ranges_to_device( + SourceInfo source_info, + list byte_ranges, + object stream=*, + DeviceMemoryResource mr=*, +) + +cpdef bytes fetch_page_index_to_host( + SourceInfo source_info, + ByteRangeInfo page_index_range, +) diff --git a/python/pylibcudf/pylibcudf/io/parquet_io_utils.pyi b/python/pylibcudf/pylibcudf/io/parquet_io_utils.pyi new file mode 100644 index 000000000000..1a18ab72f5d5 --- /dev/null +++ b/python/pylibcudf/pylibcudf/io/parquet_io_utils.pyi @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from rmm.pylibrmm.memory_resource import DeviceMemoryResource + +from pylibcudf.gpumemoryview import gpumemoryview +from pylibcudf.io.text import ByteRangeInfo +from pylibcudf.io.types import SourceInfo +from pylibcudf.utils import CudaStreamLike + +__all__ = ["fetch_byte_ranges_to_device", "fetch_page_index_to_host"] + +def fetch_byte_ranges_to_device( + source_info: SourceInfo, + byte_ranges: list[ByteRangeInfo], + stream: CudaStreamLike | None = None, + mr: DeviceMemoryResource | None = None, +) -> list[gpumemoryview]: ... +def fetch_page_index_to_host( + source_info: SourceInfo, + page_index_range: ByteRangeInfo, +) -> bytes: ... diff --git a/python/pylibcudf/pylibcudf/io/parquet_io_utils.pyx b/python/pylibcudf/pylibcudf/io/parquet_io_utils.pyx new file mode 100644 index 000000000000..1a9366dcf75e --- /dev/null +++ b/python/pylibcudf/pylibcudf/io/parquet_io_utils.pyx @@ -0,0 +1,151 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""IO utilities for the Parquet.""" + +from libc.stddef cimport size_t +from libc.stdint cimport uint8_t, uintptr_t +from libcpp.memory cimport make_unique, unique_ptr +from libcpp.pair cimport pair +from libcpp.utility cimport move +from libcpp.vector cimport vector +from cython.operator cimport dereference + +from rmm.librmm.device_buffer cimport device_buffer +from rmm.pylibrmm.device_buffer cimport DeviceBuffer +from rmm.pylibrmm.memory_resource cimport DeviceMemoryResource +from rmm.pylibrmm.stream cimport Stream + +from pylibcudf.gpumemoryview cimport gpumemoryview +from pylibcudf.io.text cimport ByteRangeInfo +from pylibcudf.io.types cimport SourceInfo +from pylibcudf.libcudf.io.datasource cimport datasource, make_datasources +from pylibcudf.libcudf.io.parquet_io_utils cimport ( + const_byte_range_info, + const_uint8_t, + cpp_fetch_byte_ranges_to_device, + fetch_page_index_to_host as cpp_fetch_page_index_to_host, +) + +from pylibcudf.libcudf.io.text cimport byte_range_info +from pylibcudf.libcudf.utilities.span cimport device_span, host_span +from pylibcudf.utils cimport _get_memory_resource, _get_stream + +__all__ = ["fetch_byte_ranges_to_device", "fetch_page_index_to_host"] + + +cpdef list fetch_byte_ranges_to_device( + SourceInfo source_info, + list byte_ranges, + object stream=None, + DeviceMemoryResource mr=None, +): + """Fetch byte ranges from a Parquet source into device memory. + + Parameters + ---------- + source_info : SourceInfo + Source describing a single Parquet file. + byte_ranges : list[ByteRangeInfo] + Byte ranges to fetch, as returned by + :meth:`~pylibcudf.io.experimental.HybridScanReader.filter_column_chunks_byte_ranges`, + :meth:`~pylibcudf.io.experimental.HybridScanReader.payload_column_chunks_byte_ranges`, + or + :meth:`~pylibcudf.io.experimental.HybridScanReader.all_column_chunks_byte_ranges`. + stream : Stream, optional + CUDA stream. + mr : DeviceMemoryResource, optional + Device memory resource. + + Returns + ------- + list[gpumemoryview] + One view per byte range. Each view holds a reference to the + :class:`~rmm.DeviceBuffer` that owns its memory, keeping the + allocation alive for as long as the view is referenced. + + Raises + ------ + ValueError + If ``source_info`` does not describe exactly one source. + """ + cdef Stream _stream = _get_stream(stream) + cdef DeviceMemoryResource _mr = _get_memory_resource(mr) + cdef vector[unique_ptr[datasource]] sources = make_datasources(source_info.c_obj) + if sources.size() != 1: + raise ValueError( + f"fetch_byte_ranges_to_device requires exactly one source, " + f"got {sources.size()}" + ) + + cdef vector[byte_range_info] ranges_vec + cdef ByteRangeInfo bri + for bri in byte_ranges: + ranges_vec.push_back(bri.c_obj) + + cdef pair[vector[device_buffer], vector[device_span[const_uint8_t]]] fetched + with nogil: + fetched = cpp_fetch_byte_ranges_to_device( + dereference(sources[0]), + host_span[const_byte_range_info](ranges_vec.data(), ranges_vec.size()), + _stream.view(), + _mr.get_mr(), + ) + + cdef DeviceBuffer owner = DeviceBuffer.c_from_unique_ptr( + make_unique[device_buffer](move(fetched.first[0])), + _stream, + _mr, + ) + cdef gpumemoryview owner_gv = gpumemoryview(owner) + cdef uintptr_t base = owner_gv.ptr + cdef uintptr_t ptr + cdef size_t n + result = [] + for i in range(fetched.second.size()): + ptr = fetched.second[i].data() + n = fetched.second[i].size() + result.append(owner_gv[ptr - base : ptr - base + n]) + return result + + +cpdef bytes fetch_page_index_to_host( + SourceInfo source_info, + ByteRangeInfo page_index_range, +): + """Fetch parquet page index bytes to host memory. + + Parameters + ---------- + source_info : SourceInfo + Source describing a single Parquet file. + page_index_range : ByteRangeInfo + Byte range of the page index, as returned by + :meth:`~pylibcudf.io.experimental.HybridScanReader.page_index_byte_range`. + + Returns + ------- + bytes + Raw page index bytes copied to Python host memory. + + Raises + ------ + ValueError + If ``source_info`` does not describe exactly one source. + """ + cdef vector[unique_ptr[datasource]] sources = make_datasources(source_info.c_obj) + if sources.size() != 1: + raise ValueError( + f"fetch_page_index_to_host requires exactly one source, " + f"got {sources.size()}" + ) + + cdef unique_ptr[datasource.buffer] buf + with nogil: + buf = move(cpp_fetch_page_index_to_host( + dereference(sources[0]), + (page_index_range).c_obj, + )) + + cdef const uint8_t* ptr = buf.get().data() + cdef size_t n = buf.get().size() + return bytes(ptr[:n]) diff --git a/python/pylibcudf/pylibcudf/io/parquet_metadata.pyx b/python/pylibcudf/pylibcudf/io/parquet_metadata.pyx index f8aca270642e..c0f7f36e8cd2 100644 --- a/python/pylibcudf/pylibcudf/io/parquet_metadata.pyx +++ b/python/pylibcudf/pylibcudf/io/parquet_metadata.pyx @@ -458,7 +458,7 @@ cdef class FileMetaData: See Also -------- - read_parquet_footers + pylibcudf.io.parquet_metadata.read_parquet_footers Read one ``FileMetaData`` per source directly from :class:`pylibcudf.io.types.SourceInfo`. """ diff --git a/python/pylibcudf/pylibcudf/libcudf/io/parquet_io_utils.pxd b/python/pylibcudf/pylibcudf/libcudf/io/parquet_io_utils.pxd new file mode 100644 index 000000000000..ac8246e70cc8 --- /dev/null +++ b/python/pylibcudf/pylibcudf/libcudf/io/parquet_io_utils.pxd @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from libc.stdint cimport uint8_t +from libcpp.memory cimport unique_ptr +from libcpp.pair cimport pair +from libcpp.vector cimport vector + +from rmm.librmm.cuda_stream_view cimport cuda_stream_view +from rmm.librmm.device_buffer cimport device_buffer +from rmm.librmm.memory_resource cimport device_async_resource_ref + +from pylibcudf.exception_handler cimport libcudf_exception_handler +from pylibcudf.libcudf.io.datasource cimport datasource +from pylibcudf.libcudf.io.text cimport byte_range_info +from pylibcudf.libcudf.utilities.span cimport device_span, host_span + +ctypedef const uint8_t const_uint8_t +ctypedef const byte_range_info const_byte_range_info + +cdef extern from * nogil: + """ + #include + #include + + static std::pair, + std::vector>> + cpp_fetch_byte_ranges_to_device( + cudf::io::datasource& datasource, + cudf::host_span byte_ranges, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) + { + auto [buffers, spans, fut] = + cudf::io::parquet::fetch_byte_ranges_to_device_async( + datasource, byte_ranges, stream, mr); + fut.get(); + return {std::move(buffers), std::move(spans)}; + } + """ + pair[vector[device_buffer], vector[device_span[const_uint8_t]]] \ + cpp_fetch_byte_ranges_to_device( + datasource& source, + host_span[const_byte_range_info] byte_ranges, + cuda_stream_view stream, + device_async_resource_ref mr, + ) except + + +cdef extern from "cudf/io/parquet_io_utils.hpp" \ + namespace "cudf::io::parquet" nogil: + + unique_ptr[datasource.buffer] fetch_page_index_to_host( + datasource& ds, + byte_range_info page_index_bytes, + ) except +libcudf_exception_handler From 1f3530f4e61c446e71edca9357ac2be25fabf354 Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Tue, 4 Aug 2026 22:21:53 +0000 Subject: [PATCH 02/18] use byte_slice --- python/pylibcudf/pylibcudf/io/parquet_io_utils.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pylibcudf/pylibcudf/io/parquet_io_utils.pyx b/python/pylibcudf/pylibcudf/io/parquet_io_utils.pyx index 1a9366dcf75e..8ea42a9c96ee 100644 --- a/python/pylibcudf/pylibcudf/io/parquet_io_utils.pyx +++ b/python/pylibcudf/pylibcudf/io/parquet_io_utils.pyx @@ -104,7 +104,7 @@ cpdef list fetch_byte_ranges_to_device( for i in range(fetched.second.size()): ptr = fetched.second[i].data() n = fetched.second[i].size() - result.append(owner_gv[ptr - base : ptr - base + n]) + result.append(owner_gv.byte_slice(slice(ptr - base, ptr - base + n))) return result From eab58d09b43b1377e4119f0d0d1069bb5d6cc7f5 Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Thu, 6 Aug 2026 02:45:05 +0000 Subject: [PATCH 03/18] add missing bindings --- python/pylibcudf/pylibcudf/libcudf/io/datasource.pxd | 7 +++++-- python/pylibcudf/pylibcudf/libcudf/utilities/span.pxd | 3 ++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/python/pylibcudf/pylibcudf/libcudf/io/datasource.pxd b/python/pylibcudf/pylibcudf/libcudf/io/datasource.pxd index 36c5bf928850..738dd1dda448 100644 --- a/python/pylibcudf/pylibcudf/libcudf/io/datasource.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/io/datasource.pxd @@ -1,6 +1,7 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libc.stddef cimport size_t +from libc.stdint cimport uint8_t from libcpp.memory cimport unique_ptr from libcpp.vector cimport vector from pylibcudf.libcudf.io.types cimport source_info @@ -11,7 +12,9 @@ cdef extern from "cudf/io/datasource.hpp" \ namespace "cudf::io" nogil: cdef cppclass datasource: - pass + cdef cppclass buffer: + const uint8_t* data() noexcept + size_t size() noexcept cdef vector[unique_ptr[datasource]] make_datasources( source_info info diff --git a/python/pylibcudf/pylibcudf/libcudf/utilities/span.pxd b/python/pylibcudf/pylibcudf/libcudf/utilities/span.pxd index f2bf388e4d4c..379701e6d4c8 100644 --- a/python/pylibcudf/pylibcudf/libcudf/utilities/span.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/utilities/span.pxd @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2021-2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libcpp.vector cimport vector from pylibcudf.exception_handler cimport libcudf_exception_handler @@ -15,3 +15,4 @@ cdef extern from "cudf/utilities/span.hpp" namespace "cudf" nogil: device_span() noexcept device_span(T *data, size_type size) noexcept T *data() noexcept + size_type size() noexcept From 09c4ad3afe83c99d3146105bb598a8611604f5a1 Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Wed, 12 Aug 2026 20:57:11 +0000 Subject: [PATCH 04/18] remove cdef --- python/pylibcudf/pylibcudf/libcudf/io/datasource.pxd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pylibcudf/pylibcudf/libcudf/io/datasource.pxd b/python/pylibcudf/pylibcudf/libcudf/io/datasource.pxd index 738dd1dda448..e342274b9cce 100644 --- a/python/pylibcudf/pylibcudf/libcudf/io/datasource.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/io/datasource.pxd @@ -12,7 +12,7 @@ cdef extern from "cudf/io/datasource.hpp" \ namespace "cudf::io" nogil: cdef cppclass datasource: - cdef cppclass buffer: + cppclass buffer: const uint8_t* data() noexcept size_t size() noexcept From 007007b3b74e1a853d022de84a37e8247bf40537 Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Wed, 12 Aug 2026 21:22:51 +0000 Subject: [PATCH 05/18] add negative test cases --- .../io/experimental/hybrid_scan_filters_test.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/cpp/tests/io/experimental/hybrid_scan_filters_test.cpp b/cpp/tests/io/experimental/hybrid_scan_filters_test.cpp index 0cec880a6adb..505c5da05a57 100644 --- a/cpp/tests/io/experimental/hybrid_scan_filters_test.cpp +++ b/cpp/tests/io/experimental/hybrid_scan_filters_test.cpp @@ -1797,6 +1797,20 @@ TEST_F(HybridScanFiltersTest, FetchByteRangesInvalidRanges) stream, mr), cudf::logic_error); + + EXPECT_THROW( + cudf::io::parquet::fetch_byte_ranges_to_device_async( + *datasource, + std::vector{cudf::io::text::byte_range_info{0, -1}}, + stream, + mr), + cudf::logic_error); + + EXPECT_NO_THROW(cudf::io::parquet::fetch_byte_ranges_to_device_async( + *datasource, + std::vector{cudf::io::text::byte_range_info{1023, 1}}, + stream, + mr)); } class DictionaryFilterGapTest : public HybridScanFiltersTest, From b95b6b1c38b788736632315ecc604895530628c2 Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Wed, 12 Aug 2026 21:25:05 +0000 Subject: [PATCH 06/18] doc location --- docs/cudf/source/pylibcudf/api_docs/io/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cudf/source/pylibcudf/api_docs/io/index.rst b/docs/cudf/source/pylibcudf/api_docs/io/index.rst index 45b4def70051..2ec579c4463a 100644 --- a/docs/cudf/source/pylibcudf/api_docs/io/index.rst +++ b/docs/cudf/source/pylibcudf/api_docs/io/index.rst @@ -17,7 +17,7 @@ I/O Functions avro csv - experimental + ../experimental/index json orc parquet From 0568a90eae13f40c7473e2fe567e538cba07f88a Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Wed, 12 Aug 2026 21:25:39 +0000 Subject: [PATCH 07/18] fix docstring --- python/pylibcudf/pylibcudf/io/parquet_io_utils.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pylibcudf/pylibcudf/io/parquet_io_utils.pyx b/python/pylibcudf/pylibcudf/io/parquet_io_utils.pyx index 8ea42a9c96ee..31f249002f6a 100644 --- a/python/pylibcudf/pylibcudf/io/parquet_io_utils.pyx +++ b/python/pylibcudf/pylibcudf/io/parquet_io_utils.pyx @@ -1,6 +1,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""IO utilities for the Parquet.""" +"""IO utilities for Parquet.""" from libc.stddef cimport size_t from libc.stdint cimport uint8_t, uintptr_t From fbdfdf288192598d9d72242ad7d8bb2f55affac9 Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Wed, 12 Aug 2026 21:26:15 +0000 Subject: [PATCH 08/18] use libcudf_exception_handler --- python/pylibcudf/pylibcudf/libcudf/io/parquet_io_utils.pxd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pylibcudf/pylibcudf/libcudf/io/parquet_io_utils.pxd b/python/pylibcudf/pylibcudf/libcudf/io/parquet_io_utils.pxd index ac8246e70cc8..3b40ac368159 100644 --- a/python/pylibcudf/pylibcudf/libcudf/io/parquet_io_utils.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/io/parquet_io_utils.pxd @@ -44,7 +44,7 @@ cdef extern from * nogil: host_span[const_byte_range_info] byte_ranges, cuda_stream_view stream, device_async_resource_ref mr, - ) except + + ) except +libcudf_exception_handler cdef extern from "cudf/io/parquet_io_utils.hpp" \ namespace "cudf::io::parquet" nogil: From 337fbad1e94133df4182fb3f069dc250ebedf426 Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Wed, 12 Aug 2026 21:28:04 +0000 Subject: [PATCH 09/18] raise for more than one device buffer --- python/pylibcudf/pylibcudf/io/parquet_io_utils.pyx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/pylibcudf/pylibcudf/io/parquet_io_utils.pyx b/python/pylibcudf/pylibcudf/io/parquet_io_utils.pyx index 31f249002f6a..2186f15562f4 100644 --- a/python/pylibcudf/pylibcudf/io/parquet_io_utils.pyx +++ b/python/pylibcudf/pylibcudf/io/parquet_io_utils.pyx @@ -91,6 +91,10 @@ cpdef list fetch_byte_ranges_to_device( _mr.get_mr(), ) + if fetched.first.size() != 1: + raise RuntimeError( + f"Expected exactly one device buffer, got {fetched.first.size()}" + ) cdef DeviceBuffer owner = DeviceBuffer.c_from_unique_ptr( make_unique[device_buffer](move(fetched.first[0])), _stream, From 9855657ca68c41f066c6e0da589385014f75a173 Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Wed, 12 Aug 2026 21:30:04 +0000 Subject: [PATCH 10/18] raise if pointer is NULL --- python/pylibcudf/pylibcudf/io/parquet_io_utils.pyx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/pylibcudf/pylibcudf/io/parquet_io_utils.pyx b/python/pylibcudf/pylibcudf/io/parquet_io_utils.pyx index 2186f15562f4..e54d879fa5c2 100644 --- a/python/pylibcudf/pylibcudf/io/parquet_io_utils.pyx +++ b/python/pylibcudf/pylibcudf/io/parquet_io_utils.pyx @@ -150,6 +150,8 @@ cpdef bytes fetch_page_index_to_host( (page_index_range).c_obj, )) + if buf.get() is NULL: + raise RuntimeError("fetch_page_index_to_host returned no buffer") cdef const uint8_t* ptr = buf.get().data() cdef size_t n = buf.get().size() return bytes(ptr[:n]) From ed680f222bfba502745eeaf46e5826445b3188a5 Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Wed, 12 Aug 2026 21:30:45 +0000 Subject: [PATCH 11/18] add consts to decl --- python/pylibcudf/pylibcudf/libcudf/io/datasource.pxd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/pylibcudf/pylibcudf/libcudf/io/datasource.pxd b/python/pylibcudf/pylibcudf/libcudf/io/datasource.pxd index e342274b9cce..3e220197f950 100644 --- a/python/pylibcudf/pylibcudf/libcudf/io/datasource.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/io/datasource.pxd @@ -13,8 +13,8 @@ cdef extern from "cudf/io/datasource.hpp" \ cdef cppclass datasource: cppclass buffer: - const uint8_t* data() noexcept - size_t size() noexcept + const uint8_t* data() const noexcept + size_t size() const noexcept cdef vector[unique_ptr[datasource]] make_datasources( source_info info From 3dd055e1dfbf41cf6720838b7d5a0b77526a88d5 Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Wed, 12 Aug 2026 21:33:50 +0000 Subject: [PATCH 12/18] use dize_t --- python/pylibcudf/pylibcudf/libcudf/utilities/span.pxd | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/python/pylibcudf/pylibcudf/libcudf/utilities/span.pxd b/python/pylibcudf/pylibcudf/libcudf/utilities/span.pxd index 379701e6d4c8..dc768ed677e6 100644 --- a/python/pylibcudf/pylibcudf/libcudf/utilities/span.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/utilities/span.pxd @@ -1,5 +1,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +from libc.stddef cimport size_t from libcpp.vector cimport vector from pylibcudf.exception_handler cimport libcudf_exception_handler from pylibcudf.libcudf.types cimport size_type @@ -13,6 +14,6 @@ cdef extern from "cudf/utilities/span.hpp" namespace "cudf" nogil: cdef cppclass device_span[T]: device_span() noexcept - device_span(T *data, size_type size) noexcept - T *data() noexcept - size_type size() noexcept + device_span(T *data, size_t size) noexcept + T *data() const noexcept + size_t size() const noexcept From 53df6491ebb203b6da8696f13d5d527e424063ec Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Thu, 13 Aug 2026 12:36:23 +0000 Subject: [PATCH 13/18] remove const --- python/pylibcudf/pylibcudf/libcudf/io/datasource.pxd | 4 ++-- python/pylibcudf/pylibcudf/libcudf/utilities/span.pxd | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/python/pylibcudf/pylibcudf/libcudf/io/datasource.pxd b/python/pylibcudf/pylibcudf/libcudf/io/datasource.pxd index 3e220197f950..e342274b9cce 100644 --- a/python/pylibcudf/pylibcudf/libcudf/io/datasource.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/io/datasource.pxd @@ -13,8 +13,8 @@ cdef extern from "cudf/io/datasource.hpp" \ cdef cppclass datasource: cppclass buffer: - const uint8_t* data() const noexcept - size_t size() const noexcept + const uint8_t* data() noexcept + size_t size() noexcept cdef vector[unique_ptr[datasource]] make_datasources( source_info info diff --git a/python/pylibcudf/pylibcudf/libcudf/utilities/span.pxd b/python/pylibcudf/pylibcudf/libcudf/utilities/span.pxd index dc768ed677e6..ecdbb2e16318 100644 --- a/python/pylibcudf/pylibcudf/libcudf/utilities/span.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/utilities/span.pxd @@ -15,5 +15,5 @@ cdef extern from "cudf/utilities/span.hpp" namespace "cudf" nogil: cdef cppclass device_span[T]: device_span() noexcept device_span(T *data, size_t size) noexcept - T *data() const noexcept - size_t size() const noexcept + T *data() noexcept + size_t size() noexcept From ef2129d8b6e8dfa709ba23d235d06f526e884209 Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Mon, 17 Aug 2026 13:37:00 +0000 Subject: [PATCH 14/18] docs --- docs/cudf/source/pylibcudf/api_docs/io/experimental.rst | 8 ++++++++ docs/cudf/source/pylibcudf/api_docs/io/index.rst | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 docs/cudf/source/pylibcudf/api_docs/io/experimental.rst diff --git a/docs/cudf/source/pylibcudf/api_docs/io/experimental.rst b/docs/cudf/source/pylibcudf/api_docs/io/experimental.rst new file mode 100644 index 000000000000..92e9360fdb42 --- /dev/null +++ b/docs/cudf/source/pylibcudf/api_docs/io/experimental.rst @@ -0,0 +1,8 @@ +============ +Experimental +============ + +APIs in this namespace are experimental and may change without warning in the future. + +.. automodule:: pylibcudf.io.experimental + :members: diff --git a/docs/cudf/source/pylibcudf/api_docs/io/index.rst b/docs/cudf/source/pylibcudf/api_docs/io/index.rst index 2ec579c4463a..45b4def70051 100644 --- a/docs/cudf/source/pylibcudf/api_docs/io/index.rst +++ b/docs/cudf/source/pylibcudf/api_docs/io/index.rst @@ -17,7 +17,7 @@ I/O Functions avro csv - ../experimental/index + experimental json orc parquet From 4544758c74bbb3285214b5bb6afd66f613ff4424 Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Tue, 18 Aug 2026 15:25:31 +0000 Subject: [PATCH 15/18] add note about resolving future --- python/pylibcudf/pylibcudf/libcudf/io/parquet_io_utils.pxd | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/pylibcudf/pylibcudf/libcudf/io/parquet_io_utils.pxd b/python/pylibcudf/pylibcudf/libcudf/io/parquet_io_utils.pxd index 3b40ac368159..47f871439b99 100644 --- a/python/pylibcudf/pylibcudf/libcudf/io/parquet_io_utils.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/io/parquet_io_utils.pxd @@ -34,6 +34,8 @@ cdef extern from * nogil: auto [buffers, spans, fut] = cudf::io::parquet::fetch_byte_ranges_to_device_async( datasource, byte_ranges, stream, mr); + // Block until the async fetch completes so the returned buffers/spans + // are fully populated. This also avoids exposing std::future to Cython. fut.get(); return {std::move(buffers), std::move(spans)}; } From 760509e8dcd81003ddc78aaffe92951e346a01cd Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Tue, 18 Aug 2026 15:28:43 +0000 Subject: [PATCH 16/18] remove redundant check --- cpp/src/io/parquet/io_utils/parquet_io_utils.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/cpp/src/io/parquet/io_utils/parquet_io_utils.cpp b/cpp/src/io/parquet/io_utils/parquet_io_utils.cpp index abcec26d10d5..bd685d238a7e 100644 --- a/cpp/src/io/parquet/io_utils/parquet_io_utils.cpp +++ b/cpp/src/io/parquet/io_utils/parquet_io_utils.cpp @@ -326,8 +326,6 @@ fetch_byte_ranges_to_device_async_impl( auto const source_size = datasources[source_idx].get().size(); auto const buffer_size = std::accumulate( byte_ranges.begin(), byte_ranges.end(), std::size_t{0}, [&](auto acc, auto const& range) { - CUDF_EXPECTS(range.offset() >= 0 and range.size() >= 0, - "Byte range offset and size must be non-negative"); CUDF_EXPECTS( static_cast(range.offset()) + static_cast(range.size()) <= source_size, "Byte range exceeds datasource size"); From 1996cf0b485442dfa60c9fff2e8657e20219055f Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Tue, 18 Aug 2026 15:41:08 +0000 Subject: [PATCH 17/18] fix cython signature --- python/pylibcudf/pylibcudf/libcudf/io/datasource.pxd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/pylibcudf/pylibcudf/libcudf/io/datasource.pxd b/python/pylibcudf/pylibcudf/libcudf/io/datasource.pxd index e342274b9cce..0e047b2fd836 100644 --- a/python/pylibcudf/pylibcudf/libcudf/io/datasource.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/io/datasource.pxd @@ -13,8 +13,8 @@ cdef extern from "cudf/io/datasource.hpp" \ cdef cppclass datasource: cppclass buffer: - const uint8_t* data() noexcept - size_t size() noexcept + const uint8_t* data() except +libcudf_exception_handler const + size_t size() except +libcudf_exception_handler const cdef vector[unique_ptr[datasource]] make_datasources( source_info info From 20df1377c9ab05833d01645b0b54d78ffa3c86c0 Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Tue, 18 Aug 2026 15:44:25 +0000 Subject: [PATCH 18/18] fix cython signature for device_span methods --- python/pylibcudf/pylibcudf/libcudf/utilities/span.pxd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/pylibcudf/pylibcudf/libcudf/utilities/span.pxd b/python/pylibcudf/pylibcudf/libcudf/utilities/span.pxd index ecdbb2e16318..4f4669d7f6f8 100644 --- a/python/pylibcudf/pylibcudf/libcudf/utilities/span.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/utilities/span.pxd @@ -15,5 +15,5 @@ cdef extern from "cudf/utilities/span.hpp" namespace "cudf" nogil: cdef cppclass device_span[T]: device_span() noexcept device_span(T *data, size_t size) noexcept - T *data() noexcept - size_t size() noexcept + T *data() noexcept const + size_t size() noexcept const