Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 25 additions & 2 deletions parquet-variant-compute/src/variant_array.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@ use crate::type_conversion::{
primitive_conversion_single_value,
};
use arrow::array::{
Array, ArrayRef, AsArray, StructArray, StructArrayAssembler, downcast_dictionary_array,
downcast_run_array, new_null_array,
Array, ArrayRef, AsArray, BinaryViewArray, StructArray, StructArrayAssembler,
downcast_dictionary_array, downcast_run_array, new_null_array,
};
use arrow::buffer::NullBuffer;
use arrow::compute::cast;
Expand Down Expand Up @@ -60,6 +60,29 @@ pub(crate) fn binary_array_value(array: &dyn Array, index: usize) -> Option<&[u8
}
}

/// A binary-like column whose array type is resolved once, not on every row.
pub(crate) enum BinaryColumn<'a> {
View(&'a BinaryViewArray),
Other(&'a dyn Array),
}

impl<'a> BinaryColumn<'a> {
pub(crate) fn new(array: &'a dyn Array) -> Self {
match array.as_binary_view_opt() {
Some(array) => Self::View(array),
None => Self::Other(array),
}
}

/// The bytes of row `index`, or `None` if the row is null or the column is not binary-like.
pub(crate) fn value(&self, index: usize) -> Option<&'a [u8]> {
match self {
Self::View(array) => array.is_valid(index).then(|| array.value(index)),
Self::Other(array) => binary_array_value(*array, index),
}
}
}

/// Returns a [`Variant`] from a `metadata` and `value` byte arrays, returns `None`
/// if one of them is of invalid type.
pub(crate) fn variant_from_arrays_at<'m, 'v>(
Expand Down
60 changes: 58 additions & 2 deletions parquet-variant-compute/src/variant_get.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,10 @@ use arrow::{
error::Result,
};
use arrow_schema::{ArrowError, DataType, FieldRef};
use parquet_variant::{VariantPath, VariantPathElement};
use parquet_variant::{Variant, VariantMetadataView, VariantPath, VariantPathElement, VariantView};

use crate::ShreddingState;
use crate::variant_array::all_null_value_column;
use crate::variant_array::{BinaryColumn, all_null_value_column};
use crate::variant_to_arrow::make_variant_to_arrow_row_builder;
use crate::{VariantArray, VariantType, unshred_variant};

Expand Down Expand Up @@ -192,6 +192,59 @@ pub(crate) fn follow_shredded_path_element(
}
}

/// Extracts `path` from an unshredded variant column, reading each row's encoded bytes directly.
///
/// A row reads only the headers and field names along `path`, and decodes only the value it finds.
fn get_path_from_bytes(
target: &VariantArray,
path: &[VariantPathElement<'_>],
as_type: Option<&DataType>,
cast_options: &CastOptions,
) -> Result<ArrayRef> {
let mut builder = make_variant_to_arrow_row_builder(
target.metadata_column(),
VariantPath::default(),
as_type,
cast_options,
target.len(),
)?;
let metadata = BinaryColumn::new(target.metadata_column());
let value = BinaryColumn::new(target.value_column());

for i in 0..target.len() {
if target.is_null(i) {
builder.append_null()?;
continue;
}
// A row with neither `value` nor `typed_value` is `Variant::Null`, which has no fields.
let Some(value) = value.value(i) else {
match path.is_empty() {
true => builder.append_value(Variant::Null).map(drop)?,
false => builder.append_null()?,
}
continue;
};
let found = row_value(metadata.value(i), value)
.and_then(|row| row.get_path(path))
.and_then(|found| found.map(|found| found.to_variant()).transpose());
match found {
Ok(Some(found)) => builder.append_value(found).map(drop)?,
Ok(None) => builder.append_null()?,
Err(_) if cast_options.safe => builder.append_null()?,
Err(e) => return Err(e),
}
}
builder.finish()
}

/// Views a row of an unshredded variant column.
fn row_value<'a>(metadata: Option<&'a [u8]>, value: &'a [u8]) -> Result<VariantView<'a, 'a>> {
let metadata = metadata.ok_or_else(|| {
ArrowError::InvalidArgumentError("Variant metadata is null in a non-null row".to_string())
})?;
VariantView::try_new(VariantMetadataView::try_new(metadata)?, value)
}

/// Follows the given path as far as possible through shredded variant fields. If the path ends on a
/// shredded field, return it directly. Otherwise, use a row shredder to follow the rest of the path
/// and extract the requested value on a per-row basis.
Expand Down Expand Up @@ -249,6 +302,9 @@ fn shredded_get_path(
} else {
as_field.map(|f| f.data_type())
};
if target.typed_value_column().is_none() {
return get_path_from_bytes(&target, &path, as_type, cast_options);
}
let mut builder = make_variant_to_arrow_row_builder(
target.metadata_column(),
path,
Expand Down
17 changes: 17 additions & 0 deletions parquet-variant/src/builder/metadata.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements

use std::cmp::Ordering;
use std::collections::HashMap;

use arrow_schema::ArrowError;
Expand Down Expand Up @@ -30,6 +31,13 @@ pub trait MetadataBuilder: std::fmt::Debug {
/// [`Self::num_field_names`]. Panics if the field id is out of bounds.
fn field_name(&self, field_id: usize) -> &str;

/// Compares the field names of two field ids, which must be less than
/// [`Self::num_field_names`]. Panics if a field id is out of bounds.
fn cmp_field_names(&self, field_a_id: u32, field_b_id: u32) -> Ordering {
self.field_name(field_a_id as _)
.cmp(self.field_name(field_b_id as _))
}

/// Returns the number of field names stored in this metadata builder. Any number less than this
/// is a valid field id. The builder can be reverted back to this size later on (discarding any
/// newer/higher field ids) by calling [`Self::truncate_field_names`].
Expand Down Expand Up @@ -117,6 +125,15 @@ impl MetadataBuilder for ReadOnlyMetadataBuilder<'_> {
fn field_name(&self, field_id: usize) -> &str {
&self.metadata[field_id]
}
// Byte order is `str` order, so the comparison needs no UTF-8 validation.
fn cmp_field_names(&self, field_a_id: u32, field_b_id: u32) -> Ordering {
let view = self.metadata.view();
let name = |id: u32| {
view.entry(id as _)
.expect("Invalid metadata dictionary entry")
};
name(field_a_id).cmp(name(field_b_id))
}
fn num_field_names(&self) -> usize {
self.metadata.len()
}
Expand Down
4 changes: 1 addition & 3 deletions parquet-variant/src/builder/object.rs
Original file line number Diff line number Diff line change
Expand Up @@ -251,9 +251,7 @@ impl<'a, S: BuilderSpecificState> ObjectBuilder<'a, S> {
let metadata_builder = self.parent_state.metadata_builder();

self.fields.sort_by(|&field_a_id, _, &field_b_id, _| {
let field_a_name = metadata_builder.field_name(field_a_id as usize);
let field_b_name = metadata_builder.field_name(field_b_id as usize);
field_a_name.cmp(field_b_name)
metadata_builder.cmp_field_names(field_a_id, field_b_id)
});

let max_id = self.fields.iter().map(|(i, _)| *i).max().unwrap_or(0);
Expand Down
66 changes: 43 additions & 23 deletions parquet-variant/src/decoder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -137,16 +137,6 @@ impl OffsetSizeBytes {
Ok(result)
}

/// Return one unsigned little-endian value from `bytes`.
///
/// * `bytes` – the byte buffer to index
/// * `index` – 0-based index into the buffer
///
/// Each value is `self as u32` bytes wide (1, 2, 3 or 4), zero-extended to 32 bits as needed.
pub(crate) fn unpack_u32(&self, bytes: &[u8], index: usize) -> Result<u32, ArrowError> {
self.unpack_u32_at_offset(bytes, 0, index)
}

/// Return one unsigned little-endian value from `bytes`.
///
/// * `bytes` – the byte buffer to index
Expand All @@ -157,17 +147,17 @@ impl OffsetSizeBytes {
///
/// Each value is `self as u32` bytes wide (1, 2, 3 or 4), zero-extended to 32 bits as needed.
pub(crate) fn unpack_u32_at_offset(
&self,
self,
bytes: &[u8],
byte_offset: usize, // how many bytes to skip
offset_index: usize, // which offset in an array of offsets
) -> Result<u32, ArrowError> {
use OffsetSizeBytes::*;

// Index into the byte array:
// byte_offset + (*self as usize) * offset_index
// byte_offset + (self as usize) * offset_index
let offset = offset_index
.checked_mul(*self as usize)
.checked_mul(self as usize)
.and_then(|n| n.checked_add(byte_offset))
.ok_or_else(|| overflow_error("unpacking offset array value"))?;
let value = match self {
Expand Down Expand Up @@ -637,48 +627,78 @@ mod tests {
fn unpack_u32_all_widths() {
// One-byte offsets
let buf_one = [0x01u8, 0xAB, 0xCD];
assert_eq!(OffsetSizeBytes::One.unpack_u32(&buf_one, 0).unwrap(), 0x01);
assert_eq!(OffsetSizeBytes::One.unpack_u32(&buf_one, 2).unwrap(), 0xCD);
assert_eq!(
OffsetSizeBytes::One
.unpack_u32_at_offset(&buf_one, 0, 0)
.unwrap(),
0x01
);
assert_eq!(
OffsetSizeBytes::One
.unpack_u32_at_offset(&buf_one, 0, 2)
.unwrap(),
0xCD
);

// Two-byte offsets (little-endian 0x1234, 0x5678)
let buf_two = [0x34, 0x12, 0x78, 0x56];
assert_eq!(
OffsetSizeBytes::Two.unpack_u32(&buf_two, 0).unwrap(),
OffsetSizeBytes::Two
.unpack_u32_at_offset(&buf_two, 0, 0)
.unwrap(),
0x1234
);
assert_eq!(
OffsetSizeBytes::Two.unpack_u32(&buf_two, 1).unwrap(),
OffsetSizeBytes::Two
.unpack_u32_at_offset(&buf_two, 0, 1)
.unwrap(),
0x5678
);

// Three-byte offsets (0x030201 and 0x0000FF)
let buf_three = [0x01, 0x02, 0x03, 0xFF, 0x00, 0x00];
assert_eq!(
OffsetSizeBytes::Three.unpack_u32(&buf_three, 0).unwrap(),
OffsetSizeBytes::Three
.unpack_u32_at_offset(&buf_three, 0, 0)
.unwrap(),
0x030201
);
assert_eq!(
OffsetSizeBytes::Three.unpack_u32(&buf_three, 1).unwrap(),
OffsetSizeBytes::Three
.unpack_u32_at_offset(&buf_three, 0, 1)
.unwrap(),
0x0000FF
);

// Four-byte offsets (0x12345678, 0x90ABCDEF)
let buf_four = [0x78, 0x56, 0x34, 0x12, 0xEF, 0xCD, 0xAB, 0x90];
assert_eq!(
OffsetSizeBytes::Four.unpack_u32(&buf_four, 0).unwrap(),
OffsetSizeBytes::Four
.unpack_u32_at_offset(&buf_four, 0, 0)
.unwrap(),
0x1234_5678
);
assert_eq!(
OffsetSizeBytes::Four.unpack_u32(&buf_four, 1).unwrap(),
OffsetSizeBytes::Four
.unpack_u32_at_offset(&buf_four, 0, 1)
.unwrap(),
0x90AB_CDEF
);
}

#[test]
fn unpack_u32_out_of_bounds() {
let tiny = [0x00u8]; // deliberately too short
assert!(OffsetSizeBytes::Two.unpack_u32(&tiny, 0).is_err());
assert!(OffsetSizeBytes::Three.unpack_u32(&tiny, 0).is_err());
assert!(
OffsetSizeBytes::Two
.unpack_u32_at_offset(&tiny, 0, 0)
.is_err()
);
assert!(
OffsetSizeBytes::Three
.unpack_u32_at_offset(&tiny, 0, 0)
.is_err()
);
}

#[test]
Expand Down
32 changes: 28 additions & 4 deletions parquet-variant/src/utils.rs
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ pub(crate) fn array_from_slice<const N: usize>(
.map_err(|e: TryFromSliceError| ArrowError::InvalidArgumentError(e.to_string()))
}

#[inline]
pub(crate) fn first_byte_from_slice(slice: &[u8]) -> Result<u8, ArrowError> {
slice
.first()
Expand All @@ -71,24 +72,47 @@ pub(crate) fn string_from_slice(
offset: usize,
range: Range<usize>,
) -> Result<&str, ArrowError> {
let offset_buffer = slice_from_slice_at_offset(slice, offset, range)?;
str_from_bytes(slice_from_slice_at_offset(slice, offset, range)?)
}

/// Interprets `bytes` as a UTF-8 string.
#[inline]
pub(crate) fn str_from_bytes(bytes: &[u8]) -> Result<&str, ArrowError> {
//Use simdutf8 by default
#[cfg(feature = "simdutf8")]
{
simdutf8::basic::from_utf8(offset_buffer).map_err(|_| {
simdutf8::basic::from_utf8(bytes).map_err(|_| {
// Use simdutf8::compat to return details about the decoding error
let e = simdutf8::compat::from_utf8(offset_buffer).unwrap_err();
let e = simdutf8::compat::from_utf8(bytes).unwrap_err();
ArrowError::InvalidArgumentError(format!("encountered non UTF-8 data: {e}"))
})
}

//Use std::str if simdutf8 is not enabled
#[cfg(not(feature = "simdutf8"))]
str::from_utf8(offset_buffer)
str::from_utf8(bytes)
.map_err(|_| ArrowError::InvalidArgumentError("invalid UTF-8 string".to_string()))
}

/// Binary search over `0..len` with a fallible comparison, which returns the first error it hits.
///
/// Returns the index for which `cmp` returns [`Ordering::Equal`], if any.
pub(crate) fn try_binary_search(
len: usize,
cmp: impl Fn(usize) -> Result<Ordering, ArrowError>,
) -> Result<Option<usize>, ArrowError> {
let (mut low, mut high) = (0, len);
while low < high {
let mid = low + (high - low) / 2;
match cmp(mid)? {
Ordering::Less => low = mid + 1,
Ordering::Greater => high = mid,
Ordering::Equal => return Ok(Some(mid)),
}
}
Ok(None)
}

/// Performs a binary search over a range using a fallible key extraction function; a failed key
/// extraction immediately terminates the search.
///
Expand Down
9 changes: 5 additions & 4 deletions parquet-variant/src/variant.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,14 @@ pub use self::metadata::{
EMPTY_VARIANT_METADATA, EMPTY_VARIANT_METADATA_BYTES, VARIANT_SPEC_VERSION, VariantMetadata,
};
pub use self::object::VariantObject;
pub use self::view::{VariantListView, VariantMetadataView, VariantObjectView, VariantView};

// Publicly export types used in the API
pub use crate::decoder::VariantBasicType;
pub use half::f16;
pub use uuid::Uuid;

use crate::decoder::{
self, VariantBasicType, VariantPrimitiveType, get_basic_type, get_primitive_type,
};
use crate::decoder::{self, VariantPrimitiveType, get_basic_type, get_primitive_type};
use crate::path::{VariantPath, VariantPathElement};
use crate::utils::{first_byte_from_slice, slice_from_slice};
use std::ops::Deref;
Expand All @@ -27,6 +27,7 @@ mod decimal;
mod list;
mod metadata;
mod object;
mod view;

const MAX_SHORT_STRING_BYTES: usize = 0x3F;

Expand Down Expand Up @@ -381,7 +382,7 @@ impl<'m, 'v> Variant<'m, 'v> {
}

// The actual constructor, which only performs shallow (constant-time) validation.
fn try_new_with_metadata_and_shallow_validation(
pub(crate) fn try_new_with_metadata_and_shallow_validation(
metadata: VariantMetadata<'m>,
value: &'v [u8],
) -> Result<Self, ArrowError> {
Expand Down
Loading