Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 38 additions & 10 deletions parquet-variant-compute/src/shred_variant.rs
Original file line number Diff line number Diff line change
Expand Up @@ -339,9 +339,18 @@ impl<'a> VariantToShreddedArrayVariantRowBuilder<'a> {
}
}

fn duplicate_field_name(field_name: &[u8]) -> ArrowError {
ArrowError::InvalidArgumentError(format!(
"Duplicate field name: {}",
String::from_utf8_lossy(field_name)
))
}

pub(crate) struct VariantToShreddedObjectVariantRowBuilder<'a> {
value_builder: VariantValueArrayBuilder,
typed_value_builders: IndexMap<&'a str, VariantToShreddedVariantRowBuilder<'a>>,
/// The names of `typed_value_builders` as bytes, sorted, each with its builder's index.
sorted_field_names: Vec<(&'a [u8], usize)>,
typed_value_nulls: NullBufferBuilder,
nulls: NullBufferBuilder,
null_value: NullValue,
Expand Down Expand Up @@ -369,8 +378,15 @@ impl<'a> VariantToShreddedObjectVariantRowBuilder<'a> {
Ok((field.name().as_str(), builder))
});
let typed_value_builders: IndexMap<_, _> = typed_value_builders.collect::<Result<_>>()?;
let mut sorted_field_names: Vec<_> = typed_value_builders
.keys()
.enumerate()
.map(|(index, name)| (name.as_bytes(), index))
.collect();
sorted_field_names.sort_unstable();
Ok(Self {
value_builder: VariantValueArrayBuilder::new(capacity),
sorted_field_names,
seen: vec![false; typed_value_builders.len()],
typed_value_builders,
typed_value_nulls: NullBufferBuilder::new(capacity),
Expand Down Expand Up @@ -401,32 +417,44 @@ impl<'a> VariantToShreddedObjectVariantRowBuilder<'a> {
return Ok(false);
};

// Route the object's fields by name as either shredded or unshredded
// Route the object's fields by name as either shredded or unshredded. An unshredded field
// keeps its field id and its encoded bytes, because the leftover `value` column uses the
// same metadata dictionary.
let Self {
value_builder,
typed_value_builders,
sorted_field_names,
seen,
..
} = self;
seen.fill(false);
let object = obj.view();
let mut builder = value_builder.builder_ext(value.metadata());
// Error on a repeated unshredded field instead of overwriting it
let mut object_builder = builder.try_new_object()?.with_validate_unique_fields(true);
let mut partially_shredded = false;
for (field_name, value) in obj.iter() {
match typed_value_builders.get_full_mut(field_name) {
Some((index, _, typed_value_builder)) => {
// The object builder only catches a repeated field id. A sorted dictionary holds each name
// once, so that suffices; an unsorted one may repeat a name under distinct ids. The spec
// lists an object's fields in name order, so such a repeat follows its first occurrence.
let check_adjacent_names = !value.metadata().is_sorted();
for i in 0..object.len() {
let field_name = object.field_name(i)?;
let field = object.field(i)?;
match sorted_field_names.binary_search_by(|(name, _)| name.cmp(&field_name)) {
Ok(position) => {
let index = sorted_field_names[position].1;
// A repeated field would append a second row to this builder
if seen[index] {
return Err(ArrowError::InvalidArgumentError(format!(
"Duplicate field name: {field_name}"
)));
return Err(duplicate_field_name(field_name));
}
typed_value_builder.append_value(value)?;
typed_value_builders[index].append_value(field.to_variant()?)?;
seen[index] = true;
}
None => {
object_builder.try_insert_bytes(field_name, value)?;
Err(_) => {
if check_adjacent_names && i > 0 && object.field_name(i - 1)? == field_name {
return Err(duplicate_field_name(field_name));
}
object_builder.insert_encoded(object.field_id(i)?, field.bytes()?)?;
partially_shredded = true;
}
}
Expand Down
24 changes: 18 additions & 6 deletions parquet-variant-compute/src/unshred_variant.rs
Original file line number Diff line number Diff line change
Expand Up @@ -582,12 +582,14 @@ struct StructUnshredVariantBuilder<'a> {
value: Option<&'a ArrayRef>,
typed_value: &'a arrow::array::StructArray,
field_unshredders: IndexMap<&'a str, Option<UnshredVariantRowBuilder<'a>>>,
/// The names of `field_unshredders` as bytes, sorted.
sorted_field_names: Vec<&'a [u8]>,
}

impl<'a> StructUnshredVariantBuilder<'a> {
fn try_new(value: Option<&'a ArrayRef>, typed_value: &'a StructArray) -> Result<Self> {
// Create unshredders for each field in constructor
let mut field_unshredders = IndexMap::new();
let mut field_unshredders: IndexMap<&str, _> = IndexMap::new();
for (field, field_array) in typed_value.fields().iter().zip(typed_value.columns()) {
// Factory returns None for None/None case -- these are missing fields we should skip
let Some(field_array) = field_array.as_struct_opt() else {
Expand All @@ -600,10 +602,16 @@ impl<'a> StructUnshredVariantBuilder<'a> {
field_unshredders.insert(field.name().as_ref(), field_unshredder);
}

let mut sorted_field_names: Vec<_> = field_unshredders
.keys()
.map(|name| name.as_bytes())
.collect();
sorted_field_names.sort_unstable();
Ok(Self {
value,
typed_value,
field_unshredders,
sorted_field_names,
})
}

Expand Down Expand Up @@ -634,14 +642,18 @@ impl<'a> StructUnshredVariantBuilder<'a> {
));
};

for entry in object.iter_try() {
let (field_name, field_value) = entry?;
if self.field_unshredders.contains_key(field_name) {
// The leftover fields keep their field ids and encoded bytes, because the output uses
// the same metadata dictionary.
let object = object.view();
for i in 0..object.len() {
let field_name = object.field_name(i)?;
if self.sorted_field_names.binary_search(&field_name).is_ok() {
return Err(ArrowError::InvalidArgumentError(format!(
"Field '{field_name}' appears in both typed_value and value",
"Field '{}' appears in both typed_value and value",
String::from_utf8_lossy(field_name),
)));
}
object_builder.insert_bytes(field_name, field_value);
object_builder.insert_encoded(object.field_id(i)?, object.field(i)?.bytes()?)?;
}
}

Expand Down
27 changes: 25 additions & 2 deletions parquet-variant-compute/src/variant_array.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@ use crate::type_conversion::{
primitive_conversion_single_value,
};
use arrow::array::{
Array, ArrayRef, AsArray, StructArray, StructArrayAssembler, downcast_dictionary_array,
downcast_run_array, new_null_array,
Array, ArrayRef, AsArray, BinaryViewArray, StructArray, StructArrayAssembler,
downcast_dictionary_array, downcast_run_array, new_null_array,
};
use arrow::buffer::NullBuffer;
use arrow::compute::cast;
Expand Down Expand Up @@ -60,6 +60,29 @@ pub(crate) fn binary_array_value(array: &dyn Array, index: usize) -> Option<&[u8
}
}

/// A binary-like column whose array type is resolved once, not on every row.
pub(crate) enum BinaryColumn<'a> {
View(&'a BinaryViewArray),
Other(&'a dyn Array),
}

impl<'a> BinaryColumn<'a> {
pub(crate) fn new(array: &'a dyn Array) -> Self {
match array.as_binary_view_opt() {
Some(array) => Self::View(array),
None => Self::Other(array),
}
}

/// The bytes of row `index`, or `None` if the row is null or the column is not binary-like.
pub(crate) fn value(&self, index: usize) -> Option<&'a [u8]> {
match self {
Self::View(array) => array.is_valid(index).then(|| array.value(index)),
Self::Other(array) => binary_array_value(*array, index),
}
}
}

/// Returns a [`Variant`] from a `metadata` and `value` byte arrays, returns `None`
/// if one of them is of invalid type.
pub(crate) fn variant_from_arrays_at<'m, 'v>(
Expand Down
60 changes: 58 additions & 2 deletions parquet-variant-compute/src/variant_get.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,10 @@ use arrow::{
error::Result,
};
use arrow_schema::{ArrowError, DataType, FieldRef};
use parquet_variant::{VariantPath, VariantPathElement};
use parquet_variant::{Variant, VariantMetadataView, VariantPath, VariantPathElement, VariantView};

use crate::ShreddingState;
use crate::variant_array::all_null_value_column;
use crate::variant_array::{BinaryColumn, all_null_value_column};
use crate::variant_to_arrow::make_variant_to_arrow_row_builder;
use crate::{VariantArray, VariantType, unshred_variant};

Expand Down Expand Up @@ -192,6 +192,59 @@ pub(crate) fn follow_shredded_path_element(
}
}

/// Extracts `path` from an unshredded variant column, reading each row's encoded bytes directly.
///
/// A row reads only the headers and field names along `path`, and decodes only the value it finds.
fn get_path_from_bytes(
target: &VariantArray,
path: &[VariantPathElement<'_>],
as_type: Option<&DataType>,
cast_options: &CastOptions,
) -> Result<ArrayRef> {
let mut builder = make_variant_to_arrow_row_builder(
target.metadata_column(),
VariantPath::default(),
as_type,
cast_options,
target.len(),
)?;
let metadata = BinaryColumn::new(target.metadata_column());
let value = BinaryColumn::new(target.value_column());

for i in 0..target.len() {
if target.is_null(i) {
builder.append_null()?;
continue;
}
// A row with neither `value` nor `typed_value` is `Variant::Null`, which has no fields.
let Some(value) = value.value(i) else {
match path.is_empty() {
true => builder.append_value(Variant::Null).map(drop)?,
false => builder.append_null()?,
}
continue;
};
let found = row_value(metadata.value(i), value)
.and_then(|row| row.get_path(path))
.and_then(|found| found.map(|found| found.to_variant()).transpose());
match found {
Ok(Some(found)) => builder.append_value(found).map(drop)?,
Ok(None) => builder.append_null()?,
Err(_) if cast_options.safe => builder.append_null()?,
Err(e) => return Err(e),
}
}
builder.finish()
}

/// Views a row of an unshredded variant column.
fn row_value<'a>(metadata: Option<&'a [u8]>, value: &'a [u8]) -> Result<VariantView<'a, 'a>> {
let metadata = metadata.ok_or_else(|| {
ArrowError::InvalidArgumentError("Variant metadata is null in a non-null row".to_string())
})?;
VariantView::try_new(VariantMetadataView::try_new(metadata)?, value)
}

/// Follows the given path as far as possible through shredded variant fields. If the path ends on a
/// shredded field, return it directly. Otherwise, use a row shredder to follow the rest of the path
/// and extract the requested value on a per-row basis.
Expand Down Expand Up @@ -249,6 +302,9 @@ fn shredded_get_path(
} else {
as_field.map(|f| f.data_type())
};
if target.typed_value_column().is_none() {
return get_path_from_bytes(&target, &path, as_type, cast_options);
}
let mut builder = make_variant_to_arrow_row_builder(
target.metadata_column(),
path,
Expand Down
17 changes: 17 additions & 0 deletions parquet-variant/src/builder/metadata.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements

use std::cmp::Ordering;
use std::collections::HashMap;

use arrow_schema::ArrowError;
Expand Down Expand Up @@ -30,6 +31,13 @@ pub trait MetadataBuilder: std::fmt::Debug {
/// [`Self::num_field_names`]. Panics if the field id is out of bounds.
fn field_name(&self, field_id: usize) -> &str;

/// Compares the field names of two field ids, which must be less than
/// [`Self::num_field_names`]. Panics if a field id is out of bounds.
fn cmp_field_names(&self, field_a_id: u32, field_b_id: u32) -> Ordering {
self.field_name(field_a_id as _)
.cmp(self.field_name(field_b_id as _))
}

/// Returns the number of field names stored in this metadata builder. Any number less than this
/// is a valid field id. The builder can be reverted back to this size later on (discarding any
/// newer/higher field ids) by calling [`Self::truncate_field_names`].
Expand Down Expand Up @@ -117,6 +125,15 @@ impl MetadataBuilder for ReadOnlyMetadataBuilder<'_> {
fn field_name(&self, field_id: usize) -> &str {
&self.metadata[field_id]
}
// Byte order is `str` order, so the comparison needs no UTF-8 validation.
fn cmp_field_names(&self, field_a_id: u32, field_b_id: u32) -> Ordering {
let view = self.metadata.view();
let name = |id: u32| {
view.entry(id as _)
.expect("Invalid metadata dictionary entry")
};
name(field_a_id).cmp(name(field_b_id))
}
fn num_field_names(&self) -> usize {
self.metadata.len()
}
Expand Down
23 changes: 20 additions & 3 deletions parquet-variant/src/builder/object.rs
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,25 @@ impl<'a, S: BuilderSpecificState> ObjectBuilder<'a, S> {
Ok(())
}

/// Adds the field named by entry `field_id` of the metadata dictionary, copying the encoded
/// `value` as is.
///
/// The caller must ensure that `field_id`, and every field id inside `value`, belong to the
/// metadata dictionary that this builder writes against. Copying a field out of an object that
/// uses that same dictionary, as shredding does, satisfies this.
pub fn insert_encoded(&mut self, field_id: u32, value: &[u8]) -> Result<(), ArrowError> {
let state = &mut self.parent_state;
let field_start = state.value_builder.offset() - state.saved_value_builder_offset;
if self.fields.insert(field_id, field_start).is_some() && self.validate_unique_fields {
return Err(ArrowError::InvalidArgumentError(format!(
"Duplicate field name: {}",
state.metadata_builder.field_name(field_id as _)
)));
}
state.value_builder.append_slice(value);
Ok(())
}

/// Builder style API for adding a field with key and value to the object
///
/// Same as [`ObjectBuilder::insert`], but returns `self` for chaining.
Expand Down Expand Up @@ -251,9 +270,7 @@ impl<'a, S: BuilderSpecificState> ObjectBuilder<'a, S> {
let metadata_builder = self.parent_state.metadata_builder();

self.fields.sort_by(|&field_a_id, _, &field_b_id, _| {
let field_a_name = metadata_builder.field_name(field_a_id as usize);
let field_b_name = metadata_builder.field_name(field_b_id as usize);
field_a_name.cmp(field_b_name)
metadata_builder.cmp_field_names(field_a_id, field_b_id)
});

let max_id = self.fields.iter().map(|(i, _)| *i).max().unwrap_or(0);
Expand Down
Loading