diff --git a/parquet-variant-compute/src/shred_variant.rs b/parquet-variant-compute/src/shred_variant.rs index 3ded8ecc9f7b..66f130af165b 100644 --- a/parquet-variant-compute/src/shred_variant.rs +++ b/parquet-variant-compute/src/shred_variant.rs @@ -339,9 +339,18 @@ impl<'a> VariantToShreddedArrayVariantRowBuilder<'a> { } } +fn duplicate_field_name(field_name: &[u8]) -> ArrowError { + ArrowError::InvalidArgumentError(format!( + "Duplicate field name: {}", + String::from_utf8_lossy(field_name) + )) +} + pub(crate) struct VariantToShreddedObjectVariantRowBuilder<'a> { value_builder: VariantValueArrayBuilder, typed_value_builders: IndexMap<&'a str, VariantToShreddedVariantRowBuilder<'a>>, + /// The names of `typed_value_builders` as bytes, sorted, each with its builder's index. + sorted_field_names: Vec<(&'a [u8], usize)>, typed_value_nulls: NullBufferBuilder, nulls: NullBufferBuilder, null_value: NullValue, @@ -369,8 +378,15 @@ impl<'a> VariantToShreddedObjectVariantRowBuilder<'a> { Ok((field.name().as_str(), builder)) }); let typed_value_builders: IndexMap<_, _> = typed_value_builders.collect::>()?; + let mut sorted_field_names: Vec<_> = typed_value_builders + .keys() + .enumerate() + .map(|(index, name)| (name.as_bytes(), index)) + .collect(); + sorted_field_names.sort_unstable(); Ok(Self { value_builder: VariantValueArrayBuilder::new(capacity), + sorted_field_names, seen: vec![false; typed_value_builders.len()], typed_value_builders, typed_value_nulls: NullBufferBuilder::new(capacity), @@ -401,32 +417,44 @@ impl<'a> VariantToShreddedObjectVariantRowBuilder<'a> { return Ok(false); }; - // Route the object's fields by name as either shredded or unshredded + // Route the object's fields by name as either shredded or unshredded. An unshredded field + // keeps its field id and its encoded bytes, because the leftover `value` column uses the + // same metadata dictionary. let Self { value_builder, typed_value_builders, + sorted_field_names, seen, .. } = self; seen.fill(false); + let object = obj.view(); let mut builder = value_builder.builder_ext(value.metadata()); // Error on a repeated unshredded field instead of overwriting it let mut object_builder = builder.try_new_object()?.with_validate_unique_fields(true); let mut partially_shredded = false; - for (field_name, value) in obj.iter() { - match typed_value_builders.get_full_mut(field_name) { - Some((index, _, typed_value_builder)) => { + // The object builder only catches a repeated field id. A sorted dictionary holds each name + // once, so that suffices; an unsorted one may repeat a name under distinct ids. The spec + // lists an object's fields in name order, so such a repeat follows its first occurrence. + let check_adjacent_names = !value.metadata().is_sorted(); + for i in 0..object.len() { + let field_name = object.field_name(i)?; + let field = object.field(i)?; + match sorted_field_names.binary_search_by(|(name, _)| name.cmp(&field_name)) { + Ok(position) => { + let index = sorted_field_names[position].1; // A repeated field would append a second row to this builder if seen[index] { - return Err(ArrowError::InvalidArgumentError(format!( - "Duplicate field name: {field_name}" - ))); + return Err(duplicate_field_name(field_name)); } - typed_value_builder.append_value(value)?; + typed_value_builders[index].append_value(field.to_variant()?)?; seen[index] = true; } - None => { - object_builder.try_insert_bytes(field_name, value)?; + Err(_) => { + if check_adjacent_names && i > 0 && object.field_name(i - 1)? == field_name { + return Err(duplicate_field_name(field_name)); + } + object_builder.insert_encoded(object.field_id(i)?, field.bytes()?)?; partially_shredded = true; } } diff --git a/parquet-variant-compute/src/unshred_variant.rs b/parquet-variant-compute/src/unshred_variant.rs index 6a8255ed3782..323e48cb4d3d 100644 --- a/parquet-variant-compute/src/unshred_variant.rs +++ b/parquet-variant-compute/src/unshred_variant.rs @@ -582,12 +582,14 @@ struct StructUnshredVariantBuilder<'a> { value: Option<&'a ArrayRef>, typed_value: &'a arrow::array::StructArray, field_unshredders: IndexMap<&'a str, Option>>, + /// The names of `field_unshredders` as bytes, sorted. + sorted_field_names: Vec<&'a [u8]>, } impl<'a> StructUnshredVariantBuilder<'a> { fn try_new(value: Option<&'a ArrayRef>, typed_value: &'a StructArray) -> Result { // Create unshredders for each field in constructor - let mut field_unshredders = IndexMap::new(); + let mut field_unshredders: IndexMap<&str, _> = IndexMap::new(); for (field, field_array) in typed_value.fields().iter().zip(typed_value.columns()) { // Factory returns None for None/None case -- these are missing fields we should skip let Some(field_array) = field_array.as_struct_opt() else { @@ -600,10 +602,16 @@ impl<'a> StructUnshredVariantBuilder<'a> { field_unshredders.insert(field.name().as_ref(), field_unshredder); } + let mut sorted_field_names: Vec<_> = field_unshredders + .keys() + .map(|name| name.as_bytes()) + .collect(); + sorted_field_names.sort_unstable(); Ok(Self { value, typed_value, field_unshredders, + sorted_field_names, }) } @@ -634,14 +642,18 @@ impl<'a> StructUnshredVariantBuilder<'a> { )); }; - for entry in object.iter_try() { - let (field_name, field_value) = entry?; - if self.field_unshredders.contains_key(field_name) { + // The leftover fields keep their field ids and encoded bytes, because the output uses + // the same metadata dictionary. + let object = object.view(); + for i in 0..object.len() { + let field_name = object.field_name(i)?; + if self.sorted_field_names.binary_search(&field_name).is_ok() { return Err(ArrowError::InvalidArgumentError(format!( - "Field '{field_name}' appears in both typed_value and value", + "Field '{}' appears in both typed_value and value", + String::from_utf8_lossy(field_name), ))); } - object_builder.insert_bytes(field_name, field_value); + object_builder.insert_encoded(object.field_id(i)?, object.field(i)?.bytes()?)?; } } diff --git a/parquet-variant-compute/src/variant_array.rs b/parquet-variant-compute/src/variant_array.rs index bbd17c46114a..414c758aff60 100644 --- a/parquet-variant-compute/src/variant_array.rs +++ b/parquet-variant-compute/src/variant_array.rs @@ -10,8 +10,8 @@ use crate::type_conversion::{ primitive_conversion_single_value, }; use arrow::array::{ - Array, ArrayRef, AsArray, StructArray, StructArrayAssembler, downcast_dictionary_array, - downcast_run_array, new_null_array, + Array, ArrayRef, AsArray, BinaryViewArray, StructArray, StructArrayAssembler, + downcast_dictionary_array, downcast_run_array, new_null_array, }; use arrow::buffer::NullBuffer; use arrow::compute::cast; @@ -60,6 +60,29 @@ pub(crate) fn binary_array_value(array: &dyn Array, index: usize) -> Option<&[u8 } } +/// A binary-like column whose array type is resolved once, not on every row. +pub(crate) enum BinaryColumn<'a> { + View(&'a BinaryViewArray), + Other(&'a dyn Array), +} + +impl<'a> BinaryColumn<'a> { + pub(crate) fn new(array: &'a dyn Array) -> Self { + match array.as_binary_view_opt() { + Some(array) => Self::View(array), + None => Self::Other(array), + } + } + + /// The bytes of row `index`, or `None` if the row is null or the column is not binary-like. + pub(crate) fn value(&self, index: usize) -> Option<&'a [u8]> { + match self { + Self::View(array) => array.is_valid(index).then(|| array.value(index)), + Self::Other(array) => binary_array_value(*array, index), + } + } +} + /// Returns a [`Variant`] from a `metadata` and `value` byte arrays, returns `None` /// if one of them is of invalid type. pub(crate) fn variant_from_arrays_at<'m, 'v>( diff --git a/parquet-variant-compute/src/variant_get.rs b/parquet-variant-compute/src/variant_get.rs index b0f1ceb33958..5810a4d3856a 100644 --- a/parquet-variant-compute/src/variant_get.rs +++ b/parquet-variant-compute/src/variant_get.rs @@ -13,10 +13,10 @@ use arrow::{ error::Result, }; use arrow_schema::{ArrowError, DataType, FieldRef}; -use parquet_variant::{VariantPath, VariantPathElement}; +use parquet_variant::{Variant, VariantMetadataView, VariantPath, VariantPathElement, VariantView}; use crate::ShreddingState; -use crate::variant_array::all_null_value_column; +use crate::variant_array::{BinaryColumn, all_null_value_column}; use crate::variant_to_arrow::make_variant_to_arrow_row_builder; use crate::{VariantArray, VariantType, unshred_variant}; @@ -192,6 +192,59 @@ pub(crate) fn follow_shredded_path_element( } } +/// Extracts `path` from an unshredded variant column, reading each row's encoded bytes directly. +/// +/// A row reads only the headers and field names along `path`, and decodes only the value it finds. +fn get_path_from_bytes( + target: &VariantArray, + path: &[VariantPathElement<'_>], + as_type: Option<&DataType>, + cast_options: &CastOptions, +) -> Result { + let mut builder = make_variant_to_arrow_row_builder( + target.metadata_column(), + VariantPath::default(), + as_type, + cast_options, + target.len(), + )?; + let metadata = BinaryColumn::new(target.metadata_column()); + let value = BinaryColumn::new(target.value_column()); + + for i in 0..target.len() { + if target.is_null(i) { + builder.append_null()?; + continue; + } + // A row with neither `value` nor `typed_value` is `Variant::Null`, which has no fields. + let Some(value) = value.value(i) else { + match path.is_empty() { + true => builder.append_value(Variant::Null).map(drop)?, + false => builder.append_null()?, + } + continue; + }; + let found = row_value(metadata.value(i), value) + .and_then(|row| row.get_path(path)) + .and_then(|found| found.map(|found| found.to_variant()).transpose()); + match found { + Ok(Some(found)) => builder.append_value(found).map(drop)?, + Ok(None) => builder.append_null()?, + Err(_) if cast_options.safe => builder.append_null()?, + Err(e) => return Err(e), + } + } + builder.finish() +} + +/// Views a row of an unshredded variant column. +fn row_value<'a>(metadata: Option<&'a [u8]>, value: &'a [u8]) -> Result> { + let metadata = metadata.ok_or_else(|| { + ArrowError::InvalidArgumentError("Variant metadata is null in a non-null row".to_string()) + })?; + VariantView::try_new(VariantMetadataView::try_new(metadata)?, value) +} + /// Follows the given path as far as possible through shredded variant fields. If the path ends on a /// shredded field, return it directly. Otherwise, use a row shredder to follow the rest of the path /// and extract the requested value on a per-row basis. @@ -249,6 +302,9 @@ fn shredded_get_path( } else { as_field.map(|f| f.data_type()) }; + if target.typed_value_column().is_none() { + return get_path_from_bytes(&target, &path, as_type, cast_options); + } let mut builder = make_variant_to_arrow_row_builder( target.metadata_column(), path, diff --git a/parquet-variant/src/builder/metadata.rs b/parquet-variant/src/builder/metadata.rs index 38582ed32f87..f55eb348a772 100644 --- a/parquet-variant/src/builder/metadata.rs +++ b/parquet-variant/src/builder/metadata.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements +use std::cmp::Ordering; use std::collections::HashMap; use arrow_schema::ArrowError; @@ -30,6 +31,13 @@ pub trait MetadataBuilder: std::fmt::Debug { /// [`Self::num_field_names`]. Panics if the field id is out of bounds. fn field_name(&self, field_id: usize) -> &str; + /// Compares the field names of two field ids, which must be less than + /// [`Self::num_field_names`]. Panics if a field id is out of bounds. + fn cmp_field_names(&self, field_a_id: u32, field_b_id: u32) -> Ordering { + self.field_name(field_a_id as _) + .cmp(self.field_name(field_b_id as _)) + } + /// Returns the number of field names stored in this metadata builder. Any number less than this /// is a valid field id. The builder can be reverted back to this size later on (discarding any /// newer/higher field ids) by calling [`Self::truncate_field_names`]. @@ -117,6 +125,15 @@ impl MetadataBuilder for ReadOnlyMetadataBuilder<'_> { fn field_name(&self, field_id: usize) -> &str { &self.metadata[field_id] } + // Byte order is `str` order, so the comparison needs no UTF-8 validation. + fn cmp_field_names(&self, field_a_id: u32, field_b_id: u32) -> Ordering { + let view = self.metadata.view(); + let name = |id: u32| { + view.entry(id as _) + .expect("Invalid metadata dictionary entry") + }; + name(field_a_id).cmp(name(field_b_id)) + } fn num_field_names(&self) -> usize { self.metadata.len() } diff --git a/parquet-variant/src/builder/object.rs b/parquet-variant/src/builder/object.rs index 58ba95117405..ce385ff1fff5 100644 --- a/parquet-variant/src/builder/object.rs +++ b/parquet-variant/src/builder/object.rs @@ -151,6 +151,25 @@ impl<'a, S: BuilderSpecificState> ObjectBuilder<'a, S> { Ok(()) } + /// Adds the field named by entry `field_id` of the metadata dictionary, copying the encoded + /// `value` as is. + /// + /// The caller must ensure that `field_id`, and every field id inside `value`, belong to the + /// metadata dictionary that this builder writes against. Copying a field out of an object that + /// uses that same dictionary, as shredding does, satisfies this. + pub fn insert_encoded(&mut self, field_id: u32, value: &[u8]) -> Result<(), ArrowError> { + let state = &mut self.parent_state; + let field_start = state.value_builder.offset() - state.saved_value_builder_offset; + if self.fields.insert(field_id, field_start).is_some() && self.validate_unique_fields { + return Err(ArrowError::InvalidArgumentError(format!( + "Duplicate field name: {}", + state.metadata_builder.field_name(field_id as _) + ))); + } + state.value_builder.append_slice(value); + Ok(()) + } + /// Builder style API for adding a field with key and value to the object /// /// Same as [`ObjectBuilder::insert`], but returns `self` for chaining. @@ -251,9 +270,7 @@ impl<'a, S: BuilderSpecificState> ObjectBuilder<'a, S> { let metadata_builder = self.parent_state.metadata_builder(); self.fields.sort_by(|&field_a_id, _, &field_b_id, _| { - let field_a_name = metadata_builder.field_name(field_a_id as usize); - let field_b_name = metadata_builder.field_name(field_b_id as usize); - field_a_name.cmp(field_b_name) + metadata_builder.cmp_field_names(field_a_id, field_b_id) }); let max_id = self.fields.iter().map(|(i, _)| *i).max().unwrap_or(0); diff --git a/parquet-variant/src/decoder.rs b/parquet-variant/src/decoder.rs index 7d95a8d685d1..e99005e5c865 100644 --- a/parquet-variant/src/decoder.rs +++ b/parquet-variant/src/decoder.rs @@ -137,16 +137,6 @@ impl OffsetSizeBytes { Ok(result) } - /// Return one unsigned little-endian value from `bytes`. - /// - /// * `bytes` – the byte buffer to index - /// * `index` – 0-based index into the buffer - /// - /// Each value is `self as u32` bytes wide (1, 2, 3 or 4), zero-extended to 32 bits as needed. - pub(crate) fn unpack_u32(&self, bytes: &[u8], index: usize) -> Result { - self.unpack_u32_at_offset(bytes, 0, index) - } - /// Return one unsigned little-endian value from `bytes`. /// /// * `bytes` – the byte buffer to index @@ -157,7 +147,7 @@ impl OffsetSizeBytes { /// /// Each value is `self as u32` bytes wide (1, 2, 3 or 4), zero-extended to 32 bits as needed. pub(crate) fn unpack_u32_at_offset( - &self, + self, bytes: &[u8], byte_offset: usize, // how many bytes to skip offset_index: usize, // which offset in an array of offsets @@ -165,9 +155,9 @@ impl OffsetSizeBytes { use OffsetSizeBytes::*; // Index into the byte array: - // byte_offset + (*self as usize) * offset_index + // byte_offset + (self as usize) * offset_index let offset = offset_index - .checked_mul(*self as usize) + .checked_mul(self as usize) .and_then(|n| n.checked_add(byte_offset)) .ok_or_else(|| overflow_error("unpacking offset array value"))?; let value = match self { @@ -637,39 +627,61 @@ mod tests { fn unpack_u32_all_widths() { // One-byte offsets let buf_one = [0x01u8, 0xAB, 0xCD]; - assert_eq!(OffsetSizeBytes::One.unpack_u32(&buf_one, 0).unwrap(), 0x01); - assert_eq!(OffsetSizeBytes::One.unpack_u32(&buf_one, 2).unwrap(), 0xCD); + assert_eq!( + OffsetSizeBytes::One + .unpack_u32_at_offset(&buf_one, 0, 0) + .unwrap(), + 0x01 + ); + assert_eq!( + OffsetSizeBytes::One + .unpack_u32_at_offset(&buf_one, 0, 2) + .unwrap(), + 0xCD + ); // Two-byte offsets (little-endian 0x1234, 0x5678) let buf_two = [0x34, 0x12, 0x78, 0x56]; assert_eq!( - OffsetSizeBytes::Two.unpack_u32(&buf_two, 0).unwrap(), + OffsetSizeBytes::Two + .unpack_u32_at_offset(&buf_two, 0, 0) + .unwrap(), 0x1234 ); assert_eq!( - OffsetSizeBytes::Two.unpack_u32(&buf_two, 1).unwrap(), + OffsetSizeBytes::Two + .unpack_u32_at_offset(&buf_two, 0, 1) + .unwrap(), 0x5678 ); // Three-byte offsets (0x030201 and 0x0000FF) let buf_three = [0x01, 0x02, 0x03, 0xFF, 0x00, 0x00]; assert_eq!( - OffsetSizeBytes::Three.unpack_u32(&buf_three, 0).unwrap(), + OffsetSizeBytes::Three + .unpack_u32_at_offset(&buf_three, 0, 0) + .unwrap(), 0x030201 ); assert_eq!( - OffsetSizeBytes::Three.unpack_u32(&buf_three, 1).unwrap(), + OffsetSizeBytes::Three + .unpack_u32_at_offset(&buf_three, 0, 1) + .unwrap(), 0x0000FF ); // Four-byte offsets (0x12345678, 0x90ABCDEF) let buf_four = [0x78, 0x56, 0x34, 0x12, 0xEF, 0xCD, 0xAB, 0x90]; assert_eq!( - OffsetSizeBytes::Four.unpack_u32(&buf_four, 0).unwrap(), + OffsetSizeBytes::Four + .unpack_u32_at_offset(&buf_four, 0, 0) + .unwrap(), 0x1234_5678 ); assert_eq!( - OffsetSizeBytes::Four.unpack_u32(&buf_four, 1).unwrap(), + OffsetSizeBytes::Four + .unpack_u32_at_offset(&buf_four, 0, 1) + .unwrap(), 0x90AB_CDEF ); } @@ -677,8 +689,16 @@ mod tests { #[test] fn unpack_u32_out_of_bounds() { let tiny = [0x00u8]; // deliberately too short - assert!(OffsetSizeBytes::Two.unpack_u32(&tiny, 0).is_err()); - assert!(OffsetSizeBytes::Three.unpack_u32(&tiny, 0).is_err()); + assert!( + OffsetSizeBytes::Two + .unpack_u32_at_offset(&tiny, 0, 0) + .is_err() + ); + assert!( + OffsetSizeBytes::Three + .unpack_u32_at_offset(&tiny, 0, 0) + .is_err() + ); } #[test] diff --git a/parquet-variant/src/utils.rs b/parquet-variant/src/utils.rs index d950dcfdf19e..2db1e62f1722 100644 --- a/parquet-variant/src/utils.rs +++ b/parquet-variant/src/utils.rs @@ -57,6 +57,7 @@ pub(crate) fn array_from_slice( .map_err(|e: TryFromSliceError| ArrowError::InvalidArgumentError(e.to_string())) } +#[inline] pub(crate) fn first_byte_from_slice(slice: &[u8]) -> Result { slice .first() @@ -71,24 +72,47 @@ pub(crate) fn string_from_slice( offset: usize, range: Range, ) -> Result<&str, ArrowError> { - let offset_buffer = slice_from_slice_at_offset(slice, offset, range)?; + str_from_bytes(slice_from_slice_at_offset(slice, offset, range)?) +} +/// Interprets `bytes` as a UTF-8 string. +#[inline] +pub(crate) fn str_from_bytes(bytes: &[u8]) -> Result<&str, ArrowError> { //Use simdutf8 by default #[cfg(feature = "simdutf8")] { - simdutf8::basic::from_utf8(offset_buffer).map_err(|_| { + simdutf8::basic::from_utf8(bytes).map_err(|_| { // Use simdutf8::compat to return details about the decoding error - let e = simdutf8::compat::from_utf8(offset_buffer).unwrap_err(); + let e = simdutf8::compat::from_utf8(bytes).unwrap_err(); ArrowError::InvalidArgumentError(format!("encountered non UTF-8 data: {e}")) }) } //Use std::str if simdutf8 is not enabled #[cfg(not(feature = "simdutf8"))] - str::from_utf8(offset_buffer) + str::from_utf8(bytes) .map_err(|_| ArrowError::InvalidArgumentError("invalid UTF-8 string".to_string())) } +/// Binary search over `0..len` with a fallible comparison, which returns the first error it hits. +/// +/// Returns the index for which `cmp` returns [`Ordering::Equal`], if any. +pub(crate) fn try_binary_search( + len: usize, + cmp: impl Fn(usize) -> Result, +) -> Result, ArrowError> { + let (mut low, mut high) = (0, len); + while low < high { + let mid = low + (high - low) / 2; + match cmp(mid)? { + Ordering::Less => low = mid + 1, + Ordering::Greater => high = mid, + Ordering::Equal => return Ok(Some(mid)), + } + } + Ok(None) +} + /// Performs a binary search over a range using a fallible key extraction function; a failed key /// extraction immediately terminates the search. /// diff --git a/parquet-variant/src/variant.rs b/parquet-variant/src/variant.rs index 3bfa0f05c82e..08add346eb80 100644 --- a/parquet-variant/src/variant.rs +++ b/parquet-variant/src/variant.rs @@ -8,14 +8,14 @@ pub use self::metadata::{ EMPTY_VARIANT_METADATA, EMPTY_VARIANT_METADATA_BYTES, VARIANT_SPEC_VERSION, VariantMetadata, }; pub use self::object::VariantObject; +pub use self::view::{VariantListView, VariantMetadataView, VariantObjectView, VariantView}; // Publicly export types used in the API +pub use crate::decoder::VariantBasicType; pub use half::f16; pub use uuid::Uuid; -use crate::decoder::{ - self, VariantBasicType, VariantPrimitiveType, get_basic_type, get_primitive_type, -}; +use crate::decoder::{self, VariantPrimitiveType, get_basic_type, get_primitive_type}; use crate::path::{VariantPath, VariantPathElement}; use crate::utils::{first_byte_from_slice, slice_from_slice}; use std::ops::Deref; @@ -27,6 +27,7 @@ mod decimal; mod list; mod metadata; mod object; +mod view; const MAX_SHORT_STRING_BYTES: usize = 0x3F; @@ -381,7 +382,7 @@ impl<'m, 'v> Variant<'m, 'v> { } // The actual constructor, which only performs shallow (constant-time) validation. - fn try_new_with_metadata_and_shallow_validation( + pub(crate) fn try_new_with_metadata_and_shallow_validation( metadata: VariantMetadata<'m>, value: &'v [u8], ) -> Result { diff --git a/parquet-variant/src/variant/list.rs b/parquet-variant/src/variant/list.rs index 4172ac1d2509..83b300ef9d1b 100644 --- a/parquet-variant/src/variant/list.rs +++ b/parquet-variant/src/variant/list.rs @@ -3,9 +3,8 @@ // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements use crate::decoder::{OffsetSizeBytes, map_bytes_to_offsets}; -use crate::utils::{ - first_byte_from_slice, overflow_error, slice_from_slice, slice_from_slice_at_offset, -}; +use crate::utils::{slice_from_slice, slice_from_slice_at_offset}; +use crate::variant::view::VariantListView; use crate::variant::{MAX_NESTING_DEPTH, Variant, VariantMetadata}; use arrow_schema::ArrowError; @@ -14,26 +13,27 @@ use arrow_schema::ArrowError; const NUM_HEADER_BYTES: u32 = 1; /// A parsed version of the variant array value header byte. -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, Copy, PartialEq)] pub(crate) struct VariantListHeader { - num_elements_size: OffsetSizeBytes, - offset_size: OffsetSizeBytes, + pub(crate) num_elements_size: OffsetSizeBytes, + pub(crate) offset_size: OffsetSizeBytes, } impl VariantListHeader { // Hide the ugly casting - const fn num_elements_size(&self) -> u32 { + pub(crate) const fn num_elements_size(&self) -> u32 { self.num_elements_size as _ } - const fn offset_size(&self) -> u32 { + pub(crate) const fn offset_size(&self) -> u32 { self.offset_size as _ } // Avoid materializing this offset, since it's cheaply and safely computable - const fn first_offset_byte(&self) -> u32 { + pub(crate) const fn first_offset_byte(&self) -> u32 { NUM_HEADER_BYTES + self.num_elements_size() } + #[inline] pub(crate) fn try_new(header_byte: u8) -> Result { // The 6 first bits to the left are the value_header and the 2 bits // to the right are the basic type, so we shift to get only the value_header @@ -153,33 +153,17 @@ impl<'m, 'v> VariantList<'m, 'v> { metadata: VariantMetadata<'m>, value: &'v [u8], ) -> Result { - let header_byte = first_byte_from_slice(value)?; - let header = VariantListHeader::try_new(header_byte)?; - - // Skip the header byte to read the num_elements; the offset array immediately follows - let num_elements = - header - .num_elements_size - .unpack_u32_at_offset(value, NUM_HEADER_BYTES as _, 0)?; - - // (num_elements + 1) * offset_size + first_offset_byte - let first_value_byte = num_elements - .checked_add(1) - .and_then(|n| n.checked_mul(header.offset_size())) - .and_then(|n| n.checked_add(header.first_offset_byte())) - .ok_or_else(|| overflow_error("offset of variant list values"))?; - - let mut new_self = Self { - metadata, - value, - header, - num_elements, - first_value_byte, - validated: false, - }; + VariantListView::try_new(metadata.view(), value)?.into_list(metadata) + } - // Validate just the first and last offset, ignoring the other offsets and all value bytes. - let first_offset = new_self.get_offset(0)?; + // Promotes a view by validating just the first and last offset, ignoring the other offsets + // and all value bytes. + #[inline] + pub(crate) fn try_from_view( + view: VariantListView<'m, 'v>, + metadata: VariantMetadata<'m>, + ) -> Result { + let first_offset = view.read_offset(0)?; if first_offset != 0 { return Err(ArrowError::InvalidArgumentError(format!( "First offset is not zero: {first_offset}" @@ -187,12 +171,27 @@ impl<'m, 'v> VariantList<'m, 'v> { } // Use the last offset to upper-bound the value buffer - let last_offset = new_self - .get_offset(num_elements as _)? - .checked_add(first_value_byte) - .ok_or_else(|| overflow_error("variant array size"))?; - new_self.value = slice_from_slice(value, ..last_offset as _)?; - Ok(new_self) + let last_offset = view.byte_len()?; + Ok(Self { + metadata, + value: slice_from_slice(view.value, ..last_offset)?, + header: view.header, + num_elements: view.num_elements, + first_value_byte: view.first_value_byte, + validated: false, + }) + } + + /// A view of this list, which reads only the elements it is asked for. + #[inline] + pub fn view(&self) -> VariantListView<'m, 'v> { + VariantListView { + metadata: self.metadata.view(), + value: self.value, + header: self.header, + num_elements: self.num_elements, + first_value_byte: self.first_value_byte, + } } /// True if this instance is fully [validated] for panic-free infallible accesses. @@ -288,9 +287,24 @@ impl<'m, 'v> VariantList<'m, 'v> { // Fallible version of `get`, performing only basic (constant-time) validation. fn try_get_with_shallow_validation(&self, index: usize) -> Result, ArrowError> { + if index >= self.len() { + return Err(ArrowError::InvalidArgumentError(format!( + "Variant list index {index} is out of bounds for length {}", + self.len() + ))); + } + self.element(&self.view(), index) + } + + // Reads element `index`, which the caller has bounded by `self.len()`, through `view`. + fn element( + &self, + view: &VariantListView<'m, 'v>, + index: usize, + ) -> Result, ArrowError> { // Fetch the value bytes between the two offsets for this index, from the value array region // of the byte buffer - let byte_range = self.get_offset(index)? as _..self.get_offset(index + 1)? as _; + let byte_range = view.read_offset(index)? as _..view.read_offset(index + 1)? as _; let value_bytes = slice_from_slice_at_offset(self.value, self.first_value_byte as _, byte_range)?; Variant::try_new_with_metadata_and_shallow_validation(self.metadata.clone(), value_bytes) @@ -320,14 +334,9 @@ impl<'m, 'v> VariantList<'m, 'v> { fn iter_try_with_shallow_validation( &self, ) -> impl Iterator, ArrowError>> + '_ { - (0..self.len()).map(|i| self.try_get_with_shallow_validation(i)) - } - - // Attempts to retrieve the ith offset from the offset array region of the byte buffer. - fn get_offset(&self, index: usize) -> Result { - let byte_range = self.header.first_offset_byte() as _..self.first_value_byte as _; - let offset_bytes = slice_from_slice(self.value, byte_range)?; - self.header.offset_size.unpack_u32(offset_bytes, index) + // One view serves every element, and the range bounds each element index. + let view = self.view(); + (0..self.len()).map(move |i| self.element(&view, i)) } } diff --git a/parquet-variant/src/variant/metadata.rs b/parquet-variant/src/variant/metadata.rs index b51b4ffbd932..ef8571397869 100644 --- a/parquet-variant/src/variant/metadata.rs +++ b/parquet-variant/src/variant/metadata.rs @@ -4,9 +4,9 @@ use crate::decoder::{OffsetSizeBytes, map_bytes_to_offsets}; use crate::utils::{ - first_byte_from_slice, overflow_error, slice_from_slice, slice_from_slice_at_offset, - string_from_slice, try_binary_search_range_by, + overflow_error, slice_from_slice, str_from_bytes, string_from_slice, try_binary_search_range_by, }; +use crate::variant::view::VariantMetadataView; use arrow_schema::ArrowError; @@ -14,9 +14,9 @@ use arrow_schema::ArrowError; #[derive(Debug, Clone, Copy, PartialEq)] pub(crate) struct VariantMetadataHeader { version: u8, - is_sorted: bool, + pub(crate) is_sorted: bool, /// Note: This is `offset_size_minus_one` + 1 - offset_size: OffsetSizeBytes, + pub(crate) offset_size: OffsetSizeBytes, } /// The version of the Variant specification implemented by this crate. @@ -32,12 +32,12 @@ const NUM_HEADER_BYTES: u32 = 1; impl VariantMetadataHeader { // Hide the cast - const fn offset_size(&self) -> u32 { + pub(crate) const fn offset_size(&self) -> u32 { self.offset_size as u32 } // Avoid materializing this offset, since it's cheaply and safely computable - const fn first_offset_byte(&self) -> u32 { + pub(crate) const fn first_offset_byte(&self) -> u32 { NUM_HEADER_BYTES + self.offset_size() } @@ -57,6 +57,7 @@ impl VariantMetadataHeader { /// - sorted_strings is a 1-bit value indicating whether dictionary strings are sorted and unique. /// - offset_size_minus_one is a 2-bit value providing the number of bytes per dictionary size and offset field. /// - The actual number of bytes, offset_size, is offset_size_minus_one + 1 + #[inline] pub(crate) fn try_new(header_byte: u8) -> Result { let version = header_byte & 0x0F; // First four bits if version != VARIANT_SPEC_VERSION { @@ -199,35 +200,14 @@ impl<'m> VariantMetadata<'m> { // The actual constructor, which performs only basic (constant-const) validation. pub(crate) fn try_new_with_shallow_validation(bytes: &'m [u8]) -> Result { - let header_byte = first_byte_from_slice(bytes)?; - let header = VariantMetadataHeader::try_new(header_byte)?; - - // First element after header is dictionary size; the offset array immediately follows. - let dictionary_size = - header - .offset_size - .unpack_u32_at_offset(bytes, NUM_HEADER_BYTES as usize, 0)?; - - // Calculate the starting offset of the dictionary string bytes. - // - // There are dict_size + 1 offsets, and the value bytes immediately follow - // = (dict_size + 1) * offset_size + header.first_offset_byte() - let first_value_byte = dictionary_size - .checked_add(1) - .and_then(|n| n.checked_mul(header.offset_size())) - .and_then(|n| n.checked_add(header.first_offset_byte())) - .ok_or_else(|| overflow_error("offset of variant metadata dictionary"))?; - - let mut new_self = Self { - bytes, - header, - dictionary_size, - first_value_byte, - validated: false, - }; + VariantMetadataView::try_new(bytes)?.into_metadata() + } - // Validate just the first and last offset, ignoring the other offsets and all value bytes. - let first_offset = new_self.get_offset(0)?; + // Promotes a view by validating just the first and last offset, ignoring the other offsets + // and all value bytes. + #[inline] + pub(crate) fn try_from_view(view: VariantMetadataView<'m>) -> Result { + let first_offset = view.offset(0)?; if first_offset != 0 { return Err(ArrowError::InvalidArgumentError(format!( "First offset is not zero: {first_offset}" @@ -235,12 +215,28 @@ impl<'m> VariantMetadata<'m> { } // Use the last offset to upper-bound the byte slice - let last_offset = new_self - .get_offset(dictionary_size as _)? - .checked_add(first_value_byte) + let last_offset = view + .offset(view.len())? + .checked_add(view.first_value_byte) .ok_or_else(|| overflow_error("variant metadata size"))?; - new_self.bytes = slice_from_slice(bytes, ..last_offset as _)?; - Ok(new_self) + Ok(Self { + bytes: slice_from_slice(view.bytes, ..last_offset as _)?, + header: view.header, + dictionary_size: view.dictionary_size, + first_value_byte: view.first_value_byte, + validated: false, + }) + } + + /// A view of this dictionary, which reads only the entries it is asked for. + #[inline] + pub fn view(&self) -> VariantMetadataView<'m> { + VariantMetadataView { + bytes: self.bytes, + header: self.header, + dictionary_size: self.dictionary_size, + first_value_byte: self.first_value_byte, + } } /// The number of metadata dictionary entries @@ -334,16 +330,6 @@ impl<'m> VariantMetadata<'m> { self.header.version } - /// Gets an offset into the dictionary entry by index. - /// - /// This offset is an index into the dictionary, at the boundary between string `i-1` and string - /// `i`. See [`Self::get`] to retrieve a specific dictionary entry. - fn get_offset(&self, i: usize) -> Result { - let offset_byte_range = self.header.first_offset_byte() as _..self.first_value_byte as _; - let bytes = slice_from_slice(self.bytes, offset_byte_range)?; - self.header.offset_size.unpack_u32(bytes, i) - } - /// Returns the total size, in bytes, of the metadata. /// /// Note this value may be smaller than what was passed to [`Self::new`] or @@ -358,16 +344,13 @@ impl<'m> VariantMetadata<'m> { /// /// [invalid]: Self#Validation pub fn get(&self, i: usize) -> Result<&'m str, ArrowError> { - let byte_range = self.get_offset(i)? as _..self.get_offset(i + 1)? as _; + let value_bytes = self.view().entry(i)?; if !self.validated { - return string_from_slice(self.bytes, self.first_value_byte as _, byte_range); + return str_from_bytes(value_bytes); } // Full validation already proved every dictionary entry is valid UTF-8, so validating // again here would charge that cost once per field access instead of once per buffer. - let value_bytes = - slice_from_slice_at_offset(self.bytes, self.first_value_byte as _, byte_range)?; - debug_assert!(std::str::from_utf8(value_bytes).is_ok()); // SAFETY: `validated` is set only by `with_full_validation`, which proved that every @@ -441,7 +424,7 @@ impl<'m> VariantMetadata<'m> { let cmp = |i| { Some( offset_size - .unpack_u32(offsets, i) + .unpack_u32_at_offset(offsets, 0, i) .ok()? .cmp(&field_name_offset), ) @@ -449,7 +432,9 @@ impl<'m> VariantMetadata<'m> { let field_id = try_binary_search_range_by(0..self.len(), cmp)?.ok()?; // Verify that this entry has exactly `field_name`'s bytes; see "Correctness" above. - let entry_end = offset_size.unpack_u32(offsets, field_id + 1).ok()?; + let entry_end = offset_size + .unpack_u32_at_offset(offsets, 0, field_id + 1) + .ok()?; let entry_len = entry_end.checked_sub(field_name_offset)?; (entry_len as usize == field_name.len()).then_some(field_id as u32) } @@ -470,15 +455,11 @@ impl<'m> VariantMetadata<'m> { return Some((field_id, self.get_impl(field_id as _))); } - let field_id = if self.is_sorted() && self.len() > 10 { - // Binary search is faster for a not-tiny sorted metadata dictionary - let cmp = |i| Some(self.get_impl(i).cmp(field_name)); - try_binary_search_range_by(0..self.len(), cmp)?.ok()? - } else { - // Fall back to Linear search for tiny or unsorted dictionary - (0..self.len()).find(|i| self.get_impl(*i) == field_name)? - }; - Some((field_id as u32, self.get_impl(field_id))) + let field_id = self + .view() + .find(field_name.as_bytes()) + .expect("Invalid metadata dictionary entry")?; + Some((field_id, self.get_impl(field_id as _))) } /// Returns an iterator that attempts to visit all dictionary entries, producing `Err` if the @@ -547,11 +528,11 @@ mod tests { assert_eq!(&md[1], "dog"); // Offsets - assert_eq!(md.get_offset(0).unwrap(), 0x00); - assert_eq!(md.get_offset(1).unwrap(), 0x03); - assert_eq!(md.get_offset(2).unwrap(), 0x06); + assert_eq!(md.view().offset(0).unwrap(), 0x00); + assert_eq!(md.view().offset(1).unwrap(), 0x03); + assert_eq!(md.view().offset(2).unwrap(), 0x06); - let err = md.get_offset(3).unwrap_err(); + let err = md.get(2).unwrap_err(); assert!( matches!(err, ArrowError::InvalidArgumentError(_)), "unexpected error: {err:?}" diff --git a/parquet-variant/src/variant/object.rs b/parquet-variant/src/variant/object.rs index 3c21f12e603a..23b396a2c8df 100644 --- a/parquet-variant/src/variant/object.rs +++ b/parquet-variant/src/variant/object.rs @@ -3,9 +3,8 @@ // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements use crate::decoder::{OffsetSizeBytes, map_bytes_to_offsets}; -use crate::utils::{ - first_byte_from_slice, overflow_error, slice_from_slice, try_binary_search_range_by, -}; +use crate::utils::slice_from_slice; +use crate::variant::view::VariantObjectView; use crate::variant::{MAX_NESTING_DEPTH, Variant, VariantMetadata}; use arrow_schema::ArrowError; @@ -14,30 +13,31 @@ use arrow_schema::ArrowError; const NUM_HEADER_BYTES: u32 = 1; /// Header structure for [`VariantObject`] -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, Copy, PartialEq)] pub(crate) struct VariantObjectHeader { - num_elements_size: OffsetSizeBytes, - field_id_size: OffsetSizeBytes, - field_offset_size: OffsetSizeBytes, + pub(crate) num_elements_size: OffsetSizeBytes, + pub(crate) field_id_size: OffsetSizeBytes, + pub(crate) field_offset_size: OffsetSizeBytes, } impl VariantObjectHeader { // Hide the ugly casting - const fn num_elements_size(&self) -> u32 { + pub(crate) const fn num_elements_size(&self) -> u32 { self.num_elements_size as _ } - const fn field_id_size(&self) -> u32 { + pub(crate) const fn field_id_size(&self) -> u32 { self.field_id_size as _ } - const fn field_offset_size(&self) -> u32 { + pub(crate) const fn field_offset_size(&self) -> u32 { self.field_offset_size as _ } // Avoid materializing this offset, since it's cheaply and safely computable - const fn field_ids_start_byte(&self) -> u32 { + pub(crate) const fn field_ids_start_byte(&self) -> u32 { NUM_HEADER_BYTES + self.num_elements_size() } + #[inline] pub(crate) fn try_new(header_byte: u8) -> Result { // Parse the header byte to get object parameters let value_header = header_byte >> 2; @@ -153,48 +153,42 @@ impl<'m, 'v> VariantObject<'m, 'v> { metadata: VariantMetadata<'m>, value: &'v [u8], ) -> Result { - let header_byte = first_byte_from_slice(value)?; - let header = VariantObjectHeader::try_new(header_byte)?; - - // Determine num_elements size based on is_large flag and fetch the value - let num_elements = - header - .num_elements_size - .unpack_u32_at_offset(value, NUM_HEADER_BYTES as _, 0)?; - - // Calculate byte offsets for field offsets and values with overflow protection, and verify - // they're in bounds - let first_field_offset_byte = num_elements - .checked_mul(header.field_id_size()) - .and_then(|n| n.checked_add(header.field_ids_start_byte())) - .ok_or_else(|| overflow_error("offset of variant object field offsets"))?; - - let first_value_byte = num_elements - .checked_add(1) - .and_then(|n| n.checked_mul(header.field_offset_size())) - .and_then(|n| n.checked_add(first_field_offset_byte)) - .ok_or_else(|| overflow_error("offset of variant object field values"))?; - - let mut new_self = Self { - metadata, - value, - header, - num_elements, - first_field_offset_byte, - first_value_byte, - validated: false, - }; + VariantObjectView::try_new(metadata.view(), value)?.into_object(metadata) + } + // Promotes a view by validating just the last field offset. + #[inline] + pub(crate) fn try_from_view( + view: VariantObjectView<'m, 'v>, + metadata: VariantMetadata<'m>, + ) -> Result { // Spec says: "The last field_offset points to the byte after the end of the last value" // // Use it to upper-bound the value bytes, which also verifies that the field id and field // offset arrays are in bounds. - let last_offset = new_self - .get_offset(num_elements as _)? - .checked_add(first_value_byte) - .ok_or_else(|| overflow_error("variant object size"))?; - new_self.value = slice_from_slice(value, ..last_offset as _)?; - Ok(new_self) + let last_offset = view.byte_len()?; + Ok(Self { + metadata, + value: slice_from_slice(view.value, ..last_offset)?, + header: view.header, + num_elements: view.num_elements, + first_field_offset_byte: view.first_field_offset_byte, + first_value_byte: view.first_value_byte, + validated: false, + }) + } + + /// A view of this object, which reads only the fields it is asked for. + #[inline] + pub fn view(&self) -> VariantObjectView<'m, 'v> { + VariantObjectView { + metadata: self.metadata.view(), + value: self.value, + header: self.header, + num_elements: self.num_elements, + first_field_offset_byte: self.first_field_offset_byte, + first_value_byte: self.first_value_byte, + } } /// True if this instance is fully [validated] for panic-free infallible accesses. @@ -351,16 +345,8 @@ impl<'m, 'v> VariantObject<'m, 'v> { // Attempts to retrieve the ith field value from the value region of the byte buffer; it // performs only basic (constant-cost) validation. fn try_field_with_shallow_validation(&self, i: usize) -> Result, ArrowError> { - let value_bytes = slice_from_slice(self.value, self.first_value_byte as _..)?; - let value_bytes = slice_from_slice(value_bytes, self.get_offset(i)? as _..)?; - Variant::try_new_with_metadata_and_shallow_validation(self.metadata.clone(), value_bytes) - } - - // Attempts to retrieve the ith offset from the field offset region of the byte buffer. - fn get_offset(&self, i: usize) -> Result { - let byte_range = self.first_field_offset_byte as _..self.first_value_byte as _; - let field_offsets = slice_from_slice(self.value, byte_range)?; - self.header.field_offset_size.unpack_u32(field_offsets, i) + let field = self.view().field(i)?; + Variant::try_new_with_metadata_and_shallow_validation(self.metadata.clone(), field.value) } /// Get a field's name by index in `0..self.len()` @@ -377,10 +363,7 @@ impl<'m, 'v> VariantObject<'m, 'v> { /// Fallible version of `field_name`. Returns field name by index, capturing validation errors fn try_field_name(&self, i: usize) -> Result<&'m str, ArrowError> { - let byte_range = self.header.field_ids_start_byte() as _..self.first_field_offset_byte as _; - let field_id_bytes = slice_from_slice(self.value, byte_range)?; - let field_id = self.header.field_id_size.unpack_u32(field_id_bytes, i)?; - self.metadata.get(field_id as _) + self.metadata.get(self.view().field_id(i)? as _) } /// Returns an iterator of (name, value) pairs over the fields of this object. @@ -408,9 +391,15 @@ impl<'m, 'v> VariantObject<'m, 'v> { fn iter_try_with_shallow_validation( &self, ) -> impl Iterator), ArrowError>> + '_ { - (0..self.len()).map(|i| { - let field = self.try_field_with_shallow_validation(i)?; - Ok((self.try_field_name(i)?, field)) + // One view serves every field, and the range bounds each field index. + let view = self.view(); + (0..self.len()).map(move |i| { + let field = view.read_field(i)?; + let field = Variant::try_new_with_metadata_and_shallow_validation( + self.metadata.clone(), + field.value, + )?; + Ok((self.metadata.get(view.read_field_id(i)? as _)?, field)) }) } @@ -418,13 +407,13 @@ impl<'m, 'v> VariantObject<'m, 'v> { /// /// Returns `Some(Variant)` if the field exists, or `None` if the field does not exist. pub fn get(&self, name: &str) -> Option> { - // Binary search through the field IDs of this object to find the requested field name. - // // NOTE: This does not require a sorted metadata dictionary, because the variant spec // requires object field ids to be lexically sorted by their corresponding string values, // and probing the dictionary for a field id is always O(1) work. - let cmp = |i| Some(self.field_name(i)?.cmp(name)); - let i = try_binary_search_range_by(0..self.len(), cmp)?.ok()?; + let i = self + .view() + .find(name.as_bytes()) + .expect("Invalid variant object field name")?; self.field(i) } } diff --git a/parquet-variant/src/variant/view.rs b/parquet-variant/src/variant/view.rs new file mode 100644 index 000000000000..6c9063217382 --- /dev/null +++ b/parquet-variant/src/variant/view.rs @@ -0,0 +1,936 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +//! Views over encoded variant bytes that read only what they use. See [`VariantView`]. + +use arrow_schema::ArrowError; + +use crate::decoder::{VariantBasicType, VariantPrimitiveType, get_basic_type}; +use crate::path::VariantPathElement; +use crate::utils::{ + array_from_slice, first_byte_from_slice, overflow_error, slice_from_slice, + slice_from_slice_at_offset, str_from_bytes, try_binary_search, +}; +use crate::variant::list::VariantListHeader; +use crate::variant::metadata::VariantMetadataHeader; +use crate::variant::object::VariantObjectHeader; +use crate::variant::{ + EMPTY_VARIANT_METADATA, Variant, VariantList, VariantMetadata, VariantObject, +}; + +/// The size of the header byte of a metadata dictionary, object or list. +const NUM_HEADER_BYTES: u32 = 1; + +#[inline] +fn checked(value: Option, what: &str) -> Result { + value.ok_or_else(|| overflow_error(what)) +} + +fn index_error(kind: &str, index: usize, len: usize) -> ArrowError { + ArrowError::InvalidArgumentError(format!( + "{kind} index {index} is out of bounds for length {len}" + )) +} + +/// A metadata dictionary of which only the header has been read. See [`VariantView`]. +#[derive(Debug, Clone, Copy)] +pub struct VariantMetadataView<'m> { + pub(crate) bytes: &'m [u8], + pub(crate) header: VariantMetadataHeader, + pub(crate) dictionary_size: u32, + pub(crate) first_value_byte: u32, +} + +impl<'m> VariantMetadataView<'m> { + /// Reads the header and the dictionary size of the metadata dictionary in `bytes`. + #[inline] + pub fn try_new(bytes: &'m [u8]) -> Result { + let header = VariantMetadataHeader::try_new(first_byte_from_slice(bytes)?)?; + let dictionary_size = + header + .offset_size + .unpack_u32_at_offset(bytes, NUM_HEADER_BYTES as _, 0)?; + + // There are dictionary_size + 1 offsets, and the string bytes immediately follow them. + let first_value_byte = checked( + dictionary_size + .checked_add(1) + .and_then(|n| n.checked_mul(header.offset_size())) + .and_then(|n| n.checked_add(header.first_offset_byte())), + "offset of variant metadata dictionary", + )?; + + Ok(Self { + bytes, + header, + dictionary_size, + first_value_byte, + }) + } + + /// The number of entries in the dictionary. + #[inline] + pub fn len(&self) -> usize { + self.dictionary_size as _ + } + + /// True if the dictionary has no entries. + #[inline] + pub fn is_empty(&self) -> bool { + self.dictionary_size == 0 + } + + /// True if the header declares the entries unique and sorted. + #[inline] + pub fn is_sorted(&self) -> bool { + self.header.is_sorted + } + + /// The bytes of entry `id`, without UTF-8 validation. + #[inline] + pub fn entry(&self, id: usize) -> Result<&'m [u8], ArrowError> { + if id >= self.len() { + return Err(index_error("Metadata dictionary", id, self.len())); + } + self.read_entry(id) + } + + /// The entry `id` as a string. + pub fn entry_str(&self, id: usize) -> Result<&'m str, ArrowError> { + str_from_bytes(self.entry(id)?) + } + + /// The id of the entry equal to `name`, if any. + /// + /// The search is a binary search if the dictionary is sorted and has more than a few entries, + /// and a linear scan otherwise. + pub fn find(&self, name: &[u8]) -> Result, ArrowError> { + if self.is_sorted() && self.len() > 10 { + let found = try_binary_search(self.len(), |id| Ok(self.read_entry(id)?.cmp(name)))?; + return Ok(found.map(|id| id as u32)); + } + for id in 0..self.len() { + if self.read_entry(id)? == name { + return Ok(Some(id as u32)); + } + } + Ok(None) + } + + /// Checks the first and last offsets, and returns the bounded [`VariantMetadata`]. + #[inline] + pub fn into_metadata(self) -> Result, ArrowError> { + VariantMetadata::try_from_view(self) + } + + /// Entry `i` of the offset array, for `i` in `0..=self.len()`. + #[inline] + pub(crate) fn offset(&self, i: usize) -> Result { + self.header.offset_size.unpack_u32_at_offset( + self.bytes, + self.header.first_offset_byte() as _, + i, + ) + } + + // The bytes of entry `id`, for `id` in `0..self.len()`. + #[inline] + fn read_entry(&self, id: usize) -> Result<&'m [u8], ArrowError> { + let range = self.offset(id)? as usize..self.offset(id + 1)? as usize; + slice_from_slice_at_offset(self.bytes, self.first_value_byte as _, range) + } +} + +/// A variant value of which only the header byte has been read. +/// +/// Building a view parses one header and nothing else. Each accessor reads and bounds-checks only +/// the bytes it needs, so malformed input produces an error, never a panic. A view does not check +/// UTF-8, sort order, or any offset it does not read. Field names are compared as bytes, which +/// orders them exactly as `str` does. +/// +/// Use a view to read a few values out of a variant, such as one path per row of a column, without +/// paying for checks that the access does not use. +/// +/// # Views and validation +/// +/// The views [`VariantView`], [`VariantMetadataView`], [`VariantObjectView`] and +/// [`VariantListView`] are the weakest state of a variant. The [`Variant`], [`VariantMetadata`], +/// [`VariantObject`] and [`VariantList`] types are the stronger states: +/// +/// - [`VariantView::to_variant`], [`VariantMetadataView::into_metadata`], +/// [`VariantObjectView::into_object`] and [`VariantListView::into_list`] check the first and +/// last offsets, which bound the encoding. Unvalidated instances of the stronger types guarantee +/// those checks. +/// - The `with_full_validation` methods of the stronger types check everything else. +/// +/// # Value bytes +/// +/// A child value starts at its offset and extends to the end of its parent, because object field +/// values need not be stored in field order. [`VariantView::bytes`] finds the exact encoding of a +/// value when a caller needs it. +/// +/// # Example +/// +/// ``` +/// # use parquet_variant::{Variant, VariantBuilder, VariantMetadataView, VariantPath, VariantView}; +/// # use arrow_schema::ArrowError; +/// # fn main() -> Result<(), ArrowError> { +/// let mut builder = VariantBuilder::new(); +/// let mut object = builder.new_object(); +/// object.insert("name", "Ferris"); +/// object.new_list("scores").with_value(7i8).with_value(9i8).finish(); +/// object.finish(); +/// let (metadata, value) = builder.finish(); +/// +/// let view = VariantView::try_new(VariantMetadataView::try_new(&metadata)?, &value)?; +/// let path = VariantPath::try_from("scores[1]")?; +/// let found = view.get_path(&path)?.expect("the path exists"); +/// assert_eq!(found.to_variant()?, Variant::Int8(9)); +/// assert!(view.get_field(b"missing")?.is_none()); +/// # Ok(()) +/// # } +/// ``` +#[derive(Debug, Clone, Copy)] +pub struct VariantView<'m, 'v> { + pub(crate) metadata: VariantMetadataView<'m>, + pub(crate) value: &'v [u8], + header: u8, +} + +impl<'m, 'v> VariantView<'m, 'v> { + /// Reads the header byte of the value that starts at the beginning of `value`. + /// + /// `value` may extend past the end of the value. + #[inline] + pub fn try_new(metadata: VariantMetadataView<'m>, value: &'v [u8]) -> Result { + let header = first_byte_from_slice(value)?; + Ok(Self { + metadata, + value, + header, + }) + } + + /// The metadata dictionary of this value. + #[inline] + pub fn metadata(&self) -> VariantMetadataView<'m> { + self.metadata + } + + /// The basic type of this value. + #[inline] + pub fn basic_type(&self) -> VariantBasicType { + get_basic_type(self.header) + } + + /// This value as an object, or `None` if it is not an object. + pub fn as_object(&self) -> Result>, ArrowError> { + match self.basic_type() { + VariantBasicType::Object => { + VariantObjectView::try_new(self.metadata, self.value).map(Some) + } + _ => Ok(None), + } + } + + /// This value as a list, or `None` if it is not a list. + pub fn as_list(&self) -> Result>, ArrowError> { + match self.basic_type() { + VariantBasicType::Array => { + VariantListView::try_new(self.metadata, self.value).map(Some) + } + _ => Ok(None), + } + } + + /// The value of field `name`, or `None` if this value is not an object or has no such field. + pub fn get_field(&self, name: &[u8]) -> Result, ArrowError> { + match self.as_object()? { + Some(object) => object.get(name), + None => Ok(None), + } + } + + /// Element `index`, or `None` if this value is not a list or has no such element. + pub fn get_index(&self, index: usize) -> Result, ArrowError> { + match self.as_list()? { + Some(list) => list.get(index), + None => Ok(None), + } + } + + /// Follows `path` from this value, or returns `None` if a step does not exist. + /// + /// A [`VariantPathElement::ListElement`] step does not exist, as in [`Variant::get_path`]. + #[inline] + pub fn get_path(&self, path: &[VariantPathElement<'_>]) -> Result, ArrowError> { + let mut current = *self; + for element in path { + let next = match element { + VariantPathElement::Field { name } => current.get_field(name.as_bytes())?, + VariantPathElement::Index { index } => current.get_index(*index)?, + VariantPathElement::ListElement => None, + }; + match next { + Some(next) => current = next, + None => return Ok(None), + } + } + Ok(Some(current)) + } + + /// Decodes this value as an unvalidated [`Variant`]. + /// + /// Only an object or a list reads the metadata dictionary, so only they check its bounds. + #[inline] + pub fn to_variant(&self) -> Result, ArrowError> { + let metadata = match self.basic_type() { + VariantBasicType::Object | VariantBasicType::Array => self.metadata.into_metadata()?, + VariantBasicType::Primitive | VariantBasicType::ShortString => EMPTY_VARIANT_METADATA, + }; + Variant::try_new_with_metadata_and_shallow_validation(metadata, self.value) + } + + /// The exact encoding of this value, without any bytes that follow it. + pub fn bytes(&self) -> Result<&'v [u8], ArrowError> { + let len = match self.basic_type() { + VariantBasicType::Primitive => primitive_len(self.header, self.value)?, + VariantBasicType::ShortString => 1 + (self.header >> 2) as usize, + VariantBasicType::Object => { + VariantObjectView::try_new(self.metadata, self.value)?.byte_len()? + } + VariantBasicType::Array => { + VariantListView::try_new(self.metadata, self.value)?.byte_len()? + } + }; + slice_from_slice(self.value, ..len) + } +} + +/// The encoded size of the primitive value with header byte `header` at the start of `value`. +fn primitive_len(header: u8, value: &[u8]) -> Result { + use VariantPrimitiveType::*; + let payload = match VariantPrimitiveType::try_from(header >> 2)? { + Null | BooleanTrue | BooleanFalse => 0, + Int8 => 1, + Int16 => 2, + Int32 | Date | Float => 4, + Decimal4 => 5, + Int64 | Double | TimestampMicros | TimestampNtzMicros | TimestampNanos + | TimestampNtzNanos | Time => 8, + Decimal8 => 9, + Uuid => 16, + Decimal16 => 17, + Binary | String => 4 + u32::from_le_bytes(array_from_slice(value, 1)?) as usize, + }; + Ok(1 + payload) +} + +/// A variant object of which only the header has been read. See [`VariantView`]. +#[derive(Debug, Clone, Copy)] +pub struct VariantObjectView<'m, 'v> { + pub(crate) metadata: VariantMetadataView<'m>, + pub(crate) value: &'v [u8], + pub(crate) header: VariantObjectHeader, + pub(crate) num_elements: u32, + pub(crate) first_field_offset_byte: u32, + pub(crate) first_value_byte: u32, +} + +impl<'m, 'v> VariantObjectView<'m, 'v> { + /// Reads the header and the field count of the object at the start of `value`. + #[inline] + pub fn try_new(metadata: VariantMetadataView<'m>, value: &'v [u8]) -> Result { + let header = VariantObjectHeader::try_new(first_byte_from_slice(value)?)?; + let num_elements = + header + .num_elements_size + .unpack_u32_at_offset(value, NUM_HEADER_BYTES as _, 0)?; + + let first_field_offset_byte = checked( + num_elements + .checked_mul(header.field_id_size()) + .and_then(|n| n.checked_add(header.field_ids_start_byte())), + "offset of variant object field offsets", + )?; + let first_value_byte = checked( + num_elements + .checked_add(1) + .and_then(|n| n.checked_mul(header.field_offset_size())) + .and_then(|n| n.checked_add(first_field_offset_byte)), + "offset of variant object field values", + )?; + + Ok(Self { + metadata, + value, + header, + num_elements, + first_field_offset_byte, + first_value_byte, + }) + } + + /// The number of fields. + #[inline] + pub fn len(&self) -> usize { + self.num_elements as _ + } + + /// True if the object has no fields. + #[inline] + pub fn is_empty(&self) -> bool { + self.num_elements == 0 + } + + /// The metadata dictionary id of field `i`. + pub fn field_id(&self, i: usize) -> Result { + self.check_index(i)?; + self.read_field_id(i) + } + + /// The name of field `i`, without UTF-8 validation. + pub fn field_name(&self, i: usize) -> Result<&'m [u8], ArrowError> { + self.check_index(i)?; + self.read_field_name(i) + } + + /// The value of field `i`. + pub fn field(&self, i: usize) -> Result, ArrowError> { + self.check_index(i)?; + self.read_field(i) + } + + /// The position of the field named `name`, if any. + /// + /// The spec requires fields to be sorted by name, so this is a binary search. + pub fn find(&self, name: &[u8]) -> Result, ArrowError> { + try_binary_search(self.len(), |i| Ok(self.read_field_name(i)?.cmp(name))) + } + + /// The value of the field named `name`, if any. + pub fn get(&self, name: &[u8]) -> Result>, ArrowError> { + self.find(name)?.map(|i| self.read_field(i)).transpose() + } + + /// The size of the encoded object. + /// + /// The spec makes the last field offset point one byte past the end of the last value. + pub fn byte_len(&self) -> Result { + let end = checked( + self.read_offset(self.len())? + .checked_add(self.first_value_byte), + "variant object size", + )?; + Ok(end as _) + } + + /// Checks the last field offset, and returns the bounded [`VariantObject`] that uses + /// `metadata`, the promoted form of this view's metadata dictionary. + #[inline] + pub fn into_object( + self, + metadata: VariantMetadata<'m>, + ) -> Result, ArrowError> { + VariantObject::try_from_view(self, metadata) + } + + #[inline] + fn check_index(&self, i: usize) -> Result<(), ArrowError> { + match i < self.len() { + true => Ok(()), + false => Err(index_error("Variant object field", i, self.len())), + } + } + + // The reads below take a field index in `0..self.len()`, which the caller has checked. + // `read_offset` also takes `self.len()`, for the end of the last value. + + #[inline] + pub(crate) fn read_field_id(&self, i: usize) -> Result { + self.header.field_id_size.unpack_u32_at_offset( + self.value, + self.header.field_ids_start_byte() as _, + i, + ) + } + + #[inline] + fn read_field_name(&self, i: usize) -> Result<&'m [u8], ArrowError> { + self.metadata.entry(self.read_field_id(i)? as _) + } + + #[inline] + fn read_offset(&self, i: usize) -> Result { + self.header.field_offset_size.unpack_u32_at_offset( + self.value, + self.first_field_offset_byte as _, + i, + ) + } + + #[inline] + pub(crate) fn read_field(&self, i: usize) -> Result, ArrowError> { + let start = checked( + self.read_offset(i)?.checked_add(self.first_value_byte), + "variant object field start", + )?; + VariantView::try_new( + self.metadata, + slice_from_slice(self.value, start as usize..)?, + ) + } +} + +/// A variant list of which only the header has been read. See [`VariantView`]. +#[derive(Debug, Clone, Copy)] +pub struct VariantListView<'m, 'v> { + pub(crate) metadata: VariantMetadataView<'m>, + pub(crate) value: &'v [u8], + pub(crate) header: VariantListHeader, + pub(crate) num_elements: u32, + pub(crate) first_value_byte: u32, +} + +impl<'m, 'v> VariantListView<'m, 'v> { + /// Reads the header and the element count of the list at the start of `value`. + #[inline] + pub fn try_new(metadata: VariantMetadataView<'m>, value: &'v [u8]) -> Result { + let header = VariantListHeader::try_new(first_byte_from_slice(value)?)?; + let num_elements = + header + .num_elements_size + .unpack_u32_at_offset(value, NUM_HEADER_BYTES as _, 0)?; + + // There are num_elements + 1 offsets, and the values immediately follow them. + let first_value_byte = checked( + num_elements + .checked_add(1) + .and_then(|n| n.checked_mul(header.offset_size())) + .and_then(|n| n.checked_add(header.first_offset_byte())), + "offset of variant list values", + )?; + + Ok(Self { + metadata, + value, + header, + num_elements, + first_value_byte, + }) + } + + /// The number of elements. + #[inline] + pub fn len(&self) -> usize { + self.num_elements as _ + } + + /// True if the list has no elements. + #[inline] + pub fn is_empty(&self) -> bool { + self.num_elements == 0 + } + + /// Element `index`, or `None` if `index` is out of bounds. + pub fn get(&self, index: usize) -> Result>, ArrowError> { + if index >= self.len() { + return Ok(None); + } + let start = checked( + self.read_offset(index)?.checked_add(self.first_value_byte), + "variant list element start", + )?; + VariantView::try_new( + self.metadata, + slice_from_slice(self.value, start as usize..)?, + ) + .map(Some) + } + + /// The size of the encoded list. + pub fn byte_len(&self) -> Result { + let end = checked( + self.read_offset(self.len())? + .checked_add(self.first_value_byte), + "variant array size", + )?; + Ok(end as _) + } + + /// Checks the first and last offsets, and returns the bounded [`VariantList`] that uses + /// `metadata`, the promoted form of this view's metadata dictionary. + #[inline] + pub fn into_list( + self, + metadata: VariantMetadata<'m>, + ) -> Result, ArrowError> { + VariantList::try_from_view(self, metadata) + } + + /// Entry `i` of the offset array, for `i` in `0..=self.len()`. + #[inline] + pub(crate) fn read_offset(&self, i: usize) -> Result { + self.header.offset_size.unpack_u32_at_offset( + self.value, + self.header.first_offset_byte() as _, + i, + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{VariantBuilder, VariantDecimal4, VariantDecimal8, VariantDecimal16, VariantPath}; + use chrono::{NaiveDate, NaiveTime}; + use uuid::Uuid; + + const LONG_STRING: &str = + "a string that is longer than the sixty-three bytes that a short string can hold"; + + /// `{"a": 1, "b": "short", "list": [1, "two", {"x": true}], "nested": {"k": LONG_STRING}}` + fn sample() -> (Vec, Vec) { + let mut builder = VariantBuilder::new(); + let mut object = builder.new_object(); + object.insert("a", 1i8); + object.insert("b", "short"); + let mut list = object.new_list("list"); + list.append_value(1i8); + list.append_value("two"); + list.new_object().with_field("x", true).finish(); + list.finish(); + object + .new_object("nested") + .with_field("k", LONG_STRING) + .finish(); + object.finish(); + builder.finish() + } + + fn view<'a>(metadata: &'a [u8], value: &'a [u8]) -> VariantView<'a, 'a> { + VariantView::try_new(VariantMetadataView::try_new(metadata).unwrap(), value).unwrap() + } + + fn metadata_with_names(names: &[&str]) -> Vec { + let mut builder = VariantBuilder::new().with_field_names(names.iter().copied()); + builder.append_value(1i8); + builder.finish().0 + } + + fn assert_metadata_view_matches(bytes: &[u8]) { + let metadata = VariantMetadata::try_new(bytes).unwrap(); + let view = VariantMetadataView::try_new(bytes).unwrap(); + assert_eq!(view.len(), metadata.len()); + assert_eq!(view.is_empty(), metadata.is_empty()); + assert_eq!(view.is_sorted(), metadata.is_sorted()); + for id in 0..metadata.len() { + let name = metadata.get(id).unwrap(); + assert_eq!(view.entry(id).unwrap(), name.as_bytes()); + assert_eq!(view.entry_str(id).unwrap(), name); + assert_eq!(view.find(name.as_bytes()).unwrap(), Some(id as u32)); + } + assert_eq!(view.find(b"no such name").unwrap(), None); + assert!(view.entry(view.len()).is_err()); + assert!(view.entry_str(view.len()).is_err()); + // Promotion gives the unvalidated form. + assert_eq!(view.into_metadata().unwrap(), VariantMetadata::new(bytes)); + } + + #[test] + fn test_metadata_view_sorted() { + // More than 10 entries, so `find` uses its binary search. + let names: Vec = (0..64).map(|i| format!("field_{i:03}")).collect(); + let names: Vec<&str> = names.iter().map(String::as_str).collect(); + let bytes = metadata_with_names(&names); + assert!(VariantMetadataView::try_new(&bytes).unwrap().is_sorted()); + assert_metadata_view_matches(&bytes); + } + + #[test] + fn test_metadata_view_unsorted() { + let bytes = metadata_with_names(&["zebra", "apple", "mango", "kiwi"]); + assert!(!VariantMetadataView::try_new(&bytes).unwrap().is_sorted()); + assert_metadata_view_matches(&bytes); + } + + #[test] + fn test_metadata_view_empty() { + let view = VariantMetadataView::try_new(crate::EMPTY_VARIANT_METADATA_BYTES).unwrap(); + assert!(view.is_empty()); + assert_eq!(view.find(b"a").unwrap(), None); + assert!(view.entry(0).is_err()); + } + + #[test] + fn test_metadata_view_entry_is_not_utf8_validated() { + // Dictionary with one entry, the invalid UTF-8 byte 0xFF. + let bytes = [0b0000_0001, 1, 0, 1, 0xFF]; + let view = VariantMetadataView::try_new(&bytes).unwrap(); + assert_eq!(view.entry(0).unwrap(), &[0xFF]); + assert_eq!(view.find(&[0xFF]).unwrap(), Some(0)); + assert!(view.entry_str(0).is_err()); + } + + #[test] + fn test_object_view_matches_object() { + let (metadata, value) = sample(); + let variant = Variant::try_new(&metadata, &value).unwrap(); + let object = variant.as_object().unwrap(); + let root = view(&metadata, &value); + let object_view = root.as_object().unwrap().unwrap(); + + assert_eq!(object_view.len(), object.len()); + assert!(!object_view.is_empty()); + for i in 0..object.len() { + let name = object.field_name(i).unwrap(); + assert_eq!(object_view.field_name(i).unwrap(), name.as_bytes()); + let field_id = object_view.field_id(i).unwrap(); + assert_eq!(root.metadata().entry_str(field_id as _).unwrap(), name); + assert_eq!( + object_view.field(i).unwrap().to_variant().unwrap(), + object.field(i).unwrap() + ); + assert_eq!(object_view.find(name.as_bytes()).unwrap(), Some(i)); + let found = object_view.get(name.as_bytes()).unwrap().unwrap(); + assert_eq!(found.to_variant().unwrap(), object.get(name).unwrap()); + } + + assert_eq!(object_view.find(b"missing").unwrap(), None); + assert!(object_view.get(b"missing").unwrap().is_none()); + assert!(object_view.field(object.len()).is_err()); + assert!(object_view.field_id(object.len()).is_err()); + assert!(object_view.field_name(object.len()).is_err()); + + assert_eq!(object_view.byte_len().unwrap(), value.len()); + let metadata = root.metadata().into_metadata().unwrap(); + assert_eq!(&object_view.into_object(metadata).unwrap(), object); + } + + #[test] + fn test_list_view_matches_list() { + let (metadata, value) = sample(); + let variant = Variant::try_new(&metadata, &value).unwrap(); + let variant = variant.get_object_field("list").unwrap(); + let list = variant.as_list().unwrap(); + let element = view(&metadata, &value).get_field(b"list").unwrap().unwrap(); + let list_view = element.as_list().unwrap().unwrap(); + + assert_eq!(list_view.len(), list.len()); + assert!(!list_view.is_empty()); + for i in 0..list.len() { + let found = list_view.get(i).unwrap().unwrap(); + assert_eq!(found.to_variant().unwrap(), list.get(i).unwrap()); + } + assert!(list_view.get(list.len()).unwrap().is_none()); + + assert_eq!( + list_view.byte_len().unwrap(), + element.bytes().unwrap().len() + ); + let metadata = element.metadata().into_metadata().unwrap(); + assert_eq!(&list_view.into_list(metadata).unwrap(), list); + } + + #[test] + fn test_value_view_types() { + let (metadata, value) = sample(); + let root = view(&metadata, &value); + assert_eq!(root.basic_type(), VariantBasicType::Object); + assert!(root.as_object().unwrap().is_some()); + assert!(root.as_list().unwrap().is_none()); + assert!(root.get_index(0).unwrap().is_none()); + + let list = root.get_field(b"list").unwrap().unwrap(); + assert_eq!(list.basic_type(), VariantBasicType::Array); + assert!(list.as_list().unwrap().is_some()); + assert!(list.as_object().unwrap().is_none()); + assert!(list.get_field(b"a").unwrap().is_none()); + + let primitive = root.get_field(b"a").unwrap().unwrap(); + assert_eq!(primitive.basic_type(), VariantBasicType::Primitive); + assert!(primitive.as_object().unwrap().is_none()); + assert!(primitive.as_list().unwrap().is_none()); + + let short_string = root.get_field(b"b").unwrap().unwrap(); + assert_eq!(short_string.basic_type(), VariantBasicType::ShortString); + } + + #[test] + fn test_get_path_matches_variant() { + let (metadata, value) = sample(); + let variant = Variant::try_new(&metadata, &value).unwrap(); + let root = view(&metadata, &value); + + for path in ["", "a", "list[1]", "list[2].x", "nested.k", "nested"] { + let path = VariantPath::try_from(path).unwrap(); + let found = root.get_path(&path).unwrap().unwrap(); + assert_eq!( + found.to_variant().unwrap(), + variant.get_path(&path).unwrap() + ); + } + for path in ["missing", "a.b", "a[0]", "list[3]", "list.x", "nested.k.z"] { + let path = VariantPath::try_from(path).unwrap(); + assert!(root.get_path(&path).unwrap().is_none(), "{path:?}"); + assert!(variant.get_path(&path).is_none(), "{path:?}"); + } + assert!( + root.get_path(&[VariantPathElement::ListElement]) + .unwrap() + .is_none() + ); + } + + /// The exact encodings of the children of a container must tile its value region. + #[test] + fn test_bytes_of_every_type() { + let date = NaiveDate::from_ymd_opt(2026, 11, 18).unwrap(); + let timestamp = date.and_hms_micro_opt(1, 2, 3, 456).unwrap(); + let values = [ + Variant::Null, + Variant::BooleanTrue, + Variant::BooleanFalse, + Variant::Int8(1), + Variant::Int16(2), + Variant::Int32(3), + Variant::Int64(4), + Variant::Float(5.0), + Variant::Double(6.0), + Variant::Decimal4(VariantDecimal4::try_new(7, 1).unwrap()), + Variant::Decimal8(VariantDecimal8::try_new(8, 1).unwrap()), + Variant::Decimal16(VariantDecimal16::try_new(9, 1).unwrap()), + Variant::Date(date), + Variant::TimestampMicros(timestamp.and_utc()), + Variant::TimestampNtzMicros(timestamp), + Variant::TimestampNanos(timestamp.and_utc()), + Variant::TimestampNtzNanos(timestamp), + Variant::Time(NaiveTime::from_hms_opt(1, 2, 3).unwrap()), + Variant::Uuid(Uuid::from_u128(10)), + Variant::Binary(&[1, 2, 3]), + Variant::from("short"), + Variant::from(LONG_STRING), + ]; + + let mut builder = VariantBuilder::new(); + let mut list = builder.new_list(); + for value in &values { + list.append_value(value.clone()); + } + list.new_object().with_field("x", 1i8).finish(); + list.new_list().with_value(1i8).finish(); + list.finish(); + let (metadata, value) = builder.finish(); + + let root = view(&metadata, &value); + assert_eq!(root.bytes().unwrap(), value.as_slice()); + let list = root.as_list().unwrap().unwrap(); + assert_eq!(list.len(), values.len() + 2); + + let mut total = 0; + for i in 0..list.len() { + let element = list.get(i).unwrap().unwrap(); + let bytes = element.bytes().unwrap(); + assert_eq!( + Variant::try_new(&metadata, bytes).unwrap(), + element.to_variant().unwrap() + ); + if let Some(expected) = values.get(i) { + assert_eq!(&element.to_variant().unwrap(), expected); + } + total += bytes.len(); + } + assert_eq!(total, value.len() - list.first_value_byte as usize); + } + + #[test] + fn test_bytes_of_object_fields() { + let (metadata, value) = sample(); + let object = view(&metadata, &value).as_object().unwrap().unwrap(); + let total: usize = (0..object.len()) + .map(|i| object.field(i).unwrap().bytes().unwrap().len()) + .sum(); + assert_eq!(total, value.len() - object.first_value_byte as usize); + } + + /// Reads everything reachable from `value` through the views, ignoring errors. + fn walk(value: VariantView<'_, '_>) { + // Malformed input may declare more children than its bytes can hold. + const MAX_CHILDREN: usize = 64; + + let _ = value.to_variant(); + let _ = value.bytes(); + let metadata = value.metadata(); + for id in 0..metadata.len().min(MAX_CHILDREN) { + let _ = metadata.entry(id); + let _ = metadata.entry_str(id); + } + let _ = metadata.find(b"k"); + let _ = metadata.into_metadata(); + + if let Ok(Some(object)) = value.as_object() { + let _ = object.byte_len(); + let _ = object.find(b"k"); + let _ = object.get(b"k"); + for i in 0..object.len().min(MAX_CHILDREN) { + let _ = object.field_id(i); + let _ = object.field_name(i); + if let Ok(field) = object.field(i) { + walk(field); + } + } + if let Ok(metadata) = metadata.into_metadata() { + let _ = object.into_object(metadata); + } + } + if let Ok(Some(list)) = value.as_list() { + let _ = list.byte_len(); + for i in 0..list.len().min(MAX_CHILDREN) { + if let Ok(Some(element)) = list.get(i) { + walk(element); + } + } + if let Ok(metadata) = metadata.into_metadata() { + let _ = list.into_list(metadata); + } + } + } + + fn walk_bytes(metadata: &[u8], value: &[u8]) { + let Ok(metadata) = VariantMetadataView::try_new(metadata) else { + return; + }; + if let Ok(value) = VariantView::try_new(metadata, value) { + walk(value); + } + } + + #[test] + fn test_truncated_input_does_not_panic() { + let (metadata, value) = sample(); + for len in 0..metadata.len() { + walk_bytes(&metadata[..len], &value); + } + for len in 0..value.len() { + walk_bytes(&metadata, &value[..len]); + } + } + + #[test] + fn test_corrupt_input_does_not_panic() { + let (metadata, value) = sample(); + for corrupt in [0x00, 0x7F, 0xFF] { + for i in 0..metadata.len() { + let mut metadata = metadata.clone(); + metadata[i] = corrupt; + walk_bytes(&metadata, &value); + } + for i in 0..value.len() { + let mut value = value.clone(); + value[i] = corrupt; + walk_bytes(&metadata, &value); + } + } + } +}