Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
133 changes: 71 additions & 62 deletions arrow-schema/src/extension/canonical/fixed_shape_tensor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,28 @@ impl FixedShapeTensor {
})
}

fn validate_impl<'a>(
data_type: &'a DataType,
metadata: &FixedShapeTensorMetadata,
) -> Result<&'a DataType, ArrowError> {
match data_type {
DataType::FixedSizeList(field, list_size) if !field.is_nullable() => {
metadata.validate()?;
let expected_size = i32::try_from(metadata.list_size()).expect("overflow");
if *list_size != expected_size {
Err(ArrowError::InvalidArgumentError(format!(
"FixedShapeTensor list size mismatch, expected {expected_size} (metadata), found {list_size} (data type)"
)))
} else {
Ok(field.data_type())
}
}
data_type => Err(ArrowError::InvalidArgumentError(format!(
"FixedShapeTensor data type mismatch, expected FixedSizeList with non-nullable field, found {data_type}"
))),
}
}

/// Returns the value type of the individual tensor elements.
pub fn value_type(&self) -> &DataType {
&self.value_type
Expand Down Expand Up @@ -321,45 +343,47 @@ impl FixedShapeTensorMetadata {
dimension_names: Option<Vec<String>>,
permutations: Option<Vec<usize>>,
) -> Result<Self, ArrowError> {
let shape = shape.into_iter().collect::<Vec<_>>();
let dimensions = shape.len();

let dim_names = dimension_names.map(|dimension_names| {
if dimension_names.len() != dimensions {
Err(ArrowError::InvalidArgumentError(format!(
"FixedShapeTensor dimension names size mismatch, expected {dimensions}, found {}", dimension_names.len()
)))
} else {
Ok(dimension_names)
}
}).transpose()?;
let metadata = Self {
shape: shape.into_iter().collect(),
dim_names: dimension_names,
permutations,
};
metadata.validate()?;
Ok(metadata)
}

let permutations = permutations
.map(|permutations| {
if permutations.len() != dimensions {
Err(ArrowError::InvalidArgumentError(format!(
"FixedShapeTensor permutations size mismatch, expected {dimensions}, found {}",
permutations.len()
)))
} else {
let mut sorted_permutations = permutations.clone();
sorted_permutations.sort_unstable();
if (0..dimensions).zip(sorted_permutations).any(|(a, b)| a != b) {
Err(ArrowError::InvalidArgumentError(format!(
"FixedShapeTensor permutations invalid, expected a permutation of [0, 1, .., N-1], where N is the number of dimensions: {dimensions}"
)))
} else {
Ok(permutations)
}
}
})
.transpose()?;
fn validate(&self) -> Result<(), ArrowError> {
let dimensions = self.dimensions();

Ok(Self {
shape,
dim_names,
permutations,
})
if let Some(dimension_names) = &self.dim_names
&& dimension_names.len() != dimensions
{
return Err(ArrowError::InvalidArgumentError(format!(
"FixedShapeTensor dimension names size mismatch, expected {dimensions}, found {}",
dimension_names.len()
)));
}

if let Some(permutations) = &self.permutations {
if permutations.len() != dimensions {
return Err(ArrowError::InvalidArgumentError(format!(
"FixedShapeTensor permutations size mismatch, expected {dimensions}, found {}",
permutations.len()
)));
}
let mut sorted_permutations = permutations.clone();
sorted_permutations.sort_unstable();
if (0..dimensions)
.zip(sorted_permutations)
.any(|(a, b)| a != b)
{
return Err(ArrowError::InvalidArgumentError(format!(
"FixedShapeTensor permutations invalid, expected a permutation of [0, 1, .., N-1], where N is the number of dimensions: {dimensions}"
)));
}
}

Ok(())
}

/// Returns the physical shape of the contained tensors.
Expand Down Expand Up @@ -447,31 +471,15 @@ impl ExtensionType for FixedShapeTensor {
}

fn try_new(data_type: &DataType, metadata: Self::Metadata) -> Result<Self, ArrowError> {
match data_type {
DataType::FixedSizeList(field, list_size) if !field.is_nullable() => {
// Make sure the metadata is valid.
let metadata = FixedShapeTensorMetadata::try_new(
metadata.shape,
metadata.dim_names,
metadata.permutations,
)?;
// Make sure it is compatible with this data type.
let expected_size = i32::try_from(metadata.list_size()).expect("overflow");
if *list_size != expected_size {
Err(ArrowError::InvalidArgumentError(format!(
"FixedShapeTensor list size mismatch, expected {expected_size} (metadata), found {list_size} (data type)"
)))
} else {
Ok(Self {
value_type: field.data_type().clone(),
metadata,
})
}
}
data_type => Err(ArrowError::InvalidArgumentError(format!(
"FixedShapeTensor data type mismatch, expected FixedSizeList with non-nullable field, found {data_type}"
))),
}
let value_type = Self::validate_impl(data_type, &metadata)?;
Ok(Self {
value_type: value_type.clone(),
metadata,
})
}

fn validate(data_type: &DataType, metadata: Self::Metadata) -> Result<(), ArrowError> {
Self::validate_impl(data_type, &metadata).map(|_| ())
}
}

Expand Down Expand Up @@ -501,6 +509,7 @@ mod tests {
false,
);
field.try_with_extension_type(fixed_shape_tensor.clone())?;
assert!(field.has_valid_extension_type::<FixedShapeTensor>());
assert_eq!(
field.try_extension_type::<FixedShapeTensor>()?,
fixed_shape_tensor
Expand Down
28 changes: 18 additions & 10 deletions arrow-schema/src/extension/canonical/json.rs
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,17 @@ use crate::{ArrowError, DataType, extension::ExtensionType};
#[derive(Debug, Clone, Default, PartialEq)]
pub struct Json(JsonMetadata);

impl Json {
fn validate_impl(data_type: &DataType) -> Result<(), ArrowError> {
match data_type {
DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => Ok(()),
data_type => Err(ArrowError::InvalidArgumentError(format!(
"Json data type mismatch, expected one of Utf8, LargeUtf8, Utf8View, found {data_type}"
))),
}
}
}

/// Empty object
#[derive(Debug, Clone, Copy, PartialEq)]
struct Empty {}
Expand Down Expand Up @@ -147,22 +158,16 @@ impl ExtensionType for Json {
}

fn supports_data_type(&self, data_type: &DataType) -> Result<(), ArrowError> {
match data_type {
DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => Ok(()),
data_type => Err(ArrowError::InvalidArgumentError(format!(
"Json data type mismatch, expected one of Utf8, LargeUtf8, Utf8View, found {data_type}"
))),
}
Self::validate_impl(data_type)
}

fn try_new(data_type: &DataType, metadata: Self::Metadata) -> Result<Self, ArrowError> {
let json = Self(metadata);
json.supports_data_type(data_type)?;
Ok(json)
Self::validate_impl(data_type)?;
Ok(Self(metadata))
}

fn validate(data_type: &DataType, _metadata: Self::Metadata) -> Result<(), ArrowError> {
Self::default().supports_data_type(data_type)
Self::validate_impl(data_type)
}
}

Expand All @@ -181,6 +186,7 @@ mod tests {
fn valid() -> Result<(), ArrowError> {
let mut field = Field::new("", DataType::Utf8, false);
field.try_with_extension_type(Json::default())?;
assert!(field.has_valid_extension_type::<Json>());
assert_eq!(
field.metadata().get(EXTENSION_TYPE_METADATA_KEY),
Some(&String::new())
Expand All @@ -192,6 +198,7 @@ mod tests {

let mut field = Field::new("", DataType::LargeUtf8, false);
field.try_with_extension_type(Json(JsonMetadata(Some(Empty {}))))?;
assert!(field.has_valid_extension_type::<Json>());
assert_eq!(
field.metadata().get(EXTENSION_TYPE_METADATA_KEY),
Some(&"{}".to_owned())
Expand All @@ -203,6 +210,7 @@ mod tests {

let mut field = Field::new("", DataType::Utf8View, false);
field.try_with_extension_type(Json::default())?;
assert!(field.has_valid_extension_type::<Json>());
field.try_extension_type::<Json>()?;
#[cfg(feature = "canonical_extension_types")]
assert_eq!(
Expand Down
Loading
Loading