From 9f929734d29a75691e8e9567edcd30f4d6ff65c5 Mon Sep 17 00:00:00 2001 From: Jiawei Zhao Date: Mon, 16 Nov 2026 13:09:41 +0800 Subject: [PATCH] refactor: separate extension validation Some extension checks construct descriptors only to discard them. Keep validation independent of construction without changing the ExtensionType contract. Refs #9708 --- .../extension/canonical/fixed_shape_tensor.rs | 133 +++++++------ arrow-schema/src/extension/canonical/json.rs | 28 ++- .../canonical/variable_shape_tensor.rs | 176 +++++++++--------- parquet-geospatial/src/types.rs | 26 ++- 4 files changed, 197 insertions(+), 166 deletions(-) diff --git a/arrow-schema/src/extension/canonical/fixed_shape_tensor.rs b/arrow-schema/src/extension/canonical/fixed_shape_tensor.rs index 26bbc9ce5b78..9e0733052536 100644 --- a/arrow-schema/src/extension/canonical/fixed_shape_tensor.rs +++ b/arrow-schema/src/extension/canonical/fixed_shape_tensor.rs @@ -90,6 +90,28 @@ impl FixedShapeTensor { }) } + fn validate_impl<'a>( + data_type: &'a DataType, + metadata: &FixedShapeTensorMetadata, + ) -> Result<&'a DataType, ArrowError> { + match data_type { + DataType::FixedSizeList(field, list_size) if !field.is_nullable() => { + metadata.validate()?; + let expected_size = i32::try_from(metadata.list_size()).expect("overflow"); + if *list_size != expected_size { + Err(ArrowError::InvalidArgumentError(format!( + "FixedShapeTensor list size mismatch, expected {expected_size} (metadata), found {list_size} (data type)" + ))) + } else { + Ok(field.data_type()) + } + } + data_type => Err(ArrowError::InvalidArgumentError(format!( + "FixedShapeTensor data type mismatch, expected FixedSizeList with non-nullable field, found {data_type}" + ))), + } + } + /// Returns the value type of the individual tensor elements. pub fn value_type(&self) -> &DataType { &self.value_type @@ -321,45 +343,47 @@ impl FixedShapeTensorMetadata { dimension_names: Option>, permutations: Option>, ) -> Result { - let shape = shape.into_iter().collect::>(); - let dimensions = shape.len(); - - let dim_names = dimension_names.map(|dimension_names| { - if dimension_names.len() != dimensions { - Err(ArrowError::InvalidArgumentError(format!( - "FixedShapeTensor dimension names size mismatch, expected {dimensions}, found {}", dimension_names.len() - ))) - } else { - Ok(dimension_names) - } - }).transpose()?; + let metadata = Self { + shape: shape.into_iter().collect(), + dim_names: dimension_names, + permutations, + }; + metadata.validate()?; + Ok(metadata) + } - let permutations = permutations - .map(|permutations| { - if permutations.len() != dimensions { - Err(ArrowError::InvalidArgumentError(format!( - "FixedShapeTensor permutations size mismatch, expected {dimensions}, found {}", - permutations.len() - ))) - } else { - let mut sorted_permutations = permutations.clone(); - sorted_permutations.sort_unstable(); - if (0..dimensions).zip(sorted_permutations).any(|(a, b)| a != b) { - Err(ArrowError::InvalidArgumentError(format!( - "FixedShapeTensor permutations invalid, expected a permutation of [0, 1, .., N-1], where N is the number of dimensions: {dimensions}" - ))) - } else { - Ok(permutations) - } - } - }) - .transpose()?; + fn validate(&self) -> Result<(), ArrowError> { + let dimensions = self.dimensions(); - Ok(Self { - shape, - dim_names, - permutations, - }) + if let Some(dimension_names) = &self.dim_names + && dimension_names.len() != dimensions + { + return Err(ArrowError::InvalidArgumentError(format!( + "FixedShapeTensor dimension names size mismatch, expected {dimensions}, found {}", + dimension_names.len() + ))); + } + + if let Some(permutations) = &self.permutations { + if permutations.len() != dimensions { + return Err(ArrowError::InvalidArgumentError(format!( + "FixedShapeTensor permutations size mismatch, expected {dimensions}, found {}", + permutations.len() + ))); + } + let mut sorted_permutations = permutations.clone(); + sorted_permutations.sort_unstable(); + if (0..dimensions) + .zip(sorted_permutations) + .any(|(a, b)| a != b) + { + return Err(ArrowError::InvalidArgumentError(format!( + "FixedShapeTensor permutations invalid, expected a permutation of [0, 1, .., N-1], where N is the number of dimensions: {dimensions}" + ))); + } + } + + Ok(()) } /// Returns the physical shape of the contained tensors. @@ -447,31 +471,15 @@ impl ExtensionType for FixedShapeTensor { } fn try_new(data_type: &DataType, metadata: Self::Metadata) -> Result { - match data_type { - DataType::FixedSizeList(field, list_size) if !field.is_nullable() => { - // Make sure the metadata is valid. - let metadata = FixedShapeTensorMetadata::try_new( - metadata.shape, - metadata.dim_names, - metadata.permutations, - )?; - // Make sure it is compatible with this data type. - let expected_size = i32::try_from(metadata.list_size()).expect("overflow"); - if *list_size != expected_size { - Err(ArrowError::InvalidArgumentError(format!( - "FixedShapeTensor list size mismatch, expected {expected_size} (metadata), found {list_size} (data type)" - ))) - } else { - Ok(Self { - value_type: field.data_type().clone(), - metadata, - }) - } - } - data_type => Err(ArrowError::InvalidArgumentError(format!( - "FixedShapeTensor data type mismatch, expected FixedSizeList with non-nullable field, found {data_type}" - ))), - } + let value_type = Self::validate_impl(data_type, &metadata)?; + Ok(Self { + value_type: value_type.clone(), + metadata, + }) + } + + fn validate(data_type: &DataType, metadata: Self::Metadata) -> Result<(), ArrowError> { + Self::validate_impl(data_type, &metadata).map(|_| ()) } } @@ -501,6 +509,7 @@ mod tests { false, ); field.try_with_extension_type(fixed_shape_tensor.clone())?; + assert!(field.has_valid_extension_type::()); assert_eq!( field.try_extension_type::()?, fixed_shape_tensor diff --git a/arrow-schema/src/extension/canonical/json.rs b/arrow-schema/src/extension/canonical/json.rs index ccc2688d08f3..2c0e3a5a3248 100644 --- a/arrow-schema/src/extension/canonical/json.rs +++ b/arrow-schema/src/extension/canonical/json.rs @@ -31,6 +31,17 @@ use crate::{ArrowError, DataType, extension::ExtensionType}; #[derive(Debug, Clone, Default, PartialEq)] pub struct Json(JsonMetadata); +impl Json { + fn validate_impl(data_type: &DataType) -> Result<(), ArrowError> { + match data_type { + DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => Ok(()), + data_type => Err(ArrowError::InvalidArgumentError(format!( + "Json data type mismatch, expected one of Utf8, LargeUtf8, Utf8View, found {data_type}" + ))), + } + } +} + /// Empty object #[derive(Debug, Clone, Copy, PartialEq)] struct Empty {} @@ -147,22 +158,16 @@ impl ExtensionType for Json { } fn supports_data_type(&self, data_type: &DataType) -> Result<(), ArrowError> { - match data_type { - DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => Ok(()), - data_type => Err(ArrowError::InvalidArgumentError(format!( - "Json data type mismatch, expected one of Utf8, LargeUtf8, Utf8View, found {data_type}" - ))), - } + Self::validate_impl(data_type) } fn try_new(data_type: &DataType, metadata: Self::Metadata) -> Result { - let json = Self(metadata); - json.supports_data_type(data_type)?; - Ok(json) + Self::validate_impl(data_type)?; + Ok(Self(metadata)) } fn validate(data_type: &DataType, _metadata: Self::Metadata) -> Result<(), ArrowError> { - Self::default().supports_data_type(data_type) + Self::validate_impl(data_type) } } @@ -181,6 +186,7 @@ mod tests { fn valid() -> Result<(), ArrowError> { let mut field = Field::new("", DataType::Utf8, false); field.try_with_extension_type(Json::default())?; + assert!(field.has_valid_extension_type::()); assert_eq!( field.metadata().get(EXTENSION_TYPE_METADATA_KEY), Some(&String::new()) @@ -192,6 +198,7 @@ mod tests { let mut field = Field::new("", DataType::LargeUtf8, false); field.try_with_extension_type(Json(JsonMetadata(Some(Empty {}))))?; + assert!(field.has_valid_extension_type::()); assert_eq!( field.metadata().get(EXTENSION_TYPE_METADATA_KEY), Some(&"{}".to_owned()) @@ -203,6 +210,7 @@ mod tests { let mut field = Field::new("", DataType::Utf8View, false); field.try_with_extension_type(Json::default())?; + assert!(field.has_valid_extension_type::()); field.try_extension_type::()?; #[cfg(feature = "canonical_extension_types")] assert_eq!( diff --git a/arrow-schema/src/extension/canonical/variable_shape_tensor.rs b/arrow-schema/src/extension/canonical/variable_shape_tensor.rs index bd6b7a49808a..8bbb64f2b205 100644 --- a/arrow-schema/src/extension/canonical/variable_shape_tensor.rs +++ b/arrow-schema/src/extension/canonical/variable_shape_tensor.rs @@ -98,6 +98,40 @@ impl VariableShapeTensor { }) } + fn validate_impl<'a>( + data_type: &'a DataType, + metadata: &VariableShapeTensorMetadata, + ) -> Result<(&'a DataType, usize), ArrowError> { + match data_type { + DataType::Struct(fields) + if fields.len() == 2 + && matches!(fields.find("data"), Some((0, _))) + && matches!(fields.find("shape"), Some((1, _))) => + { + let shape_field = &fields[1]; + match shape_field.data_type() { + DataType::FixedSizeList(_, list_size) => { + let dimensions = usize::try_from(*list_size).expect("conversion failed"); + metadata.validate(dimensions)?; + let data_field = &fields[0]; + match data_field.data_type() { + DataType::List(field) => Ok((field.data_type(), dimensions)), + data_type => Err(ArrowError::InvalidArgumentError(format!( + "VariableShapeTensor data type mismatch, expected List for data field, found {data_type}" + ))), + } + } + data_type => Err(ArrowError::InvalidArgumentError(format!( + "VariableShapeTensor data type mismatch, expected FixedSizeList for shape field, found {data_type}" + ))), + } + } + data_type => Err(ArrowError::InvalidArgumentError(format!( + "VariableShapeTensor data type mismatch, expected Struct with 2 fields (data and shape), found {data_type}" + ))), + } + } + /// Returns the value type of the individual tensor elements. pub fn value_type(&self) -> &DataType { &self.value_type @@ -291,55 +325,54 @@ impl VariableShapeTensorMetadata { permutations: Option>, uniform_shapes: Option>>, ) -> Result { - let dim_names = dimension_names.map(|dimension_names| { - if dimension_names.len() != dimensions { - Err(ArrowError::InvalidArgumentError(format!( - "VariableShapeTensor dimension names size mismatch, expected {dimensions}, found {}", dimension_names.len() - ))) - } else { - Ok(dimension_names) + let metadata = Self { + dim_names: dimension_names, + permutations, + uniform_shape: uniform_shapes, + }; + metadata.validate(dimensions)?; + Ok(metadata) + } + + fn validate(&self, dimensions: usize) -> Result<(), ArrowError> { + if let Some(dimension_names) = &self.dim_names + && dimension_names.len() != dimensions + { + return Err(ArrowError::InvalidArgumentError(format!( + "VariableShapeTensor dimension names size mismatch, expected {dimensions}, found {}", + dimension_names.len() + ))); + } + + if let Some(permutations) = &self.permutations { + if permutations.len() != dimensions { + return Err(ArrowError::InvalidArgumentError(format!( + "VariableShapeTensor permutations size mismatch, expected {dimensions}, found {}", + permutations.len() + ))); } - }).transpose()?; - - let permutations = permutations - .map(|permutations| { - if permutations.len() != dimensions { - Err(ArrowError::InvalidArgumentError(format!( - "VariableShapeTensor permutations size mismatch, expected {dimensions}, found {}", - permutations.len() - ))) - } else { - let mut sorted_permutations = permutations.clone(); - sorted_permutations.sort_unstable(); - if (0..dimensions).zip(sorted_permutations).any(|(a, b)| a != b) { - Err(ArrowError::InvalidArgumentError(format!( - "VariableShapeTensor permutations invalid, expected a permutation of [0, 1, .., N-1], where N is the number of dimensions: {dimensions}" - ))) - } else { - Ok(permutations) - } - } - }) - .transpose()?; - - let uniform_shape = uniform_shapes - .map(|uniform_shapes| { - if uniform_shapes.len() != dimensions { - Err(ArrowError::InvalidArgumentError(format!( - "VariableShapeTensor uniform shapes size mismatch, expected {dimensions}, found {}", - uniform_shapes.len() - ))) - } else { - Ok(uniform_shapes) - } - }) - .transpose()?; + let mut sorted_permutations = permutations.clone(); + sorted_permutations.sort_unstable(); + if (0..dimensions) + .zip(sorted_permutations) + .any(|(a, b)| a != b) + { + return Err(ArrowError::InvalidArgumentError(format!( + "VariableShapeTensor permutations invalid, expected a permutation of [0, 1, .., N-1], where N is the number of dimensions: {dimensions}" + ))); + } + } - Ok(Self { - dim_names, - permutations, - uniform_shape, - }) + if let Some(uniform_shapes) = &self.uniform_shape + && uniform_shapes.len() != dimensions + { + return Err(ArrowError::InvalidArgumentError(format!( + "VariableShapeTensor uniform shapes size mismatch, expected {dimensions}, found {}", + uniform_shapes.len() + ))); + } + + Ok(()) } /// Returns the names of the dimensions in this variable shape tensor, if @@ -424,44 +457,16 @@ impl ExtensionType for VariableShapeTensor { } fn try_new(data_type: &DataType, metadata: Self::Metadata) -> Result { - match data_type { - DataType::Struct(fields) - if fields.len() == 2 - && matches!(fields.find("data"), Some((0, _))) - && matches!(fields.find("shape"), Some((1, _))) => - { - let shape_field = &fields[1]; - match shape_field.data_type() { - DataType::FixedSizeList(_, list_size) => { - let dimensions = usize::try_from(*list_size).expect("conversion failed"); - // Make sure the metadata is valid. - let metadata = VariableShapeTensorMetadata::try_new( - dimensions, - metadata.dim_names, - metadata.permutations, - metadata.uniform_shape, - )?; - let data_field = &fields[0]; - match data_field.data_type() { - DataType::List(field) => Ok(Self { - value_type: field.data_type().clone(), - dimensions, - metadata, - }), - data_type => Err(ArrowError::InvalidArgumentError(format!( - "VariableShapeTensor data type mismatch, expected List for data field, found {data_type}" - ))), - } - } - data_type => Err(ArrowError::InvalidArgumentError(format!( - "VariableShapeTensor data type mismatch, expected FixedSizeList for shape field, found {data_type}" - ))), - } - } - data_type => Err(ArrowError::InvalidArgumentError(format!( - "VariableShapeTensor data type mismatch, expected Struct with 2 fields (data and shape), found {data_type}" - ))), - } + let (value_type, dimensions) = Self::validate_impl(data_type, &metadata)?; + Ok(Self { + value_type: value_type.clone(), + dimensions, + metadata, + }) + } + + fn validate(data_type: &DataType, metadata: Self::Metadata) -> Result<(), ArrowError> { + Self::validate_impl(data_type, &metadata).map(|_| ()) } } @@ -503,6 +508,7 @@ mod tests { false, ); field.try_with_extension_type(variable_shape_tensor.clone())?; + assert!(field.has_valid_extension_type::()); assert_eq!( field.try_extension_type::()?, variable_shape_tensor diff --git a/parquet-geospatial/src/types.rs b/parquet-geospatial/src/types.rs index 287a2caf821f..04ccc33d804c 100644 --- a/parquet-geospatial/src/types.rs +++ b/parquet-geospatial/src/types.rs @@ -118,6 +118,15 @@ impl WkbType { pub fn new(metadata: Option) -> Self { Self(metadata.unwrap_or_default()) } + + fn validate_impl(data_type: &DataType) -> ArrowResult<()> { + match data_type { + DataType::Binary | DataType::LargeBinary | DataType::BinaryView => Ok(()), + dt => Err(ArrowError::InvalidArgumentError(format!( + "Geometry data type mismatch, expected one of Binary, LargeBinary, BinaryView. Found {dt}" + ))), + } + } } type ArrowResult = Result; @@ -143,18 +152,16 @@ impl ExtensionType for WkbType { } fn supports_data_type(&self, data_type: &arrow_schema::DataType) -> ArrowResult<()> { - match data_type { - DataType::Binary | DataType::LargeBinary | DataType::BinaryView => Ok(()), - dt => Err(ArrowError::InvalidArgumentError(format!( - "Geometry data type mismatch, expected one of Binary, LargeBinary, BinaryView. Found {dt}" - ))), - } + Self::validate_impl(data_type) } fn try_new(data_type: &arrow_schema::DataType, metadata: Self::Metadata) -> ArrowResult { - let wkb = Self(metadata); - wkb.supports_data_type(data_type)?; - Ok(wkb) + Self::validate_impl(data_type)?; + Ok(Self(metadata)) + } + + fn validate(data_type: &DataType, _metadata: Self::Metadata) -> ArrowResult<()> { + Self::validate_impl(data_type) } } @@ -304,6 +311,7 @@ mod tests { let mut field = Field::new("geometry", DataType::Binary, false); field.try_with_extension_type(wkb_type)?; + assert!(field.has_valid_extension_type::()); // Verify we can extract the extension type back let extracted = field.try_extension_type::()?;