Skip to content

Commit bccaa42

Browse files
Jefffreyneilconway
andauthored
[58_maintenance] fix(arrow-data): fix gaps in ListView equality (#11703)
- Backport #11345 - For inclusion in #11665 Cherry-pick of 862b1e0 Co-authored-by: Neil Conway <neil.conway@gmail.com>
1 parent 473171d commit bccaa42

3 files changed

Lines changed: 44 additions & 8 deletions

File tree

‎arrow-data/src/equal/list_view.rs‎

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717

1818
use crate::ArrayData;
1919
use crate::data::count_nulls;
20-
use crate::equal::equal_values;
20+
use crate::equal::equal_range;
2121
use arrow_buffer::ArrowNativeType;
2222
use num_integer::Integer;
2323

@@ -74,8 +74,7 @@ pub(super) fn list_view_equal<T: ArrowNativeType + Integer>(
7474
let rhs_offset = rhs_offset.to_usize().unwrap();
7575
let size = size.to_usize().unwrap();
7676

77-
// Check if offsets are valid for the given range
78-
if !equal_values(lhs_data, rhs_data, lhs_offset, rhs_offset, size) {
77+
if !equal_range(lhs_data, rhs_data, lhs_offset, rhs_offset, size) {
7978
return false;
8079
}
8180
}
@@ -101,10 +100,10 @@ pub(super) fn list_view_equal<T: ArrowNativeType + Integer>(
101100
return false;
102101
}
103102

104-
for (index, ((&lhs_offset, &rhs_offset), &size)) in lhs_range_offsets
103+
for (index, ((&lhs_offset, &rhs_offset), (&lhs_size, &rhs_size))) in lhs_range_offsets
105104
.iter()
106105
.zip(rhs_range_offsets)
107-
.zip(lhs_range_sizes)
106+
.zip(lhs_range_sizes.iter().zip(rhs_range_sizes))
108107
.enumerate()
109108
{
110109
let lhs_is_null = lhs_nulls.is_null(index);
@@ -114,12 +113,20 @@ pub(super) fn list_view_equal<T: ArrowNativeType + Integer>(
114113
return false;
115114
}
116115

116+
// Null rows are equal whatever their sizes and values
117+
if lhs_is_null {
118+
continue;
119+
}
120+
121+
if lhs_size != rhs_size {
122+
return false;
123+
}
124+
117125
let lhs_offset = lhs_offset.to_usize().unwrap();
118126
let rhs_offset = rhs_offset.to_usize().unwrap();
119-
let size = size.to_usize().unwrap();
127+
let size = lhs_size.to_usize().unwrap();
120128

121-
// Check if values match in the range
122-
if !lhs_is_null && !equal_values(lhs_data, rhs_data, lhs_offset, rhs_offset, size) {
129+
if !equal_range(lhs_data, rhs_data, lhs_offset, rhs_offset, size) {
123130
return false;
124131
}
125132
}

‎arrow-data/src/equal/mod.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,9 @@ use variable_size::variable_sized_equal;
5858

5959
/// Compares the values of two [ArrayData] starting at `lhs_start` and `rhs_start` respectively
6060
/// for `len` slots.
61+
///
62+
/// Assumes the null masks of the two ranges are already known to be equal. To compare
63+
/// ranges of child data, use [`equal_range`], which also compares the null masks.
6164
#[inline]
6265
fn equal_values(
6366
lhs: &ArrayData,

‎arrow/tests/array_equal.rs‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -822,6 +822,22 @@ fn test_test_list_view_array<T: OffsetSizeTrait>() {
822822
test_equal(&a, &b, true);
823823
}
824824

825+
fn test_list_view_sizes_and_child_nulls<T: OffsetSizeTrait>() {
826+
// Same first elements but different sizes, with a null row
827+
let a = create_list_view_array::<T, _, _>([Some(vec![Some(1), Some(2)]), None]);
828+
let b = create_list_view_array::<T, _, _>([Some(vec![Some(1), Some(2), Some(3)]), None]);
829+
test_equal(&a, &b, false);
830+
831+
// Child values that differ only in their null masks, without and with a null row
832+
let a = create_list_view_array::<T, _, _>([Some(vec![Some(1), Some(0)])]);
833+
let b = create_list_view_array::<T, _, _>([Some(vec![Some(1), None])]);
834+
test_equal(&a, &b, false);
835+
836+
let a = create_list_view_array::<T, _, _>([None, Some(vec![Some(1), Some(0)])]);
837+
let b = create_list_view_array::<T, _, _>([None, Some(vec![Some(1), None])]);
838+
test_equal(&a, &b, false);
839+
}
840+
825841
// Special test for List<ListView<i32>>.
826842
// This tests the equal_ranges kernel
827843
fn test_sliced_list_of_list_view<T: OffsetSizeTrait>() {
@@ -871,6 +887,16 @@ fn test_large_list_view_array() {
871887
test_test_list_view_array::<i64>();
872888
}
873889

890+
#[test]
891+
fn test_list_view_array_sizes_and_child_nulls() {
892+
test_list_view_sizes_and_child_nulls::<i32>();
893+
}
894+
895+
#[test]
896+
fn test_large_list_view_array_sizes_and_child_nulls() {
897+
test_list_view_sizes_and_child_nulls::<i64>();
898+
}
899+
874900
#[test]
875901
fn test_nested_list_view_array() {
876902
test_sliced_list_of_list_view::<i32>();

0 commit comments

Comments
 (0)