diff --git a/arrow-arith/src/numeric.rs b/arrow-arith/src/numeric.rs index 6238a6a7caf1..5bd0616c24bd 100644 --- a/arrow-arith/src/numeric.rs +++ b/arrow-arith/src/numeric.rs @@ -5,6 +5,7 @@ //! Defines numeric arithmetic kernels on [`PrimitiveArray`], such as [`add`] use std::cmp::Ordering; +use std::collections::TryReserveError; use std::convert::Infallible; use std::fmt::Formatter; use std::sync::Arc; @@ -15,8 +16,8 @@ use arrow_array::timezone::Tz; use arrow_array::types::*; use arrow_array::*; use arrow_buffer::{ - ArrowNativeType, DivisorI32, DivisorI64, DivisorU32, DivisorU64, IntervalDayTime, - IntervalMonthDayNano, + ArrowNativeType, BooleanBuffer, DivisorI32, DivisorI64, DivisorU32, DivisorU64, + IntervalDayTime, IntervalMonthDayNano, NullBuffer, i256, }; use arrow_schema::{ArrowError, DataType, IntervalUnit, TimeUnit}; use num_traits::{CheckedMul, ToPrimitive}; @@ -327,6 +328,313 @@ macro_rules! try_op_ref { }}; } +/// Wrapping `+`, `-` and `*` that also report whether they overflowed, written +/// so that LLVM vectorizes a loop over them. +/// +/// The signed `overflowing_add` and `overflowing_sub` lower to +/// `llvm.sadd.with.overflow` and `llvm.ssub.with.overflow`, which LLVM doesn't +/// vectorize, so the signed impls test the sign bits instead. +trait OverflowingOp: Sized { + /// Whether the checked kernels use the one-pass path for this type. When + /// it's false, [`overflowing_op`] returns `None` without doing any work. + const ONE_PASS: bool = true; + + fn add_overflowing(self, rhs: Self) -> (Self, bool); + fn sub_overflowing(self, rhs: Self) -> (Self, bool); + fn mul_overflowing(self, rhs: Self) -> (Self, bool); +} + +macro_rules! signed_overflowing_op { + ($($t:ty $(=> $wide:ty)?),*) => {$( + impl OverflowingOp for $t { + #[inline] + fn add_overflowing(self, rhs: Self) -> (Self, bool) { + let sum = self.wrapping_add(rhs); + // Overflow iff both operands have the same sign and `sum` has the other. + (sum, (self ^ sum) & (rhs ^ sum) < 0) + } + + #[inline] + fn sub_overflowing(self, rhs: Self) -> (Self, bool) { + let difference = self.wrapping_sub(rhs); + // Overflow iff the operands have different signs and `difference` + // doesn't have the sign of `self`. + (difference, (self ^ rhs) & (self ^ difference) < 0) + } + + #[inline] + fn mul_overflowing(self, rhs: Self) -> (Self, bool) { + signed_mul_overflowing!(self, rhs, $t $(, $wide)?) + } + } + )*}; +} + +/// The wrapping product of two signed integers and whether it overflowed. +macro_rules! signed_mul_overflowing { + // The truncating cast is the wrapping product. It overflowed iff widening + // it back doesn't give the full product. + ($l:expr, $r:expr, $t:ty, $wide:ty) => {{ + let product = <$wide>::from($l) * <$wide>::from($r); + (product as $t, <$wide>::from(product as $t) != product) + }}; + // Widened to `i128`, an `i64` multiply stays a scalar `imul` on x86-64, and + // LLVM packs each row's overflow flag into vector registers around it, + // which made the loop slower than the checked kernel with + // `-C target-cpu=x86-64-v3`. + ($l:expr, $r:expr, $t:ty) => { + $l.overflowing_mul($r) + }; +} + +signed_overflowing_op!(i8 => i16, i16 => i32, i32 => i64, i64); + +macro_rules! unsigned_overflowing_op { + ($($t:ty),*) => {$( + // LLVM vectorizes the unsigned `overflowing_add` and `overflowing_sub`. + // It folds a widening unsigned multiply back into `llvm.umul.with.overflow`, + // which it doesn't vectorize, so `mul` stays scalar either way. + impl OverflowingOp for $t { + #[inline] + fn add_overflowing(self, rhs: Self) -> (Self, bool) { + self.overflowing_add(rhs) + } + + #[inline] + fn sub_overflowing(self, rhs: Self) -> (Self, bool) { + self.overflowing_sub(rhs) + } + + #[inline] + fn mul_overflowing(self, rhs: Self) -> (Self, bool) { + self.overflowing_mul(rhs) + } + } + )*}; +} + +unsigned_overflowing_op!(u8, u16, u32, u64); + +/// Applies `op` to every slot of `l` and `r`, including null slots, without +/// branching on overflow. +/// +/// Returns `Ok(None)` if a valid row overflowed or the array lengths differ, +/// so the caller can fall back to the checked kernel to report the error. +fn overflowing_op( + l: &PrimitiveArray, + l_s: bool, + r: &PrimitiveArray, + r_s: bool, + op: F, +) -> Result>, ArrowError> +where + T: ArrowPrimitiveType, + T::Native: OverflowingOp, + F: Fn(T::Native, T::Native) -> (T::Native, bool), +{ + if !T::Native::ONE_PASS { + return Ok(None); + } + match (l_s, r_s) { + (true, true) | (false, false) => { + if l.len() != r.len() { + return Ok(None); + } + let pairs = l.values().iter().zip(r.values()); + let values = collect_overflowing(pairs.map(|(&l, &r)| op(l, r)))?; + let nulls = NullBuffer::union(l.nulls(), r.nulls()); + let (l, r) = (l.values(), r.values()); + Ok(unless_valid_overflow(values, nulls, |i| op(l[i], r[i]).1)) + } + (true, false) => match (l.null_count() == 0).then(|| l.value(0)) { + None => Ok(Some(PrimitiveArray::new_null(r.len()))), + Some(l) => { + let values = collect_overflowing(r.values().iter().map(|&r| op(l, r)))?; + let (nulls, r) = (r.nulls().cloned(), r.values()); + Ok(unless_valid_overflow(values, nulls, |i| op(l, r[i]).1)) + } + }, + (false, true) => match (r.null_count() == 0).then(|| r.value(0)) { + None => Ok(Some(PrimitiveArray::new_null(l.len()))), + Some(r) => { + let values = collect_overflowing(l.values().iter().map(|&l| op(l, r)))?; + let (nulls, l) = (l.nulls().cloned(), l.values()); + Ok(unless_valid_overflow(values, nulls, |i| op(l[i], r).1)) + } + }, + } +} + +/// Builds the array from the output of [`collect_overflowing`], or returns +/// `None` if a valid row overflowed. +/// +/// A null slot can hold any value, such as the 0 that builders write, which +/// overflows unsigned `0 - 1`. So when a slot overflowed, this recomputes the +/// overflow of every row with `overflowed` and masks it with the validity +/// bits, instead of making the caller rerun the whole batch. +fn unless_valid_overflow( + (values, overflow): (Vec, bool), + nulls: Option, + overflowed: impl Fn(usize) -> bool, +) -> Option> { + let valid_overflow = overflow + && nulls + .as_ref() + .is_none_or(|n| any_valid(n, &BooleanBuffer::collect_bool(n.len(), overflowed))); + (!valid_overflow).then(|| PrimitiveArray::new(values.into(), nulls)) +} + +/// Whether any valid row of `nulls` is set in `overflowed` +fn any_valid(nulls: &NullBuffer, overflowed: &BooleanBuffer) -> bool { + let valid = nulls.inner().bit_chunks().iter_padded(); + valid + .zip(overflowed.bit_chunks().iter_padded()) + .any(|(v, o)| v & o != 0) +} + +/// Converts an allocation failure in [`collect_overflowing`]. +/// +/// One shared, out-of-line copy keeps the formatting code out of every +/// instance of [`collect_overflowing`], as [`mul_checked_cold`] does for the +/// decimal `div` and `rem` closures. +#[cold] +#[inline(never)] +fn memory_error(e: TryReserveError) -> ArrowError { + ArrowError::MemoryError(e.to_string()) +} + +/// Collects the values of `iter`, and whether any was flagged as overflowed. +/// +/// This intentional `inline(never)` keeps `overflow` in a register. Inlined +/// into its caller under thin LTO, LLVM keeps the flag in memory and loads +/// and stores it on every row, which serializes the loop and keeps it from +/// vectorizing. The loop writes into the `Vec`'s spare capacity itself, +/// because whether LLVM inlines the loop inside `collect` depends on the +/// closure and the target, and when it doesn't, that loop reaches the flag +/// through a pointer. The checked `add` cases of the `arithmetic_kernels` +/// benchmark measure this. +#[inline(never)] +fn collect_overflowing( + iter: impl ExactSizeIterator, +) -> Result<(Vec, bool), ArrowError> { + let mut values = Vec::new(); + values.try_reserve_exact(iter.len()).map_err(memory_error)?; + let mut overflow = false; + let mut written = 0; + for (slot, (value, overflowed)) in values.spare_capacity_mut().iter_mut().zip(iter) { + slot.write(value); + overflow |= overflowed; + written += 1; + } + debug_assert!(written <= values.capacity()); + // SAFETY: the loop initialized the first `written` slots of the spare + // capacity, and `written` counts those writes, so it doesn't rely on + // `iter.len()`. + unsafe { values.set_len(written) }; + Ok((values, overflow)) +} + +// No vector lanes are 128 or 256 bits wide, so loops over these types stay +// scalar. Their `overflowing_*` methods don't branch on overflow, which still +// removes the early return of the checked kernel. +impl OverflowingOp for i128 { + #[inline] + fn add_overflowing(self, rhs: Self) -> (Self, bool) { + self.overflowing_add(rhs) + } + + #[inline] + fn sub_overflowing(self, rhs: Self) -> (Self, bool) { + self.overflowing_sub(rhs) + } + + #[inline] + fn mul_overflowing(self, rhs: Self) -> (Self, bool) { + // Decimal values usually fit in 64 bits, and the product of two `i64` + // can't overflow `i128`, so this skips the full 128-bit multiply. + match (i64::try_from(self), i64::try_from(rhs)) { + (Ok(l), Ok(r)) => (i128::from(l) * i128::from(r), false), + _ => self.overflowing_mul(rhs), + } + } +} + +// On `x86-64-v3`, LLVM vectorizes the one-pass `i256` add loop by packing the +// carries into vector registers, which is slower than the checked kernel. +impl OverflowingOp for i256 { + const ONE_PASS: bool = false; + + #[inline] + fn add_overflowing(self, rhs: Self) -> (Self, bool) { + self.overflowing_add(rhs) + } + + #[inline] + fn sub_overflowing(self, rhs: Self) -> (Self, bool) { + self.overflowing_sub(rhs) + } + + #[inline] + fn mul_overflowing(self, rhs: Self) -> (Self, bool) { + // `i256` has no `overflowing_mul`. + match self.checked_mul(rhs) { + Some(product) => (product, false), + None => (self.wrapping_mul(rhs), true), + } + } +} + +/// Multiplies `l` by `mul`, then applies `op`, reporting whether either step +/// overflowed. With unequal scales only the operand with the smaller scale is +/// multiplied, because the other multiplier is one. +#[inline] +fn rescale_left(l: N, mul: N, r: N, op: impl Fn(N, N) -> (N, bool)) -> (N, bool) { + let (l, l_overflow) = l.mul_overflowing(mul); + let (value, overflow) = op(l, r); + (value, l_overflow | overflow) +} + +/// Multiplies the value of the decimal scalar `array` by `mul`, so that the +/// kernel doesn't rescale it on every row. +/// +/// Returns `None` if `array` isn't a valid scalar, `mul` is one, the product +/// overflows, or its allocation fails. The kernel then rescales the scalar on +/// every row, so that an overflow is an error only if a valid row needs the +/// value, and an allocation failure reaches the kernel's fallible reservation. +fn rescale_scalar( + array: &PrimitiveArray, + is_scalar: bool, + mul: T::Native, +) -> Option> +where + T::Native: OverflowingOp, +{ + if !is_scalar || mul.is_eq(T::Native::ONE) || array.is_null(0) { + return None; + } + let (value, overflow) = array.value(0).mul_overflowing(mul); + if overflow { + return None; + } + let mut values = Vec::new(); + values.try_reserve_exact(1).ok()?; + values.push(value); + Some(PrimitiveArray::new(values.into(), None)) +} + +/// Like [`rescale_left`], but multiplies `r` by `mul`. +#[inline] +fn rescale_right( + l: N, + r: N, + mul: N, + op: impl Fn(N, N) -> (N, bool), +) -> (N, bool) { + let (r, r_overflow) = r.mul_overflowing(mul); + let (value, overflow) = op(l, r); + (value, r_overflow | overflow) +} + /// Defines `$fn`, which computes `/` or `%` of an array by a scalar with a precomputed divisor. /// It returns `None` for an op other than `/` and `%`, when the scalar is null, or when a value /// could fail to divide by it (zero, or `$fallible` for `/`), so that `integer_op` returns an @@ -1164,7 +1472,7 @@ fn decimal_op( r_s: bool, ) -> Result where - T::Native: CheckedMul, + T::Native: CheckedMul + OverflowingOp, { let l = l.as_primitive::(); let r = r.as_primitive::(); @@ -1192,31 +1500,72 @@ where let l_mul = T::Native::usize_as(10).pow_checked((result_scale - s1) as _)?; let r_mul = T::Native::usize_as(10).pow_checked((result_scale - s2) as _)?; + // A scalar with the smaller scale is rescaled once here, rather than + // on every row, which makes its multiplier one. + let (l_rescaled, r_rescaled) = + (rescale_scalar(l, l_s, l_mul), rescale_scalar(r, r_s, r_mul)); + let one = T::Native::ONE; + let (l, l_mul) = l_rescaled.as_ref().map_or((l, l_mul), |l| (l, one)); + let (r, r_mul) = r_rescaled.as_ref().map_or((r, r_mul), |r| (r, one)); + let unscaled = l_mul.is_eq(one) && r_mul.is_eq(one); + match op { - // Equal scales make both decimal multipliers one. - Op::Add | Op::AddWrapping if s1 == s2 => { - try_op!(l, l_s, r, r_s, l.add_checked(r)) + Op::Add | Op::AddWrapping if unscaled => { + match overflowing_op(l, l_s, r, r_s, T::Native::add_overflowing)? { + Some(array) => array, + None => try_op!(l, l_s, r, r_s, l.add_checked(r)), + } } - Op::Sub | Op::SubWrapping if s1 == s2 => { - try_op!(l, l_s, r, r_s, l.sub_checked(r)) + Op::Sub | Op::SubWrapping if unscaled => { + match overflowing_op(l, l_s, r, r_s, T::Native::sub_overflowing)? { + Some(array) => array, + None => try_op!(l, l_s, r, r_s, l.sub_checked(r)), + } } + // `move` copies the multipliers into the closures. Borrowed, they + // keep the loop from vectorizing under thin LTO, like the borrowed + // captures #11525 fixed in `unary`. Op::Add | Op::AddWrapping => { - try_op!( - l, - l_s, - r, - r_s, - l.mul_checked(l_mul)?.add_checked(r.mul_checked(r_mul)?) - ) + let add = T::Native::add_overflowing; + let array = match s1 < s2 { + true => overflowing_op(l, l_s, r, r_s, move |l, r| { + rescale_left(l, l_mul, r, add) + })?, + false => overflowing_op(l, l_s, r, r_s, move |l, r| { + rescale_right(l, r, r_mul, add) + })?, + }; + match array { + Some(array) => array, + None => try_op!( + l, + l_s, + r, + r_s, + l.mul_checked(l_mul)?.add_checked(r.mul_checked(r_mul)?) + ), + } } Op::Sub | Op::SubWrapping => { - try_op!( - l, - l_s, - r, - r_s, - l.mul_checked(l_mul)?.sub_checked(r.mul_checked(r_mul)?) - ) + let sub = T::Native::sub_overflowing; + let array = match s1 < s2 { + true => overflowing_op(l, l_s, r, r_s, move |l, r| { + rescale_left(l, l_mul, r, sub) + })?, + false => overflowing_op(l, l_s, r, r_s, move |l, r| { + rescale_right(l, r, r_mul, sub) + })?, + }; + match array { + Some(array) => array, + None => try_op!( + l, + l_s, + r, + r_s, + l.mul_checked(l_mul)?.sub_checked(r.mul_checked(r_mul)?) + ), + } } _ => unreachable!(), } @@ -1244,8 +1593,11 @@ where )) })?; - try_op!(l, l_s, r, r_s, l.mul_checked(r)) - .with_precision_and_scale(result_precision, result_scale)? + match overflowing_op(l, l_s, r, r_s, T::Native::mul_overflowing)? { + Some(array) => array, + None => try_op!(l, l_s, r, r_s, l.mul_checked(r)), + } + .with_precision_and_scale(result_precision, result_scale)? } Op::Div => { @@ -1812,6 +2164,335 @@ mod tests { assert_eq!(err, "Divide by zero error"); } + type Kernel = fn(&dyn Datum, &dyn Datum) -> Result; + type Checked = fn(N, N) -> Result; + + fn assert_decimal_result( + actual: Result, + expected: Result>, ArrowError>, + ) { + match (actual, expected) { + (Ok(actual), Ok(expected)) => { + let actual: Vec<_> = actual.as_primitive::().iter().collect(); + assert_eq!(actual, expected) + } + (Err(actual), Err(expected)) => assert_eq!(actual.to_string(), expected.to_string()), + (actual, expected) => panic!("got {actual:?}, expected {expected:?}"), + } + } + + /// Checks `kernel` against `checked` for every pair of `edges`, with + /// arrays on both sides, with each pair as a one-row slice, and with a + /// scalar on either side. Every third row of the left array is null, so + /// null slots also hold values that overflow. + fn test_checked_decimal_op( + edges: &[T::Native], + (l_scale, r_scale): (i8, i8), + kernel: Kernel, + checked: impl Fn(T::Native, T::Native) -> Result, + ) { + let decimal = |array: PrimitiveArray, scale| { + array + .with_precision_and_scale(T::MAX_PRECISION, scale) + .unwrap() + }; + let (l, r): (Vec<_>, Vec<_>) = edges + .iter() + .flat_map(|&x| edges.iter().map(move |&y| (x, y))) + .unzip(); + let nulls = NullBuffer::from_iter((0..l.len()).map(|i| i % 3 != 2)); + let l_array = decimal( + PrimitiveArray::new(l.clone().into(), Some(nulls.clone())), + l_scale, + ); + let r_array = decimal(PrimitiveArray::from_iter_values(r.iter().copied()), r_scale); + let row = |i: usize, x, y| match nulls.is_valid(i) { + true => checked(x, y).map(Some), + false => Ok(None), + }; + + // The kernels report the first valid row that overflows. + let expected = (0..l.len()).map(|i| row(i, l[i], r[i])).collect(); + assert_decimal_result::(kernel(&l_array, &r_array), expected); + + // An overflow in one row sends the whole array to the checked kernel, + // which would hide a missed overflow in another row, so also check each + // row alone. + for i in 0..l.len() { + let actual = kernel(&l_array.slice(i, 1), &r_array.slice(i, 1)); + assert_decimal_result::(actual, row(i, l[i], r[i]).map(|v| vec![v])); + } + + for &x in edges { + let l_scalar = Scalar::new(decimal(PrimitiveArray::from_iter_values([x]), l_scale)); + let expected = r.iter().map(|&y| checked(x, y).map(Some)).collect(); + assert_decimal_result::(kernel(&l_scalar, &r_array), expected); + let r_scalar = Scalar::new(decimal(PrimitiveArray::from_iter_values([x]), r_scale)); + let expected = (0..l.len()).map(|i| row(i, l[i], x)).collect(); + assert_decimal_result::(kernel(&l_array, &r_scalar), expected); + } + } + + /// Checks `add`, `sub` and `mul` of `T` at its edge values, with equal + /// scales and with either side rescaled by 100. + fn test_checked_decimal_ops() { + let one = T::Native::ONE; + let three = T::Native::usize_as(3); + let two = T::Native::usize_as(2); + let hundred = T::Native::usize_as(100); + let (min, max) = (T::Native::MIN_TOTAL_ORDER, T::Native::MAX_TOTAL_ORDER); + let edges = [ + min, + min.add_wrapping(one), + min.div_wrapping(two), + min.div_wrapping(hundred), + min.div_wrapping(hundred).sub_wrapping(one), + three.neg_wrapping(), + one.neg_wrapping(), + T::Native::ZERO, + one, + three, + max.div_wrapping(hundred), + max.div_wrapping(hundred).add_wrapping(one), + max.div_wrapping(two), + max.div_wrapping(two).add_wrapping(one), + max.sub_wrapping(one), + max, + ]; + + for scales in [(2, 2), (2, 4), (4, 2)] { + let (l_mul, r_mul) = match scales { + (2, 4) => (hundred, one), + (4, 2) => (one, hundred), + _ => (one, one), + }; + let rescale = move |l: T::Native, r: T::Native| { + Ok::<_, ArrowError>((l.mul_checked(l_mul)?, r.mul_checked(r_mul)?)) + }; + test_checked_decimal_op::(&edges, scales, add, |l, r| { + let (l, r) = rescale(l, r)?; + l.add_checked(r) + }); + test_checked_decimal_op::(&edges, scales, sub, |l, r| { + let (l, r) = rescale(l, r)?; + l.sub_checked(r) + }); + test_checked_decimal_op::(&edges, scales, add_wrapping, |l, r| { + let (l, r) = rescale(l, r)?; + l.add_checked(r) + }); + } + // Multiplication doesn't rescale its operands. + test_checked_decimal_op::(&edges, (2, 2), mul, |l, r| l.mul_checked(r)); + test_checked_decimal_op::(&edges, (2, 4), mul_wrapping, |l, r| l.mul_checked(r)); + } + + #[test] + fn test_checked_decimal_edges() { + test_checked_decimal_ops::(); + test_checked_decimal_ops::(); + test_checked_decimal_ops::(); + test_checked_decimal_ops::(); + } + + #[test] + fn test_checked_decimal128_mul_i64_boundary() { + // `i128` multiplies through `i64` when both operands fit, so check both + // sides of each `i64` bound. `i64::MIN * i64::MIN` is 2^126. + let (min, max) = (i64::MIN as i128, i64::MAX as i128); + let edges = [min - 1, min, min + 1, -1, 0, 1, max - 1, max, max + 1]; + test_checked_decimal_op::(&edges, (2, 2), mul, |l, r| l.mul_checked(r)); + test_checked_decimal_op::(&edges, (2, 4), add, |l, r| { + l.mul_checked(100)?.add_checked(r) + }); + } + + /// Moves one row that overflows through a batch, so that each lane of the + /// vector loop and each row of the scalar remainder holds it once, with an + /// array and a scalar on either side. The rescaled cases overflow only in + /// the rescale. The same row is also checked as a null slot, which is not + /// an error. + fn test_checked_decimal_overflow_position() { + // Three iterations of the widest vector loop we build, 8 `i32` lanes of + // AVX2 unrolled four times, plus a remainder. + const LEN: usize = 100; + let one = T::Native::ONE; + let two = T::Native::usize_as(2); + let hundred = T::Native::usize_as(100); + let (min, max) = (T::Native::MIN_TOTAL_ORDER, T::Native::MAX_TOTAL_ORDER); + // Multiplying by 100 overflows, but adding or subtracting 1 doesn't. + let rescale_overflow = max.div_wrapping(hundred).add_wrapping(one); + // The kernel, its checked scalar operation, whether it rescales, the + // scales of the batch and the other operand, the value that overflows, + // and the other operand. + type Case = (Kernel, Checked, bool, (i8, i8), N, N); + let cases: [Case; 5] = [ + (add, |l, r| l.add_checked(r), true, (2, 2), max, one), + (sub, |l, r| l.sub_checked(r), true, (2, 2), min, one), + (mul, |l, r| l.mul_checked(r), false, (2, 2), max, two), + ( + add, + |l, r| l.add_checked(r), + true, + (2, 4), + rescale_overflow, + one, + ), + ( + sub, + |l, r| l.sub_checked(r), + true, + (2, 4), + rescale_overflow, + one, + ), + ]; + let decimal = |array: PrimitiveArray, scale| { + array + .with_precision_and_scale(T::MAX_PRECISION, scale) + .unwrap() + }; + for (kernel, checked, rescales, (batch_scale, other_scale), edge, other) in cases { + // The checked result of `l op r`, rescaling the operand with the smaller scale. + let row = |l: T::Native, l_scale, r: T::Native, r_scale| { + if !rescales { + return checked(l, r); + } + let l = if l_scale < r_scale { + l.mul_checked(hundred)? + } else { + l + }; + let r = if r_scale < l_scale { + r.mul_checked(hundred)? + } else { + r + }; + checked(l, r) + }; + let other_array = decimal(PrimitiveArray::from_value(other, LEN), other_scale); + let other_scalar = + Scalar::new(decimal(PrimitiveArray::from_value(other, 1), other_scale)); + for i in 0..LEN { + let mut values = vec![one; LEN]; + values[i] = edge; + for null in [None, Some(i)] { + let nulls = null.map(|i| NullBuffer::from_iter((0..LEN).map(|j| j != i))); + let array = decimal( + PrimitiveArray::new(values.clone().into(), nulls), + batch_scale, + ); + let expected = + |batch_left: bool| -> Result>, ArrowError> { + (0..LEN) + .map(|j| match (Some(j) == null, batch_left) { + (true, _) => Ok(None), + (false, true) => { + row(values[j], batch_scale, other, other_scale).map(Some) + } + (false, false) => { + row(other, other_scale, values[j], batch_scale).map(Some) + } + }) + .collect() + }; + assert_decimal_result::(kernel(&array, &other_array), expected(true)); + assert_decimal_result::(kernel(&other_array, &array), expected(false)); + assert_decimal_result::(kernel(&array, &other_scalar), expected(true)); + assert_decimal_result::(kernel(&other_scalar, &array), expected(false)); + } + } + } + } + + #[test] + fn test_checked_decimal_null_slot_overflow_sliced() { + // Sliced at 3, the validity bits start mid-byte and the 200 rows span + // three full 64-row chunks plus a remainder. Row 5 is null and always + // overflows, so every case checks the valid rows. + const LEN: usize = 203; + let nulls = NullBuffer::from_iter((0..LEN).map(|j| j % 5 != 0)); + let decimal = |array: Decimal64Array| array.with_precision_and_scale(18, 2).unwrap(); + let one = Scalar::new(decimal(Decimal64Array::from_iter_values([1]))); + for i in 3..LEN { + let mut values = vec![1; LEN]; + values[5] = i64::MAX; + values[i] = i64::MAX; + let array = decimal(Decimal64Array::new(values.into(), Some(nulls.clone()))); + let result = add(&array.slice(3, LEN - 3), &one); + match nulls.is_valid(i) { + true => assert!(result.is_err(), "row {i}"), + false => { + let actual: Vec<_> = result + .unwrap() + .as_primitive::() + .iter() + .collect(); + let expected: Vec<_> = + (3..LEN).map(|j| nulls.is_valid(j).then_some(2)).collect(); + assert_eq!(actual, expected, "row {i}"); + } + } + } + } + + #[test] + #[cfg_attr(miri, ignore)] // Checks the lanes of vectorized loops, which Miri doesn't run + fn test_checked_decimal_overflow_positions() { + test_checked_decimal_overflow_position::(); + test_checked_decimal_overflow_position::(); + test_checked_decimal_overflow_position::(); + test_checked_decimal_overflow_position::(); + } + + #[test] + fn test_decimal_rescaled_scalar() { + let decimal = |values: Vec>, scale| { + Decimal128Array::from(values) + .with_precision_and_scale(38, scale) + .unwrap() + }; + let values = + |array: ArrayRef| -> Vec<_> { array.as_primitive::().iter().collect() }; + // `col + 1.5` and `1.5 - col`, with `col` at scale 4. + let col = decimal(vec![Some(10_000), None, Some(-25_000)], 4); + let scalar = Scalar::new(decimal(vec![Some(150)], 2)); + assert_eq!( + values(add(&col, &scalar).unwrap()), + [Some(25_000), None, Some(-10_000)] + ); + assert_eq!( + values(sub(&scalar, &col).unwrap()), + [Some(5_000), None, Some(40_000)] + ); + + // A scalar whose rescale overflows is an error only if a valid row needs it. + let big_value = i128::MAX / 10; + let big = Scalar::new(decimal(vec![Some(big_value)], 2)); + let err = big_value.mul_checked(100).unwrap_err().to_string(); + assert_eq!(add(&col, &big).unwrap_err().to_string(), err); + assert_eq!(sub(&big, &col).unwrap_err().to_string(), err); + let all_null = decimal(vec![None, None], 4); + assert_eq!(values(add(&all_null, &big).unwrap()), [None, None]); + assert_eq!(values(sub(&big, &all_null).unwrap()), [None, None]); + let empty = decimal(vec![], 4); + assert!(add(&empty, &big).unwrap().is_empty()); + + let null = Scalar::new(decimal(vec![None], 2)); + assert_eq!(values(add(&col, &null).unwrap()), [None, None, None]); + + // `1.50 + 1.0000` and `1.0000 - 1.50`, with both operands scalars. + let ten_thousandths = Scalar::new(decimal(vec![Some(10_000)], 4)); + assert_eq!( + values(add(&scalar, &ten_thousandths).unwrap()), + [Some(25_000)] + ); + assert_eq!( + values(sub(&ten_thousandths, &scalar).unwrap()), + [Some(-5_000)] + ); + } + #[test] fn test_decimal256_add_sub_negative_scale_metadata() { let a = Decimal256Array::from(vec![Some(i256::ONE), Some(i256::MINUS_ONE), None]) diff --git a/arrow/benches/arithmetic_kernels.rs b/arrow/benches/arithmetic_kernels.rs index 5b7d6c124495..4e1365c36361 100644 --- a/arrow/benches/arithmetic_kernels.rs +++ b/arrow/benches/arithmetic_kernels.rs @@ -5,10 +5,17 @@ use criterion::*; use arrow::compute::kernels::numeric::*; -use arrow::datatypes::Float32Type; +use arrow::datatypes::{ + ArrowPrimitiveType, Decimal32Type, Decimal64Type, Decimal128Type, Decimal256Type, Float32Type, +}; use arrow::util::bench_util::*; -use arrow_array::Scalar; +use arrow_array::{Array, Decimal128Array, PrimitiveArray, Scalar}; +use arrow_buffer::i256; +use rand::distr::uniform::SampleUniform; +use rand::rngs::StdRng; +use rand::{RngExt, SeedableRng}; use std::hint; +use std::ops::Range; fn add_benchmark(c: &mut Criterion) { const BATCH_SIZE: usize = 64 * 1024; @@ -60,5 +67,160 @@ fn add_benchmark(c: &mut Criterion) { } } -criterion_group!(benches, add_benchmark); +/// Like `create_primitive_array_range`, but with its own `seed`, so that two +/// arrays hold different values and nulls. +fn seeded_array( + size: usize, + null_density: f32, + range: Range, + seed: u64, +) -> PrimitiveArray +where + T: ArrowPrimitiveType, + T::Native: SampleUniform, +{ + let mut rng = StdRng::seed_from_u64(seed); + (0..size) + .map(|_| (rng.random::() >= null_density).then(|| rng.random_range(range.clone()))) + .collect() +} + +/// Decimal arithmetic at DataFusion's default batch size. +/// +/// Values stay within six digits, so the cases never overflow unless the case +/// name says so. `add` and `add_wrapping` run the same checked kernel for +/// decimals, so only `add` is measured. +fn decimal_benchmark(c: &mut Criterion) { + const BATCH_SIZE: usize = 8192; + let decimal128 = |null_density, scale, seed| { + seeded_array::(BATCH_SIZE, null_density, -999_999..1_000_000, seed) + .with_precision_and_scale(18, scale) + .unwrap() + }; + let decimal256 = |array: &Decimal128Array| { + let array: PrimitiveArray = + array.iter().map(|v| v.map(i256::from)).collect(); + array.with_precision_and_scale(38, 2).unwrap() + }; + for null_density in [0., 0.1] { + let a = decimal128(null_density, 2, 1); + let b = decimal128(null_density, 2, 2); + let b_scale4 = decimal128(null_density, 4, 3); + // `1.5`, which has a smaller scale than `b_scale4`. + let scalar = Scalar::new( + Decimal128Array::from_iter_values([150]) + .with_precision_and_scale(18, 2) + .unwrap(), + ); + let a64 = seeded_array::(BATCH_SIZE, null_density, -999_999..1_000_000, 1) + .with_precision_and_scale(18, 2) + .unwrap(); + let b64 = seeded_array::(BATCH_SIZE, null_density, -999_999..1_000_000, 2) + .with_precision_and_scale(18, 2) + .unwrap(); + // Four digits, so that products fit in `i32`. + let decimal32 = |seed| { + seeded_array::(BATCH_SIZE, null_density, -9_999..10_000, seed) + .with_precision_and_scale(9, 2) + .unwrap() + }; + let (a32, b32) = (decimal32(1), decimal32(2)); + let a256 = decimal256(&a); + let b256 = decimal256(&b); + // Each value fits in `i64` or not at random, which mispredicts the branch + // to the `i64` multiplication in `i128` `mul_overflowing`. + let mut rng = StdRng::seed_from_u64(4); + let wide: Decimal128Array = (0..BATCH_SIZE) + .map(|_| { + (rng.random::() >= null_density).then(|| match rng.random_bool(0.5) { + true => rng.random_range(-999_999..1_000_000), + false => { + rng.random_range(1 << 63..1 << 80) + * if rng.random_bool(0.5) { 1 } else { -1 } + } + }) + }) + .collect(); + let wide = wide.with_precision_and_scale(38, 2).unwrap(); + + c.bench_function(&format!("decimal128_add({null_density})"), |bench| { + bench.iter(|| hint::black_box(add(&a, &b).unwrap())) + }); + c.bench_function( + &format!("decimal128_add_rescale({null_density})"), + |bench| bench.iter(|| hint::black_box(add(&a, &b_scale4).unwrap())), + ); + c.bench_function( + &format!("decimal128_add_rescale_scalar({null_density})"), + |bench| bench.iter(|| hint::black_box(add(&b_scale4, &scalar).unwrap())), + ); + c.bench_function(&format!("decimal128_subtract({null_density})"), |bench| { + bench.iter(|| hint::black_box(sub(&a, &b).unwrap())) + }); + c.bench_function(&format!("decimal128_multiply({null_density})"), |bench| { + bench.iter(|| hint::black_box(mul(&a, &b).unwrap())) + }); + c.bench_function( + &format!("decimal128_multiply_wide({null_density})"), + |bench| bench.iter(|| hint::black_box(mul(&wide, &b).unwrap())), + ); + c.bench_function(&format!("decimal128_divide({null_density})"), |bench| { + bench.iter(|| hint::black_box(div(&a, &b).unwrap())) + }); + c.bench_function(&format!("decimal64_add({null_density})"), |bench| { + bench.iter(|| hint::black_box(add(&a64, &b64).unwrap())) + }); + c.bench_function(&format!("decimal64_subtract({null_density})"), |bench| { + bench.iter(|| hint::black_box(sub(&a64, &b64).unwrap())) + }); + c.bench_function(&format!("decimal32_add({null_density})"), |bench| { + bench.iter(|| hint::black_box(add(&a32, &b32).unwrap())) + }); + c.bench_function(&format!("decimal32_subtract({null_density})"), |bench| { + bench.iter(|| hint::black_box(sub(&a32, &b32).unwrap())) + }); + c.bench_function(&format!("decimal32_multiply({null_density})"), |bench| { + bench.iter(|| hint::black_box(mul(&a32, &b32).unwrap())) + }); + c.bench_function(&format!("decimal64_multiply({null_density})"), |bench| { + bench.iter(|| hint::black_box(mul(&a64, &b64).unwrap())) + }); + c.bench_function(&format!("decimal256_add({null_density})"), |bench| { + bench.iter(|| hint::black_box(add(&a256, &b256).unwrap())) + }); + c.bench_function(&format!("decimal256_multiply({null_density})"), |bench| { + bench.iter(|| hint::black_box(mul(&a256, &b256).unwrap())) + }); + } + + // Overflow in the last row, so the whole array is processed before the error. + let mut values = decimal128(0., 2, 5).values().to_vec(); + values[BATCH_SIZE - 1] = i128::MAX; + let overflow = PrimitiveArray::::from_iter_values(values) + .with_precision_and_scale(38, 2) + .unwrap(); + let one = Scalar::new( + PrimitiveArray::::from_iter_values([1]) + .with_precision_and_scale(38, 2) + .unwrap(), + ); + c.bench_function("decimal128_add_overflow", |bench| { + bench.iter(|| hint::black_box(add(&overflow, &one).unwrap_err())) + }); + + // Null slots hold `i128::MAX`, which overflows, but no valid row does. + let a = decimal128(0.1, 2, 6); + let values = (0..BATCH_SIZE).map(|i| if a.is_null(i) { i128::MAX } else { a.value(i) }); + let null_overflow = PrimitiveArray::::new(values.collect(), a.nulls().cloned()) + .with_precision_and_scale(18, 2) + .unwrap(); + let b = seeded_array::(BATCH_SIZE, 0., 1..1_000_000, 7) + .with_precision_and_scale(18, 2) + .unwrap(); + c.bench_function("decimal128_add_null_slot_overflow", |bench| { + bench.iter(|| hint::black_box(add(&null_overflow, &b).unwrap())) + }); +} + +criterion_group!(benches, add_benchmark, decimal_benchmark); criterion_main!(benches);