From 8009ae02169e07e8319611a4cba5e4660d408e02 Mon Sep 17 00:00:00 2001 From: Mola-maker <2249464964@qq.com> Date: Mon, 16 Nov 2026 00:29:44 +0000 Subject: [PATCH] fix(arrow-ipc): do not write a validity bitmap for RunEndEncoded in V4 The reader never consumes a validity buffer for RunEndEncoded arrays, in any metadata version, so the extra buffer written for V4 shifted every later buffer by one and the run-ends buffer was read as empty. Align the V4 branch of has_validity_bitmap with V5 and with the reader. Closes #12248 --- arrow-ipc/src/reader.rs | 33 +++++++++++++++++++++++++++++++++ arrow-ipc/src/writer.rs | 7 +++---- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/arrow-ipc/src/reader.rs b/arrow-ipc/src/reader.rs index 8564103dfc17..a2ec722f0484 100644 --- a/arrow-ipc/src/reader.rs +++ b/arrow-ipc/src/reader.rs @@ -3166,6 +3166,39 @@ mod tests { assert_eq!(run_array_1_unsliced, output_batch.column(0).into_data()); } + // Regression test for https://github.com/apache/arrow-rs/issues/12248: + // a RunEndEncoded array written with MetadataVersion::V4 did not read + // back, because the writer emitted a validity buffer for it while the + // reader never consumes one, shifting every later buffer by one. + #[test] + fn test_roundtrip_stream_run_array_v4() { + use crate::MetadataVersion; + use crate::writer::{IpcWriteOptions, StreamWriter}; + + let run_ends = Int32Array::from(vec![2, 5]); + let values = Int32Array::from(vec![10, 20]); + let array = RunArray::::try_new(&run_ends, &values).unwrap(); + let schema = Arc::new(Schema::new(vec![Field::new( + "a", + array.data_type().clone(), + false, + )])); + let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(array)]).unwrap(); + + let mut buf = Vec::new(); + { + let options = IpcWriteOptions::try_new(8, false, MetadataVersion::V4).unwrap(); + let mut writer = + StreamWriter::try_new_with_options(&mut buf, &schema, options).unwrap(); + writer.write(&batch).unwrap(); + writer.finish().unwrap(); + } + + let mut reader = StreamReader::try_new(buf.as_slice(), None).unwrap(); + let read = reader.next().unwrap().unwrap(); + assert_eq!(read, batch); + } + #[test] fn test_roundtrip_stream_nested_dict() { let xs = vec!["AA", "BB", "AA", "CC", "BB"]; diff --git a/arrow-ipc/src/writer.rs b/arrow-ipc/src/writer.rs index 33bf06cb6157..755ac82cddf6 100644 --- a/arrow-ipc/src/writer.rs +++ b/arrow-ipc/src/writer.rs @@ -2255,12 +2255,11 @@ pub fn write_message( writer.write_encoded_data(encoded, write_options) } -/// In V4, null types have no validity bitmap -/// In V5 and later, null and union types have no validity bitmap -/// Run end encoded type has no validity bitmap. +/// In V4, null and run end encoded types have no validity bitmap +/// In V5 and later, null, union and run end encoded types have no validity bitmap fn has_validity_bitmap(data_type: &DataType, write_options: &IpcWriteOptions) -> bool { if write_options.metadata_version < crate::MetadataVersion::V5 { - !matches!(data_type, DataType::Null) + !matches!(data_type, DataType::Null | DataType::RunEndEncoded(_, _)) } else { !matches!( data_type,