Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions parquet/src/file/properties.rs
Original file line number Diff line number Diff line change
Expand Up @@ -998,6 +998,11 @@ impl WriterPropertiesBuilder {
}

/// Sets FileEncryptionProperties (defaults to `None`)
///
/// # Notes
///
/// Writing encrypted bloom filters is not yet supported. See
/// <https://github.com/apache/arrow-rs/issues/12166>
#[cfg(feature = "encryption")]
pub fn with_file_encryption_properties(
mut self,
Expand Down
21 changes: 18 additions & 3 deletions parquet/src/file/writer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -218,9 +218,24 @@ impl<W: Write + Send> SerializedFileWriter<W> {
if let Some(file_encryption_properties) = properties.file_encryption_properties() {
file_encryption_properties.validate_encrypted_column_names(schema_descriptor)?;

Ok(Some(Arc::new(FileEncryptor::new(Arc::clone(
file_encryption_properties,
))?)))
let file_encryptor = FileEncryptor::new(Arc::clone(file_encryption_properties))?;

// Writing encrypted bloom filters not supported. See
// https://github.com/apache/arrow-rs/issues/12166
for column in schema_descriptor.columns() {
let path = column.path().string();
if file_encryptor.is_column_encrypted(&path)
&& properties.bloom_filter_properties(column.path()).is_some()
{
return Err(nyi_err!(
"Encryption is not currently supported with bloom filters, \
but bloom filters are enabled for encrypted column '{}'",
path
));
}
}

Ok(Some(Arc::new(file_encryptor)))
} else {
Ok(None)
}
Expand Down
99 changes: 96 additions & 3 deletions parquet/tests/encryption/encryption.rs
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ use arrow::array::*;
use arrow::error::Result as ArrowResult;
use arrow_array::{Int32Array, RecordBatch};
use arrow_schema::{DataType as ArrowDataType, DataType, Field, Schema};
use bytes::Bytes;
use parquet::arrow::ArrowWriter;
use parquet::arrow::arrow_reader::{
ArrowReaderMetadata, ArrowReaderOptions, ParquetRecordBatchReaderBuilder, RowSelection,
Expand Down Expand Up @@ -1047,7 +1048,9 @@ fn write_and_read_stats(

let props = WriterProperties::builder()
.with_file_encryption_properties(encryption_properties)
.set_bloom_filter_enabled(true)
// TODO: enable bloom filters when supported
// https://github.com/apache/arrow-rs/issues/12166
//.set_bloom_filter_enabled(true)
.build();

let temp_file = tempfile::tempfile().unwrap();
Expand All @@ -1073,8 +1076,10 @@ fn write_and_read_stats(
assert_eq!(column_stats.min_bytes_opt(), Some(expected_min.as_slice()));
assert_eq!(column_stats.max_bytes_opt(), Some(expected_max.as_slice()));
}
assert!(column.bloom_filter_offset().is_some());
assert!(column.bloom_filter_length().is_some());
// Enable when bloom filters supported
// https://github.com/apache/arrow-rs/issues/12166
// assert!(column.bloom_filter_offset().is_some());
// assert!(column.bloom_filter_length().is_some());
} else {
assert!(column.statistics().is_none());
assert!(column.page_encoding_stats().is_none());
Expand Down Expand Up @@ -1107,6 +1112,94 @@ fn write_and_read_stats(
}
}

/// Returns a batch with two Int32 columns, "x" and "z"
fn bloom_filter_test_batch() -> RecordBatch {
let values = Arc::new(Int32Array::from(vec![8, 3, 4, 19, 5]));
let schema = Arc::new(Schema::new(vec![
Field::new("x", DataType::Int32, true),
Field::new("z", DataType::Int32, true),
]));
RecordBatch::try_new(schema, vec![values.clone(), values]).unwrap()
}

/// Writing a bloom filter for an encrypted column is not supported, and must
/// error rather than silently writing the bloom filter in plaintext
#[test]
fn test_write_bloom_filter_for_encrypted_column_errors() {
let footer_key = AES_128_FOOTER_KEY.to_vec();
let column_key = AES_128_COLUMN_KEYS[0].to_vec();
let batch = bloom_filter_test_batch();

let encryption_properties = FileEncryptionProperties::builder(footer_key.clone())
.with_column_key("x", column_key.clone())
.build()
.unwrap();

// Bloom filters enabled for all columns results in an error
let props = WriterProperties::builder()
.with_file_encryption_properties(Arc::clone(&encryption_properties))
.set_bloom_filter_enabled(true)
.build();
let err = ArrowWriter::try_new(Vec::new(), batch.schema(), Some(props)).unwrap_err();
assert_eq!(
err.to_string(),
"NYI: Encryption is not currently supported with bloom filters, \
but bloom filters are enabled for encrypted column 'x'"
);

// Uniform encryption (all columns encrypted) also results in an error
let uniform_properties = FileEncryptionProperties::builder(footer_key.clone())
.build()
.unwrap();
let props = WriterProperties::builder()
.with_file_encryption_properties(uniform_properties)
.set_bloom_filter_enabled(true)
.build();
let err = ArrowWriter::try_new(Vec::new(), batch.schema(), Some(props)).unwrap_err();
assert!(matches!(err, ParquetError::NYI(_)), "{err}");

let decryption_properties = FileDecryptionProperties::builder(footer_key.clone())
.with_column_key("x", column_key.clone())
.build()
.unwrap();

// However, we can write bloom filters for unencrypted columns even when
// some are still encrypted.
let props = WriterProperties::builder()
.with_file_encryption_properties(encryption_properties)
.set_column_bloom_filter_enabled("z".into(), true)
.build();

let mut output = Vec::new();
let mut writer = ArrowWriter::try_new(&mut output, batch.schema(), Some(props)).unwrap();
writer.write(&batch).unwrap();
writer.close().unwrap();

let options =
ArrowReaderOptions::default().with_file_decryption_properties(decryption_properties);
let builder =
ParquetRecordBatchReaderBuilder::try_new_with_options(Bytes::from(output), options)
.unwrap();

let row_group = builder.metadata().row_group(0);
assert!(row_group.column(0).bloom_filter_offset().is_none());
assert!(row_group.column(1).bloom_filter_offset().is_some()); // wrote bloom filter

let sbbf = builder
.get_row_group_column_bloom_filter(0, 1)
.unwrap()
.expect("bloom filter for column z");
assert!(sbbf.check(&19_i32));
assert!(!sbbf.check(&42_i32));

let batches = builder
.build()
.unwrap()
.collect::<ArrowResult<Vec<_>>>()
.unwrap();
assert_eq!(batches, vec![batch.clone()]);
}

#[test]
fn test_write_uniform_encryption() {
fn write_uniform_encryption(footer_key: &[u8]) {
Expand Down
Loading