From 850020055d69ae6f66c21934ce44b0a647c94c88 Mon Sep 17 00:00:00 2001 From: Momcilo Mrkaic Date: Fri, 17 Apr 2026 17:02:41 +0000 Subject: [PATCH 1/5] fix: propagate null bitmap in evaluate_map_to_struct When the input MapArray has null rows (e.g. for remove/metadata/protocol actions where the parent `add` struct is null), evaluate_map_to_struct appends null values to all child field builders but creates the output StructArray with no null buffer (None). This causes Arrow validation to reject the array with "Found unmasked nulls for non-nullable StructArray field" when any output field is declared non-nullable. This manifests during checkpoint creation for partitioned tables with NOT NULL partition columns and delta.checkpoint.writeStatsAsStruct=true. The COALESCE(partitionValues_parsed, MAP_TO_STRUCT(partitionValues)) expression evaluates MAP_TO_STRUCT for all rows including those where the map is null, triggering the validation error. The fix propagates the input MapArray's null bitmap to the output StructArray, matching the pattern established in PR #1645 for nested transform expressions. Co-authored-by: Isaac --- kernel/src/engine/arrow_expression/evaluate_expression.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/src/engine/arrow_expression/evaluate_expression.rs b/kernel/src/engine/arrow_expression/evaluate_expression.rs index 000ceda865..ca4722476d 100644 --- a/kernel/src/engine/arrow_expression/evaluate_expression.rs +++ b/kernel/src/engine/arrow_expression/evaluate_expression.rs @@ -905,7 +905,7 @@ fn evaluate_map_to_struct( Ok(StructArray::try_new( arrow_fields.into(), output_columns, - None, + map_array.nulls().cloned(), )?) } From 87b097520a3d53f126c4f9cab246a7f33f50cd36 Mon Sep 17 00:00:00 2001 From: Momcilo Mrkaic Date: Mon, 20 Apr 2026 08:43:39 +0000 Subject: [PATCH 2/5] test: add tests for MAP_TO_STRUCT null bitmap propagation Add three tests covering the null bitmap fix in evaluate_map_to_struct: 1. test_map_to_struct_propagates_null_bitmap_for_non_nullable_fields: Direct test -- null map rows with non-nullable output fields. Verifies the output struct is null where the input map is null, preventing "Found unmasked nulls for non-nullable StructArray field" errors. 2. test_coalesce_map_to_struct_with_null_map_non_nullable_fields: Simulates the checkpoint expression COALESCE(partitionValues_parsed, MAP_TO_STRUCT(partitionValues)) with a null map row and NOT NULL partition column. This is the exact pattern that fails during checkpoint creation. 3. test_map_to_struct_all_null_maps_with_non_nullable_fields: Edge case where every map row is null. Verifies the output struct is entirely null despite non-nullable child fields. Co-authored-by: Isaac --- .../arrow_expression/evaluate_expression.rs | 159 ++++++++++++++++++ 1 file changed, 159 insertions(+) diff --git a/kernel/src/engine/arrow_expression/evaluate_expression.rs b/kernel/src/engine/arrow_expression/evaluate_expression.rs index ca4722476d..e89f57aaa5 100644 --- a/kernel/src/engine/arrow_expression/evaluate_expression.rs +++ b/kernel/src/engine/arrow_expression/evaluate_expression.rs @@ -2130,6 +2130,165 @@ mod tests { assert_eq!(col.value(0), "last"); } + /// Null map rows with non-nullable output fields must not trigger Arrow validation errors. + /// Without propagating the input MapArray's null bitmap to the output StructArray, Arrow + /// rejects the result with "Found unmasked nulls for non-nullable StructArray field". + #[test] + fn test_map_to_struct_propagates_null_bitmap_for_non_nullable_fields() { + use crate::arrow::array::{MapBuilder, StringBuilder}; + + let mut builder = MapBuilder::new(None, StringBuilder::new(), StringBuilder::new()); + + // Row 0: valid map {"region": "us", "id": "42"} + builder.keys().append_value("region"); + builder.values().append_value("us"); + builder.keys().append_value("id"); + builder.values().append_value("42"); + builder.append(true).unwrap(); + + // Row 1: null map (e.g. a null parent struct for remove/metadata/protocol actions) + builder.append(false).unwrap(); + + // Row 2: valid map {"region": "eu", "id": "7"} + builder.keys().append_value("region"); + builder.values().append_value("eu"); + builder.keys().append_value("id"); + builder.values().append_value("7"); + builder.append(true).unwrap(); + + let map_array = builder.finish(); + let schema = ArrowSchema::new(vec![ArrowField::new( + "pv", + map_array.data_type().clone(), + true, + )]); + let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(map_array)]).unwrap(); + + // Non-nullable output fields: triggers the bug when null bitmap is not propagated + let output_schema = StructType::new_unchecked(vec![ + StructField::new("region", DataType::STRING, false), + StructField::new("id", DataType::INTEGER, false), + ]); + let result_type = DataType::Struct(Box::new(output_schema)); + let expr = Expr::map_to_struct(column_expr!("pv")); + let result = evaluate_expression(&expr, &batch, Some(&result_type)).unwrap(); + let structs = result.as_any().downcast_ref::().unwrap(); + + // The StructArray must carry the null bitmap from the input MapArray + assert!(structs.is_valid(0)); + assert!(structs.is_null(1)); + assert!(structs.is_valid(2)); + + let regions = structs + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let ids = structs + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + + assert_eq!(regions.value(0), "us"); + assert_eq!(ids.value(0), 42); + + assert_eq!(regions.value(2), "eu"); + assert_eq!(ids.value(2), 7); + } + + /// Simulates the checkpoint COALESCE pattern: + /// COALESCE(partitionValues_parsed, MAP_TO_STRUCT(partitionValues)) + /// with a null map row and non-nullable output fields. This is the exact + /// expression that fails during checkpoint creation for partitioned tables + /// with NOT NULL partition columns and writeStatsAsStruct=true. + #[test] + fn test_coalesce_map_to_struct_with_null_map_non_nullable_fields() { + use crate::arrow::array::{MapBuilder, StringBuilder}; + + let mut builder = MapBuilder::new(None, StringBuilder::new(), StringBuilder::new()); + + // Row 0: add action with partition value + builder.keys().append_value("date"); + builder.values().append_value("2024-01-15"); + builder.append(true).unwrap(); + + // Row 1: null map (remove/metadata action where add is null) + builder.append(false).unwrap(); + + let map_array = builder.finish(); + let map_type = map_array.data_type().clone(); + let schema = Arc::new(ArrowSchema::new(vec![ + // First column: existing partitionValues_parsed (null for commit-sourced rows) + ArrowField::new( + "pv_parsed", + ArrowDataType::Struct( + vec![ArrowField::new("date", ArrowDataType::Date32, false)].into(), + ), + true, + ), + // Second column: partitionValues map + ArrowField::new("pv", map_type, true), + ])); + + // pv_parsed: all null (simulates data from commit JSON that lacks partitionValues_parsed) + let pv_parsed = new_null_array(schema.field(0).data_type(), 2); + let batch = + RecordBatch::try_new(schema, vec![pv_parsed, Arc::new(map_array)]).unwrap(); + + let output_schema = StructType::new_unchecked(vec![StructField::new( + "date", + DataType::DATE, + false, // NOT NULL partition column + )]); + let result_type = DataType::Struct(Box::new(output_schema)); + + // COALESCE(pv_parsed, MAP_TO_STRUCT(pv)) + let expr = Expr::coalesce([ + Expr::column(["pv_parsed"]), + Expr::map_to_struct(column_expr!("pv")), + ]); + let result = evaluate_expression(&expr, &batch, Some(&result_type)).unwrap(); + let structs = result.as_any().downcast_ref::().unwrap(); + + // Row 0: pv_parsed is null, MAP_TO_STRUCT succeeds -> non-null struct + assert!(structs.is_valid(0)); + // Row 1: pv_parsed is null, MAP_TO_STRUCT gets null map -> null struct + assert!(structs.is_null(1)); + } + + /// All map rows are null. The output StructArray should be entirely null. + #[test] + fn test_map_to_struct_all_null_maps_with_non_nullable_fields() { + use crate::arrow::array::{MapBuilder, StringBuilder}; + + let mut builder = MapBuilder::new(None, StringBuilder::new(), StringBuilder::new()); + builder.append(false).unwrap(); + builder.append(false).unwrap(); + + let map_array = builder.finish(); + let schema = ArrowSchema::new(vec![ArrowField::new( + "pv", + map_array.data_type().clone(), + true, + )]); + let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(map_array)]).unwrap(); + + let output_schema = StructType::new_unchecked(vec![StructField::new( + "key", + DataType::STRING, + false, + )]); + let result_type = DataType::Struct(Box::new(output_schema)); + let expr = Expr::map_to_struct(column_expr!("pv")); + let result = evaluate_expression(&expr, &batch, Some(&result_type)).unwrap(); + let structs = result.as_any().downcast_ref::().unwrap(); + + assert_eq!(structs.len(), 2); + assert!(structs.is_null(0)); + assert!(structs.is_null(1)); + } + #[test] fn test_map_to_struct_non_map_input() { let schema = ArrowSchema::new(vec![ArrowField::new("s", ArrowDataType::Utf8, true)]); From 3a303d0c2fa907ff9fd4264ca0d66faa8fde4d47 Mon Sep 17 00:00:00 2001 From: Momcilo Mrkaic Date: Mon, 20 Apr 2026 09:41:53 +0000 Subject: [PATCH 3/5] make test more consistent --- .../arrow_expression/evaluate_expression.rs | 96 ++++++++----------- 1 file changed, 38 insertions(+), 58 deletions(-) diff --git a/kernel/src/engine/arrow_expression/evaluate_expression.rs b/kernel/src/engine/arrow_expression/evaluate_expression.rs index e89f57aaa5..263bc0002e 100644 --- a/kernel/src/engine/arrow_expression/evaluate_expression.rs +++ b/kernel/src/engine/arrow_expression/evaluate_expression.rs @@ -2130,26 +2130,23 @@ mod tests { assert_eq!(col.value(0), "last"); } - /// Null map rows with non-nullable output fields must not trigger Arrow validation errors. - /// Without propagating the input MapArray's null bitmap to the output StructArray, Arrow - /// rejects the result with "Found unmasked nulls for non-nullable StructArray field". #[test] - fn test_map_to_struct_propagates_null_bitmap_for_non_nullable_fields() { + fn test_map_to_struct_null_map_with_non_nullable_fields() { use crate::arrow::array::{MapBuilder, StringBuilder}; let mut builder = MapBuilder::new(None, StringBuilder::new(), StringBuilder::new()); - // Row 0: valid map {"region": "us", "id": "42"} + // Row 0: {"region": "us", "id": "42"} builder.keys().append_value("region"); builder.values().append_value("us"); builder.keys().append_value("id"); builder.values().append_value("42"); builder.append(true).unwrap(); - // Row 1: null map (e.g. a null parent struct for remove/metadata/protocol actions) + // Row 1: null map builder.append(false).unwrap(); - // Row 2: valid map {"region": "eu", "id": "7"} + // Row 2: {"region": "eu", "id": "7"} builder.keys().append_value("region"); builder.values().append_value("eu"); builder.keys().append_value("id"); @@ -2164,7 +2161,6 @@ mod tests { )]); let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(map_array)]).unwrap(); - // Non-nullable output fields: triggers the bug when null bitmap is not propagated let output_schema = StructType::new_unchecked(vec![ StructField::new("region", DataType::STRING, false), StructField::new("id", DataType::INTEGER, false), @@ -2174,7 +2170,6 @@ mod tests { let result = evaluate_expression(&expr, &batch, Some(&result_type)).unwrap(); let structs = result.as_any().downcast_ref::().unwrap(); - // The StructArray must carry the null bitmap from the input MapArray assert!(structs.is_valid(0)); assert!(structs.is_null(1)); assert!(structs.is_valid(2)); @@ -2197,29 +2192,50 @@ mod tests { assert_eq!(ids.value(2), 7); } - /// Simulates the checkpoint COALESCE pattern: - /// COALESCE(partitionValues_parsed, MAP_TO_STRUCT(partitionValues)) - /// with a null map row and non-nullable output fields. This is the exact - /// expression that fails during checkpoint creation for partitioned tables - /// with NOT NULL partition columns and writeStatsAsStruct=true. #[test] - fn test_coalesce_map_to_struct_with_null_map_non_nullable_fields() { + fn test_map_to_struct_all_null_maps_with_non_nullable_fields() { use crate::arrow::array::{MapBuilder, StringBuilder}; let mut builder = MapBuilder::new(None, StringBuilder::new(), StringBuilder::new()); + builder.append(false).unwrap(); + builder.append(false).unwrap(); - // Row 0: add action with partition value + let map_array = builder.finish(); + let schema = ArrowSchema::new(vec![ArrowField::new( + "pv", + map_array.data_type().clone(), + true, + )]); + let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(map_array)]).unwrap(); + + let output_schema = StructType::new_unchecked(vec![StructField::new( + "key", + DataType::STRING, + false, + )]); + let result_type = DataType::Struct(Box::new(output_schema)); + let expr = Expr::map_to_struct(column_expr!("pv")); + let result = evaluate_expression(&expr, &batch, Some(&result_type)).unwrap(); + let structs = result.as_any().downcast_ref::().unwrap(); + + assert_eq!(structs.len(), 2); + assert!(structs.is_null(0)); + assert!(structs.is_null(1)); + } + + #[test] + fn test_coalesce_map_to_struct_with_null_map_non_nullable_fields() { + use crate::arrow::array::{MapBuilder, StringBuilder}; + + let mut builder = MapBuilder::new(None, StringBuilder::new(), StringBuilder::new()); builder.keys().append_value("date"); builder.values().append_value("2024-01-15"); builder.append(true).unwrap(); - - // Row 1: null map (remove/metadata action where add is null) builder.append(false).unwrap(); let map_array = builder.finish(); let map_type = map_array.data_type().clone(); let schema = Arc::new(ArrowSchema::new(vec![ - // First column: existing partitionValues_parsed (null for commit-sourced rows) ArrowField::new( "pv_parsed", ArrowDataType::Struct( @@ -2227,11 +2243,9 @@ mod tests { ), true, ), - // Second column: partitionValues map ArrowField::new("pv", map_type, true), ])); - // pv_parsed: all null (simulates data from commit JSON that lacks partitionValues_parsed) let pv_parsed = new_null_array(schema.field(0).data_type(), 2); let batch = RecordBatch::try_new(schema, vec![pv_parsed, Arc::new(map_array)]).unwrap(); @@ -2239,11 +2253,9 @@ mod tests { let output_schema = StructType::new_unchecked(vec![StructField::new( "date", DataType::DATE, - false, // NOT NULL partition column + false, )]); let result_type = DataType::Struct(Box::new(output_schema)); - - // COALESCE(pv_parsed, MAP_TO_STRUCT(pv)) let expr = Expr::coalesce([ Expr::column(["pv_parsed"]), Expr::map_to_struct(column_expr!("pv")), @@ -2251,41 +2263,9 @@ mod tests { let result = evaluate_expression(&expr, &batch, Some(&result_type)).unwrap(); let structs = result.as_any().downcast_ref::().unwrap(); - // Row 0: pv_parsed is null, MAP_TO_STRUCT succeeds -> non-null struct + // Row 0: pv_parsed null, MAP_TO_STRUCT succeeds assert!(structs.is_valid(0)); - // Row 1: pv_parsed is null, MAP_TO_STRUCT gets null map -> null struct - assert!(structs.is_null(1)); - } - - /// All map rows are null. The output StructArray should be entirely null. - #[test] - fn test_map_to_struct_all_null_maps_with_non_nullable_fields() { - use crate::arrow::array::{MapBuilder, StringBuilder}; - - let mut builder = MapBuilder::new(None, StringBuilder::new(), StringBuilder::new()); - builder.append(false).unwrap(); - builder.append(false).unwrap(); - - let map_array = builder.finish(); - let schema = ArrowSchema::new(vec![ArrowField::new( - "pv", - map_array.data_type().clone(), - true, - )]); - let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(map_array)]).unwrap(); - - let output_schema = StructType::new_unchecked(vec![StructField::new( - "key", - DataType::STRING, - false, - )]); - let result_type = DataType::Struct(Box::new(output_schema)); - let expr = Expr::map_to_struct(column_expr!("pv")); - let result = evaluate_expression(&expr, &batch, Some(&result_type)).unwrap(); - let structs = result.as_any().downcast_ref::().unwrap(); - - assert_eq!(structs.len(), 2); - assert!(structs.is_null(0)); + // Row 1: pv_parsed null, map null → null struct assert!(structs.is_null(1)); } From 9e146389fc8ecd0f91652f366ad30b4355744650 Mon Sep 17 00:00:00 2001 From: Momcilo Mrkaic Date: Mon, 20 Apr 2026 09:51:22 +0000 Subject: [PATCH 4/5] add comment --- .../engine/arrow_expression/evaluate_expression.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/kernel/src/engine/arrow_expression/evaluate_expression.rs b/kernel/src/engine/arrow_expression/evaluate_expression.rs index 263bc0002e..8edf8db805 100644 --- a/kernel/src/engine/arrow_expression/evaluate_expression.rs +++ b/kernel/src/engine/arrow_expression/evaluate_expression.rs @@ -902,6 +902,18 @@ fn evaluate_map_to_struct( .map(|f| ArrowField::try_from_kernel(*f)) .try_collect()?; + // Propagate the input map's null bitmap to the output struct. This is critical: + // when a map row is null, the loop above appends null to every child builder + // (since no keys match). Without this null bitmap, the output struct row appears + // valid (non-null) to Arrow, but its children contain nulls. If any child field + // is non-nullable, Arrow rejects this as "Found unmasked nulls for non-nullable + // StructArray field". With the bitmap, the struct row is marked null, which masks + // the child nulls and satisfies Arrow's validation. + // + // This matters during checkpoint creation: the COALESCE expression evaluates + // MAP_TO_STRUCT for all rows including non-add actions (remove, metadata, protocol) + // where the partition values map is null. Partition columns declared NOT NULL would + // cause the checkpoint to fail without this propagation. Ok(StructArray::try_new( arrow_fields.into(), output_columns, From 0fde57279d5d6de4aa039082218f746e515d1a3f Mon Sep 17 00:00:00 2001 From: Momcilo Mrkaic Date: Tue, 21 Apr 2026 14:57:45 +0000 Subject: [PATCH 5/5] test: address review feedback on MAP_TO_STRUCT tests - Hoist MapBuilder and StringBuilder imports to module level (removes 6 inline use statements across existing and new tests) - Consolidate the two MAP_TO_STRUCT null propagation tests into a single parametrized rstest with mixed_nulls and all_nulls cases - Fix rustfmt formatting in test_coalesce_map_to_struct Co-authored-by: Isaac --- .../arrow_expression/evaluate_expression.rs | 126 +++++------------- 1 file changed, 37 insertions(+), 89 deletions(-) diff --git a/kernel/src/engine/arrow_expression/evaluate_expression.rs b/kernel/src/engine/arrow_expression/evaluate_expression.rs index 8edf8db805..57a6445d57 100644 --- a/kernel/src/engine/arrow_expression/evaluate_expression.rs +++ b/kernel/src/engine/arrow_expression/evaluate_expression.rs @@ -936,7 +936,8 @@ mod tests { use super::*; use crate::arrow::array::{ - ArrayRef, BooleanArray, Int32Array, Int64Array, StringArray, StructArray, + ArrayRef, BooleanArray, Int32Array, Int64Array, MapBuilder, StringArray, StringBuilder, + StructArray, }; use crate::arrow::datatypes::{ DataType as ArrowDataType, Field as ArrowField, Schema as ArrowSchema, @@ -1983,8 +1984,6 @@ mod tests { /// Helper: creates a RecordBatch with a `pv` column of type Map. fn create_partition_map_batch() -> RecordBatch { - use crate::arrow::array::{MapBuilder, StringBuilder}; - let mut builder = MapBuilder::new(None, StringBuilder::new(), StringBuilder::new()); // Row 0: {"date": "2024-01-15", "region": "us", "id": "42"} @@ -2085,8 +2084,6 @@ mod tests { #[test] fn test_map_to_struct_parse_error() { - use crate::arrow::array::{MapBuilder, StringBuilder}; - let mut builder = MapBuilder::new(None, StringBuilder::new(), StringBuilder::new()); builder.keys().append_value("count"); builder.values().append_value("not_a_number"); @@ -2110,8 +2107,6 @@ mod tests { #[test] fn test_map_to_struct_duplicate_keys() { - use crate::arrow::array::{MapBuilder, StringBuilder}; - let mut builder = MapBuilder::new(None, StringBuilder::new(), StringBuilder::new()); builder.keys().append_value("x"); builder.values().append_value("first"); @@ -2142,29 +2137,35 @@ mod tests { assert_eq!(col.value(0), "last"); } - #[test] - fn test_map_to_struct_null_map_with_non_nullable_fields() { - use crate::arrow::array::{MapBuilder, StringBuilder}; - + #[rstest] + #[case::mixed_nulls( + vec![ + Some(vec![("region", "us"), ("id", "42")]), + None, + Some(vec![("region", "eu"), ("id", "7")]), + ], + vec![true, false, true], + )] + #[case::all_nulls(vec![None, None], vec![false, false])] + fn test_map_to_struct_null_propagation_with_non_nullable_fields( + #[case] rows: Vec>>, + #[case] expected_validity: Vec, + ) { let mut builder = MapBuilder::new(None, StringBuilder::new(), StringBuilder::new()); - - // Row 0: {"region": "us", "id": "42"} - builder.keys().append_value("region"); - builder.values().append_value("us"); - builder.keys().append_value("id"); - builder.values().append_value("42"); - builder.append(true).unwrap(); - - // Row 1: null map - builder.append(false).unwrap(); - - // Row 2: {"region": "eu", "id": "7"} - builder.keys().append_value("region"); - builder.values().append_value("eu"); - builder.keys().append_value("id"); - builder.values().append_value("7"); - builder.append(true).unwrap(); - + for row in &rows { + match row { + Some(entries) => { + for (k, v) in entries { + builder.keys().append_value(k); + builder.values().append_value(v); + } + builder.append(true).unwrap(); + } + None => { + builder.append(false).unwrap(); + } + } + } let map_array = builder.finish(); let schema = ArrowSchema::new(vec![ArrowField::new( "pv", @@ -2182,63 +2183,14 @@ mod tests { let result = evaluate_expression(&expr, &batch, Some(&result_type)).unwrap(); let structs = result.as_any().downcast_ref::().unwrap(); - assert!(structs.is_valid(0)); - assert!(structs.is_null(1)); - assert!(structs.is_valid(2)); - - let regions = structs - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - let ids = structs - .column(1) - .as_any() - .downcast_ref::() - .unwrap(); - - assert_eq!(regions.value(0), "us"); - assert_eq!(ids.value(0), 42); - - assert_eq!(regions.value(2), "eu"); - assert_eq!(ids.value(2), 7); - } - - #[test] - fn test_map_to_struct_all_null_maps_with_non_nullable_fields() { - use crate::arrow::array::{MapBuilder, StringBuilder}; - - let mut builder = MapBuilder::new(None, StringBuilder::new(), StringBuilder::new()); - builder.append(false).unwrap(); - builder.append(false).unwrap(); - - let map_array = builder.finish(); - let schema = ArrowSchema::new(vec![ArrowField::new( - "pv", - map_array.data_type().clone(), - true, - )]); - let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(map_array)]).unwrap(); - - let output_schema = StructType::new_unchecked(vec![StructField::new( - "key", - DataType::STRING, - false, - )]); - let result_type = DataType::Struct(Box::new(output_schema)); - let expr = Expr::map_to_struct(column_expr!("pv")); - let result = evaluate_expression(&expr, &batch, Some(&result_type)).unwrap(); - let structs = result.as_any().downcast_ref::().unwrap(); - - assert_eq!(structs.len(), 2); - assert!(structs.is_null(0)); - assert!(structs.is_null(1)); + assert_eq!(structs.len(), expected_validity.len()); + for (i, &valid) in expected_validity.iter().enumerate() { + assert_eq!(structs.is_valid(i), valid, "row {i} validity mismatch"); + } } #[test] fn test_coalesce_map_to_struct_with_null_map_non_nullable_fields() { - use crate::arrow::array::{MapBuilder, StringBuilder}; - let mut builder = MapBuilder::new(None, StringBuilder::new(), StringBuilder::new()); builder.keys().append_value("date"); builder.values().append_value("2024-01-15"); @@ -2259,14 +2211,10 @@ mod tests { ])); let pv_parsed = new_null_array(schema.field(0).data_type(), 2); - let batch = - RecordBatch::try_new(schema, vec![pv_parsed, Arc::new(map_array)]).unwrap(); + let batch = RecordBatch::try_new(schema, vec![pv_parsed, Arc::new(map_array)]).unwrap(); - let output_schema = StructType::new_unchecked(vec![StructField::new( - "date", - DataType::DATE, - false, - )]); + let output_schema = + StructType::new_unchecked(vec![StructField::new("date", DataType::DATE, false)]); let result_type = DataType::Struct(Box::new(output_schema)); let expr = Expr::coalesce([ Expr::column(["pv_parsed"]),