feat: auto-enable invariants writer feature for non-null columns in CREATE TABLE (#2418)

## What changes are proposed in this pull request?

Auto-adds the invariants writer feature to the protocol when a CREATE
TABLE schema contains any user-controlled non-null field (nullable:
false). Delta-Spark treats non-nullable schema columns as implicit
invariants and requires this feature to be listed in writerFeatures.
Previously kernel rejected non-null columns at CREATE; this change makes
such tables compatible with Spark readers/writers.

Explicit delta.invariants metadata annotations (SQL expression
invariants like x > 3) remain rejected because kernel cannot evaluate
them. The existing write guard in
TableConfiguration::ensure_write_supported blocks writes to any table
with such metadata; we now reject it at CREATE time for a clearer,
path-aware error.

Also adds TableFeature::Invariants to ALLOWED_DELTA_FEATURES so users
can pre-enable the feature on an all-nullable table via
delta.feature.invariants=supported. This is useful when a later ALTER
TABLE ADD COLUMN NOT NULL (by Spark) should not need a protocol upgrade.

Core changes:
- validate_schema_for_create no longer takes invariants_enabled and
instead rejects fields with delta.invariants metadata
- New schema_contains_non_null_fields helper
- New maybe_enable_invariants in the CREATE builder, following the
maybe_enable_variant_type / maybe_enable_timestamp_ntz pattern; adds
schema driven Invariants to writer_features only
- TableFeature::Invariants added to ALLOWED_DELTA_FEATURES

## How was this change tested?

- Unit: schema_contains_non_null_fields (top-level, nested, array, map,
all-nullable, Variant-skip), test_maybe_enable_invariants (feature flips
on for non-null only), invariants_metadata_rejected (all nesting paths),
and invariants case in test_feature_signal_accepted
- Integration:
test_create_table_with_non_null_columns_auto_enables_invariants,
test_create_table_with_invariants_feature_signal_allowed,
test_create_table_rejects_delta_invariants_metadata
- Full cargo test -p delta_kernel --all-features --locked --lib --tests
passes
- cargo clippy --tests --benches -- -D warnings clean

Co-authored-by: Sanuj Basu <sanujbasu@users.noreply.github.com>

Author: sanujbasu

CLAUDE.md

@@ -204,6 +204,9 @@ Keep this list updated when new protocol features are added to kernel.
 - Prefer `==` over `matches!` for simple single-variant enum comparisons. `matches!` is
   for patterns with bindings or guards. For example: `self == Variant` not
   `matches!(self, Variant)`.
+- Prefer `StructField::nullable` / `StructField::not_null` over
+  `StructField::new(name, type, bool)` when nullability is known at compile time.
+  Reserve `StructField::new` for cases where nullability is a runtime value.
 - NEVER panic in production code -- use errors instead. Panicking
   (including `unwrap()`, `expect()`, `panic!()`, `unreachable!()`, etc) is acceptable in test code only.
 

kernel/src/schema/mod.rs

@@ -446,7 +446,7 @@ impl StructField {
         MakePhysical::new(column_mapping_mode).run_field(self)
     }
 
-    fn has_invariants(&self) -> bool {
+    pub(crate) fn has_invariants(&self) -> bool {
         self.metadata
             .contains_key(ColumnMetadataKey::Invariants.as_ref())
     }
@@ -1291,6 +1291,40 @@ pub(crate) fn schema_has_invariants(schema: &Schema) -> bool {
     checker.0
 }
 
+/// Visitor that reports whether any non-null (`nullable: false`) field exists in a schema.
+/// Walks the full schema tree including nested struct, array element, and map value structs.
+struct NonNullFieldChecker {
+    found: bool,
+}
+
+impl<'a> SchemaTransform<'a> for NonNullFieldChecker {
+    /// Skip recursion into variant internals. The `metadata` and `value` fields inside a
+    /// `Variant` are protocol-defined, always non-null, and not user-controlled, so they
+    /// must not be treated as user-declared non-null columns.
+    fn transform_variant(&mut self, stype: &'a StructType) -> Option<Cow<'a, StructType>> {
+        Some(Cow::Borrowed(stype))
+    }
+
+    fn transform_struct_field(&mut self, field: &'a StructField) -> Option<Cow<'a, StructField>> {
+        if !field.is_nullable() {
+            self.found = true;
+        } else if !self.found {
+            let _ = self.recurse_into_struct_field(field);
+        }
+        Some(Cow::Borrowed(field))
+    }
+}
+
+/// Checks if any user-controlled column in the schema (including nested columns) is declared
+/// non-null (`nullable: false`).
+///
+/// Skips `Variant` internal struct fields, which are protocol-defined and always non-null.
+pub(crate) fn schema_contains_non_null_fields(schema: &Schema) -> bool {
+    let mut checker = NonNullFieldChecker { found: false };
+    let _ = checker.transform_struct(schema);
+    checker.found
+}
+
 /// Normalizes column name field names to match the casing in the schema.
 ///
 /// Walks each field name through the schema's struct hierarchy, replacing user-provided
@@ -2733,6 +2767,70 @@ mod tests {
         assert!(schema_has_invariants(&schema));
     }
 
+    fn all_nullable_schema() -> StructType {
+        StructType::new_unchecked([
+            StructField::nullable("a", DataType::STRING),
+            StructField::nullable("b", DataType::INTEGER),
+        ])
+    }
+
+    fn top_level_non_null_schema() -> StructType {
+        StructType::new_unchecked([
+            StructField::not_null("id", DataType::INTEGER),
+            StructField::nullable("name", DataType::STRING),
+        ])
+    }
+
+    fn nested_non_null_schema() -> StructType {
+        let nested_field = StructField::nullable(
+            "parent",
+            DataType::try_struct_type([StructField::not_null("child", DataType::INTEGER)]).unwrap(),
+        );
+        StructType::new_unchecked([StructField::nullable("a", DataType::STRING), nested_field])
+    }
+
+    fn array_non_null_schema() -> StructType {
+        let array_field = StructField::nullable(
+            "arr",
+            ArrayType::new(
+                DataType::try_struct_type([StructField::not_null("child", DataType::INTEGER)])
+                    .unwrap(),
+                true,
+            ),
+        );
+        StructType::new_unchecked([array_field])
+    }
+
+    fn map_non_null_schema() -> StructType {
+        let map_field = StructField::nullable(
+            "map",
+            MapType::new(
+                DataType::STRING,
+                DataType::try_struct_type([StructField::not_null("child", DataType::INTEGER)])
+                    .unwrap(),
+                true,
+            ),
+        );
+        StructType::new_unchecked([map_field])
+    }
+
+    fn variant_only_schema() -> StructType {
+        // Variant internal fields (metadata, value) are protocol-defined non-null but
+        // must NOT be counted as user-controlled non-null fields.
+        StructType::new_unchecked([StructField::nullable("v", DataType::unshredded_variant())])
+    }
+
+    #[rstest]
+    #[case::all_nullable(all_nullable_schema(), false)]
+    #[case::top_level(top_level_non_null_schema(), true)]
+    #[case::nested_struct(nested_non_null_schema(), true)]
+    #[case::array_element(array_non_null_schema(), true)]
+    #[case::map_value(map_non_null_schema(), true)]
+    #[case::variant_skipped(variant_only_schema(), false)]
+    fn test_schema_contains_non_null_fields(#[case] schema: StructType, #[case] expected: bool) {
+        assert_eq!(schema_contains_non_null_fields(&schema), expected);
+    }
+
     #[test]
     fn test_struct_type_iterator_basic() {
         let fields = vec![