diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 366bb9cc12..abe64ad3b1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -115,6 +115,13 @@ jobs: repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Build + if: matrix.os != 'windows-latest' + run: make build + + - name: Build + if: matrix.os == 'windows-latest' + env: + AWS_LC_SYS_C_STD: "11" run: make build build_with_no_default_features: diff --git a/Cargo.lock b/Cargo.lock index 1acf14d6db..3e7f198258 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1331,9 +1331,9 @@ dependencies = [ [[package]] name = "cmake" -version = "0.1.54" +version = "0.1.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" dependencies = [ "cc", ] diff --git a/crates/catalog/glue/src/schema.rs b/crates/catalog/glue/src/schema.rs index 864320dae4..9829af9571 100644 --- a/crates/catalog/glue/src/schema.rs +++ b/crates/catalog/glue/src/schema.rs @@ -165,7 +165,7 @@ impl SchemaVisitor for GlueSchemaBuilder { PrimitiveType::Date => "date".to_string(), PrimitiveType::Timestamp => "timestamp".to_string(), PrimitiveType::TimestampNs => "timestamp_ns".to_string(), - PrimitiveType::Timestamptz | PrimitiveType::TimestamptzNs => { + PrimitiveType::Timestamptz | PrimitiveType::TimestamptzNs | PrimitiveType::Variant => { return Err(Error::new( ErrorKind::FeatureUnsupported, format!("Conversion from {p:?} is not supported"), diff --git a/crates/catalog/hms/src/schema.rs b/crates/catalog/hms/src/schema.rs index c23b48719d..71fa2377d4 100644 --- a/crates/catalog/hms/src/schema.rs +++ b/crates/catalog/hms/src/schema.rs @@ -122,7 +122,7 @@ impl SchemaVisitor for HiveSchemaBuilder { PrimitiveType::Date => "date".to_string(), PrimitiveType::Timestamp => "timestamp".to_string(), PrimitiveType::TimestampNs => "timestamp_ns".to_string(), - PrimitiveType::Timestamptz | PrimitiveType::TimestamptzNs => { + PrimitiveType::Timestamptz | PrimitiveType::TimestamptzNs | PrimitiveType::Variant => { return Err(Error::new( ErrorKind::FeatureUnsupported, format!("Conversion from {p:?} is not supported"), diff --git a/crates/iceberg/src/arrow/schema.rs b/crates/iceberg/src/arrow/schema.rs index 9ee7897cb6..021c868363 100644 --- a/crates/iceberg/src/arrow/schema.rs +++ b/crates/iceberg/src/arrow/schema.rs @@ -687,6 +687,9 @@ impl SchemaVisitor for ToArrowSchemaConverter { crate::spec::PrimitiveType::Binary => { Ok(ArrowSchemaOrFieldOrType::Type(DataType::LargeBinary)) } + crate::spec::PrimitiveType::Variant => { + Ok(ArrowSchemaOrFieldOrType::Type(DataType::LargeBinary)) + } } } } @@ -1119,6 +1122,7 @@ pub fn datum_to_arrow_type_with_ree(datum: &Datum) -> DataType { PrimitiveType::Uuid => make_ree(DataType::Binary), PrimitiveType::Fixed(_) => make_ree(DataType::Binary), PrimitiveType::Binary => make_ree(DataType::Binary), + PrimitiveType::Variant => make_ree(DataType::Binary), PrimitiveType::Decimal { precision, scale } => { make_ree(DataType::Decimal128(*precision as u8, *scale as i8)) } diff --git a/crates/iceberg/src/arrow/value.rs b/crates/iceberg/src/arrow/value.rs index 30b47d83fc..6ecafc9317 100644 --- a/crates/iceberg/src/arrow/value.rs +++ b/crates/iceberg/src/arrow/value.rs @@ -424,6 +424,10 @@ impl SchemaWithPartnerVisitor for ArrowArrayToIcebergStructConverter { )) } } + PrimitiveType::Variant => Err(Error::new( + ErrorKind::FeatureUnsupported, + "Converting arrow array to variant literals is not supported yet.", + )), } } } diff --git a/crates/iceberg/src/avro/schema.rs b/crates/iceberg/src/avro/schema.rs index fdbc680977..e0e1d98141 100644 --- a/crates/iceberg/src/avro/schema.rs +++ b/crates/iceberg/src/avro/schema.rs @@ -237,6 +237,12 @@ impl SchemaVisitor for SchemaToAvroSchema { PrimitiveType::Uuid => AvroSchema::Uuid, PrimitiveType::Fixed(len) => avro_fixed_schema((*len) as usize)?, PrimitiveType::Binary => AvroSchema::Bytes, + PrimitiveType::Variant => { + return Err(Error::new( + ErrorKind::FeatureUnsupported, + "Converting variant type to avro schema is not supported yet.", + )); + } PrimitiveType::Decimal { precision, scale } => { avro_decimal_schema(*precision as usize, *scale as usize)? } diff --git a/crates/iceberg/src/spec/datatypes.rs b/crates/iceberg/src/spec/datatypes.rs index 0379465584..6dd27cff0c 100644 --- a/crates/iceberg/src/spec/datatypes.rs +++ b/crates/iceberg/src/spec/datatypes.rs @@ -241,6 +241,8 @@ pub enum PrimitiveType { TimestamptzNs, /// Arbitrary-length character sequences encoded in utf-8 String, + /// Semi-structured data encoded as an Iceberg variant. + Variant, /// Universally Unique Identifiers, should use 16-byte fixed Uuid, /// Fixed length byte array @@ -379,6 +381,7 @@ impl fmt::Display for PrimitiveType { PrimitiveType::TimestampNs => write!(f, "timestamp_ns"), PrimitiveType::TimestamptzNs => write!(f, "timestamptz_ns"), PrimitiveType::String => write!(f, "string"), + PrimitiveType::Variant => write!(f, "variant"), PrimitiveType::Uuid => write!(f, "uuid"), PrimitiveType::Fixed(size) => write!(f, "fixed({size})"), PrimitiveType::Binary => write!(f, "binary"), @@ -868,7 +871,8 @@ mod tests { {"id": 13, "name": "uuid_field", "required": true, "type": "uuid"}, {"id": 14, "name": "fixed_field", "required": true, "type": "fixed[10]"}, {"id": 15, "name": "binary_field", "required": true, "type": "binary"}, - {"id": 16, "name": "string_field", "required": true, "type": "string"} + {"id": 16, "name": "string_field", "required": true, "type": "string"}, + {"id": 17, "name": "variant_field", "required": true, "type": "variant"} ] } "#; @@ -948,6 +952,12 @@ mod tests { Type::Primitive(PrimitiveType::String), ) .into(), + NestedField::required( + 17, + "variant_field", + Type::Primitive(PrimitiveType::Variant), + ) + .into(), ], id_lookup: OnceLock::default(), name_lookup: OnceLock::default(), diff --git a/crates/iceberg/src/spec/table_metadata.rs b/crates/iceberg/src/spec/table_metadata.rs index 48b715da59..415fdb736c 100644 --- a/crates/iceberg/src/spec/table_metadata.rs +++ b/crates/iceberg/src/spec/table_metadata.rs @@ -1732,6 +1732,12 @@ mod tests { "name": "ts", "required": true, "type": "timestamp" + }, + { + "id": 5, + "name": "payload", + "required": false, + "type": "variant" } ] } @@ -1798,11 +1804,18 @@ mod tests { let schema = Schema::builder() .with_schema_id(1) - .with_fields(vec![Arc::new(NestedField::required( - 4, - "ts", - Type::Primitive(PrimitiveType::Timestamp), - ))]) + .with_fields(vec![ + Arc::new(NestedField::required( + 4, + "ts", + Type::Primitive(PrimitiveType::Timestamp), + )), + Arc::new(NestedField::optional( + 5, + "payload", + Type::Primitive(PrimitiveType::Variant), + )), + ]) .build() .unwrap(); diff --git a/crates/iceberg/src/spec/values/datum.rs b/crates/iceberg/src/spec/values/datum.rs index 88209ae95c..0e904f905a 100644 --- a/crates/iceberg/src/spec/values/datum.rs +++ b/crates/iceberg/src/spec/values/datum.rs @@ -406,6 +406,12 @@ impl Datum { } PrimitiveType::Fixed(_) => PrimitiveLiteral::Binary(Vec::from(bytes)), PrimitiveType::Binary => PrimitiveLiteral::Binary(Vec::from(bytes)), + PrimitiveType::Variant => { + return Err(Error::new( + ErrorKind::FeatureUnsupported, + "Converting bytes to variant datum is not supported yet.", + )); + } PrimitiveType::Decimal { .. } => { let unscaled_value = BigInt::from_signed_bytes_be(bytes); PrimitiveLiteral::Int128(unscaled_value.to_i128().ok_or_else(|| {