Upgrade some dependencies

guilload · guilload · commit a5b9ade3bfc4 · 2026-04-14T16:04:16.000-04:00
diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv
diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock
diff --git a/quickwit/Cargo.toml b/quickwit/Cargo.toml
@@ -87,7 +87,7 @@ license = "Apache-2.0"
 [workspace.dependencies]
 anyhow = "1"
 arc-swap = "1.8"
-arrow = { version = "57", default-features = false, features = ["ipc"] }
+arrow = { version = "58", default-features = false, features = ["ipc"] }
 assert-json-diff = "2"
 async-compression = { version = "0.4", features = ["tokio", "gzip"] }
 async-speed-limit = "0.4"
@@ -109,7 +109,7 @@ coarsetime = "0.1"
 colored = "3.0"
 console-subscriber = "0.5"
 criterion = { version = "0.8", features = ["async_tokio"] }
-cron = "0.15"
+cron = "0.16"
 dialoguer = { version = "0.12", default-features = false }
 dotenvy = "0.15"
 dyn-clone = "1.0"
@@ -165,15 +165,15 @@ mrecordlog = { git = "https://github.com/quickwit-oss/mrecordlog", rev = "306c0a
 new_string_template = "1.5"
 nom = "8.0"
 numfmt = "1.2"
-oneshot = "0.1"
+oneshot = { version = "0.2", features = ["async", "std"] }
 openssl = { version = "0.10", default-features = false }
 openssl-probe = "0.1"
 opentelemetry = "0.31"
 opentelemetry-appender-tracing = "0.31"
 opentelemetry_sdk = { version = "0.31", features = ["rt-tokio"] }
 opentelemetry-otlp = { version = "0.31", features = ["grpc-tonic", "http-json"] }
 ouroboros = "0.18"
-parquet = { version = "57", default-features = false, features = ["arrow", "zstd", "snap", "variant_experimental"] }
+parquet = { version = "58", default-features = false, features = ["arrow", "snap", "variant_experimental", "zstd"] }
 percent-encoding = "2.3"
 pin-project = "1.1"
 pnet = { version = "0.35", features = ["std"] }
@@ -198,10 +198,10 @@ pulsar = { version = "6.6", default-features = false, features = [
 ] }
 quick_cache = "0.6.18"
 quote = "1.0"
-rand = "0.9"
-rand_distr = "0.5"
+rand = "0.10"
+rand_distr = "0.6"
 rayon = "1.11"
-rdkafka = { version = "0.38", default-features = false, features = [
+rdkafka = { version = "0.39", default-features = false, features = [
   "cmake-build",
   "libz",
   "ssl",
@@ -233,9 +233,9 @@ serde_qs = { version = "0.15" }
 serde_with = "3.16"
 serde_yaml = "0.9"
 serial_test = { version = "3.2", features = ["file_locks"] }
-sha2 = "0.10"
+sha2 = "0.11"
 siphasher = "1.0"
-stateright = "0.30"
+stateright = "0.31"
 smallvec = "1"
 sqlx = { version = "0.8", features = [
   "migrate",
@@ -245,7 +245,7 @@ sqlx = { version = "0.8", features = [
 ] }
 syn = { version = "2.0", features = ["extra-traits", "full", "parsing"] }
 sync_wrapper = "1"
-sysinfo = { version = "0.37", default-features = false, features = ["disk"] }
+sysinfo = { version = "0.38", default-features = false, features = ["disk"] }
 tabled = { version = "0.20", features = ["ansi"] }
 tempfile = "3"
 thiserror = "2"
@@ -304,7 +304,7 @@ username = "0.2"
 # in `quickwit-serve`. This code is fundamentally incompatible with version 5.x.
 utoipa = { version = "4.2", features = ["time", "ulid"] }
 uuid = { version = "1.19", features = ["v4", "serde"] }
-vrl = { version = "0.29", default-features = false, features = [
+vrl = { version = "0.31", default-features = false, features = [
   "compiler",
   "diagnostic",
   "stdlib",
@@ -318,7 +318,7 @@ aws-config = "1.8"
 aws-credential-types = { version = "1.2", features = ["hardcoded-credentials"] }
 aws-runtime = "1.5"
 aws-sdk-kinesis = "1.97"
-aws-sdk-s3 = "=1.62"
+aws-sdk-s3 = "1.129"
 aws-sdk-lambda = "1"
 aws-sdk-sqs = "1.91"
 aws-smithy-async = "1.2"
diff --git a/quickwit/quickwit-cluster/src/cluster.rs b/quickwit/quickwit-cluster/src/cluster.rs
@@ -782,7 +782,7 @@ mod tests {
     use quickwit_config::service::QuickwitService;
     use quickwit_proto::indexing::IndexingTask;
     use quickwit_proto::types::IndexUid;
-    use rand::Rng;
+    use rand::RngExt;
 
     use super::*;
 
diff --git a/quickwit/quickwit-cluster/src/grpc_gossip.rs b/quickwit/quickwit-cluster/src/grpc_gossip.rs
@@ -193,7 +193,7 @@ fn select_gossip_candidates(
             find_gossip_candidate_grpc_addr(self_chitchat_id, node_state)
                 .map(|grpc_addr| (&node_state.chitchat_id().node_id, grpc_addr))
         })
-        .choose_multiple(&mut rand::rng(), MAX_GOSSIP_PEERS)
+        .sample(&mut rand::rng(), MAX_GOSSIP_PEERS)
         .into_iter()
         .map(|(node_id, grpc_addr)| (node_id.clone(), grpc_addr))
         .unzip()
diff --git a/quickwit/quickwit-common/src/rand.rs b/quickwit/quickwit-common/src/rand.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use rand::Rng;
+use rand::RngExt;
 use rand::distr::Alphanumeric;
 
 /// Appends a random suffix composed of a hyphen and five random alphanumeric characters.
diff --git a/quickwit/quickwit-common/src/retry.rs b/quickwit/quickwit-common/src/retry.rs
@@ -17,7 +17,7 @@ use std::time::Duration;
 
 use async_trait::async_trait;
 use futures::Future;
-use rand::Rng;
+use rand::RngExt;
 use tracing::{debug, warn};
 
 pub trait Retryable {
diff --git a/quickwit/quickwit-common/src/temp_dir.rs b/quickwit/quickwit-common/src/temp_dir.rs
@@ -216,7 +216,7 @@ impl<'a> Builder<'a> {
 mod tests {
     use std::cmp;
 
-    use rand::Rng;
+    use rand::RngExt;
 
     use super::*;
 
diff --git a/quickwit/quickwit-config/src/storage_config.rs b/quickwit/quickwit-config/src/storage_config.rs
@@ -329,6 +329,8 @@ pub struct S3StorageConfig {
     pub disable_multi_object_delete: bool,
     #[serde(default)]
     pub disable_multipart_upload: bool,
+    #[serde(default)]
+    pub disable_checksums: bool,
 }
 
 impl S3StorageConfig {
@@ -337,18 +339,22 @@ impl S3StorageConfig {
             Some(StorageBackendFlavor::DigitalOcean) => {
                 self.force_path_style_access = true;
                 self.disable_multi_object_delete = true;
+                self.disable_checksums = true;
             }
             Some(StorageBackendFlavor::Garage) => {
                 self.region = Some("garage".to_string());
                 self.force_path_style_access = true;
+                self.disable_checksums = true;
             }
             Some(StorageBackendFlavor::Gcs) => {
                 self.disable_multi_object_delete = true;
                 self.disable_multipart_upload = true;
+                self.disable_checksums = true;
             }
             Some(StorageBackendFlavor::MinIO) => {
                 self.region = Some("minio".to_string());
                 self.force_path_style_access = true;
+                self.disable_checksums = true;
             }
             _ => {}
         }
@@ -627,6 +633,7 @@ mod tests {
                 force_path_style_access: true
                 disable_multi_object_delete_requests: true
                 disable_multipart_upload: true
+                disable_checksums: true
             "#;
             let s3_storage_config: S3StorageConfig =
                 serde_yaml::from_str(s3_storage_config_yaml).unwrap();
@@ -637,6 +644,7 @@ mod tests {
                 force_path_style_access: true,
                 disable_multi_object_delete: true,
                 disable_multipart_upload: true,
+                disable_checksums: true,
                 ..Default::default()
             };
             assert_eq!(s3_storage_config, expected_s3_config);
diff --git a/quickwit/quickwit-control-plane/src/ingest/ingest_controller.rs b/quickwit/quickwit-control-plane/src/ingest/ingest_controller.rs
@@ -49,7 +49,7 @@ use quickwit_proto::types::{IndexUid, NodeId, NodeIdRef, Position, ShardId, Sour
 use rand::prelude::IndexedRandom;
 use rand::rngs::ThreadRng;
 use rand::seq::SliceRandom;
-use rand::{Rng, RngCore, rng};
+use rand::{Rng, RngExt, rng};
 use serde::{Deserialize, Serialize};
 use tokio::sync::{Mutex, OwnedMutexGuard};
 use tracing::{Level, debug, enabled, error, info, instrument, warn};
diff --git a/quickwit/quickwit-ingest/src/ingest_v2/routing_table.rs b/quickwit/quickwit-ingest/src/ingest_v2/routing_table.rs
@@ -62,7 +62,7 @@ impl RoutingEntry {
 /// Higher capacity_score wins; tiebreak on more open_shard_count (more landing spots).
 fn power_of_two_choices<'a>(candidates: &[&'a IngesterNode]) -> &'a IngesterNode {
     debug_assert!(candidates.len() >= 2);
-    let mut iter = candidates.choose_multiple(&mut rng(), 2);
+    let mut iter = candidates.sample(&mut rng(), 2);
     let (&a, &b) = (iter.next().unwrap(), iter.next().unwrap());
 
     if (a.capacity_score, a.open_shard_count) >= (b.capacity_score, b.open_shard_count) {
diff --git a/quickwit/quickwit-ingest/src/queue.rs b/quickwit/quickwit-ingest/src/queue.rs
@@ -444,7 +444,7 @@ mod tests {
         use std::iter::repeat_with;
 
         use rand::rngs::StdRng;
-        use rand::{Rng, SeedableRng};
+        use rand::{RngExt, SeedableRng};
         use rand_distr::weighted::WeightedIndex;
         use rand_distr::{Distribution, LogNormal};
 
diff --git a/quickwit/quickwit-lambda-client/build.rs b/quickwit/quickwit-lambda-client/build.rs
@@ -20,6 +20,7 @@
 //! The Lambda binary is built separately in CI and published as a GitHub release.
 
 use std::env;
+use std::fmt::Write;
 use std::path::{Path, PathBuf};
 
 use sha2::{Digest, Sha256};
@@ -95,7 +96,12 @@ fn fetch_lambda_zip(local_cache_path: &Path) {
 }
 
 fn sha256_hex(data: &[u8]) -> String {
-    format!("{:x}", Sha256::digest(data))
+    let hash = Sha256::digest(data);
+    let mut hex = String::with_capacity(64);
+    for byte in hash {
+        write!(hex, "{byte:02x}").unwrap();
+    }
+    hex
 }
 
 fn download_lambda_zip(url: &str) -> Result<Vec<u8>, String> {
diff --git a/quickwit/quickwit-metastore/src/metastore/file_backed/mod.rs b/quickwit/quickwit-metastore/src/metastore/file_backed/mod.rs
@@ -1490,7 +1490,7 @@ mod tests {
     use quickwit_proto::types::SourceId;
     use quickwit_query::query_ast::qast_helper;
     use quickwit_storage::{MockStorage, RamStorage, Storage, StorageErrorKind};
-    use rand::Rng;
+    use rand::RngExt;
     use tests::manifest::{IndexStatus, Manifest};
     use time::OffsetDateTime;
     use tokio::time::Duration;
diff --git a/quickwit/quickwit-parquet-engine/src/storage/config.rs b/quickwit/quickwit-parquet-engine/src/storage/config.rs
@@ -142,7 +142,7 @@ impl ParquetWriterConfig {
         kv_metadata: Option<Vec<KeyValue>>,
     ) -> WriterProperties {
         let mut builder = WriterProperties::builder()
-            .set_max_row_group_size(self.row_group_size)
+            .set_max_row_group_row_count(Some(self.row_group_size))
             .set_data_page_size_limit(self.data_page_size)
             .set_write_batch_size(self.write_batch_size)
             .set_column_index_truncate_length(Some(64))
@@ -268,23 +268,23 @@ mod tests {
         let config = ParquetWriterConfig::default();
         let schema = create_test_schema();
         let props = config.to_writer_properties(&schema);
-        assert!(props.max_row_group_size() == 128 * 1024);
+        assert!(props.max_row_group_row_count() == Some(128 * 1024));
     }
 
     #[test]
     fn test_to_writer_properties_snappy() {
         let config = ParquetWriterConfig::new().with_compression(Compression::Snappy);
         let schema = create_test_schema();
         let props = config.to_writer_properties(&schema);
-        assert!(props.max_row_group_size() == 128 * 1024);
+        assert!(props.max_row_group_row_count() == Some(128 * 1024));
     }
 
     #[test]
     fn test_to_writer_properties_uncompressed() {
         let config = ParquetWriterConfig::new().with_compression(Compression::Uncompressed);
         let schema = create_test_schema();
         let props = config.to_writer_properties(&schema);
-        assert!(props.max_row_group_size() == 128 * 1024);
+        assert!(props.max_row_group_row_count() == Some(128 * 1024));
     }
 
     #[test]
diff --git a/quickwit/quickwit-search/src/leaf.rs b/quickwit/quickwit-search/src/leaf.rs
@@ -1924,7 +1924,7 @@ mod tests {
     use quickwit_config::{LambdaConfig, SearcherConfig};
     use quickwit_directories::write_hotcache;
     use quickwit_proto::search::LambdaSingleSplitResult;
-    use rand::Rng;
+    use rand::RngExt;
     use tantivy::TantivyDocument;
     use tantivy::directory::RamDirectory;
     use tantivy::schema::{
diff --git a/quickwit/quickwit-storage/src/object_storage/s3_compatible_storage.rs b/quickwit/quickwit-storage/src/object_storage/s3_compatible_storage.rs
@@ -24,7 +24,9 @@ use anyhow::{Context as AnyhhowContext, anyhow};
 use async_trait::async_trait;
 use aws_credential_types::provider::SharedCredentialsProvider;
 use aws_sdk_s3::Client as S3Client;
-use aws_sdk_s3::config::{Credentials, Region};
+use aws_sdk_s3::config::{
+    Credentials, Region, RequestChecksumCalculation, ResponseChecksumValidation,
+};
 use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
 use aws_sdk_s3::operation::delete_objects::DeleteObjectsOutput;
 use aws_sdk_s3::operation::get_object::{GetObjectError, GetObjectOutput};
@@ -144,6 +146,11 @@ pub async fn create_s3_client(s3_storage_config: &S3StorageConfig) -> S3Client {
     s3_config.set_stalled_stream_protection(aws_config.stalled_stream_protection());
     s3_config.set_timeout_config(aws_config.timeout_config().cloned());
 
+    if s3_storage_config.disable_checksums {
+        s3_config.set_request_checksum_calculation(Some(RequestChecksumCalculation::WhenRequired));
+        s3_config.set_response_checksum_validation(Some(ResponseChecksumValidation::WhenRequired));
+    }
+
     if let Some(endpoint) = s3_storage_config.endpoint() {
         info!(endpoint=%endpoint, "using S3 endpoint defined in storage config or environment variable");
         s3_config.set_endpoint_url(Some(endpoint));
