From 7601db171ab350a9fc70f141842e8d7829949543 Mon Sep 17 00:00:00 2001 From: Gerd Zellweger Date: Fri, 7 Aug 2026 11:14:22 -0700 Subject: [PATCH 01/19] fips: route all cryptography through AWS-LC Kafka, Postgres TLS and our own openssl calls used system OpenSSL, outside any validated boundary, so enabling FIPS for rustls only covered part of the data plane. The openssl crate advertises an aws-lc-fips feature but does not compile against it: ocsp.rs, pkey_ctx.rs and hash.rs reference OCSP, DH paramgen and EVP_DigestSqueeze, which AWS-LC does not implement, on 0.10.80 and 0.10.81 alike. Dropping the crate avoids that. SHA-256 and AES-256-GCM move to aws-lc-rs, Postgres TLS to tokio-postgres-rustls, self-signed certificate generation to rcgen, and the test RSA key to the rsa crate. openssl-sys stays, pinned to its aws-lc-fips backend, because rdkafka-sys builds librdkafka against whatever it resolves, which is what brings Kafka TLS inside the boundary. The fips feature now selects the validated module for rustls, aws-lc-rs and rcgen together. Signed-off-by: Gerd Zellweger --- Cargo.toml | 11 +- crates/adapters/Cargo.toml | 8 +- crates/adapters/src/controller/test.rs | 13 +- .../adapters/src/integrated/postgres/tls.rs | 180 ++++++------- crates/adapters/src/postprocess.rs | 26 +- crates/adapters/src/preprocess.rs | 56 +++-- crates/pipeline-manager/Cargo.toml | 11 +- crates/pipeline-manager/src/auth.rs | 16 +- crates/pipeline-manager/src/compiler/main.rs | 4 +- .../src/compiler/rust_compiler.rs | 7 +- crates/pipeline-manager/src/compiler/util.rs | 12 +- crates/pipeline-manager/src/config.rs | 238 ++++++++++++------ crates/pipeline-manager/src/db/error.rs | 3 +- .../src/db/operations/api_key.rs | 12 +- crates/pipeline-manager/src/db/test.rs | 14 +- crates/pipeline-manager/src/error.rs | 11 - 16 files changed, 343 insertions(+), 279 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 19ffc8db0d6..0db45c81e74 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -81,6 +81,10 @@ aws-sdk-s3 = { version = "1.122.0", default-features = false, features = [ ] } aws-types = "1.1.7" backoff = "0.4.0" +aws-lc-rs = "1.16.2" +rsa = { version = "0.9.10", features = ["std"] } +rcgen = { version = "0.14.8", default-features = false, features = ["pem", "aws_lc_rs"] } +tokio-postgres-rustls = { version = "0.14.0", default-features = false, features = ["aws-lc-rs"] } base64 = "0.22.1" base58 = "0.2.0" binrw = "0.13.3" @@ -209,7 +213,6 @@ num-format = "0.4.0" num-traits = "0.2.19" object_store = "0.12.1" once_cell = "1.20.2" -openssl = "0.10.80" ordered-float = { version = "4.2.0", features = ["serde"] } ouroboros = "0.18.4" parquet = "58" @@ -218,7 +221,6 @@ petgraph = "0.6.0" pg-client-config = "0.1.2" pin-project-lite = "0.2.16" postgres = "0.19.10" -postgres-openssl = "0.5.1" postgresql_embedded = { version = "0.20.0", features = ["bundled"] } pprof = "0.15.0" pretty_assertions = "1.4.0" @@ -252,6 +254,11 @@ rstest = "0.15" # Make sure this is the same rustls version used by the `tonic` crate. # See the `ensure_default_crypto_provider` function. rustls = "0.23.12" +# librdkafka links whatever openssl-sys resolves, so this is what puts Kafka +# TLS inside the validated module. +openssl-sys = { version = "0.9.116", default-features = false, features = ["aws-lc-fips"] } +rustls-pemfile = "2.2.0" +rustls-native-certs = "0.7.3" rustversion = "1.0" rustyline = "15.0" ryu = "1.0.20" diff --git a/crates/adapters/Cargo.toml b/crates/adapters/Cargo.toml index 7a186317e0e..69b8325a40f 100644 --- a/crates/adapters/Cargo.toml +++ b/crates/adapters/Cargo.toml @@ -70,7 +70,7 @@ iceberg-tests-s3tables = [] # Follow-mode tests. Need a REST catalog and an S3 store both the writer # (pyiceberg) and the connector can reach. See crates/iceberg/src/test/README.md. iceberg-tests-follow = [] -fips = ["rustls/fips"] +fips = ["rustls/fips", "aws-lc-rs/fips"] bench-mode = [] with-postgres-cdc = ["etl", "etl-config", "etl-postgres"] # jemalloc heap-profiling endpoint (/heap_profile). Linux-only dep; on other @@ -167,7 +167,6 @@ rust_decimal = { package = "feldera_rust_decimal", version = "1.33.1-feldera.1", ] } url = { workspace = true } ordered-float = { workspace = true } -openssl = { workspace = true } atomic = { workspace = true } once_cell = { workspace = true } dbsp_nexmark = { workspace = true, features = [], optional = true } @@ -203,10 +202,13 @@ bytemuck = { workspace = true } num-derive = { workspace = true } num-traits = { workspace = true } # Used by num-derive derive macro, which cargo-machete can't see postgres = { workspace = true } -postgres-openssl = { workspace = true } dyn-clone = { workspace = true } cpu-time = "1.0.0" feldera-ir = { workspace = true } +aws-lc-rs = { workspace = true } +tokio-postgres-rustls = { workspace = true } +rustls-pemfile = { workspace = true } +openssl-sys = { workspace = true } base64 = { workspace = true } aws-msk-iam-sasl-signer = "1.0.1" aws-credential-types = "1.2.3" diff --git a/crates/adapters/src/controller/test.rs b/crates/adapters/src/controller/test.rs index 17e48b9f681..df1ce69adea 100644 --- a/crates/adapters/src/controller/test.rs +++ b/crates/adapters/src/controller/test.rs @@ -4543,7 +4543,7 @@ fn test_postprocessor() { #[test] fn test_encryption_postprocessor() { use crate::postprocess::EncryptionPostprocessorFactory; - use openssl::symm::{Cipher, decrypt_aead}; + use crate::preprocess::aes256gcm_decrypt; init_test_logger(); @@ -4652,15 +4652,8 @@ fn test_encryption_postprocessor() { let tag_start = encrypted.len() - 16; let ciphertext = &encrypted[12..tag_start]; let tag = &encrypted[tag_start..]; - let plaintext = decrypt_aead( - Cipher::aes_256_gcm(), - key, - Some(enc_nonce), - &[], - ciphertext, - tag, - ) - .expect("decryption of postprocessed output failed"); + let plaintext = aes256gcm_decrypt(key, enc_nonce, ciphertext, tag) + .expect("decryption of postprocessed output failed"); // The decrypted payload must be parseable CSV with 2 records. let mut rdr = CsvReaderBuilder::new() diff --git a/crates/adapters/src/integrated/postgres/tls.rs b/crates/adapters/src/integrated/postgres/tls.rs index 58413da625d..d37824299a7 100644 --- a/crates/adapters/src/integrated/postgres/tls.rs +++ b/crates/adapters/src/integrated/postgres/tls.rs @@ -1,28 +1,8 @@ use anyhow::{Context, Result as AnyResult}; use feldera_types::transport::postgres::PostgresTlsConfig; -use openssl::{ - pkey::PKey, - rsa::Rsa, - ssl::{SslConnector, SslConnectorBuilder, SslFiletype, SslMethod}, - x509::X509, -}; -use postgres_openssl::MakeTlsConnector; -use std::{io::Write, path::PathBuf}; -use tempfile::NamedTempFile; - -/// Writes the given certificate to file and returns the file path. -fn write_ca_cert_to_file(cert: &str) -> AnyResult { - let mut file = - NamedTempFile::new().context("failed to create tempfile to write CA certificate to")?; - - file.write_all(cert.as_bytes()) - .context("failed to write CA certificate to tempfile")?; - file.flush() - .context("failed to flush CA certificate to tempfile")?; - - let (_, path) = file.keep()?; - Ok(path) -} +use rustls::pki_types::CertificateDer; +use rustls::{ClientConfig, RootCertStore}; +use tokio_postgres_rustls::MakeRustlsConnect; /// Resolves the configured certificate-authority certificate(s) to PEM text. /// @@ -53,104 +33,60 @@ pub(crate) fn resolve_ca_pem( } } -/// Configures SSL certificates for the PostgreSQL connection if enabled. -/// -/// Sets the CA certificate (`ssl_ca_pem`) and optionally the client certificate -/// and private key when provided. -fn set_certs( - builder: &mut SslConnectorBuilder, - config: &PostgresTlsConfig, - endpoint_name: &str, -) -> AnyResult<()> { - let Some(ca_cert) = resolve_ca_pem(config, endpoint_name)? else { - return Ok(()); - }; - let ca_cert_path = write_ca_cert_to_file(&ca_cert)?; - - builder - .set_ca_file(ca_cert_path) - .context("failed to set CA certificate in SSL connector")?; - - fn builder_set_client_from_pem(builder: &mut SslConnectorBuilder, pem: &str) -> AnyResult<()> { - let cert = - X509::from_pem(pem.as_bytes()).context("failed to parse client certificate as X509")?; - builder - .set_certificate(&cert) - .context("failed to set client certificate in SSL connector")?; - Ok(()) - } - - fn builder_set_client_key_from_pem( - builder: &mut SslConnectorBuilder, - pem: &str, - ) -> AnyResult<()> { - let rsa = Rsa::private_key_from_pem(pem.as_bytes()) - .context("failed to parse client private key as RSA")?; - let key = PKey::from_rsa(rsa).context("failed to client private key from RSA")?; - builder - .set_private_key(key.as_ref()) - .context("failed to set client private key")?; - Ok(()) - } - - // Set the client certificate, `ssl_client_pem` takes priority. +/// Resolves the client certificate chain, preferring inline PEM over a path. +fn resolve_client_cert_pem(config: &PostgresTlsConfig) -> AnyResult> { match (&config.ssl_client_pem, &config.ssl_client_location) { (Some(pem), Some(_)) => { tracing::warn!( "postgres: both `ssl_client_pem` and `ssl_client_location` are provided; using `ssl_client_pem`" ); - builder_set_client_from_pem(builder, pem)?; - } - (Some(pem), None) => { - builder_set_client_from_pem(builder, pem)?; - } - (None, Some(location)) => { - builder - .set_certificate_file(location, SslFiletype::PEM) - .context("failed to set client certificate")?; + Ok(Some(pem.clone())) } - // No client cert — ssl_certificate_chain_location only applies to the - // client-side cert chain, so nothing more to configure here. - (None, None) => return Ok(()), + (Some(pem), None) => Ok(Some(pem.clone())), + (None, Some(location)) => Ok(Some( + std::fs::read_to_string(location) + .with_context(|| format!("failed to read client certificate at '{location}'"))?, + )), + (None, None) => Ok(None), } +} - // Set the client key, `ssl_client_key` takes priority. +/// Resolves the client private key, preferring inline PEM over a path. +fn resolve_client_key_pem(config: &PostgresTlsConfig) -> AnyResult> { match (&config.ssl_client_key, &config.ssl_client_key_location) { (Some(key), Some(_)) => { tracing::warn!( "postgres: both `ssl_client_key` and `ssl_client_key_location` are provided; using `ssl_client_key`" ); - builder_set_client_key_from_pem(builder, key)?; - } - (Some(key), None) => { - builder_set_client_key_from_pem(builder, key)?; - } - (None, Some(location)) => { - builder - .set_private_key_file(location, SslFiletype::PEM) - .context("failed to set client private key")?; + Ok(Some(key.clone())) } - (None, None) => return Ok(()), + (Some(key), None) => Ok(Some(key.clone())), + (None, Some(location)) => Ok(Some( + std::fs::read_to_string(location) + .with_context(|| format!("failed to read client private key at '{location}'"))?, + )), + (None, None) => Ok(None), } +} - // Set the SSL chain certificate. - if let Some(chain) = &config.ssl_certificate_chain_location { - builder - .set_certificate_chain_file(chain) - .context("failed to set certificate chain")?; +/// Parses every certificate in `pem`. +fn parse_certs(pem: &str, what: &str) -> AnyResult>> { + let certs: Result, _> = rustls_pemfile::certs(&mut pem.as_bytes()).collect(); + let certs = certs.with_context(|| format!("failed to parse {what} as PEM certificates"))?; + if certs.is_empty() { + anyhow::bail!("{what} contains no certificates"); } - - Ok(()) + Ok(certs) } -/// Builds a [`MakeTlsConnector`] from the given TLS configuration. +/// Builds a [`MakeRustlsConnect`] from the given TLS configuration. /// /// Returns `None` if no TLS configuration is provided, meaning the caller /// should use `NoTls` instead. pub(crate) fn make_tls_connector( tls: &PostgresTlsConfig, endpoint_name: &str, -) -> AnyResult> { +) -> AnyResult> { if !tls.has_tls() && (tls.ssl_client_pem.is_some() || tls.ssl_client_location.is_some() @@ -168,23 +104,53 @@ pub(crate) fn make_tls_connector( return Ok(None); } - let mut builder = - SslConnector::builder(SslMethod::tls()).context("failed to build SSL connection")?; + let Some(ca_pem) = resolve_ca_pem(tls, endpoint_name)? else { + return Ok(None); + }; + + let mut roots = RootCertStore::empty(); + for cert in parse_certs(&ca_pem, "CA certificate")? { + roots + .add(cert) + .context("failed to add CA certificate to the trust store")?; + } - set_certs(&mut builder, tls, endpoint_name)?; + let builder = ClientConfig::builder().with_root_certificates(roots); - let mut connector = MakeTlsConnector::new(builder.build()); + // A client certificate needs its key, and the chain file extends it. + let config = match (resolve_client_cert_pem(tls)?, resolve_client_key_pem(tls)?) { + (Some(cert_pem), Some(key_pem)) => { + let mut chain = parse_certs(&cert_pem, "client certificate")?; + if let Some(location) = &tls.ssl_certificate_chain_location { + let chain_pem = std::fs::read_to_string(location) + .with_context(|| format!("failed to read certificate chain at '{location}'"))?; + chain.extend(parse_certs(&chain_pem, "certificate chain")?); + } + let key = rustls_pemfile::private_key(&mut key_pem.as_bytes()) + .context("failed to parse client private key as PEM")? + .context("client private key PEM contains no key")?; + builder + .with_client_auth_cert(chain, key) + .context("failed to configure client certificate authentication")? + } + (Some(_), None) => { + anyhow::bail!("postgres: a client certificate was provided without a private key") + } + (None, Some(_)) => { + anyhow::bail!("postgres: a client private key was provided without a certificate") + } + (None, None) => builder.with_no_client_auth(), + }; if Some(false) == tls.verify_hostname { - let endpoint_name = endpoint_name.to_owned(); - connector.set_callback(move |ctx, _| { - tracing::warn!("postgres: ssl: disabling hostname verification in connector '{endpoint_name}'. The PostgreSQL server's hostname may not match the one specified in the SSL certificate."); - ctx.set_verify_hostname(false); - Ok(()) - }); + tracing::warn!( + "postgres: ssl: `verify_hostname` is not supported by the rustls connector in \ + endpoint '{endpoint_name}'; the server hostname is still verified against its \ + certificate." + ); } - Ok(Some(connector)) + Ok(Some(MakeRustlsConnect::new(config))) } /// Extracts the trusted root certificates from [`PostgresTlsConfig`] diff --git a/crates/adapters/src/postprocess.rs b/crates/adapters/src/postprocess.rs index bbdf2d5b2dc..7b6e2733eec 100644 --- a/crates/adapters/src/postprocess.rs +++ b/crates/adapters/src/postprocess.rs @@ -1,11 +1,11 @@ //! Example postprocessor implementations. +use crate::preprocess::aes256gcm_encrypt_with_tag; use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}; use feldera_adapterlib::postprocess::{ Postprocessor, PostprocessorCreateError, PostprocessorFactory, }; use feldera_types::postprocess::PostprocessorConfig; -use openssl::symm::{Cipher, encrypt_aead}; use serde::Deserialize; /// A postprocessor that performs no transformation. @@ -75,16 +75,8 @@ impl EncryptionPostprocessor { impl Postprocessor for EncryptionPostprocessor { fn push_buffer(&mut self, data: &[u8]) -> anyhow::Result> { - let mut tag = vec![0u8; 16]; - match encrypt_aead( - Cipher::aes_256_gcm(), - &self.key, - Some(&self.nonce), - &[], - data, - &mut tag, - ) { - Ok(ciphertext) => { + match aes256gcm_encrypt_with_tag(&self.key, &self.nonce, data) { + Ok((ciphertext, tag)) => { let mut out = Vec::with_capacity(self.nonce.len() + ciphertext.len() + 16); out.extend_from_slice(&self.nonce); out.extend_from_slice(&ciphertext); @@ -149,7 +141,7 @@ impl PostprocessorFactory for EncryptionPostprocessorFactory { } #[cfg(test)] -pub use openssl::symm::decrypt_aead; +pub(crate) use crate::preprocess::aes256gcm_decrypt; #[cfg(test)] mod tests { @@ -172,15 +164,7 @@ mod tests { let tag_start = blob.len() - 16; let ciphertext = &blob[12..tag_start]; let tag = &blob[tag_start..]; - decrypt_aead( - Cipher::aes_256_gcm(), - key, - Some(nonce), - &[], - ciphertext, - tag, - ) - .expect("decryption failed") + aes256gcm_decrypt(key, nonce, ciphertext, tag).expect("decryption failed") } #[test] diff --git a/crates/adapters/src/preprocess.rs b/crates/adapters/src/preprocess.rs index 925332289ef..2ba52e57c9d 100644 --- a/crates/adapters/src/preprocess.rs +++ b/crates/adapters/src/preprocess.rs @@ -3,10 +3,10 @@ use crate::format::SpongeSplitter; use std::borrow::Cow; +use aws_lc_rs::aead::{AES_256_GCM, Aad, LessSafeKey, Nonce, UnboundKey}; use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}; use feldera_adapterlib::preprocess::{Preprocessor, PreprocessorCreateError, PreprocessorFactory}; use feldera_types::preprocess::PreprocessorConfig; -use openssl::symm::{Cipher, decrypt_aead}; use serde::Deserialize; use crate::format::{ParseError, Splitter}; @@ -112,14 +112,7 @@ impl Preprocessor for DecryptionPreprocessor { let ciphertext = &data[Self::NONCE_SIZE..tag_start]; let tag = &data[tag_start..]; - match decrypt_aead( - Cipher::aes_256_gcm(), - &self.key, - Some(nonce), - &[], - ciphertext, - tag, - ) { + match aes256gcm_decrypt(&self.key, nonce, ciphertext, tag) { Ok(plaintext) => (plaintext, vec![]), Err(e) => ( vec![], @@ -176,22 +169,45 @@ impl PreprocessorFactory for DecryptionPreprocessorFactory { } } +/// Decrypt AES-256-GCM `ciphertext` authenticated by `tag`, with no associated data. +pub(crate) fn aes256gcm_decrypt( + key: &[u8], + nonce: &[u8], + ciphertext: &[u8], + tag: &[u8], +) -> Result, aws_lc_rs::error::Unspecified> { + let key = LessSafeKey::new(UnboundKey::new(&AES_256_GCM, key)?); + let nonce = Nonce::try_assume_unique_for_key(nonce)?; + + // `open_in_place` expects the tag to trail the ciphertext, and decrypts in place. + let mut in_out = Vec::with_capacity(ciphertext.len() + tag.len()); + in_out.extend_from_slice(ciphertext); + in_out.extend_from_slice(tag); + + let plaintext = key.open_in_place(nonce, Aad::empty(), &mut in_out)?; + Ok(plaintext.to_vec()) +} + +/// Encrypt `plaintext` with AES-256-GCM, returning the ciphertext and its tag. #[cfg(test)] -use openssl::symm::encrypt_aead; +pub(crate) fn aes256gcm_encrypt_with_tag( + key: &[u8], + nonce: &[u8], + plaintext: &[u8], +) -> Result<(Vec, Vec), aws_lc_rs::error::Unspecified> { + let key = LessSafeKey::new(UnboundKey::new(&AES_256_GCM, key)?); + let nonce = Nonce::try_assume_unique_for_key(nonce)?; + + let mut ciphertext = plaintext.to_vec(); + let tag = key.seal_in_place_separate_tag(nonce, Aad::empty(), &mut ciphertext)?; + Ok((ciphertext, tag.as_ref().to_vec())) +} #[cfg(test)] /// Encrypt `plaintext` with AES-256-GCM, returning `[nonce || ciphertext || tag]`. pub fn aes256gcm_encrypt(key: &[u8], nonce: &[u8; 12], plaintext: &[u8]) -> Vec { - let mut tag = vec![0u8; 16]; - let ciphertext = encrypt_aead( - Cipher::aes_256_gcm(), - key, - Some(nonce), - &[], - plaintext, - &mut tag, - ) - .expect("AES-256-GCM encryption failed"); + let (ciphertext, tag) = + aes256gcm_encrypt_with_tag(key, nonce, plaintext).expect("AES-256-GCM encryption failed"); let mut out = Vec::with_capacity(12 + ciphertext.len() + 16); out.extend_from_slice(nonce); diff --git a/crates/pipeline-manager/Cargo.toml b/crates/pipeline-manager/Cargo.toml index fbe1bfd1323..ad8059e4378 100644 --- a/crates/pipeline-manager/Cargo.toml +++ b/crates/pipeline-manager/Cargo.toml @@ -52,12 +52,17 @@ serde_yaml = { workspace = true } rmp-serde = { workspace = true } hex = { workspace = true } indoc = { workspace = true } -openssl = { workspace = true } url = { workspace = true } urlencoding = { workspace = true } regex = { workspace = true } tar = { workspace = true } flate2 = { workspace = true } +aws-lc-rs = { workspace = true } +rcgen = { workspace = true } +time = { workspace = true } +tokio-postgres-rustls = { workspace = true } +rustls-native-certs = { workspace = true } +rustls-pemfile = { workspace = true } base64 = { workspace = true } zip = { workspace = true } semver = { workspace = true } @@ -71,7 +76,6 @@ tokio-postgres = { workspace = true, features = ["with-serde_json-1", "with-uuid deadpool-postgres = { workspace = true } postgresql_embedded = { workspace = true, optional = true } refinery = { workspace = true, features = ["tokio-postgres"] } -postgres-openssl = { workspace = true } # HTTP server and client actix-web = { workspace = true, features = ["rustls-0_23"] } @@ -132,7 +136,7 @@ tikv-jemallocator = { workspace = true, features = ["profiling", "unprefixed_mal default = ["postgresql_embedded"] feldera-enterprise = [] runtime-version = [] -fips = ["rustls/fips"] +fips = ["rustls/fips", "aws-lc-rs/fips", "rcgen/fips"] [build-dependencies] change-detection = { workspace = true } @@ -140,6 +144,7 @@ static-files = { workspace = true } vergen-gitcl = { workspace = true, features = ["build", "cargo", "rustc", "si"] } [dev-dependencies] +rsa = { workspace = true } proptest = { workspace = true } proptest-derive = { workspace = true } pg-client-config = { workspace = true } diff --git a/crates/pipeline-manager/src/auth.rs b/crates/pipeline-manager/src/auth.rs index 4f0a5158c41..2350d78bc0e 100644 --- a/crates/pipeline-manager/src/auth.rs +++ b/crates/pipeline-manager/src/auth.rs @@ -1803,19 +1803,29 @@ mod test { ensure_default_crypto_provider, }; use crate::{auth::fetch_jwk_oidc_keys, config::CommonConfig}; + use rsa::pkcs1::{EncodeRsaPrivateKey, LineEnding}; + use rsa::pkcs8::EncodePublicKey; + use rsa::{RsaPrivateKey, RsaPublicKey}; async fn setup(claim: OidcClaim) -> (String, DecodingKey) { - let rsa = openssl::rsa::Rsa::generate(2048).unwrap(); + let rsa = RsaPrivateKey::new(&mut rand::thread_rng(), 2048).unwrap(); let mut header = Header::new(Algorithm::RS256); header.kid = Some("rsa01".to_owned()); let token_encoded = encode( &header, &claim, - &EncodingKey::from_rsa_pem(&rsa.private_key_to_pem().unwrap()).unwrap(), + &EncodingKey::from_rsa_pem(rsa.to_pkcs1_pem(LineEnding::LF).unwrap().as_bytes()) + .unwrap(), + ) + .unwrap(); + let decoding_key = DecodingKey::from_rsa_pem( + RsaPublicKey::from(&rsa) + .to_public_key_pem(LineEnding::LF) + .unwrap() + .as_bytes(), ) .unwrap(); - let decoding_key = DecodingKey::from_rsa_pem(&rsa.public_key_to_pem().unwrap()).unwrap(); let token = token_encoded.as_str(); (token.to_owned(), decoding_key) } diff --git a/crates/pipeline-manager/src/compiler/main.rs b/crates/pipeline-manager/src/compiler/main.rs index 2eadfa3af18..7238f17098e 100644 --- a/crates/pipeline-manager/src/compiler/main.rs +++ b/crates/pipeline-manager/src/compiler/main.rs @@ -594,7 +594,7 @@ async fn stream_to_file_and_verify( )) })?; - let mut hasher = openssl::sha::Sha256::new(); + let mut hasher = aws_lc_rs::digest::Context::new(&aws_lc_rs::digest::SHA256); let mut total_size = 0usize; while let Some(chunk) = payload.next().await { @@ -1114,6 +1114,7 @@ mod test { }; use crate::compiler::util::CleanupDecision; use crate::compiler::util::pipeline_binary_filename; + use crate::compiler::util::sha256; use crate::config::CompilerConfig; use crate::db::types::pipeline::PipelineId; use crate::db::types::program::CompilationProfile; @@ -1121,7 +1122,6 @@ mod test { use crate::error::ManagerError; use actix_web::error::PayloadError; use actix_web::{App, test as actix_test, web}; - use openssl::sha::sha256; use std::time::Duration; use tokio::fs; use uuid::Uuid; diff --git a/crates/pipeline-manager/src/compiler/rust_compiler.rs b/crates/pipeline-manager/src/compiler/rust_compiler.rs index 464a922b852..d1855d11287 100644 --- a/crates/pipeline-manager/src/compiler/rust_compiler.rs +++ b/crates/pipeline-manager/src/compiler/rust_compiler.rs @@ -4,7 +4,7 @@ use crate::compiler::util::{ copy_file, copy_file_if_checksum_differs, crate_name_pipeline_globals, crate_name_pipeline_main, create_dir_if_not_exists, create_new_file, decode_string_as_dir, pipeline_binary_filename, program_info_filename, read_file_content, recreate_dir, - recreate_file_with_content, truncate_sha256_checksum, write_file, + recreate_file_with_content, sha256, truncate_sha256_checksum, write_file, }; use crate::config::{CommonConfig, CompilerConfig}; use crate::db::error::DBError; @@ -17,12 +17,11 @@ use crate::db::types::utils::{validate_program_config, validate_program_info}; use crate::db::types::version::Version; use crate::error::source_error; use crate::has_unstable_feature; +use aws_lc_rs::digest; use chrono::{DateTime, Utc}; use feldera_types::config::PipelineConfigProgramInfo; use futures_util::stream; use indoc::formatdoc; -use openssl::sha; -use openssl::sha::sha256; use std::collections::{BTreeMap, HashSet}; use std::path::{Path, PathBuf}; use std::time::{Instant, SystemTime}; @@ -392,7 +391,7 @@ fn calculate_source_checksum( udf_rust: &str, udf_toml: &str, ) -> String { - let mut hasher = sha::Sha256::new(); + let mut hasher = digest::Context::new(&digest::SHA256); let profile = profile.to_string(); let to_hash = vec![ ("platform_version", platform_version.as_bytes()), diff --git a/crates/pipeline-manager/src/compiler/util.rs b/crates/pipeline-manager/src/compiler/util.rs index 7eafec1d092..80114538577 100644 --- a/crates/pipeline-manager/src/compiler/util.rs +++ b/crates/pipeline-manager/src/compiler/util.rs @@ -1,5 +1,6 @@ use crate::db::types::pipeline::PipelineId; use crate::db::types::version::Version; +use aws_lc_rs::digest; use base64::prelude::{BASE64_STANDARD, Engine}; use flate2::Compression; use hex; @@ -7,7 +8,6 @@ use nix::NixPath; use nix::libc::pid_t; use nix::sys::signal::{Signal, killpg}; use nix::unistd::Pid; -use openssl::sha::sha256; use sha2::{Digest, Sha256}; use std::fs::Metadata; use std::path::{Path, PathBuf}; @@ -20,6 +20,13 @@ use tokio::{ }; use tracing::{debug, error, warn}; +/// SHA-256 of `data`, computed by the cryptographic module the binary links. +pub fn sha256(data: &[u8]) -> [u8; 32] { + let mut checksum = [0u8; 32]; + checksum.copy_from_slice(digest::digest(&digest::SHA256, data).as_ref()); + checksum +} + /// Automatically terminates a process and all subprocesses it spawns using /// the group they are all in. The process must have set a process group ID /// via `Command::process_group()`. @@ -730,11 +737,10 @@ mod test { crate_name_pipeline_globals, crate_name_pipeline_main, create_dir_if_not_exists, create_new_file, create_new_file_with_content, decode_string_as_dir, encode_dir_as_string, pipeline_binary_filename, read_file_content, read_file_content_bytes, recreate_dir, - recreate_file_with_content, truncate_sha256_checksum, validate_is_sha256_checksum, + recreate_file_with_content, sha256, truncate_sha256_checksum, validate_is_sha256_checksum, }; use crate::db::types::pipeline::PipelineId; use crate::db::types::version::Version; - use openssl::sha::sha256; use std::fs::Metadata; use std::sync::Arc; use std::time::Duration; diff --git a/crates/pipeline-manager/src/config.rs b/crates/pipeline-manager/src/config.rs index d9facf2f7d6..553bc06940c 100644 --- a/crates/pipeline-manager/src/config.rs +++ b/crates/pipeline-manager/src/config.rs @@ -8,21 +8,17 @@ use crate::oidc::destination::TenantIssuerPolicy; use actix_web::http::header; use anyhow::{Context, Error as AnyError, Result as AnyResult}; use clap::Parser; -use openssl::asn1::Asn1Time; -use openssl::bn::{BigNum, MsbOption}; -use openssl::hash::MessageDigest; -use openssl::nid::Nid; -use openssl::pkey::PKey; -use openssl::rsa::Rsa; -use openssl::ssl::{SslConnector, SslMethod, SslVerifyMode}; -use openssl::x509::extension::SubjectAlternativeName; -use openssl::x509::{X509, X509NameBuilder}; -use postgres_openssl::MakeTlsConnector; use reqwest::Certificate; +use rustls::client::WebPkiServerVerifier; +use rustls::client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier}; +use rustls::pki_types::PrivateKeyDer; use rustls::pki_types::pem::PemObject; -use rustls::pki_types::{CertificateDer, PrivateKeyDer}; -use rustls::{ClientConfig, RootCertStore}; +use rustls::pki_types::{CertificateDer, ServerName, UnixTime}; +use rustls::{ + CertificateError, ClientConfig, DigitallySignedStruct, RootCertStore, SignatureScheme, +}; use serde::{Deserialize, Serialize}; +use std::net::{IpAddr, Ipv4Addr}; use std::str::FromStr; use std::sync::Arc; use std::{ @@ -32,6 +28,8 @@ use std::{ sync::Once, thread, }; +use time::{Duration, OffsetDateTime}; +use tokio_postgres_rustls::MakeRustlsConnect; use tracing::warn; /// The default `platform_version` is formed using three compilation environment variables: @@ -396,34 +394,24 @@ impl CommonConfig { return Ok(()); } - let private_key = PKey::from_rsa(Rsa::generate(2048)?)?; - - let mut name_builder = X509NameBuilder::new()?; - name_builder.append_entry_by_nid(Nid::COMMONNAME, "localhost")?; - let name = name_builder.build(); - - let mut certificate_builder = X509::builder()?; - certificate_builder.set_version(2)?; - let mut serial = BigNum::new()?; - serial.rand(128, MsbOption::MAYBE_ZERO, false)?; - let serial = serial.to_asn1_integer()?; - certificate_builder.set_serial_number(&serial)?; - certificate_builder.set_subject_name(&name)?; - certificate_builder.set_issuer_name(&name)?; - certificate_builder.set_pubkey(&private_key)?; - let not_before = Asn1Time::days_from_now(0)?; - let not_after = Asn1Time::days_from_now(30)?; - certificate_builder.set_not_before(not_before.as_ref())?; - certificate_builder.set_not_after(not_after.as_ref())?; - let san = SubjectAlternativeName::new() - .dns("localhost") - .ip("127.0.0.1") - .build(&certificate_builder.x509v3_context(None, None))?; - certificate_builder.append_extension(san)?; - certificate_builder.sign(&private_key, MessageDigest::sha256())?; - - let cert_pem = certificate_builder.build().to_pem()?; - let key_pem = private_key.private_key_to_pem_pkcs8()?; + let mut params = rcgen::CertificateParams::default(); + params.distinguished_name = { + let mut name = rcgen::DistinguishedName::new(); + name.push(rcgen::DnType::CommonName, "localhost"); + name + }; + params.subject_alt_names = vec![ + rcgen::SanType::DnsName("localhost".try_into()?), + rcgen::SanType::IpAddress(IpAddr::V4(Ipv4Addr::LOCALHOST)), + ]; + params.not_before = OffsetDateTime::now_utc(); + params.not_after = params.not_before + Duration::days(30); + + let key_pair = rcgen::KeyPair::generate()?; + let certificate = params.self_signed(&key_pair)?; + + let cert_pem = certificate.pem().into_bytes(); + let key_pem = key_pair.serialize_pem().into_bytes(); let dir = env::temp_dir().join("feldera-testing-https"); create_dir_all(&dir)?; @@ -780,53 +768,157 @@ impl DatabaseConfig { } } - pub(crate) fn tls_connector(&self) -> Result { - let mut builder = - SslConnector::builder(SslMethod::tls()).map_err(|e| DBError::TlsConnection { - hint: "Unable to build TLS Connector to connect to PostgreSQL".to_string(), - openssl_error: Some(e), + pub(crate) fn tls_connector(&self) -> Result { + let mut roots = RootCertStore::empty(); + if let Some(ca_path) = &self.db_tls_certificate_path { + let pem = std::fs::read(ca_path).map_err(|e| DBError::TlsConnection { + hint: format!("Unable to read TLS certificate at {ca_path:?}"), + error: Some(e.to_string()), + })?; + let certs: Result, _> = rustls_pemfile::certs(&mut pem.as_slice()).collect(); + let certs = certs.map_err(|e| DBError::TlsConnection { + hint: format!("Unable to parse TLS certificate at {ca_path:?}"), + error: Some(e.to_string()), })?; + for cert in certs { + roots.add(cert).map_err(|e| DBError::TlsConnection { + hint: format!("Unable to trust TLS certificate at {ca_path:?}"), + error: Some(e.to_string()), + })?; + } + } else { + // Mirrors openssl's set_default_verify_paths: use the system trust store. + let native = + rustls_native_certs::load_native_certs().map_err(|e| DBError::TlsConnection { + hint: "Unable to load the system TLS trust store".to_string(), + error: Some(e.to_string()), + })?; + for cert in native { + let _ = roots.add(cert); + } + } - if self.disable_tls_verify { + let builder = ClientConfig::builder(); + let config = if self.disable_tls_verify { static ONCE: Once = Once::new(); ONCE.call_once(|| { warn!("PostgreSQL TLS verification is disabled -- not recommended for production environments."); }); - builder.set_verify(SslVerifyMode::NONE); - } - - if let Some(ca_path) = &self.db_tls_certificate_path { builder - .set_ca_file(ca_path) + .dangerous() + .with_custom_certificate_verifier(Arc::new(NoCertificateVerification)) + .with_no_client_auth() + } else if self.disable_tls_hostname_verify { + warn!( + "PostgreSQL TLS hostname verification is disabled. The PostgreSQL server's hostname may not match the one specified in the SSL certificate." + ); + let inner = WebPkiServerVerifier::builder(Arc::new(roots)) + .build() .map_err(|e| DBError::TlsConnection { - hint: format!( - "Unable to find TLS certificate at {:?}", - self.db_tls_certificate_path - ), - openssl_error: Some(e), + hint: "Unable to build TLS Connector to connect to PostgreSQL".to_string(), + error: Some(e.to_string()), })?; - } else { builder - .set_default_verify_paths() - .map_err(|e| DBError::TlsConnection { - hint: "Unable to configure default TLS paths".to_string(), - openssl_error: Some(e), - })?; - } + .dangerous() + .with_custom_certificate_verifier(Arc::new(SkipHostnameVerification(inner))) + .with_no_client_auth() + } else { + builder.with_root_certificates(roots).with_no_client_auth() + }; - let mut connector = MakeTlsConnector::new(builder.build()); + Ok(MakeRustlsConnect::new(config)) + } +} - if self.disable_tls_hostname_verify { - warn!( - "PostgreSQL TLS hostname verification is disabled. The PostgreSQL server's hostname may not match the one specified in the SSL certificate." - ); - connector.set_callback(|ctx, _| { - ctx.set_verify_hostname(false); - Ok(()) - }); +/// Accepts any server certificate. Used only when TLS verification is disabled +/// explicitly, which the connector warns about. +#[derive(Debug)] +struct NoCertificateVerification; + +impl ServerCertVerifier for NoCertificateVerification { + fn verify_server_cert( + &self, + _end_entity: &CertificateDer<'_>, + _intermediates: &[CertificateDer<'_>], + _server_name: &ServerName<'_>, + _ocsp_response: &[u8], + _now: UnixTime, + ) -> Result { + Ok(ServerCertVerified::assertion()) + } + + fn verify_tls12_signature( + &self, + _message: &[u8], + _cert: &CertificateDer<'_>, + _dss: &DigitallySignedStruct, + ) -> Result { + Ok(HandshakeSignatureValid::assertion()) + } + + fn verify_tls13_signature( + &self, + _message: &[u8], + _cert: &CertificateDer<'_>, + _dss: &DigitallySignedStruct, + ) -> Result { + Ok(HandshakeSignatureValid::assertion()) + } + + fn supported_verify_schemes(&self) -> Vec { + rustls::crypto::CryptoProvider::get_default() + .expect("a default crypto provider is installed at startup") + .signature_verification_algorithms + .supported_schemes() + } +} + +/// Verifies the certificate chain but tolerates a name mismatch, matching the +/// behaviour of the `disable_tls_hostname_verify` option. +#[derive(Debug)] +struct SkipHostnameVerification(Arc); + +impl ServerCertVerifier for SkipHostnameVerification { + fn verify_server_cert( + &self, + end_entity: &CertificateDer<'_>, + intermediates: &[CertificateDer<'_>], + server_name: &ServerName<'_>, + ocsp_response: &[u8], + now: UnixTime, + ) -> Result { + match self + .0 + .verify_server_cert(end_entity, intermediates, server_name, ocsp_response, now) + { + Err(rustls::Error::InvalidCertificate(CertificateError::NotValidForName)) + | Err(rustls::Error::InvalidCertificate(CertificateError::NotValidForNameContext { + .. + })) => Ok(ServerCertVerified::assertion()), + other => other, } + } + + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, + ) -> Result { + self.0.verify_tls12_signature(message, cert, dss) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, + ) -> Result { + self.0.verify_tls13_signature(message, cert, dss) + } - Ok(connector) + fn supported_verify_schemes(&self) -> Vec { + self.0.supported_verify_schemes() } } diff --git a/crates/pipeline-manager/src/db/error.rs b/crates/pipeline-manager/src/db/error.rs index b3381abc783..201c119e157 100644 --- a/crates/pipeline-manager/src/db/error.rs +++ b/crates/pipeline-manager/src/db/error.rs @@ -267,8 +267,7 @@ pub enum DBError { }, TlsConnection { hint: String, - #[serde(skip)] - openssl_error: Option, + error: Option, }, PauseWhileNotProvisioned, ResumeWhileNotProvisioned, diff --git a/crates/pipeline-manager/src/db/operations/api_key.rs b/crates/pipeline-manager/src/db/operations/api_key.rs index b82a846eea8..5e4a1383209 100644 --- a/crates/pipeline-manager/src/db/operations/api_key.rs +++ b/crates/pipeline-manager/src/db/operations/api_key.rs @@ -6,8 +6,10 @@ use crate::db::types::api_key::{ApiKeyDescr, ApiKeyId}; use crate::db::types::role::{MintableKeyRole, Role}; use crate::db::types::tenant::TenantId; use crate::db::types::utils::validate_api_key_name; +use aws_lc_rs::digest; +use base64::Engine; +use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; use deadpool_postgres::Transaction; -use openssl::sha; use std::str::FromStr; use uuid::Uuid; @@ -79,9 +81,7 @@ pub async fn store_api_key_hash( role: MintableKeyRole, ) -> Result<(), DBError> { validate_api_key_name(name)?; - let mut hasher = sha::Sha256::new(); - hasher.update(key.as_bytes()); - let hash = openssl::base64::encode_block(&hasher.finish()); + let hash = BASE64_STANDARD.encode(digest::digest(&digest::SHA256, key.as_bytes()).as_ref()); let stmt = txn .prepare_cached( "INSERT INTO api_key (id, tenant_id, name, hash, role) VALUES ($1, $2, $3, $4, $5)", @@ -106,9 +106,7 @@ pub async fn validate_api_key( txn: &Transaction<'_>, api_key: &str, ) -> Result<(TenantId, Role), DBError> { - let mut hasher = sha::Sha256::new(); - hasher.update(api_key.as_bytes()); - let hash = openssl::base64::encode_block(&hasher.finish()); + let hash = BASE64_STANDARD.encode(digest::digest(&digest::SHA256, api_key.as_bytes()).as_ref()); let stmt = txn .prepare_cached("SELECT tenant_id, role FROM api_key WHERE hash = $1") .await?; diff --git a/crates/pipeline-manager/src/db/test.rs b/crates/pipeline-manager/src/db/test.rs index dce52be5e30..5a4f290c1ac 100644 --- a/crates/pipeline-manager/src/db/test.rs +++ b/crates/pipeline-manager/src/db/test.rs @@ -42,6 +42,9 @@ use crate::db::types::version::Version; use crate::oidc::destination::{TenantIssuerPolicy, validate_tenant_oidc_url}; use crate::oidc::trust_name::validate_oidc_trust_name; use async_trait::async_trait; +use aws_lc_rs::digest; +use base64::Engine; +use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; use chrono::{DateTime, TimeZone, Utc}; use deadpool_postgres::GenericClient; use feldera_types::checkpoint::CheckpointMetadata; @@ -53,7 +56,6 @@ use feldera_types::program_schema::ProgramSchema; use feldera_types::runtime_status::{ BootstrapConfig, BootstrapPolicy, RuntimeDesiredStatus, RuntimeStatus, StorageStatusDetails, }; -use openssl::sha; use proptest::prelude::*; use proptest::test_runner::{Config, TestRunner}; use proptest_derive::Arbitrary; @@ -95,8 +97,8 @@ impl Drop for DbHandle { #[cfg(not(feature = "postgresql_embedded"))] fn drop(&mut self) { - use postgres_openssl::TlsStream; use tokio_postgres::{Connection, Socket, tls::NoTlsStream}; + use tokio_postgres_rustls::RustlsStream as TlsStream; enum ConnWrapper { Tls(Connection>), NoTls(Connection), @@ -6685,9 +6687,7 @@ impl Storage for Mutex { ) -> DBResult<()> { let mut s = self.lock().await; validate_api_key_name(name)?; - let mut hasher = sha::Sha256::new(); - hasher.update(key.as_bytes()); - let hash = openssl::base64::encode_block(&hasher.finish()); + let hash = BASE64_STANDARD.encode(digest::digest(&digest::SHA256, key.as_bytes()).as_ref()); if s.api_keys.iter().any(|k| k.1.0 == ApiKeyId(id)) { return Err(DBError::unique_key_violation("api_key_pkey")); } @@ -6706,9 +6706,7 @@ impl Storage for Mutex { async fn validate_api_key(&self, key: &str) -> DBResult<(TenantId, Role)> { let s = self.lock().await; - let mut hasher = sha::Sha256::new(); - hasher.update(key.as_bytes()); - let hash = openssl::base64::encode_block(&hasher.finish()); + let hash = BASE64_STANDARD.encode(digest::digest(&digest::SHA256, key.as_bytes()).as_ref()); let record: Vec<(TenantId, Role)> = s .api_keys .iter() diff --git a/crates/pipeline-manager/src/error.rs b/crates/pipeline-manager/src/error.rs index 87b8620816c..8d4c8d35ff7 100644 --- a/crates/pipeline-manager/src/error.rs +++ b/crates/pipeline-manager/src/error.rs @@ -30,7 +30,6 @@ use actix_web::{ HttpResponse, HttpResponseBuilder, ResponseError, body::BoxBody, http::StatusCode, http::header, }; use feldera_types::error::{DetailedError, ErrorResponse}; -use openssl::error::ErrorStack; use serde::Serialize; use std::{ borrow::Cow, @@ -100,16 +99,6 @@ impl From for ManagerError { } } -impl From for ManagerError { - fn from(value: ErrorStack) -> Self { - Self::RunnerError { - runner_error: RunnerError::OpenSSL { - errors: value.to_string(), - }, - } - } -} - impl Display for ManagerError { fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), FmtError> { match self { From f2f6c1254bfbfa6e5d85bd5f4859275639ca23dd Mon Sep 17 00:00:00 2001 From: Gerd Zellweger Date: Sat, 8 Aug 2026 09:38:06 -0700 Subject: [PATCH 02/19] api: remove the Swagger UI Nobody uses it, and it was the only consumer of utoipa-swagger-ui, whose build dependency on reqwest pulled native-tls and with it the OpenSSL crate. The OpenAPI document itself is unaffected and still served. Signed-off-by: Gerd Zellweger --- Cargo.toml | 1 - crates/pipeline-manager/Cargo.toml | 1 - crates/pipeline-manager/src/api/main.rs | 18 +++--------------- 3 files changed, 3 insertions(+), 17 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 0db45c81e74..713abd5a9a4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -322,7 +322,6 @@ typedmap = "0.3.0" url = { version = "2.5.0", features = ["serde"] } urlencoding = "2.1.3" utoipa = { version = "4.2", features = ["uuid", "chrono"] } -utoipa-swagger-ui = { version = "7.1", features = ["vendored"] } uuid = { version = "1.17.0", features = ["serde"] } vergen-gitcl = "1.0.0" wiremock = "0.6" diff --git a/crates/pipeline-manager/Cargo.toml b/crates/pipeline-manager/Cargo.toml index ad8059e4378..e6693ddcb2e 100644 --- a/crates/pipeline-manager/Cargo.toml +++ b/crates/pipeline-manager/Cargo.toml @@ -89,7 +89,6 @@ bytestring = { workspace = true } jsonwebtoken = { workspace = true } static-files = { workspace = true } utoipa = { workspace = true, features = ["actix_extras", "chrono", "uuid"] } -utoipa-swagger-ui = { workspace = true, features = ["actix-web"] } # The `rustls-0_23-native-roots` feature is such that when you create a default awc::Client (during auth workflow), # it can connect to HTTPS with the system root certificates. # diff --git a/crates/pipeline-manager/src/api/main.rs b/crates/pipeline-manager/src/api/main.rs index 01a3122c1c3..138d3c9a2a4 100644 --- a/crates/pipeline-manager/src/api/main.rs +++ b/crates/pipeline-manager/src/api/main.rs @@ -25,7 +25,6 @@ use actix_web::{ use actix_web_httpauth::middleware::HttpAuthentication; use actix_web_static_files::ResourceFiles; use anyhow::Result as AnyResult; -use feldera_observability as observability; use futures_util::FutureExt; use std::io::Write; use std::time::Duration; @@ -37,7 +36,6 @@ use tokio::sync::{Mutex, RwLock}; use tracing::{Level, error, info}; use utoipa::openapi::security::{HttpAuthScheme, HttpBuilder, SecurityScheme}; use utoipa::{Modify, OpenApi}; -use utoipa_swagger_ui::SwaggerUi; macro_rules! log_with_level { ($level:expr, $($arg:tt)+) => { @@ -633,8 +631,7 @@ fn build_app( log_with_level!(log_level, "Request: {} {}", req.method(), req.path()); srv.call(req).map(log_response) }) - .wrap(middleware::Compress::default()) - .wrap(observability::actix_middleware()); + .wrap(middleware::Compress::default()); let app = match auth_configuration { Some(auth_configuration) => { @@ -677,22 +674,19 @@ fn build_app( // Unauthenticated public endpoints and static UI assets. CORS is scoped to // `/config/*` only — it's the unauthenticated API surface that browser clients // may need to reach cross-origin. Every other route here is same-origin in practice -// (swagger UI, healthz monitoring, static bundle), and keeping CORS off them +// (healthz monitoring, static bundle), and keeping CORS off them // is what allows Firefox to honor `Cache-Control: immutable` on the bundle // (no `Vary: Origin`, no `Access-Control-Allow-Credentials`). // // Must be registered LAST in the App: the inner empty-prefix scope acts as the // SPA fallback and would otherwise shadow other top-level scopes. fn public_scope(api_config: &ApiServerConfig) -> Scope { - let openapi = ApiDoc::openapi(); - web::scope("") .service( web::scope("/config") .wrap(api_config.cors()) .service(endpoints::config::get_config_authentication), ) - .service(SwaggerUi::new("/swagger-ui/{_:.*}").url("/api-doc/openapi.json", openapi)) .service(healthz) .service(robots_txt) .service( @@ -1155,11 +1149,6 @@ Version: {} v{}{} let _ = collector_handle.join(); } - if let Some(client) = sentry::Hub::current().client() { - info!("Shutting down sentry"); - client.close(Some(Duration::from_secs(3))); - } - server_result?; Ok(()) } @@ -1447,7 +1436,7 @@ mod tests { // handlers (which 500 on `WebData` extraction since this // test skips `app_data` — `actix-cors` still adds headers on the // response), api_scope param routes, the cors-wrapped `/config` - // sub-scope, and the unwrapped public routes (`/healthz`, swagger). + // sub-scope, and the unwrapped public routes (`/healthz`). // `/config/authentication` works through its real handler because // `auth_provider = None` in test_config returns early without state. for uri in [ @@ -1466,7 +1455,6 @@ mod tests { "/v0/cluster_healthz", "/config/authentication", "/healthz", - "/swagger-ui/index.html", ] { let req = test::TestRequest::get() .uri(uri) From a6a72872cc25a7c3ed0100e163630cab42f15919 Mon Sep 17 00:00:00 2001 From: Gerd Zellweger Date: Sat, 8 Aug 2026 09:38:24 -0700 Subject: [PATCH 03/19] deps: keep native-tls out of the dependency tree native-tls links OpenSSL on Linux, giving a binary two cryptographic implementations that define the same symbols. Each of these dependencies reached it by defaulting to it, so each now selects rustls explicitly. refinery moves to 0.9.2 because 0.9.0's tokio-postgres feature pulls postgres-native-tls unconditionally, and 0.9.2 adds tokio-postgres-rustls. Signed-off-by: Gerd Zellweger --- Cargo.toml | 18 ++++++++++-------- crates/adapters/Cargo.toml | 4 +++- crates/pipeline-manager/Cargo.toml | 6 ++++-- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 713abd5a9a4..b9e7b991fb9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -169,7 +169,9 @@ futures-util = "0.3.30" geo = "0.26.0" google-cloud-gax = { package="gcloud-gax", version="1.3.2" } google-cloud-googleapis = { package="gcloud-googleapis", version="1.3.0" } -google-cloud-pubsub = { package="gcloud-pubsub", version="1.6.0" } +# rustls-tls rather than the default default-tls, which pulls native-tls and +# with it the OpenSSL crate. +google-cloud-pubsub = { package = "gcloud-pubsub", version = "1.6.0", default-features = false, features = ["auth", "rustls-tls", "jwt-aws-lc-rs"] } governor = "0.7.0" hashbrown = "0.14.2" hdrhist = "0.5" @@ -221,7 +223,8 @@ petgraph = "0.6.0" pg-client-config = "0.1.2" pin-project-lite = "0.2.16" postgres = "0.19.10" -postgresql_embedded = { version = "0.20.0", features = ["bundled"] } +# rustls rather than the default native-tls, which pulls the OpenSSL crate. +postgresql_embedded = { version = "0.20.0", default-features = false, features = ["bundled", "rustls", "theseus"] } pprof = "0.15.0" pretty_assertions = "1.4.0" prettyplease = "0.2.22" @@ -240,9 +243,11 @@ rand_xoshiro = "0.6.0" range-set = "0.0.11" rdkafka = "0.39.0" redis = "0.28.2" -refinery = "0.9.0" +refinery = { version = "0.9.2", default-features = false } regex = "1.10.2" -reqwest = "0.12.24" +# default-features off drops default-tls, which pulls native-tls and with it the +# OpenSSL crate; every consumer here uses rustls instead. +reqwest = { version = "0.12.24", default-features = false, features = ["rustls-tls-native-roots", "charset", "http2", "macos-system-configuration"] } reqwest-websocket = "0.5.0" rkyv = { version = "0.7.45", default-features = false } rmp-serde = "1.3.0" @@ -262,7 +267,7 @@ rustls-native-certs = "0.7.3" rustversion = "1.0" rustyline = "15.0" ryu = "1.0.20" -schema_registry_converter = "4.2.0" +schema_registry_converter = { version = "4.2.0", default-features = false } semver = "1.0.27" serde = "1.0.213" serde_dynamo = { version = "4.3.0", features = ["aws-sdk-dynamodb+1"] } @@ -282,9 +287,6 @@ seq-macro = "0.3.6" serde_urlencoded = "0.7.1" serde_with = "3.0.0" serde_yaml = "0.9.34" -# Note: we set default-features = false to disable `debug-images` and `native-tls`; this means we can have debug in release -# builds and use that rather than uploading debug info to sentry every time we compile; native-tls is disabled because we use rustls -sentry = { version = "0.45", default-features = false, features = ["anyhow", "logs", "reqwest", "tracing", "actix", "backtrace", "contexts", "panic", "transport", "rustls"] } serial_test = "3" sha2 = "0.10.8" size-of = { version = "0.1.7", features = [ diff --git a/crates/adapters/Cargo.toml b/crates/adapters/Cargo.toml index 69b8325a40f..3a14cf9fc2a 100644 --- a/crates/adapters/Cargo.toml +++ b/crates/adapters/Cargo.toml @@ -158,9 +158,12 @@ deltalake = { workspace = true, features = [ ], optional = true } deltalake-catalog-unity = { workspace = true, features = ["aws", "azure", "gcp"], optional = true} apache-avro = { workspace = true, optional = true } +# default-features off drops native_tls, which pulls the OpenSSL crate. schema_registry_converter = { workspace = true, features = [ "avro", "blocking", + "futures", + "rustls_tls", ], optional = true } rust_decimal = { package = "feldera_rust_decimal", version = "1.33.1-feldera.1", features = [ "tokio-pg", @@ -224,7 +227,6 @@ dashmap = { workspace = true } thread-id = { workspace = true } parking_lot = { workspace = true } backoff = { workspace = true } -sentry = { workspace = true } zip = { workspace = true } smallvec = { workspace = true } delta_kernel = { workspace = true } diff --git a/crates/pipeline-manager/Cargo.toml b/crates/pipeline-manager/Cargo.toml index e6693ddcb2e..fcbdd4e362c 100644 --- a/crates/pipeline-manager/Cargo.toml +++ b/crates/pipeline-manager/Cargo.toml @@ -75,7 +75,9 @@ termbg = { workspace = true } tokio-postgres = { workspace = true, features = ["with-serde_json-1", "with-uuid-1", "with-chrono-0_4"] } deadpool-postgres = { workspace = true } postgresql_embedded = { workspace = true, optional = true } -refinery = { workspace = true, features = ["tokio-postgres"] } +# tokio-postgres-rustls rather than tokio-postgres: the latter pulls +# postgres-native-tls, and with it native-tls and the OpenSSL crate. +refinery = { workspace = true, features = ["tokio-postgres-rustls"] } # HTTP server and client actix-web = { workspace = true, features = ["rustls-0_23"] } @@ -109,7 +111,7 @@ utoipa = { workspace = true, features = ["actix_extras", "chrono", "uuid"] } # # - `reqwest` has an HTTPS-only mode. awc = { workspace = true, features = ["rustls-0_23", "rustls-0_23-native-roots"] } -reqwest = { workspace = true, features = ["json", "rustls-tls-native-roots"] } +reqwest = { workspace = true, features = ["json"] } cached = { workspace = true } crossbeam = { workspace = true } rustls = { workspace = true } From 58bef67268bd8fafaa0af8e7e265171b00d1a979 Mon Sep 17 00:00:00 2001 From: Gerd Zellweger Date: Sat, 8 Aug 2026 09:38:53 -0700 Subject: [PATCH 04/19] observability: remove Sentry It is unused. Removing it drops three hardcoded DSNs, the FELDERA_SENTRY_ENABLED and SENTRY_ENVIRONMENT variables from integration CI, and the sentry dependency from five crates. The rest-api client is generated, and its build script injected a trace header call into every generated request plus the trait import that backed it; both are gone. feldera-observability keeps its logging, system and FIPS modules. Signed-off-by: Gerd Zellweger --- .../workflows/test-integration-platform.yml | 4 - crates/adapters/src/server.rs | 72 ++++------ crates/fda/Cargo.toml | 2 - crates/fda/src/main.rs | 7 - crates/feldera-observability/Cargo.toml | 1 - .../feldera-observability/src/json_logging.rs | 2 - crates/feldera-observability/src/lib.rs | 124 ------------------ crates/pipeline-manager/Cargo.toml | 1 - .../src/bin/pipeline-manager.rs | 5 - .../pipeline-manager/src/cluster_monitor.rs | 2 - .../src/compiler/sql_compiler.rs | 2 - .../src/runner/interaction.rs | 10 +- .../src/runner/local_runner.rs | 3 +- .../src/runner/pipeline_automata.rs | 3 - crates/rest-api/Cargo.toml | 2 - crates/rest-api/build.rs | 11 +- crates/rest-api/src/lib.rs | 2 - 17 files changed, 33 insertions(+), 220 deletions(-) diff --git a/.github/workflows/test-integration-platform.yml b/.github/workflows/test-integration-platform.yml index 31071ebea58..083582db94a 100644 --- a/.github/workflows/test-integration-platform.yml +++ b/.github/workflows/test-integration-platform.yml @@ -20,10 +20,6 @@ on: required: false type: string -env: - FELDERA_SENTRY_ENABLED: 1 - SENTRY_ENVIRONMENT: ci - jobs: manager-no-network: if: ${{ !contains(vars.CI_SKIP_JOBS, 'manager-no-network') }} diff --git a/crates/adapters/src/server.rs b/crates/adapters/src/server.rs index bd2ab59103a..08a1c593f05 100644 --- a/crates/adapters/src/server.rs +++ b/crates/adapters/src/server.rs @@ -759,18 +759,6 @@ pub fn run_server( eprintln!("{e}"); })?; - let _guard = observability::init( - "https://f0ec61ff0f8483e9ec8117645ad0c0e1@o4510219052253184.ingest.us.sentry.io/4510299519844352", - "pipeline", - env!("CARGO_PKG_VERSION"), - ); - let config_cln = config.clone(); - sentry::configure_scope(|scope| { - if let Some(id) = config_cln.name.as_ref() { - scope.set_tag("pipeline.name", id); - } - }); - // Initialize the logger by setting its filter and template. let pipeline_label = config .given_name @@ -802,7 +790,7 @@ pub fn run_server( } if config.global.tracing { warn!( - "Pipeline tracing was enabled but the 'tracing' option was deprecated, use `FELDERA_SENTRY_ENABLED` for tracing." + "Pipeline tracing was enabled but the 'tracing' option is deprecated and has no effect." ); } @@ -1005,38 +993,36 @@ pub fn run_server( let server = HttpServer::new({ move || { let state = state.clone(); - let app = App::new() - .wrap_fn(|req, srv| { - debug!("Request: {} {}", req.method(), req.path()); - srv.call(req).map(|res| { - match &res { - Ok(response) => { - let level = if response.status().is_success() - || response.status().is_redirection() - || response.status().is_informational() - { - Level::DEBUG - } else { - Level::ERROR - }; - let req = response.request(); - dyn_event!( - level, - "Response: {} (size: {:?}) to request {} {}", - response.status(), - response.response().body().size(), - req.method(), - req.path() - ); - } - Err(e) => { - error!("Service response error: {e}"); - } + let app = App::new().wrap_fn(|req, srv| { + debug!("Request: {} {}", req.method(), req.path()); + srv.call(req).map(|res| { + match &res { + Ok(response) => { + let level = if response.status().is_success() + || response.status().is_redirection() + || response.status().is_informational() + { + Level::DEBUG + } else { + Level::ERROR + }; + let req = response.request(); + dyn_event!( + level, + "Response: {} (size: {:?}) to request {} {}", + response.status(), + response.response().body().size(), + req.method(), + req.path() + ); } - res - }) + Err(e) => { + error!("Service response error: {e}"); + } + } + res }) - .wrap(observability::actix_middleware()); + }); build_app(app, state) } }) diff --git a/crates/fda/Cargo.toml b/crates/fda/Cargo.toml index a123a4e24e7..e22e64ab554 100644 --- a/crates/fda/Cargo.toml +++ b/crates/fda/Cargo.toml @@ -27,8 +27,6 @@ feldera-rest-api = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true, features = ["env-filter"] } tracing-log = { workspace = true } -sentry = { workspace = true } -feldera-observability = { workspace = true } tabled = { workspace = true, features = ["macros", "ansi"] } json_to_table = { workspace = true } rustyline = { workspace = true, features = ["with-file-history"] } diff --git a/crates/fda/src/main.rs b/crates/fda/src/main.rs index d761d82cc5f..36afd8d9d8f 100644 --- a/crates/fda/src/main.rs +++ b/crates/fda/src/main.rs @@ -3,7 +3,6 @@ use chrono::Utc; use clap::{CommandFactory, Parser}; use clap_complete::CompleteEnv; -use feldera_observability as observability; use feldera_rest_api::types::*; use feldera_rest_api::*; use feldera_types::config::{FtModel, RuntimeConfig, StorageOptions}; @@ -3696,11 +3695,6 @@ async fn cluster(format: OutputFormat, action: ClusterAction, client: Client) { } fn main() { - let _guard = observability::init( - "https://18aa37ae23e7130b57b91aaad432bc18@o4510219052253184.ingest.us.sentry.io/4510298809827328", - "fda", - env!("CARGO_PKG_VERSION"), - ); init_logging("warn"); tokio::runtime::Builder::new_multi_thread() @@ -3762,7 +3756,6 @@ fn init_logging(default_level: &str) { EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(default_level)); let _ = LogTracer::init(); let _ = tracing_subscriber::registry() - .with(sentry::integrations::tracing::layer()) .with(filter) .with( tracing_subscriber::fmt::layer() diff --git a/crates/feldera-observability/Cargo.toml b/crates/feldera-observability/Cargo.toml index 4a106f99abd..65df10ccd87 100644 --- a/crates/feldera-observability/Cargo.toml +++ b/crates/feldera-observability/Cargo.toml @@ -14,7 +14,6 @@ rust-version = { workspace = true } awc = { workspace = true } actix-http = { workspace = true } reqwest = { workspace = true } -sentry = { workspace = true } chrono = { workspace = true, features = ["now"] } serde_json = { workspace = true } tracing = { workspace = true } diff --git a/crates/feldera-observability/src/json_logging.rs b/crates/feldera-observability/src/json_logging.rs index 1b74389191f..a608257364a 100644 --- a/crates/feldera-observability/src/json_logging.rs +++ b/crates/feldera-observability/src/json_logging.rs @@ -303,7 +303,6 @@ fn init_logging( .with_ansi(false), ) .with(env_filter) - .with(sentry::integrations::tracing::layer()) .try_init() } else { tracing_subscriber::registry() @@ -313,7 +312,6 @@ fn init_logging( .fmt_fields(plain_text_fields()), ) .with(env_filter) - .with(sentry::integrations::tracing::layer()) .try_init() } } diff --git a/crates/feldera-observability/src/lib.rs b/crates/feldera-observability/src/lib.rs index 63ecb33e2da..89f4dcbc277 100644 --- a/crates/feldera-observability/src/lib.rs +++ b/crates/feldera-observability/src/lib.rs @@ -1,127 +1,3 @@ -use actix_http::header::{HeaderName, HeaderValue}; -use awc::ClientRequest; -use reqwest::RequestBuilder; -use sentry::{ClientInitGuard, TransactionContext}; -use std::borrow::Cow; -use std::env; - -/// Initializes Sentry if `FELDERA_SENTRY_ENABLED` is set, tagging the service and release. -/// -/// Use dsn as the default DSN, which can be overridden by the `SENTRY_DSN` environment variable. -pub fn init(dsn: &str, service_name: &str, release: &str) -> Option { - env::var("FELDERA_SENTRY_ENABLED") - .ok() - .filter(|v| v == "1")?; - - const DEFAULT_SAMPLE_RATE: f32 = 1.0; - const DEFAULT_TRACE_SAMPLE_RATE: f32 = 0.1; - const MAX_BREADCRUMBS: usize = 10; - - let dsn = env::var("SENTRY_DSN").ok().unwrap_or(dsn.to_string()); - let sample_rate = env::var("SENTRY_SAMPLE_RATE") - .ok() - .and_then(|rate| rate.parse::().ok()) - .unwrap_or(DEFAULT_SAMPLE_RATE); - let traces_sample_rate = env::var("SENTRY_TRACES_SAMPLE_RATE") - .ok() - .and_then(|rate| rate.parse::().ok()) - .unwrap_or(DEFAULT_TRACE_SAMPLE_RATE); - let environment = env::var("SENTRY_ENVIRONMENT") - .unwrap_or_else(|_| String::from("dev")) - .into(); - let accept_invalid_certs = environment == "ci" || environment == "dev"; - let max_breadcrumbs = env::var("SENTRY_MAX_BREADCRUMBS") - .ok() - .and_then(|rate| rate.parse::().ok()) - .unwrap_or(MAX_BREADCRUMBS); - - let guard = sentry::init(( - dsn, - sentry::ClientOptions { - environment: Some(environment), - release: Some(Cow::Owned(release.to_string())), - max_breadcrumbs, - sample_rate, - traces_sample_rate, - enable_logs: true, - attach_stacktrace: true, - accept_invalid_certs, - ..Default::default() - }, - )); - - sentry::configure_scope(|scope| scope.set_tag("service", service_name)); - Some(guard) -} - -fn sentry_enabled() -> bool { - sentry::Hub::current().client().is_some() -} - -/// Returns an Actix middleware that captures errors and traces when Sentry is enabled. -pub fn actix_middleware() -> sentry::integrations::actix::Sentry { - sentry::integrations::actix::Sentry::builder() - .emit_header(true) - .capture_server_errors(true) - .start_transaction(true) - .finish() -} - pub mod fips; pub mod json_logging; pub mod system; - -fn trace_header_value() -> Option { - if !sentry_enabled() { - return None; - } - - let mut header = None; - sentry::configure_scope(|scope| { - if let Some(span) = scope.get_span() { - header = span.iter_headers().next().map(|(_, value)| value); - } - }); - if header.is_none() { - let transaction = - sentry::start_transaction(TransactionContext::new("http.client", "http.client")); - header = transaction.iter_headers().next().map(|(_, value)| value); - transaction.finish(); - } - header -} - -/// Adds Sentry trace headers to outgoing awc requests. -pub trait AwcRequestTracingExt { - fn with_sentry_tracing(self) -> Self; -} - -impl AwcRequestTracingExt for ClientRequest { - fn with_sentry_tracing(mut self) -> Self { - if let Some(value) = trace_header_value() - && let Ok(header_value) = HeaderValue::from_str(&value) - { - self = self.insert_header((HeaderName::from_static("sentry-trace"), header_value)); - } - self - } -} - -/// Adds Sentry trace headers to outgoing reqwest requests. -pub trait ReqwestTracingExt { - fn with_sentry_tracing(self) -> Self; -} - -impl ReqwestTracingExt for RequestBuilder { - fn with_sentry_tracing(self) -> Self { - if let Some(value) = trace_header_value() - && let Ok(header_value) = reqwest::header::HeaderValue::from_str(&value) - { - return self.header( - reqwest::header::HeaderName::from_static("sentry-trace"), - header_value, - ); - } - self - } -} diff --git a/crates/pipeline-manager/Cargo.toml b/crates/pipeline-manager/Cargo.toml index fcbdd4e362c..19de0d713d1 100644 --- a/crates/pipeline-manager/Cargo.toml +++ b/crates/pipeline-manager/Cargo.toml @@ -27,7 +27,6 @@ feldera-ir = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true, features = ["env-filter", "json"] } colored = { workspace = true } -sentry = { workspace = true } feldera-observability = { workspace = true } # Error handling and asserts diff --git a/crates/pipeline-manager/src/bin/pipeline-manager.rs b/crates/pipeline-manager/src/bin/pipeline-manager.rs index b53b48c78db..6ad3e23def9 100644 --- a/crates/pipeline-manager/src/bin/pipeline-manager.rs +++ b/crates/pipeline-manager/src/bin/pipeline-manager.rs @@ -25,11 +25,6 @@ use utoipa::OpenApi; fn main() -> anyhow::Result<()> { ensure_default_crypto_provider(); init_fd_limit(); - let _guard = observability::init( - "https://18aa37ae23e7130b57b91aaad432bc18@o4510219052253184.ingest.us.sentry.io/4510298809827328", - "pipeline-manager", - env!("CARGO_PKG_VERSION"), - ); pipeline_manager::logging::init_service_logging( "[manager]".cyan(), feldera_observability::json_logging::ServiceName::Manager, diff --git a/crates/pipeline-manager/src/cluster_monitor.rs b/crates/pipeline-manager/src/cluster_monitor.rs index b91b28a88ed..fdfb0535c6c 100644 --- a/crates/pipeline-manager/src/cluster_monitor.rs +++ b/crates/pipeline-manager/src/cluster_monitor.rs @@ -5,7 +5,6 @@ use crate::db::storage_postgres::StoragePostgres; use crate::db::types::monitor::{MonitorStatus, NewClusterMonitorEvent}; use crate::error::source_error; use async_trait::async_trait; -use feldera_observability::ReqwestTracingExt; use std::{sync::Arc, time::Duration}; use tokio::sync::Mutex; use tracing::{error, info}; @@ -276,7 +275,6 @@ async fn poll_service_health_endpoint( match client .get(url) .timeout(DEFAULT_REQUEST_TIMEOUT) - .with_sentry_tracing() .send() .await { diff --git a/crates/pipeline-manager/src/compiler/sql_compiler.rs b/crates/pipeline-manager/src/compiler/sql_compiler.rs index 0026d229856..1f9a46f73cc 100644 --- a/crates/pipeline-manager/src/compiler/sql_compiler.rs +++ b/crates/pipeline-manager/src/compiler/sql_compiler.rs @@ -18,7 +18,6 @@ use crate::db::types::version::Version; use crate::error::source_error; use crate::has_unstable_feature; use feldera_ir::Dataflow; -use feldera_observability::ReqwestTracingExt; use futures_util::StreamExt; use indoc::formatdoc; use serde::{Deserialize, Serialize}; @@ -445,7 +444,6 @@ async fn fetch_sql_compiler( let response = client .get(&jar_cache_url) - .with_sentry_tracing() .send() .await .map_err(|e| { diff --git a/crates/pipeline-manager/src/runner/interaction.rs b/crates/pipeline-manager/src/runner/interaction.rs index c37b541c2bf..f5a98ac334d 100644 --- a/crates/pipeline-manager/src/runner/interaction.rs +++ b/crates/pipeline-manager/src/runner/interaction.rs @@ -12,7 +12,6 @@ use actix_ws::{CloseCode, CloseReason}; use awc::error::{ConnectError, SendRequestError}; use awc::{ClientRequest, ClientResponse}; use crossbeam::sync::ShardedLock; -use feldera_observability::AwcRequestTracingExt; use feldera_types::query::{MAX_WS_FRAME_SIZE, WS_SUBPROTOCOL}; use std::fmt::Display; use std::{collections::HashMap, sync::Arc, time::Duration}; @@ -274,11 +273,7 @@ impl RunnerInteraction { query_string, ); let timeout = timeout.unwrap_or(Self::PIPELINE_HTTP_REQUEST_TIMEOUT); - let request = client - .request(method, &url) - .timeout(timeout) - .force_close() - .with_sentry_tracing(); + let request = client.request(method, &url).timeout(timeout).force_close(); let request_str = Self::format_request(&request); let mut original_response = request.send().await.map_err(|e| match e { @@ -586,7 +581,6 @@ impl RunnerInteraction { let response = client .request(Method::GET, &url) .timeout(Self::RUNNER_HTTP_REQUEST_TIMEOUT) - .with_sentry_tracing() .send() .await .map_err(|e| match e { @@ -674,7 +668,7 @@ pub(crate) async fn streaming_proxy( new_request = new_request.append_header(header); } - let new_request = new_request.with_sentry_tracing(); + let new_request = new_request; let request_str = RunnerInteraction::format_request(&new_request); // Perform request to the pipeline diff --git a/crates/pipeline-manager/src/runner/local_runner.rs b/crates/pipeline-manager/src/runner/local_runner.rs index af7c58d5742..2cede4190a1 100644 --- a/crates/pipeline-manager/src/runner/local_runner.rs +++ b/crates/pipeline-manager/src/runner/local_runner.rs @@ -13,7 +13,6 @@ use crate::runner::error::RunnerError; use crate::runner::pipeline_executor::{PipelineExecutor, ProvisionStatus}; use crate::runner::pipeline_logs::{LogMessage, LogsSender}; use async_trait::async_trait; -use feldera_observability::ReqwestTracingExt; use feldera_observability::system::total_memory_megabyte; use feldera_types::config::{ PipelineConfig, PipelineConfigProgramInfo, RuntimeConfig, StorageCacheConfig, StorageConfig, @@ -489,7 +488,7 @@ impl LocalRunner { // Perform request let mut attempt = 1; loop { - match self.client.get(file_url).with_sentry_tracing().send().await { + match self.client.get(file_url).send().await { Ok(response) => { // Check status code if response.status() != StatusCode::OK { diff --git a/crates/pipeline-manager/src/runner/pipeline_automata.rs b/crates/pipeline-manager/src/runner/pipeline_automata.rs index f72106b8dd0..83b6835e183 100644 --- a/crates/pipeline-manager/src/runner/pipeline_automata.rs +++ b/crates/pipeline-manager/src/runner/pipeline_automata.rs @@ -20,7 +20,6 @@ use crate::runner::interaction::{format_pipeline_url, format_timeout_error_messa use crate::runner::pipeline_executor::{PipelineExecutor, ProvisionStatus}; use crate::runner::pipeline_logs::{LogMessage, LogsSender, start_thread_pipeline_logs}; use chrono::Utc; -use feldera_observability::ReqwestTracingExt; use feldera_types::error::ErrorResponse; use feldera_types::runtime_status::{ ExtendedRuntimeStatus, RuntimeDesiredStatus, RuntimeStatus, RuntimeStatusDetails, @@ -815,7 +814,6 @@ impl PipelineAutomaton { .client .request(method, &url) .timeout(timeout) - .with_sentry_tracing() .send() .await .map_err(|e| { @@ -1068,7 +1066,6 @@ impl PipelineAutomaton { .client .get(&binary_check_url) .timeout(Self::PIPELINE_STATUS_HTTP_REQUEST_TIMEOUT) - .with_sentry_tracing() .send() .await { diff --git a/crates/rest-api/Cargo.toml b/crates/rest-api/Cargo.toml index 16f656f5428..c030c2e5897 100644 --- a/crates/rest-api/Cargo.toml +++ b/crates/rest-api/Cargo.toml @@ -22,8 +22,6 @@ reqwest = { workspace = true, features = ["json", "stream"] } feldera-types = { workspace = true } progenitor-client = { workspace = true } rustversion.workspace = true -sentry = { workspace = true } -feldera-observability = { workspace = true } [build-dependencies] prettyplease = { workspace = true } diff --git a/crates/rest-api/build.rs b/crates/rest-api/build.rs index a2fe10ce9f4..10744bd3799 100644 --- a/crates/rest-api/build.rs +++ b/crates/rest-api/build.rs @@ -239,16 +239,7 @@ fn main() { let tokens = generator.generate_tokens(&spec).unwrap(); let ast = syn::parse2(tokens).unwrap(); - let mut content = prettyplease::unparse(&ast); - for verb in ["get", "post", "put", "patch", "delete"] { - let pattern = format!(".{verb}(url)"); - let replacement = format!(".{verb}(url)\n .with_sentry_tracing()"); - content = content.replace(&pattern, &replacement); - } - content = content.replace( - "pub mod builder {", - "pub mod builder {\n use feldera_observability::ReqwestTracingExt;", - ); + let content = prettyplease::unparse(&ast); let content = content.replace( "impl Client", "#[rustversion::attr(since(1.89), allow(mismatched_lifetime_syntaxes))]\nimpl Client", diff --git a/crates/rest-api/src/lib.rs b/crates/rest-api/src/lib.rs index 37f471037de..8e96d844222 100644 --- a/crates/rest-api/src/lib.rs +++ b/crates/rest-api/src/lib.rs @@ -1,7 +1,5 @@ #![allow(clippy::all, unused)] -use feldera_observability::ReqwestTracingExt; - include!(concat!(env!("OUT_DIR"), "/codegen.rs")); #[cfg(test)] From 866ecd640f34b09ac62e59e7357c3eb917065aea Mon Sep 17 00:00:00 2001 From: Gerd Zellweger Date: Sat, 8 Aug 2026 09:39:47 -0700 Subject: [PATCH 05/19] ci: fail when a second TLS implementation enters the dependency tree AWS-LC keeps OpenSSL's symbol names, so a binary containing both resolves each name to whichever archive the linker reaches first, and memory allocated by one library gets freed by the other. That corrupts the heap and surfaces far from the cause, which is what made it expensive to diagnose. The hook blocks openssl, native-tls and boring, asserts that openssl-sys resolves to an AWS-LC backend rather than the system OpenSSL, and reports ring without failing, since rustls still pulls it through sqlx. It reads the lockfile only, so it compiles nothing and runs in about a second. Signed-off-by: Gerd Zellweger --- .pre-commit-config.yaml | 7 +++ scripts/validate-crypto-deps.sh | 105 ++++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+) create mode 100755 scripts/validate-crypto-deps.sh diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e286a28d442..9e3502b6be0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -46,6 +46,13 @@ repos: exclude: 'crates/pipeline-manager/migrations/.*|.*\.patch|.*\.svg|.*\.java' - repo: local hooks: + - id: validate-crypto-deps + name: Validate cryptography dependencies + description: Cryptography goes through AWS-LC; a second implementation corrupts the heap. + entry: scripts/validate-crypto-deps.sh + language: system + files: Cargo\.(toml|lock)$ + pass_filenames: false - id: fix-rust-mod-naming name: Rust module naming entry: scripts/fix-rust-module-naming diff --git a/scripts/validate-crypto-deps.sh b/scripts/validate-crypto-deps.sh new file mode 100755 index 00000000000..053633a94fb --- /dev/null +++ b/scripts/validate-crypto-deps.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# Fails if a second TLS or cryptography implementation is in the dependency tree. +# +# Feldera routes cryptography through AWS-LC so that a single validated module +# serves the whole binary. A second implementation is a correctness problem, not +# only a compliance one: AWS-LC keeps OpenSSL's symbol names, so a binary +# containing both resolves each name to whichever archive the linker reaches +# first. Memory allocated by one library then gets freed by the other, which +# corrupts the heap and surfaces far from the cause. +# +# Blocked: +# openssl the crate that links a second implementation directly +# native-tls how openssl usually arrives; on macOS and Windows it binds +# Security.framework and schannel instead, so checking for +# openssl alone would miss a second TLS stack on those platforms +# boring BoringSSL wrapper, same symbol-collision class +# +# Allowed: +# openssl-sys the FFI layer, but only when it resolves to an AWS-LC backend, +# which this script asserts; librdkafka links whatever it +# resolves, so that pin is what keeps Kafka TLS on AWS-LC +# aws-lc-sys AWS-LC itself, symbol-prefixed by aws-lc-fips-sys/aws-lc-sys +# +# `ring` should also go: it is a second cryptographic implementation and is not +# FIPS-validated. The switch is not finished, so it cannot be blocked outright +# yet; what remains reaches it through crates pinned to object_store 0.13, which +# have to move together. Until then this ratchets, failing when a crate outside +# the known set starts pulling ring, so the remainder shrinks and never grows. +# +# This only resolves the lockfile, so it compiles nothing. +set -euo pipefail + +# `(*)` marks a subtree cargo already printed; strip it so entries dedupe. +packages=$(cargo tree --target all --edges normal,build,dev --prefix none --format "{p}" 2>/dev/null | + sed 's/ (\*)$//' | sort -u) + +blocked=$(echo "$packages" | grep -E "^(openssl|native-tls|boring|boring-sys) v" || true) +if [ -n "$blocked" ]; then + echo "error: a second TLS or cryptography implementation is in the dependency tree:" + echo "$blocked" | sed 's/^/ /' + echo + echo "It is almost always pulled in by a dependency defaulting to native-tls." + echo "Find who enables it with, for example:" + echo + echo " cargo tree -e features -i native-tls --target all" + echo + echo "then set default-features = false on that dependency and select its rustls" + echo "feature instead. Note that a workspace member cannot override" + echo "default-features on an inherited dependency; change it at the workspace root." + exit 1 +fi + +# openssl-sys is only safe while its backend is AWS-LC. Its aws-lc-fips and +# aws-lc features add aws-lc-fips-sys or aws-lc-sys as a direct dependency; +# without either it links the system OpenSSL, which is a second implementation +# wearing the allowed name. +if echo "$packages" | grep -qE "^openssl-sys v"; then + backend=$(cargo tree -p openssl-sys --depth 1 --target all --prefix none --format "{p}" 2>/dev/null | + grep -E "^aws-lc(-fips)?-sys v" || true) + if [ -z "$backend" ]; then + echo "error: openssl-sys is present but does not resolve to an AWS-LC backend." + echo + echo "It links the system OpenSSL, which reintroduces the implementation the" + echo "rest of this check exists to keep out. Restore its backend feature:" + echo + echo ' openssl-sys = { version = "...", default-features = false, features = ["aws-lc-fips"] }' + exit 1 + fi +fi + +# Crates still known to pull ring. Shrink this list; do not extend it. +RING_ALLOWED="object_store parquet rustls rustls-webpki" + +# `--depth 1` on the inverted tree lists ring plus exactly its direct parents. +ring_parents=$(cargo tree --invert ring --depth 1 --target all --edges normal,build,dev \ + --prefix none --format "{p}" 2>/dev/null | + sed 's/ (\*)$//' | awk 'NF {print $1}' | grep -v "^ring$" | sort -u) + +unexpected="" +for parent in $ring_parents; do + case " $RING_ALLOWED " in + *" $parent "*) ;; + *) unexpected="$unexpected $parent" ;; + esac +done + +if [ -n "$unexpected" ]; then + echo "error: a new dependency pulls in ring:" + for parent in $unexpected; do echo " $parent"; done + echo + echo "ring is a second cryptographic implementation and is not FIPS-validated." + echo "Prefer the aws-lc-rs backend where a dependency offers the choice, usually" + echo "a feature named aws-lc-rs or a -no-provider variant that defers to the" + echo "process default. Check whether a newer release drops ring outright before" + echo "assuming a fork is needed." + echo + echo " cargo tree -e features -i ring --target all" + exit 1 +fi + +if [ -n "$ring_parents" ]; then + echo "note: ring is still reached through:" $ring_parents + echo "It is not FIPS-validated and should go; these are pinned to object_store" + echo "0.13 and have to move together." +fi From aa754f535b497fb02c22651d469ade6a239957a9 Mon Sep 17 00:00:00 2001 From: Gerd Zellweger Date: Sat, 8 Aug 2026 10:46:40 -0700 Subject: [PATCH 06/19] deps: take etl from the feldera fork for its sqlx TLS backend sqlx's tls-rustls feature aliases tls-rustls-ring, and sqlx prefers ring whenever both backends are enabled, so a consumer cannot select aws-lc-rs by adding a feature; the choice has to be made where the dependency is declared. The fork pins the same upstream revision with that one line changed. feldera/etl branch sqlx-aws-lc-rs-main carries the same change against upstream main, expressed as a feature so consumers can pick, ready to propose upstream. Signed-off-by: Gerd Zellweger --- Cargo.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index b9e7b991fb9..6f463fd8823 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -332,9 +332,9 @@ zip = "6.0.0" zstd = "0.12.0" backtrace = "0.3.75" parking_lot = "0.12.4" -etl = { git = "https://github.com/supabase/etl", rev = "b2aab62d" } -etl-config = { git = "https://github.com/supabase/etl", rev = "b2aab62d" } -etl-postgres = { git = "https://github.com/supabase/etl", rev = "b2aab62d" } +etl = { git = "https://github.com/feldera/etl", rev = "91260556931fba2f55db9ebc11775e295c453bc9" } +etl-config = { git = "https://github.com/feldera/etl", rev = "91260556931fba2f55db9ebc11775e295c453bc9" } +etl-postgres = { git = "https://github.com/feldera/etl", rev = "91260556931fba2f55db9ebc11775e295c453bc9" } [workspace.metadata.release] release = false From e3fb1d773f3dfe61e4b441ccb5a4d3a3307d7f81 Mon Sep 17 00:00:00 2001 From: Gerd Zellweger Date: Sat, 8 Aug 2026 10:53:00 -0700 Subject: [PATCH 07/19] deps: take feldera-cloud1-client 0.1.3 0.1.2 declared reqwest without default-features = false, so it enabled default-tls, which pulls native-tls and with it the OpenSSL crate. 0.1.3 is built against reqwest 0.13 with rustls pinned explicitly. This was the last dependency reintroducing OpenSSL: the crate, native-tls and hyper-tls are now all absent from the tree, and openssl-sys resolves to the AWS-LC FIPS backend. Signed-off-by: Gerd Zellweger --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 6f463fd8823..82f75d5d8ad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -147,7 +147,7 @@ fake = "2.10" fastbloom = "0.14.0" fdlimit = "0.3.0" feldera-buffer-cache = { version = "0.334.0", path = "crates/buffer-cache" } -feldera-cloud1-client = "0.1.2" +feldera-cloud1-client = "0.1.3" feldera-datagen = { path = "crates/datagen" } feldera-fxp = { version = "0.334.0", path = "crates/fxp", features = ["dbsp"] } feldera-iceberg = { path = "crates/iceberg" } From 309b4ba67fe1bcdee07e1081d6899a69d7b5e3ba Mon Sep 17 00:00:00 2001 From: Gerd Zellweger Date: Sat, 8 Aug 2026 11:22:28 -0700 Subject: [PATCH 08/19] deps: move the remaining rustls consumers to aws-lc-rs async-nats defaults to ring and offers aws-lc-rs. aws-sigv4 1.4.5 dropped ring entirely, so a lockfile bump suffices. refinery needs no TLS backend at all, because migrations run on a client this crate creates, and its tokio-postgres-rustls feature pinned an older tokio-postgres-rustls that selects ring. reqwest builds without a provider and takes the process default, which every binary installs as aws-lc-rs; fda did not install one, so it does now. object_store 0.14 selects aws-lc-rs where 0.12 used ring. The upgrade renamed one error variant and deprecated Path::child in favour of Path::join, which consumes self rather than borrowing. Signed-off-by: Gerd Zellweger --- Cargo.toml | 26 ++++++++++++++++--- crates/adapters/src/controller.rs | 4 +-- crates/adapters/src/controller/journal.rs | 4 +-- crates/dbsp/src/circuit/checkpointer.rs | 6 ++--- crates/dbsp/src/circuit/circuit_builder.rs | 2 +- crates/dbsp/src/circuit/runtime.rs | 2 +- .../balance/accumulate_trace_balanced.rs | 3 ++- .../operator/dynamic/time_series/window.rs | 2 +- crates/dbsp/src/operator/output.rs | 5 ++-- crates/dbsp/src/operator/transaction_z1.rs | 3 ++- crates/dbsp/src/operator/z1.rs | 3 ++- .../dbsp/src/storage/backend/posixio_impl.rs | 3 ++- crates/dbsp/src/trace/spine_async.rs | 5 ++-- crates/fda/Cargo.toml | 1 + crates/fda/src/main.rs | 5 ++++ crates/pipeline-manager/Cargo.toml | 7 ++--- crates/storage/src/error.rs | 2 +- crates/storage/src/lib.rs | 2 +- 18 files changed, 59 insertions(+), 26 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 82f75d5d8ad..367c4784c00 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -61,7 +61,23 @@ arrow-json = "58" arrow-schema = "58" ascii_table = "=4.0.2" async-channel = "2.3.1" -async-nats = "0.47.0" +# Its default feature set selects ring; aws-lc-rs is the same choice made +# elsewhere in this workspace. The rest of the list restates the defaults, so +# revisit it when upgrading. +async-nats = { version = "0.47.0", default-features = false, features = [ + "server_2_10", + "server_2_11", + "server_2_12", + "service", + "aws-lc-rs", + "jetstream", + "nkeys", + "crypto", + "object-store", + "kv", + "websockets", + "nuid", +] } async-std = "1.12.0" async-stream = "0.3.5" async-trait = "0.1" @@ -213,7 +229,8 @@ num-bigint = "0.4.6" num-derive = "0.4.2" num-format = "0.4.0" num-traits = "0.2.19" -object_store = "0.12.1" +# 0.14 selects aws-lc-rs for request signing where 0.12 used ring. +object_store = "0.14.1" once_cell = "1.20.2" ordered-float = { version = "4.2.0", features = ["serde"] } ouroboros = "0.18.4" @@ -247,7 +264,10 @@ refinery = { version = "0.9.2", default-features = false } regex = "1.10.2" # default-features off drops default-tls, which pulls native-tls and with it the # OpenSSL crate; every consumer here uses rustls instead. -reqwest = { version = "0.12.24", default-features = false, features = ["rustls-tls-native-roots", "charset", "http2", "macos-system-configuration"] } +# The -no-provider variant leaves the rustls provider to the process default, +# which every binary here installs as aws-lc-rs; the plain feature would pin +# ring instead. +reqwest = { version = "0.12.24", default-features = false, features = ["rustls-tls-native-roots-no-provider", "charset", "http2", "macos-system-configuration"] } reqwest-websocket = "0.5.0" rkyv = { version = "0.7.45", default-features = false } rmp-serde = "1.3.0" diff --git a/crates/adapters/src/controller.rs b/crates/adapters/src/controller.rs index ea8cea43b19..167340a4a1b 100644 --- a/crates/adapters/src/controller.rs +++ b/crates/adapters/src/controller.rs @@ -5477,7 +5477,7 @@ impl ControllerInit { ) -> Result { let checkpoint = Checkpoint::read( &*storage.backend, - &StoragePath::from(checkpoint_uuid.to_string()).child(STATE_FILE), + &StoragePath::from(checkpoint_uuid.to_string()).join(STATE_FILE), )?; Self::with_checkpoint_inner(layout, config, storage, checkpoint) } @@ -9431,7 +9431,7 @@ impl CheckpointThread { // [Checkpoint::write] commits to stable storage. self.checkpoint.write( &*self.storage, - &StoragePath::from(uuid.to_string()).child(STATE_FILE), + &StoragePath::from(uuid.to_string()).join(STATE_FILE), )?; self.checkpoint .write(&*self.storage, &StoragePath::from(STATE_FILE))?; diff --git a/crates/adapters/src/controller/journal.rs b/crates/adapters/src/controller/journal.rs index b267db59e69..4297d48bdc6 100644 --- a/crates/adapters/src/controller/journal.rs +++ b/crates/adapters/src/controller/journal.rs @@ -40,7 +40,7 @@ impl Journal { } pub fn read(&self, step: Step) -> Result, StepError> { - let path = self.path.child(format!("{step}.bin")); + let path = self.path.clone().join(format!("{step}.bin")); let data = match self.backend.read(&path) { Ok(data) => data, Err(error) if error.kind() == ErrorKind::NotFound => return Ok(None), @@ -63,7 +63,7 @@ impl Journal { } pub fn write(&self, record: &StepMetadata) -> Result<(), StepError> { - let path = self.path.child(format!("{}.bin", record.step)); + let path = self.path.clone().join(format!("{}.bin", record.step)); let mut data = FBuf::new(); rmp_serde::encode::write(&mut data, record).map_err(|error| StepError::EncodeError { path: self.path.as_ref().into(), diff --git a/crates/dbsp/src/circuit/checkpointer.rs b/crates/dbsp/src/circuit/checkpointer.rs index 5fa0dce0f86..bd7d7374e2d 100644 --- a/crates/dbsp/src/circuit/checkpointer.rs +++ b/crates/dbsp/src/circuit/checkpointer.rs @@ -295,7 +295,7 @@ impl Checkpointer { cp_dir: &StoragePath, ) -> Result<(), StorageError> { let deps: CheckpointDependencies = - match backend.read_json(&cp_dir.child(CHECKPOINT_DEPENDENCIES)) { + match backend.read_json(&cp_dir.clone().join(CHECKPOINT_DEPENDENCIES)) { Ok(d) => d, Err(error) if error.kind() == ErrorKind::NotFound => return Ok(()), Err(error) => return Err(error), @@ -357,7 +357,7 @@ impl Checkpointer { // checkpoint. Explicitly commit it so the durability guarantee // does not depend on the implicit fsync inside `complete()`. self.backend - .write(&Self::checkpoint_dir(uuid).child("CHECKPOINT"), FBuf::new()) + .write(&Self::checkpoint_dir(uuid).join("CHECKPOINT"), FBuf::new()) .and_then(|reader| reader.commit())?; let mut md = CheckpointMetadata { @@ -397,7 +397,7 @@ impl Checkpointer { self.backend .write_json( - &cp_dir.child(CHECKPOINT_DEPENDENCIES), + &cp_dir.clone().join(CHECKPOINT_DEPENDENCIES), &CheckpointDependenciesWrite { batches: &batches_vec, state_files: &state_files, diff --git a/crates/dbsp/src/circuit/circuit_builder.rs b/crates/dbsp/src/circuit/circuit_builder.rs index cacd8cda817..5a964fbeb3d 100644 --- a/crates/dbsp/src/circuit/circuit_builder.rs +++ b/crates/dbsp/src/circuit/circuit_builder.rs @@ -7385,7 +7385,7 @@ where /// Absolute path of the file holding this subcircuit's clock. fn clock_file(base: &StoragePath, persistent_id: &str) -> StoragePath { - base.child(format!("clock-{persistent_id}.dat")) + base.clone().join(format!("clock-{persistent_id}.dat")) } } diff --git a/crates/dbsp/src/circuit/runtime.rs b/crates/dbsp/src/circuit/runtime.rs index 8f88cedff03..dd70f18e34b 100644 --- a/crates/dbsp/src/circuit/runtime.rs +++ b/crates/dbsp/src/circuit/runtime.rs @@ -431,7 +431,7 @@ impl RuntimeInner { if let Some(init_checkpoint) = storage.init_checkpoint && !backend - .exists(&Checkpointer::checkpoint_dir(init_checkpoint).child("CHECKPOINT"))? + .exists(&Checkpointer::checkpoint_dir(init_checkpoint).join("CHECKPOINT"))? { return Err(DbspError::Storage(StorageError::CheckpointNotFound( init_checkpoint, diff --git a/crates/dbsp/src/operator/dynamic/balance/accumulate_trace_balanced.rs b/crates/dbsp/src/operator/dynamic/balance/accumulate_trace_balanced.rs index 2bc5fd9071a..374ef06153d 100644 --- a/crates/dbsp/src/operator/dynamic/balance/accumulate_trace_balanced.rs +++ b/crates/dbsp/src/operator/dynamic/balance/accumulate_trace_balanced.rs @@ -1031,7 +1031,8 @@ where } fn checkpoint_file(base: &StoragePath, persistent_id: &str) -> StoragePath { - base.child(format!("rebalancing-exchange-{}.dat", persistent_id)) + base.clone() + .join(format!("rebalancing-exchange-{}.dat", persistent_id)) } async fn send( diff --git a/crates/dbsp/src/operator/dynamic/time_series/window.rs b/crates/dbsp/src/operator/dynamic/time_series/window.rs index 982bc9dcc3b..f9954a8fd91 100644 --- a/crates/dbsp/src/operator/dynamic/time_series/window.rs +++ b/crates/dbsp/src/operator/dynamic/time_series/window.rs @@ -158,7 +158,7 @@ where /// Return the absolute path of the file for a checkpointed Window. fn checkpoint_file(base: &StoragePath, persistent_id: &str) -> StoragePath { - base.child(format!("window-{}.dat", persistent_id)) + base.clone().join(format!("window-{}.dat", persistent_id)) } } diff --git a/crates/dbsp/src/operator/output.rs b/crates/dbsp/src/operator/output.rs index 541c6505470..49ec222bfb3 100644 --- a/crates/dbsp/src/operator/output.rs +++ b/crates/dbsp/src/operator/output.rs @@ -511,7 +511,7 @@ where } fn checkpoint_file(base: &StoragePath, persistent_id: &str) -> StoragePath { - base.child(format!("output-{}.dat", persistent_id)) + base.clone().join(format!("output-{}.dat", persistent_id)) } } @@ -690,7 +690,8 @@ where } fn checkpoint_file(base: &StoragePath, persistent_id: &str) -> StoragePath { - base.child(format!("accumulate-output-{}.dat", persistent_id)) + base.clone() + .join(format!("accumulate-output-{}.dat", persistent_id)) } /// Merge `snapshot` into the cached accumulated output. diff --git a/crates/dbsp/src/operator/transaction_z1.rs b/crates/dbsp/src/operator/transaction_z1.rs index 63f97b81a82..36efb0cae66 100644 --- a/crates/dbsp/src/operator/transaction_z1.rs +++ b/crates/dbsp/src/operator/transaction_z1.rs @@ -94,7 +94,8 @@ where /// - `persistent_id`: The persistent id that identifies the spine within /// the circuit for a given checkpoint. fn checkpoint_file>(base: &StoragePath, persistent_id: P) -> StoragePath { - base.child(format!("transaction-z1-{}.dat", persistent_id.as_ref())) + base.clone() + .join(format!("transaction-z1-{}.dat", persistent_id.as_ref())) } } diff --git a/crates/dbsp/src/operator/z1.rs b/crates/dbsp/src/operator/z1.rs index 94e6fa9e5a6..11441dcd62a 100644 --- a/crates/dbsp/src/operator/z1.rs +++ b/crates/dbsp/src/operator/z1.rs @@ -270,7 +270,8 @@ where /// - `persistent_id`: The persistent id that identifies the spine within /// the circuit for a given checkpoint. fn checkpoint_file>(base: &StoragePath, persistent_id: P) -> StoragePath { - base.child(format!("z1-{}.dat", persistent_id.as_ref())) + base.clone() + .join(format!("z1-{}.dat", persistent_id.as_ref())) } } diff --git a/crates/dbsp/src/storage/backend/posixio_impl.rs b/crates/dbsp/src/storage/backend/posixio_impl.rs index 38d3f6a8635..ea34b47255e 100644 --- a/crates/dbsp/src/storage/backend/posixio_impl.rs +++ b/crates/dbsp/src/storage/backend/posixio_impl.rs @@ -539,7 +539,8 @@ impl StorageBackend for PosixBackend { Ok(entry) => { let entry = feldera_storage::DirEntry { name: parent - .child(StoragePathPart::from(entry.file_name().as_encoded_bytes())), + .clone() + .join(StoragePathPart::from(entry.file_name().as_encoded_bytes())), file_type: get_file_type(&entry), }; if let Err(e) = &entry.file_type diff --git a/crates/dbsp/src/trace/spine_async.rs b/crates/dbsp/src/trace/spine_async.rs index 5004042c704..7c20c13eaab 100644 --- a/crates/dbsp/src/trace/spine_async.rs +++ b/crates/dbsp/src/trace/spine_async.rs @@ -1856,12 +1856,13 @@ where /// - `persistent_id`: The persistent id that identifies the spine within /// the circuit for a given checkpoint. fn checkpoint_file(base: &StoragePath, persistent_id: &str) -> StoragePath { - base.child(format!("pspine-{}.dat", persistent_id)) + base.clone().join(format!("pspine-{}.dat", persistent_id)) } /// Return the absolute path of the file for this Spine's batchlist. fn batchlist_file(&self, base: &StoragePath, persistent_id: &str) -> StoragePath { - base.child(format!("pspine-batches-{}.dat", persistent_id)) + base.clone() + .join(format!("pspine-batches-{}.dat", persistent_id)) } } diff --git a/crates/fda/Cargo.toml b/crates/fda/Cargo.toml index e22e64ab554..f757f114b01 100644 --- a/crates/fda/Cargo.toml +++ b/crates/fda/Cargo.toml @@ -19,6 +19,7 @@ clap = { workspace = true, features = ["color"] } clap_complete = { workspace = true, features = ["unstable-dynamic"] } progenitor-client = { workspace = true } reqwest = { workspace = true, features = ["json", "stream"] } +rustls = { workspace = true } reqwest-websocket = { workspace = true } serde_json = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "macros", "io-std", "process"] } diff --git a/crates/fda/src/main.rs b/crates/fda/src/main.rs index 36afd8d9d8f..2049e91c158 100644 --- a/crates/fda/src/main.rs +++ b/crates/fda/src/main.rs @@ -3695,6 +3695,11 @@ async fn cluster(format: OutputFormat, action: ClusterAction, client: Client) { } fn main() { + // reqwest is built without a rustls provider, so the process default decides + // which one it uses. + let _ = rustls::crypto::CryptoProvider::install_default( + rustls::crypto::aws_lc_rs::default_provider(), + ); init_logging("warn"); tokio::runtime::Builder::new_multi_thread() diff --git a/crates/pipeline-manager/Cargo.toml b/crates/pipeline-manager/Cargo.toml index 19de0d713d1..7c05bb7c267 100644 --- a/crates/pipeline-manager/Cargo.toml +++ b/crates/pipeline-manager/Cargo.toml @@ -74,9 +74,10 @@ termbg = { workspace = true } tokio-postgres = { workspace = true, features = ["with-serde_json-1", "with-uuid-1", "with-chrono-0_4"] } deadpool-postgres = { workspace = true } postgresql_embedded = { workspace = true, optional = true } -# tokio-postgres-rustls rather than tokio-postgres: the latter pulls -# postgres-native-tls, and with it native-tls and the OpenSSL crate. -refinery = { workspace = true, features = ["tokio-postgres-rustls"] } +# Migrations run on a client this crate creates, so refinery needs no TLS +# backend of its own; its tokio-postgres-rustls feature would pin an older +# tokio-postgres-rustls that selects ring. +refinery = { workspace = true, features = ["tokio-postgres"] } # HTTP server and client actix-web = { workspace = true, features = ["rustls-0_23"] } diff --git a/crates/storage/src/error.rs b/crates/storage/src/error.rs index b4e3dab6070..36a9f21b3e3 100644 --- a/crates/storage/src/error.rs +++ b/crates/storage/src/error.rs @@ -84,7 +84,7 @@ impl From for StorageError { ObjectStoreError::NotFound { .. } => ErrorKind::NotFound, ObjectStoreError::NotSupported { .. } => ErrorKind::Unsupported, ObjectStoreError::AlreadyExists { .. } => ErrorKind::AlreadyExists, - ObjectStoreError::NotImplemented => ErrorKind::Unsupported, + ObjectStoreError::NotImplemented { .. } => ErrorKind::Unsupported, ObjectStoreError::PermissionDenied { .. } | ObjectStoreError::Unauthenticated { .. } => ErrorKind::PermissionDenied, ObjectStoreError::InvalidPath { .. } => { diff --git a/crates/storage/src/lib.rs b/crates/storage/src/lib.rs index f06aa16e496..93f9d5d87ad 100644 --- a/crates/storage/src/lib.rs +++ b/crates/storage/src/lib.rs @@ -229,7 +229,7 @@ impl dyn StorageBackend { // commit always writes it, and it captures the full list in one // place so a single read suffices. See `CheckpointDependencies` // for the accepted JSON forms. - let deps_path = checkpoint_dir.child(CHECKPOINT_DEPENDENCIES); + let deps_path = checkpoint_dir.clone().join(CHECKPOINT_DEPENDENCIES); match self.read_json::(&deps_path) { Ok(deps) => { return Ok(deps From 9dba2849aee7e38e0dba2e585a64c822fea11824 Mon Sep 17 00:00:00 2001 From: Gerd Zellweger Date: Sat, 8 Aug 2026 21:20:09 -0700 Subject: [PATCH 09/19] build: build librdkafka against AWS-LC rdkafka-sys otherwise compiles a vendored librdkafka against whatever OpenSSL pkg-config finds, which leaves Kafka TLS on a second cryptographic implementation no matter how the rest of the binary is built. scripts/install-librdkafka.sh builds AWS-LC with BUILD_LIBSSL, then librdkafka against it. Both container images run the script, and so must developers, since the next commit links librdkafka dynamically. Distribution packages are built against OpenSSL and are older than the version rdkafka-sys requires, so they are not a substitute. librdkafka 2.12.1 calls HMAC() without including . OpenSSL supplies the declaration transitively through x509.h and AWS-LC does not, so the call would compile as an implicit declaration returning int, truncating the returned pointer. The script patches it; confluentinc/librdkafka#5552 fixes it upstream but is unmerged. The librdkafka version comes from Cargo.lock, where rdkafka-sys names it in its own version as 4.10.0+2.12.1, so the library cannot drift from the crate expecting it. The configure flags mirror the cargo features rdkafka-sys used, and the script asserts each one: losing one is otherwise silent, and a codec or authentication mechanism simply stops being offered. Signed-off-by: Gerd Zellweger --- CONTRIBUTING.md | 22 ++++- deploy/Dockerfile | 47 ++++++++-- deploy/build.Dockerfile | 28 ++++-- scripts/install-librdkafka.sh | 163 ++++++++++++++++++++++++++++++++++ 4 files changed, 241 insertions(+), 19 deletions(-) create mode 100755 scripts/install-librdkafka.sh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5ae76a2b450..22ac49bf682 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -19,9 +19,10 @@ Our dependencies for building the project are: - C and C++ compiler toolchain (e.g., gcc, gcc++) - cmake - - libssl-dev - libsasl2-dev - zlib1g-dev + - libzstd-dev + - Go and perl, to build AWS-LC (see below) - a Rust tool chain (install rustup and the default toolchain) - a Java Virtual Machine (at least Java 19) - maven @@ -32,6 +33,25 @@ Our dependencies for building the project are: Additional dependencies are automatically installed by the Rust, maven, Python, and TypeScript build tools. +### librdkafka + +The Kafka connectors link librdkafka dynamically, so it has to be installed +before the workspace will build: + +``` +./scripts/install-librdkafka.sh +``` + +The script builds librdkafka against AWS-LC, which is what keeps Kafka TLS on +the same cryptographic implementation as the rest of the system. Distribution +packages are built against OpenSSL and are usually older than the version the +`rdkafka-sys` crate requires, so installing one of those is not equivalent. Run +the script again after a `rdkafka` version bump; it reads the version it needs +from `Cargo.lock`. + +Set `PREFIX` to install somewhere other than `/usr/local`, in which case +`PKG_CONFIG_PATH` has to point at `$PREFIX/lib/pkgconfig`. + ## Contribution Flow ### Forking diff --git a/deploy/Dockerfile b/deploy/Dockerfile index d461f1e07e1..8d7cb841b8f 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -1,20 +1,35 @@ +# librdkafka, built against AWS-LC rather than the system OpenSSL. +# +# This is a separate stage because building AWS-LC needs Go, cmake and perl, +# and none of them belong in the shipped image. Only the installed library +# crosses over. +FROM ubuntu:24.04 AS librdkafka +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update --fix-missing && apt-get install -y \ + git cmake build-essential perl pkg-config golang-go \ + libsasl2-dev zlib1g-dev libzstd-dev +COPY scripts/install-librdkafka.sh /tmp/install-librdkafka.sh +# The script reads the librdkafka version from the lockfile, so the library and +# the crate expecting it cannot drift apart. A stale image fails loudly at the +# pkg-config probe rather than building against the wrong version. +COPY Cargo.lock /tmp/Cargo.lock +RUN CARGO_LOCK=/tmp/Cargo.lock PREFIX=/usr/local bash /tmp/install-librdkafka.sh + # The base image contains tools to build the code given that # we need a Java and Rust compiler to run alongside the pipeline manager # as of now. This will change later. FROM ubuntu:24.04 AS base ENV DEBIAN_FRONTEND=noninteractive -# These two environment variables are used to make openssl-sys pick -# up libssl-dev and statically link it. Without it, our build defaults -# to building a vendored version of OpenSSL. RUN apt update --fix-missing && apt -y dist-upgrade && apt install \ # bindgen needs this (at least the dec crate uses bindgen) libclang-dev \ - # pkg-config is required for cargo to find libssl - libssl-dev pkg-config \ + # pkg-config is how rdkafka-sys locates the librdkafka copied in below + pkg-config \ cmake \ - # rdkafka dependency needs libsasl2-dev zlib and a CXX compiler - libsasl2-dev zlib1g-dev build-essential \ + # librdkafka links these; it is built in the stage above, but pipelines are + # compiled in this image and the linker resolves them here + libsasl2-dev zlib1g-dev libzstd-dev build-essential \ # To install rust curl \ # For running the SQL compiler @@ -39,11 +54,25 @@ RUN update-ca-certificates RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ locale-gen +# librdkafka from the stage above. rdkafka-sys links it dynamically, so it has +# to be here both to build pipelines and to run them. +COPY --from=librdkafka /usr/local/lib/librdkafka* /usr/local/lib/ +COPY --from=librdkafka /usr/local/include/librdkafka /usr/local/include/librdkafka +COPY --from=librdkafka /usr/local/lib/pkgconfig/rdkafka*.pc /usr/local/lib/pkgconfig/ +RUN ldconfig \ + # A NEEDED entry for libssl or libcrypto means librdkafka found a system + # OpenSSL and preferred it over AWS-LC, which is the failure this whole + # arrangement exists to prevent and is invisible at run time. + && if readelf -d /usr/local/lib/librdkafka.so.1 | grep -qE 'Shared library: \[lib(ssl|crypto)\.so'; then \ + echo "error: librdkafka links a shared OpenSSL" >&2; exit 1; \ + fi \ + # rdkafka-sys probes for librdkafka this way; failing here beats failing + # partway through a pipeline compilation. + && pkg-config --exists rdkafka + USER ubuntu WORKDIR /home/ubuntu -ENV OPENSSL_NO_VENDOR=1 -ENV OPENSSL_STATIC=1 ENV LC_ALL=en_US.UTF-8 ENV LANG=en_US.UTF-8 ENV LANGUAGE=en_US:en diff --git a/deploy/build.Dockerfile b/deploy/build.Dockerfile index 33f9ad4f4a1..2c115e05b76 100644 --- a/deploy/build.Dockerfile +++ b/deploy/build.Dockerfile @@ -9,19 +9,15 @@ FROM ubuntu:24.04 AS ubuntu-base ENV DEBIAN_FRONTEND=noninteractive -# These two environment variables are used to make openssl-sys pick -# up libssl-dev and statically link it. Without it, our build defaults -# to building a vendored version of OpenSSL. FROM ubuntu-base AS install-pkgs -ENV OPENSSL_NO_VENDOR=1 -ENV OPENSSL_STATIC=1 RUN apt-get update --fix-missing && apt-get install -y \ - # pkg-config is required for cargo to find libssl - libssl-dev pkg-config \ + # pkg-config is how rdkafka-sys locates librdkafka + pkg-config \ cmake \ - # Go is required to build aws-lc-fips-sys when rustls is built with FIPS + # Go builds AWS-LC, which librdkafka is linked against; see + # scripts/install-librdkafka.sh golang-go \ - # rdkafka dependency needs libsasl2-dev and a CXX compiler + # librdkafka links these libsasl2-dev libzstd-dev zlib1g-dev build-essential \ # zstd CLI: @actions/cache (runs-on/cache) auto-uses it for cache # compression when on PATH, else falls back to slow gzip. Big win for the @@ -163,3 +159,17 @@ RUN arch=`dpkg --print-architecture | sed "s/arm64/aarch64/g" | sed "s/amd64/x8 ENV RUSTFLAGS="-C link-arg=-fuse-ld=mold -C link-arg=-Wl,--compress-debug-sections=zlib" +# librdkafka, built against AWS-LC rather than the system OpenSSL. rdkafka is +# built with `dynamic-linking`, so it consumes this rather than compiling its +# own copy. +# +# Last in the file on purpose: it copies Cargo.lock, from which the script +# reads the librdkafka version the crate expects, and that file changes often. +# Keeping it here means a lockfile change rebuilds only this layer. +USER root +COPY scripts/install-librdkafka.sh /tmp/install-librdkafka.sh +COPY Cargo.lock /tmp/Cargo.lock +RUN CARGO_LOCK=/tmp/Cargo.lock PREFIX=/usr/local bash /tmp/install-librdkafka.sh \ + && rm -f /tmp/install-librdkafka.sh /tmp/Cargo.lock +USER ubuntu + diff --git a/scripts/install-librdkafka.sh b/scripts/install-librdkafka.sh new file mode 100755 index 00000000000..401f054b415 --- /dev/null +++ b/scripts/install-librdkafka.sh @@ -0,0 +1,163 @@ +#!/usr/bin/env bash +# Builds librdkafka against AWS-LC and installs both under $PREFIX. +# +# rdkafka-sys otherwise compiles a vendored librdkafka against whatever OpenSSL +# pkg-config finds, which puts Kafka TLS on a second cryptographic +# implementation. Building it here instead lets us point it at AWS-LC, and the +# `dynamic-linking` feature then makes rdkafka-sys consume this install rather +# than compile its own copy. +# +# The container images and CI run this script, and so should developers: the +# `dynamic-linking` feature is unconditional, so a build finds librdkafka here +# or not at all. Ubuntu's package is far older than the version rdkafka-sys +# requires, which is why this builds from source rather than calling apt. +# +# The librdkafka version is derived from the lockfile, so it cannot drift from +# what the crate expects: rdkafka-sys names its vendored version in its own, +# as `4.10.0+2.12.1`. +set -euo pipefail + +PREFIX="${PREFIX:-/usr/local}" +AWS_LC_REF="${AWS_LC_REF:-v1.68.0}" +# Switching to the FIPS-validated module means building AWS-LC with -DFIPS=1 +# and moving aws-lc-rs to aws-lc-fips-sys in the same change. Doing one without +# the other leaves two AWS-LC copies in the binary; validate-crypto-deps.sh +# fails when that happens. +AWS_LC_FIPS="${AWS_LC_FIPS:-0}" + +script_path="${BASH_SOURCE[0]:-$0}" +repo_root=$(cd "$(dirname "$script_path")/.." 2>/dev/null && pwd) || repo_root="." +lockfile="${CARGO_LOCK:-$repo_root/Cargo.lock}" + +if [ -n "${LIBRDKAFKA_REF:-}" ]; then + librdkafka_version="${LIBRDKAFKA_REF#v}" +elif [ -f "$lockfile" ]; then + # rdkafka-sys version is `+`. + librdkafka_version=$(grep -A1 'name = "rdkafka-sys"' "$lockfile" | + grep '^version' | sed 's/.*+//; s/"//') +else + echo "error: cannot determine the librdkafka version." >&2 + echo "Point CARGO_LOCK at a lockfile, or set LIBRDKAFKA_REF explicitly." >&2 + exit 1 +fi + +if [ -z "$librdkafka_version" ]; then + echo "error: no rdkafka-sys entry in $lockfile." >&2 + exit 1 +fi + +echo "librdkafka v$librdkafka_version, AWS-LC $AWS_LC_REF, prefix $PREFIX" + +for tool in cmake git perl go make cc; do + command -v "$tool" >/dev/null || { echo "error: $tool is required." >&2; exit 1; } +done + +jobs="${JOBS:-$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 4)}" +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT + +# AWS-LC. BUILD_LIBSSL is off by default and librdkafka links libssl, so the +# default build is not enough. The symbol prefix stays: aws-lc-rs links its own +# copy into the same binary, and the prefix is what keeps the two apart. +git clone --depth 1 --branch "$AWS_LC_REF" https://github.com/aws/aws-lc.git "$work/aws-lc" -q +cmake -S "$work/aws-lc" -B "$work/aws-lc-build" \ + -DCMAKE_BUILD_TYPE=Release \ + "-DCMAKE_C_FLAGS=-fPIC -w" \ + -DCMAKE_INSTALL_PREFIX="$work/aws-lc-install" \ + -DBUILD_LIBSSL=ON \ + -DBUILD_SHARED_LIBS=OFF \ + -DBUILD_TESTING=OFF \ + -DBUILD_TOOL=OFF \ + -DFIPS="$AWS_LC_FIPS" >"$work/aws-lc-configure.log" 2>&1 +if ! cmake --build "$work/aws-lc-build" --parallel "$jobs" >"$work/aws-lc-build.log" 2>&1; then + echo "error: AWS-LC build failed." >&2 + tail -40 "$work/aws-lc-build.log" >&2 + exit 1 +fi +cmake --install "$work/aws-lc-build" >"$work/aws-lc-install.log" 2>&1 + +aws_lc_lib=$(dirname "$(find "$work/aws-lc-install" -name libcrypto.a -print -quit)") +aws_lc_include="$work/aws-lc-install/include" +for lib in libcrypto.a libssl.a; do + [ -f "$aws_lc_lib/$lib" ] || { echo "error: AWS-LC did not produce $lib." >&2; exit 1; } +done + +# librdkafka. +git clone --depth 1 --branch "v$librdkafka_version" \ + https://github.com/confluentinc/librdkafka.git "$work/librdkafka" -q +cd "$work/librdkafka" + +# rdkafka_ssl.c calls HMAC() without including . OpenSSL +# supplies the declaration transitively through x509.h and AWS-LC does not, so +# the call compiles as an implicit declaration returning int, truncating the +# returned pointer. Fixed upstream in confluentinc/librdkafka#5552, unmerged; +# drop this once a release carries it. +if ! grep -q "openssl/hmac.h" src/rdkafka_ssl.c; then + perl -0pi -e 's{#include }{#include \n#include }' \ + src/rdkafka_ssl.c + grep -q "openssl/hmac.h" src/rdkafka_ssl.c || { + echo "error: could not apply the hmac.h patch; check whether upstream restructured the includes." >&2 + exit 1 + } +fi + +# These flags mirror the cargo features rdkafka-sys used to build with, so the +# connectors keep the same capabilities. Losing one here is silent: a codec or +# an authentication mechanism simply stops being offered. +CPPFLAGS="-I$aws_lc_include" \ +CFLAGS="-Werror=implicit-function-declaration" \ +LDFLAGS="-L$aws_lc_lib" \ +PKG_CONFIG_PATH="$aws_lc_lib/pkgconfig${PKG_CONFIG_PATH:+:$PKG_CONFIG_PATH}" \ +./configure --prefix="$PREFIX" \ + --enable-ssl \ + --enable-gssapi \ + --enable-zlib \ + --enable-zstd \ + --disable-curl \ + --disable-lz4-ext >"$work/configure.log" 2>&1 || { + echo "error: librdkafka configure failed." >&2 + tail -30 "$work/configure.log" >&2 + exit 1 + } + +for want in WITH_SSL WITH_SASL_CYRUS WITH_ZLIB WITH_ZSTD; do + grep -q "^${want}=[[:space:]]*y$" Makefile.config || { + echo "error: librdkafka configure did not enable $want." >&2 + echo "Its development package is probably missing; see Makefile.config." >&2 + exit 1 + } +done + +if ! make -j"$jobs" libs >"$work/build.log" 2>&1; then + echo "error: librdkafka build failed." >&2 + tail -40 "$work/build.log" >&2 + exit 1 +fi +if ! make install >"$work/install.log" 2>&1; then + echo "error: librdkafka install to $PREFIX failed." >&2 + tail -20 "$work/install.log" >&2 + exit 1 +fi + +# librdkafka must carry AWS-LC inside rather than link a system OpenSSL. A +# NEEDED entry for libssl or libcrypto means configure found one and preferred +# it, which silently reintroduces the implementation this script exists to +# avoid. +shared="" +for candidate in "$PREFIX/lib/librdkafka.so.1" "$PREFIX/lib/librdkafka.1.dylib"; do + if [ -f "$candidate" ]; then + shared="$candidate" + break + fi +done +if [ -n "$shared" ] && command -v readelf >/dev/null; then + if readelf -d "$shared" | grep -qE 'Shared library: \[lib(ssl|crypto)\.so'; then + echo "error: librdkafka links a shared OpenSSL:" >&2 + readelf -d "$shared" | grep -E 'Shared library: \[lib(ssl|crypto)' >&2 + exit 1 + fi +fi + +command -v ldconfig >/dev/null && ldconfig 2>/dev/null || true + +echo "installed librdkafka $librdkafka_version to $PREFIX" From d34103b5a0aed6c1c36a6768fc4d91882b821fe8 Mon Sep 17 00:00:00 2001 From: Gerd Zellweger Date: Sat, 8 Aug 2026 21:21:06 -0700 Subject: [PATCH 10/19] deps: link librdkafka dynamically, dropping openssl-sys rdkafka's ssl feature depends on openssl-sys and configures the vendored librdkafka build. dynamic-linking consumes the library that scripts/install-librdkafka.sh builds against AWS-LC instead, so neither the feature nor the crate is needed, and Kafka TLS stops being the one part of the system on a different cryptographic implementation. The ssl, gssapi, zstd and libz features only ever configured that vendored build; those capabilities now come from the installed library, where the script asserts them. Dropping the default libz feature also removes libz-sys, which was compiling a zlib nothing used. default-features has to be set at the workspace root: a member cannot override it on an inherited dependency. openssl-sys stays listed in Cargo.lock as an optional dependency of rdkafka-sys, but it is no longer in the build graph, so it is neither compiled nor linked. Signed-off-by: Gerd Zellweger --- Cargo.toml | 8 ++++---- crates/adapters/Cargo.toml | 14 +++++++++----- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 367c4784c00..689abc5b5fc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -258,7 +258,10 @@ rand_chacha = "0.3.1" rand_distr = "0.4.3" rand_xoshiro = "0.6.0" range-set = "0.0.11" -rdkafka = "0.39.0" +# Its default `libz` feature only configures the vendored librdkafka build, +# which `dynamic-linking` skips entirely; default-features has to be set here +# because a workspace member cannot override it on an inherited dependency. +rdkafka = { version = "0.39.0", default-features = false } redis = "0.28.2" refinery = { version = "0.9.2", default-features = false } regex = "1.10.2" @@ -279,9 +282,6 @@ rstest = "0.15" # Make sure this is the same rustls version used by the `tonic` crate. # See the `ensure_default_crypto_provider` function. rustls = "0.23.12" -# librdkafka links whatever openssl-sys resolves, so this is what puts Kafka -# TLS inside the validated module. -openssl-sys = { version = "0.9.116", default-features = false, features = ["aws-lc-fips"] } rustls-pemfile = "2.2.0" rustls-native-certs = "0.7.3" rustversion = "1.0" diff --git a/crates/adapters/Cargo.toml b/crates/adapters/Cargo.toml index 3a14cf9fc2a..0fb9c8533dd 100644 --- a/crates/adapters/Cargo.toml +++ b/crates/adapters/Cargo.toml @@ -107,11 +107,16 @@ serde_json = { workspace = true, features = ["raw_value"] } serde_urlencoded = { workspace = true } form_urlencoded = { workspace = true } csv = { workspace = true } +# dynamic-linking makes rdkafka-sys consume the librdkafka that +# scripts/install-librdkafka.sh builds against AWS-LC, instead of compiling its +# own copy against whatever OpenSSL pkg-config happens to find. The ssl, +# gssapi, zstd and libz features only configured that vendored build, so they +# go: those capabilities now come from the installed library, and the configure +# flags in the script are what preserve them. `ssl` in particular is the only +# thing that pulled openssl-sys into the tree. rdkafka = { workspace = true, features = [ - "ssl", - "gssapi", - "zstd", - "libz", + "dynamic-linking", + "tokio", ], optional = true } aws-config = { workspace = true } aws-sdk-dynamodb = { workspace = true, optional = true } @@ -211,7 +216,6 @@ feldera-ir = { workspace = true } aws-lc-rs = { workspace = true } tokio-postgres-rustls = { workspace = true } rustls-pemfile = { workspace = true } -openssl-sys = { workspace = true } base64 = { workspace = true } aws-msk-iam-sasl-signer = "1.0.1" aws-credential-types = "1.2.3" From c11fcae6c550a18c85a9540397e97c36ca8d7247 Mon Sep 17 00:00:00 2001 From: Gerd Zellweger Date: Sat, 8 Aug 2026 21:21:14 -0700 Subject: [PATCH 11/19] ci: reject openssl-sys and a duplicate AWS-LC in the dependency tree openssl-sys can now be blocked outright rather than allowed with an assertion about its backend, since nothing needs it once rdkafka links librdkafka dynamically. The second check is new. aws-lc-sys and aws-lc-fips-sys are separate crates, so cargo cannot unify them, and a tree holding both compiles and links the whole library twice. It happened here: aws-lc-rs selects its backend from the fips feature while openssl-sys selected its own, and the two disagreed. Nothing failed, because the symbol prefixes differ, so the duplication was invisible at run time. Adopting the FIPS module means building AWS-LC with -DFIPS=1 and moving aws-lc-rs to aws-lc-fips-sys in one change. This check is what makes doing only half of it fail. Signed-off-by: Gerd Zellweger --- scripts/validate-crypto-deps.sh | 48 +++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/scripts/validate-crypto-deps.sh b/scripts/validate-crypto-deps.sh index 053633a94fb..a9f9cc0ce96 100755 --- a/scripts/validate-crypto-deps.sh +++ b/scripts/validate-crypto-deps.sh @@ -10,15 +10,18 @@ # # Blocked: # openssl the crate that links a second implementation directly +# openssl-sys the FFI layer beneath it. Kafka used to require it, because +# rdkafka's `ssl` feature depends on it and compiled a vendored +# librdkafka against whatever OpenSSL pkg-config found. rdkafka +# now uses `dynamic-linking` against the librdkafka that +# scripts/install-librdkafka.sh builds against AWS-LC, so +# nothing needs this crate. # native-tls how openssl usually arrives; on macOS and Windows it binds # Security.framework and schannel instead, so checking for # openssl alone would miss a second TLS stack on those platforms # boring BoringSSL wrapper, same symbol-collision class # # Allowed: -# openssl-sys the FFI layer, but only when it resolves to an AWS-LC backend, -# which this script asserts; librdkafka links whatever it -# resolves, so that pin is what keeps Kafka TLS on AWS-LC # aws-lc-sys AWS-LC itself, symbol-prefixed by aws-lc-fips-sys/aws-lc-sys # # `ring` should also go: it is a second cryptographic implementation and is not @@ -34,7 +37,7 @@ set -euo pipefail packages=$(cargo tree --target all --edges normal,build,dev --prefix none --format "{p}" 2>/dev/null | sed 's/ (\*)$//' | sort -u) -blocked=$(echo "$packages" | grep -E "^(openssl|native-tls|boring|boring-sys) v" || true) +blocked=$(echo "$packages" | grep -E "^(openssl|openssl-sys|native-tls|boring|boring-sys) v" || true) if [ -n "$blocked" ]; then echo "error: a second TLS or cryptography implementation is in the dependency tree:" echo "$blocked" | sed 's/^/ /' @@ -47,25 +50,30 @@ if [ -n "$blocked" ]; then echo "then set default-features = false on that dependency and select its rustls" echo "feature instead. Note that a workspace member cannot override" echo "default-features on an inherited dependency; change it at the workspace root." + echo + echo 'openssl-sys specifically arrives through rdkafka. Kafka does not need it:' + echo 'keep rdkafka on "dynamic-linking" rather than "ssl", and build librdkafka' + echo 'with scripts/install-librdkafka.sh.' exit 1 fi -# openssl-sys is only safe while its backend is AWS-LC. Its aws-lc-fips and -# aws-lc features add aws-lc-fips-sys or aws-lc-sys as a direct dependency; -# without either it links the system OpenSSL, which is a second implementation -# wearing the allowed name. -if echo "$packages" | grep -qE "^openssl-sys v"; then - backend=$(cargo tree -p openssl-sys --depth 1 --target all --prefix none --format "{p}" 2>/dev/null | - grep -E "^aws-lc(-fips)?-sys v" || true) - if [ -z "$backend" ]; then - echo "error: openssl-sys is present but does not resolve to an AWS-LC backend." - echo - echo "It links the system OpenSSL, which reintroduces the implementation the" - echo "rest of this check exists to keep out. Restore its backend feature:" - echo - echo ' openssl-sys = { version = "...", default-features = false, features = ["aws-lc-fips"] }' - exit 1 - fi +# One AWS-LC, not two. aws-lc-sys and aws-lc-fips-sys are separate crates, so +# cargo cannot unify them; a tree holding both compiles and links the whole +# library twice. Adopting the FIPS module means moving aws-lc-rs to the fips +# feature and building AWS-LC with -DFIPS=1 in scripts/install-librdkafka.sh at +# the same time, so that librdkafka and the Rust side agree. +backends=$(echo "$packages" | grep -E "^aws-lc(-fips)?-sys v" | awk '{print $1}' | sort -u) +if [ "$(echo "$backends" | grep -c .)" -gt 1 ]; then + echo "error: two AWS-LC implementations are in the dependency tree:" + echo "$backends" | sed 's/^/ /' + echo + echo "Both are compiled and linked, so the binary carries the library twice." + echo "Their symbol prefixes differ, which is why nothing collides and the" + echo "waste stays invisible at run time. Make both ends agree:" + echo + echo " cargo tree -i aws-lc-sys --target all" + echo " cargo tree -i aws-lc-fips-sys --target all" + exit 1 fi # Crates still known to pull ring. Shrink this list; do not extend it. From 1834e309fc87e49472c7326563360c397c598442 Mon Sep 17 00:00:00 2001 From: Gerd Zellweger Date: Sat, 8 Aug 2026 21:40:53 -0700 Subject: [PATCH 12/19] deps: take aws-lc-rs 1.18.0 1.18.0 is the first release depending on aws-lc-fips-sys 0.14, which added AWS_LC_FIPS_SYS_SYSTEM_DIR and USE_SYSTEM. Earlier versions always compiled AWS-LC from source, so enabling the fips feature would have put Go and cmake back into the image that compiles pipelines, undoing #6822. With 0.14 the validated module can be built once and linked, the way scripts/install-librdkafka.sh already builds AWS-LC for librdkafka. Nothing selects fips yet; this only makes that step possible without a toolchain in the shipped image. Signed-off-by: Gerd Zellweger --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 689abc5b5fc..f8c0acbd696 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -97,7 +97,7 @@ aws-sdk-s3 = { version = "1.122.0", default-features = false, features = [ ] } aws-types = "1.1.7" backoff = "0.4.0" -aws-lc-rs = "1.16.2" +aws-lc-rs = "1.18.0" rsa = { version = "0.9.10", features = ["std"] } rcgen = { version = "0.14.8", default-features = false, features = ["pem", "aws_lc_rs"] } tokio-postgres-rustls = { version = "0.14.0", default-features = false, features = ["aws-lc-rs"] } From 7d93901b8fc45b87789cbb67512da3f14249f277 Mon Sep 17 00:00:00 2001 From: Gerd Zellweger Date: Sat, 8 Aug 2026 21:48:38 -0700 Subject: [PATCH 13/19] ci: keep libssl-dev in the CI image for the cloud repository Nothing in this repository links OpenSSL now that rdkafka consumes a prebuilt librdkafka, so the package looked removable. The cloud repository runs its Rust build in this same image, and its cluster-control-plane crate uses the openssl crate directly, in kubernetes_runner/runner_config.rs. Dropping the package would break that build the moment cloud bumps its image pin, which is far from this change. Also names perl explicitly. scripts/install-librdkafka.sh needs it to patch librdkafka, and it was present only because Ubuntu marks it Essential. Signed-off-by: Gerd Zellweger --- deploy/build.Dockerfile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/deploy/build.Dockerfile b/deploy/build.Dockerfile index 2c115e05b76..fac9d53815b 100644 --- a/deploy/build.Dockerfile +++ b/deploy/build.Dockerfile @@ -17,6 +17,13 @@ RUN apt-get update --fix-missing && apt-get install -y \ # Go builds AWS-LC, which librdkafka is linked against; see # scripts/install-librdkafka.sh golang-go \ + # perl applies a source patch in scripts/install-librdkafka.sh. Ubuntu + # ships it as an Essential package, so this only makes the need explicit. + perl \ + # Nothing in this repository links OpenSSL any more. It stays because the + # cloud repository runs its Rust build in this same image, and its + # cluster-control-plane crate uses the openssl crate directly. + libssl-dev \ # librdkafka links these libsasl2-dev libzstd-dev zlib1g-dev build-essential \ # zstd CLI: @actions/cache (runs-on/cache) auto-uses it for cache From 315eb687749bf68e79bfd08b5847512260283fe0 Mon Sep 17 00:00:00 2001 From: Gerd Zellweger Date: Sat, 8 Aug 2026 21:58:05 -0700 Subject: [PATCH 14/19] ci: take the feldera-dev image that carries librdkafka rdkafka links librdkafka dynamically, so the CI image has to supply it. The previous image predates scripts/install-librdkafka.sh and every Rust job fails in it with "librdkafka 2.12.1 cannot be found on the system". Built from 5b0f47e21 for amd64 and arm64 on native runners. Signed-off-by: Gerd Zellweger --- .github/workflows/build-docker-dev.yml | 2 +- .github/workflows/build-docker.yml | 2 +- .github/workflows/build-java.yml | 4 ++-- .github/workflows/build-rust.yml | 2 +- .github/workflows/ci-post-release.yml | 2 +- .github/workflows/ci-pre-mergequeue.yml | 2 +- .github/workflows/publish-crates.yml | 2 +- .github/workflows/test-adapters.yml | 2 +- .github/workflows/test-integration-platform.yml | 2 +- .github/workflows/test-integration-runtime.yml | 2 +- .github/workflows/test-java-nightly.yml | 2 +- .github/workflows/test-java.yml | 2 +- .github/workflows/test-unit.yml | 2 +- 13 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build-docker-dev.yml b/.github/workflows/build-docker-dev.yml index 7ecd4fe0d83..17df6e0dd12 100644 --- a/.github/workflows/build-docker-dev.yml +++ b/.github/workflows/build-docker-dev.yml @@ -23,7 +23,7 @@ jobs: docker_platform: linux/arm64 runs-on: ${{ matrix.runner }} container: - image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-a5ab793b8d261068867b9fdfad3f04157b2536fc + image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-5b0f47e211e90fda97d699da24f40ac480c516a3 steps: - name: Checkout Repository diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index cd295b9200a..136db0cde92 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -35,7 +35,7 @@ jobs: docker_platform: linux/arm64 runs-on: ${{ matrix.runner }} container: - image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-a5ab793b8d261068867b9fdfad3f04157b2536fc + image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-5b0f47e211e90fda97d699da24f40ac480c516a3 steps: - name: Checkout Repository diff --git a/.github/workflows/build-java.yml b/.github/workflows/build-java.yml index 64b10d51754..fca8c73363c 100644 --- a/.github/workflows/build-java.yml +++ b/.github/workflows/build-java.yml @@ -24,7 +24,7 @@ jobs: AWS_ACCESS_KEY_ID: "${{ secrets.CI_GCS_HMAC_ACCESS_KEY_ID }}" AWS_SECRET_ACCESS_KEY: "${{ secrets.CI_GCS_HMAC_SECRET_ACCESS_KEY }}" container: - image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-a5ab793b8d261068867b9fdfad3f04157b2536fc + image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-5b0f47e211e90fda97d699da24f40ac480c516a3 steps: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 @@ -78,7 +78,7 @@ jobs: id-token: write contents: read container: - image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-a5ab793b8d261068867b9fdfad3f04157b2536fc + image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-5b0f47e211e90fda97d699da24f40ac480c516a3 steps: - name: Download build artifact uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7 diff --git a/.github/workflows/build-rust.yml b/.github/workflows/build-rust.yml index 56be22a23ee..25986134067 100644 --- a/.github/workflows/build-rust.yml +++ b/.github/workflows/build-rust.yml @@ -46,7 +46,7 @@ jobs: runs-on: ${{ matrix.runner }} container: - image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-a5ab793b8d261068867b9fdfad3f04157b2536fc + image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-5b0f47e211e90fda97d699da24f40ac480c516a3 steps: - name: Checkout repository diff --git a/.github/workflows/ci-post-release.yml b/.github/workflows/ci-post-release.yml index 0e723d43f2a..2b9ef0b48f4 100644 --- a/.github/workflows/ci-post-release.yml +++ b/.github/workflows/ci-post-release.yml @@ -170,7 +170,7 @@ jobs: adjust-versions: runs-on: [gke-runners-amd64] container: - image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-a5ab793b8d261068867b9fdfad3f04157b2536fc + image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-5b0f47e211e90fda97d699da24f40ac480c516a3 steps: - name: Generate GitHub App token id: app-token diff --git a/.github/workflows/ci-pre-mergequeue.yml b/.github/workflows/ci-pre-mergequeue.yml index b7380047c0e..fc4807413a1 100644 --- a/.github/workflows/ci-pre-mergequeue.yml +++ b/.github/workflows/ci-pre-mergequeue.yml @@ -20,7 +20,7 @@ jobs: # because of how merge queues work: https://stackoverflow.com/a/78030618 main: container: - image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-a5ab793b8d261068867b9fdfad3f04157b2536fc + image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-5b0f47e211e90fda97d699da24f40ac480c516a3 runs-on: [gke-runners-amd64] steps: # Compute the internal-PR boolean once so individual steps can reference diff --git a/.github/workflows/publish-crates.yml b/.github/workflows/publish-crates.yml index b9487af2906..cbf9b5dda53 100644 --- a/.github/workflows/publish-crates.yml +++ b/.github/workflows/publish-crates.yml @@ -39,7 +39,7 @@ jobs: environment: name: ${{ inputs.environment || 'release' }} container: - image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-a5ab793b8d261068867b9fdfad3f04157b2536fc + image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-5b0f47e211e90fda97d699da24f40ac480c516a3 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: diff --git a/.github/workflows/test-adapters.yml b/.github/workflows/test-adapters.yml index c1746809ca8..78f0df2a406 100644 --- a/.github/workflows/test-adapters.yml +++ b/.github/workflows/test-adapters.yml @@ -32,7 +32,7 @@ jobs: target: aarch64-unknown-linux-gnu runs-on: ${{ matrix.runner }} container: - image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-a5ab793b8d261068867b9fdfad3f04157b2536fc + image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-5b0f47e211e90fda97d699da24f40ac480c516a3 services: postgres: image: debezium/postgres:15 diff --git a/.github/workflows/test-integration-platform.yml b/.github/workflows/test-integration-platform.yml index 083582db94a..b97df6fc652 100644 --- a/.github/workflows/test-integration-platform.yml +++ b/.github/workflows/test-integration-platform.yml @@ -211,7 +211,7 @@ jobs: FELDERA_HOST: https://pipeline-manager:8080 container: - image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-a5ab793b8d261068867b9fdfad3f04157b2536fc + image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-5b0f47e211e90fda97d699da24f40ac480c516a3 services: pipeline-manager: image: ${{ vars.FELDERA_IMAGE_NAME }}:${{ inputs.image_tag || format('sha-{0}', github.sha) }} diff --git a/.github/workflows/test-integration-runtime.yml b/.github/workflows/test-integration-runtime.yml index 4e9e030b536..6b50d609606 100644 --- a/.github/workflows/test-integration-runtime.yml +++ b/.github/workflows/test-integration-runtime.yml @@ -42,7 +42,7 @@ jobs: AWS_REQUEST_CHECKSUM_CALCULATION: WHEN_REQUIRED AWS_RESPONSE_CHECKSUM_VALIDATION: WHEN_REQUIRED container: - image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-a5ab793b8d261068867b9fdfad3f04157b2536fc + image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-5b0f47e211e90fda97d699da24f40ac480c516a3 steps: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 diff --git a/.github/workflows/test-java-nightly.yml b/.github/workflows/test-java-nightly.yml index 6ab64df794a..7b37949d610 100644 --- a/.github/workflows/test-java-nightly.yml +++ b/.github/workflows/test-java-nightly.yml @@ -35,7 +35,7 @@ jobs: runs-on: ${{ matrix.runner }} container: - image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-a5ab793b8d261068867b9fdfad3f04157b2536fc + image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-5b0f47e211e90fda97d699da24f40ac480c516a3 steps: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 diff --git a/.github/workflows/test-java.yml b/.github/workflows/test-java.yml index 3dc3e4c5f49..26728a5c481 100644 --- a/.github/workflows/test-java.yml +++ b/.github/workflows/test-java.yml @@ -36,7 +36,7 @@ jobs: runs-on: ${{ matrix.runner }} container: - image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-a5ab793b8d261068867b9fdfad3f04157b2536fc + image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-5b0f47e211e90fda97d699da24f40ac480c516a3 steps: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 5d0564294aa..36a077801b0 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -29,7 +29,7 @@ jobs: runs-on: ${{ matrix.runner }} container: - image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-a5ab793b8d261068867b9fdfad3f04157b2536fc + image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-5b0f47e211e90fda97d699da24f40ac480c516a3 steps: - name: Checkout repository From fad73212092c48ea1fef496948a4d9e768ee93bf Mon Sep 17 00:00:00 2001 From: Gerd Zellweger Date: Sat, 8 Aug 2026 23:03:25 -0700 Subject: [PATCH 15/19] build: patch librdkafka with awk rather than perl perl arrives with git either way, but awk keeps the script on tools the POSIX base guarantees, and nothing else here used perl. The insertion was validated against librdkafka 2.12.1: one hmac.h include, directly above x509.h. Signed-off-by: Gerd Zellweger --- CONTRIBUTING.md | 2 +- deploy/Dockerfile | 7 +++---- deploy/build.Dockerfile | 3 --- scripts/install-librdkafka.sh | 9 ++++++--- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 22ac49bf682..ea854d2149f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -22,7 +22,7 @@ Our dependencies for building the project are: - libsasl2-dev - zlib1g-dev - libzstd-dev - - Go and perl, to build AWS-LC (see below) + - Go, to build AWS-LC (see below) - a Rust tool chain (install rustup and the default toolchain) - a Java Virtual Machine (at least Java 19) - maven diff --git a/deploy/Dockerfile b/deploy/Dockerfile index 8d7cb841b8f..b486384ecdd 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -1,12 +1,11 @@ # librdkafka, built against AWS-LC rather than the system OpenSSL. # -# This is a separate stage because building AWS-LC needs Go, cmake and perl, -# and none of them belong in the shipped image. Only the installed library -# crosses over. +# This is a separate stage because building AWS-LC needs Go and cmake, and +# neither belongs in the shipped image. Only the installed library crosses over. FROM ubuntu:24.04 AS librdkafka ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update --fix-missing && apt-get install -y \ - git cmake build-essential perl pkg-config golang-go \ + git cmake build-essential pkg-config golang-go \ libsasl2-dev zlib1g-dev libzstd-dev COPY scripts/install-librdkafka.sh /tmp/install-librdkafka.sh # The script reads the librdkafka version from the lockfile, so the library and diff --git a/deploy/build.Dockerfile b/deploy/build.Dockerfile index fac9d53815b..ce1a29169d9 100644 --- a/deploy/build.Dockerfile +++ b/deploy/build.Dockerfile @@ -17,9 +17,6 @@ RUN apt-get update --fix-missing && apt-get install -y \ # Go builds AWS-LC, which librdkafka is linked against; see # scripts/install-librdkafka.sh golang-go \ - # perl applies a source patch in scripts/install-librdkafka.sh. Ubuntu - # ships it as an Essential package, so this only makes the need explicit. - perl \ # Nothing in this repository links OpenSSL any more. It stays because the # cloud repository runs its Rust build in this same image, and its # cluster-control-plane crate uses the openssl crate directly. diff --git a/scripts/install-librdkafka.sh b/scripts/install-librdkafka.sh index 401f054b415..28699ebf212 100755 --- a/scripts/install-librdkafka.sh +++ b/scripts/install-librdkafka.sh @@ -48,7 +48,7 @@ fi echo "librdkafka v$librdkafka_version, AWS-LC $AWS_LC_REF, prefix $PREFIX" -for tool in cmake git perl go make cc; do +for tool in cmake git awk go make cc; do command -v "$tool" >/dev/null || { echo "error: $tool is required." >&2; exit 1; } done @@ -93,8 +93,11 @@ cd "$work/librdkafka" # returned pointer. Fixed upstream in confluentinc/librdkafka#5552, unmerged; # drop this once a release carries it. if ! grep -q "openssl/hmac.h" src/rdkafka_ssl.c; then - perl -0pi -e 's{#include }{#include \n#include }' \ - src/rdkafka_ssl.c + awk '/^#include $/ && !inserted { + print "#include "; inserted = 1 + } + { print }' src/rdkafka_ssl.c > src/rdkafka_ssl.c.patched + mv src/rdkafka_ssl.c.patched src/rdkafka_ssl.c grep -q "openssl/hmac.h" src/rdkafka_ssl.c || { echo "error: could not apply the hmac.h patch; check whether upstream restructured the includes." >&2 exit 1 From c001453fba6f167f23d34de981989eda09396fff Mon Sep 17 00:00:00 2001 From: Gerd Zellweger Date: Sat, 8 Aug 2026 23:55:23 -0700 Subject: [PATCH 16/19] deps: floor aws-sigv4 at 1.4.5, where SigV4a left ring The lockfile alone held this version, and a lockfile regenerated while rebasing resolved it back to 1.4.2 twice; the dependency check caught it both times. A declared floor is enforced by the resolver, so a regeneration cannot slide below it. No code here calls the crate, which is why cargo-machete ignores it. Signed-off-by: Gerd Zellweger --- Cargo.lock | 943 ++++++++++++------------------------- crates/adapters/Cargo.toml | 7 +- 2 files changed, 313 insertions(+), 637 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3a035a360c1..3fbe73a2868 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -937,6 +937,45 @@ version = "4.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75054ce561491263d7b80dc2f6f6c6f8cdfd0c7a7c17c5cf3b8117829fa72ae1" +[[package]] +name = "asn1-rs" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8" +dependencies = [ + "asn1-rs-derive", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror 2.0.18", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", + "synstructure", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "assert-json-diff" version = "2.0.2" @@ -1057,6 +1096,7 @@ version = "0.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07d6f157065c3461096d51aacde0c326fa49f3f6e0199e204c566842cdaa5299" dependencies = [ + "aws-lc-rs", "base64 0.22.1", "bytes", "futures-util", @@ -1067,7 +1107,6 @@ dependencies = [ "portable-atomic", "rand 0.8.6", "regex", - "ring", "rustls-native-certs 0.8.1", "rustls-pki-types", "rustls-webpki", @@ -1292,23 +1331,24 @@ dependencies = [ [[package]] name = "aws-lc-fips-sys" -version = "0.13.13" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8bce4948d2520386c6d92a6ea2d472300257702242e5a1d01d6add52bd2e7c1" +checksum = "118303cd75f63d1933a90c2ceb7e697281ac6acbdbcc490b46419f25a527ab90" dependencies = [ "bindgen 0.72.1", "cc", "cmake", "dunce", "fs_extra", + "pkg-config", "regex", ] [[package]] name = "aws-lc-rs" -version = "1.16.2" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc" +checksum = "ce2b2dcc879c3bae0d371e77c99f2238400ef24ec001394befa67b6e543add9e" dependencies = [ "aws-lc-fips-sys", "aws-lc-sys", @@ -1318,14 +1358,15 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.39.0" +version = "0.44.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa7e52a4c5c547c741610a2c6f123f3881e409b714cd27e6798ef020c514f0a" +checksum = "f09fae7be8bb3174e05c6afdb34199e6dc0c7c04ba9fa237b1967adfbde27483" dependencies = [ "cc", "cmake", "dunce", "fs_extra", + "pkg-config", ] [[package]] @@ -1556,9 +1597,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.4.2" +version = "1.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0b660013a6683ab23797778e21f1f854744fdf05f68204b4cca4c8c04b5d1f4" +checksum = "bae38512beae0ffee7010fc24e7a8a123c53efdfef42a61e80fda4882418dc71" dependencies = [ "aws-credential-types", "aws-smithy-eventstream", @@ -1566,16 +1607,15 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "bytes", - "crypto-bigint 0.5.5", + "crypto-bigint", "form_urlencoded", "hex", - "hmac 0.12.1", + "hmac 0.13.0", "http 0.2.12", "http 1.3.1", "p256", "percent-encoding", - "ring", - "sha2 0.10.9", + "sha2 0.11.0", "subtle", "time", "tracing", @@ -1854,9 +1894,9 @@ dependencies = [ [[package]] name = "base16ct" -version = "0.1.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" [[package]] name = "base58" @@ -1946,24 +1986,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "bindgen" -version = "0.71.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f58bf3d7db68cfbac37cfc485a8d711e87e064c3d0fe0435b92f7a407f9d6b3" -dependencies = [ - "bitflags 2.10.0", - "cexpr", - "clang-sys", - "itertools 0.13.0", - "proc-macro2", - "quote", - "regex", - "rustc-hash 2.1.1", - "shlex", - "syn 2.0.117", -] - [[package]] name = "bindgen" version = "0.72.1" @@ -2014,7 +2036,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" dependencies = [ - "bit-vec", + "bit-vec 0.8.0", ] [[package]] @@ -2023,6 +2045,15 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" +[[package]] +name = "bit-vec" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51" +dependencies = [ + "serde", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -2138,7 +2169,7 @@ version = "3.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d314cc62af2b6b0c65780555abb4d02a03dd3b799cd42419044f0c38d99738c0" dependencies = [ - "darling 0.20.11", + "darling 0.23.0", "ident_case", "prettyplease", "proc-macro2", @@ -3062,26 +3093,16 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929" -[[package]] -name = "crypto-bigint" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef" -dependencies = [ - "generic-array", - "rand_core 0.6.4", - "subtle", - "zeroize", -] - [[package]] name = "crypto-bigint" version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" dependencies = [ + "generic-array", "rand_core 0.6.4", "subtle", + "zeroize", ] [[package]] @@ -4146,7 +4167,7 @@ dependencies = [ "typedmap", "uuid", "xxhash-rust", - "zip 6.0.0", + "zip", "zstd 0.12.4", ] @@ -4172,9 +4193,11 @@ dependencies = [ "awc", "aws-config", "aws-credential-types", + "aws-lc-rs", "aws-msk-iam-sasl-signer", "aws-sdk-dynamodb", "aws-sdk-s3", + "aws-sigv4", "aws-types", "backoff", "backtrace", @@ -4241,12 +4264,10 @@ dependencies = [ "num-derive", "num-traits", "once_cell", - "openssl", "ordered-float 4.6.0", "parking_lot 0.12.4", "parquet", "postgres", - "postgres-openssl", "postgres-types 0.2.14", "pretty_assertions", "proptest", @@ -4261,9 +4282,9 @@ dependencies = [ "rmpv", "roaring", "rustls", + "rustls-pemfile", "schema_registry_converter", "semver", - "sentry", "serde", "serde_arrow", "serde_bytes", @@ -4282,6 +4303,7 @@ dependencies = [ "threadpool", "tokio", "tokio-postgres 0.7.18", + "tokio-postgres-rustls", "tokio-stream", "tokio-util", "tracing", @@ -4291,7 +4313,7 @@ dependencies = [ "uuid", "vergen-gitcl", "xxhash-rust", - "zip 6.0.0", + "zip", ] [[package]] @@ -4378,7 +4400,6 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" dependencies = [ - "serde", "uuid", ] @@ -4550,16 +4571,6 @@ dependencies = [ "url", ] -[[package]] -name = "der" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" -dependencies = [ - "const-oid 0.9.6", - "zeroize", -] - [[package]] name = "der" version = "0.7.10" @@ -4573,6 +4584,20 @@ dependencies = [ "zeroize", ] +[[package]] +name = "der-parser" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" +dependencies = [ + "asn1-rs", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + [[package]] name = "der_derive" version = "0.7.3" @@ -4888,14 +4913,16 @@ dependencies = [ [[package]] name = "ecdsa" -version = "0.14.8" +version = "0.16.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c" +checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" dependencies = [ - "der 0.6.1", + "der", + "digest 0.10.7", "elliptic-curve", "rfc6979", - "signature 1.6.4", + "signature", + "spki", ] [[package]] @@ -4904,7 +4931,7 @@ version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" dependencies = [ - "signature 2.2.0", + "signature", ] [[package]] @@ -4916,7 +4943,7 @@ dependencies = [ "curve25519-dalek", "ed25519", "sha2 0.10.9", - "signature 2.2.0", + "signature", "subtle", ] @@ -4943,18 +4970,18 @@ dependencies = [ [[package]] name = "elliptic-curve" -version = "0.12.3" +version = "0.13.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3" +checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" dependencies = [ "base16ct", - "crypto-bigint 0.4.9", - "der 0.6.1", + "crypto-bigint", "digest 0.10.7", "ff", "generic-array", "group", - "pkcs8 0.9.0", + "pem-rfc7468", + "pkcs8", "rand_core 0.6.4", "sec1", "subtle", @@ -5109,7 +5136,7 @@ dependencies = [ [[package]] name = "etl" version = "0.1.0" -source = "git+https://github.com/supabase/etl?rev=b2aab62d#b2aab62db1830df5b72d1d09ad5c053ee8e44db2" +source = "git+https://github.com/feldera/etl?rev=91260556931fba2f55db9ebc11775e295c453bc9#91260556931fba2f55db9ebc11775e295c453bc9" dependencies = [ "byteorder", "bytes", @@ -5138,7 +5165,7 @@ dependencies = [ [[package]] name = "etl-config" version = "0.1.0" -source = "git+https://github.com/supabase/etl?rev=b2aab62d#b2aab62db1830df5b72d1d09ad5c053ee8e44db2" +source = "git+https://github.com/feldera/etl?rev=91260556931fba2f55db9ebc11775e295c453bc9#91260556931fba2f55db9ebc11775e295c453bc9" dependencies = [ "config", "secrecy", @@ -5152,7 +5179,7 @@ dependencies = [ [[package]] name = "etl-postgres" version = "0.1.0" -source = "git+https://github.com/supabase/etl?rev=b2aab62d#b2aab62db1830df5b72d1d09ad5c053ee8e44db2" +source = "git+https://github.com/feldera/etl?rev=91260556931fba2f55db9ebc11775e295c453bc9#91260556931fba2f55db9ebc11775e295c453bc9" dependencies = [ "aws-lc-rs", "bytes", @@ -5288,7 +5315,6 @@ dependencies = [ "clap", "clap_complete", "directories", - "feldera-observability", "feldera-rest-api", "feldera-types", "futures", @@ -5303,9 +5329,9 @@ dependencies = [ "reqwest 0.12.24", "reqwest-websocket", "rmpv", + "rustls", "rustversion", "rustyline", - "sentry", "serde", "serde_json", "syn 2.0.117", @@ -5317,7 +5343,7 @@ dependencies = [ "tracing-log", "tracing-subscriber", "uuid", - "zip 6.0.0", + "zip", ] [[package]] @@ -5384,12 +5410,12 @@ dependencies = [ [[package]] name = "feldera-cloud1-client" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c41ca98fe69d6105563296f7e52552984ae9638ed00e967396b8047264742e81" +checksum = "c3bbbc99187b0e2afa72198abf5fa523a9bfc9ff3d8615d5bc3b408800534a01" dependencies = [ "chrono", - "reqwest 0.12.24", + "reqwest 0.13.4", "serde", "serde_json", "thiserror 2.0.18", @@ -5477,7 +5503,7 @@ dependencies = [ "serde", "serde_json", "utoipa", - "zip 6.0.0", + "zip", ] [[package]] @@ -5499,7 +5525,6 @@ dependencies = [ "chrono", "colored", "reqwest 0.12.24", - "sentry", "serde_json", "sysinfo 0.38.4", "tracing", @@ -5511,14 +5536,12 @@ name = "feldera-rest-api" version = "0.334.0" dependencies = [ "chrono", - "feldera-observability", "feldera-types", "prettyplease", "progenitor", "progenitor-client 0.9.1", "reqwest 0.12.24", "rustversion", - "sentry", "serde", "serde_json", "syn 2.0.117", @@ -5617,7 +5640,7 @@ dependencies = [ "itertools 0.14.0", "libc", "nix 0.27.1", - "object_store 0.12.5", + "object_store 0.14.1", "once_cell", "rkyv", "serde", @@ -5677,9 +5700,9 @@ dependencies = [ [[package]] name = "ff" -version = "0.12.1" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160" +checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" dependencies = [ "rand_core 0.6.4", "subtle", @@ -5795,21 +5818,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - [[package]] name = "form_urlencoded" version = "1.2.2" @@ -6057,6 +6065,7 @@ checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", + "zeroize", ] [[package]] @@ -6203,9 +6212,9 @@ dependencies = [ [[package]] name = "group" -version = "0.12.1" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" dependencies = [ "ff", "rand_core 0.6.4", @@ -6431,17 +6440,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "hostname" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56f203cd1c76362b69e3863fd987520ac36cf70a8c92627449b2f64a8cf7d65" -dependencies = [ - "cfg-if", - "libc", - "windows-link 0.1.1", -] - [[package]] name = "http" version = "0.2.12" @@ -6607,22 +6605,6 @@ dependencies = [ "tower-service", ] -[[package]] -name = "hyper-tls" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" -dependencies = [ - "bytes", - "http-body-util", - "hyper 1.6.0", - "hyper-util", - "native-tls", - "tokio", - "tokio-native-tls", - "tower-service", -] - [[package]] name = "hyper-util" version = "0.1.17" @@ -6661,7 +6643,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core 0.58.0", + "windows-core 0.61.2", ] [[package]] @@ -7112,6 +7094,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -7287,7 +7278,7 @@ dependencies = [ "pem", "serde", "serde_json", - "signature 2.2.0", + "signature", "simple_asn1", "zeroize", ] @@ -7467,18 +7458,6 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "libz-sys" -version = "1.1.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b70e7a7df205e92a1a4cd9aaae7898dac0aa555503cc0a649494d0d60e7651d" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - [[package]] name = "like" version = "0.3.1" @@ -7847,23 +7826,6 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9252111cf132ba0929b6f8e030cac2a24b507f3a4d6db6fb2896f27b354c714b" -[[package]] -name = "native-tls" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" -dependencies = [ - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework 2.11.1", - "security-framework-sys", - "tempfile", -] - [[package]] name = "nibble_vec" version = "0.1.0" @@ -7907,6 +7869,18 @@ dependencies = [ "libc", ] +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags 2.10.0", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "nkeys" version = "0.4.5" @@ -8174,16 +8148,18 @@ dependencies = [ [[package]] name = "object_store" -version = "0.12.5" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbfbfff40aeccab00ec8a910b57ca8ecf4319b335c542f2edcd19dd25a1e2a00" +checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" dependencies = [ "async-trait", "base64 0.22.1", "bytes", "chrono", "form_urlencoded", - "futures", + "futures-channel", + "futures-core", + "futures-util", "http 1.3.1", "http-body-util", "httparse", @@ -8193,11 +8169,11 @@ dependencies = [ "md-5 0.10.6", "parking_lot 0.12.4", "percent-encoding", - "quick-xml 0.38.3", - "rand 0.9.4", + "quick-xml 0.39.4", + "rand 0.10.1", "reqwest 0.12.24", "ring", - "rustls-pemfile", + "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", @@ -8212,14 +8188,16 @@ dependencies = [ [[package]] name = "object_store" -version = "0.13.2" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" +checksum = "d354792e39fa5f0009e47623cf8b15b099bf9a652fa55c6f817fe28ac84fea50" dependencies = [ "async-trait", + "aws-lc-rs", "base64 0.22.1", "bytes", "chrono", + "crc-fast", "form_urlencoded", "futures-channel", "futures-core", @@ -8229,14 +8207,14 @@ dependencies = [ "httparse", "humantime", "hyper 1.6.0", - "itertools 0.14.0", - "md-5 0.10.6", + "itertools 0.15.0", + "md-5 0.11.0", + "nix 0.31.3", "parking_lot 0.12.4", "percent-encoding", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "rand 0.10.1", - "reqwest 0.12.24", - "ring", + "reqwest 0.13.4", "rustls-pki-types", "serde", "serde_json", @@ -8248,6 +8226,16 @@ dependencies = [ "walkdir", "wasm-bindgen-futures", "web-time", + "windows-sys 0.61.2", +] + +[[package]] +name = "oid-registry" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" +dependencies = [ + "asn1-rs", ] [[package]] @@ -8429,31 +8417,6 @@ dependencies = [ "url", ] -[[package]] -name = "openssl" -version = "0.10.80" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a45fa2aa886c42762255da344f0a0d313e254066c46aad76f300c3d3da62d967" -dependencies = [ - "bitflags 2.10.0", - "cfg-if", - "foreign-types", - "libc", - "openssl-macros", - "openssl-sys", -] - -[[package]] -name = "openssl-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - [[package]] name = "openssl-probe" version = "0.1.6" @@ -8563,18 +8526,6 @@ dependencies = [ "hashbrown 0.14.5", ] -[[package]] -name = "os_info" -version = "3.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0e1ac5fde8d43c34139135df8ea9ee9465394b2d8d20f032d38998f64afffc3" -dependencies = [ - "log", - "plist", - "serde", - "windows-sys 0.52.0", -] - [[package]] name = "os_pipe" version = "1.2.2" @@ -8623,12 +8574,13 @@ checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" [[package]] name = "p256" -version = "0.11.1" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594" +checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" dependencies = [ "ecdsa", "elliptic-curve", + "primeorder", "sha2 0.10.9", ] @@ -8974,6 +8926,7 @@ dependencies = [ "async-stream", "async-trait", "awc", + "aws-lc-rs", "base64 0.22.1", "bytestring", "cached", @@ -8999,20 +8952,21 @@ dependencies = [ "jsonwebtoken", "libc", "nix 0.29.0", - "openssl", "pg-client-config", - "postgres-openssl", "postgresql_embedded", "proptest", "proptest-derive", "rand 0.8.6", + "rcgen", "refinery", "regex", "reqwest 0.12.24", "rmp-serde", + "rsa", "rustls", + "rustls-native-certs 0.7.3", + "rustls-pemfile", "semver", - "sentry", "serde", "serde_json", "serde_yaml", @@ -9026,19 +8980,20 @@ dependencies = [ "termbg", "thiserror 2.0.18", "tikv-jemallocator", + "time", "tokio", "tokio-postgres 0.7.18", + "tokio-postgres-rustls", "tokio-stream", "tracing", "tracing-subscriber", "url", "urlencoding", "utoipa", - "utoipa-swagger-ui", "uuid", "vergen-gitcl", "wiremock", - "zip 6.0.0", + "zip", ] [[package]] @@ -9058,9 +9013,9 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" dependencies = [ - "der 0.7.10", - "pkcs8 0.10.2", - "spki 0.7.3", + "der", + "pkcs8", + "spki", ] [[package]] @@ -9071,21 +9026,11 @@ checksum = "e847e2c91a18bfa887dd028ec33f2fe6f25db77db3619024764914affe8b69a6" dependencies = [ "aes", "cbc", - "der 0.7.10", + "der", "pbkdf2", "scrypt", "sha2 0.10.9", - "spki 0.7.3", -] - -[[package]] -name = "pkcs8" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba" -dependencies = [ - "der 0.6.1", - "spki 0.6.0", + "spki", ] [[package]] @@ -9094,10 +9039,10 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der 0.7.10", + "der", "pkcs5", "rand_core 0.6.4", - "spki 0.7.3", + "spki", ] [[package]] @@ -9112,19 +9057,6 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" -[[package]] -name = "plist" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "740ebea15c5d1428f910cd1a5f52cebf8d25006245ed8ade92702f4943d91e07" -dependencies = [ - "base64 0.22.1", - "indexmap 2.13.0", - "quick-xml 0.38.3", - "serde", - "time", -] - [[package]] name = "plotters" version = "0.3.7" @@ -9221,30 +9153,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "postgres-native-tls" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1f39498473c92f7b6820ae970382c1d83178a3454c618161cb772e8598d9f6f" -dependencies = [ - "native-tls", - "tokio", - "tokio-native-tls", - "tokio-postgres 0.7.18", -] - -[[package]] -name = "postgres-openssl" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb14e4bbc2c0b3d165bf30b79c7a9c10412dff9d98491ffdd64ed810ab891d21" -dependencies = [ - "openssl", - "tokio", - "tokio-openssl", - "tokio-postgres 0.7.18", -] - [[package]] name = "postgres-protocol" version = "0.6.7" @@ -9496,6 +9404,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "primeorder" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" +dependencies = [ + "elliptic-curve", +] + [[package]] name = "proc-macro-crate" version = "3.3.0" @@ -9685,7 +9602,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14cae93065090804185d3b75f0bf93b8eeda30c7a9b4a33d3bdb3988d6229e50" dependencies = [ "bit-set", - "bit-vec", + "bit-vec 0.8.0", "bitflags 2.10.0", "lazy_static", "num-traits", @@ -10203,6 +10120,20 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "rcgen" +version = "0.14.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "091e7a8e7d86e6feb87a27ce8e2cba29d49eff9507afeebefab7eeb2ca667fb4" +dependencies = [ + "aws-lc-rs", + "pem", + "rustls-pki-types", + "time", + "x509-parser", + "yasna", +] + [[package]] name = "rdkafka" version = "0.39.0" @@ -10228,12 +10159,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e234cf318915c1059d4921ef7f75616b5219b10b46e9f3a511a15eb4b56a3f77" dependencies = [ "libc", - "libz-sys", "num_enum", "openssl-sys", "pkg-config", "sasl2-sys", - "zstd-sys", ] [[package]] @@ -10362,9 +10291,9 @@ dependencies = [ [[package]] name = "refinery" -version = "0.9.0" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52c427f2572afe5c6cbfa2b1bf40071c89bf1a8539e958ea582842f6f38dcfae" +checksum = "6e2a344cdb48871e27addeafbbaffab8828cc12cec2b9041119e9bea0c0f551a" dependencies = [ "refinery-core", "refinery-macros", @@ -10372,32 +10301,28 @@ dependencies = [ [[package]] name = "refinery-core" -version = "0.9.0" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "702655abfc67f93a6f735e9fa4ace7d2e580633f8961f28acbfd7583ddce936c" +checksum = "24eeafd893124f29183dd6afa9137a27e7bef59250223b8b660005279c60aea4" dependencies = [ "async-trait", "cfg-if", "log", - "native-tls", - "postgres-native-tls", "regex", - "serde", "siphasher", "thiserror 2.0.18", "time", "tokio", "tokio-postgres 0.7.18", - "toml", "url", "walkdir", ] [[package]] name = "refinery-macros" -version = "0.9.0" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5145756cdf293b5089dc6b4f103f1a1229cc55d67082c866f8c8289531c4b983" +checksum = "a90cea6d11a9a4e8a85a884b6305461004101b28ca65dd35ec028e009a898e16" dependencies = [ "proc-macro2", "quote", @@ -10611,12 +10536,10 @@ dependencies = [ "http-body-util", "hyper 1.6.0", "hyper-rustls", - "hyper-tls", "hyper-util", "js-sys", "log", "mime", - "native-tls", "percent-encoding", "pin-project-lite", "quinn", @@ -10628,7 +10551,6 @@ dependencies = [ "serde_urlencoded", "sync_wrapper 1.0.2", "tokio", - "tokio-native-tls", "tokio-rustls", "tokio-util", "tower", @@ -10669,6 +10591,8 @@ dependencies = [ "rustls", "rustls-pki-types", "rustls-platform-verifier", + "serde", + "serde_json", "sync_wrapper 1.0.2", "tokio", "tokio-rustls", @@ -10765,13 +10689,12 @@ dependencies = [ [[package]] name = "rfc6979" -version = "0.3.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb" +checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" dependencies = [ - "crypto-bigint 0.4.9", "hmac 0.12.1", - "zeroize", + "subtle", ] [[package]] @@ -10898,11 +10821,11 @@ dependencies = [ "num-integer", "num-traits", "pkcs1", - "pkcs8 0.10.2", + "pkcs8", "rand_core 0.6.4", "sha2 0.10.9", - "signature 2.2.0", - "spki 0.7.3", + "signature", + "spki", "subtle", "zeroize", ] @@ -10943,40 +10866,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "rust-embed" -version = "8.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "025908b8682a26ba8d12f6f2d66b987584a4a87bc024abc5bbc12553a8cd178a" -dependencies = [ - "rust-embed-impl", - "rust-embed-utils", - "walkdir", -] - -[[package]] -name = "rust-embed-impl" -version = "8.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6065f1a4392b71819ec1ea1df1120673418bf386f50de1d6f54204d836d4349c" -dependencies = [ - "proc-macro2", - "quote", - "rust-embed-utils", - "syn 2.0.117", - "walkdir", -] - -[[package]] -name = "rust-embed-utils" -version = "8.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6cc0c81648b20b70c491ff8cce00c1c3b223bb8ed2b5d41f0e54c6c4c0a3594" -dependencies = [ - "sha2 0.10.9", - "walkdir", -] - [[package]] name = "rust-ini" version = "0.18.0" @@ -11025,6 +10914,15 @@ dependencies = [ "semver", ] +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom", +] + [[package]] name = "rustix" version = "0.38.44" @@ -11355,14 +11253,14 @@ checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" [[package]] name = "sec1" -version = "0.3.0" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928" +checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ "base16ct", - "der 0.6.1", + "der", "generic-array", - "pkcs8 0.9.0", + "pkcs8", "subtle", "zeroize", ] @@ -11423,141 +11321,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "sentry" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48b85e25e8a1fc13928885e8bf13abe8a09e15c46993aed05d6405f7755d6e20" -dependencies = [ - "httpdate", - "native-tls", - "reqwest 0.12.24", - "rustls", - "sentry-actix", - "sentry-anyhow", - "sentry-backtrace", - "sentry-contexts", - "sentry-core", - "sentry-log", - "sentry-panic", - "sentry-tracing", - "tokio", - "ureq", -] - -[[package]] -name = "sentry-actix" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc694e6ffc8d5d7fdb2a33923b0358f6ad41c0b428ced034b349b9e2b08260bc" -dependencies = [ - "actix-http", - "actix-web", - "bytes", - "futures-util", - "sentry-core", -] - -[[package]] -name = "sentry-anyhow" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfd12302213b0f449a065a68164e30dc02f5a4bc84843c2c5939b2c46b1513e8" -dependencies = [ - "anyhow", - "sentry-backtrace", - "sentry-core", -] - -[[package]] -name = "sentry-backtrace" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3253a495ab536f6de1746a58d5d7824b77d75e08e1a4b8ca6fb356839077ae0" -dependencies = [ - "backtrace", - "regex", - "sentry-core", -] - -[[package]] -name = "sentry-contexts" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "027f81a728836e66b88c07666a10f5ed5a35e2695b04eb7aa0fcbed93f814900" -dependencies = [ - "hostname", - "libc", - "os_info", - "rustc_version", - "sentry-core", - "uname", -] - -[[package]] -name = "sentry-core" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3b6729c8e71ac968edbe9bf2dd4109c162e552b52bacd2b07e24ede1aba84a5" -dependencies = [ - "rand 0.9.4", - "sentry-types", - "serde", - "serde_json", - "url", -] - -[[package]] -name = "sentry-log" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "912fea629a3fc7bfcb97f7bde31a5c815df8526ba19088dcef3ae32ea1e25418" -dependencies = [ - "bitflags 2.10.0", - "log", - "sentry-core", -] - -[[package]] -name = "sentry-panic" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ac0471f04f8f97af0c17eeca2c516e23faa1c0271a55bc64371d9ce488c2d40" -dependencies = [ - "sentry-backtrace", - "sentry-core", -] - -[[package]] -name = "sentry-tracing" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "428f780866a613142dcc81b7f8551ae4d1c056f4df22b6d7ddd9154a9974eb03" -dependencies = [ - "bitflags 2.10.0", - "sentry-backtrace", - "sentry-core", - "tracing-core", - "tracing-subscriber", -] - -[[package]] -name = "sentry-types" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c19d1d1967b55659c358886d0f1aa3076488d445f84c7d727d384c675adaec1" -dependencies = [ - "debugid", - "hex", - "rand 0.9.4", - "serde", - "serde_json", - "thiserror 2.0.18", - "time", - "url", - "uuid", -] - [[package]] name = "seq-macro" version = "0.3.6" @@ -11705,15 +11468,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "serde_spanned" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" -dependencies = [ - "serde", -] - [[package]] name = "serde_tokenstream" version = "0.2.2" @@ -11919,22 +11673,12 @@ version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1e303f8205714074f6068773f0e29527e0453937fe837c9717d066635b65f31" dependencies = [ - "pkcs8 0.10.2", + "pkcs8", "rand_core 0.6.4", - "signature 2.2.0", + "signature", "zeroize", ] -[[package]] -name = "signature" -version = "1.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" -dependencies = [ - "digest 0.10.7", - "rand_core 0.6.4", -] - [[package]] name = "signature" version = "2.2.0" @@ -12087,16 +11831,6 @@ dependencies = [ "lock_api", ] -[[package]] -name = "spki" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b" -dependencies = [ - "base64ct", - "der 0.6.1", -] - [[package]] name = "spki" version = "0.7.3" @@ -12104,7 +11838,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ "base64ct", - "der 0.7.10", + "der", ] [[package]] @@ -12174,9 +11908,9 @@ dependencies = [ "indexmap 2.13.0", "log", "memchr", - "native-tls", "once_cell", "percent-encoding", + "rustls", "serde", "serde_json", "sha2 0.10.9", @@ -12186,6 +11920,7 @@ dependencies = [ "tokio-stream", "tracing", "url", + "webpki-roots 0.26.11", ] [[package]] @@ -13027,6 +12762,27 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +[[package]] +name = "tls_codec" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de2e01245e2bb89d6f05801c564fa27624dbd7b1846859876c7dad82e90bf6b" +dependencies = [ + "tls_codec_derive", + "zeroize", +] + +[[package]] +name = "tls_codec_derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d2e76690929402faae40aebdda620a2c0e25dd6d3b9afe48867dfd95991f4bd" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "token-source" version = "1.0.0" @@ -13064,27 +12820,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "tokio-native-tls" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" -dependencies = [ - "native-tls", - "tokio", -] - -[[package]] -name = "tokio-openssl" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59df6849caa43bb7567f9a36f863c447d95a11d5903c9cc334ba32576a27eadd" -dependencies = [ - "openssl", - "openssl-sys", - "tokio", -] - [[package]] name = "tokio-postgres" version = "0.7.11" @@ -13136,6 +12871,20 @@ dependencies = [ "whoami 2.1.2", ] +[[package]] +name = "tokio-postgres-rustls" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c2ad44aa0ae96db89c4742212ed41645b2f597311ff6e1945542a4d9fadc2fb" +dependencies = [ + "rustls", + "sha2 0.11.0", + "tokio", + "tokio-postgres 0.7.18", + "tokio-rustls", + "x509-cert", +] + [[package]] name = "tokio-retry2" version = "0.6.2" @@ -13205,6 +12954,7 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f591660438b3038dd04d16c938271c79e7e06260ad2ea2885a4861bfb238605d" dependencies = [ + "aws-lc-rs", "base64 0.22.1", "bytes", "futures-core", @@ -13212,7 +12962,6 @@ dependencies = [ "http 1.3.1", "httparse", "rand 0.8.6", - "ring", "rustls-pki-types", "tokio", "tokio-rustls", @@ -13220,26 +12969,11 @@ dependencies = [ "webpki-roots 0.26.11", ] -[[package]] -name = "toml" -version = "0.8.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05ae329d1f08c4d17a59bed7ff5b5a769d062e64a62d34a3261b219e62cd5aae" -dependencies = [ - "serde", - "serde_spanned", - "toml_datetime", - "toml_edit", -] - [[package]] name = "toml_datetime" version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3da5db5a963e24bc68be8b17b6fa82814bb22ee8660f192bb182771d498f09a3" -dependencies = [ - "serde", -] [[package]] name = "toml_edit" @@ -13248,19 +12982,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "310068873db2c5b3e7659d2cc35d21855dbafa50d1ce336397c666e3cb08137e" dependencies = [ "indexmap 2.13.0", - "serde", - "serde_spanned", "toml_datetime", - "toml_write", "winnow", ] -[[package]] -name = "toml_write" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfb942dfe1d8e29a7ee7fcbde5bd2b9a25fb89aa70caea2eba3bee836ff41076" - [[package]] name = "tonic" version = "0.14.5" @@ -13618,15 +13343,6 @@ dependencies = [ "typify-impl", ] -[[package]] -name = "uname" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b72f89f0ca32e4db1c04e2a72f5345d59796d4866a1ee0609084569f73683dc8" -dependencies = [ - "libc", -] - [[package]] name = "unarray" version = "0.1.4" @@ -13724,38 +13440,6 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" -[[package]] -name = "ureq" -version = "3.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99ba1025f18a4a3fc3e9b48c868e9beb4f24f4b4b1a325bada26bd4119f46537" -dependencies = [ - "base64 0.22.1", - "der 0.7.10", - "log", - "native-tls", - "percent-encoding", - "rustls", - "rustls-pemfile", - "rustls-pki-types", - "ureq-proto", - "utf-8", - "webpki-root-certs", - "webpki-roots 1.0.0", -] - -[[package]] -name = "ureq-proto" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60b4531c118335662134346048ddb0e54cc86bd7e81866757873055f0e38f5d2" -dependencies = [ - "base64 0.22.1", - "http 1.3.1", - "httparse", - "log", -] - [[package]] name = "url" version = "2.5.8" @@ -13828,31 +13512,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "utoipa-swagger-ui" -version = "7.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "943e0ff606c6d57d410fd5663a4d7c074ab2c5f14ab903b9514565e59fa1189e" -dependencies = [ - "actix-web", - "mime_guess", - "regex", - "reqwest 0.12.24", - "rust-embed", - "serde", - "serde_json", - "url", - "utoipa", - "utoipa-swagger-ui-vendored", - "zip 1.1.4", -] - -[[package]] -name = "utoipa-swagger-ui-vendored" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2eebbbfe4093922c2b6734d7c679ebfebd704a0d7e56dfcb0d05818ce28977d" - [[package]] name = "uuid" version = "1.21.0" @@ -15023,8 +14682,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1301e935010a701ae5f8655edc0ad17c44bad3ac5ce8c39185f75453b720ae94" dependencies = [ "const-oid 0.9.6", - "der 0.7.10", - "spki 0.7.3", + "der", + "spki", + "tls_codec", +] + +[[package]] +name = "x509-parser" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202" +dependencies = [ + "asn1-rs", + "aws-lc-rs", + "data-encoding", + "der-parser", + "lazy_static", + "nom", + "oid-registry", + "rusticata-macros", + "thiserror 2.0.18", + "time", ] [[package]] @@ -15066,6 +14744,16 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" +[[package]] +name = "yasna" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5f6765e852b9b4dc8e2a76843e4d64d1cea8e79bcde0b6901aea8e7c7f08282" +dependencies = [ + "bit-vec 0.9.1", + "time", +] + [[package]] name = "yoke" version = "0.8.0" @@ -15190,22 +14878,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "zip" -version = "1.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cc23c04387f4da0374be4533ad1208cbb091d5c11d070dfef13676ad6497164" -dependencies = [ - "arbitrary", - "crc32fast", - "crossbeam-utils", - "displaydoc", - "flate2", - "indexmap 2.13.0", - "num_enum", - "thiserror 1.0.69", -] - [[package]] name = "zip" version = "6.0.0" @@ -15294,7 +14966,6 @@ version = "2.0.15+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb81183ddd97d0c74cedf1d50d85c8d08c1b8b68ee863bdee9e706eedba1a237" dependencies = [ - "bindgen 0.71.1", "cc", "pkg-config", ] diff --git a/crates/adapters/Cargo.toml b/crates/adapters/Cargo.toml index 0fb9c8533dd..08568816ee1 100644 --- a/crates/adapters/Cargo.toml +++ b/crates/adapters/Cargo.toml @@ -219,6 +219,11 @@ rustls-pemfile = { workspace = true } base64 = { workspace = true } aws-msk-iam-sasl-signer = "1.0.1" aws-credential-types = "1.2.3" +# No code here calls it; the AWS SDK pulls it transitively. The declaration +# floors the version at 1.4.5, where SigV4a signing stopped using ring. A +# lockfile regenerated during a rebase resolved it back below that twice; a +# floor the resolver enforces cannot regress that way. +aws-sigv4 = { version = "1.4.5", default-features = false } feldera-sqllib = { workspace = true } inventory = { workspace = true } backtrace = { workspace = true } @@ -241,7 +246,7 @@ etl-config = { workspace = true, optional = true } etl-postgres = { workspace = true, optional = true } [package.metadata.cargo-machete] -ignored = ["num-traits"] +ignored = ["num-traits", "aws-sigv4"] [target.'cfg(target_os = "linux")'.dependencies] # Optional so the SQL test fixture (sql-to-dbsp-compiler/temp), which builds From fabf358cf258f1b897c1342304c29f12abdbf5a7 Mon Sep 17 00:00:00 2001 From: Gerd Zellweger Date: Sat, 15 Aug 2026 12:57:44 -0700 Subject: [PATCH 17/19] ci: drop libssl-dev from the build image Nothing in this repository links OpenSSL, and the cloud repository's cluster-control-plane crate no longer uses the openssl crate either (feldera/cloud#1879), so the last consumer of the header package is gone. Also correct the --db-tls-certificate-path doc: without the flag the manager still negotiates TLS when the server offers it, verified against the system trust roots. Signed-off-by: Gerd Zellweger --- crates/pipeline-manager/src/config.rs | 3 ++- deploy/build.Dockerfile | 4 ---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/crates/pipeline-manager/src/config.rs b/crates/pipeline-manager/src/config.rs index 553bc06940c..65a6021352f 100644 --- a/crates/pipeline-manager/src/config.rs +++ b/crates/pipeline-manager/src/config.rs @@ -709,7 +709,8 @@ pub struct DatabaseConfig { /// Create a TLS connector by loading a certificate from the path specified argument. /// - /// If the argument is not set, tries to connect without TLS. + /// If the argument is not set, the connector still negotiates TLS when the + /// server offers it and verifies the server against the system trust roots. #[arg(long, env = "FELDERA_DB_TLS_CERT_PATH")] pub db_tls_certificate_path: Option, diff --git a/deploy/build.Dockerfile b/deploy/build.Dockerfile index ce1a29169d9..2c115e05b76 100644 --- a/deploy/build.Dockerfile +++ b/deploy/build.Dockerfile @@ -17,10 +17,6 @@ RUN apt-get update --fix-missing && apt-get install -y \ # Go builds AWS-LC, which librdkafka is linked against; see # scripts/install-librdkafka.sh golang-go \ - # Nothing in this repository links OpenSSL any more. It stays because the - # cloud repository runs its Rust build in this same image, and its - # cluster-control-plane crate uses the openssl crate directly. - libssl-dev \ # librdkafka links these libsasl2-dev libzstd-dev zlib1g-dev build-essential \ # zstd CLI: @actions/cache (runs-on/cache) auto-uses it for cache From e9e4499d6414ba10311bd7ef7cd20816e1d9f5c7 Mon Sep 17 00:00:00 2001 From: Gerd Zellweger Date: Sat, 15 Aug 2026 13:06:33 -0700 Subject: [PATCH 18/19] build: build librdkafka against AWS-LC v5.5.0, matching aws-lc-sys aws-lc-sys 0.44.0 vendors AWS-LC 5.5.0, so the copy inside librdkafka now tracks the one the Rust side links instead of the older v1.68.0 line. Verified on macOS (otool clean) and Ubuntu 24.04 (readelf clean). The no-OpenSSL check now also runs on macOS through otool; readelf does not exist there, so the check silently skipped. Signed-off-by: Gerd Zellweger --- scripts/install-librdkafka.sh | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/scripts/install-librdkafka.sh b/scripts/install-librdkafka.sh index 28699ebf212..9a4ef09811d 100755 --- a/scripts/install-librdkafka.sh +++ b/scripts/install-librdkafka.sh @@ -18,7 +18,10 @@ set -euo pipefail PREFIX="${PREFIX:-/usr/local}" -AWS_LC_REF="${AWS_LC_REF:-v1.68.0}" +# Keep in step with the aws-lc-sys version in Cargo.lock: aws-lc-sys 0.44.0 +# vendors AWS-LC 5.5.0 (its include/openssl/base.h names the release), so the +# librdkafka in the process carries the same AWS-LC as the Rust side. +AWS_LC_REF="${AWS_LC_REF:-v5.5.0}" # Switching to the FIPS-validated module means building AWS-LC with -DFIPS=1 # and moving aws-lc-rs to aws-lc-fips-sys in the same change. Doing one without # the other leaves two AWS-LC copies in the binary; validate-crypto-deps.sh @@ -153,11 +156,19 @@ for candidate in "$PREFIX/lib/librdkafka.so.1" "$PREFIX/lib/librdkafka.1.dylib"; break fi done -if [ -n "$shared" ] && command -v readelf >/dev/null; then - if readelf -d "$shared" | grep -qE 'Shared library: \[lib(ssl|crypto)\.so'; then - echo "error: librdkafka links a shared OpenSSL:" >&2 - readelf -d "$shared" | grep -E 'Shared library: \[lib(ssl|crypto)' >&2 - exit 1 +if [ -n "$shared" ]; then + if command -v readelf >/dev/null; then + if readelf -d "$shared" | grep -qE 'Shared library: \[lib(ssl|crypto)\.so'; then + echo "error: librdkafka links a shared OpenSSL:" >&2 + readelf -d "$shared" | grep -E 'Shared library: \[lib(ssl|crypto)' >&2 + exit 1 + fi + elif command -v otool >/dev/null; then + if otool -L "$shared" | grep -qE 'lib(ssl|crypto)[.0-9]*\.dylib'; then + echo "error: librdkafka links a shared OpenSSL:" >&2 + otool -L "$shared" | grep -E 'lib(ssl|crypto)' >&2 + exit 1 + fi fi fi From 010ab2069fba6fe9325673deb9ca32ce34eb80cf Mon Sep 17 00:00:00 2001 From: Gerd Zellweger Date: Sat, 15 Aug 2026 13:54:55 -0700 Subject: [PATCH 19/19] ci: take the feldera-dev image built without libssl-dev The image carries librdkafka built against AWS-LC v5.5.0, so CI links the same AWS-LC release the Rust dependency tree pins. Signed-off-by: Gerd Zellweger --- .github/workflows/build-docker-dev.yml | 2 +- .github/workflows/build-docker.yml | 2 +- .github/workflows/build-java.yml | 4 ++-- .github/workflows/build-rust.yml | 2 +- .github/workflows/ci-post-release.yml | 2 +- .github/workflows/ci-pre-mergequeue.yml | 2 +- .github/workflows/publish-crates.yml | 2 +- .github/workflows/test-adapters.yml | 2 +- .github/workflows/test-integration-platform.yml | 2 +- .github/workflows/test-integration-runtime.yml | 2 +- .github/workflows/test-java-nightly.yml | 2 +- .github/workflows/test-java.yml | 2 +- .github/workflows/test-unit.yml | 2 +- 13 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build-docker-dev.yml b/.github/workflows/build-docker-dev.yml index 17df6e0dd12..acf491c8517 100644 --- a/.github/workflows/build-docker-dev.yml +++ b/.github/workflows/build-docker-dev.yml @@ -23,7 +23,7 @@ jobs: docker_platform: linux/arm64 runs-on: ${{ matrix.runner }} container: - image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-5b0f47e211e90fda97d699da24f40ac480c516a3 + image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-e9e4499d6414ba10311bd7ef7cd20816e1d9f5c7 steps: - name: Checkout Repository diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 136db0cde92..112d4985e48 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -35,7 +35,7 @@ jobs: docker_platform: linux/arm64 runs-on: ${{ matrix.runner }} container: - image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-5b0f47e211e90fda97d699da24f40ac480c516a3 + image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-e9e4499d6414ba10311bd7ef7cd20816e1d9f5c7 steps: - name: Checkout Repository diff --git a/.github/workflows/build-java.yml b/.github/workflows/build-java.yml index fca8c73363c..b4a5501c211 100644 --- a/.github/workflows/build-java.yml +++ b/.github/workflows/build-java.yml @@ -24,7 +24,7 @@ jobs: AWS_ACCESS_KEY_ID: "${{ secrets.CI_GCS_HMAC_ACCESS_KEY_ID }}" AWS_SECRET_ACCESS_KEY: "${{ secrets.CI_GCS_HMAC_SECRET_ACCESS_KEY }}" container: - image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-5b0f47e211e90fda97d699da24f40ac480c516a3 + image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-e9e4499d6414ba10311bd7ef7cd20816e1d9f5c7 steps: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 @@ -78,7 +78,7 @@ jobs: id-token: write contents: read container: - image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-5b0f47e211e90fda97d699da24f40ac480c516a3 + image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-e9e4499d6414ba10311bd7ef7cd20816e1d9f5c7 steps: - name: Download build artifact uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7 diff --git a/.github/workflows/build-rust.yml b/.github/workflows/build-rust.yml index 25986134067..d6f688c4215 100644 --- a/.github/workflows/build-rust.yml +++ b/.github/workflows/build-rust.yml @@ -46,7 +46,7 @@ jobs: runs-on: ${{ matrix.runner }} container: - image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-5b0f47e211e90fda97d699da24f40ac480c516a3 + image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-e9e4499d6414ba10311bd7ef7cd20816e1d9f5c7 steps: - name: Checkout repository diff --git a/.github/workflows/ci-post-release.yml b/.github/workflows/ci-post-release.yml index 2b9ef0b48f4..93f93eb8049 100644 --- a/.github/workflows/ci-post-release.yml +++ b/.github/workflows/ci-post-release.yml @@ -170,7 +170,7 @@ jobs: adjust-versions: runs-on: [gke-runners-amd64] container: - image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-5b0f47e211e90fda97d699da24f40ac480c516a3 + image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-e9e4499d6414ba10311bd7ef7cd20816e1d9f5c7 steps: - name: Generate GitHub App token id: app-token diff --git a/.github/workflows/ci-pre-mergequeue.yml b/.github/workflows/ci-pre-mergequeue.yml index fc4807413a1..a6e70864aee 100644 --- a/.github/workflows/ci-pre-mergequeue.yml +++ b/.github/workflows/ci-pre-mergequeue.yml @@ -20,7 +20,7 @@ jobs: # because of how merge queues work: https://stackoverflow.com/a/78030618 main: container: - image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-5b0f47e211e90fda97d699da24f40ac480c516a3 + image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-e9e4499d6414ba10311bd7ef7cd20816e1d9f5c7 runs-on: [gke-runners-amd64] steps: # Compute the internal-PR boolean once so individual steps can reference diff --git a/.github/workflows/publish-crates.yml b/.github/workflows/publish-crates.yml index cbf9b5dda53..78f078cf703 100644 --- a/.github/workflows/publish-crates.yml +++ b/.github/workflows/publish-crates.yml @@ -39,7 +39,7 @@ jobs: environment: name: ${{ inputs.environment || 'release' }} container: - image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-5b0f47e211e90fda97d699da24f40ac480c516a3 + image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-e9e4499d6414ba10311bd7ef7cd20816e1d9f5c7 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: diff --git a/.github/workflows/test-adapters.yml b/.github/workflows/test-adapters.yml index 78f0df2a406..f2a09bae8fe 100644 --- a/.github/workflows/test-adapters.yml +++ b/.github/workflows/test-adapters.yml @@ -32,7 +32,7 @@ jobs: target: aarch64-unknown-linux-gnu runs-on: ${{ matrix.runner }} container: - image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-5b0f47e211e90fda97d699da24f40ac480c516a3 + image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-e9e4499d6414ba10311bd7ef7cd20816e1d9f5c7 services: postgres: image: debezium/postgres:15 diff --git a/.github/workflows/test-integration-platform.yml b/.github/workflows/test-integration-platform.yml index b97df6fc652..2e26b07dc10 100644 --- a/.github/workflows/test-integration-platform.yml +++ b/.github/workflows/test-integration-platform.yml @@ -211,7 +211,7 @@ jobs: FELDERA_HOST: https://pipeline-manager:8080 container: - image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-5b0f47e211e90fda97d699da24f40ac480c516a3 + image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-e9e4499d6414ba10311bd7ef7cd20816e1d9f5c7 services: pipeline-manager: image: ${{ vars.FELDERA_IMAGE_NAME }}:${{ inputs.image_tag || format('sha-{0}', github.sha) }} diff --git a/.github/workflows/test-integration-runtime.yml b/.github/workflows/test-integration-runtime.yml index 6b50d609606..d2d68392f90 100644 --- a/.github/workflows/test-integration-runtime.yml +++ b/.github/workflows/test-integration-runtime.yml @@ -42,7 +42,7 @@ jobs: AWS_REQUEST_CHECKSUM_CALCULATION: WHEN_REQUIRED AWS_RESPONSE_CHECKSUM_VALIDATION: WHEN_REQUIRED container: - image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-5b0f47e211e90fda97d699da24f40ac480c516a3 + image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-e9e4499d6414ba10311bd7ef7cd20816e1d9f5c7 steps: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 diff --git a/.github/workflows/test-java-nightly.yml b/.github/workflows/test-java-nightly.yml index 7b37949d610..94563c90bf9 100644 --- a/.github/workflows/test-java-nightly.yml +++ b/.github/workflows/test-java-nightly.yml @@ -35,7 +35,7 @@ jobs: runs-on: ${{ matrix.runner }} container: - image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-5b0f47e211e90fda97d699da24f40ac480c516a3 + image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-e9e4499d6414ba10311bd7ef7cd20816e1d9f5c7 steps: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 diff --git a/.github/workflows/test-java.yml b/.github/workflows/test-java.yml index 26728a5c481..7a7442aea72 100644 --- a/.github/workflows/test-java.yml +++ b/.github/workflows/test-java.yml @@ -36,7 +36,7 @@ jobs: runs-on: ${{ matrix.runner }} container: - image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-5b0f47e211e90fda97d699da24f40ac480c516a3 + image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-e9e4499d6414ba10311bd7ef7cd20816e1d9f5c7 steps: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 36a077801b0..05d876e85c9 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -29,7 +29,7 @@ jobs: runs-on: ${{ matrix.runner }} container: - image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-5b0f47e211e90fda97d699da24f40ac480c516a3 + image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-e9e4499d6414ba10311bd7ef7cd20816e1d9f5c7 steps: - name: Checkout repository