diff --git a/.github/workflows/build-docker-dev.yml b/.github/workflows/build-docker-dev.yml index 7ecd4fe0d83..acf491c8517 100644 --- a/.github/workflows/build-docker-dev.yml +++ b/.github/workflows/build-docker-dev.yml @@ -23,7 +23,7 @@ jobs: docker_platform: linux/arm64 runs-on: ${{ matrix.runner }} container: - image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-a5ab793b8d261068867b9fdfad3f04157b2536fc + image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-e9e4499d6414ba10311bd7ef7cd20816e1d9f5c7 steps: - name: Checkout Repository diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index cd295b9200a..112d4985e48 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -35,7 +35,7 @@ jobs: docker_platform: linux/arm64 runs-on: ${{ matrix.runner }} container: - image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-a5ab793b8d261068867b9fdfad3f04157b2536fc + image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-e9e4499d6414ba10311bd7ef7cd20816e1d9f5c7 steps: - name: Checkout Repository diff --git a/.github/workflows/build-java.yml b/.github/workflows/build-java.yml index 64b10d51754..b4a5501c211 100644 --- a/.github/workflows/build-java.yml +++ b/.github/workflows/build-java.yml @@ -24,7 +24,7 @@ jobs: AWS_ACCESS_KEY_ID: "${{ secrets.CI_GCS_HMAC_ACCESS_KEY_ID }}" AWS_SECRET_ACCESS_KEY: "${{ secrets.CI_GCS_HMAC_SECRET_ACCESS_KEY }}" container: - image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-a5ab793b8d261068867b9fdfad3f04157b2536fc + image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-e9e4499d6414ba10311bd7ef7cd20816e1d9f5c7 steps: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 @@ -78,7 +78,7 @@ jobs: id-token: write contents: read container: - image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-a5ab793b8d261068867b9fdfad3f04157b2536fc + image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-e9e4499d6414ba10311bd7ef7cd20816e1d9f5c7 steps: - name: Download build artifact uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7 diff --git a/.github/workflows/build-rust.yml b/.github/workflows/build-rust.yml index 56be22a23ee..d6f688c4215 100644 --- a/.github/workflows/build-rust.yml +++ b/.github/workflows/build-rust.yml @@ -46,7 +46,7 @@ jobs: runs-on: ${{ matrix.runner }} container: - image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-a5ab793b8d261068867b9fdfad3f04157b2536fc + image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-e9e4499d6414ba10311bd7ef7cd20816e1d9f5c7 steps: - name: Checkout repository diff --git a/.github/workflows/ci-post-release.yml b/.github/workflows/ci-post-release.yml index 0e723d43f2a..93f93eb8049 100644 --- a/.github/workflows/ci-post-release.yml +++ b/.github/workflows/ci-post-release.yml @@ -170,7 +170,7 @@ jobs: adjust-versions: runs-on: [gke-runners-amd64] container: - image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-a5ab793b8d261068867b9fdfad3f04157b2536fc + image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-e9e4499d6414ba10311bd7ef7cd20816e1d9f5c7 steps: - name: Generate GitHub App token id: app-token diff --git a/.github/workflows/ci-pre-mergequeue.yml b/.github/workflows/ci-pre-mergequeue.yml index b7380047c0e..a6e70864aee 100644 --- a/.github/workflows/ci-pre-mergequeue.yml +++ b/.github/workflows/ci-pre-mergequeue.yml @@ -20,7 +20,7 @@ jobs: # because of how merge queues work: https://stackoverflow.com/a/78030618 main: container: - image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-a5ab793b8d261068867b9fdfad3f04157b2536fc + image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-e9e4499d6414ba10311bd7ef7cd20816e1d9f5c7 runs-on: [gke-runners-amd64] steps: # Compute the internal-PR boolean once so individual steps can reference diff --git a/.github/workflows/publish-crates.yml b/.github/workflows/publish-crates.yml index b9487af2906..78f078cf703 100644 --- a/.github/workflows/publish-crates.yml +++ b/.github/workflows/publish-crates.yml @@ -39,7 +39,7 @@ jobs: environment: name: ${{ inputs.environment || 'release' }} container: - image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-a5ab793b8d261068867b9fdfad3f04157b2536fc + image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-e9e4499d6414ba10311bd7ef7cd20816e1d9f5c7 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: diff --git a/.github/workflows/test-adapters.yml b/.github/workflows/test-adapters.yml index c1746809ca8..f2a09bae8fe 100644 --- a/.github/workflows/test-adapters.yml +++ b/.github/workflows/test-adapters.yml @@ -32,7 +32,7 @@ jobs: target: aarch64-unknown-linux-gnu runs-on: ${{ matrix.runner }} container: - image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-a5ab793b8d261068867b9fdfad3f04157b2536fc + image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-e9e4499d6414ba10311bd7ef7cd20816e1d9f5c7 services: postgres: image: debezium/postgres:15 diff --git a/.github/workflows/test-integration-platform.yml b/.github/workflows/test-integration-platform.yml index 31071ebea58..2e26b07dc10 100644 --- a/.github/workflows/test-integration-platform.yml +++ b/.github/workflows/test-integration-platform.yml @@ -20,10 +20,6 @@ on: required: false type: string -env: - FELDERA_SENTRY_ENABLED: 1 - SENTRY_ENVIRONMENT: ci - jobs: manager-no-network: if: ${{ !contains(vars.CI_SKIP_JOBS, 'manager-no-network') }} @@ -215,7 +211,7 @@ jobs: FELDERA_HOST: https://pipeline-manager:8080 container: - image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-a5ab793b8d261068867b9fdfad3f04157b2536fc + image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-e9e4499d6414ba10311bd7ef7cd20816e1d9f5c7 services: pipeline-manager: image: ${{ vars.FELDERA_IMAGE_NAME }}:${{ inputs.image_tag || format('sha-{0}', github.sha) }} diff --git a/.github/workflows/test-integration-runtime.yml b/.github/workflows/test-integration-runtime.yml index 4e9e030b536..d2d68392f90 100644 --- a/.github/workflows/test-integration-runtime.yml +++ b/.github/workflows/test-integration-runtime.yml @@ -42,7 +42,7 @@ jobs: AWS_REQUEST_CHECKSUM_CALCULATION: WHEN_REQUIRED AWS_RESPONSE_CHECKSUM_VALIDATION: WHEN_REQUIRED container: - image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-a5ab793b8d261068867b9fdfad3f04157b2536fc + image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-e9e4499d6414ba10311bd7ef7cd20816e1d9f5c7 steps: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 diff --git a/.github/workflows/test-java-nightly.yml b/.github/workflows/test-java-nightly.yml index 6ab64df794a..94563c90bf9 100644 --- a/.github/workflows/test-java-nightly.yml +++ b/.github/workflows/test-java-nightly.yml @@ -35,7 +35,7 @@ jobs: runs-on: ${{ matrix.runner }} container: - image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-a5ab793b8d261068867b9fdfad3f04157b2536fc + image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-e9e4499d6414ba10311bd7ef7cd20816e1d9f5c7 steps: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 diff --git a/.github/workflows/test-java.yml b/.github/workflows/test-java.yml index 3dc3e4c5f49..7a7442aea72 100644 --- a/.github/workflows/test-java.yml +++ b/.github/workflows/test-java.yml @@ -36,7 +36,7 @@ jobs: runs-on: ${{ matrix.runner }} container: - image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-a5ab793b8d261068867b9fdfad3f04157b2536fc + image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-e9e4499d6414ba10311bd7ef7cd20816e1d9f5c7 steps: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 5d0564294aa..05d876e85c9 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -29,7 +29,7 @@ jobs: runs-on: ${{ matrix.runner }} container: - image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-a5ab793b8d261068867b9fdfad3f04157b2536fc + image: us-central1-docker.pkg.dev/feldera-ci/ghcr-remote/feldera/feldera-dev:sha-e9e4499d6414ba10311bd7ef7cd20816e1d9f5c7 steps: - name: Checkout repository diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e286a28d442..9e3502b6be0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -46,6 +46,13 @@ repos: exclude: 'crates/pipeline-manager/migrations/.*|.*\.patch|.*\.svg|.*\.java' - repo: local hooks: + - id: validate-crypto-deps + name: Validate cryptography dependencies + description: Cryptography goes through AWS-LC; a second implementation corrupts the heap. + entry: scripts/validate-crypto-deps.sh + language: system + files: Cargo\.(toml|lock)$ + pass_filenames: false - id: fix-rust-mod-naming name: Rust module naming entry: scripts/fix-rust-module-naming diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5ae76a2b450..ea854d2149f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -19,9 +19,10 @@ Our dependencies for building the project are: - C and C++ compiler toolchain (e.g., gcc, gcc++) - cmake - - libssl-dev - libsasl2-dev - zlib1g-dev + - libzstd-dev + - Go, to build AWS-LC (see below) - a Rust tool chain (install rustup and the default toolchain) - a Java Virtual Machine (at least Java 19) - maven @@ -32,6 +33,25 @@ Our dependencies for building the project are: Additional dependencies are automatically installed by the Rust, maven, Python, and TypeScript build tools. +### librdkafka + +The Kafka connectors link librdkafka dynamically, so it has to be installed +before the workspace will build: + +``` +./scripts/install-librdkafka.sh +``` + +The script builds librdkafka against AWS-LC, which is what keeps Kafka TLS on +the same cryptographic implementation as the rest of the system. Distribution +packages are built against OpenSSL and are usually older than the version the +`rdkafka-sys` crate requires, so installing one of those is not equivalent. Run +the script again after a `rdkafka` version bump; it reads the version it needs +from `Cargo.lock`. + +Set `PREFIX` to install somewhere other than `/usr/local`, in which case +`PKG_CONFIG_PATH` has to point at `$PREFIX/lib/pkgconfig`. + ## Contribution Flow ### Forking diff --git a/Cargo.lock b/Cargo.lock index 3a035a360c1..3fbe73a2868 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -937,6 +937,45 @@ version = "4.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75054ce561491263d7b80dc2f6f6c6f8cdfd0c7a7c17c5cf3b8117829fa72ae1" +[[package]] +name = "asn1-rs" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8" +dependencies = [ + "asn1-rs-derive", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror 2.0.18", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", + "synstructure", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "assert-json-diff" version = "2.0.2" @@ -1057,6 +1096,7 @@ version = "0.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07d6f157065c3461096d51aacde0c326fa49f3f6e0199e204c566842cdaa5299" dependencies = [ + "aws-lc-rs", "base64 0.22.1", "bytes", "futures-util", @@ -1067,7 +1107,6 @@ dependencies = [ "portable-atomic", "rand 0.8.6", "regex", - "ring", "rustls-native-certs 0.8.1", "rustls-pki-types", "rustls-webpki", @@ -1292,23 +1331,24 @@ dependencies = [ [[package]] name = "aws-lc-fips-sys" -version = "0.13.13" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8bce4948d2520386c6d92a6ea2d472300257702242e5a1d01d6add52bd2e7c1" +checksum = "118303cd75f63d1933a90c2ceb7e697281ac6acbdbcc490b46419f25a527ab90" dependencies = [ "bindgen 0.72.1", "cc", "cmake", "dunce", "fs_extra", + "pkg-config", "regex", ] [[package]] name = "aws-lc-rs" -version = "1.16.2" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc" +checksum = "ce2b2dcc879c3bae0d371e77c99f2238400ef24ec001394befa67b6e543add9e" dependencies = [ "aws-lc-fips-sys", "aws-lc-sys", @@ -1318,14 +1358,15 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.39.0" +version = "0.44.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa7e52a4c5c547c741610a2c6f123f3881e409b714cd27e6798ef020c514f0a" +checksum = "f09fae7be8bb3174e05c6afdb34199e6dc0c7c04ba9fa237b1967adfbde27483" dependencies = [ "cc", "cmake", "dunce", "fs_extra", + "pkg-config", ] [[package]] @@ -1556,9 +1597,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.4.2" +version = "1.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0b660013a6683ab23797778e21f1f854744fdf05f68204b4cca4c8c04b5d1f4" +checksum = "bae38512beae0ffee7010fc24e7a8a123c53efdfef42a61e80fda4882418dc71" dependencies = [ "aws-credential-types", "aws-smithy-eventstream", @@ -1566,16 +1607,15 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "bytes", - "crypto-bigint 0.5.5", + "crypto-bigint", "form_urlencoded", "hex", - "hmac 0.12.1", + "hmac 0.13.0", "http 0.2.12", "http 1.3.1", "p256", "percent-encoding", - "ring", - "sha2 0.10.9", + "sha2 0.11.0", "subtle", "time", "tracing", @@ -1854,9 +1894,9 @@ dependencies = [ [[package]] name = "base16ct" -version = "0.1.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" [[package]] name = "base58" @@ -1946,24 +1986,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "bindgen" -version = "0.71.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f58bf3d7db68cfbac37cfc485a8d711e87e064c3d0fe0435b92f7a407f9d6b3" -dependencies = [ - "bitflags 2.10.0", - "cexpr", - "clang-sys", - "itertools 0.13.0", - "proc-macro2", - "quote", - "regex", - "rustc-hash 2.1.1", - "shlex", - "syn 2.0.117", -] - [[package]] name = "bindgen" version = "0.72.1" @@ -2014,7 +2036,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" dependencies = [ - "bit-vec", + "bit-vec 0.8.0", ] [[package]] @@ -2023,6 +2045,15 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" +[[package]] +name = "bit-vec" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51" +dependencies = [ + "serde", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -2138,7 +2169,7 @@ version = "3.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d314cc62af2b6b0c65780555abb4d02a03dd3b799cd42419044f0c38d99738c0" dependencies = [ - "darling 0.20.11", + "darling 0.23.0", "ident_case", "prettyplease", "proc-macro2", @@ -3062,26 +3093,16 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929" -[[package]] -name = "crypto-bigint" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef" -dependencies = [ - "generic-array", - "rand_core 0.6.4", - "subtle", - "zeroize", -] - [[package]] name = "crypto-bigint" version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" dependencies = [ + "generic-array", "rand_core 0.6.4", "subtle", + "zeroize", ] [[package]] @@ -4146,7 +4167,7 @@ dependencies = [ "typedmap", "uuid", "xxhash-rust", - "zip 6.0.0", + "zip", "zstd 0.12.4", ] @@ -4172,9 +4193,11 @@ dependencies = [ "awc", "aws-config", "aws-credential-types", + "aws-lc-rs", "aws-msk-iam-sasl-signer", "aws-sdk-dynamodb", "aws-sdk-s3", + "aws-sigv4", "aws-types", "backoff", "backtrace", @@ -4241,12 +4264,10 @@ dependencies = [ "num-derive", "num-traits", "once_cell", - "openssl", "ordered-float 4.6.0", "parking_lot 0.12.4", "parquet", "postgres", - "postgres-openssl", "postgres-types 0.2.14", "pretty_assertions", "proptest", @@ -4261,9 +4282,9 @@ dependencies = [ "rmpv", "roaring", "rustls", + "rustls-pemfile", "schema_registry_converter", "semver", - "sentry", "serde", "serde_arrow", "serde_bytes", @@ -4282,6 +4303,7 @@ dependencies = [ "threadpool", "tokio", "tokio-postgres 0.7.18", + "tokio-postgres-rustls", "tokio-stream", "tokio-util", "tracing", @@ -4291,7 +4313,7 @@ dependencies = [ "uuid", "vergen-gitcl", "xxhash-rust", - "zip 6.0.0", + "zip", ] [[package]] @@ -4378,7 +4400,6 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" dependencies = [ - "serde", "uuid", ] @@ -4550,16 +4571,6 @@ dependencies = [ "url", ] -[[package]] -name = "der" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" -dependencies = [ - "const-oid 0.9.6", - "zeroize", -] - [[package]] name = "der" version = "0.7.10" @@ -4573,6 +4584,20 @@ dependencies = [ "zeroize", ] +[[package]] +name = "der-parser" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" +dependencies = [ + "asn1-rs", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + [[package]] name = "der_derive" version = "0.7.3" @@ -4888,14 +4913,16 @@ dependencies = [ [[package]] name = "ecdsa" -version = "0.14.8" +version = "0.16.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c" +checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" dependencies = [ - "der 0.6.1", + "der", + "digest 0.10.7", "elliptic-curve", "rfc6979", - "signature 1.6.4", + "signature", + "spki", ] [[package]] @@ -4904,7 +4931,7 @@ version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" dependencies = [ - "signature 2.2.0", + "signature", ] [[package]] @@ -4916,7 +4943,7 @@ dependencies = [ "curve25519-dalek", "ed25519", "sha2 0.10.9", - "signature 2.2.0", + "signature", "subtle", ] @@ -4943,18 +4970,18 @@ dependencies = [ [[package]] name = "elliptic-curve" -version = "0.12.3" +version = "0.13.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3" +checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" dependencies = [ "base16ct", - "crypto-bigint 0.4.9", - "der 0.6.1", + "crypto-bigint", "digest 0.10.7", "ff", "generic-array", "group", - "pkcs8 0.9.0", + "pem-rfc7468", + "pkcs8", "rand_core 0.6.4", "sec1", "subtle", @@ -5109,7 +5136,7 @@ dependencies = [ [[package]] name = "etl" version = "0.1.0" -source = "git+https://github.com/supabase/etl?rev=b2aab62d#b2aab62db1830df5b72d1d09ad5c053ee8e44db2" +source = "git+https://github.com/feldera/etl?rev=91260556931fba2f55db9ebc11775e295c453bc9#91260556931fba2f55db9ebc11775e295c453bc9" dependencies = [ "byteorder", "bytes", @@ -5138,7 +5165,7 @@ dependencies = [ [[package]] name = "etl-config" version = "0.1.0" -source = "git+https://github.com/supabase/etl?rev=b2aab62d#b2aab62db1830df5b72d1d09ad5c053ee8e44db2" +source = "git+https://github.com/feldera/etl?rev=91260556931fba2f55db9ebc11775e295c453bc9#91260556931fba2f55db9ebc11775e295c453bc9" dependencies = [ "config", "secrecy", @@ -5152,7 +5179,7 @@ dependencies = [ [[package]] name = "etl-postgres" version = "0.1.0" -source = "git+https://github.com/supabase/etl?rev=b2aab62d#b2aab62db1830df5b72d1d09ad5c053ee8e44db2" +source = "git+https://github.com/feldera/etl?rev=91260556931fba2f55db9ebc11775e295c453bc9#91260556931fba2f55db9ebc11775e295c453bc9" dependencies = [ "aws-lc-rs", "bytes", @@ -5288,7 +5315,6 @@ dependencies = [ "clap", "clap_complete", "directories", - "feldera-observability", "feldera-rest-api", "feldera-types", "futures", @@ -5303,9 +5329,9 @@ dependencies = [ "reqwest 0.12.24", "reqwest-websocket", "rmpv", + "rustls", "rustversion", "rustyline", - "sentry", "serde", "serde_json", "syn 2.0.117", @@ -5317,7 +5343,7 @@ dependencies = [ "tracing-log", "tracing-subscriber", "uuid", - "zip 6.0.0", + "zip", ] [[package]] @@ -5384,12 +5410,12 @@ dependencies = [ [[package]] name = "feldera-cloud1-client" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c41ca98fe69d6105563296f7e52552984ae9638ed00e967396b8047264742e81" +checksum = "c3bbbc99187b0e2afa72198abf5fa523a9bfc9ff3d8615d5bc3b408800534a01" dependencies = [ "chrono", - "reqwest 0.12.24", + "reqwest 0.13.4", "serde", "serde_json", "thiserror 2.0.18", @@ -5477,7 +5503,7 @@ dependencies = [ "serde", "serde_json", "utoipa", - "zip 6.0.0", + "zip", ] [[package]] @@ -5499,7 +5525,6 @@ dependencies = [ "chrono", "colored", "reqwest 0.12.24", - "sentry", "serde_json", "sysinfo 0.38.4", "tracing", @@ -5511,14 +5536,12 @@ name = "feldera-rest-api" version = "0.334.0" dependencies = [ "chrono", - "feldera-observability", "feldera-types", "prettyplease", "progenitor", "progenitor-client 0.9.1", "reqwest 0.12.24", "rustversion", - "sentry", "serde", "serde_json", "syn 2.0.117", @@ -5617,7 +5640,7 @@ dependencies = [ "itertools 0.14.0", "libc", "nix 0.27.1", - "object_store 0.12.5", + "object_store 0.14.1", "once_cell", "rkyv", "serde", @@ -5677,9 +5700,9 @@ dependencies = [ [[package]] name = "ff" -version = "0.12.1" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160" +checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" dependencies = [ "rand_core 0.6.4", "subtle", @@ -5795,21 +5818,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - [[package]] name = "form_urlencoded" version = "1.2.2" @@ -6057,6 +6065,7 @@ checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", + "zeroize", ] [[package]] @@ -6203,9 +6212,9 @@ dependencies = [ [[package]] name = "group" -version = "0.12.1" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" dependencies = [ "ff", "rand_core 0.6.4", @@ -6431,17 +6440,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "hostname" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56f203cd1c76362b69e3863fd987520ac36cf70a8c92627449b2f64a8cf7d65" -dependencies = [ - "cfg-if", - "libc", - "windows-link 0.1.1", -] - [[package]] name = "http" version = "0.2.12" @@ -6607,22 +6605,6 @@ dependencies = [ "tower-service", ] -[[package]] -name = "hyper-tls" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" -dependencies = [ - "bytes", - "http-body-util", - "hyper 1.6.0", - "hyper-util", - "native-tls", - "tokio", - "tokio-native-tls", - "tower-service", -] - [[package]] name = "hyper-util" version = "0.1.17" @@ -6661,7 +6643,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core 0.58.0", + "windows-core 0.61.2", ] [[package]] @@ -7112,6 +7094,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -7287,7 +7278,7 @@ dependencies = [ "pem", "serde", "serde_json", - "signature 2.2.0", + "signature", "simple_asn1", "zeroize", ] @@ -7467,18 +7458,6 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "libz-sys" -version = "1.1.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b70e7a7df205e92a1a4cd9aaae7898dac0aa555503cc0a649494d0d60e7651d" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - [[package]] name = "like" version = "0.3.1" @@ -7847,23 +7826,6 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9252111cf132ba0929b6f8e030cac2a24b507f3a4d6db6fb2896f27b354c714b" -[[package]] -name = "native-tls" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" -dependencies = [ - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework 2.11.1", - "security-framework-sys", - "tempfile", -] - [[package]] name = "nibble_vec" version = "0.1.0" @@ -7907,6 +7869,18 @@ dependencies = [ "libc", ] +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags 2.10.0", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "nkeys" version = "0.4.5" @@ -8174,16 +8148,18 @@ dependencies = [ [[package]] name = "object_store" -version = "0.12.5" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbfbfff40aeccab00ec8a910b57ca8ecf4319b335c542f2edcd19dd25a1e2a00" +checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" dependencies = [ "async-trait", "base64 0.22.1", "bytes", "chrono", "form_urlencoded", - "futures", + "futures-channel", + "futures-core", + "futures-util", "http 1.3.1", "http-body-util", "httparse", @@ -8193,11 +8169,11 @@ dependencies = [ "md-5 0.10.6", "parking_lot 0.12.4", "percent-encoding", - "quick-xml 0.38.3", - "rand 0.9.4", + "quick-xml 0.39.4", + "rand 0.10.1", "reqwest 0.12.24", "ring", - "rustls-pemfile", + "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", @@ -8212,14 +8188,16 @@ dependencies = [ [[package]] name = "object_store" -version = "0.13.2" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" +checksum = "d354792e39fa5f0009e47623cf8b15b099bf9a652fa55c6f817fe28ac84fea50" dependencies = [ "async-trait", + "aws-lc-rs", "base64 0.22.1", "bytes", "chrono", + "crc-fast", "form_urlencoded", "futures-channel", "futures-core", @@ -8229,14 +8207,14 @@ dependencies = [ "httparse", "humantime", "hyper 1.6.0", - "itertools 0.14.0", - "md-5 0.10.6", + "itertools 0.15.0", + "md-5 0.11.0", + "nix 0.31.3", "parking_lot 0.12.4", "percent-encoding", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "rand 0.10.1", - "reqwest 0.12.24", - "ring", + "reqwest 0.13.4", "rustls-pki-types", "serde", "serde_json", @@ -8248,6 +8226,16 @@ dependencies = [ "walkdir", "wasm-bindgen-futures", "web-time", + "windows-sys 0.61.2", +] + +[[package]] +name = "oid-registry" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" +dependencies = [ + "asn1-rs", ] [[package]] @@ -8429,31 +8417,6 @@ dependencies = [ "url", ] -[[package]] -name = "openssl" -version = "0.10.80" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a45fa2aa886c42762255da344f0a0d313e254066c46aad76f300c3d3da62d967" -dependencies = [ - "bitflags 2.10.0", - "cfg-if", - "foreign-types", - "libc", - "openssl-macros", - "openssl-sys", -] - -[[package]] -name = "openssl-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - [[package]] name = "openssl-probe" version = "0.1.6" @@ -8563,18 +8526,6 @@ dependencies = [ "hashbrown 0.14.5", ] -[[package]] -name = "os_info" -version = "3.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0e1ac5fde8d43c34139135df8ea9ee9465394b2d8d20f032d38998f64afffc3" -dependencies = [ - "log", - "plist", - "serde", - "windows-sys 0.52.0", -] - [[package]] name = "os_pipe" version = "1.2.2" @@ -8623,12 +8574,13 @@ checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" [[package]] name = "p256" -version = "0.11.1" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594" +checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" dependencies = [ "ecdsa", "elliptic-curve", + "primeorder", "sha2 0.10.9", ] @@ -8974,6 +8926,7 @@ dependencies = [ "async-stream", "async-trait", "awc", + "aws-lc-rs", "base64 0.22.1", "bytestring", "cached", @@ -8999,20 +8952,21 @@ dependencies = [ "jsonwebtoken", "libc", "nix 0.29.0", - "openssl", "pg-client-config", - "postgres-openssl", "postgresql_embedded", "proptest", "proptest-derive", "rand 0.8.6", + "rcgen", "refinery", "regex", "reqwest 0.12.24", "rmp-serde", + "rsa", "rustls", + "rustls-native-certs 0.7.3", + "rustls-pemfile", "semver", - "sentry", "serde", "serde_json", "serde_yaml", @@ -9026,19 +8980,20 @@ dependencies = [ "termbg", "thiserror 2.0.18", "tikv-jemallocator", + "time", "tokio", "tokio-postgres 0.7.18", + "tokio-postgres-rustls", "tokio-stream", "tracing", "tracing-subscriber", "url", "urlencoding", "utoipa", - "utoipa-swagger-ui", "uuid", "vergen-gitcl", "wiremock", - "zip 6.0.0", + "zip", ] [[package]] @@ -9058,9 +9013,9 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" dependencies = [ - "der 0.7.10", - "pkcs8 0.10.2", - "spki 0.7.3", + "der", + "pkcs8", + "spki", ] [[package]] @@ -9071,21 +9026,11 @@ checksum = "e847e2c91a18bfa887dd028ec33f2fe6f25db77db3619024764914affe8b69a6" dependencies = [ "aes", "cbc", - "der 0.7.10", + "der", "pbkdf2", "scrypt", "sha2 0.10.9", - "spki 0.7.3", -] - -[[package]] -name = "pkcs8" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba" -dependencies = [ - "der 0.6.1", - "spki 0.6.0", + "spki", ] [[package]] @@ -9094,10 +9039,10 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der 0.7.10", + "der", "pkcs5", "rand_core 0.6.4", - "spki 0.7.3", + "spki", ] [[package]] @@ -9112,19 +9057,6 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" -[[package]] -name = "plist" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "740ebea15c5d1428f910cd1a5f52cebf8d25006245ed8ade92702f4943d91e07" -dependencies = [ - "base64 0.22.1", - "indexmap 2.13.0", - "quick-xml 0.38.3", - "serde", - "time", -] - [[package]] name = "plotters" version = "0.3.7" @@ -9221,30 +9153,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "postgres-native-tls" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1f39498473c92f7b6820ae970382c1d83178a3454c618161cb772e8598d9f6f" -dependencies = [ - "native-tls", - "tokio", - "tokio-native-tls", - "tokio-postgres 0.7.18", -] - -[[package]] -name = "postgres-openssl" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb14e4bbc2c0b3d165bf30b79c7a9c10412dff9d98491ffdd64ed810ab891d21" -dependencies = [ - "openssl", - "tokio", - "tokio-openssl", - "tokio-postgres 0.7.18", -] - [[package]] name = "postgres-protocol" version = "0.6.7" @@ -9496,6 +9404,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "primeorder" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" +dependencies = [ + "elliptic-curve", +] + [[package]] name = "proc-macro-crate" version = "3.3.0" @@ -9685,7 +9602,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14cae93065090804185d3b75f0bf93b8eeda30c7a9b4a33d3bdb3988d6229e50" dependencies = [ "bit-set", - "bit-vec", + "bit-vec 0.8.0", "bitflags 2.10.0", "lazy_static", "num-traits", @@ -10203,6 +10120,20 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "rcgen" +version = "0.14.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "091e7a8e7d86e6feb87a27ce8e2cba29d49eff9507afeebefab7eeb2ca667fb4" +dependencies = [ + "aws-lc-rs", + "pem", + "rustls-pki-types", + "time", + "x509-parser", + "yasna", +] + [[package]] name = "rdkafka" version = "0.39.0" @@ -10228,12 +10159,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e234cf318915c1059d4921ef7f75616b5219b10b46e9f3a511a15eb4b56a3f77" dependencies = [ "libc", - "libz-sys", "num_enum", "openssl-sys", "pkg-config", "sasl2-sys", - "zstd-sys", ] [[package]] @@ -10362,9 +10291,9 @@ dependencies = [ [[package]] name = "refinery" -version = "0.9.0" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52c427f2572afe5c6cbfa2b1bf40071c89bf1a8539e958ea582842f6f38dcfae" +checksum = "6e2a344cdb48871e27addeafbbaffab8828cc12cec2b9041119e9bea0c0f551a" dependencies = [ "refinery-core", "refinery-macros", @@ -10372,32 +10301,28 @@ dependencies = [ [[package]] name = "refinery-core" -version = "0.9.0" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "702655abfc67f93a6f735e9fa4ace7d2e580633f8961f28acbfd7583ddce936c" +checksum = "24eeafd893124f29183dd6afa9137a27e7bef59250223b8b660005279c60aea4" dependencies = [ "async-trait", "cfg-if", "log", - "native-tls", - "postgres-native-tls", "regex", - "serde", "siphasher", "thiserror 2.0.18", "time", "tokio", "tokio-postgres 0.7.18", - "toml", "url", "walkdir", ] [[package]] name = "refinery-macros" -version = "0.9.0" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5145756cdf293b5089dc6b4f103f1a1229cc55d67082c866f8c8289531c4b983" +checksum = "a90cea6d11a9a4e8a85a884b6305461004101b28ca65dd35ec028e009a898e16" dependencies = [ "proc-macro2", "quote", @@ -10611,12 +10536,10 @@ dependencies = [ "http-body-util", "hyper 1.6.0", "hyper-rustls", - "hyper-tls", "hyper-util", "js-sys", "log", "mime", - "native-tls", "percent-encoding", "pin-project-lite", "quinn", @@ -10628,7 +10551,6 @@ dependencies = [ "serde_urlencoded", "sync_wrapper 1.0.2", "tokio", - "tokio-native-tls", "tokio-rustls", "tokio-util", "tower", @@ -10669,6 +10591,8 @@ dependencies = [ "rustls", "rustls-pki-types", "rustls-platform-verifier", + "serde", + "serde_json", "sync_wrapper 1.0.2", "tokio", "tokio-rustls", @@ -10765,13 +10689,12 @@ dependencies = [ [[package]] name = "rfc6979" -version = "0.3.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb" +checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" dependencies = [ - "crypto-bigint 0.4.9", "hmac 0.12.1", - "zeroize", + "subtle", ] [[package]] @@ -10898,11 +10821,11 @@ dependencies = [ "num-integer", "num-traits", "pkcs1", - "pkcs8 0.10.2", + "pkcs8", "rand_core 0.6.4", "sha2 0.10.9", - "signature 2.2.0", - "spki 0.7.3", + "signature", + "spki", "subtle", "zeroize", ] @@ -10943,40 +10866,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "rust-embed" -version = "8.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "025908b8682a26ba8d12f6f2d66b987584a4a87bc024abc5bbc12553a8cd178a" -dependencies = [ - "rust-embed-impl", - "rust-embed-utils", - "walkdir", -] - -[[package]] -name = "rust-embed-impl" -version = "8.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6065f1a4392b71819ec1ea1df1120673418bf386f50de1d6f54204d836d4349c" -dependencies = [ - "proc-macro2", - "quote", - "rust-embed-utils", - "syn 2.0.117", - "walkdir", -] - -[[package]] -name = "rust-embed-utils" -version = "8.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6cc0c81648b20b70c491ff8cce00c1c3b223bb8ed2b5d41f0e54c6c4c0a3594" -dependencies = [ - "sha2 0.10.9", - "walkdir", -] - [[package]] name = "rust-ini" version = "0.18.0" @@ -11025,6 +10914,15 @@ dependencies = [ "semver", ] +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom", +] + [[package]] name = "rustix" version = "0.38.44" @@ -11355,14 +11253,14 @@ checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" [[package]] name = "sec1" -version = "0.3.0" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928" +checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ "base16ct", - "der 0.6.1", + "der", "generic-array", - "pkcs8 0.9.0", + "pkcs8", "subtle", "zeroize", ] @@ -11423,141 +11321,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "sentry" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48b85e25e8a1fc13928885e8bf13abe8a09e15c46993aed05d6405f7755d6e20" -dependencies = [ - "httpdate", - "native-tls", - "reqwest 0.12.24", - "rustls", - "sentry-actix", - "sentry-anyhow", - "sentry-backtrace", - "sentry-contexts", - "sentry-core", - "sentry-log", - "sentry-panic", - "sentry-tracing", - "tokio", - "ureq", -] - -[[package]] -name = "sentry-actix" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc694e6ffc8d5d7fdb2a33923b0358f6ad41c0b428ced034b349b9e2b08260bc" -dependencies = [ - "actix-http", - "actix-web", - "bytes", - "futures-util", - "sentry-core", -] - -[[package]] -name = "sentry-anyhow" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfd12302213b0f449a065a68164e30dc02f5a4bc84843c2c5939b2c46b1513e8" -dependencies = [ - "anyhow", - "sentry-backtrace", - "sentry-core", -] - -[[package]] -name = "sentry-backtrace" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3253a495ab536f6de1746a58d5d7824b77d75e08e1a4b8ca6fb356839077ae0" -dependencies = [ - "backtrace", - "regex", - "sentry-core", -] - -[[package]] -name = "sentry-contexts" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "027f81a728836e66b88c07666a10f5ed5a35e2695b04eb7aa0fcbed93f814900" -dependencies = [ - "hostname", - "libc", - "os_info", - "rustc_version", - "sentry-core", - "uname", -] - -[[package]] -name = "sentry-core" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3b6729c8e71ac968edbe9bf2dd4109c162e552b52bacd2b07e24ede1aba84a5" -dependencies = [ - "rand 0.9.4", - "sentry-types", - "serde", - "serde_json", - "url", -] - -[[package]] -name = "sentry-log" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "912fea629a3fc7bfcb97f7bde31a5c815df8526ba19088dcef3ae32ea1e25418" -dependencies = [ - "bitflags 2.10.0", - "log", - "sentry-core", -] - -[[package]] -name = "sentry-panic" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ac0471f04f8f97af0c17eeca2c516e23faa1c0271a55bc64371d9ce488c2d40" -dependencies = [ - "sentry-backtrace", - "sentry-core", -] - -[[package]] -name = "sentry-tracing" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "428f780866a613142dcc81b7f8551ae4d1c056f4df22b6d7ddd9154a9974eb03" -dependencies = [ - "bitflags 2.10.0", - "sentry-backtrace", - "sentry-core", - "tracing-core", - "tracing-subscriber", -] - -[[package]] -name = "sentry-types" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c19d1d1967b55659c358886d0f1aa3076488d445f84c7d727d384c675adaec1" -dependencies = [ - "debugid", - "hex", - "rand 0.9.4", - "serde", - "serde_json", - "thiserror 2.0.18", - "time", - "url", - "uuid", -] - [[package]] name = "seq-macro" version = "0.3.6" @@ -11705,15 +11468,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "serde_spanned" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" -dependencies = [ - "serde", -] - [[package]] name = "serde_tokenstream" version = "0.2.2" @@ -11919,22 +11673,12 @@ version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1e303f8205714074f6068773f0e29527e0453937fe837c9717d066635b65f31" dependencies = [ - "pkcs8 0.10.2", + "pkcs8", "rand_core 0.6.4", - "signature 2.2.0", + "signature", "zeroize", ] -[[package]] -name = "signature" -version = "1.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" -dependencies = [ - "digest 0.10.7", - "rand_core 0.6.4", -] - [[package]] name = "signature" version = "2.2.0" @@ -12087,16 +11831,6 @@ dependencies = [ "lock_api", ] -[[package]] -name = "spki" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b" -dependencies = [ - "base64ct", - "der 0.6.1", -] - [[package]] name = "spki" version = "0.7.3" @@ -12104,7 +11838,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ "base64ct", - "der 0.7.10", + "der", ] [[package]] @@ -12174,9 +11908,9 @@ dependencies = [ "indexmap 2.13.0", "log", "memchr", - "native-tls", "once_cell", "percent-encoding", + "rustls", "serde", "serde_json", "sha2 0.10.9", @@ -12186,6 +11920,7 @@ dependencies = [ "tokio-stream", "tracing", "url", + "webpki-roots 0.26.11", ] [[package]] @@ -13027,6 +12762,27 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +[[package]] +name = "tls_codec" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de2e01245e2bb89d6f05801c564fa27624dbd7b1846859876c7dad82e90bf6b" +dependencies = [ + "tls_codec_derive", + "zeroize", +] + +[[package]] +name = "tls_codec_derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d2e76690929402faae40aebdda620a2c0e25dd6d3b9afe48867dfd95991f4bd" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "token-source" version = "1.0.0" @@ -13064,27 +12820,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "tokio-native-tls" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" -dependencies = [ - "native-tls", - "tokio", -] - -[[package]] -name = "tokio-openssl" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59df6849caa43bb7567f9a36f863c447d95a11d5903c9cc334ba32576a27eadd" -dependencies = [ - "openssl", - "openssl-sys", - "tokio", -] - [[package]] name = "tokio-postgres" version = "0.7.11" @@ -13136,6 +12871,20 @@ dependencies = [ "whoami 2.1.2", ] +[[package]] +name = "tokio-postgres-rustls" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c2ad44aa0ae96db89c4742212ed41645b2f597311ff6e1945542a4d9fadc2fb" +dependencies = [ + "rustls", + "sha2 0.11.0", + "tokio", + "tokio-postgres 0.7.18", + "tokio-rustls", + "x509-cert", +] + [[package]] name = "tokio-retry2" version = "0.6.2" @@ -13205,6 +12954,7 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f591660438b3038dd04d16c938271c79e7e06260ad2ea2885a4861bfb238605d" dependencies = [ + "aws-lc-rs", "base64 0.22.1", "bytes", "futures-core", @@ -13212,7 +12962,6 @@ dependencies = [ "http 1.3.1", "httparse", "rand 0.8.6", - "ring", "rustls-pki-types", "tokio", "tokio-rustls", @@ -13220,26 +12969,11 @@ dependencies = [ "webpki-roots 0.26.11", ] -[[package]] -name = "toml" -version = "0.8.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05ae329d1f08c4d17a59bed7ff5b5a769d062e64a62d34a3261b219e62cd5aae" -dependencies = [ - "serde", - "serde_spanned", - "toml_datetime", - "toml_edit", -] - [[package]] name = "toml_datetime" version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3da5db5a963e24bc68be8b17b6fa82814bb22ee8660f192bb182771d498f09a3" -dependencies = [ - "serde", -] [[package]] name = "toml_edit" @@ -13248,19 +12982,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "310068873db2c5b3e7659d2cc35d21855dbafa50d1ce336397c666e3cb08137e" dependencies = [ "indexmap 2.13.0", - "serde", - "serde_spanned", "toml_datetime", - "toml_write", "winnow", ] -[[package]] -name = "toml_write" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfb942dfe1d8e29a7ee7fcbde5bd2b9a25fb89aa70caea2eba3bee836ff41076" - [[package]] name = "tonic" version = "0.14.5" @@ -13618,15 +13343,6 @@ dependencies = [ "typify-impl", ] -[[package]] -name = "uname" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b72f89f0ca32e4db1c04e2a72f5345d59796d4866a1ee0609084569f73683dc8" -dependencies = [ - "libc", -] - [[package]] name = "unarray" version = "0.1.4" @@ -13724,38 +13440,6 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" -[[package]] -name = "ureq" -version = "3.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99ba1025f18a4a3fc3e9b48c868e9beb4f24f4b4b1a325bada26bd4119f46537" -dependencies = [ - "base64 0.22.1", - "der 0.7.10", - "log", - "native-tls", - "percent-encoding", - "rustls", - "rustls-pemfile", - "rustls-pki-types", - "ureq-proto", - "utf-8", - "webpki-root-certs", - "webpki-roots 1.0.0", -] - -[[package]] -name = "ureq-proto" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60b4531c118335662134346048ddb0e54cc86bd7e81866757873055f0e38f5d2" -dependencies = [ - "base64 0.22.1", - "http 1.3.1", - "httparse", - "log", -] - [[package]] name = "url" version = "2.5.8" @@ -13828,31 +13512,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "utoipa-swagger-ui" -version = "7.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "943e0ff606c6d57d410fd5663a4d7c074ab2c5f14ab903b9514565e59fa1189e" -dependencies = [ - "actix-web", - "mime_guess", - "regex", - "reqwest 0.12.24", - "rust-embed", - "serde", - "serde_json", - "url", - "utoipa", - "utoipa-swagger-ui-vendored", - "zip 1.1.4", -] - -[[package]] -name = "utoipa-swagger-ui-vendored" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2eebbbfe4093922c2b6734d7c679ebfebd704a0d7e56dfcb0d05818ce28977d" - [[package]] name = "uuid" version = "1.21.0" @@ -15023,8 +14682,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1301e935010a701ae5f8655edc0ad17c44bad3ac5ce8c39185f75453b720ae94" dependencies = [ "const-oid 0.9.6", - "der 0.7.10", - "spki 0.7.3", + "der", + "spki", + "tls_codec", +] + +[[package]] +name = "x509-parser" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202" +dependencies = [ + "asn1-rs", + "aws-lc-rs", + "data-encoding", + "der-parser", + "lazy_static", + "nom", + "oid-registry", + "rusticata-macros", + "thiserror 2.0.18", + "time", ] [[package]] @@ -15066,6 +14744,16 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" +[[package]] +name = "yasna" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5f6765e852b9b4dc8e2a76843e4d64d1cea8e79bcde0b6901aea8e7c7f08282" +dependencies = [ + "bit-vec 0.9.1", + "time", +] + [[package]] name = "yoke" version = "0.8.0" @@ -15190,22 +14878,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "zip" -version = "1.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cc23c04387f4da0374be4533ad1208cbb091d5c11d070dfef13676ad6497164" -dependencies = [ - "arbitrary", - "crc32fast", - "crossbeam-utils", - "displaydoc", - "flate2", - "indexmap 2.13.0", - "num_enum", - "thiserror 1.0.69", -] - [[package]] name = "zip" version = "6.0.0" @@ -15294,7 +14966,6 @@ version = "2.0.15+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb81183ddd97d0c74cedf1d50d85c8d08c1b8b68ee863bdee9e706eedba1a237" dependencies = [ - "bindgen 0.71.1", "cc", "pkg-config", ] diff --git a/Cargo.toml b/Cargo.toml index 19ffc8db0d6..f8c0acbd696 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -61,7 +61,23 @@ arrow-json = "58" arrow-schema = "58" ascii_table = "=4.0.2" async-channel = "2.3.1" -async-nats = "0.47.0" +# Its default feature set selects ring; aws-lc-rs is the same choice made +# elsewhere in this workspace. The rest of the list restates the defaults, so +# revisit it when upgrading. +async-nats = { version = "0.47.0", default-features = false, features = [ + "server_2_10", + "server_2_11", + "server_2_12", + "service", + "aws-lc-rs", + "jetstream", + "nkeys", + "crypto", + "object-store", + "kv", + "websockets", + "nuid", +] } async-std = "1.12.0" async-stream = "0.3.5" async-trait = "0.1" @@ -81,6 +97,10 @@ aws-sdk-s3 = { version = "1.122.0", default-features = false, features = [ ] } aws-types = "1.1.7" backoff = "0.4.0" +aws-lc-rs = "1.18.0" +rsa = { version = "0.9.10", features = ["std"] } +rcgen = { version = "0.14.8", default-features = false, features = ["pem", "aws_lc_rs"] } +tokio-postgres-rustls = { version = "0.14.0", default-features = false, features = ["aws-lc-rs"] } base64 = "0.22.1" base58 = "0.2.0" binrw = "0.13.3" @@ -143,7 +163,7 @@ fake = "2.10" fastbloom = "0.14.0" fdlimit = "0.3.0" feldera-buffer-cache = { version = "0.334.0", path = "crates/buffer-cache" } -feldera-cloud1-client = "0.1.2" +feldera-cloud1-client = "0.1.3" feldera-datagen = { path = "crates/datagen" } feldera-fxp = { version = "0.334.0", path = "crates/fxp", features = ["dbsp"] } feldera-iceberg = { path = "crates/iceberg" } @@ -165,7 +185,9 @@ futures-util = "0.3.30" geo = "0.26.0" google-cloud-gax = { package="gcloud-gax", version="1.3.2" } google-cloud-googleapis = { package="gcloud-googleapis", version="1.3.0" } -google-cloud-pubsub = { package="gcloud-pubsub", version="1.6.0" } +# rustls-tls rather than the default default-tls, which pulls native-tls and +# with it the OpenSSL crate. +google-cloud-pubsub = { package = "gcloud-pubsub", version = "1.6.0", default-features = false, features = ["auth", "rustls-tls", "jwt-aws-lc-rs"] } governor = "0.7.0" hashbrown = "0.14.2" hdrhist = "0.5" @@ -207,9 +229,9 @@ num-bigint = "0.4.6" num-derive = "0.4.2" num-format = "0.4.0" num-traits = "0.2.19" -object_store = "0.12.1" +# 0.14 selects aws-lc-rs for request signing where 0.12 used ring. +object_store = "0.14.1" once_cell = "1.20.2" -openssl = "0.10.80" ordered-float = { version = "4.2.0", features = ["serde"] } ouroboros = "0.18.4" parquet = "58" @@ -218,8 +240,8 @@ petgraph = "0.6.0" pg-client-config = "0.1.2" pin-project-lite = "0.2.16" postgres = "0.19.10" -postgres-openssl = "0.5.1" -postgresql_embedded = { version = "0.20.0", features = ["bundled"] } +# rustls rather than the default native-tls, which pulls the OpenSSL crate. +postgresql_embedded = { version = "0.20.0", default-features = false, features = ["bundled", "rustls", "theseus"] } pprof = "0.15.0" pretty_assertions = "1.4.0" prettyplease = "0.2.22" @@ -236,11 +258,19 @@ rand_chacha = "0.3.1" rand_distr = "0.4.3" rand_xoshiro = "0.6.0" range-set = "0.0.11" -rdkafka = "0.39.0" +# Its default `libz` feature only configures the vendored librdkafka build, +# which `dynamic-linking` skips entirely; default-features has to be set here +# because a workspace member cannot override it on an inherited dependency. +rdkafka = { version = "0.39.0", default-features = false } redis = "0.28.2" -refinery = "0.9.0" +refinery = { version = "0.9.2", default-features = false } regex = "1.10.2" -reqwest = "0.12.24" +# default-features off drops default-tls, which pulls native-tls and with it the +# OpenSSL crate; every consumer here uses rustls instead. +# The -no-provider variant leaves the rustls provider to the process default, +# which every binary here installs as aws-lc-rs; the plain feature would pin +# ring instead. +reqwest = { version = "0.12.24", default-features = false, features = ["rustls-tls-native-roots-no-provider", "charset", "http2", "macos-system-configuration"] } reqwest-websocket = "0.5.0" rkyv = { version = "0.7.45", default-features = false } rmp-serde = "1.3.0" @@ -252,10 +282,12 @@ rstest = "0.15" # Make sure this is the same rustls version used by the `tonic` crate. # See the `ensure_default_crypto_provider` function. rustls = "0.23.12" +rustls-pemfile = "2.2.0" +rustls-native-certs = "0.7.3" rustversion = "1.0" rustyline = "15.0" ryu = "1.0.20" -schema_registry_converter = "4.2.0" +schema_registry_converter = { version = "4.2.0", default-features = false } semver = "1.0.27" serde = "1.0.213" serde_dynamo = { version = "4.3.0", features = ["aws-sdk-dynamodb+1"] } @@ -275,9 +307,6 @@ seq-macro = "0.3.6" serde_urlencoded = "0.7.1" serde_with = "3.0.0" serde_yaml = "0.9.34" -# Note: we set default-features = false to disable `debug-images` and `native-tls`; this means we can have debug in release -# builds and use that rather than uploading debug info to sentry every time we compile; native-tls is disabled because we use rustls -sentry = { version = "0.45", default-features = false, features = ["anyhow", "logs", "reqwest", "tracing", "actix", "backtrace", "contexts", "panic", "transport", "rustls"] } serial_test = "3" sha2 = "0.10.8" size-of = { version = "0.1.7", features = [ @@ -315,7 +344,6 @@ typedmap = "0.3.0" url = { version = "2.5.0", features = ["serde"] } urlencoding = "2.1.3" utoipa = { version = "4.2", features = ["uuid", "chrono"] } -utoipa-swagger-ui = { version = "7.1", features = ["vendored"] } uuid = { version = "1.17.0", features = ["serde"] } vergen-gitcl = "1.0.0" wiremock = "0.6" @@ -324,9 +352,9 @@ zip = "6.0.0" zstd = "0.12.0" backtrace = "0.3.75" parking_lot = "0.12.4" -etl = { git = "https://github.com/supabase/etl", rev = "b2aab62d" } -etl-config = { git = "https://github.com/supabase/etl", rev = "b2aab62d" } -etl-postgres = { git = "https://github.com/supabase/etl", rev = "b2aab62d" } +etl = { git = "https://github.com/feldera/etl", rev = "91260556931fba2f55db9ebc11775e295c453bc9" } +etl-config = { git = "https://github.com/feldera/etl", rev = "91260556931fba2f55db9ebc11775e295c453bc9" } +etl-postgres = { git = "https://github.com/feldera/etl", rev = "91260556931fba2f55db9ebc11775e295c453bc9" } [workspace.metadata.release] release = false diff --git a/crates/adapters/Cargo.toml b/crates/adapters/Cargo.toml index 7a186317e0e..08568816ee1 100644 --- a/crates/adapters/Cargo.toml +++ b/crates/adapters/Cargo.toml @@ -70,7 +70,7 @@ iceberg-tests-s3tables = [] # Follow-mode tests. Need a REST catalog and an S3 store both the writer # (pyiceberg) and the connector can reach. See crates/iceberg/src/test/README.md. iceberg-tests-follow = [] -fips = ["rustls/fips"] +fips = ["rustls/fips", "aws-lc-rs/fips"] bench-mode = [] with-postgres-cdc = ["etl", "etl-config", "etl-postgres"] # jemalloc heap-profiling endpoint (/heap_profile). Linux-only dep; on other @@ -107,11 +107,16 @@ serde_json = { workspace = true, features = ["raw_value"] } serde_urlencoded = { workspace = true } form_urlencoded = { workspace = true } csv = { workspace = true } +# dynamic-linking makes rdkafka-sys consume the librdkafka that +# scripts/install-librdkafka.sh builds against AWS-LC, instead of compiling its +# own copy against whatever OpenSSL pkg-config happens to find. The ssl, +# gssapi, zstd and libz features only configured that vendored build, so they +# go: those capabilities now come from the installed library, and the configure +# flags in the script are what preserve them. `ssl` in particular is the only +# thing that pulled openssl-sys into the tree. rdkafka = { workspace = true, features = [ - "ssl", - "gssapi", - "zstd", - "libz", + "dynamic-linking", + "tokio", ], optional = true } aws-config = { workspace = true } aws-sdk-dynamodb = { workspace = true, optional = true } @@ -158,16 +163,18 @@ deltalake = { workspace = true, features = [ ], optional = true } deltalake-catalog-unity = { workspace = true, features = ["aws", "azure", "gcp"], optional = true} apache-avro = { workspace = true, optional = true } +# default-features off drops native_tls, which pulls the OpenSSL crate. schema_registry_converter = { workspace = true, features = [ "avro", "blocking", + "futures", + "rustls_tls", ], optional = true } rust_decimal = { package = "feldera_rust_decimal", version = "1.33.1-feldera.1", features = [ "tokio-pg", ] } url = { workspace = true } ordered-float = { workspace = true } -openssl = { workspace = true } atomic = { workspace = true } once_cell = { workspace = true } dbsp_nexmark = { workspace = true, features = [], optional = true } @@ -203,13 +210,20 @@ bytemuck = { workspace = true } num-derive = { workspace = true } num-traits = { workspace = true } # Used by num-derive derive macro, which cargo-machete can't see postgres = { workspace = true } -postgres-openssl = { workspace = true } dyn-clone = { workspace = true } cpu-time = "1.0.0" feldera-ir = { workspace = true } +aws-lc-rs = { workspace = true } +tokio-postgres-rustls = { workspace = true } +rustls-pemfile = { workspace = true } base64 = { workspace = true } aws-msk-iam-sasl-signer = "1.0.1" aws-credential-types = "1.2.3" +# No code here calls it; the AWS SDK pulls it transitively. The declaration +# floors the version at 1.4.5, where SigV4a signing stopped using ring. A +# lockfile regenerated during a rebase resolved it back below that twice; a +# floor the resolver enforces cannot regress that way. +aws-sigv4 = { version = "1.4.5", default-features = false } feldera-sqllib = { workspace = true } inventory = { workspace = true } backtrace = { workspace = true } @@ -222,7 +236,6 @@ dashmap = { workspace = true } thread-id = { workspace = true } parking_lot = { workspace = true } backoff = { workspace = true } -sentry = { workspace = true } zip = { workspace = true } smallvec = { workspace = true } delta_kernel = { workspace = true } @@ -233,7 +246,7 @@ etl-config = { workspace = true, optional = true } etl-postgres = { workspace = true, optional = true } [package.metadata.cargo-machete] -ignored = ["num-traits"] +ignored = ["num-traits", "aws-sigv4"] [target.'cfg(target_os = "linux")'.dependencies] # Optional so the SQL test fixture (sql-to-dbsp-compiler/temp), which builds diff --git a/crates/adapters/src/controller.rs b/crates/adapters/src/controller.rs index ea8cea43b19..167340a4a1b 100644 --- a/crates/adapters/src/controller.rs +++ b/crates/adapters/src/controller.rs @@ -5477,7 +5477,7 @@ impl ControllerInit { ) -> Result { let checkpoint = Checkpoint::read( &*storage.backend, - &StoragePath::from(checkpoint_uuid.to_string()).child(STATE_FILE), + &StoragePath::from(checkpoint_uuid.to_string()).join(STATE_FILE), )?; Self::with_checkpoint_inner(layout, config, storage, checkpoint) } @@ -9431,7 +9431,7 @@ impl CheckpointThread { // [Checkpoint::write] commits to stable storage. self.checkpoint.write( &*self.storage, - &StoragePath::from(uuid.to_string()).child(STATE_FILE), + &StoragePath::from(uuid.to_string()).join(STATE_FILE), )?; self.checkpoint .write(&*self.storage, &StoragePath::from(STATE_FILE))?; diff --git a/crates/adapters/src/controller/journal.rs b/crates/adapters/src/controller/journal.rs index b267db59e69..4297d48bdc6 100644 --- a/crates/adapters/src/controller/journal.rs +++ b/crates/adapters/src/controller/journal.rs @@ -40,7 +40,7 @@ impl Journal { } pub fn read(&self, step: Step) -> Result, StepError> { - let path = self.path.child(format!("{step}.bin")); + let path = self.path.clone().join(format!("{step}.bin")); let data = match self.backend.read(&path) { Ok(data) => data, Err(error) if error.kind() == ErrorKind::NotFound => return Ok(None), @@ -63,7 +63,7 @@ impl Journal { } pub fn write(&self, record: &StepMetadata) -> Result<(), StepError> { - let path = self.path.child(format!("{}.bin", record.step)); + let path = self.path.clone().join(format!("{}.bin", record.step)); let mut data = FBuf::new(); rmp_serde::encode::write(&mut data, record).map_err(|error| StepError::EncodeError { path: self.path.as_ref().into(), diff --git a/crates/adapters/src/controller/test.rs b/crates/adapters/src/controller/test.rs index 17e48b9f681..df1ce69adea 100644 --- a/crates/adapters/src/controller/test.rs +++ b/crates/adapters/src/controller/test.rs @@ -4543,7 +4543,7 @@ fn test_postprocessor() { #[test] fn test_encryption_postprocessor() { use crate::postprocess::EncryptionPostprocessorFactory; - use openssl::symm::{Cipher, decrypt_aead}; + use crate::preprocess::aes256gcm_decrypt; init_test_logger(); @@ -4652,15 +4652,8 @@ fn test_encryption_postprocessor() { let tag_start = encrypted.len() - 16; let ciphertext = &encrypted[12..tag_start]; let tag = &encrypted[tag_start..]; - let plaintext = decrypt_aead( - Cipher::aes_256_gcm(), - key, - Some(enc_nonce), - &[], - ciphertext, - tag, - ) - .expect("decryption of postprocessed output failed"); + let plaintext = aes256gcm_decrypt(key, enc_nonce, ciphertext, tag) + .expect("decryption of postprocessed output failed"); // The decrypted payload must be parseable CSV with 2 records. let mut rdr = CsvReaderBuilder::new() diff --git a/crates/adapters/src/integrated/postgres/tls.rs b/crates/adapters/src/integrated/postgres/tls.rs index 58413da625d..d37824299a7 100644 --- a/crates/adapters/src/integrated/postgres/tls.rs +++ b/crates/adapters/src/integrated/postgres/tls.rs @@ -1,28 +1,8 @@ use anyhow::{Context, Result as AnyResult}; use feldera_types::transport::postgres::PostgresTlsConfig; -use openssl::{ - pkey::PKey, - rsa::Rsa, - ssl::{SslConnector, SslConnectorBuilder, SslFiletype, SslMethod}, - x509::X509, -}; -use postgres_openssl::MakeTlsConnector; -use std::{io::Write, path::PathBuf}; -use tempfile::NamedTempFile; - -/// Writes the given certificate to file and returns the file path. -fn write_ca_cert_to_file(cert: &str) -> AnyResult { - let mut file = - NamedTempFile::new().context("failed to create tempfile to write CA certificate to")?; - - file.write_all(cert.as_bytes()) - .context("failed to write CA certificate to tempfile")?; - file.flush() - .context("failed to flush CA certificate to tempfile")?; - - let (_, path) = file.keep()?; - Ok(path) -} +use rustls::pki_types::CertificateDer; +use rustls::{ClientConfig, RootCertStore}; +use tokio_postgres_rustls::MakeRustlsConnect; /// Resolves the configured certificate-authority certificate(s) to PEM text. /// @@ -53,104 +33,60 @@ pub(crate) fn resolve_ca_pem( } } -/// Configures SSL certificates for the PostgreSQL connection if enabled. -/// -/// Sets the CA certificate (`ssl_ca_pem`) and optionally the client certificate -/// and private key when provided. -fn set_certs( - builder: &mut SslConnectorBuilder, - config: &PostgresTlsConfig, - endpoint_name: &str, -) -> AnyResult<()> { - let Some(ca_cert) = resolve_ca_pem(config, endpoint_name)? else { - return Ok(()); - }; - let ca_cert_path = write_ca_cert_to_file(&ca_cert)?; - - builder - .set_ca_file(ca_cert_path) - .context("failed to set CA certificate in SSL connector")?; - - fn builder_set_client_from_pem(builder: &mut SslConnectorBuilder, pem: &str) -> AnyResult<()> { - let cert = - X509::from_pem(pem.as_bytes()).context("failed to parse client certificate as X509")?; - builder - .set_certificate(&cert) - .context("failed to set client certificate in SSL connector")?; - Ok(()) - } - - fn builder_set_client_key_from_pem( - builder: &mut SslConnectorBuilder, - pem: &str, - ) -> AnyResult<()> { - let rsa = Rsa::private_key_from_pem(pem.as_bytes()) - .context("failed to parse client private key as RSA")?; - let key = PKey::from_rsa(rsa).context("failed to client private key from RSA")?; - builder - .set_private_key(key.as_ref()) - .context("failed to set client private key")?; - Ok(()) - } - - // Set the client certificate, `ssl_client_pem` takes priority. +/// Resolves the client certificate chain, preferring inline PEM over a path. +fn resolve_client_cert_pem(config: &PostgresTlsConfig) -> AnyResult> { match (&config.ssl_client_pem, &config.ssl_client_location) { (Some(pem), Some(_)) => { tracing::warn!( "postgres: both `ssl_client_pem` and `ssl_client_location` are provided; using `ssl_client_pem`" ); - builder_set_client_from_pem(builder, pem)?; - } - (Some(pem), None) => { - builder_set_client_from_pem(builder, pem)?; - } - (None, Some(location)) => { - builder - .set_certificate_file(location, SslFiletype::PEM) - .context("failed to set client certificate")?; + Ok(Some(pem.clone())) } - // No client cert — ssl_certificate_chain_location only applies to the - // client-side cert chain, so nothing more to configure here. - (None, None) => return Ok(()), + (Some(pem), None) => Ok(Some(pem.clone())), + (None, Some(location)) => Ok(Some( + std::fs::read_to_string(location) + .with_context(|| format!("failed to read client certificate at '{location}'"))?, + )), + (None, None) => Ok(None), } +} - // Set the client key, `ssl_client_key` takes priority. +/// Resolves the client private key, preferring inline PEM over a path. +fn resolve_client_key_pem(config: &PostgresTlsConfig) -> AnyResult> { match (&config.ssl_client_key, &config.ssl_client_key_location) { (Some(key), Some(_)) => { tracing::warn!( "postgres: both `ssl_client_key` and `ssl_client_key_location` are provided; using `ssl_client_key`" ); - builder_set_client_key_from_pem(builder, key)?; - } - (Some(key), None) => { - builder_set_client_key_from_pem(builder, key)?; - } - (None, Some(location)) => { - builder - .set_private_key_file(location, SslFiletype::PEM) - .context("failed to set client private key")?; + Ok(Some(key.clone())) } - (None, None) => return Ok(()), + (Some(key), None) => Ok(Some(key.clone())), + (None, Some(location)) => Ok(Some( + std::fs::read_to_string(location) + .with_context(|| format!("failed to read client private key at '{location}'"))?, + )), + (None, None) => Ok(None), } +} - // Set the SSL chain certificate. - if let Some(chain) = &config.ssl_certificate_chain_location { - builder - .set_certificate_chain_file(chain) - .context("failed to set certificate chain")?; +/// Parses every certificate in `pem`. +fn parse_certs(pem: &str, what: &str) -> AnyResult>> { + let certs: Result, _> = rustls_pemfile::certs(&mut pem.as_bytes()).collect(); + let certs = certs.with_context(|| format!("failed to parse {what} as PEM certificates"))?; + if certs.is_empty() { + anyhow::bail!("{what} contains no certificates"); } - - Ok(()) + Ok(certs) } -/// Builds a [`MakeTlsConnector`] from the given TLS configuration. +/// Builds a [`MakeRustlsConnect`] from the given TLS configuration. /// /// Returns `None` if no TLS configuration is provided, meaning the caller /// should use `NoTls` instead. pub(crate) fn make_tls_connector( tls: &PostgresTlsConfig, endpoint_name: &str, -) -> AnyResult> { +) -> AnyResult> { if !tls.has_tls() && (tls.ssl_client_pem.is_some() || tls.ssl_client_location.is_some() @@ -168,23 +104,53 @@ pub(crate) fn make_tls_connector( return Ok(None); } - let mut builder = - SslConnector::builder(SslMethod::tls()).context("failed to build SSL connection")?; + let Some(ca_pem) = resolve_ca_pem(tls, endpoint_name)? else { + return Ok(None); + }; + + let mut roots = RootCertStore::empty(); + for cert in parse_certs(&ca_pem, "CA certificate")? { + roots + .add(cert) + .context("failed to add CA certificate to the trust store")?; + } - set_certs(&mut builder, tls, endpoint_name)?; + let builder = ClientConfig::builder().with_root_certificates(roots); - let mut connector = MakeTlsConnector::new(builder.build()); + // A client certificate needs its key, and the chain file extends it. + let config = match (resolve_client_cert_pem(tls)?, resolve_client_key_pem(tls)?) { + (Some(cert_pem), Some(key_pem)) => { + let mut chain = parse_certs(&cert_pem, "client certificate")?; + if let Some(location) = &tls.ssl_certificate_chain_location { + let chain_pem = std::fs::read_to_string(location) + .with_context(|| format!("failed to read certificate chain at '{location}'"))?; + chain.extend(parse_certs(&chain_pem, "certificate chain")?); + } + let key = rustls_pemfile::private_key(&mut key_pem.as_bytes()) + .context("failed to parse client private key as PEM")? + .context("client private key PEM contains no key")?; + builder + .with_client_auth_cert(chain, key) + .context("failed to configure client certificate authentication")? + } + (Some(_), None) => { + anyhow::bail!("postgres: a client certificate was provided without a private key") + } + (None, Some(_)) => { + anyhow::bail!("postgres: a client private key was provided without a certificate") + } + (None, None) => builder.with_no_client_auth(), + }; if Some(false) == tls.verify_hostname { - let endpoint_name = endpoint_name.to_owned(); - connector.set_callback(move |ctx, _| { - tracing::warn!("postgres: ssl: disabling hostname verification in connector '{endpoint_name}'. The PostgreSQL server's hostname may not match the one specified in the SSL certificate."); - ctx.set_verify_hostname(false); - Ok(()) - }); + tracing::warn!( + "postgres: ssl: `verify_hostname` is not supported by the rustls connector in \ + endpoint '{endpoint_name}'; the server hostname is still verified against its \ + certificate." + ); } - Ok(Some(connector)) + Ok(Some(MakeRustlsConnect::new(config))) } /// Extracts the trusted root certificates from [`PostgresTlsConfig`] diff --git a/crates/adapters/src/postprocess.rs b/crates/adapters/src/postprocess.rs index bbdf2d5b2dc..7b6e2733eec 100644 --- a/crates/adapters/src/postprocess.rs +++ b/crates/adapters/src/postprocess.rs @@ -1,11 +1,11 @@ //! Example postprocessor implementations. +use crate::preprocess::aes256gcm_encrypt_with_tag; use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}; use feldera_adapterlib::postprocess::{ Postprocessor, PostprocessorCreateError, PostprocessorFactory, }; use feldera_types::postprocess::PostprocessorConfig; -use openssl::symm::{Cipher, encrypt_aead}; use serde::Deserialize; /// A postprocessor that performs no transformation. @@ -75,16 +75,8 @@ impl EncryptionPostprocessor { impl Postprocessor for EncryptionPostprocessor { fn push_buffer(&mut self, data: &[u8]) -> anyhow::Result> { - let mut tag = vec![0u8; 16]; - match encrypt_aead( - Cipher::aes_256_gcm(), - &self.key, - Some(&self.nonce), - &[], - data, - &mut tag, - ) { - Ok(ciphertext) => { + match aes256gcm_encrypt_with_tag(&self.key, &self.nonce, data) { + Ok((ciphertext, tag)) => { let mut out = Vec::with_capacity(self.nonce.len() + ciphertext.len() + 16); out.extend_from_slice(&self.nonce); out.extend_from_slice(&ciphertext); @@ -149,7 +141,7 @@ impl PostprocessorFactory for EncryptionPostprocessorFactory { } #[cfg(test)] -pub use openssl::symm::decrypt_aead; +pub(crate) use crate::preprocess::aes256gcm_decrypt; #[cfg(test)] mod tests { @@ -172,15 +164,7 @@ mod tests { let tag_start = blob.len() - 16; let ciphertext = &blob[12..tag_start]; let tag = &blob[tag_start..]; - decrypt_aead( - Cipher::aes_256_gcm(), - key, - Some(nonce), - &[], - ciphertext, - tag, - ) - .expect("decryption failed") + aes256gcm_decrypt(key, nonce, ciphertext, tag).expect("decryption failed") } #[test] diff --git a/crates/adapters/src/preprocess.rs b/crates/adapters/src/preprocess.rs index 925332289ef..2ba52e57c9d 100644 --- a/crates/adapters/src/preprocess.rs +++ b/crates/adapters/src/preprocess.rs @@ -3,10 +3,10 @@ use crate::format::SpongeSplitter; use std::borrow::Cow; +use aws_lc_rs::aead::{AES_256_GCM, Aad, LessSafeKey, Nonce, UnboundKey}; use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}; use feldera_adapterlib::preprocess::{Preprocessor, PreprocessorCreateError, PreprocessorFactory}; use feldera_types::preprocess::PreprocessorConfig; -use openssl::symm::{Cipher, decrypt_aead}; use serde::Deserialize; use crate::format::{ParseError, Splitter}; @@ -112,14 +112,7 @@ impl Preprocessor for DecryptionPreprocessor { let ciphertext = &data[Self::NONCE_SIZE..tag_start]; let tag = &data[tag_start..]; - match decrypt_aead( - Cipher::aes_256_gcm(), - &self.key, - Some(nonce), - &[], - ciphertext, - tag, - ) { + match aes256gcm_decrypt(&self.key, nonce, ciphertext, tag) { Ok(plaintext) => (plaintext, vec![]), Err(e) => ( vec![], @@ -176,22 +169,45 @@ impl PreprocessorFactory for DecryptionPreprocessorFactory { } } +/// Decrypt AES-256-GCM `ciphertext` authenticated by `tag`, with no associated data. +pub(crate) fn aes256gcm_decrypt( + key: &[u8], + nonce: &[u8], + ciphertext: &[u8], + tag: &[u8], +) -> Result, aws_lc_rs::error::Unspecified> { + let key = LessSafeKey::new(UnboundKey::new(&AES_256_GCM, key)?); + let nonce = Nonce::try_assume_unique_for_key(nonce)?; + + // `open_in_place` expects the tag to trail the ciphertext, and decrypts in place. + let mut in_out = Vec::with_capacity(ciphertext.len() + tag.len()); + in_out.extend_from_slice(ciphertext); + in_out.extend_from_slice(tag); + + let plaintext = key.open_in_place(nonce, Aad::empty(), &mut in_out)?; + Ok(plaintext.to_vec()) +} + +/// Encrypt `plaintext` with AES-256-GCM, returning the ciphertext and its tag. #[cfg(test)] -use openssl::symm::encrypt_aead; +pub(crate) fn aes256gcm_encrypt_with_tag( + key: &[u8], + nonce: &[u8], + plaintext: &[u8], +) -> Result<(Vec, Vec), aws_lc_rs::error::Unspecified> { + let key = LessSafeKey::new(UnboundKey::new(&AES_256_GCM, key)?); + let nonce = Nonce::try_assume_unique_for_key(nonce)?; + + let mut ciphertext = plaintext.to_vec(); + let tag = key.seal_in_place_separate_tag(nonce, Aad::empty(), &mut ciphertext)?; + Ok((ciphertext, tag.as_ref().to_vec())) +} #[cfg(test)] /// Encrypt `plaintext` with AES-256-GCM, returning `[nonce || ciphertext || tag]`. pub fn aes256gcm_encrypt(key: &[u8], nonce: &[u8; 12], plaintext: &[u8]) -> Vec { - let mut tag = vec![0u8; 16]; - let ciphertext = encrypt_aead( - Cipher::aes_256_gcm(), - key, - Some(nonce), - &[], - plaintext, - &mut tag, - ) - .expect("AES-256-GCM encryption failed"); + let (ciphertext, tag) = + aes256gcm_encrypt_with_tag(key, nonce, plaintext).expect("AES-256-GCM encryption failed"); let mut out = Vec::with_capacity(12 + ciphertext.len() + 16); out.extend_from_slice(nonce); diff --git a/crates/adapters/src/server.rs b/crates/adapters/src/server.rs index bd2ab59103a..08a1c593f05 100644 --- a/crates/adapters/src/server.rs +++ b/crates/adapters/src/server.rs @@ -759,18 +759,6 @@ pub fn run_server( eprintln!("{e}"); })?; - let _guard = observability::init( - "https://f0ec61ff0f8483e9ec8117645ad0c0e1@o4510219052253184.ingest.us.sentry.io/4510299519844352", - "pipeline", - env!("CARGO_PKG_VERSION"), - ); - let config_cln = config.clone(); - sentry::configure_scope(|scope| { - if let Some(id) = config_cln.name.as_ref() { - scope.set_tag("pipeline.name", id); - } - }); - // Initialize the logger by setting its filter and template. let pipeline_label = config .given_name @@ -802,7 +790,7 @@ pub fn run_server( } if config.global.tracing { warn!( - "Pipeline tracing was enabled but the 'tracing' option was deprecated, use `FELDERA_SENTRY_ENABLED` for tracing." + "Pipeline tracing was enabled but the 'tracing' option is deprecated and has no effect." ); } @@ -1005,38 +993,36 @@ pub fn run_server( let server = HttpServer::new({ move || { let state = state.clone(); - let app = App::new() - .wrap_fn(|req, srv| { - debug!("Request: {} {}", req.method(), req.path()); - srv.call(req).map(|res| { - match &res { - Ok(response) => { - let level = if response.status().is_success() - || response.status().is_redirection() - || response.status().is_informational() - { - Level::DEBUG - } else { - Level::ERROR - }; - let req = response.request(); - dyn_event!( - level, - "Response: {} (size: {:?}) to request {} {}", - response.status(), - response.response().body().size(), - req.method(), - req.path() - ); - } - Err(e) => { - error!("Service response error: {e}"); - } + let app = App::new().wrap_fn(|req, srv| { + debug!("Request: {} {}", req.method(), req.path()); + srv.call(req).map(|res| { + match &res { + Ok(response) => { + let level = if response.status().is_success() + || response.status().is_redirection() + || response.status().is_informational() + { + Level::DEBUG + } else { + Level::ERROR + }; + let req = response.request(); + dyn_event!( + level, + "Response: {} (size: {:?}) to request {} {}", + response.status(), + response.response().body().size(), + req.method(), + req.path() + ); } - res - }) + Err(e) => { + error!("Service response error: {e}"); + } + } + res }) - .wrap(observability::actix_middleware()); + }); build_app(app, state) } }) diff --git a/crates/dbsp/src/circuit/checkpointer.rs b/crates/dbsp/src/circuit/checkpointer.rs index 5fa0dce0f86..bd7d7374e2d 100644 --- a/crates/dbsp/src/circuit/checkpointer.rs +++ b/crates/dbsp/src/circuit/checkpointer.rs @@ -295,7 +295,7 @@ impl Checkpointer { cp_dir: &StoragePath, ) -> Result<(), StorageError> { let deps: CheckpointDependencies = - match backend.read_json(&cp_dir.child(CHECKPOINT_DEPENDENCIES)) { + match backend.read_json(&cp_dir.clone().join(CHECKPOINT_DEPENDENCIES)) { Ok(d) => d, Err(error) if error.kind() == ErrorKind::NotFound => return Ok(()), Err(error) => return Err(error), @@ -357,7 +357,7 @@ impl Checkpointer { // checkpoint. Explicitly commit it so the durability guarantee // does not depend on the implicit fsync inside `complete()`. self.backend - .write(&Self::checkpoint_dir(uuid).child("CHECKPOINT"), FBuf::new()) + .write(&Self::checkpoint_dir(uuid).join("CHECKPOINT"), FBuf::new()) .and_then(|reader| reader.commit())?; let mut md = CheckpointMetadata { @@ -397,7 +397,7 @@ impl Checkpointer { self.backend .write_json( - &cp_dir.child(CHECKPOINT_DEPENDENCIES), + &cp_dir.clone().join(CHECKPOINT_DEPENDENCIES), &CheckpointDependenciesWrite { batches: &batches_vec, state_files: &state_files, diff --git a/crates/dbsp/src/circuit/circuit_builder.rs b/crates/dbsp/src/circuit/circuit_builder.rs index cacd8cda817..5a964fbeb3d 100644 --- a/crates/dbsp/src/circuit/circuit_builder.rs +++ b/crates/dbsp/src/circuit/circuit_builder.rs @@ -7385,7 +7385,7 @@ where /// Absolute path of the file holding this subcircuit's clock. fn clock_file(base: &StoragePath, persistent_id: &str) -> StoragePath { - base.child(format!("clock-{persistent_id}.dat")) + base.clone().join(format!("clock-{persistent_id}.dat")) } } diff --git a/crates/dbsp/src/circuit/runtime.rs b/crates/dbsp/src/circuit/runtime.rs index 8f88cedff03..dd70f18e34b 100644 --- a/crates/dbsp/src/circuit/runtime.rs +++ b/crates/dbsp/src/circuit/runtime.rs @@ -431,7 +431,7 @@ impl RuntimeInner { if let Some(init_checkpoint) = storage.init_checkpoint && !backend - .exists(&Checkpointer::checkpoint_dir(init_checkpoint).child("CHECKPOINT"))? + .exists(&Checkpointer::checkpoint_dir(init_checkpoint).join("CHECKPOINT"))? { return Err(DbspError::Storage(StorageError::CheckpointNotFound( init_checkpoint, diff --git a/crates/dbsp/src/operator/dynamic/balance/accumulate_trace_balanced.rs b/crates/dbsp/src/operator/dynamic/balance/accumulate_trace_balanced.rs index 2bc5fd9071a..374ef06153d 100644 --- a/crates/dbsp/src/operator/dynamic/balance/accumulate_trace_balanced.rs +++ b/crates/dbsp/src/operator/dynamic/balance/accumulate_trace_balanced.rs @@ -1031,7 +1031,8 @@ where } fn checkpoint_file(base: &StoragePath, persistent_id: &str) -> StoragePath { - base.child(format!("rebalancing-exchange-{}.dat", persistent_id)) + base.clone() + .join(format!("rebalancing-exchange-{}.dat", persistent_id)) } async fn send( diff --git a/crates/dbsp/src/operator/dynamic/time_series/window.rs b/crates/dbsp/src/operator/dynamic/time_series/window.rs index 982bc9dcc3b..f9954a8fd91 100644 --- a/crates/dbsp/src/operator/dynamic/time_series/window.rs +++ b/crates/dbsp/src/operator/dynamic/time_series/window.rs @@ -158,7 +158,7 @@ where /// Return the absolute path of the file for a checkpointed Window. fn checkpoint_file(base: &StoragePath, persistent_id: &str) -> StoragePath { - base.child(format!("window-{}.dat", persistent_id)) + base.clone().join(format!("window-{}.dat", persistent_id)) } } diff --git a/crates/dbsp/src/operator/output.rs b/crates/dbsp/src/operator/output.rs index 541c6505470..49ec222bfb3 100644 --- a/crates/dbsp/src/operator/output.rs +++ b/crates/dbsp/src/operator/output.rs @@ -511,7 +511,7 @@ where } fn checkpoint_file(base: &StoragePath, persistent_id: &str) -> StoragePath { - base.child(format!("output-{}.dat", persistent_id)) + base.clone().join(format!("output-{}.dat", persistent_id)) } } @@ -690,7 +690,8 @@ where } fn checkpoint_file(base: &StoragePath, persistent_id: &str) -> StoragePath { - base.child(format!("accumulate-output-{}.dat", persistent_id)) + base.clone() + .join(format!("accumulate-output-{}.dat", persistent_id)) } /// Merge `snapshot` into the cached accumulated output. diff --git a/crates/dbsp/src/operator/transaction_z1.rs b/crates/dbsp/src/operator/transaction_z1.rs index 63f97b81a82..36efb0cae66 100644 --- a/crates/dbsp/src/operator/transaction_z1.rs +++ b/crates/dbsp/src/operator/transaction_z1.rs @@ -94,7 +94,8 @@ where /// - `persistent_id`: The persistent id that identifies the spine within /// the circuit for a given checkpoint. fn checkpoint_file>(base: &StoragePath, persistent_id: P) -> StoragePath { - base.child(format!("transaction-z1-{}.dat", persistent_id.as_ref())) + base.clone() + .join(format!("transaction-z1-{}.dat", persistent_id.as_ref())) } } diff --git a/crates/dbsp/src/operator/z1.rs b/crates/dbsp/src/operator/z1.rs index 94e6fa9e5a6..11441dcd62a 100644 --- a/crates/dbsp/src/operator/z1.rs +++ b/crates/dbsp/src/operator/z1.rs @@ -270,7 +270,8 @@ where /// - `persistent_id`: The persistent id that identifies the spine within /// the circuit for a given checkpoint. fn checkpoint_file>(base: &StoragePath, persistent_id: P) -> StoragePath { - base.child(format!("z1-{}.dat", persistent_id.as_ref())) + base.clone() + .join(format!("z1-{}.dat", persistent_id.as_ref())) } } diff --git a/crates/dbsp/src/storage/backend/posixio_impl.rs b/crates/dbsp/src/storage/backend/posixio_impl.rs index 38d3f6a8635..ea34b47255e 100644 --- a/crates/dbsp/src/storage/backend/posixio_impl.rs +++ b/crates/dbsp/src/storage/backend/posixio_impl.rs @@ -539,7 +539,8 @@ impl StorageBackend for PosixBackend { Ok(entry) => { let entry = feldera_storage::DirEntry { name: parent - .child(StoragePathPart::from(entry.file_name().as_encoded_bytes())), + .clone() + .join(StoragePathPart::from(entry.file_name().as_encoded_bytes())), file_type: get_file_type(&entry), }; if let Err(e) = &entry.file_type diff --git a/crates/dbsp/src/trace/spine_async.rs b/crates/dbsp/src/trace/spine_async.rs index 5004042c704..7c20c13eaab 100644 --- a/crates/dbsp/src/trace/spine_async.rs +++ b/crates/dbsp/src/trace/spine_async.rs @@ -1856,12 +1856,13 @@ where /// - `persistent_id`: The persistent id that identifies the spine within /// the circuit for a given checkpoint. fn checkpoint_file(base: &StoragePath, persistent_id: &str) -> StoragePath { - base.child(format!("pspine-{}.dat", persistent_id)) + base.clone().join(format!("pspine-{}.dat", persistent_id)) } /// Return the absolute path of the file for this Spine's batchlist. fn batchlist_file(&self, base: &StoragePath, persistent_id: &str) -> StoragePath { - base.child(format!("pspine-batches-{}.dat", persistent_id)) + base.clone() + .join(format!("pspine-batches-{}.dat", persistent_id)) } } diff --git a/crates/fda/Cargo.toml b/crates/fda/Cargo.toml index a123a4e24e7..f757f114b01 100644 --- a/crates/fda/Cargo.toml +++ b/crates/fda/Cargo.toml @@ -19,6 +19,7 @@ clap = { workspace = true, features = ["color"] } clap_complete = { workspace = true, features = ["unstable-dynamic"] } progenitor-client = { workspace = true } reqwest = { workspace = true, features = ["json", "stream"] } +rustls = { workspace = true } reqwest-websocket = { workspace = true } serde_json = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "macros", "io-std", "process"] } @@ -27,8 +28,6 @@ feldera-rest-api = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true, features = ["env-filter"] } tracing-log = { workspace = true } -sentry = { workspace = true } -feldera-observability = { workspace = true } tabled = { workspace = true, features = ["macros", "ansi"] } json_to_table = { workspace = true } rustyline = { workspace = true, features = ["with-file-history"] } diff --git a/crates/fda/src/main.rs b/crates/fda/src/main.rs index d761d82cc5f..2049e91c158 100644 --- a/crates/fda/src/main.rs +++ b/crates/fda/src/main.rs @@ -3,7 +3,6 @@ use chrono::Utc; use clap::{CommandFactory, Parser}; use clap_complete::CompleteEnv; -use feldera_observability as observability; use feldera_rest_api::types::*; use feldera_rest_api::*; use feldera_types::config::{FtModel, RuntimeConfig, StorageOptions}; @@ -3696,10 +3695,10 @@ async fn cluster(format: OutputFormat, action: ClusterAction, client: Client) { } fn main() { - let _guard = observability::init( - "https://18aa37ae23e7130b57b91aaad432bc18@o4510219052253184.ingest.us.sentry.io/4510298809827328", - "fda", - env!("CARGO_PKG_VERSION"), + // reqwest is built without a rustls provider, so the process default decides + // which one it uses. + let _ = rustls::crypto::CryptoProvider::install_default( + rustls::crypto::aws_lc_rs::default_provider(), ); init_logging("warn"); @@ -3762,7 +3761,6 @@ fn init_logging(default_level: &str) { EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(default_level)); let _ = LogTracer::init(); let _ = tracing_subscriber::registry() - .with(sentry::integrations::tracing::layer()) .with(filter) .with( tracing_subscriber::fmt::layer() diff --git a/crates/feldera-observability/Cargo.toml b/crates/feldera-observability/Cargo.toml index 4a106f99abd..65df10ccd87 100644 --- a/crates/feldera-observability/Cargo.toml +++ b/crates/feldera-observability/Cargo.toml @@ -14,7 +14,6 @@ rust-version = { workspace = true } awc = { workspace = true } actix-http = { workspace = true } reqwest = { workspace = true } -sentry = { workspace = true } chrono = { workspace = true, features = ["now"] } serde_json = { workspace = true } tracing = { workspace = true } diff --git a/crates/feldera-observability/src/json_logging.rs b/crates/feldera-observability/src/json_logging.rs index 1b74389191f..a608257364a 100644 --- a/crates/feldera-observability/src/json_logging.rs +++ b/crates/feldera-observability/src/json_logging.rs @@ -303,7 +303,6 @@ fn init_logging( .with_ansi(false), ) .with(env_filter) - .with(sentry::integrations::tracing::layer()) .try_init() } else { tracing_subscriber::registry() @@ -313,7 +312,6 @@ fn init_logging( .fmt_fields(plain_text_fields()), ) .with(env_filter) - .with(sentry::integrations::tracing::layer()) .try_init() } } diff --git a/crates/feldera-observability/src/lib.rs b/crates/feldera-observability/src/lib.rs index 63ecb33e2da..89f4dcbc277 100644 --- a/crates/feldera-observability/src/lib.rs +++ b/crates/feldera-observability/src/lib.rs @@ -1,127 +1,3 @@ -use actix_http::header::{HeaderName, HeaderValue}; -use awc::ClientRequest; -use reqwest::RequestBuilder; -use sentry::{ClientInitGuard, TransactionContext}; -use std::borrow::Cow; -use std::env; - -/// Initializes Sentry if `FELDERA_SENTRY_ENABLED` is set, tagging the service and release. -/// -/// Use dsn as the default DSN, which can be overridden by the `SENTRY_DSN` environment variable. -pub fn init(dsn: &str, service_name: &str, release: &str) -> Option { - env::var("FELDERA_SENTRY_ENABLED") - .ok() - .filter(|v| v == "1")?; - - const DEFAULT_SAMPLE_RATE: f32 = 1.0; - const DEFAULT_TRACE_SAMPLE_RATE: f32 = 0.1; - const MAX_BREADCRUMBS: usize = 10; - - let dsn = env::var("SENTRY_DSN").ok().unwrap_or(dsn.to_string()); - let sample_rate = env::var("SENTRY_SAMPLE_RATE") - .ok() - .and_then(|rate| rate.parse::().ok()) - .unwrap_or(DEFAULT_SAMPLE_RATE); - let traces_sample_rate = env::var("SENTRY_TRACES_SAMPLE_RATE") - .ok() - .and_then(|rate| rate.parse::().ok()) - .unwrap_or(DEFAULT_TRACE_SAMPLE_RATE); - let environment = env::var("SENTRY_ENVIRONMENT") - .unwrap_or_else(|_| String::from("dev")) - .into(); - let accept_invalid_certs = environment == "ci" || environment == "dev"; - let max_breadcrumbs = env::var("SENTRY_MAX_BREADCRUMBS") - .ok() - .and_then(|rate| rate.parse::().ok()) - .unwrap_or(MAX_BREADCRUMBS); - - let guard = sentry::init(( - dsn, - sentry::ClientOptions { - environment: Some(environment), - release: Some(Cow::Owned(release.to_string())), - max_breadcrumbs, - sample_rate, - traces_sample_rate, - enable_logs: true, - attach_stacktrace: true, - accept_invalid_certs, - ..Default::default() - }, - )); - - sentry::configure_scope(|scope| scope.set_tag("service", service_name)); - Some(guard) -} - -fn sentry_enabled() -> bool { - sentry::Hub::current().client().is_some() -} - -/// Returns an Actix middleware that captures errors and traces when Sentry is enabled. -pub fn actix_middleware() -> sentry::integrations::actix::Sentry { - sentry::integrations::actix::Sentry::builder() - .emit_header(true) - .capture_server_errors(true) - .start_transaction(true) - .finish() -} - pub mod fips; pub mod json_logging; pub mod system; - -fn trace_header_value() -> Option { - if !sentry_enabled() { - return None; - } - - let mut header = None; - sentry::configure_scope(|scope| { - if let Some(span) = scope.get_span() { - header = span.iter_headers().next().map(|(_, value)| value); - } - }); - if header.is_none() { - let transaction = - sentry::start_transaction(TransactionContext::new("http.client", "http.client")); - header = transaction.iter_headers().next().map(|(_, value)| value); - transaction.finish(); - } - header -} - -/// Adds Sentry trace headers to outgoing awc requests. -pub trait AwcRequestTracingExt { - fn with_sentry_tracing(self) -> Self; -} - -impl AwcRequestTracingExt for ClientRequest { - fn with_sentry_tracing(mut self) -> Self { - if let Some(value) = trace_header_value() - && let Ok(header_value) = HeaderValue::from_str(&value) - { - self = self.insert_header((HeaderName::from_static("sentry-trace"), header_value)); - } - self - } -} - -/// Adds Sentry trace headers to outgoing reqwest requests. -pub trait ReqwestTracingExt { - fn with_sentry_tracing(self) -> Self; -} - -impl ReqwestTracingExt for RequestBuilder { - fn with_sentry_tracing(self) -> Self { - if let Some(value) = trace_header_value() - && let Ok(header_value) = reqwest::header::HeaderValue::from_str(&value) - { - return self.header( - reqwest::header::HeaderName::from_static("sentry-trace"), - header_value, - ); - } - self - } -} diff --git a/crates/pipeline-manager/Cargo.toml b/crates/pipeline-manager/Cargo.toml index fbe1bfd1323..7c05bb7c267 100644 --- a/crates/pipeline-manager/Cargo.toml +++ b/crates/pipeline-manager/Cargo.toml @@ -27,7 +27,6 @@ feldera-ir = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true, features = ["env-filter", "json"] } colored = { workspace = true } -sentry = { workspace = true } feldera-observability = { workspace = true } # Error handling and asserts @@ -52,12 +51,17 @@ serde_yaml = { workspace = true } rmp-serde = { workspace = true } hex = { workspace = true } indoc = { workspace = true } -openssl = { workspace = true } url = { workspace = true } urlencoding = { workspace = true } regex = { workspace = true } tar = { workspace = true } flate2 = { workspace = true } +aws-lc-rs = { workspace = true } +rcgen = { workspace = true } +time = { workspace = true } +tokio-postgres-rustls = { workspace = true } +rustls-native-certs = { workspace = true } +rustls-pemfile = { workspace = true } base64 = { workspace = true } zip = { workspace = true } semver = { workspace = true } @@ -70,8 +74,10 @@ termbg = { workspace = true } tokio-postgres = { workspace = true, features = ["with-serde_json-1", "with-uuid-1", "with-chrono-0_4"] } deadpool-postgres = { workspace = true } postgresql_embedded = { workspace = true, optional = true } +# Migrations run on a client this crate creates, so refinery needs no TLS +# backend of its own; its tokio-postgres-rustls feature would pin an older +# tokio-postgres-rustls that selects ring. refinery = { workspace = true, features = ["tokio-postgres"] } -postgres-openssl = { workspace = true } # HTTP server and client actix-web = { workspace = true, features = ["rustls-0_23"] } @@ -85,7 +91,6 @@ bytestring = { workspace = true } jsonwebtoken = { workspace = true } static-files = { workspace = true } utoipa = { workspace = true, features = ["actix_extras", "chrono", "uuid"] } -utoipa-swagger-ui = { workspace = true, features = ["actix-web"] } # The `rustls-0_23-native-roots` feature is such that when you create a default awc::Client (during auth workflow), # it can connect to HTTPS with the system root certificates. # @@ -106,7 +111,7 @@ utoipa-swagger-ui = { workspace = true, features = ["actix-web"] } # # - `reqwest` has an HTTPS-only mode. awc = { workspace = true, features = ["rustls-0_23", "rustls-0_23-native-roots"] } -reqwest = { workspace = true, features = ["json", "rustls-tls-native-roots"] } +reqwest = { workspace = true, features = ["json"] } cached = { workspace = true } crossbeam = { workspace = true } rustls = { workspace = true } @@ -132,7 +137,7 @@ tikv-jemallocator = { workspace = true, features = ["profiling", "unprefixed_mal default = ["postgresql_embedded"] feldera-enterprise = [] runtime-version = [] -fips = ["rustls/fips"] +fips = ["rustls/fips", "aws-lc-rs/fips", "rcgen/fips"] [build-dependencies] change-detection = { workspace = true } @@ -140,6 +145,7 @@ static-files = { workspace = true } vergen-gitcl = { workspace = true, features = ["build", "cargo", "rustc", "si"] } [dev-dependencies] +rsa = { workspace = true } proptest = { workspace = true } proptest-derive = { workspace = true } pg-client-config = { workspace = true } diff --git a/crates/pipeline-manager/src/api/main.rs b/crates/pipeline-manager/src/api/main.rs index 01a3122c1c3..138d3c9a2a4 100644 --- a/crates/pipeline-manager/src/api/main.rs +++ b/crates/pipeline-manager/src/api/main.rs @@ -25,7 +25,6 @@ use actix_web::{ use actix_web_httpauth::middleware::HttpAuthentication; use actix_web_static_files::ResourceFiles; use anyhow::Result as AnyResult; -use feldera_observability as observability; use futures_util::FutureExt; use std::io::Write; use std::time::Duration; @@ -37,7 +36,6 @@ use tokio::sync::{Mutex, RwLock}; use tracing::{Level, error, info}; use utoipa::openapi::security::{HttpAuthScheme, HttpBuilder, SecurityScheme}; use utoipa::{Modify, OpenApi}; -use utoipa_swagger_ui::SwaggerUi; macro_rules! log_with_level { ($level:expr, $($arg:tt)+) => { @@ -633,8 +631,7 @@ fn build_app( log_with_level!(log_level, "Request: {} {}", req.method(), req.path()); srv.call(req).map(log_response) }) - .wrap(middleware::Compress::default()) - .wrap(observability::actix_middleware()); + .wrap(middleware::Compress::default()); let app = match auth_configuration { Some(auth_configuration) => { @@ -677,22 +674,19 @@ fn build_app( // Unauthenticated public endpoints and static UI assets. CORS is scoped to // `/config/*` only — it's the unauthenticated API surface that browser clients // may need to reach cross-origin. Every other route here is same-origin in practice -// (swagger UI, healthz monitoring, static bundle), and keeping CORS off them +// (healthz monitoring, static bundle), and keeping CORS off them // is what allows Firefox to honor `Cache-Control: immutable` on the bundle // (no `Vary: Origin`, no `Access-Control-Allow-Credentials`). // // Must be registered LAST in the App: the inner empty-prefix scope acts as the // SPA fallback and would otherwise shadow other top-level scopes. fn public_scope(api_config: &ApiServerConfig) -> Scope { - let openapi = ApiDoc::openapi(); - web::scope("") .service( web::scope("/config") .wrap(api_config.cors()) .service(endpoints::config::get_config_authentication), ) - .service(SwaggerUi::new("/swagger-ui/{_:.*}").url("/api-doc/openapi.json", openapi)) .service(healthz) .service(robots_txt) .service( @@ -1155,11 +1149,6 @@ Version: {} v{}{} let _ = collector_handle.join(); } - if let Some(client) = sentry::Hub::current().client() { - info!("Shutting down sentry"); - client.close(Some(Duration::from_secs(3))); - } - server_result?; Ok(()) } @@ -1447,7 +1436,7 @@ mod tests { // handlers (which 500 on `WebData` extraction since this // test skips `app_data` — `actix-cors` still adds headers on the // response), api_scope param routes, the cors-wrapped `/config` - // sub-scope, and the unwrapped public routes (`/healthz`, swagger). + // sub-scope, and the unwrapped public routes (`/healthz`). // `/config/authentication` works through its real handler because // `auth_provider = None` in test_config returns early without state. for uri in [ @@ -1466,7 +1455,6 @@ mod tests { "/v0/cluster_healthz", "/config/authentication", "/healthz", - "/swagger-ui/index.html", ] { let req = test::TestRequest::get() .uri(uri) diff --git a/crates/pipeline-manager/src/auth.rs b/crates/pipeline-manager/src/auth.rs index 4f0a5158c41..2350d78bc0e 100644 --- a/crates/pipeline-manager/src/auth.rs +++ b/crates/pipeline-manager/src/auth.rs @@ -1803,19 +1803,29 @@ mod test { ensure_default_crypto_provider, }; use crate::{auth::fetch_jwk_oidc_keys, config::CommonConfig}; + use rsa::pkcs1::{EncodeRsaPrivateKey, LineEnding}; + use rsa::pkcs8::EncodePublicKey; + use rsa::{RsaPrivateKey, RsaPublicKey}; async fn setup(claim: OidcClaim) -> (String, DecodingKey) { - let rsa = openssl::rsa::Rsa::generate(2048).unwrap(); + let rsa = RsaPrivateKey::new(&mut rand::thread_rng(), 2048).unwrap(); let mut header = Header::new(Algorithm::RS256); header.kid = Some("rsa01".to_owned()); let token_encoded = encode( &header, &claim, - &EncodingKey::from_rsa_pem(&rsa.private_key_to_pem().unwrap()).unwrap(), + &EncodingKey::from_rsa_pem(rsa.to_pkcs1_pem(LineEnding::LF).unwrap().as_bytes()) + .unwrap(), + ) + .unwrap(); + let decoding_key = DecodingKey::from_rsa_pem( + RsaPublicKey::from(&rsa) + .to_public_key_pem(LineEnding::LF) + .unwrap() + .as_bytes(), ) .unwrap(); - let decoding_key = DecodingKey::from_rsa_pem(&rsa.public_key_to_pem().unwrap()).unwrap(); let token = token_encoded.as_str(); (token.to_owned(), decoding_key) } diff --git a/crates/pipeline-manager/src/bin/pipeline-manager.rs b/crates/pipeline-manager/src/bin/pipeline-manager.rs index b53b48c78db..6ad3e23def9 100644 --- a/crates/pipeline-manager/src/bin/pipeline-manager.rs +++ b/crates/pipeline-manager/src/bin/pipeline-manager.rs @@ -25,11 +25,6 @@ use utoipa::OpenApi; fn main() -> anyhow::Result<()> { ensure_default_crypto_provider(); init_fd_limit(); - let _guard = observability::init( - "https://18aa37ae23e7130b57b91aaad432bc18@o4510219052253184.ingest.us.sentry.io/4510298809827328", - "pipeline-manager", - env!("CARGO_PKG_VERSION"), - ); pipeline_manager::logging::init_service_logging( "[manager]".cyan(), feldera_observability::json_logging::ServiceName::Manager, diff --git a/crates/pipeline-manager/src/cluster_monitor.rs b/crates/pipeline-manager/src/cluster_monitor.rs index b91b28a88ed..fdfb0535c6c 100644 --- a/crates/pipeline-manager/src/cluster_monitor.rs +++ b/crates/pipeline-manager/src/cluster_monitor.rs @@ -5,7 +5,6 @@ use crate::db::storage_postgres::StoragePostgres; use crate::db::types::monitor::{MonitorStatus, NewClusterMonitorEvent}; use crate::error::source_error; use async_trait::async_trait; -use feldera_observability::ReqwestTracingExt; use std::{sync::Arc, time::Duration}; use tokio::sync::Mutex; use tracing::{error, info}; @@ -276,7 +275,6 @@ async fn poll_service_health_endpoint( match client .get(url) .timeout(DEFAULT_REQUEST_TIMEOUT) - .with_sentry_tracing() .send() .await { diff --git a/crates/pipeline-manager/src/compiler/main.rs b/crates/pipeline-manager/src/compiler/main.rs index 2eadfa3af18..7238f17098e 100644 --- a/crates/pipeline-manager/src/compiler/main.rs +++ b/crates/pipeline-manager/src/compiler/main.rs @@ -594,7 +594,7 @@ async fn stream_to_file_and_verify( )) })?; - let mut hasher = openssl::sha::Sha256::new(); + let mut hasher = aws_lc_rs::digest::Context::new(&aws_lc_rs::digest::SHA256); let mut total_size = 0usize; while let Some(chunk) = payload.next().await { @@ -1114,6 +1114,7 @@ mod test { }; use crate::compiler::util::CleanupDecision; use crate::compiler::util::pipeline_binary_filename; + use crate::compiler::util::sha256; use crate::config::CompilerConfig; use crate::db::types::pipeline::PipelineId; use crate::db::types::program::CompilationProfile; @@ -1121,7 +1122,6 @@ mod test { use crate::error::ManagerError; use actix_web::error::PayloadError; use actix_web::{App, test as actix_test, web}; - use openssl::sha::sha256; use std::time::Duration; use tokio::fs; use uuid::Uuid; diff --git a/crates/pipeline-manager/src/compiler/rust_compiler.rs b/crates/pipeline-manager/src/compiler/rust_compiler.rs index 464a922b852..d1855d11287 100644 --- a/crates/pipeline-manager/src/compiler/rust_compiler.rs +++ b/crates/pipeline-manager/src/compiler/rust_compiler.rs @@ -4,7 +4,7 @@ use crate::compiler::util::{ copy_file, copy_file_if_checksum_differs, crate_name_pipeline_globals, crate_name_pipeline_main, create_dir_if_not_exists, create_new_file, decode_string_as_dir, pipeline_binary_filename, program_info_filename, read_file_content, recreate_dir, - recreate_file_with_content, truncate_sha256_checksum, write_file, + recreate_file_with_content, sha256, truncate_sha256_checksum, write_file, }; use crate::config::{CommonConfig, CompilerConfig}; use crate::db::error::DBError; @@ -17,12 +17,11 @@ use crate::db::types::utils::{validate_program_config, validate_program_info}; use crate::db::types::version::Version; use crate::error::source_error; use crate::has_unstable_feature; +use aws_lc_rs::digest; use chrono::{DateTime, Utc}; use feldera_types::config::PipelineConfigProgramInfo; use futures_util::stream; use indoc::formatdoc; -use openssl::sha; -use openssl::sha::sha256; use std::collections::{BTreeMap, HashSet}; use std::path::{Path, PathBuf}; use std::time::{Instant, SystemTime}; @@ -392,7 +391,7 @@ fn calculate_source_checksum( udf_rust: &str, udf_toml: &str, ) -> String { - let mut hasher = sha::Sha256::new(); + let mut hasher = digest::Context::new(&digest::SHA256); let profile = profile.to_string(); let to_hash = vec![ ("platform_version", platform_version.as_bytes()), diff --git a/crates/pipeline-manager/src/compiler/sql_compiler.rs b/crates/pipeline-manager/src/compiler/sql_compiler.rs index 0026d229856..1f9a46f73cc 100644 --- a/crates/pipeline-manager/src/compiler/sql_compiler.rs +++ b/crates/pipeline-manager/src/compiler/sql_compiler.rs @@ -18,7 +18,6 @@ use crate::db::types::version::Version; use crate::error::source_error; use crate::has_unstable_feature; use feldera_ir::Dataflow; -use feldera_observability::ReqwestTracingExt; use futures_util::StreamExt; use indoc::formatdoc; use serde::{Deserialize, Serialize}; @@ -445,7 +444,6 @@ async fn fetch_sql_compiler( let response = client .get(&jar_cache_url) - .with_sentry_tracing() .send() .await .map_err(|e| { diff --git a/crates/pipeline-manager/src/compiler/util.rs b/crates/pipeline-manager/src/compiler/util.rs index 7eafec1d092..80114538577 100644 --- a/crates/pipeline-manager/src/compiler/util.rs +++ b/crates/pipeline-manager/src/compiler/util.rs @@ -1,5 +1,6 @@ use crate::db::types::pipeline::PipelineId; use crate::db::types::version::Version; +use aws_lc_rs::digest; use base64::prelude::{BASE64_STANDARD, Engine}; use flate2::Compression; use hex; @@ -7,7 +8,6 @@ use nix::NixPath; use nix::libc::pid_t; use nix::sys::signal::{Signal, killpg}; use nix::unistd::Pid; -use openssl::sha::sha256; use sha2::{Digest, Sha256}; use std::fs::Metadata; use std::path::{Path, PathBuf}; @@ -20,6 +20,13 @@ use tokio::{ }; use tracing::{debug, error, warn}; +/// SHA-256 of `data`, computed by the cryptographic module the binary links. +pub fn sha256(data: &[u8]) -> [u8; 32] { + let mut checksum = [0u8; 32]; + checksum.copy_from_slice(digest::digest(&digest::SHA256, data).as_ref()); + checksum +} + /// Automatically terminates a process and all subprocesses it spawns using /// the group they are all in. The process must have set a process group ID /// via `Command::process_group()`. @@ -730,11 +737,10 @@ mod test { crate_name_pipeline_globals, crate_name_pipeline_main, create_dir_if_not_exists, create_new_file, create_new_file_with_content, decode_string_as_dir, encode_dir_as_string, pipeline_binary_filename, read_file_content, read_file_content_bytes, recreate_dir, - recreate_file_with_content, truncate_sha256_checksum, validate_is_sha256_checksum, + recreate_file_with_content, sha256, truncate_sha256_checksum, validate_is_sha256_checksum, }; use crate::db::types::pipeline::PipelineId; use crate::db::types::version::Version; - use openssl::sha::sha256; use std::fs::Metadata; use std::sync::Arc; use std::time::Duration; diff --git a/crates/pipeline-manager/src/config.rs b/crates/pipeline-manager/src/config.rs index d9facf2f7d6..65a6021352f 100644 --- a/crates/pipeline-manager/src/config.rs +++ b/crates/pipeline-manager/src/config.rs @@ -8,21 +8,17 @@ use crate::oidc::destination::TenantIssuerPolicy; use actix_web::http::header; use anyhow::{Context, Error as AnyError, Result as AnyResult}; use clap::Parser; -use openssl::asn1::Asn1Time; -use openssl::bn::{BigNum, MsbOption}; -use openssl::hash::MessageDigest; -use openssl::nid::Nid; -use openssl::pkey::PKey; -use openssl::rsa::Rsa; -use openssl::ssl::{SslConnector, SslMethod, SslVerifyMode}; -use openssl::x509::extension::SubjectAlternativeName; -use openssl::x509::{X509, X509NameBuilder}; -use postgres_openssl::MakeTlsConnector; use reqwest::Certificate; +use rustls::client::WebPkiServerVerifier; +use rustls::client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier}; +use rustls::pki_types::PrivateKeyDer; use rustls::pki_types::pem::PemObject; -use rustls::pki_types::{CertificateDer, PrivateKeyDer}; -use rustls::{ClientConfig, RootCertStore}; +use rustls::pki_types::{CertificateDer, ServerName, UnixTime}; +use rustls::{ + CertificateError, ClientConfig, DigitallySignedStruct, RootCertStore, SignatureScheme, +}; use serde::{Deserialize, Serialize}; +use std::net::{IpAddr, Ipv4Addr}; use std::str::FromStr; use std::sync::Arc; use std::{ @@ -32,6 +28,8 @@ use std::{ sync::Once, thread, }; +use time::{Duration, OffsetDateTime}; +use tokio_postgres_rustls::MakeRustlsConnect; use tracing::warn; /// The default `platform_version` is formed using three compilation environment variables: @@ -396,34 +394,24 @@ impl CommonConfig { return Ok(()); } - let private_key = PKey::from_rsa(Rsa::generate(2048)?)?; - - let mut name_builder = X509NameBuilder::new()?; - name_builder.append_entry_by_nid(Nid::COMMONNAME, "localhost")?; - let name = name_builder.build(); - - let mut certificate_builder = X509::builder()?; - certificate_builder.set_version(2)?; - let mut serial = BigNum::new()?; - serial.rand(128, MsbOption::MAYBE_ZERO, false)?; - let serial = serial.to_asn1_integer()?; - certificate_builder.set_serial_number(&serial)?; - certificate_builder.set_subject_name(&name)?; - certificate_builder.set_issuer_name(&name)?; - certificate_builder.set_pubkey(&private_key)?; - let not_before = Asn1Time::days_from_now(0)?; - let not_after = Asn1Time::days_from_now(30)?; - certificate_builder.set_not_before(not_before.as_ref())?; - certificate_builder.set_not_after(not_after.as_ref())?; - let san = SubjectAlternativeName::new() - .dns("localhost") - .ip("127.0.0.1") - .build(&certificate_builder.x509v3_context(None, None))?; - certificate_builder.append_extension(san)?; - certificate_builder.sign(&private_key, MessageDigest::sha256())?; - - let cert_pem = certificate_builder.build().to_pem()?; - let key_pem = private_key.private_key_to_pem_pkcs8()?; + let mut params = rcgen::CertificateParams::default(); + params.distinguished_name = { + let mut name = rcgen::DistinguishedName::new(); + name.push(rcgen::DnType::CommonName, "localhost"); + name + }; + params.subject_alt_names = vec![ + rcgen::SanType::DnsName("localhost".try_into()?), + rcgen::SanType::IpAddress(IpAddr::V4(Ipv4Addr::LOCALHOST)), + ]; + params.not_before = OffsetDateTime::now_utc(); + params.not_after = params.not_before + Duration::days(30); + + let key_pair = rcgen::KeyPair::generate()?; + let certificate = params.self_signed(&key_pair)?; + + let cert_pem = certificate.pem().into_bytes(); + let key_pem = key_pair.serialize_pem().into_bytes(); let dir = env::temp_dir().join("feldera-testing-https"); create_dir_all(&dir)?; @@ -721,7 +709,8 @@ pub struct DatabaseConfig { /// Create a TLS connector by loading a certificate from the path specified argument. /// - /// If the argument is not set, tries to connect without TLS. + /// If the argument is not set, the connector still negotiates TLS when the + /// server offers it and verifies the server against the system trust roots. #[arg(long, env = "FELDERA_DB_TLS_CERT_PATH")] pub db_tls_certificate_path: Option, @@ -780,53 +769,157 @@ impl DatabaseConfig { } } - pub(crate) fn tls_connector(&self) -> Result { - let mut builder = - SslConnector::builder(SslMethod::tls()).map_err(|e| DBError::TlsConnection { - hint: "Unable to build TLS Connector to connect to PostgreSQL".to_string(), - openssl_error: Some(e), + pub(crate) fn tls_connector(&self) -> Result { + let mut roots = RootCertStore::empty(); + if let Some(ca_path) = &self.db_tls_certificate_path { + let pem = std::fs::read(ca_path).map_err(|e| DBError::TlsConnection { + hint: format!("Unable to read TLS certificate at {ca_path:?}"), + error: Some(e.to_string()), + })?; + let certs: Result, _> = rustls_pemfile::certs(&mut pem.as_slice()).collect(); + let certs = certs.map_err(|e| DBError::TlsConnection { + hint: format!("Unable to parse TLS certificate at {ca_path:?}"), + error: Some(e.to_string()), })?; + for cert in certs { + roots.add(cert).map_err(|e| DBError::TlsConnection { + hint: format!("Unable to trust TLS certificate at {ca_path:?}"), + error: Some(e.to_string()), + })?; + } + } else { + // Mirrors openssl's set_default_verify_paths: use the system trust store. + let native = + rustls_native_certs::load_native_certs().map_err(|e| DBError::TlsConnection { + hint: "Unable to load the system TLS trust store".to_string(), + error: Some(e.to_string()), + })?; + for cert in native { + let _ = roots.add(cert); + } + } - if self.disable_tls_verify { + let builder = ClientConfig::builder(); + let config = if self.disable_tls_verify { static ONCE: Once = Once::new(); ONCE.call_once(|| { warn!("PostgreSQL TLS verification is disabled -- not recommended for production environments."); }); - builder.set_verify(SslVerifyMode::NONE); - } - - if let Some(ca_path) = &self.db_tls_certificate_path { builder - .set_ca_file(ca_path) + .dangerous() + .with_custom_certificate_verifier(Arc::new(NoCertificateVerification)) + .with_no_client_auth() + } else if self.disable_tls_hostname_verify { + warn!( + "PostgreSQL TLS hostname verification is disabled. The PostgreSQL server's hostname may not match the one specified in the SSL certificate." + ); + let inner = WebPkiServerVerifier::builder(Arc::new(roots)) + .build() .map_err(|e| DBError::TlsConnection { - hint: format!( - "Unable to find TLS certificate at {:?}", - self.db_tls_certificate_path - ), - openssl_error: Some(e), + hint: "Unable to build TLS Connector to connect to PostgreSQL".to_string(), + error: Some(e.to_string()), })?; - } else { builder - .set_default_verify_paths() - .map_err(|e| DBError::TlsConnection { - hint: "Unable to configure default TLS paths".to_string(), - openssl_error: Some(e), - })?; - } + .dangerous() + .with_custom_certificate_verifier(Arc::new(SkipHostnameVerification(inner))) + .with_no_client_auth() + } else { + builder.with_root_certificates(roots).with_no_client_auth() + }; - let mut connector = MakeTlsConnector::new(builder.build()); + Ok(MakeRustlsConnect::new(config)) + } +} - if self.disable_tls_hostname_verify { - warn!( - "PostgreSQL TLS hostname verification is disabled. The PostgreSQL server's hostname may not match the one specified in the SSL certificate." - ); - connector.set_callback(|ctx, _| { - ctx.set_verify_hostname(false); - Ok(()) - }); +/// Accepts any server certificate. Used only when TLS verification is disabled +/// explicitly, which the connector warns about. +#[derive(Debug)] +struct NoCertificateVerification; + +impl ServerCertVerifier for NoCertificateVerification { + fn verify_server_cert( + &self, + _end_entity: &CertificateDer<'_>, + _intermediates: &[CertificateDer<'_>], + _server_name: &ServerName<'_>, + _ocsp_response: &[u8], + _now: UnixTime, + ) -> Result { + Ok(ServerCertVerified::assertion()) + } + + fn verify_tls12_signature( + &self, + _message: &[u8], + _cert: &CertificateDer<'_>, + _dss: &DigitallySignedStruct, + ) -> Result { + Ok(HandshakeSignatureValid::assertion()) + } + + fn verify_tls13_signature( + &self, + _message: &[u8], + _cert: &CertificateDer<'_>, + _dss: &DigitallySignedStruct, + ) -> Result { + Ok(HandshakeSignatureValid::assertion()) + } + + fn supported_verify_schemes(&self) -> Vec { + rustls::crypto::CryptoProvider::get_default() + .expect("a default crypto provider is installed at startup") + .signature_verification_algorithms + .supported_schemes() + } +} + +/// Verifies the certificate chain but tolerates a name mismatch, matching the +/// behaviour of the `disable_tls_hostname_verify` option. +#[derive(Debug)] +struct SkipHostnameVerification(Arc); + +impl ServerCertVerifier for SkipHostnameVerification { + fn verify_server_cert( + &self, + end_entity: &CertificateDer<'_>, + intermediates: &[CertificateDer<'_>], + server_name: &ServerName<'_>, + ocsp_response: &[u8], + now: UnixTime, + ) -> Result { + match self + .0 + .verify_server_cert(end_entity, intermediates, server_name, ocsp_response, now) + { + Err(rustls::Error::InvalidCertificate(CertificateError::NotValidForName)) + | Err(rustls::Error::InvalidCertificate(CertificateError::NotValidForNameContext { + .. + })) => Ok(ServerCertVerified::assertion()), + other => other, } + } + + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, + ) -> Result { + self.0.verify_tls12_signature(message, cert, dss) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, + ) -> Result { + self.0.verify_tls13_signature(message, cert, dss) + } - Ok(connector) + fn supported_verify_schemes(&self) -> Vec { + self.0.supported_verify_schemes() } } diff --git a/crates/pipeline-manager/src/db/error.rs b/crates/pipeline-manager/src/db/error.rs index b3381abc783..201c119e157 100644 --- a/crates/pipeline-manager/src/db/error.rs +++ b/crates/pipeline-manager/src/db/error.rs @@ -267,8 +267,7 @@ pub enum DBError { }, TlsConnection { hint: String, - #[serde(skip)] - openssl_error: Option, + error: Option, }, PauseWhileNotProvisioned, ResumeWhileNotProvisioned, diff --git a/crates/pipeline-manager/src/db/operations/api_key.rs b/crates/pipeline-manager/src/db/operations/api_key.rs index b82a846eea8..5e4a1383209 100644 --- a/crates/pipeline-manager/src/db/operations/api_key.rs +++ b/crates/pipeline-manager/src/db/operations/api_key.rs @@ -6,8 +6,10 @@ use crate::db::types::api_key::{ApiKeyDescr, ApiKeyId}; use crate::db::types::role::{MintableKeyRole, Role}; use crate::db::types::tenant::TenantId; use crate::db::types::utils::validate_api_key_name; +use aws_lc_rs::digest; +use base64::Engine; +use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; use deadpool_postgres::Transaction; -use openssl::sha; use std::str::FromStr; use uuid::Uuid; @@ -79,9 +81,7 @@ pub async fn store_api_key_hash( role: MintableKeyRole, ) -> Result<(), DBError> { validate_api_key_name(name)?; - let mut hasher = sha::Sha256::new(); - hasher.update(key.as_bytes()); - let hash = openssl::base64::encode_block(&hasher.finish()); + let hash = BASE64_STANDARD.encode(digest::digest(&digest::SHA256, key.as_bytes()).as_ref()); let stmt = txn .prepare_cached( "INSERT INTO api_key (id, tenant_id, name, hash, role) VALUES ($1, $2, $3, $4, $5)", @@ -106,9 +106,7 @@ pub async fn validate_api_key( txn: &Transaction<'_>, api_key: &str, ) -> Result<(TenantId, Role), DBError> { - let mut hasher = sha::Sha256::new(); - hasher.update(api_key.as_bytes()); - let hash = openssl::base64::encode_block(&hasher.finish()); + let hash = BASE64_STANDARD.encode(digest::digest(&digest::SHA256, api_key.as_bytes()).as_ref()); let stmt = txn .prepare_cached("SELECT tenant_id, role FROM api_key WHERE hash = $1") .await?; diff --git a/crates/pipeline-manager/src/db/test.rs b/crates/pipeline-manager/src/db/test.rs index dce52be5e30..5a4f290c1ac 100644 --- a/crates/pipeline-manager/src/db/test.rs +++ b/crates/pipeline-manager/src/db/test.rs @@ -42,6 +42,9 @@ use crate::db::types::version::Version; use crate::oidc::destination::{TenantIssuerPolicy, validate_tenant_oidc_url}; use crate::oidc::trust_name::validate_oidc_trust_name; use async_trait::async_trait; +use aws_lc_rs::digest; +use base64::Engine; +use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; use chrono::{DateTime, TimeZone, Utc}; use deadpool_postgres::GenericClient; use feldera_types::checkpoint::CheckpointMetadata; @@ -53,7 +56,6 @@ use feldera_types::program_schema::ProgramSchema; use feldera_types::runtime_status::{ BootstrapConfig, BootstrapPolicy, RuntimeDesiredStatus, RuntimeStatus, StorageStatusDetails, }; -use openssl::sha; use proptest::prelude::*; use proptest::test_runner::{Config, TestRunner}; use proptest_derive::Arbitrary; @@ -95,8 +97,8 @@ impl Drop for DbHandle { #[cfg(not(feature = "postgresql_embedded"))] fn drop(&mut self) { - use postgres_openssl::TlsStream; use tokio_postgres::{Connection, Socket, tls::NoTlsStream}; + use tokio_postgres_rustls::RustlsStream as TlsStream; enum ConnWrapper { Tls(Connection>), NoTls(Connection), @@ -6685,9 +6687,7 @@ impl Storage for Mutex { ) -> DBResult<()> { let mut s = self.lock().await; validate_api_key_name(name)?; - let mut hasher = sha::Sha256::new(); - hasher.update(key.as_bytes()); - let hash = openssl::base64::encode_block(&hasher.finish()); + let hash = BASE64_STANDARD.encode(digest::digest(&digest::SHA256, key.as_bytes()).as_ref()); if s.api_keys.iter().any(|k| k.1.0 == ApiKeyId(id)) { return Err(DBError::unique_key_violation("api_key_pkey")); } @@ -6706,9 +6706,7 @@ impl Storage for Mutex { async fn validate_api_key(&self, key: &str) -> DBResult<(TenantId, Role)> { let s = self.lock().await; - let mut hasher = sha::Sha256::new(); - hasher.update(key.as_bytes()); - let hash = openssl::base64::encode_block(&hasher.finish()); + let hash = BASE64_STANDARD.encode(digest::digest(&digest::SHA256, key.as_bytes()).as_ref()); let record: Vec<(TenantId, Role)> = s .api_keys .iter() diff --git a/crates/pipeline-manager/src/error.rs b/crates/pipeline-manager/src/error.rs index 87b8620816c..8d4c8d35ff7 100644 --- a/crates/pipeline-manager/src/error.rs +++ b/crates/pipeline-manager/src/error.rs @@ -30,7 +30,6 @@ use actix_web::{ HttpResponse, HttpResponseBuilder, ResponseError, body::BoxBody, http::StatusCode, http::header, }; use feldera_types::error::{DetailedError, ErrorResponse}; -use openssl::error::ErrorStack; use serde::Serialize; use std::{ borrow::Cow, @@ -100,16 +99,6 @@ impl From for ManagerError { } } -impl From for ManagerError { - fn from(value: ErrorStack) -> Self { - Self::RunnerError { - runner_error: RunnerError::OpenSSL { - errors: value.to_string(), - }, - } - } -} - impl Display for ManagerError { fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), FmtError> { match self { diff --git a/crates/pipeline-manager/src/runner/interaction.rs b/crates/pipeline-manager/src/runner/interaction.rs index c37b541c2bf..f5a98ac334d 100644 --- a/crates/pipeline-manager/src/runner/interaction.rs +++ b/crates/pipeline-manager/src/runner/interaction.rs @@ -12,7 +12,6 @@ use actix_ws::{CloseCode, CloseReason}; use awc::error::{ConnectError, SendRequestError}; use awc::{ClientRequest, ClientResponse}; use crossbeam::sync::ShardedLock; -use feldera_observability::AwcRequestTracingExt; use feldera_types::query::{MAX_WS_FRAME_SIZE, WS_SUBPROTOCOL}; use std::fmt::Display; use std::{collections::HashMap, sync::Arc, time::Duration}; @@ -274,11 +273,7 @@ impl RunnerInteraction { query_string, ); let timeout = timeout.unwrap_or(Self::PIPELINE_HTTP_REQUEST_TIMEOUT); - let request = client - .request(method, &url) - .timeout(timeout) - .force_close() - .with_sentry_tracing(); + let request = client.request(method, &url).timeout(timeout).force_close(); let request_str = Self::format_request(&request); let mut original_response = request.send().await.map_err(|e| match e { @@ -586,7 +581,6 @@ impl RunnerInteraction { let response = client .request(Method::GET, &url) .timeout(Self::RUNNER_HTTP_REQUEST_TIMEOUT) - .with_sentry_tracing() .send() .await .map_err(|e| match e { @@ -674,7 +668,7 @@ pub(crate) async fn streaming_proxy( new_request = new_request.append_header(header); } - let new_request = new_request.with_sentry_tracing(); + let new_request = new_request; let request_str = RunnerInteraction::format_request(&new_request); // Perform request to the pipeline diff --git a/crates/pipeline-manager/src/runner/local_runner.rs b/crates/pipeline-manager/src/runner/local_runner.rs index af7c58d5742..2cede4190a1 100644 --- a/crates/pipeline-manager/src/runner/local_runner.rs +++ b/crates/pipeline-manager/src/runner/local_runner.rs @@ -13,7 +13,6 @@ use crate::runner::error::RunnerError; use crate::runner::pipeline_executor::{PipelineExecutor, ProvisionStatus}; use crate::runner::pipeline_logs::{LogMessage, LogsSender}; use async_trait::async_trait; -use feldera_observability::ReqwestTracingExt; use feldera_observability::system::total_memory_megabyte; use feldera_types::config::{ PipelineConfig, PipelineConfigProgramInfo, RuntimeConfig, StorageCacheConfig, StorageConfig, @@ -489,7 +488,7 @@ impl LocalRunner { // Perform request let mut attempt = 1; loop { - match self.client.get(file_url).with_sentry_tracing().send().await { + match self.client.get(file_url).send().await { Ok(response) => { // Check status code if response.status() != StatusCode::OK { diff --git a/crates/pipeline-manager/src/runner/pipeline_automata.rs b/crates/pipeline-manager/src/runner/pipeline_automata.rs index f72106b8dd0..83b6835e183 100644 --- a/crates/pipeline-manager/src/runner/pipeline_automata.rs +++ b/crates/pipeline-manager/src/runner/pipeline_automata.rs @@ -20,7 +20,6 @@ use crate::runner::interaction::{format_pipeline_url, format_timeout_error_messa use crate::runner::pipeline_executor::{PipelineExecutor, ProvisionStatus}; use crate::runner::pipeline_logs::{LogMessage, LogsSender, start_thread_pipeline_logs}; use chrono::Utc; -use feldera_observability::ReqwestTracingExt; use feldera_types::error::ErrorResponse; use feldera_types::runtime_status::{ ExtendedRuntimeStatus, RuntimeDesiredStatus, RuntimeStatus, RuntimeStatusDetails, @@ -815,7 +814,6 @@ impl PipelineAutomaton { .client .request(method, &url) .timeout(timeout) - .with_sentry_tracing() .send() .await .map_err(|e| { @@ -1068,7 +1066,6 @@ impl PipelineAutomaton { .client .get(&binary_check_url) .timeout(Self::PIPELINE_STATUS_HTTP_REQUEST_TIMEOUT) - .with_sentry_tracing() .send() .await { diff --git a/crates/rest-api/Cargo.toml b/crates/rest-api/Cargo.toml index 16f656f5428..c030c2e5897 100644 --- a/crates/rest-api/Cargo.toml +++ b/crates/rest-api/Cargo.toml @@ -22,8 +22,6 @@ reqwest = { workspace = true, features = ["json", "stream"] } feldera-types = { workspace = true } progenitor-client = { workspace = true } rustversion.workspace = true -sentry = { workspace = true } -feldera-observability = { workspace = true } [build-dependencies] prettyplease = { workspace = true } diff --git a/crates/rest-api/build.rs b/crates/rest-api/build.rs index a2fe10ce9f4..10744bd3799 100644 --- a/crates/rest-api/build.rs +++ b/crates/rest-api/build.rs @@ -239,16 +239,7 @@ fn main() { let tokens = generator.generate_tokens(&spec).unwrap(); let ast = syn::parse2(tokens).unwrap(); - let mut content = prettyplease::unparse(&ast); - for verb in ["get", "post", "put", "patch", "delete"] { - let pattern = format!(".{verb}(url)"); - let replacement = format!(".{verb}(url)\n .with_sentry_tracing()"); - content = content.replace(&pattern, &replacement); - } - content = content.replace( - "pub mod builder {", - "pub mod builder {\n use feldera_observability::ReqwestTracingExt;", - ); + let content = prettyplease::unparse(&ast); let content = content.replace( "impl Client", "#[rustversion::attr(since(1.89), allow(mismatched_lifetime_syntaxes))]\nimpl Client", diff --git a/crates/rest-api/src/lib.rs b/crates/rest-api/src/lib.rs index 37f471037de..8e96d844222 100644 --- a/crates/rest-api/src/lib.rs +++ b/crates/rest-api/src/lib.rs @@ -1,7 +1,5 @@ #![allow(clippy::all, unused)] -use feldera_observability::ReqwestTracingExt; - include!(concat!(env!("OUT_DIR"), "/codegen.rs")); #[cfg(test)] diff --git a/crates/storage/src/error.rs b/crates/storage/src/error.rs index b4e3dab6070..36a9f21b3e3 100644 --- a/crates/storage/src/error.rs +++ b/crates/storage/src/error.rs @@ -84,7 +84,7 @@ impl From for StorageError { ObjectStoreError::NotFound { .. } => ErrorKind::NotFound, ObjectStoreError::NotSupported { .. } => ErrorKind::Unsupported, ObjectStoreError::AlreadyExists { .. } => ErrorKind::AlreadyExists, - ObjectStoreError::NotImplemented => ErrorKind::Unsupported, + ObjectStoreError::NotImplemented { .. } => ErrorKind::Unsupported, ObjectStoreError::PermissionDenied { .. } | ObjectStoreError::Unauthenticated { .. } => ErrorKind::PermissionDenied, ObjectStoreError::InvalidPath { .. } => { diff --git a/crates/storage/src/lib.rs b/crates/storage/src/lib.rs index f06aa16e496..93f9d5d87ad 100644 --- a/crates/storage/src/lib.rs +++ b/crates/storage/src/lib.rs @@ -229,7 +229,7 @@ impl dyn StorageBackend { // commit always writes it, and it captures the full list in one // place so a single read suffices. See `CheckpointDependencies` // for the accepted JSON forms. - let deps_path = checkpoint_dir.child(CHECKPOINT_DEPENDENCIES); + let deps_path = checkpoint_dir.clone().join(CHECKPOINT_DEPENDENCIES); match self.read_json::(&deps_path) { Ok(deps) => { return Ok(deps diff --git a/deploy/Dockerfile b/deploy/Dockerfile index d461f1e07e1..b486384ecdd 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -1,20 +1,34 @@ +# librdkafka, built against AWS-LC rather than the system OpenSSL. +# +# This is a separate stage because building AWS-LC needs Go and cmake, and +# neither belongs in the shipped image. Only the installed library crosses over. +FROM ubuntu:24.04 AS librdkafka +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update --fix-missing && apt-get install -y \ + git cmake build-essential pkg-config golang-go \ + libsasl2-dev zlib1g-dev libzstd-dev +COPY scripts/install-librdkafka.sh /tmp/install-librdkafka.sh +# The script reads the librdkafka version from the lockfile, so the library and +# the crate expecting it cannot drift apart. A stale image fails loudly at the +# pkg-config probe rather than building against the wrong version. +COPY Cargo.lock /tmp/Cargo.lock +RUN CARGO_LOCK=/tmp/Cargo.lock PREFIX=/usr/local bash /tmp/install-librdkafka.sh + # The base image contains tools to build the code given that # we need a Java and Rust compiler to run alongside the pipeline manager # as of now. This will change later. FROM ubuntu:24.04 AS base ENV DEBIAN_FRONTEND=noninteractive -# These two environment variables are used to make openssl-sys pick -# up libssl-dev and statically link it. Without it, our build defaults -# to building a vendored version of OpenSSL. RUN apt update --fix-missing && apt -y dist-upgrade && apt install \ # bindgen needs this (at least the dec crate uses bindgen) libclang-dev \ - # pkg-config is required for cargo to find libssl - libssl-dev pkg-config \ + # pkg-config is how rdkafka-sys locates the librdkafka copied in below + pkg-config \ cmake \ - # rdkafka dependency needs libsasl2-dev zlib and a CXX compiler - libsasl2-dev zlib1g-dev build-essential \ + # librdkafka links these; it is built in the stage above, but pipelines are + # compiled in this image and the linker resolves them here + libsasl2-dev zlib1g-dev libzstd-dev build-essential \ # To install rust curl \ # For running the SQL compiler @@ -39,11 +53,25 @@ RUN update-ca-certificates RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ locale-gen +# librdkafka from the stage above. rdkafka-sys links it dynamically, so it has +# to be here both to build pipelines and to run them. +COPY --from=librdkafka /usr/local/lib/librdkafka* /usr/local/lib/ +COPY --from=librdkafka /usr/local/include/librdkafka /usr/local/include/librdkafka +COPY --from=librdkafka /usr/local/lib/pkgconfig/rdkafka*.pc /usr/local/lib/pkgconfig/ +RUN ldconfig \ + # A NEEDED entry for libssl or libcrypto means librdkafka found a system + # OpenSSL and preferred it over AWS-LC, which is the failure this whole + # arrangement exists to prevent and is invisible at run time. + && if readelf -d /usr/local/lib/librdkafka.so.1 | grep -qE 'Shared library: \[lib(ssl|crypto)\.so'; then \ + echo "error: librdkafka links a shared OpenSSL" >&2; exit 1; \ + fi \ + # rdkafka-sys probes for librdkafka this way; failing here beats failing + # partway through a pipeline compilation. + && pkg-config --exists rdkafka + USER ubuntu WORKDIR /home/ubuntu -ENV OPENSSL_NO_VENDOR=1 -ENV OPENSSL_STATIC=1 ENV LC_ALL=en_US.UTF-8 ENV LANG=en_US.UTF-8 ENV LANGUAGE=en_US:en diff --git a/deploy/build.Dockerfile b/deploy/build.Dockerfile index 33f9ad4f4a1..2c115e05b76 100644 --- a/deploy/build.Dockerfile +++ b/deploy/build.Dockerfile @@ -9,19 +9,15 @@ FROM ubuntu:24.04 AS ubuntu-base ENV DEBIAN_FRONTEND=noninteractive -# These two environment variables are used to make openssl-sys pick -# up libssl-dev and statically link it. Without it, our build defaults -# to building a vendored version of OpenSSL. FROM ubuntu-base AS install-pkgs -ENV OPENSSL_NO_VENDOR=1 -ENV OPENSSL_STATIC=1 RUN apt-get update --fix-missing && apt-get install -y \ - # pkg-config is required for cargo to find libssl - libssl-dev pkg-config \ + # pkg-config is how rdkafka-sys locates librdkafka + pkg-config \ cmake \ - # Go is required to build aws-lc-fips-sys when rustls is built with FIPS + # Go builds AWS-LC, which librdkafka is linked against; see + # scripts/install-librdkafka.sh golang-go \ - # rdkafka dependency needs libsasl2-dev and a CXX compiler + # librdkafka links these libsasl2-dev libzstd-dev zlib1g-dev build-essential \ # zstd CLI: @actions/cache (runs-on/cache) auto-uses it for cache # compression when on PATH, else falls back to slow gzip. Big win for the @@ -163,3 +159,17 @@ RUN arch=`dpkg --print-architecture | sed "s/arm64/aarch64/g" | sed "s/amd64/x8 ENV RUSTFLAGS="-C link-arg=-fuse-ld=mold -C link-arg=-Wl,--compress-debug-sections=zlib" +# librdkafka, built against AWS-LC rather than the system OpenSSL. rdkafka is +# built with `dynamic-linking`, so it consumes this rather than compiling its +# own copy. +# +# Last in the file on purpose: it copies Cargo.lock, from which the script +# reads the librdkafka version the crate expects, and that file changes often. +# Keeping it here means a lockfile change rebuilds only this layer. +USER root +COPY scripts/install-librdkafka.sh /tmp/install-librdkafka.sh +COPY Cargo.lock /tmp/Cargo.lock +RUN CARGO_LOCK=/tmp/Cargo.lock PREFIX=/usr/local bash /tmp/install-librdkafka.sh \ + && rm -f /tmp/install-librdkafka.sh /tmp/Cargo.lock +USER ubuntu + diff --git a/scripts/install-librdkafka.sh b/scripts/install-librdkafka.sh new file mode 100755 index 00000000000..9a4ef09811d --- /dev/null +++ b/scripts/install-librdkafka.sh @@ -0,0 +1,177 @@ +#!/usr/bin/env bash +# Builds librdkafka against AWS-LC and installs both under $PREFIX. +# +# rdkafka-sys otherwise compiles a vendored librdkafka against whatever OpenSSL +# pkg-config finds, which puts Kafka TLS on a second cryptographic +# implementation. Building it here instead lets us point it at AWS-LC, and the +# `dynamic-linking` feature then makes rdkafka-sys consume this install rather +# than compile its own copy. +# +# The container images and CI run this script, and so should developers: the +# `dynamic-linking` feature is unconditional, so a build finds librdkafka here +# or not at all. Ubuntu's package is far older than the version rdkafka-sys +# requires, which is why this builds from source rather than calling apt. +# +# The librdkafka version is derived from the lockfile, so it cannot drift from +# what the crate expects: rdkafka-sys names its vendored version in its own, +# as `4.10.0+2.12.1`. +set -euo pipefail + +PREFIX="${PREFIX:-/usr/local}" +# Keep in step with the aws-lc-sys version in Cargo.lock: aws-lc-sys 0.44.0 +# vendors AWS-LC 5.5.0 (its include/openssl/base.h names the release), so the +# librdkafka in the process carries the same AWS-LC as the Rust side. +AWS_LC_REF="${AWS_LC_REF:-v5.5.0}" +# Switching to the FIPS-validated module means building AWS-LC with -DFIPS=1 +# and moving aws-lc-rs to aws-lc-fips-sys in the same change. Doing one without +# the other leaves two AWS-LC copies in the binary; validate-crypto-deps.sh +# fails when that happens. +AWS_LC_FIPS="${AWS_LC_FIPS:-0}" + +script_path="${BASH_SOURCE[0]:-$0}" +repo_root=$(cd "$(dirname "$script_path")/.." 2>/dev/null && pwd) || repo_root="." +lockfile="${CARGO_LOCK:-$repo_root/Cargo.lock}" + +if [ -n "${LIBRDKAFKA_REF:-}" ]; then + librdkafka_version="${LIBRDKAFKA_REF#v}" +elif [ -f "$lockfile" ]; then + # rdkafka-sys version is `+`. + librdkafka_version=$(grep -A1 'name = "rdkafka-sys"' "$lockfile" | + grep '^version' | sed 's/.*+//; s/"//') +else + echo "error: cannot determine the librdkafka version." >&2 + echo "Point CARGO_LOCK at a lockfile, or set LIBRDKAFKA_REF explicitly." >&2 + exit 1 +fi + +if [ -z "$librdkafka_version" ]; then + echo "error: no rdkafka-sys entry in $lockfile." >&2 + exit 1 +fi + +echo "librdkafka v$librdkafka_version, AWS-LC $AWS_LC_REF, prefix $PREFIX" + +for tool in cmake git awk go make cc; do + command -v "$tool" >/dev/null || { echo "error: $tool is required." >&2; exit 1; } +done + +jobs="${JOBS:-$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 4)}" +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT + +# AWS-LC. BUILD_LIBSSL is off by default and librdkafka links libssl, so the +# default build is not enough. The symbol prefix stays: aws-lc-rs links its own +# copy into the same binary, and the prefix is what keeps the two apart. +git clone --depth 1 --branch "$AWS_LC_REF" https://github.com/aws/aws-lc.git "$work/aws-lc" -q +cmake -S "$work/aws-lc" -B "$work/aws-lc-build" \ + -DCMAKE_BUILD_TYPE=Release \ + "-DCMAKE_C_FLAGS=-fPIC -w" \ + -DCMAKE_INSTALL_PREFIX="$work/aws-lc-install" \ + -DBUILD_LIBSSL=ON \ + -DBUILD_SHARED_LIBS=OFF \ + -DBUILD_TESTING=OFF \ + -DBUILD_TOOL=OFF \ + -DFIPS="$AWS_LC_FIPS" >"$work/aws-lc-configure.log" 2>&1 +if ! cmake --build "$work/aws-lc-build" --parallel "$jobs" >"$work/aws-lc-build.log" 2>&1; then + echo "error: AWS-LC build failed." >&2 + tail -40 "$work/aws-lc-build.log" >&2 + exit 1 +fi +cmake --install "$work/aws-lc-build" >"$work/aws-lc-install.log" 2>&1 + +aws_lc_lib=$(dirname "$(find "$work/aws-lc-install" -name libcrypto.a -print -quit)") +aws_lc_include="$work/aws-lc-install/include" +for lib in libcrypto.a libssl.a; do + [ -f "$aws_lc_lib/$lib" ] || { echo "error: AWS-LC did not produce $lib." >&2; exit 1; } +done + +# librdkafka. +git clone --depth 1 --branch "v$librdkafka_version" \ + https://github.com/confluentinc/librdkafka.git "$work/librdkafka" -q +cd "$work/librdkafka" + +# rdkafka_ssl.c calls HMAC() without including . OpenSSL +# supplies the declaration transitively through x509.h and AWS-LC does not, so +# the call compiles as an implicit declaration returning int, truncating the +# returned pointer. Fixed upstream in confluentinc/librdkafka#5552, unmerged; +# drop this once a release carries it. +if ! grep -q "openssl/hmac.h" src/rdkafka_ssl.c; then + awk '/^#include $/ && !inserted { + print "#include "; inserted = 1 + } + { print }' src/rdkafka_ssl.c > src/rdkafka_ssl.c.patched + mv src/rdkafka_ssl.c.patched src/rdkafka_ssl.c + grep -q "openssl/hmac.h" src/rdkafka_ssl.c || { + echo "error: could not apply the hmac.h patch; check whether upstream restructured the includes." >&2 + exit 1 + } +fi + +# These flags mirror the cargo features rdkafka-sys used to build with, so the +# connectors keep the same capabilities. Losing one here is silent: a codec or +# an authentication mechanism simply stops being offered. +CPPFLAGS="-I$aws_lc_include" \ +CFLAGS="-Werror=implicit-function-declaration" \ +LDFLAGS="-L$aws_lc_lib" \ +PKG_CONFIG_PATH="$aws_lc_lib/pkgconfig${PKG_CONFIG_PATH:+:$PKG_CONFIG_PATH}" \ +./configure --prefix="$PREFIX" \ + --enable-ssl \ + --enable-gssapi \ + --enable-zlib \ + --enable-zstd \ + --disable-curl \ + --disable-lz4-ext >"$work/configure.log" 2>&1 || { + echo "error: librdkafka configure failed." >&2 + tail -30 "$work/configure.log" >&2 + exit 1 + } + +for want in WITH_SSL WITH_SASL_CYRUS WITH_ZLIB WITH_ZSTD; do + grep -q "^${want}=[[:space:]]*y$" Makefile.config || { + echo "error: librdkafka configure did not enable $want." >&2 + echo "Its development package is probably missing; see Makefile.config." >&2 + exit 1 + } +done + +if ! make -j"$jobs" libs >"$work/build.log" 2>&1; then + echo "error: librdkafka build failed." >&2 + tail -40 "$work/build.log" >&2 + exit 1 +fi +if ! make install >"$work/install.log" 2>&1; then + echo "error: librdkafka install to $PREFIX failed." >&2 + tail -20 "$work/install.log" >&2 + exit 1 +fi + +# librdkafka must carry AWS-LC inside rather than link a system OpenSSL. A +# NEEDED entry for libssl or libcrypto means configure found one and preferred +# it, which silently reintroduces the implementation this script exists to +# avoid. +shared="" +for candidate in "$PREFIX/lib/librdkafka.so.1" "$PREFIX/lib/librdkafka.1.dylib"; do + if [ -f "$candidate" ]; then + shared="$candidate" + break + fi +done +if [ -n "$shared" ]; then + if command -v readelf >/dev/null; then + if readelf -d "$shared" | grep -qE 'Shared library: \[lib(ssl|crypto)\.so'; then + echo "error: librdkafka links a shared OpenSSL:" >&2 + readelf -d "$shared" | grep -E 'Shared library: \[lib(ssl|crypto)' >&2 + exit 1 + fi + elif command -v otool >/dev/null; then + if otool -L "$shared" | grep -qE 'lib(ssl|crypto)[.0-9]*\.dylib'; then + echo "error: librdkafka links a shared OpenSSL:" >&2 + otool -L "$shared" | grep -E 'lib(ssl|crypto)' >&2 + exit 1 + fi + fi +fi + +command -v ldconfig >/dev/null && ldconfig 2>/dev/null || true + +echo "installed librdkafka $librdkafka_version to $PREFIX" diff --git a/scripts/validate-crypto-deps.sh b/scripts/validate-crypto-deps.sh new file mode 100755 index 00000000000..a9f9cc0ce96 --- /dev/null +++ b/scripts/validate-crypto-deps.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +# Fails if a second TLS or cryptography implementation is in the dependency tree. +# +# Feldera routes cryptography through AWS-LC so that a single validated module +# serves the whole binary. A second implementation is a correctness problem, not +# only a compliance one: AWS-LC keeps OpenSSL's symbol names, so a binary +# containing both resolves each name to whichever archive the linker reaches +# first. Memory allocated by one library then gets freed by the other, which +# corrupts the heap and surfaces far from the cause. +# +# Blocked: +# openssl the crate that links a second implementation directly +# openssl-sys the FFI layer beneath it. Kafka used to require it, because +# rdkafka's `ssl` feature depends on it and compiled a vendored +# librdkafka against whatever OpenSSL pkg-config found. rdkafka +# now uses `dynamic-linking` against the librdkafka that +# scripts/install-librdkafka.sh builds against AWS-LC, so +# nothing needs this crate. +# native-tls how openssl usually arrives; on macOS and Windows it binds +# Security.framework and schannel instead, so checking for +# openssl alone would miss a second TLS stack on those platforms +# boring BoringSSL wrapper, same symbol-collision class +# +# Allowed: +# aws-lc-sys AWS-LC itself, symbol-prefixed by aws-lc-fips-sys/aws-lc-sys +# +# `ring` should also go: it is a second cryptographic implementation and is not +# FIPS-validated. The switch is not finished, so it cannot be blocked outright +# yet; what remains reaches it through crates pinned to object_store 0.13, which +# have to move together. Until then this ratchets, failing when a crate outside +# the known set starts pulling ring, so the remainder shrinks and never grows. +# +# This only resolves the lockfile, so it compiles nothing. +set -euo pipefail + +# `(*)` marks a subtree cargo already printed; strip it so entries dedupe. +packages=$(cargo tree --target all --edges normal,build,dev --prefix none --format "{p}" 2>/dev/null | + sed 's/ (\*)$//' | sort -u) + +blocked=$(echo "$packages" | grep -E "^(openssl|openssl-sys|native-tls|boring|boring-sys) v" || true) +if [ -n "$blocked" ]; then + echo "error: a second TLS or cryptography implementation is in the dependency tree:" + echo "$blocked" | sed 's/^/ /' + echo + echo "It is almost always pulled in by a dependency defaulting to native-tls." + echo "Find who enables it with, for example:" + echo + echo " cargo tree -e features -i native-tls --target all" + echo + echo "then set default-features = false on that dependency and select its rustls" + echo "feature instead. Note that a workspace member cannot override" + echo "default-features on an inherited dependency; change it at the workspace root." + echo + echo 'openssl-sys specifically arrives through rdkafka. Kafka does not need it:' + echo 'keep rdkafka on "dynamic-linking" rather than "ssl", and build librdkafka' + echo 'with scripts/install-librdkafka.sh.' + exit 1 +fi + +# One AWS-LC, not two. aws-lc-sys and aws-lc-fips-sys are separate crates, so +# cargo cannot unify them; a tree holding both compiles and links the whole +# library twice. Adopting the FIPS module means moving aws-lc-rs to the fips +# feature and building AWS-LC with -DFIPS=1 in scripts/install-librdkafka.sh at +# the same time, so that librdkafka and the Rust side agree. +backends=$(echo "$packages" | grep -E "^aws-lc(-fips)?-sys v" | awk '{print $1}' | sort -u) +if [ "$(echo "$backends" | grep -c .)" -gt 1 ]; then + echo "error: two AWS-LC implementations are in the dependency tree:" + echo "$backends" | sed 's/^/ /' + echo + echo "Both are compiled and linked, so the binary carries the library twice." + echo "Their symbol prefixes differ, which is why nothing collides and the" + echo "waste stays invisible at run time. Make both ends agree:" + echo + echo " cargo tree -i aws-lc-sys --target all" + echo " cargo tree -i aws-lc-fips-sys --target all" + exit 1 +fi + +# Crates still known to pull ring. Shrink this list; do not extend it. +RING_ALLOWED="object_store parquet rustls rustls-webpki" + +# `--depth 1` on the inverted tree lists ring plus exactly its direct parents. +ring_parents=$(cargo tree --invert ring --depth 1 --target all --edges normal,build,dev \ + --prefix none --format "{p}" 2>/dev/null | + sed 's/ (\*)$//' | awk 'NF {print $1}' | grep -v "^ring$" | sort -u) + +unexpected="" +for parent in $ring_parents; do + case " $RING_ALLOWED " in + *" $parent "*) ;; + *) unexpected="$unexpected $parent" ;; + esac +done + +if [ -n "$unexpected" ]; then + echo "error: a new dependency pulls in ring:" + for parent in $unexpected; do echo " $parent"; done + echo + echo "ring is a second cryptographic implementation and is not FIPS-validated." + echo "Prefer the aws-lc-rs backend where a dependency offers the choice, usually" + echo "a feature named aws-lc-rs or a -no-provider variant that defers to the" + echo "process default. Check whether a newer release drops ring outright before" + echo "assuming a fork is needed." + echo + echo " cargo tree -e features -i ring --target all" + exit 1 +fi + +if [ -n "$ring_parents" ]; then + echo "note: ring is still reached through:" $ring_parents + echo "It is not FIPS-validated and should go; these are pinned to object_store" + echo "0.13 and have to move together." +fi