From 8905bf817391d1bad422ebe69e72cefa51825849 Mon Sep 17 00:00:00 2001 From: Tom Parker-Shemilt Date: Thu, 9 Apr 2026 15:54:51 +0100 Subject: [PATCH 1/9] Build MacOS lre-rs image --- .github/workflows/image.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/image.yaml b/.github/workflows/image.yaml index 4d68cd746..f56b87aaa 100644 --- a/.github/workflows/image.yaml +++ b/.github/workflows/image.yaml @@ -23,8 +23,12 @@ jobs: fail-fast: false matrix: image: [image, nativelink-worker-init, nativelink-worker-lre-cc, nativelink-worker-lre-rs] - name: Publish ${{ matrix.image }} - runs-on: ubuntu-24.04 + os: [ubuntu-24.04] + include: + - os: macos-26 + image: nativelink-worker-lre-rs + name: Publish ${{ matrix.image }} / ${{ matrix.os }} + runs-on: ${{ matrix.os }} permissions: packages: write id-token: write From 2535de14a04c3584eb22ff224999e086f79c0d27 Mon Sep 17 00:00:00 2001 From: Tom Parker-Shemilt Date: Thu, 4 Jun 2026 13:02:37 +0100 Subject: [PATCH 2/9] Inline skopeo copy --- tools/public/default.nix | 4 +++- tools/public/local-image-test.nix | 13 ++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/tools/public/default.nix b/tools/public/default.nix index 35886cc36..674551644 100644 --- a/tools/public/default.nix +++ b/tools/public/default.nix @@ -3,7 +3,9 @@ # Note: Only put tools here that should be usable from external flakes. nativelink-tools = { - local-image-test = final.callPackage ./local-image-test.nix {}; + local-image-test = final.callPackage ./local-image-test.nix { + skopeo = nix2container.packages.${final.stdenv.hostPlatform.system}.skopeo-nix2container; + }; publish-ghcr = final.callPackage ./publish-ghcr.nix {}; create-local-image = final.callPackage ./create-local-image.nix {}; diff --git a/tools/public/local-image-test.nix b/tools/public/local-image-test.nix index 6899d5030..a3d84ee88 100644 --- a/tools/public/local-image-test.nix +++ b/tools/public/local-image-test.nix @@ -2,11 +2,13 @@ dive, trivy, writeShellScriptBin, + skopeo, }: writeShellScriptBin "local-image-test" '' - set -xeuo pipefail + set -euo pipefail echo "Testing image: $1" + set -x # Commit hashes would not be a good choice here as they are not # fully dependent on the inputs to the image. For instance, amending @@ -15,9 +17,14 @@ writeShellScriptBin "local-image-test" '' # didn't change. IMAGE_TAG=$(nix eval .#$1.imageTag --raw) IMAGE_NAME=$(nix eval .#$1.imageName --raw) + IMAGE=$(nix eval .#$1 --raw) - nix run .#$1.copyTo \ - docker-daemon:''${IMAGE_NAME}:''${IMAGE_TAG} + # Inlining from https://github.com/nlewo/nix2container/blob/76be9608a7f4d6c985d28b0e7be903ae2547df3e/default.nix#L88 + # so we can run --debug on skopeo + # nix run .#$1.copyTo \ + # docker-daemon:''${IMAGE_NAME}:''${IMAGE_TAG} + nix build .#$1 + ${skopeo}/bin/skopeo --debug --insecure-policy copy nix:''${IMAGE} docker-daemon:''${IMAGE_NAME}:''${IMAGE_TAG} # Ensure that the image has minimal closure size. # TODO(palfrey): The default allows 10% inefficiency. Since we control all From 10366e6c17cd9c9b882145034dfe6c9dd928547b Mon Sep 17 00:00:00 2001 From: Tom Parker-Shemilt Date: Thu, 4 Jun 2026 15:24:01 +0100 Subject: [PATCH 3/9] Check if we're running out of disk before running skopeo --- tools/public/local-image-test.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/public/local-image-test.nix b/tools/public/local-image-test.nix index a3d84ee88..527354a53 100644 --- a/tools/public/local-image-test.nix +++ b/tools/public/local-image-test.nix @@ -24,6 +24,8 @@ writeShellScriptBin "local-image-test" '' # nix run .#$1.copyTo \ # docker-daemon:''${IMAGE_NAME}:''${IMAGE_TAG} nix build .#$1 + # Double-check available disk before we run skopeo + df -h ${skopeo}/bin/skopeo --debug --insecure-policy copy nix:''${IMAGE} docker-daemon:''${IMAGE_NAME}:''${IMAGE_TAG} # Ensure that the image has minimal closure size. From 3f2835cb1eab9ad333a2dddc8603ba369765efce Mon Sep 17 00:00:00 2001 From: Tom Parker-Shemilt Date: Thu, 4 Jun 2026 15:43:25 +0100 Subject: [PATCH 4/9] Try docker setup to fix lre-rs build --- .github/workflows/image.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/image.yaml b/.github/workflows/image.yaml index f56b87aaa..8f99e8aac 100644 --- a/.github/workflows/image.yaml +++ b/.github/workflows/image.yaml @@ -44,6 +44,10 @@ jobs: with: nativelink_attic_token: ${{ secrets.NATIVELINK_ATTIC_TOKEN }} + - name: Set up Docker + uses: >- # v5.2.0 + docker/setup-docker-action@0234bb73ccb40f0c430b795634f9247e2b5c2d23 + - name: Test image run: | nix run --fallback .#local-image-test ${{ matrix.image }} From 67bb2bdb993212e5fdba768251d75b49cd9f16c9 Mon Sep 17 00:00:00 2001 From: Tom Parker-Shemilt Date: Thu, 4 Jun 2026 16:11:23 +0100 Subject: [PATCH 5/9] Redo Docker setup just for macs --- .github/workflows/image.yaml | 15 +++++++++++++++ typos.toml | 4 +++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/.github/workflows/image.yaml b/.github/workflows/image.yaml index 8f99e8aac..3b263e6fa 100644 --- a/.github/workflows/image.yaml +++ b/.github/workflows/image.yaml @@ -47,6 +47,21 @@ jobs: - name: Set up Docker uses: >- # v5.2.0 docker/setup-docker-action@0234bb73ccb40f0c430b795634f9247e2b5c2d23 + if: runner.os == 'macOS' # already setup for Linux + env: + # m1 chips, as seen in the GitHub CI, don't support nested hardware virtualization. + # + # We need `-machine virt` to stop Lima from providing `accel=hvf`due to this + # [QEMU issue](https://gitlab.com/qemu-project/qemu/-/issues/2981). + # + # We also need `-cpu *` to stop Lima from providing `-cpu host` as: + # 1) `-cpu host` requires `kvm` or `hvf` (which we don't have) + # 2) `-cpu host` implies `host-phys-bits=on` which is borked in this environment + # From which CPU to actually emulate, we choose what will be the most performant. + # + # We accomplish both of these goals with the methodology from + # [this comment](https://github.com/lima-vm/lima/pull/3173#issuecomment-2623130310). + QEMU_SYSTEM_AARCH64: "qemu-system-aarch64 -machine virt -cpu max,pmu=off,sve=on,sve128=on,sme=off,pauth=off" - name: Test image run: | diff --git a/typos.toml b/typos.toml index 356f7e4ba..fab7c522e 100644 --- a/typos.toml +++ b/typos.toml @@ -3,10 +3,12 @@ conly = "conly" [default] -# Old wrong spelling support extend-ignore-re = [ + # Old wrong spelling support "alias = \"Cachable\"", "old_unique_qualifier_cachable_works", + # Argument in .github/workflows/image.yaml + "sme=off", ] [files] From dfc78b4a093fdfb43190fb1e9fe279e83a05e518 Mon Sep 17 00:00:00 2001 From: Tom Parker-Shemilt Date: Wed, 17 Jun 2026 13:33:56 +0100 Subject: [PATCH 6/9] Set docker host for MacOS --- .github/workflows/image.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/image.yaml b/.github/workflows/image.yaml index 3b263e6fa..fd09d4fed 100644 --- a/.github/workflows/image.yaml +++ b/.github/workflows/image.yaml @@ -45,6 +45,7 @@ jobs: nativelink_attic_token: ${{ secrets.NATIVELINK_ATTIC_TOKEN }} - name: Set up Docker + id: docker uses: >- # v5.2.0 docker/setup-docker-action@0234bb73ccb40f0c430b795634f9247e2b5c2d23 if: runner.os == 'macOS' # already setup for Linux @@ -63,9 +64,14 @@ jobs: # [this comment](https://github.com/lima-vm/lima/pull/3173#issuecomment-2623130310). QEMU_SYSTEM_AARCH64: "qemu-system-aarch64 -machine virt -cpu max,pmu=off,sve=on,sve128=on,sme=off,pauth=off" + - name: Set docker host + if: runner.os == 'macOS' # already setup for Linux + run: | + echo "DOCKER_HOST=${{steps.docker.outputs.sock}}" >> "$GITHUB_ENV" + - name: Test image run: | - nix run --fallback .#local-image-test ${{ matrix.image }} + nix run --fallback --impure .#local-image-test ${{ matrix.image }} - name: Upload image run: | From 7b19fa69f21d2ffa595325bdf70afa85c06fd8cb Mon Sep 17 00:00:00 2001 From: Tom Parker-Shemilt Date: Wed, 17 Jun 2026 14:03:33 +0100 Subject: [PATCH 7/9] Add dest-daemon-host to skopeo --- tools/public/local-image-test.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/public/local-image-test.nix b/tools/public/local-image-test.nix index 527354a53..a4b0fde45 100644 --- a/tools/public/local-image-test.nix +++ b/tools/public/local-image-test.nix @@ -26,7 +26,7 @@ writeShellScriptBin "local-image-test" '' nix build .#$1 # Double-check available disk before we run skopeo df -h - ${skopeo}/bin/skopeo --debug --insecure-policy copy nix:''${IMAGE} docker-daemon:''${IMAGE_NAME}:''${IMAGE_TAG} + ${skopeo}/bin/skopeo --debug --insecure-policy copy --dest-daemon-host=''${DOCKER_HOST:-unix:///var/run/docker.sock} nix:''${IMAGE} docker-daemon:''${IMAGE_NAME}:''${IMAGE_TAG} # Ensure that the image has minimal closure size. # TODO(palfrey): The default allows 10% inefficiency. Since we control all From 0391f910040f736a5ae07f610a04bc833eb446f9 Mon Sep 17 00:00:00 2001 From: Tom Parker-Shemilt Date: Wed, 17 Jun 2026 15:00:11 +0100 Subject: [PATCH 8/9] Increase image timeout to 90m --- .github/workflows/image.yaml | 2 +- tools/public/local-image-test.nix | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/image.yaml b/.github/workflows/image.yaml index fd09d4fed..d282acb16 100644 --- a/.github/workflows/image.yaml +++ b/.github/workflows/image.yaml @@ -33,7 +33,7 @@ jobs: packages: write id-token: write security-events: write - timeout-minutes: 30 + timeout-minutes: 90 steps: - name: Checkout uses: >- # v6.0.2 diff --git a/tools/public/local-image-test.nix b/tools/public/local-image-test.nix index a4b0fde45..498213607 100644 --- a/tools/public/local-image-test.nix +++ b/tools/public/local-image-test.nix @@ -20,13 +20,13 @@ writeShellScriptBin "local-image-test" '' IMAGE=$(nix eval .#$1 --raw) # Inlining from https://github.com/nlewo/nix2container/blob/76be9608a7f4d6c985d28b0e7be903ae2547df3e/default.nix#L88 - # so we can run --debug on skopeo + # so we can add parameters to skopeo # nix run .#$1.copyTo \ # docker-daemon:''${IMAGE_NAME}:''${IMAGE_TAG} nix build .#$1 # Double-check available disk before we run skopeo df -h - ${skopeo}/bin/skopeo --debug --insecure-policy copy --dest-daemon-host=''${DOCKER_HOST:-unix:///var/run/docker.sock} nix:''${IMAGE} docker-daemon:''${IMAGE_NAME}:''${IMAGE_TAG} + ${skopeo}/bin/skopeo --insecure-policy copy --dest-daemon-host=''${DOCKER_HOST:-unix:///var/run/docker.sock} nix:''${IMAGE} docker-daemon:''${IMAGE_NAME}:''${IMAGE_TAG} # Ensure that the image has minimal closure size. # TODO(palfrey): The default allows 10% inefficiency. Since we control all From be5cf092cbb261a696ed8c3f8927367e0316b1e7 Mon Sep 17 00:00:00 2001 From: Tom Parker-Shemilt Date: Mon, 22 Jun 2026 15:25:11 +0100 Subject: [PATCH 9/9] Merge --- .bazelrc | 1 + .clusterfuzzlite/Dockerfile | 5 + .clusterfuzzlite/build.sh | 6 + .clusterfuzzlite/project.yaml | 2 + .github/actions/generate-version/action.yml | 13 + .../actions/test-and-upload-image/action.yaml | 60 + .github/images-matrix.json | 30 + .github/workflows/clusterfuzzlite.yaml | 57 + .github/workflows/custom-image.yaml | 70 +- .github/workflows/image.yaml | 60 +- .github/workflows/nix.yaml | 5 +- .github/workflows/tagged_image.yaml | 38 +- .gitignore | 1 + CHANGELOG.md | 15 + Cargo.lock | 93 +- Cargo.toml | 3 +- MODULE.bazel | 2 +- MODULE.bazel.lock | 6 +- deployment-examples/docker-compose/Dockerfile | 14 +- deployment-examples/rhel/Dockerfile.rhel8 | 8 +- flake.lock | 8 +- flake.nix | 36 +- nativelink-config/BUILD.bazel | 13 - nativelink-config/Cargo.toml | 2 +- nativelink-config/src/cas_server.rs | 15 +- nativelink-config/src/schedulers.rs | 12 +- nativelink-config/src/stores.rs | 28 +- nativelink-error/Cargo.toml | 2 +- nativelink-macro/Cargo.toml | 2 +- nativelink-metric/Cargo.toml | 2 +- .../nativelink-metric-macro-derive/Cargo.toml | 2 +- nativelink-proto/Cargo.toml | 2 +- nativelink-redis-tester/Cargo.toml | 2 +- nativelink-scheduler/Cargo.toml | 2 +- .../src/api_worker_scheduler.rs | 5 +- nativelink-scheduler/src/simple_scheduler.rs | 10 +- nativelink-service/Cargo.toml | 2 +- nativelink-service/src/bep_server.rs | 75 +- nativelink-service/tests/bep_server_test.rs | 135 +- nativelink-service/tests/cas_server_test.rs | 4 + nativelink-store/Cargo.toml | 6 +- nativelink-store/src/azure_blob_store.rs | 7 + nativelink-store/src/cache_metrics_store.rs | 5 + nativelink-store/src/common_s3_utils.rs | 8 + .../src/completeness_checking_store.rs | 10 +- nativelink-store/src/compression_store.rs | 5 + nativelink-store/src/dedup_store.rs | 9 + nativelink-store/src/existence_cache_store.rs | 5 + nativelink-store/src/fast_slow_store.rs | 10 +- nativelink-store/src/filesystem_store.rs | 4 + nativelink-store/src/gcs_client/client.rs | 3 + nativelink-store/src/gcs_store.rs | 4 + nativelink-store/src/grpc_store.rs | 4 + nativelink-store/src/memory_store.rs | 4 + nativelink-store/src/mongo_store.rs | 4 + nativelink-store/src/noop_store.rs | 4 + .../src/ontap_s3_existence_cache_store.rs | 5 + nativelink-store/src/ontap_s3_store.rs | 8 +- nativelink-store/src/redis_store.rs | 65 +- nativelink-store/src/ref_store.rs | 48 +- nativelink-store/src/s3_store.rs | 4 + nativelink-store/src/shard_store.rs | 10 + .../src/size_partitioning_store.rs | 9 + nativelink-store/src/store_manager.rs | 12 + nativelink-store/src/verify_store.rs | 5 + .../tests/fast_slow_store_test.rs | 36 + nativelink-store/tests/redis_store_test.rs | 56 +- nativelink-store/tests/ref_store_test.rs | 38 +- nativelink-test/fuzz/Cargo.lock | 2523 +++++++++++++++++ nativelink-test/fuzz/Cargo.toml | 21 + .../fuzz/fuzz_targets/cas_config.rs | 10 + nativelink-util/Cargo.toml | 2 +- nativelink-util/src/origin_event_publisher.rs | 52 +- nativelink-util/src/store_trait.rs | 4 + nativelink-util/tests/origin_event_test.rs | 135 +- nativelink-util/tests/store_trait_test.rs | 4 + nativelink-worker/Cargo.toml | 2 +- run_integration_tests.sh | 20 + src/bin/nativelink.rs | 11 +- tools/public/create-multi-arch-image.nix | 39 + tools/public/default.nix | 13 +- tools/public/local-image-test.nix | 40 +- tools/public/publish-ghcr.nix | 13 +- tools/public/regctl-ghcr-login.nix | 11 + tools/public/trivy-report.nix | 18 + tools/regclient.nix | 89 + .../getting-started/other-build-systems.mdx | 18 - .../other-build-systems/buck2.mdx | 93 + .../other-build-systems/buildstream.mdx | 117 + .../other-build-systems}/cmake-recc.mdx | 2 +- .../other-build-systems/index.mdx | 37 + .../other-build-systems/meta.json | 10 + .../other-build-systems/pants.mdx | 57 + .../other-build-systems/reclient.mdx | 82 + web/apps/docs/content/docs/rbe/meta.json | 3 +- .../reference/nativelink-config/index.mdx | 87 +- .../docs/reference/nativelink-config/main.mdx | 79 +- .../reference/nativelink-config/v1.0.0.mdx | 20 +- .../reference/nativelink-config/v1.1.0.mdx | 20 +- .../reference/nativelink-config/v1.2.0.mdx | 20 +- .../reference/nativelink-config/v1.3.0.mdx | 21 +- .../reference/nativelink-config/v1.3.1.mdx | 21 +- .../reference/nativelink-config/v1.3.2.mdx | 21 +- .../reference/nativelink-config/v1.4.0.mdx | 21 +- .../reference/nativelink-config/v1.5.0.mdx | 19 +- .../reference/nativelink-config/v1.5.1.mdx | 2041 +++++++++++++ web/apps/docs/lib/config-versions.ts | 22 +- web/apps/docs/package.json | 2 +- web/apps/docs/scripts/lib/schema-to-mdx.mjs | 20 +- web/apps/web/app/company/page.tsx | 7 +- web/apps/web/app/page.tsx | 102 +- web/apps/web/components/customer-logos.tsx | 252 ++ web/apps/web/components/terminal-data.ts | 12 +- web/apps/web/package.json | 2 +- web/bun.lock | 8 +- 115 files changed, 6895 insertions(+), 564 deletions(-) create mode 100644 .clusterfuzzlite/Dockerfile create mode 100755 .clusterfuzzlite/build.sh create mode 100644 .clusterfuzzlite/project.yaml create mode 100644 .github/actions/generate-version/action.yml create mode 100644 .github/actions/test-and-upload-image/action.yaml create mode 100644 .github/images-matrix.json create mode 100644 .github/workflows/clusterfuzzlite.yaml create mode 100644 nativelink-test/fuzz/Cargo.lock create mode 100644 nativelink-test/fuzz/Cargo.toml create mode 100644 nativelink-test/fuzz/fuzz_targets/cas_config.rs create mode 100644 tools/public/create-multi-arch-image.nix create mode 100644 tools/public/regctl-ghcr-login.nix create mode 100644 tools/public/trivy-report.nix create mode 100644 tools/regclient.nix delete mode 100644 web/apps/docs/content/docs/getting-started/other-build-systems.mdx create mode 100644 web/apps/docs/content/docs/getting-started/other-build-systems/buck2.mdx create mode 100644 web/apps/docs/content/docs/getting-started/other-build-systems/buildstream.mdx rename web/apps/docs/content/docs/{rbe => getting-started/other-build-systems}/cmake-recc.mdx (99%) create mode 100644 web/apps/docs/content/docs/getting-started/other-build-systems/index.mdx create mode 100644 web/apps/docs/content/docs/getting-started/other-build-systems/meta.json create mode 100644 web/apps/docs/content/docs/getting-started/other-build-systems/pants.mdx create mode 100644 web/apps/docs/content/docs/getting-started/other-build-systems/reclient.mdx create mode 100644 web/apps/docs/content/docs/reference/nativelink-config/v1.5.1.mdx create mode 100644 web/apps/web/components/customer-logos.tsx diff --git a/.bazelrc b/.bazelrc index 858f2c74e..273b36af5 100644 --- a/.bazelrc +++ b/.bazelrc @@ -46,6 +46,7 @@ build --incompatible_strict_action_env # change. build --@llvm//config:experimental_stub_libgcc_s=True build --@rules_cc//cc/toolchains/args/archiver_flags:use_libtool_on_macos=False +build --@rules_rust//cargo/settings:experimental_symlink_execroot # Don't use legacy repository rules. build --incompatible_disable_native_repo_rules diff --git a/.clusterfuzzlite/Dockerfile b/.clusterfuzzlite/Dockerfile new file mode 100644 index 000000000..0cef54454 --- /dev/null +++ b/.clusterfuzzlite/Dockerfile @@ -0,0 +1,5 @@ +FROM gcr.io/oss-fuzz-base/base-builder-rust@sha256:a96c8dfb5e0b4c418a3a63c6b1acd04a13d70335fbcaa370d4e1840780a58ce6 + +COPY . $SRC/nativelink +WORKDIR $SRC/nativelink +COPY .clusterfuzzlite/build.sh $SRC/ diff --git a/.clusterfuzzlite/build.sh b/.clusterfuzzlite/build.sh new file mode 100755 index 000000000..0089bb3a2 --- /dev/null +++ b/.clusterfuzzlite/build.sh @@ -0,0 +1,6 @@ +#!/bin/bash -eu + +fuzz_dir="$SRC/nativelink/nativelink-test/fuzz" + +cargo fuzz build --fuzz-dir "$fuzz_dir" +cp "$fuzz_dir/target/x86_64-unknown-linux-gnu/release/cas_config" "$OUT/cas_config" diff --git a/.clusterfuzzlite/project.yaml b/.clusterfuzzlite/project.yaml new file mode 100644 index 000000000..b35328608 --- /dev/null +++ b/.clusterfuzzlite/project.yaml @@ -0,0 +1,2 @@ +language: rust +main_repo: https://github.com/TraceMachina/nativelink diff --git a/.github/actions/generate-version/action.yml b/.github/actions/generate-version/action.yml new file mode 100644 index 000000000..b5bf2af58 --- /dev/null +++ b/.github/actions/generate-version/action.yml @@ -0,0 +1,13 @@ +name: 'Version generator' +description: 'Generate a date-prefixed image version string.' +outputs: + version-string: + description: "Version string" + value: ${{ steps.generate-version.outputs.version }} +runs: + using: "composite" + steps: + - name: Generate version + run: echo "version=$(date +%Y-%m-%d)-${{ github.sha }}" >> $GITHUB_OUTPUT + id: generate-version + shell: bash -euo pipefail {0} diff --git a/.github/actions/test-and-upload-image/action.yaml b/.github/actions/test-and-upload-image/action.yaml new file mode 100644 index 000000000..8c0cb01a5 --- /dev/null +++ b/.github/actions/test-and-upload-image/action.yaml @@ -0,0 +1,60 @@ +--- +name: Test and upload image +description: "Test and upload a Docker image" +inputs: + image: + required: true + multi-arch: + required: true + components: + required: false + tag: + required: true + testing: + required: false + default: true + GHCR_USERNAME: + required: true + GHCR_PASSWORD: + required: true +runs: + using: "composite" + steps: + # FIXME: merge the multi-arch and single-arch paths + - name: Test multi-arch image + if: ${{ inputs.multi-arch == 'true' && inputs.testing }} + run: | + nix run --fallback .#create-multi-arch-image localhost:5000 ${{ inputs.image }}:${{ inputs.tag }} ${{ inputs.components }} + nix run --fallback .#local-image-test localhost:5000/${{ inputs.image }}:${{ inputs.tag }} + shell: bash -euo pipefail {0} + + # FIXME: after we've merged this and made sure it all works _then_ remove the 2457 ref + # See https://github.com/TraceMachina/nativelink/pull/2457 + - name: Upload multi-arch image + if: ${{ inputs.multi-arch == 'true' && contains(fromJson('["refs/heads/main", "refs/pull/2457/merge"]'), github.ref) }} + run: | + nix run --fallback .#regctl-ghcr-login + nix run --fallback .#create-multi-arch-image ghcr.io ${{ github.repository_owner }}/${{ inputs.image }}:${{ inputs.tag }} ${{ inputs.components }} + env: + # FIXME: When https://github.com/github/roadmap/issues/558 gets fixed, replace with a fine-grained token + GHCR_USERNAME: ${{ inputs.GHCR_USERNAME }} + GHCR_PASSWORD: ${{ inputs.GHCR_PASSWORD }} + SKIP_SIGNING: ${{ case(inputs.testing == 'true', 'false', 'true') }} + SKIP_TRIVY: ${{ case(inputs.testing == 'true', 'false', 'true') }} + shell: bash -euo pipefail {0} + + - name: Test image + if: ${{ inputs.multi-arch == 'false' && inputs.testing }} + run: | + nix run --fallback .#local-image-test ${{ inputs.image }} + shell: bash -euo pipefail {0} + + - name: Upload image + if: ${{ inputs.multi-arch == 'false' && contains(fromJson('["refs/heads/main", "refs/pull/2457/merge"]'), github.ref) }} + run: | + nix run --fallback .#publish-ghcr ${{ inputs.image }} + env: + GHCR_REGISTRY: ghcr.io/${{ github.repository_owner }} + GHCR_USERNAME: ${{ inputs.GHCR_USERNAME }} + GHCR_PASSWORD: ${{ inputs.GHCR_PASSWORD }} + shell: bash -euo pipefail {0} diff --git a/.github/images-matrix.json b/.github/images-matrix.json new file mode 100644 index 000000000..62736684d --- /dev/null +++ b/.github/images-matrix.json @@ -0,0 +1,30 @@ +{ + "include": [ + { + "components": "nativelink-image-for-x64 nativelink-image-for-aarch64", + "image": "nativelink", + "multi-arch": true, + "os": "ubuntu-24.04" + }, + { + "image": "nativelink-worker-init", + "multi-arch": false, + "os": "ubuntu-24.04" + }, + { + "image": "nativelink-worker-lre-cc", + "multi-arch": false, + "os": "ubuntu-24.04" + }, + { + "image": "nativelink-worker-lre-rs", + "multi-arch": false, + "os": "ubuntu-24.04" + }, + { + "image": "nativelink-worker-lre-rs", + "multi-arch": false, + "os": "macos-26" + } + ] +} diff --git a/.github/workflows/clusterfuzzlite.yaml b/.github/workflows/clusterfuzzlite.yaml new file mode 100644 index 000000000..a90d2ad7d --- /dev/null +++ b/.github/workflows/clusterfuzzlite.yaml @@ -0,0 +1,57 @@ +--- +name: ClusterFuzzLite + +on: + pull_request: + branches: + - main + paths: + - ".clusterfuzzlite/**" + - "nativelink-test/fuzz/**" + - "nativelink-config/**" + - "nativelink-error/**" + - "Cargo.toml" + - "Cargo.lock" + push: + branches: + - main + paths: + - ".clusterfuzzlite/**" + - "nativelink-test/fuzz/**" + - "nativelink-config/**" + - "nativelink-error/**" + - "Cargo.toml" + - "Cargo.lock" + +permissions: read-all + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +jobs: + fuzz: + runs-on: ubuntu-24.04 + steps: + - name: Checkout + uses: >- # v6.0.2 + actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + persist-credentials: false + + - name: Build fuzzers + id: build + uses: >- # v1 + google/clusterfuzzlite/actions/build_fuzzers@884713a6c30a92e5e8544c39945cd7cb630abcd1 + with: + language: rust + sanitizer: address + + - name: Run fuzzers + uses: >- # v1 + google/clusterfuzzlite/actions/run_fuzzers@884713a6c30a92e5e8544c39945cd7cb630abcd1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + fuzz-seconds: 600 + mode: code-change + sanitizer: address diff --git a/.github/workflows/custom-image.yaml b/.github/workflows/custom-image.yaml index b6a914a4f..b90c4e909 100644 --- a/.github/workflows/custom-image.yaml +++ b/.github/workflows/custom-image.yaml @@ -23,32 +23,41 @@ on: permissions: contents: read - packages: write - pull-requests: write - id-token: write jobs: check-trigger: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 + permissions: + issues: write + pull-requests: read outputs: should_build: ${{ steps.check.outputs.should_build }} pr_sha: ${{ steps.check.outputs.pr_sha }} image: ${{ steps.check.outputs.image }} + multi-arch: ${{ steps.check.outputs.multi-arch }} + components: ${{ steps.check.outputs.components }} steps: + - name: Checkout + uses: >- # v6.0.2 + actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - name: Check trigger id: check uses: >- #v8.0.0 actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd with: script: | + var fs = require('fs'); + var matrix = JSON.parse(fs.readFileSync('.github/images-matrix.json', 'utf8')); + var validImages = []; + for (const option of matrix['include']) { + validImages.append(option['image']); + } if (context.eventName === 'workflow_dispatch') { core.setOutput('should_build', 'true'); core.setOutput('pr_sha', context.sha); - core.setOutput('image', '${{ inputs.image }}'); - return; + image = '${{ inputs.image }}'; } - - if (context.eventName === 'issue_comment') { + else if (context.eventName === 'issue_comment') { const body = context.payload.comment.body.trim(); const isPR = !!context.payload.issue.pull_request; @@ -63,7 +72,6 @@ jobs: }); const image = match[1] || 'image'; - const validImages = ['image', 'nativelink-worker-init', 'nativelink-worker-lre-cc']; if (!validImages.includes(image)) { await github.rest.issues.createComment({ @@ -86,17 +94,28 @@ jobs: comment_id: context.payload.comment.id, content: 'rocket' }); - return; + } + } else { + core.setOutput('should_build', 'false'); + return + } + core.setOutput('image', image); + for (const option of matrix['include']) { + if (option['image'] == image) { + core.setOutput('multi-arch', option['multi-arch'] ?? false); + core.setOutput('components', option['components'] ?? ""); + return } } - - core.setOutput('should_build', 'false'); build-image: name: Build and Push Image needs: check-trigger if: needs.check-trigger.outputs.should_build == 'true' runs-on: ubuntu-24.04 + permissions: + contents: read + issues: write timeout-minutes: 45 steps: - name: Checkout @@ -110,26 +129,25 @@ jobs: with: nativelink_attic_token: ${{ secrets.NATIVELINK_ATTIC_TOKEN }} + - id: version + uses: ./.github/actions/generate-version + - name: Upload image - id: upload - run: | - GIT_HASH=$(git rev-parse --short HEAD) - nix run --fallback .#publish-ghcr ${{ needs.check-trigger.outputs.image }} "$GIT_HASH" - - IMAGE_NAME=$(nix eval .#${{ needs.check-trigger.outputs.image }}.imageName --raw) - echo "image_tag=ghcr.io/${{ github.repository_owner }}/${IMAGE_NAME}:${GIT_HASH}" >> $GITHUB_OUTPUT - env: - GHCR_REGISTRY: ghcr.io/${{ github.repository_owner }} - GHCR_USERNAME: ${{ github.actor }} - GHCR_PASSWORD: ${{ secrets.GITHUB_TOKEN }} - SKIP_SIGNING: "true" - SKIP_TRIVY: "true" + uses: ./.github/actions/test-and-upload-image + with: + testing: false + image: ${{ needs.check-trigger.outputs.image }} + multi-arch: ${{ needs.check-trigger.outputs.multi-arch }} + components: ${{ needs.check-trigger.outputs.components }} + tag: ${{ steps.version.outputs.version-string }} + GHCR_USERNAME: ${{ vars.GHCR_PUBLISH_USER }} + GHCR_PASSWORD: ${{ secrets.GHCR_PUBLISH_TOKEN }} - name: Output image info run: | echo "### Published Image" >> $GITHUB_STEP_SUMMARY echo '```' >> $GITHUB_STEP_SUMMARY - echo "${{ steps.upload.outputs.image_tag }}" >> $GITHUB_STEP_SUMMARY + echo "ghcr.io/${{ github.repository_owner }}/${{ needs.check-trigger.outputs.image }}:${{ steps.version.outputs.version-string }}" >> $GITHUB_STEP_SUMMARY echo '```' >> $GITHUB_STEP_SUMMARY - name: Comment on PR diff --git a/.github/workflows/image.yaml b/.github/workflows/image.yaml index d282acb16..58ef9a80b 100644 --- a/.github/workflows/image.yaml +++ b/.github/workflows/image.yaml @@ -5,8 +5,6 @@ on: branches: - main pull_request: - branches: - - main paths-ignore: - '.github/styles/**' - 'web/**' @@ -18,22 +16,36 @@ concurrency: cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} jobs: + get-images-matrix: + runs-on: ubuntu-24.04 + outputs: + matrix: ${{ steps.setmatrix.outputs.matrix }} + steps: + - name: Checkout + uses: >- # v6.0.2 + actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + - name: Set Images matrix + id: setmatrix + run: | + MATRIX_STRINGIFIED=$(jq -c . < .github/images-matrix.json) + echo "matrix=$MATRIX_STRINGIFIED" >> $GITHUB_OUTPUT + publish-image: + needs: get-images-matrix strategy: fail-fast: false - matrix: - image: [image, nativelink-worker-init, nativelink-worker-lre-cc, nativelink-worker-lre-rs] - os: [ubuntu-24.04] - include: - - os: macos-26 - image: nativelink-worker-lre-rs - name: Publish ${{ matrix.image }} / ${{ matrix.os }} + matrix: ${{ fromJson(needs.get-images-matrix.outputs.matrix) }} + name: Publish ${{ matrix.image }} runs-on: ${{ matrix.os }} permissions: - packages: write id-token: write security-events: write timeout-minutes: 90 + services: + registry: + image: registry:3 + ports: + - 5000:5000 steps: - name: Checkout uses: >- # v6.0.2 @@ -69,25 +81,25 @@ jobs: run: | echo "DOCKER_HOST=${{steps.docker.outputs.sock}}" >> "$GITHUB_ENV" - - name: Test image - run: | - nix run --fallback --impure .#local-image-test ${{ matrix.image }} + - id: version + uses: ./.github/actions/generate-version - - name: Upload image - run: | - nix run --fallback .#publish-ghcr ${{ matrix.image }} - env: - GHCR_REGISTRY: ghcr.io/${{ github.repository_owner }} - GHCR_USERNAME: ${{ github.actor }} - GHCR_PASSWORD: ${{ secrets.GITHUB_TOKEN }} - if: github.ref == 'refs/heads/main' + - name: Test and upload ${{ matrix.image }} + uses: ./.github/actions/test-and-upload-image + with: + image: ${{ matrix.image }} + multi-arch: ${{ matrix.multi-arch }} + components: ${{ matrix.components }} + tag: ${{ steps.version.outputs.version-string }} + GHCR_USERNAME: ${{ vars.GHCR_PUBLISH_USER }} + GHCR_PASSWORD: ${{ secrets.GHCR_PUBLISH_TOKEN }} - name: Upload trivy scan results to GitHub Security tab - uses: >- # v2.16.3 - github/codeql-action/upload-sarif@592977e6ae857384aa79bb31e7a1d62d63449ec5 + uses: >- # v4.36.2 + github/codeql-action/upload-sarif@8aad20d150bbac5944a9f9d289da16a4b0d87c1e with: sarif_file: 'trivy-results.sarif' - if: github.ref == 'refs/heads/main' + if: ${{ contains(fromJson('["refs/heads/main", "refs/pull/2457/merge"]'), github.ref) }} - name: Teardown Worker uses: ./.github/actions/end-nix diff --git a/.github/workflows/nix.yaml b/.github/workflows/nix.yaml index fd70ec972..774adfb1c 100644 --- a/.github/workflows/nix.yaml +++ b/.github/workflows/nix.yaml @@ -47,7 +47,10 @@ jobs: delay=5 for attempt in 1 2 3; do if nix develop --fallback --impure --command \ - bash -c "bazel test //... --verbose_failures --lockfile_mode=error" 2>&1 | tee bazel.log; then + bazel test //... \ + --verbose_failures \ + --lockfile_mode=error \ + 2>&1 | tee bazel.log; then exit 0 fi grep -E '^ERROR:' bazel.log \ diff --git a/.github/workflows/tagged_image.yaml b/.github/workflows/tagged_image.yaml index 172005634..7ee1c7138 100644 --- a/.github/workflows/tagged_image.yaml +++ b/.github/workflows/tagged_image.yaml @@ -12,14 +12,27 @@ concurrency: cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} jobs: + get-images-matrix: + runs-on: ubuntu-24.04 + outputs: + matrix: ${{ steps.setmatrix.outputs.matrix }} + steps: + - name: Checkout + uses: >- # v6.0.2 + actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + - name: Set Images matrix + id: setmatrix + run: | + MATRIX_STRINGIFIED=$(jq -c . < .github/images-matrix.json) + echo "::set-output name=matrix::$MATRIX_STRINGIFIED" + publish-image: + needs: get-images-matrix strategy: fail-fast: false - matrix: - image: [image, nativelink-worker-init, nativelink-worker-lre-cc] + matrix: ${{ fromJson(needs.get-images-matrix.outputs.matrix) }} runs-on: ubuntu-24.04 permissions: - packages: write id-token: write timeout-minutes: 60 steps: @@ -32,17 +45,16 @@ jobs: with: nativelink_attic_token: ${{ secrets.NATIVELINK_ATTIC_TOKEN }} - - name: Test image - run: | - nix run --fallback .#local-image-test ${{ matrix.image }} - - name: Upload image - run: | - nix run --fallback .#publish-ghcr ${{ matrix.image }} ${{github.ref_name}} - env: - GHCR_REGISTRY: ghcr.io/${{ github.repository_owner }} - GHCR_USERNAME: ${{ github.actor }} - GHCR_PASSWORD: ${{ secrets.GITHUB_TOKEN }} + uses: ./.github/actions/test-and-upload-image + with: + testing: false + image: ${{ matrix.image }} + multi-arch: ${{ matrix.multi-arch }} + components: ${{ matrix.components }} + tag: ${{github.ref_name}} + GHCR_USERNAME: ${{ vars.GHCR_PUBLISH_USER }} + GHCR_PASSWORD: ${{ secrets.GHCR_PUBLISH_TOKEN }} - name: Teardown Worker uses: ./.github/actions/end-nix diff --git a/.gitignore b/.gitignore index f4c5a184b..49f46fe80 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ bazel-* target/ +nativelink-test/fuzz/target/ .vscode/ .idea/ .zed diff --git a/CHANGELOG.md b/CHANGELOG.md index ba46db0a1..45e5b7b3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,21 @@ All notable changes to this project will be documented in this file. +## [1.5.2](https://github.com/TraceMachina/nativelink/compare/v1.5.1..v1.5.2) - 2026-06-17 + +### 🐛 Bug Fixes + +- *(redis)* ride out a Sentinel failover on the write paths ([#2445](https://github.com/TraceMachina/nativelink/issues/2445)) by @amankrx - ([a776234](https://github.com/TraceMachina/nativelink/commit/a77623428757dc8d228aa4932edcfd4dac81cea8)) +- *(origin-events)* don't drop resource-usage events on transient failure ([#2442](https://github.com/TraceMachina/nativelink/issues/2442)) by @amankrx - ([78af03a](https://github.com/TraceMachina/nativelink/commit/78af03a7e5c96fb8553b903d87fe6943e34932ee)) + +### 📚 Documentation + +- Makes various docs improvements ([#2444](https://github.com/TraceMachina/nativelink/issues/2444)) by @palfrey - ([2d7d6b3](https://github.com/TraceMachina/nativelink/commit/2d7d6b325008f5e1a15657eb2bee8f1a8da0e0db)) + +### ⚙️ Miscellaneous + +- Adds latest tag with bun ([#2440](https://github.com/TraceMachina/nativelink/issues/2440)) by @MarcusSorealheis - ([3bca6f7](https://github.com/TraceMachina/nativelink/commit/3bca6f7ec75af659dd015e7c9df08a85cc163365)) + ## [1.5.1](https://github.com/TraceMachina/nativelink/compare/v1.5.0..v1.5.1) - 2026-06-16 ### ⛰️ Features diff --git a/Cargo.lock b/Cargo.lock index a86e7289a..1cc2205b9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1869,7 +1869,7 @@ dependencies = [ "gcloud-metadata", "home", "jsonwebtoken", - "reqwest", + "reqwest 0.12.24", "serde", "serde_json", "thiserror 2.0.18", @@ -1882,11 +1882,11 @@ dependencies = [ [[package]] name = "gcloud-metadata" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61f706788c1b58712c513e4d403234707fd255f49caa89d1c930197418b5fb2c" +checksum = "bd3152612316be627be52fe9ca72331eb48425059b3a6a700e7adde223e061d5" dependencies = [ - "reqwest", + "reqwest 0.13.4", "thiserror 2.0.18", "tokio", ] @@ -1908,7 +1908,7 @@ dependencies = [ "percent-encoding", "pkcs8", "regex", - "reqwest", + "reqwest 0.12.24", "reqwest-middleware", "ring", "serde", @@ -2350,7 +2350,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.4", + "socket2 0.5.10", "tokio", "tower-service", "tracing", @@ -2561,16 +2561,6 @@ dependencies = [ "serde", ] -[[package]] -name = "iri-string" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbc5ebe9c3a1a7a5127f920a418f7585e9e758e911d0466ed004f393b0e380b2" -dependencies = [ - "memchr", - "serde", -] - [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -3061,7 +3051,7 @@ checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" [[package]] name = "nativelink" -version = "1.5.1" +version = "1.5.2" dependencies = [ "async-lock", "axum", @@ -3092,7 +3082,7 @@ dependencies = [ [[package]] name = "nativelink-config" -version = "1.5.1" +version = "1.5.2" dependencies = [ "byte-unit", "humantime", @@ -3111,7 +3101,7 @@ dependencies = [ [[package]] name = "nativelink-error" -version = "1.5.1" +version = "1.5.2" dependencies = [ "mongodb", "nativelink-metric", @@ -3119,7 +3109,7 @@ dependencies = [ "prost", "prost-types", "redis", - "reqwest", + "reqwest 0.12.24", "rustls-pki-types", "serde", "serde_json5", @@ -3134,7 +3124,7 @@ dependencies = [ [[package]] name = "nativelink-macro" -version = "1.5.1" +version = "1.5.2" dependencies = [ "proc-macro2", "quote", @@ -3143,7 +3133,7 @@ dependencies = [ [[package]] name = "nativelink-metric" -version = "1.5.1" +version = "1.5.2" dependencies = [ "async-lock", "nativelink-metric-macro-derive", @@ -3154,7 +3144,7 @@ dependencies = [ [[package]] name = "nativelink-metric-macro-derive" -version = "1.5.1" +version = "1.5.2" dependencies = [ "proc-macro2", "quote", @@ -3163,7 +3153,7 @@ dependencies = [ [[package]] name = "nativelink-proto" -version = "1.5.1" +version = "1.5.2" dependencies = [ "derive_more 2.1.0", "prost", @@ -3175,7 +3165,7 @@ dependencies = [ [[package]] name = "nativelink-redis-tester" -version = "1.5.1" +version = "1.5.2" dependencies = [ "either", "nativelink-util", @@ -3188,7 +3178,7 @@ dependencies = [ [[package]] name = "nativelink-scheduler" -version = "1.5.1" +version = "1.5.2" dependencies = [ "async-lock", "async-trait", @@ -3225,7 +3215,7 @@ dependencies = [ [[package]] name = "nativelink-service" -version = "1.5.1" +version = "1.5.2" dependencies = [ "async-lock", "async-trait", @@ -3265,7 +3255,7 @@ dependencies = [ [[package]] name = "nativelink-store" -version = "1.5.1" +version = "1.5.2" dependencies = [ "async-lock", "async-trait", @@ -3318,7 +3308,7 @@ dependencies = [ "redis", "redis-test", "regex", - "reqwest", + "reqwest 0.12.24", "reqwest-middleware", "rlimit", "rustls", @@ -3342,7 +3332,7 @@ dependencies = [ [[package]] name = "nativelink-util" -version = "1.5.1" +version = "1.5.2" dependencies = [ "anyhow", "async-trait", @@ -3403,7 +3393,7 @@ dependencies = [ [[package]] name = "nativelink-worker" -version = "1.5.1" +version = "1.5.2" dependencies = [ "async-lock", "bytes", @@ -4043,7 +4033,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.6.4", + "socket2 0.5.10", "thiserror 2.0.18", "tokio", "tracing", @@ -4080,7 +4070,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.4", + "socket2 0.5.10", "tracing", "windows-sys 0.60.2", ] @@ -4414,6 +4404,35 @@ dependencies = [ "webpki-roots 1.0.3", ] +[[package]] +name = "reqwest" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-core", + "http 1.4.1", + "http-body 1.0.1", + "http-body-util", + "hyper", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "reqwest-middleware" version = "0.4.2" @@ -4423,7 +4442,7 @@ dependencies = [ "anyhow", "async-trait", "http 1.4.1", - "reqwest", + "reqwest 0.12.24", "serde", "thiserror 1.0.69", "tower-service", @@ -5460,20 +5479,20 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.6" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ "bitflags 2.11.1", "bytes", "futures-util", "http 1.4.1", "http-body 1.0.1", - "iri-string", "pin-project-lite", "tower", "tower-layer", "tower-service", + "url", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index e97557bf4..9cf585de2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ [workspace] exclude = [ "nativelink-config/generate-stores-config", + "nativelink-test/fuzz", "tools/generate-bazel-rc", ] resolver = "2" @@ -10,7 +11,7 @@ resolver = "2" edition = "2024" name = "nativelink" rust-version = "1.93.1" -version = "1.5.1" +version = "1.5.2" [profile.release] lto = true diff --git a/MODULE.bazel b/MODULE.bazel index 51917f947..eae78d033 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -1,6 +1,6 @@ module( name = "nativelink", - version = "1.5.1", + version = "1.5.2", compatibility_level = 0, ) diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index e06ad700b..05b9e99cf 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -960,7 +960,7 @@ "futures-util_0.3.32": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"std\"],\"name\":\"futures-channel\",\"optional\":true,\"req\":\"^0.3.32\"},{\"default_features\":false,\"name\":\"futures-core\",\"req\":\"^0.3.32\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"futures-io\",\"optional\":true,\"req\":\"^0.3.32\"},{\"default_features\":false,\"name\":\"futures-macro\",\"optional\":true,\"req\":\"=0.3.32\"},{\"default_features\":false,\"name\":\"futures-sink\",\"optional\":true,\"req\":\"^0.3.32\"},{\"default_features\":false,\"name\":\"futures-task\",\"req\":\"^0.3.32\"},{\"name\":\"futures_01\",\"optional\":true,\"package\":\"futures\",\"req\":\"^0.1.25\"},{\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2.26\"},{\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.2\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.6\"},{\"default_features\":false,\"name\":\"slab\",\"optional\":true,\"req\":\"^0.4.7\"},{\"name\":\"spin\",\"optional\":true,\"req\":\"^0.10.0\"},{\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^0.1.11\"},{\"name\":\"tokio-io\",\"optional\":true,\"req\":\"^0.1.9\"}],\"features\":{\"alloc\":[\"futures-core/alloc\",\"futures-task/alloc\",\"slab\"],\"async-await\":[],\"async-await-macro\":[\"async-await\",\"futures-macro\"],\"bilock\":[],\"cfg-target-has-atomic\":[],\"channel\":[\"std\",\"futures-channel\"],\"compat\":[\"std\",\"futures_01\",\"libc\"],\"default\":[\"std\",\"async-await\",\"async-await-macro\"],\"io\":[\"std\",\"futures-io\",\"memchr\"],\"io-compat\":[\"io\",\"compat\",\"tokio-io\",\"libc\"],\"portable-atomic\":[\"futures-core/portable-atomic\"],\"sink\":[\"futures-sink\"],\"std\":[\"alloc\",\"futures-core/std\",\"futures-task/std\",\"slab/std\"],\"unstable\":[\"futures-core/unstable\",\"futures-task/unstable\"],\"write-all-vectored\":[\"io\"]}}", "futures_0.3.31": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"assert_matches\",\"req\":\"^1.3.0\"},{\"default_features\":false,\"features\":[\"sink\"],\"name\":\"futures-channel\",\"req\":\"^0.3.31\"},{\"default_features\":false,\"name\":\"futures-core\",\"req\":\"^0.3.31\"},{\"default_features\":false,\"name\":\"futures-executor\",\"optional\":true,\"req\":\"^0.3.31\"},{\"default_features\":false,\"name\":\"futures-io\",\"req\":\"^0.3.31\"},{\"default_features\":false,\"name\":\"futures-sink\",\"req\":\"^0.3.31\"},{\"default_features\":false,\"name\":\"futures-task\",\"req\":\"^0.3.31\"},{\"default_features\":false,\"features\":[\"sink\"],\"name\":\"futures-util\",\"req\":\"^0.3.31\"},{\"kind\":\"dev\",\"name\":\"pin-project\",\"req\":\"^1.0.11\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^0.1.11\"}],\"features\":{\"alloc\":[\"futures-core/alloc\",\"futures-task/alloc\",\"futures-sink/alloc\",\"futures-channel/alloc\",\"futures-util/alloc\"],\"async-await\":[\"futures-util/async-await\",\"futures-util/async-await-macro\"],\"bilock\":[\"futures-util/bilock\"],\"cfg-target-has-atomic\":[],\"compat\":[\"std\",\"futures-util/compat\"],\"default\":[\"std\",\"async-await\",\"executor\"],\"executor\":[\"std\",\"futures-executor/std\"],\"io-compat\":[\"compat\",\"futures-util/io-compat\"],\"std\":[\"alloc\",\"futures-core/std\",\"futures-task/std\",\"futures-io/std\",\"futures-sink/std\",\"futures-util/std\",\"futures-util/io\",\"futures-util/channel\"],\"thread-pool\":[\"executor\",\"futures-executor/thread-pool\"],\"unstable\":[\"futures-core/unstable\",\"futures-task/unstable\",\"futures-channel/unstable\",\"futures-io/unstable\",\"futures-util/unstable\"],\"write-all-vectored\":[\"futures-util/write-all-vectored\"]}}", "gcloud-auth_1.2.0": "{\"dependencies\":[{\"name\":\"async-trait\",\"req\":\"^0.1\"},{\"name\":\"base64\",\"req\":\"^0.22\"},{\"kind\":\"dev\",\"name\":\"ctor\",\"req\":\"^0.5\"},{\"name\":\"google-cloud-metadata\",\"package\":\"gcloud-metadata\",\"req\":\"^1.0.1\"},{\"name\":\"hex\",\"optional\":true,\"req\":\"^0.4\"},{\"name\":\"hmac\",\"optional\":true,\"req\":\"^0.12\"},{\"name\":\"home\",\"req\":\"^0.5\"},{\"default_features\":false,\"features\":[\"use_pem\"],\"name\":\"jsonwebtoken\",\"req\":\"^10.2\"},{\"name\":\"path-clean\",\"optional\":true,\"req\":\"^1.0\"},{\"name\":\"percent-encoding\",\"optional\":true,\"req\":\"^2.3\"},{\"default_features\":false,\"features\":[\"json\",\"charset\"],\"name\":\"reqwest\",\"req\":\"^0.12.4\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"req\":\"^1.0\"},{\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"name\":\"sha2\",\"optional\":true,\"req\":\"^0.10\"},{\"features\":[\"async_closure\"],\"kind\":\"dev\",\"name\":\"temp-env\",\"req\":\"^0.3.6\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.8.0\"},{\"name\":\"thiserror\",\"req\":\"^2.0\"},{\"name\":\"time\",\"req\":\"^0.3\"},{\"name\":\"token-source\",\"req\":\"^1.0\"},{\"features\":[\"fs\"],\"name\":\"tokio\",\"req\":\"^1.32\"},{\"features\":[\"test-util\",\"rt-multi-thread\",\"macros\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.32\"},{\"name\":\"tracing\",\"req\":\"^0.1\"},{\"features\":[\"env-filter\",\"std\"],\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3\"},{\"name\":\"url\",\"optional\":true,\"req\":\"^2.4\"},{\"name\":\"urlencoding\",\"req\":\"^2.1\"}],\"features\":{\"default\":[\"default-tls\",\"jwt-aws-lc-rs\"],\"default-tls\":[\"reqwest/default-tls\"],\"external-account\":[\"sha2\",\"path-clean\",\"url\",\"percent-encoding\",\"hmac\",\"hex\"],\"hickory-dns\":[\"reqwest/hickory-dns\"],\"jwt-aws-lc-rs\":[\"jsonwebtoken/aws_lc_rs\"],\"jwt-rust-crypto\":[\"jsonwebtoken/rust_crypto\"],\"rustls-tls\":[\"reqwest/rustls-tls\"]}}", - "gcloud-metadata_1.0.1": "{\"dependencies\":[{\"default_features\":false,\"name\":\"reqwest\",\"req\":\"^0.12.4\"},{\"name\":\"thiserror\",\"req\":\"^2.0\"},{\"features\":[\"sync\",\"net\",\"parking_lot\"],\"name\":\"tokio\",\"req\":\"^1.32\"},{\"features\":[\"test-util\",\"rt-multi-thread\",\"macros\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.32\"}],\"features\":{}}", + "gcloud-metadata_1.0.2": "{\"dependencies\":[{\"default_features\":false,\"name\":\"reqwest\",\"req\":\"^0.13\"},{\"name\":\"thiserror\",\"req\":\"^2.0\"},{\"features\":[\"sync\",\"net\",\"parking_lot\"],\"name\":\"tokio\",\"req\":\"^1.32\"},{\"features\":[\"test-util\",\"rt-multi-thread\",\"macros\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.32\"}],\"features\":{}}", "gcloud-storage_1.1.1": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0\"},{\"name\":\"base64\",\"req\":\"^0.22\"},{\"name\":\"bytes\",\"req\":\"^1.5\"},{\"kind\":\"dev\",\"name\":\"ctor\",\"req\":\"^0.5\"},{\"name\":\"futures-util\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"google-cloud-auth\",\"optional\":true,\"package\":\"gcloud-auth\",\"req\":\"^1.1.2\"},{\"name\":\"google-cloud-metadata\",\"optional\":true,\"package\":\"gcloud-metadata\",\"req\":\"^1.0.1\"},{\"name\":\"hex\",\"req\":\"^0.4\"},{\"name\":\"once_cell\",\"req\":\"^1.18\"},{\"name\":\"percent-encoding\",\"req\":\"^2.3\"},{\"features\":[\"pem\"],\"name\":\"pkcs8\",\"req\":\"^0.10\"},{\"name\":\"regex\",\"req\":\"^1.9\"},{\"default_features\":false,\"features\":[\"json\",\"stream\",\"multipart\"],\"name\":\"reqwest\",\"req\":\"^0.12\"},{\"features\":[\"json\",\"multipart\"],\"name\":\"reqwest-middleware\",\"req\":\"^0.4\"},{\"name\":\"ring\",\"req\":\"^0.17\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"req\":\"^1.0\"},{\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serial_test\",\"req\":\"^3.1\"},{\"name\":\"sha2\",\"req\":\"^0.10\"},{\"name\":\"thiserror\",\"req\":\"^2.0\"},{\"features\":[\"std\",\"macros\",\"formatting\",\"parsing\",\"serde\"],\"name\":\"time\",\"req\":\"^0.3\"},{\"name\":\"token-source\",\"req\":\"^1.0\"},{\"features\":[\"macros\"],\"name\":\"tokio\",\"req\":\"^1.32\"},{\"features\":[\"rt-multi-thread\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.32\"},{\"name\":\"tracing\",\"req\":\"^0.1\"},{\"features\":[\"env-filter\"],\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3.17\"},{\"name\":\"url\",\"req\":\"^2.4\"}],\"features\":{\"auth\":[\"google-cloud-auth\",\"google-cloud-metadata\"],\"default\":[\"default-tls\",\"auth\"],\"default-tls\":[\"reqwest/default-tls\",\"google-cloud-auth?/default-tls\"],\"external-account\":[\"google-cloud-auth?/external-account\"],\"hickory-dns\":[\"reqwest/hickory-dns\",\"google-cloud-auth?/hickory-dns\"],\"rustls-tls\":[\"reqwest/rustls-tls\",\"google-cloud-auth?/rustls-tls\"],\"trace\":[]}}", "generic-array_0.14.9": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"name\":\"typenum\",\"req\":\"^1.12\"},{\"kind\":\"build\",\"name\":\"version_check\",\"req\":\"^0.9\"},{\"default_features\":false,\"name\":\"zeroize\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"more_lengths\":[]}}", "getrandom_0.1.16": "{\"dependencies\":[{\"name\":\"bindgen\",\"optional\":true,\"package\":\"wasm-bindgen\",\"req\":\"^0.2.29\",\"target\":\"wasm32-unknown-unknown\"},{\"name\":\"cfg-if\",\"req\":\"^1\"},{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0\"},{\"name\":\"js-sys\",\"optional\":true,\"req\":\"^0.3\",\"target\":\"wasm32-unknown-unknown\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.64\",\"target\":\"cfg(unix)\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4\"},{\"name\":\"stdweb\",\"optional\":true,\"req\":\"^0.4.18\",\"target\":\"wasm32-unknown-unknown\"},{\"name\":\"wasi\",\"req\":\"^0.9\",\"target\":\"cfg(target_os = \\\"wasi\\\")\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.2\",\"target\":\"wasm32-unknown-unknown\"}],\"features\":{\"dummy\":[],\"rustc-dep-of-std\":[\"compiler_builtins\",\"core\"],\"std\":[],\"test-in-browser\":[\"wasm-bindgen\"],\"wasm-bindgen\":[\"bindgen\",\"js-sys\"]}}", @@ -1022,7 +1022,6 @@ "io-lifetimes_2.0.4": "{\"dependencies\":[{\"features\":[\"io_safety\"],\"name\":\"async-std\",\"optional\":true,\"req\":\"^1.13.0\",\"target\":\"cfg(not(target_os = \\\"wasi\\\"))\"},{\"name\":\"hermit-abi\",\"optional\":true,\"req\":\">=0.3, <=0.4\",\"target\":\"cfg(target_os = \\\"hermit\\\")\"},{\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2.96\",\"target\":\"cfg(not(windows))\"},{\"features\":[\"net\",\"os-ext\"],\"name\":\"mio\",\"optional\":true,\"req\":\"^0.8.0\",\"target\":\"cfg(not(target_os = \\\"wasi\\\"))\"},{\"features\":[\"io_safety\"],\"name\":\"os_pipe\",\"optional\":true,\"req\":\"^1.0.0\",\"target\":\"cfg(not(target_os = \\\"wasi\\\"))\"},{\"name\":\"socket2\",\"optional\":true,\"req\":\"^0.5.0\",\"target\":\"cfg(not(target_os = \\\"wasi\\\"))\"},{\"features\":[\"io-std\",\"fs\",\"net\",\"process\"],\"name\":\"tokio\",\"optional\":true,\"req\":\"^1.6.0\",\"target\":\"cfg(not(target_os = \\\"wasi\\\"))\"},{\"features\":[\"Win32_Foundation\",\"Win32_Storage_FileSystem\",\"Win32_Networking_WinSock\",\"Win32_Security\",\"Win32_System_IO\"],\"name\":\"windows-sys\",\"optional\":true,\"req\":\">=0.52, <=0.59\",\"target\":\"cfg(windows)\"}],\"features\":{\"close\":[\"libc\",\"hermit-abi\",\"windows-sys\"],\"default\":[]}}", "ipconfig_0.3.4": "{\"dependencies\":[{\"name\":\"socket2\",\"req\":\"^0.6.0\",\"target\":\"cfg(windows)\"},{\"name\":\"widestring\",\"req\":\"^1.0.2\",\"target\":\"cfg(windows)\"},{\"name\":\"windows-registry\",\"optional\":true,\"req\":\"^0.6.1\",\"target\":\"cfg(windows)\"},{\"name\":\"windows-result\",\"optional\":true,\"req\":\"^0.4.1\",\"target\":\"cfg(windows)\"},{\"features\":[\"Win32_Foundation\",\"Win32_Networking_WinSock\",\"Win32_System_Registry\"],\"name\":\"windows-sys\",\"req\":\"^0.61\",\"target\":\"cfg(windows)\"}],\"features\":{\"computer\":[\"dep:windows-registry\",\"dep:windows-result\"],\"default\":[\"computer\"]}}", "ipnet_2.11.0": "{\"dependencies\":[{\"name\":\"heapless\",\"optional\":true,\"req\":\"^0\"},{\"name\":\"schemars\",\"optional\":true,\"req\":\"^0.8\"},{\"default_features\":false,\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"package\":\"serde\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1\"}],\"features\":{\"default\":[\"std\"],\"json\":[\"serde\",\"schemars\"],\"ser_as_str\":[\"heapless\"],\"std\":[]}}", - "iri-string_0.7.8": "{\"dependencies\":[{\"default_features\":false,\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.4.1\"},{\"default_features\":false,\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.103\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0.104\"}],\"features\":{\"alloc\":[\"serde?/alloc\"],\"default\":[\"std\"],\"std\":[\"alloc\",\"memchr?/std\",\"serde?/std\"]}}", "is_terminal_polyfill_1.70.2": "{\"dependencies\":[],\"features\":{\"default\":[]}}", "itertools_0.13.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4.0\"},{\"default_features\":false,\"name\":\"either\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"paste\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"permutohedron\",\"req\":\"^0.2\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^0.9\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.7\"}],\"features\":{\"default\":[\"use_std\"],\"use_alloc\":[],\"use_std\":[\"use_alloc\",\"either/use_std\"]}}", "itertools_0.14.0": "{\"dependencies\":[{\"features\":[\"html_reports\"],\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4.0\"},{\"default_features\":false,\"name\":\"either\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"paste\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"permutohedron\",\"req\":\"^0.2\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^0.9\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.7\"}],\"features\":{\"default\":[\"use_std\"],\"use_alloc\":[],\"use_std\":[\"use_alloc\",\"either/use_std\"]}}", @@ -1192,6 +1191,7 @@ "relative-path_2.0.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.76\"},{\"kind\":\"dev\",\"name\":\"foldhash\",\"req\":\"^0.1.5\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.160\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.160\"}],\"features\":{\"alloc\":[\"serde?/alloc\"],\"default\":[\"std\",\"alloc\"],\"serde\":[\"dep:serde\"],\"std\":[]}}", "reqwest-middleware_0.4.2": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.0\"},{\"name\":\"async-trait\",\"req\":\"^0.1.51\"},{\"name\":\"http\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"name\":\"reqwest\",\"req\":\"^0.12.0\"},{\"features\":[\"rustls-tls\"],\"kind\":\"dev\",\"name\":\"reqwest\",\"req\":\"^0.12.0\"},{\"name\":\"serde\",\"req\":\"^1.0.106\"},{\"name\":\"thiserror\",\"req\":\"^1.0.21\"},{\"features\":[\"macros\",\"rt-multi-thread\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.0.0\"},{\"name\":\"tower-service\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"wiremock\",\"req\":\"^0.6.0\"}],\"features\":{\"charset\":[\"reqwest/charset\"],\"http2\":[\"reqwest/http2\"],\"json\":[\"reqwest/json\"],\"multipart\":[\"reqwest/multipart\"],\"rustls-tls\":[\"reqwest/rustls-tls\"]}}", "reqwest_0.12.24": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"tokio\"],\"name\":\"async-compression\",\"optional\":true,\"req\":\"^0.4.0\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"name\":\"base64\",\"req\":\"^0.22\"},{\"kind\":\"dev\",\"name\":\"brotli_crate\",\"package\":\"brotli\",\"req\":\"^8\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"name\":\"bytes\",\"req\":\"^1.2\"},{\"name\":\"cookie_crate\",\"optional\":true,\"package\":\"cookie\",\"req\":\"^0.18.0\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"name\":\"cookie_store\",\"optional\":true,\"req\":\"^0.21.0\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"name\":\"encoding_rs\",\"optional\":true,\"req\":\"^0.8\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.10\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1.0.13\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"name\":\"futures-channel\",\"optional\":true,\"req\":\"^0.3\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"default_features\":false,\"name\":\"futures-core\",\"req\":\"^0.3.28\"},{\"default_features\":false,\"name\":\"futures-util\",\"optional\":true,\"req\":\"^0.3.28\"},{\"default_features\":false,\"features\":[\"std\",\"alloc\"],\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3.28\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"name\":\"h2\",\"optional\":true,\"req\":\"^0.4\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"name\":\"h3\",\"optional\":true,\"req\":\"^0.0.8\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"name\":\"h3-quinn\",\"optional\":true,\"req\":\"^0.0.10\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"features\":[\"tokio\"],\"name\":\"hickory-resolver\",\"optional\":true,\"req\":\"^0.25\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"name\":\"http\",\"req\":\"^1.1\"},{\"name\":\"http-body\",\"req\":\"^1\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"name\":\"http-body-util\",\"req\":\"^0.1\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"features\":[\"http1\",\"client\"],\"name\":\"hyper\",\"req\":\"^1.1\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"default_features\":false,\"features\":[\"http1\",\"http2\",\"client\",\"server\"],\"kind\":\"dev\",\"name\":\"hyper\",\"req\":\"^1.1.0\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"default_features\":false,\"features\":[\"http1\",\"tls12\"],\"name\":\"hyper-rustls\",\"optional\":true,\"req\":\"^0.27.0\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"name\":\"hyper-tls\",\"optional\":true,\"req\":\"^0.6\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"features\":[\"http1\",\"client\",\"client-legacy\",\"client-proxy\",\"tokio\"],\"name\":\"hyper-util\",\"req\":\"^0.1.12\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"features\":[\"http1\",\"http2\",\"client\",\"client-legacy\",\"server-auto\",\"server-graceful\",\"tokio\"],\"kind\":\"dev\",\"name\":\"hyper-util\",\"req\":\"^0.1.12\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"name\":\"js-sys\",\"req\":\"^0.3.77\",\"target\":\"cfg(target_arch = \\\"wasm32\\\")\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0\"},{\"name\":\"log\",\"req\":\"^0.4.17\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"name\":\"mime\",\"optional\":true,\"req\":\"^0.3.16\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"default_features\":false,\"name\":\"mime_guess\",\"optional\":true,\"req\":\"^2.0\"},{\"name\":\"native-tls-crate\",\"optional\":true,\"package\":\"native-tls\",\"req\":\"^0.2.10\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"kind\":\"dev\",\"name\":\"num_cpus\",\"req\":\"^1.0\"},{\"name\":\"once_cell\",\"optional\":true,\"req\":\"^1.18\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"name\":\"percent-encoding\",\"req\":\"^2.3\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.11\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"default_features\":false,\"features\":[\"rustls\",\"runtime-tokio\"],\"name\":\"quinn\",\"optional\":true,\"req\":\"^0.11.1\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"default_features\":false,\"features\":[\"std\",\"tls12\"],\"name\":\"rustls\",\"optional\":true,\"req\":\"^0.23.4\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"name\":\"rustls-native-certs\",\"optional\":true,\"req\":\"^0.8.0\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"features\":[\"std\"],\"name\":\"rustls-pki-types\",\"optional\":true,\"req\":\"^1.9.0\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"name\":\"serde\",\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"name\":\"serde_json\",\"req\":\"^1.0\",\"target\":\"cfg(target_arch = \\\"wasm32\\\")\"},{\"name\":\"serde_json\",\"optional\":true,\"req\":\"^1.0\"},{\"name\":\"serde_urlencoded\",\"req\":\"^0.7.1\"},{\"features\":[\"futures\"],\"name\":\"sync_wrapper\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"net\",\"time\"],\"name\":\"tokio\",\"req\":\"^1.0\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"default_features\":false,\"features\":[\"macros\",\"rt-multi-thread\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.0\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"name\":\"tokio-native-tls\",\"optional\":true,\"req\":\"^0.3.0\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"default_features\":false,\"features\":[\"tls12\"],\"name\":\"tokio-rustls\",\"optional\":true,\"req\":\"^0.26\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"default_features\":false,\"features\":[\"codec\",\"io\"],\"name\":\"tokio-util\",\"optional\":true,\"req\":\"^0.7.9\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"default_features\":false,\"features\":[\"retry\",\"timeout\",\"util\"],\"name\":\"tower\",\"req\":\"^0.5.2\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"default_features\":false,\"features\":[\"limit\"],\"kind\":\"dev\",\"name\":\"tower\",\"req\":\"^0.5.2\"},{\"default_features\":false,\"features\":[\"follow-redirect\"],\"name\":\"tower-http\",\"req\":\"^0.6.5\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"name\":\"tower-service\",\"req\":\"^0.3\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"name\":\"url\",\"req\":\"^2.4\"},{\"name\":\"wasm-bindgen\",\"req\":\"^0.2.89\",\"target\":\"cfg(target_arch = \\\"wasm32\\\")\"},{\"features\":[\"serde-serialize\"],\"kind\":\"dev\",\"name\":\"wasm-bindgen\",\"req\":\"^0.2.89\",\"target\":\"cfg(target_arch = \\\"wasm32\\\")\"},{\"name\":\"wasm-bindgen-futures\",\"req\":\"^0.4.18\",\"target\":\"cfg(target_arch = \\\"wasm32\\\")\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3\",\"target\":\"cfg(target_arch = \\\"wasm32\\\")\"},{\"name\":\"wasm-streams\",\"optional\":true,\"req\":\"^0.4\",\"target\":\"cfg(target_arch = \\\"wasm32\\\")\"},{\"features\":[\"AbortController\",\"AbortSignal\",\"Headers\",\"Request\",\"RequestInit\",\"RequestMode\",\"Response\",\"Window\",\"FormData\",\"Blob\",\"BlobPropertyBag\",\"ServiceWorkerGlobalScope\",\"RequestCredentials\",\"File\",\"ReadableStream\",\"RequestCache\"],\"name\":\"web-sys\",\"req\":\"^0.3.28\",\"target\":\"cfg(target_arch = \\\"wasm32\\\")\"},{\"name\":\"webpki-roots\",\"optional\":true,\"req\":\"^1\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"kind\":\"dev\",\"name\":\"zstd_crate\",\"package\":\"zstd\",\"req\":\"^0.13\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"}],\"features\":{\"__rustls\":[\"dep:hyper-rustls\",\"dep:tokio-rustls\",\"dep:rustls\",\"__tls\"],\"__rustls-ring\":[\"hyper-rustls?/ring\",\"tokio-rustls?/ring\",\"rustls?/ring\",\"quinn?/ring\"],\"__tls\":[\"dep:rustls-pki-types\",\"tokio/io-util\"],\"blocking\":[\"dep:futures-channel\",\"futures-channel?/sink\",\"dep:futures-util\",\"futures-util?/io\",\"futures-util?/sink\",\"tokio/sync\"],\"brotli\":[\"dep:async-compression\",\"async-compression?/brotli\",\"dep:futures-util\",\"dep:tokio-util\"],\"charset\":[\"dep:encoding_rs\",\"dep:mime\"],\"cookies\":[\"dep:cookie_crate\",\"dep:cookie_store\"],\"default\":[\"default-tls\",\"charset\",\"http2\",\"system-proxy\"],\"default-tls\":[\"dep:hyper-tls\",\"dep:native-tls-crate\",\"__tls\",\"dep:tokio-native-tls\"],\"deflate\":[\"dep:async-compression\",\"async-compression?/zlib\",\"dep:futures-util\",\"dep:tokio-util\"],\"gzip\":[\"dep:async-compression\",\"async-compression?/gzip\",\"dep:futures-util\",\"dep:tokio-util\"],\"hickory-dns\":[\"dep:hickory-resolver\",\"dep:once_cell\"],\"http2\":[\"h2\",\"hyper/http2\",\"hyper-util/http2\",\"hyper-rustls?/http2\"],\"http3\":[\"rustls-tls-manual-roots\",\"dep:h3\",\"dep:h3-quinn\",\"dep:quinn\",\"tokio/macros\"],\"json\":[\"dep:serde_json\"],\"macos-system-configuration\":[\"system-proxy\"],\"multipart\":[\"dep:mime_guess\",\"dep:futures-util\"],\"native-tls\":[\"default-tls\"],\"native-tls-alpn\":[\"native-tls\",\"native-tls-crate?/alpn\",\"hyper-tls?/alpn\"],\"native-tls-vendored\":[\"native-tls\",\"native-tls-crate?/vendored\"],\"rustls-tls\":[\"rustls-tls-webpki-roots\"],\"rustls-tls-manual-roots\":[\"rustls-tls-manual-roots-no-provider\",\"__rustls-ring\"],\"rustls-tls-manual-roots-no-provider\":[\"__rustls\"],\"rustls-tls-native-roots\":[\"rustls-tls-native-roots-no-provider\",\"__rustls-ring\"],\"rustls-tls-native-roots-no-provider\":[\"dep:rustls-native-certs\",\"hyper-rustls?/native-tokio\",\"__rustls\"],\"rustls-tls-no-provider\":[\"rustls-tls-manual-roots-no-provider\"],\"rustls-tls-webpki-roots\":[\"rustls-tls-webpki-roots-no-provider\",\"__rustls-ring\"],\"rustls-tls-webpki-roots-no-provider\":[\"dep:webpki-roots\",\"hyper-rustls?/webpki-tokio\",\"__rustls\"],\"socks\":[],\"stream\":[\"tokio/fs\",\"dep:futures-util\",\"dep:tokio-util\",\"dep:wasm-streams\"],\"system-proxy\":[\"hyper-util/client-proxy-system\"],\"trust-dns\":[],\"zstd\":[\"dep:async-compression\",\"async-compression?/zstd\",\"dep:futures-util\",\"dep:tokio-util\"]}}", + "reqwest_0.13.4": "{\"dependencies\":[{\"name\":\"base64\",\"req\":\"^0.22\"},{\"kind\":\"dev\",\"name\":\"brotli_crate\",\"package\":\"brotli\",\"req\":\"^8\",\"target\":\"cfg(not(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"))))\"},{\"name\":\"bytes\",\"req\":\"^1.2\"},{\"name\":\"cookie_crate\",\"optional\":true,\"package\":\"cookie\",\"req\":\"^0.18.0\",\"target\":\"cfg(not(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"))))\"},{\"name\":\"cookie_store\",\"optional\":true,\"req\":\"^0.22.0\",\"target\":\"cfg(not(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"))))\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3\",\"target\":\"cfg(not(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"))))\"},{\"name\":\"encoding_rs\",\"optional\":true,\"req\":\"^0.8\",\"target\":\"cfg(not(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"))))\"},{\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.10\",\"target\":\"cfg(not(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"))))\"},{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1.0.13\",\"target\":\"cfg(not(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"))))\"},{\"name\":\"futures-channel\",\"optional\":true,\"req\":\"^0.3\",\"target\":\"cfg(not(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"))))\"},{\"default_features\":false,\"name\":\"futures-core\",\"req\":\"^0.3.28\"},{\"default_features\":false,\"name\":\"futures-util\",\"optional\":true,\"req\":\"^0.3.28\"},{\"default_features\":false,\"features\":[\"std\",\"alloc\"],\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3.28\",\"target\":\"cfg(not(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"))))\"},{\"name\":\"h2\",\"optional\":true,\"req\":\"^0.4\",\"target\":\"cfg(not(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"))))\"},{\"name\":\"h3\",\"optional\":true,\"req\":\"^0.0.8\",\"target\":\"cfg(not(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"))))\"},{\"name\":\"h3-quinn\",\"optional\":true,\"req\":\"^0.0.10\",\"target\":\"cfg(not(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"))))\"},{\"features\":[\"tokio\"],\"name\":\"hickory-resolver\",\"optional\":true,\"req\":\"^0.26\",\"target\":\"cfg(not(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"))))\"},{\"name\":\"http\",\"req\":\"^1.1\"},{\"name\":\"http-body\",\"req\":\"^1\",\"target\":\"cfg(not(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"))))\"},{\"name\":\"http-body-util\",\"req\":\"^0.1.2\",\"target\":\"cfg(not(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"))))\"},{\"features\":[\"http1\",\"client\"],\"name\":\"hyper\",\"req\":\"^1.1\",\"target\":\"cfg(not(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"))))\"},{\"default_features\":false,\"features\":[\"http1\",\"http2\",\"client\",\"server\"],\"kind\":\"dev\",\"name\":\"hyper\",\"req\":\"^1.1.0\",\"target\":\"cfg(not(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"))))\"},{\"default_features\":false,\"features\":[\"http1\",\"tls12\"],\"name\":\"hyper-rustls\",\"optional\":true,\"req\":\"^0.27.0\",\"target\":\"cfg(not(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"))))\"},{\"name\":\"hyper-tls\",\"optional\":true,\"req\":\"^0.6\",\"target\":\"cfg(not(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"))))\"},{\"features\":[\"http1\",\"client\",\"client-legacy\",\"client-proxy\",\"tokio\"],\"name\":\"hyper-util\",\"req\":\"^0.1.12\",\"target\":\"cfg(not(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"))))\"},{\"features\":[\"http1\",\"http2\",\"client\",\"client-legacy\",\"server-auto\",\"server-graceful\",\"tokio\"],\"kind\":\"dev\",\"name\":\"hyper-util\",\"req\":\"^0.1.12\",\"target\":\"cfg(not(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"))))\"},{\"name\":\"js-sys\",\"req\":\"^0.3.77\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\")))\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0\"},{\"name\":\"log\",\"req\":\"^0.4.17\",\"target\":\"cfg(not(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"))))\"},{\"name\":\"mime\",\"optional\":true,\"req\":\"^0.3.16\",\"target\":\"cfg(not(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"))))\"},{\"default_features\":false,\"name\":\"mime_guess\",\"optional\":true,\"req\":\"^2.0\"},{\"name\":\"native-tls-crate\",\"optional\":true,\"package\":\"native-tls\",\"req\":\"^0.2.16\",\"target\":\"cfg(not(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"))))\"},{\"kind\":\"dev\",\"name\":\"num_cpus\",\"req\":\"^1.0\"},{\"name\":\"once_cell\",\"optional\":true,\"req\":\"^1.18\",\"target\":\"cfg(not(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"))))\"},{\"name\":\"percent-encoding\",\"req\":\"^2.3\",\"target\":\"cfg(not(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"))))\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.11\",\"target\":\"cfg(not(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"))))\"},{\"default_features\":false,\"features\":[\"runtime-tokio\"],\"name\":\"quinn\",\"optional\":true,\"req\":\"^0.11.1\",\"target\":\"cfg(not(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"))))\"},{\"default_features\":false,\"features\":[\"std\",\"tls12\"],\"name\":\"rustls\",\"optional\":true,\"req\":\"^0.23.4\",\"target\":\"cfg(not(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"))))\"},{\"features\":[\"std\"],\"name\":\"rustls-pki-types\",\"optional\":true,\"req\":\"^1.9.0\",\"target\":\"cfg(not(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"))))\"},{\"name\":\"rustls-platform-verifier\",\"optional\":true,\"req\":\">=0.6.0, <0.8.0\",\"target\":\"cfg(not(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"))))\"},{\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\",\"target\":\"cfg(not(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"))))\"},{\"name\":\"serde_json\",\"optional\":true,\"req\":\"^1.0\"},{\"name\":\"serde_urlencoded\",\"optional\":true,\"req\":\"^0.7.1\"},{\"features\":[\"futures\"],\"name\":\"sync_wrapper\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"net\",\"time\"],\"name\":\"tokio\",\"req\":\"^1.0\",\"target\":\"cfg(not(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"))))\"},{\"default_features\":false,\"features\":[\"macros\",\"rt-multi-thread\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.0\",\"target\":\"cfg(not(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"))))\"},{\"name\":\"tokio-native-tls\",\"optional\":true,\"req\":\"^0.3.0\",\"target\":\"cfg(not(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"))))\"},{\"default_features\":false,\"features\":[\"tls12\"],\"name\":\"tokio-rustls\",\"optional\":true,\"req\":\"^0.26\",\"target\":\"cfg(not(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"))))\"},{\"default_features\":false,\"features\":[\"io\"],\"name\":\"tokio-util\",\"optional\":true,\"req\":\"^0.7.9\",\"target\":\"cfg(not(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"))))\"},{\"default_features\":false,\"features\":[\"retry\",\"timeout\",\"util\"],\"name\":\"tower\",\"req\":\"^0.5.2\",\"target\":\"cfg(not(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"))))\"},{\"default_features\":false,\"features\":[\"limit\"],\"kind\":\"dev\",\"name\":\"tower\",\"req\":\"^0.5.2\"},{\"default_features\":false,\"features\":[\"follow-redirect\"],\"name\":\"tower-http\",\"req\":\"^0.6.8\",\"target\":\"cfg(not(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"))))\"},{\"name\":\"tower-service\",\"req\":\"^0.3\",\"target\":\"cfg(not(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"))))\"},{\"name\":\"url\",\"req\":\"^2.4\"},{\"name\":\"wasm-bindgen\",\"req\":\"^0.2.89\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\")))\"},{\"features\":[\"serde-serialize\"],\"kind\":\"dev\",\"name\":\"wasm-bindgen\",\"req\":\"^0.2.89\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\")))\"},{\"name\":\"wasm-bindgen-futures\",\"req\":\"^0.4.18\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\")))\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\")))\"},{\"name\":\"wasm-streams\",\"optional\":true,\"req\":\"^0.5\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\")))\"},{\"features\":[\"AbortController\",\"AbortSignal\",\"Headers\",\"Request\",\"RequestInit\",\"RequestMode\",\"Response\",\"Window\",\"FormData\",\"Blob\",\"BlobPropertyBag\",\"ServiceWorkerGlobalScope\",\"RequestCredentials\",\"File\",\"ReadableStream\",\"RequestCache\"],\"name\":\"web-sys\",\"req\":\"^0.3.28\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\")))\"},{\"kind\":\"dev\",\"name\":\"zstd_crate\",\"package\":\"zstd\",\"req\":\"^0.13\",\"target\":\"cfg(not(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"))))\"}],\"features\":{\"__native-tls\":[\"dep:hyper-tls\",\"dep:native-tls-crate\",\"__tls\",\"dep:tokio-native-tls\"],\"__native-tls-alpn\":[\"native-tls-crate?/alpn\",\"hyper-tls?/alpn\"],\"__rustls\":[\"dep:hyper-rustls\",\"dep:tokio-rustls\",\"dep:rustls\",\"__tls\"],\"__rustls-aws-lc-rs\":[\"hyper-rustls?/aws-lc-rs\",\"tokio-rustls?/aws-lc-rs\",\"rustls?/aws-lc-rs\",\"quinn?/rustls-aws-lc-rs\"],\"__tls\":[\"dep:rustls-pki-types\",\"tokio/io-util\"],\"blocking\":[\"dep:futures-channel\",\"futures-channel?/sink\",\"dep:futures-util\",\"futures-util?/io\",\"futures-util?/sink\",\"tokio/sync\"],\"brotli\":[\"tower-http/decompression-br\"],\"charset\":[\"dep:encoding_rs\",\"dep:mime\"],\"cookies\":[\"dep:cookie_crate\",\"dep:cookie_store\"],\"default\":[\"default-tls\",\"charset\",\"http2\",\"system-proxy\"],\"default-tls\":[\"rustls\"],\"deflate\":[\"tower-http/decompression-deflate\"],\"form\":[\"dep:serde\",\"dep:serde_urlencoded\"],\"gzip\":[\"tower-http/decompression-gzip\"],\"hickory-dns\":[\"dep:hickory-resolver\",\"dep:once_cell\"],\"http2\":[\"dep:h2\",\"hyper/http2\",\"hyper-util/http2\",\"hyper-rustls?/http2\"],\"http3\":[\"rustls\",\"dep:h3\",\"dep:h3-quinn\",\"dep:quinn\",\"tokio/macros\"],\"json\":[\"dep:serde\",\"dep:serde_json\"],\"multipart\":[\"dep:mime_guess\",\"dep:futures-util\"],\"native-tls\":[\"__native-tls\",\"__native-tls-alpn\"],\"native-tls-no-alpn\":[\"__native-tls\"],\"native-tls-vendored\":[\"__native-tls\",\"native-tls-crate?/vendored\",\"__native-tls-alpn\"],\"native-tls-vendored-no-alpn\":[\"__native-tls\",\"native-tls-crate?/vendored\"],\"query\":[\"dep:serde\",\"dep:serde_urlencoded\"],\"rustls\":[\"__rustls-aws-lc-rs\",\"dep:rustls-platform-verifier\",\"__rustls\"],\"rustls-no-provider\":[\"dep:rustls-platform-verifier\",\"__rustls\"],\"socks\":[],\"stream\":[\"tokio/fs\",\"dep:futures-util\",\"dep:tokio-util\",\"dep:wasm-streams\"],\"system-proxy\":[\"hyper-util/client-proxy-system\"],\"zstd\":[\"tower-http/decompression-zstd\"]}}", "resolv-conf_0.7.6": "{\"dependencies\":[],\"features\":{\"system\":[]}}", "rfc6979_0.4.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"hex-literal\",\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"reset\"],\"name\":\"hmac\",\"req\":\"^0.12\"},{\"kind\":\"dev\",\"name\":\"sha2\",\"req\":\"^0.10\"},{\"default_features\":false,\"name\":\"subtle\",\"req\":\"^2\"}],\"features\":{}}", "ring_0.17.14": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"build\",\"name\":\"cc\",\"req\":\"^1.2.8\"},{\"default_features\":false,\"name\":\"cfg-if\",\"req\":\"^1.0.0\"},{\"name\":\"getrandom\",\"req\":\"^0.2.10\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.148\",\"target\":\"cfg(all(any(all(target_arch = \\\"aarch64\\\", target_endian = \\\"little\\\"), all(target_arch = \\\"arm\\\", target_endian = \\\"little\\\")), any(target_os = \\\"android\\\", target_os = \\\"linux\\\")))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.155\",\"target\":\"cfg(all(all(target_arch = \\\"aarch64\\\", target_endian = \\\"little\\\"), target_vendor = \\\"apple\\\", any(target_os = \\\"ios\\\", target_os = \\\"macos\\\", target_os = \\\"tvos\\\", target_os = \\\"visionos\\\", target_os = \\\"watchos\\\")))\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2.148\",\"target\":\"cfg(any(unix, windows, target_os = \\\"wasi\\\"))\"},{\"name\":\"untrusted\",\"req\":\"^0.9\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3.37\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", target_os = \\\"unknown\\\"))\"},{\"features\":[\"Win32_Foundation\",\"Win32_System_Threading\"],\"name\":\"windows-sys\",\"req\":\"^0.52\",\"target\":\"cfg(all(all(target_arch = \\\"aarch64\\\", target_endian = \\\"little\\\"), target_os = \\\"windows\\\"))\"}],\"features\":{\"alloc\":[],\"default\":[\"alloc\",\"dev_urandom_fallback\"],\"dev_urandom_fallback\":[],\"less-safe-getrandom-custom-or-rdrand\":[],\"less-safe-getrandom-espidf\":[],\"slow_tests\":[],\"std\":[\"alloc\"],\"test_logging\":[],\"unstable-testing-arm-no-hw\":[],\"unstable-testing-arm-no-neon\":[],\"wasm32_unknown_unknown_js\":[\"getrandom/js\"]}}", @@ -1306,7 +1306,7 @@ "tonic-prost_0.14.5": "{\"dependencies\":[{\"name\":\"bytes\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"http-body\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"http-body-util\",\"req\":\"^0.1\"},{\"name\":\"prost\",\"req\":\"^0.14\"},{\"features\":[\"macros\",\"rt-multi-thread\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"tokio-stream\",\"req\":\"^0.1\"},{\"default_features\":false,\"name\":\"tonic\",\"req\":\"^0.14.0\"}],\"features\":{}}", "tonic_0.13.1": "{\"dependencies\":[{\"name\":\"async-trait\",\"optional\":true,\"req\":\"^0.1.13\"},{\"default_features\":false,\"name\":\"axum\",\"optional\":true,\"req\":\"^0.8\"},{\"name\":\"base64\",\"req\":\"^0.22\"},{\"kind\":\"dev\",\"name\":\"bencher\",\"req\":\"^0.1.5\"},{\"name\":\"bytes\",\"req\":\"^1.0\"},{\"name\":\"flate2\",\"optional\":true,\"req\":\"^1.0\"},{\"name\":\"h2\",\"optional\":true,\"req\":\"^0.4\"},{\"name\":\"http\",\"req\":\"^1\"},{\"name\":\"http-body\",\"req\":\"^1\"},{\"name\":\"http-body-util\",\"req\":\"^0.1\"},{\"features\":[\"http1\",\"http2\"],\"name\":\"hyper\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"hyper-timeout\",\"optional\":true,\"req\":\"^0.5\"},{\"features\":[\"tokio\"],\"name\":\"hyper-util\",\"optional\":true,\"req\":\"^0.1.4\"},{\"name\":\"percent-encoding\",\"req\":\"^2.1\"},{\"name\":\"pin-project\",\"req\":\"^1.0.11\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"prost\",\"optional\":true,\"req\":\"^0.13\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"quickcheck_macros\",\"req\":\"^1.0\"},{\"name\":\"rustls-native-certs\",\"optional\":true,\"req\":\"^0.8\"},{\"features\":[\"all\"],\"name\":\"socket2\",\"optional\":true,\"req\":\"^0.5\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"tokio\",\"optional\":true,\"req\":\"^1\"},{\"features\":[\"rt-multi-thread\",\"macros\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"logging\",\"tls12\"],\"name\":\"tokio-rustls\",\"optional\":true,\"req\":\"^0.26.1\"},{\"default_features\":false,\"name\":\"tokio-stream\",\"req\":\"^0.1.16\"},{\"default_features\":false,\"name\":\"tower\",\"optional\":true,\"req\":\"^0.5\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tower\",\"req\":\"^0.5\"},{\"name\":\"tower-layer\",\"req\":\"^0.3\"},{\"name\":\"tower-service\",\"req\":\"^0.3\"},{\"name\":\"tracing\",\"req\":\"^0.1\"},{\"name\":\"webpki-roots\",\"optional\":true,\"req\":\"^0.26\"},{\"name\":\"zstd\",\"optional\":true,\"req\":\"^0.13.0\"}],\"features\":{\"_tls-any\":[\"dep:tokio-rustls\",\"dep:tokio\",\"tokio?/rt\",\"tokio?/macros\"],\"channel\":[\"dep:hyper\",\"hyper?/client\",\"dep:hyper-util\",\"hyper-util?/client-legacy\",\"dep:tower\",\"tower?/balance\",\"tower?/buffer\",\"tower?/discover\",\"tower?/limit\",\"tower?/util\",\"dep:tokio\",\"tokio?/time\",\"dep:hyper-timeout\"],\"codegen\":[\"dep:async-trait\"],\"default\":[\"router\",\"transport\",\"codegen\",\"prost\"],\"deflate\":[\"dep:flate2\"],\"gzip\":[\"dep:flate2\"],\"prost\":[\"dep:prost\"],\"router\":[\"dep:axum\",\"dep:tower\",\"tower?/util\"],\"server\":[\"dep:h2\",\"dep:hyper\",\"hyper?/server\",\"dep:hyper-util\",\"hyper-util?/service\",\"hyper-util?/server-auto\",\"dep:socket2\",\"dep:tokio\",\"tokio?/macros\",\"tokio?/net\",\"tokio?/time\",\"tokio-stream/net\",\"dep:tower\",\"tower?/util\",\"tower?/limit\"],\"tls-aws-lc\":[\"_tls-any\",\"tokio-rustls/aws-lc-rs\"],\"tls-native-roots\":[\"_tls-any\",\"channel\",\"dep:rustls-native-certs\"],\"tls-ring\":[\"_tls-any\",\"tokio-rustls/ring\"],\"tls-webpki-roots\":[\"_tls-any\",\"channel\",\"dep:webpki-roots\"],\"transport\":[\"server\",\"channel\"],\"zstd\":[\"dep:zstd\"]}}", "tonic_0.14.5": "{\"dependencies\":[{\"name\":\"async-trait\",\"optional\":true,\"req\":\"^0.1.13\"},{\"default_features\":false,\"name\":\"axum\",\"optional\":true,\"req\":\"^0.8\"},{\"name\":\"base64\",\"req\":\"^0.22\"},{\"kind\":\"dev\",\"name\":\"bencher\",\"req\":\"^0.1.5\"},{\"name\":\"bytes\",\"req\":\"^1.0\"},{\"name\":\"flate2\",\"optional\":true,\"req\":\"^1.0\"},{\"name\":\"h2\",\"optional\":true,\"req\":\"^0.4\"},{\"name\":\"http\",\"req\":\"^1.1.0\"},{\"name\":\"http-body\",\"req\":\"^1\"},{\"name\":\"http-body-util\",\"req\":\"^0.1\"},{\"features\":[\"http1\",\"http2\"],\"name\":\"hyper\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"hyper-timeout\",\"optional\":true,\"req\":\"^0.5\"},{\"features\":[\"tokio\"],\"name\":\"hyper-util\",\"optional\":true,\"req\":\"^0.1.11\"},{\"name\":\"percent-encoding\",\"req\":\"^2.1\"},{\"name\":\"pin-project\",\"req\":\"^1.0.11\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"quickcheck_macros\",\"req\":\"^1.0\"},{\"name\":\"rustls-native-certs\",\"optional\":true,\"req\":\"^0.8\"},{\"features\":[\"all\"],\"name\":\"socket2\",\"optional\":true,\"req\":\"^0.6\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.0\"},{\"name\":\"sync_wrapper\",\"req\":\"^1.0.2\"},{\"default_features\":false,\"name\":\"tokio\",\"optional\":true,\"req\":\"^1\"},{\"features\":[\"rt-multi-thread\",\"macros\",\"test-util\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"logging\",\"tls12\"],\"name\":\"tokio-rustls\",\"optional\":true,\"req\":\"^0.26.1\"},{\"default_features\":false,\"name\":\"tokio-stream\",\"req\":\"^0.1.16\"},{\"default_features\":false,\"name\":\"tower\",\"optional\":true,\"req\":\"^0.5\"},{\"features\":[\"load-shed\",\"timeout\"],\"kind\":\"dev\",\"name\":\"tower\",\"req\":\"^0.5\"},{\"name\":\"tower-layer\",\"req\":\"^0.3\"},{\"name\":\"tower-service\",\"req\":\"^0.3\"},{\"name\":\"tracing\",\"req\":\"^0.1\"},{\"name\":\"webpki-roots\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"zstd\",\"optional\":true,\"req\":\"^0.13.0\"}],\"features\":{\"_tls-any\":[\"dep:tokio\",\"tokio?/rt\",\"tokio?/macros\",\"tls-connect-info\"],\"channel\":[\"dep:hyper\",\"hyper?/client\",\"dep:hyper-util\",\"hyper-util?/client-legacy\",\"dep:tower\",\"tower?/balance\",\"tower?/buffer\",\"tower?/discover\",\"tower?/limit\",\"tower?/load-shed\",\"tower?/util\",\"dep:tokio\",\"tokio?/time\",\"dep:hyper-timeout\"],\"codegen\":[\"dep:async-trait\"],\"default\":[\"router\",\"transport\",\"codegen\"],\"deflate\":[\"dep:flate2\"],\"gzip\":[\"dep:flate2\"],\"router\":[\"dep:axum\",\"dep:tower\",\"tower?/util\"],\"server\":[\"dep:h2\",\"dep:hyper\",\"hyper?/server\",\"dep:hyper-util\",\"hyper-util?/service\",\"hyper-util?/server-auto\",\"dep:socket2\",\"dep:tokio\",\"tokio?/macros\",\"tokio?/net\",\"tokio?/time\",\"tokio-stream/net\",\"dep:tower\",\"tower?/util\",\"tower?/limit\",\"tower?/load-shed\"],\"tls-aws-lc\":[\"_tls-any\",\"tokio-rustls/aws-lc-rs\"],\"tls-connect-info\":[\"dep:tokio-rustls\"],\"tls-native-roots\":[\"_tls-any\",\"channel\",\"dep:rustls-native-certs\"],\"tls-ring\":[\"_tls-any\",\"tokio-rustls/ring\"],\"tls-webpki-roots\":[\"_tls-any\",\"channel\",\"dep:webpki-roots\"],\"transport\":[\"server\",\"channel\"],\"zstd\":[\"dep:zstd\"]}}", - "tower-http_0.6.6": "{\"dependencies\":[{\"features\":[\"tokio\"],\"name\":\"async-compression\",\"optional\":true,\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"async-trait\",\"req\":\"^0.1\"},{\"name\":\"base64\",\"optional\":true,\"req\":\"^0.22\"},{\"name\":\"bitflags\",\"req\":\"^2.0.2\"},{\"kind\":\"dev\",\"name\":\"brotli\",\"req\":\"^7\"},{\"name\":\"bytes\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"bytes\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"futures-core\",\"optional\":true,\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"futures-util\",\"optional\":true,\"req\":\"^0.3.14\"},{\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3.14\"},{\"name\":\"http\",\"req\":\"^1.0\"},{\"name\":\"http-body\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"http-body\",\"req\":\"^1.0.0\"},{\"name\":\"http-body-util\",\"optional\":true,\"req\":\"^0.1.0\"},{\"kind\":\"dev\",\"name\":\"http-body-util\",\"req\":\"^0.1.0\"},{\"name\":\"http-range-header\",\"optional\":true,\"req\":\"^0.4.0\"},{\"name\":\"httpdate\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"client-legacy\",\"http1\",\"tokio\"],\"kind\":\"dev\",\"name\":\"hyper-util\",\"req\":\"^0.1\"},{\"name\":\"iri-string\",\"optional\":true,\"req\":\"^0.7.0\"},{\"default_features\":false,\"name\":\"mime\",\"optional\":true,\"req\":\"^0.3.17\"},{\"default_features\":false,\"name\":\"mime_guess\",\"optional\":true,\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1\"},{\"name\":\"percent-encoding\",\"optional\":true,\"req\":\"^2.1.0\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.7\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"sync_wrapper\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"tokio\",\"optional\":true,\"req\":\"^1.6\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"io\"],\"name\":\"tokio-util\",\"optional\":true,\"req\":\"^0.7\"},{\"name\":\"tower\",\"optional\":true,\"req\":\"^0.5\"},{\"features\":[\"buffer\",\"util\",\"retry\",\"make\",\"timeout\"],\"kind\":\"dev\",\"name\":\"tower\",\"req\":\"^0.5\"},{\"name\":\"tower-layer\",\"req\":\"^0.3.3\"},{\"name\":\"tower-service\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3\"},{\"features\":[\"v4\"],\"name\":\"uuid\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"v4\"],\"kind\":\"dev\",\"name\":\"uuid\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"zstd\",\"req\":\"^0.13\"}],\"features\":{\"add-extension\":[],\"auth\":[\"base64\",\"validate-request\"],\"catch-panic\":[\"tracing\",\"futures-util/std\",\"dep:http-body\",\"dep:http-body-util\"],\"compression-br\":[\"async-compression/brotli\",\"futures-core\",\"dep:http-body\",\"tokio-util\",\"tokio\"],\"compression-deflate\":[\"async-compression/zlib\",\"futures-core\",\"dep:http-body\",\"tokio-util\",\"tokio\"],\"compression-full\":[\"compression-br\",\"compression-deflate\",\"compression-gzip\",\"compression-zstd\"],\"compression-gzip\":[\"async-compression/gzip\",\"futures-core\",\"dep:http-body\",\"tokio-util\",\"tokio\"],\"compression-zstd\":[\"async-compression/zstd\",\"futures-core\",\"dep:http-body\",\"tokio-util\",\"tokio\"],\"cors\":[],\"decompression-br\":[\"async-compression/brotli\",\"futures-core\",\"dep:http-body\",\"dep:http-body-util\",\"tokio-util\",\"tokio\"],\"decompression-deflate\":[\"async-compression/zlib\",\"futures-core\",\"dep:http-body\",\"dep:http-body-util\",\"tokio-util\",\"tokio\"],\"decompression-full\":[\"decompression-br\",\"decompression-deflate\",\"decompression-gzip\",\"decompression-zstd\"],\"decompression-gzip\":[\"async-compression/gzip\",\"futures-core\",\"dep:http-body\",\"dep:http-body-util\",\"tokio-util\",\"tokio\"],\"decompression-zstd\":[\"async-compression/zstd\",\"futures-core\",\"dep:http-body\",\"dep:http-body-util\",\"tokio-util\",\"tokio\"],\"default\":[],\"follow-redirect\":[\"futures-util\",\"dep:http-body\",\"iri-string\",\"tower/util\"],\"fs\":[\"futures-core\",\"futures-util\",\"dep:http-body\",\"dep:http-body-util\",\"tokio/fs\",\"tokio-util/io\",\"tokio/io-util\",\"dep:http-range-header\",\"mime_guess\",\"mime\",\"percent-encoding\",\"httpdate\",\"set-status\",\"futures-util/alloc\",\"tracing\"],\"full\":[\"add-extension\",\"auth\",\"catch-panic\",\"compression-full\",\"cors\",\"decompression-full\",\"follow-redirect\",\"fs\",\"limit\",\"map-request-body\",\"map-response-body\",\"metrics\",\"normalize-path\",\"propagate-header\",\"redirect\",\"request-id\",\"sensitive-headers\",\"set-header\",\"set-status\",\"timeout\",\"trace\",\"util\",\"validate-request\"],\"limit\":[\"dep:http-body\",\"dep:http-body-util\"],\"map-request-body\":[],\"map-response-body\":[],\"metrics\":[\"dep:http-body\",\"tokio/time\"],\"normalize-path\":[],\"propagate-header\":[],\"redirect\":[],\"request-id\":[\"uuid\"],\"sensitive-headers\":[],\"set-header\":[],\"set-status\":[],\"timeout\":[\"dep:http-body\",\"tokio/time\"],\"trace\":[\"dep:http-body\",\"tracing\"],\"util\":[\"tower\"],\"validate-request\":[\"mime\"]}}", + "tower-http_0.6.11": "{\"dependencies\":[{\"features\":[\"tokio\"],\"name\":\"async-compression\",\"optional\":true,\"req\":\"^0.4\"},{\"name\":\"base64\",\"optional\":true,\"req\":\"^0.22\"},{\"name\":\"bitflags\",\"req\":\"^2.0.2\"},{\"kind\":\"dev\",\"name\":\"brotli\",\"req\":\"^8\"},{\"name\":\"bytes\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"bytes\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"futures-core\",\"optional\":true,\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"futures-util\",\"optional\":true,\"req\":\"^0.3.14\"},{\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3.14\"},{\"name\":\"http\",\"req\":\"^1.0\"},{\"name\":\"http-body\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"http-body\",\"req\":\"^1.0.0\"},{\"name\":\"http-body-util\",\"optional\":true,\"req\":\"^0.1.0\"},{\"kind\":\"dev\",\"name\":\"http-body-util\",\"req\":\"^0.1.0\"},{\"name\":\"http-range-header\",\"optional\":true,\"req\":\"^0.4.0\"},{\"name\":\"httpdate\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"hyper\",\"req\":\"^1\"},{\"features\":[\"client-legacy\",\"http1\",\"server\",\"service\",\"tokio\"],\"kind\":\"dev\",\"name\":\"hyper-util\",\"req\":\"^0.1\"},{\"default_features\":false,\"name\":\"mime\",\"optional\":true,\"req\":\"^0.3.17\"},{\"default_features\":false,\"name\":\"mime_guess\",\"optional\":true,\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1\"},{\"name\":\"percent-encoding\",\"optional\":true,\"req\":\"^2.1.0\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.7\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"sync_wrapper\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"tokio\",\"optional\":true,\"req\":\"^1.6\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"io\"],\"name\":\"tokio-util\",\"optional\":true,\"req\":\"^0.7\"},{\"name\":\"tower\",\"optional\":true,\"req\":\"^0.5\"},{\"features\":[\"buffer\",\"util\",\"retry\",\"make\",\"timeout\"],\"kind\":\"dev\",\"name\":\"tower\",\"req\":\"^0.5\"},{\"name\":\"tower-layer\",\"req\":\"^0.3.3\"},{\"name\":\"tower-service\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3\"},{\"name\":\"url\",\"optional\":true,\"req\":\"^2.5\"},{\"features\":[\"v4\"],\"name\":\"uuid\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"zstd\",\"req\":\"^0.13\"}],\"features\":{\"add-extension\":[],\"async-compression\":[],\"auth\":[\"base64\",\"validate-request\"],\"catch-panic\":[\"tracing\",\"futures-util/std\",\"dep:http-body\",\"dep:http-body-util\"],\"compression-br\":[\"dep:async-compression\",\"async-compression?/brotli\",\"futures-core\",\"dep:http-body\",\"tokio-util\",\"dep:tokio\"],\"compression-deflate\":[\"dep:async-compression\",\"async-compression?/zlib\",\"futures-core\",\"dep:http-body\",\"tokio-util\",\"dep:tokio\"],\"compression-full\":[\"compression-br\",\"compression-deflate\",\"compression-gzip\",\"compression-zstd\"],\"compression-gzip\":[\"dep:async-compression\",\"async-compression?/gzip\",\"futures-core\",\"dep:http-body\",\"tokio-util\",\"dep:tokio\"],\"compression-zstd\":[\"dep:async-compression\",\"async-compression?/zstd\",\"futures-core\",\"dep:http-body\",\"tokio-util\",\"dep:tokio\"],\"cors\":[],\"decompression-br\":[\"dep:async-compression\",\"async-compression?/brotli\",\"futures-core\",\"dep:http-body\",\"dep:http-body-util\",\"tokio-util\",\"dep:tokio\"],\"decompression-deflate\":[\"dep:async-compression\",\"async-compression?/zlib\",\"futures-core\",\"dep:http-body\",\"dep:http-body-util\",\"tokio-util\",\"dep:tokio\"],\"decompression-full\":[\"decompression-br\",\"decompression-deflate\",\"decompression-gzip\",\"decompression-zstd\"],\"decompression-gzip\":[\"dep:async-compression\",\"async-compression?/gzip\",\"futures-core\",\"dep:http-body\",\"dep:http-body-util\",\"tokio-util\",\"dep:tokio\"],\"decompression-zstd\":[\"dep:async-compression\",\"async-compression?/zstd\",\"futures-core\",\"dep:http-body\",\"dep:http-body-util\",\"tokio-util\",\"dep:tokio\"],\"default\":[],\"follow-redirect\":[\"futures-util\",\"dep:http-body\",\"dep:url\",\"tower/util\"],\"fs\":[\"dep:tokio\",\"tokio?/fs\",\"tokio?/io-util\",\"futures-core\",\"futures-util\",\"dep:http-body\",\"dep:http-body-util\",\"tokio-util/io\",\"dep:http-range-header\",\"mime_guess\",\"mime\",\"percent-encoding\",\"httpdate\",\"set-status\",\"futures-util/alloc\"],\"full\":[\"add-extension\",\"auth\",\"catch-panic\",\"compression-full\",\"cors\",\"decompression-full\",\"follow-redirect\",\"fs\",\"limit\",\"map-request-body\",\"map-response-body\",\"metrics\",\"normalize-path\",\"on-early-drop\",\"propagate-header\",\"redirect\",\"request-id\",\"sensitive-headers\",\"set-header\",\"set-status\",\"timeout\",\"trace\",\"util\",\"validate-request\"],\"limit\":[\"dep:http-body\",\"dep:http-body-util\"],\"map-request-body\":[],\"map-response-body\":[],\"metrics\":[\"dep:http-body\",\"dep:tokio\",\"tokio?/time\"],\"normalize-path\":[],\"on-early-drop\":[\"dep:http-body\"],\"propagate-header\":[],\"redirect\":[],\"request-id\":[\"uuid\"],\"sensitive-headers\":[],\"set-header\":[],\"set-status\":[],\"timeout\":[\"dep:http-body\",\"dep:tokio\",\"tokio?/time\"],\"tokio\":[],\"trace\":[\"dep:http-body\",\"tracing\"],\"util\":[\"tower\"],\"validate-request\":[\"mime\"]}}", "tower-layer_0.3.3": "{\"dependencies\":[],\"features\":{}}", "tower-service_0.3.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.22\"},{\"kind\":\"dev\",\"name\":\"http\",\"req\":\"^0.2\"},{\"features\":[\"macros\",\"time\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.6.2\"},{\"kind\":\"dev\",\"name\":\"tower-layer\",\"req\":\"^0.3\"}],\"features\":{}}", "tower_0.5.2": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.22\"},{\"name\":\"futures-core\",\"optional\":true,\"req\":\"^0.3.22\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"futures-util\",\"optional\":true,\"req\":\"^0.3.22\"},{\"default_features\":false,\"name\":\"hdrhistogram\",\"optional\":true,\"req\":\"^7.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"hdrhistogram\",\"req\":\"^7.0\"},{\"kind\":\"dev\",\"name\":\"http\",\"req\":\"^1\"},{\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.0.2\"},{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1.4.0\"},{\"name\":\"pin-project-lite\",\"optional\":true,\"req\":\"^0.2.7\"},{\"kind\":\"dev\",\"name\":\"pin-project-lite\",\"req\":\"^0.2.7\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"name\":\"slab\",\"optional\":true,\"req\":\"^0.4\"},{\"name\":\"sync_wrapper\",\"optional\":true,\"req\":\"^1\"},{\"features\":[\"sync\"],\"name\":\"tokio\",\"optional\":true,\"req\":\"^1.6.2\"},{\"features\":[\"macros\",\"sync\",\"test-util\",\"rt-multi-thread\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.6.2\"},{\"name\":\"tokio-stream\",\"optional\":true,\"req\":\"^0.1.0\"},{\"kind\":\"dev\",\"name\":\"tokio-stream\",\"req\":\"^0.1.0\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4\"},{\"default_features\":false,\"name\":\"tokio-util\",\"optional\":true,\"req\":\"^0.7.0\"},{\"name\":\"tower-layer\",\"req\":\"^0.3.3\"},{\"name\":\"tower-service\",\"req\":\"^0.3.3\"},{\"kind\":\"dev\",\"name\":\"tower-test\",\"req\":\"^0.4\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1.2\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"tracing\",\"req\":\"^0.1.2\"},{\"default_features\":false,\"features\":[\"fmt\",\"ansi\"],\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3\"}],\"features\":{\"__common\":[\"futures-core\",\"pin-project-lite\"],\"balance\":[\"discover\",\"load\",\"ready-cache\",\"make\",\"slab\",\"util\"],\"buffer\":[\"__common\",\"tokio/sync\",\"tokio/rt\",\"tokio-util\",\"tracing\"],\"discover\":[\"__common\"],\"filter\":[\"__common\",\"futures-util\"],\"full\":[\"balance\",\"buffer\",\"discover\",\"filter\",\"hedge\",\"limit\",\"load\",\"load-shed\",\"make\",\"ready-cache\",\"reconnect\",\"retry\",\"spawn-ready\",\"steer\",\"timeout\",\"util\"],\"hedge\":[\"util\",\"filter\",\"futures-util\",\"hdrhistogram\",\"tokio/time\",\"tracing\"],\"limit\":[\"__common\",\"tokio/time\",\"tokio/sync\",\"tokio-util\",\"tracing\"],\"load\":[\"__common\",\"tokio/time\",\"tracing\"],\"load-shed\":[\"__common\"],\"log\":[\"tracing/log\"],\"make\":[\"futures-util\",\"pin-project-lite\",\"tokio/io-std\"],\"ready-cache\":[\"futures-core\",\"futures-util\",\"indexmap\",\"tokio/sync\",\"tracing\",\"pin-project-lite\"],\"reconnect\":[\"make\",\"tokio/io-std\",\"tracing\"],\"retry\":[\"__common\",\"tokio/time\",\"util\"],\"spawn-ready\":[\"__common\",\"futures-util\",\"tokio/sync\",\"tokio/rt\",\"util\",\"tracing\"],\"steer\":[],\"timeout\":[\"pin-project-lite\",\"tokio/time\"],\"util\":[\"__common\",\"futures-util\",\"pin-project-lite\",\"sync_wrapper\"]}}", diff --git a/deployment-examples/docker-compose/Dockerfile b/deployment-examples/docker-compose/Dockerfile index c95796932..06d98d4c5 100644 --- a/deployment-examples/docker-compose/Dockerfile +++ b/deployment-examples/docker-compose/Dockerfile @@ -14,7 +14,8 @@ # Current supported Ubuntu version, Noble Numbat aka 24.04 LTS # Locked down to a specific revision to avoid issues with package versions -ARG OS_VERSION=noble-20250925@sha256:728785b59223d755e3e5c5af178fab1be7031f3522c5ccd7a0b32b80d8248123 +ARG BAZELISK_URL=https://github.com/bazelbuild/bazelisk/releases/download/v1.25.0/bazelisk-linux-amd64 +ARG BAZELISK_SHA256=fd8fdff418a1758887520fa42da7e6ae39aefc788cf5e7f7bb8db6934d279fc4 # `--compilation_mode` to pass into bazel (eg: opt, dbg, fastbuild). ARG OPT_LEVEL=opt # Additional bazel flags. @@ -23,12 +24,12 @@ ARG ADDITIONAL_BAZEL_FLAGS= # needed by the user. Useful if your worker needs specific dependencies installed. ARG ADDITIONAL_SETUP_WORKER_CMD= -FROM ubuntu:${OS_VERSION} AS dependencies -ARG OS_VERSION +FROM ubuntu:noble-20250925@sha256:728785b59223d755e3e5c5af178fab1be7031f3522c5ccd7a0b32b80d8248123 AS dependencies +ARG BAZELISK_URL +ARG BAZELISK_SHA256 RUN apt-get update \ && DEBIAN_FRONTEND=noninteractive \ apt-get install --no-install-recommends -y \ - npm=9.2.0~ds1-2 \ git=1:2.43.0-1ubuntu7.3 \ gcc=4:13.2.0-7ubuntu1 \ g++=4:13.2.0-7ubuntu1 \ @@ -36,7 +37,7 @@ RUN apt-get update \ ca-certificates=20240203 \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* \ - && npm install -g @bazel/bazelisk@1.25.0 + && python3 -c 'import hashlib, pathlib, stat, sys, urllib.request; data = urllib.request.urlopen(sys.argv[1], timeout=60).read(); digest = hashlib.sha256(data).hexdigest(); assert digest == sys.argv[2], f"sha256 mismatch: {digest}"; path = pathlib.Path("/usr/local/bin/bazel"); path.write_bytes(data); path.chmod(path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)' "$BAZELISK_URL" "$BAZELISK_SHA256" # Build the binary. FROM dependencies AS builder @@ -52,8 +53,7 @@ RUN bazel \ cp ./bazel-bin/nativelink /root/nativelink-bin # Go back to a fresh ubuntu container and copy only the compiled binary. -FROM ubuntu:${OS_VERSION} AS final -ARG OS_VERSION +FROM ubuntu:noble-20250925@sha256:728785b59223d755e3e5c5af178fab1be7031f3522c5ccd7a0b32b80d8248123 AS final COPY --from=builder /root/nativelink-bin /usr/local/bin/nativelink ARG ADDITIONAL_SETUP_WORKER_CMD diff --git a/deployment-examples/rhel/Dockerfile.rhel8 b/deployment-examples/rhel/Dockerfile.rhel8 index 1352ae899..fe45a9e05 100644 --- a/deployment-examples/rhel/Dockerfile.rhel8 +++ b/deployment-examples/rhel/Dockerfile.rhel8 @@ -19,20 +19,22 @@ ARG ADDITIONAL_BAZEL_FLAGS= # Bash arguments may be passed in here to install any additional dependencies # needed by the user. Useful if your worker needs specific dependencies installed. ARG ADDITIONAL_SETUP_WORKER_CMD= +ARG BAZELISK_URL=https://github.com/bazelbuild/bazelisk/releases/download/v1.25.0/bazelisk-linux-amd64 +ARG BAZELISK_SHA256=fd8fdff418a1758887520fa42da7e6ae39aefc788cf5e7f7bb8db6934d279fc4 # RHEL8-equivalent image # see https://www.redhat.com/en/blog/introducing-red-hat-universal-base-image FROM redhat/ubi8:8.10-1756195303@sha256:534c2c0efa4150ede18e3f9d7480d3b9ec2a52e62bc91cd54e08ee7336819619 AS dependencies -ARG OS_VERSION +ARG BAZELISK_URL +ARG BAZELISK_SHA256 RUN yum update --assumeyes && \ yum install --assumeyes \ - npm-1:6.14.11 \ git-2.43.7 \ gcc-8.5.0 \ gcc-c++-8.5.0 \ python3.12-3.12.12 \ ca-certificates-2025.2.80_v9.0.304 \ - && npm install -g @bazel/bazelisk@1.25.0 + && python3.12 -c 'import hashlib, pathlib, stat, sys, urllib.request; data = urllib.request.urlopen(sys.argv[1], timeout=60).read(); digest = hashlib.sha256(data).hexdigest(); assert digest == sys.argv[2], f"sha256 mismatch: {digest}"; path = pathlib.Path("/usr/local/bin/bazel"); path.write_bytes(data); path.chmod(path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)' "$BAZELISK_URL" "$BAZELISK_SHA256" # Build the binary. FROM dependencies AS builder diff --git a/flake.lock b/flake.lock index b28d4f061..19ee81e21 100644 --- a/flake.lock +++ b/flake.lock @@ -99,17 +99,17 @@ ] }, "locked": { - "lastModified": 1767430085, - "narHash": "sha256-SiXJ6xv4pS2MDUqfj0/mmG746cGeJrMQGmoFgHLS25Y=", + "lastModified": 1775487831, + "narHash": "sha256-2lguQpLPQaxpQCJjXhmEEAfabwsAhkP29Z7fgLzHARA=", "owner": "nlewo", "repo": "nix2container", - "rev": "66f4b8a47e92aa744ec43acbb5e9185078983909", + "rev": "76be9608a7f4d6c985d28b0e7be903ae2547df3e", "type": "github" }, "original": { "owner": "nlewo", "repo": "nix2container", - "rev": "66f4b8a47e92aa744ec43acbb5e9185078983909", + "rev": "76be9608a7f4d6c985d28b0e7be903ae2547df3e", "type": "github" } }, diff --git a/flake.nix b/flake.nix index d6bc99d29..72af9e7c0 100644 --- a/flake.nix +++ b/flake.nix @@ -17,7 +17,7 @@ }; nix2container = { # TODO(SchahinRohani): Use a specific commit hash until nix2container is stable. - url = "github:nlewo/nix2container/66f4b8a47e92aa744ec43acbb5e9185078983909"; + url = "github:nlewo/nix2container/76be9608a7f4d6c985d28b0e7be903ae2547df3e"; inputs.nixpkgs.follows = "nixpkgs"; }; }; @@ -109,6 +109,7 @@ commonArgsFor = p: let isLinuxBuild = p.stdenv.buildPlatform.isLinux; isLinuxTarget = p.stdenv.targetPlatform.isLinux; + isCrossCompile = p.stdenv.targetPlatform.system != pkgs.stdenv.targetPlatform.system; # Map the nix system to the Rust target triple that we'd want to target # by default. targetArch = @@ -155,9 +156,12 @@ ]; CARGO_BUILD_TARGET = targetArch; } + // (pkgs.lib.optionalAttrs (isLinuxTarget && !isCrossCompile) { + # customClang is only defined for the host compiler, so doesn't work for cross-compiling + TARGET_CC = "${pkgs.lre.clang}/bin/customClang"; # So mimalloc gets the right compiler not defaulting to gcc + }) // (pkgs.lib.optionalAttrs isLinuxTarget { CARGO_BUILD_RUSTFLAGS = "-C target-feature=+crt-static"; - TARGET_CC = "${pkgs.lre.clang}/bin/customClang"; # So mimalloc gets the right compiler not defaulting to gcc # FIXME(palfrey): Attempted workaround from https://github.com/llvm/llvm-project/issues/32849#issuecomment-2353071071 but doesn't work # CFLAGS = "-femit-all-decls"; ${linkerEnvVar} = linkerPath; @@ -204,26 +208,19 @@ inherit (nix2container.packages.${system}.nix2container) pullImage; inherit (nix2container.packages.${system}.nix2container) buildImage; - # TODO(palfrey): Allow "crosscompiling" this image. At the moment - # this would set a wrong container architecture. See: - # https://github.com/nlewo/nix2container/issues/138. - nativelink-image = let - nativelinkForImage = - if pkgs.stdenv.isx86_64 - then nativelink-x86_64-linux - else nativelink-aarch64-linux; - in + nativelinkImageFor = archPackages: arch: buildImage { + inherit arch; name = "nativelink"; copyToRoot = [ (pkgs.buildEnv { name = "nativelink-buildEnv"; - paths = [nativelinkForImage]; + paths = [archPackages]; pathsToLink = ["/bin"]; }) ]; config = { - Entrypoint = [(pkgs.lib.getExe' nativelinkForImage "nativelink")]; + Entrypoint = [(pkgs.lib.getExe' archPackages "nativelink")]; Labels = { "org.opencontainers.image.description" = "An RBE compatible, high-performance cache and remote executor."; "org.opencontainers.image.documentation" = "https://github.com/TraceMachina/nativelink"; @@ -236,6 +233,14 @@ }; }; + nativelink-image-for-x64 = nativelinkImageFor nativelink-x86_64-linux "amd64"; + nativelink-image-for-aarch64 = nativelinkImageFor nativelink-aarch64-linux "arm64"; + + nativelink-image = + if pkgs.stdenv.isx86_64 + then nativelink-image-for-x64 + else nativelink-image-for-aarch64; + nativelink-worker-init = pkgs.callPackage ./tools/nativelink-worker-init.nix {inherit buildImage self nativelink-image;}; createWorker = pkgs.nativelink-tools.lib.createWorker self; @@ -387,13 +392,15 @@ nativelinkCoverageForHost nativelink-aarch64-linux nativelink-image + nativelink-image-for-aarch64 + nativelink-image-for-x64 nativelink-is-executable-test nativelink-worker-init nativelink-x86_64-linux ; # Used by the CI - inherit (pkgs.nativelink-tools) local-image-test publish-ghcr; + inherit (pkgs.nativelink-tools) local-image-test publish-ghcr create-multi-arch-image regctl-ghcr-login; default = nativelink; @@ -547,6 +554,7 @@ pkgs.lre.lre-cc.lre-cc-configs-gen pkgs.nativelink-tools.local-image-test pkgs.nativelink-tools.create-local-image + pkgs.nativelink-tools.create-multi-arch-image pkgs.attic-client ] ++ pkgs.lib.optionals pkgs.stdenv.isDarwin [ diff --git a/nativelink-config/BUILD.bazel b/nativelink-config/BUILD.bazel index 7e500d041..0cf07ac79 100644 --- a/nativelink-config/BUILD.bazel +++ b/nativelink-config/BUILD.bazel @@ -120,19 +120,6 @@ rust_doc( visibility = ["//visibility:public"], ) -rust_doc( - name = "docs_json", - crate = ":nativelink-config", - rustdoc_flags = [ - "--output-format=json", - "-Zunstable-options", - ], - target_compatible_with = select({ - "@local-remote-execution//rust:nightly": [], - "//conditions:default": ["@platforms//:incompatible"], - }), -) - rust_doc_test( name = "doc_test", timeout = "short", diff --git a/nativelink-config/Cargo.toml b/nativelink-config/Cargo.toml index e972e090d..0b2474206 100644 --- a/nativelink-config/Cargo.toml +++ b/nativelink-config/Cargo.toml @@ -4,7 +4,7 @@ lints.workspace = true [package] edition = "2024" name = "nativelink-config" -version = "1.5.1" +version = "1.5.2" [dependencies] nativelink-error = { path = "../nativelink-error" } diff --git a/nativelink-config/src/cas_server.rs b/nativelink-config/src/cas_server.rs index dbe431881..c2a327aeb 100644 --- a/nativelink-config/src/cas_server.rs +++ b/nativelink-config/src/cas_server.rs @@ -38,6 +38,7 @@ pub type InstanceName = String; #[derive(Debug, Default, Clone, PartialEq, Eq, Deserialize, Serialize)] #[cfg_attr(feature = "dev-schema", derive(JsonSchema))] pub struct WithInstanceName { + /// Used when the config references `instance_name` in the protocol. #[serde(default)] pub instance_name: InstanceName, #[serde(flatten)] @@ -219,7 +220,7 @@ pub struct ByteStreamConfig { /// This allows clients that disconnect to reconnect and continue uploading /// the same blob. /// - /// Default: 10 (seconds) + /// Default: 10 seconds #[serde( default, deserialize_with = "convert_duration_with_shellexpand", @@ -289,7 +290,7 @@ pub struct HealthConfig { #[serde(default)] pub path: String, - // Timeout on health checks. Defaults to 5s. + /// Timeout on health checks. Default: 5s. #[serde(default)] pub timeout_seconds: u64, } @@ -611,7 +612,7 @@ pub struct EndpointConfig { pub uri: String, /// Timeout in seconds that a request should take. - /// Default: 5 (seconds) + /// Default: 5 seconds pub timeout: Option, /// The TLS configuration to use to connect to the endpoint. @@ -701,7 +702,7 @@ pub struct UploadActionResultConfig { /// if set to `SuccessOnly` then only results with an exit code of 0 will be /// uploaded, if set to Everything all completed results will be uploaded. /// - /// Default: `UploadCacheResultsStrategy::SuccessOnly` + /// Default: `SuccessOnly` #[serde(default)] pub upload_ac_results_strategy: UploadCacheResultsStrategy, @@ -715,7 +716,7 @@ pub struct UploadActionResultConfig { /// to the CAS key-value lookup format and are always a `HistoricalExecuteResponse` /// serialized message. /// - /// Default: `UploadCacheResultsStrategy::FailuresOnly` + /// Default: `FailuresOnly` #[serde(default)] pub upload_historical_results_strategy: Option, @@ -765,7 +766,7 @@ pub struct LocalWorkerConfig { /// The maximum time an action is allowed to run. If a task requests for a timeout /// longer than this time limit, the task will be rejected. Value in seconds. /// - /// Default: 1200 (seconds / 20 mins) + /// Default: 20 minutes #[serde(default, deserialize_with = "convert_duration_with_shellexpand")] pub max_action_timeout: usize, @@ -773,7 +774,7 @@ pub struct LocalWorkerConfig { /// completes. If upload takes longer than this, the action fails with /// `DeadlineExceeded` and may be retried by the scheduler. Value in seconds. /// - /// Default: 600 (seconds / 10 mins) + /// Default: 10 minutes #[serde(default, deserialize_with = "convert_duration_with_shellexpand")] pub max_upload_timeout: usize, diff --git a/nativelink-config/src/schedulers.rs b/nativelink-config/src/schedulers.rs index 9a70b25e9..3d32c9f33 100644 --- a/nativelink-config/src/schedulers.rs +++ b/nativelink-config/src/schedulers.rs @@ -115,19 +115,19 @@ pub struct SimpleSpec { /// The amount of time to retain completed actions for in case /// a `WaitExecution` is called after the action has completed. - /// Default: 60 (seconds) + /// Default: 60 seconds #[serde(default, deserialize_with = "convert_duration_with_shellexpand")] pub retain_completed_for_s: u32, /// Mark operations as completed with error if no client has updated them /// within this duration. - /// Default: 60 (seconds) + /// Default: 60 seconds #[serde(default, deserialize_with = "convert_duration_with_shellexpand")] pub client_action_timeout_s: u64, /// Remove workers from pool once the worker has not responded in this /// amount of time in seconds. - /// Default: 5 (seconds) + /// Default: 5 seconds #[serde(default, deserialize_with = "convert_duration_with_shellexpand")] pub worker_timeout_s: u64, @@ -206,11 +206,13 @@ pub struct GrpcSpec { /// Limit the number of simultaneous upstream requests to this many. A /// value of zero is treated as unlimited. If the limit is reached the /// request is queued. + /// Default: unlimited #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] pub max_concurrent_requests: usize, /// The number of connections to make to each specified endpoint to balance - /// the load over multiple TCP connections. Default 1. + /// the load over multiple TCP connections. + /// Default: 1. #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] pub connections_per_endpoint: usize, } @@ -314,7 +316,7 @@ pub struct HistoricalResourceSpec { pub hints_file: String, /// Reload interval for `hints_file`. Set to 0 to load once. - /// Default: 30 (seconds) + /// Default: 30 seconds #[serde( default = "default_historical_resource_refresh_interval_s", deserialize_with = "convert_duration_with_shellexpand" diff --git a/nativelink-config/src/stores.rs b/nativelink-config/src/stores.rs index 4af52b627..d8b7f473d 100644 --- a/nativelink-config/src/stores.rs +++ b/nativelink-config/src/stores.rs @@ -686,7 +686,7 @@ pub struct FilesystemSpec { /// The block size of the filesystem for the running machine /// value is used to determine an entry's actual size on disk consumed /// For a 4KB block size filesystem, a 1B file actually consumes 4KB - /// Default: 4096 + /// Default: 4kb #[serde(default, deserialize_with = "convert_data_size_with_shellexpand")] pub block_size: u64, @@ -696,7 +696,7 @@ pub struct FilesystemSpec { /// Limiting concurrency prevents disk saturation from blocking the async /// runtime. /// A value of 0 means unlimited (no concurrency limit). - /// Default: 0 + /// Default: unlimited #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] pub max_concurrent_writes: usize, } @@ -808,6 +808,7 @@ pub struct FastSlowSpec { /// threshold — 256 MiB is a reasonable starting point for backends where /// large-blob dedup is a net loss (followers tend to time out anyway), /// but the right value is workload-dependent. + /// Default: disabled (0) #[serde(default, deserialize_with = "convert_data_size_with_shellexpand")] pub bypass_dedup_threshold_bytes: u64, } @@ -839,7 +840,7 @@ pub struct DedupSpec { /// because it will actually not check this number of bytes when /// deciding where to partition the data. /// - /// Default: 65536 (64k) + /// Default: 64k #[serde(default, deserialize_with = "convert_data_size_with_shellexpand")] pub min_size: u32, @@ -853,13 +854,13 @@ pub struct DedupSpec { /// value will be about `normal_size * 1.3` due to implementation /// details. /// - /// Default: 262144 (256k) + /// Default: 256k #[serde(default, deserialize_with = "convert_data_size_with_shellexpand")] pub normal_size: u32, /// Maximum size a chunk is allowed to be. /// - /// Default: 524288 (512k) + /// Default: 512k #[serde(default, deserialize_with = "convert_data_size_with_shellexpand")] pub max_size: u32, @@ -951,7 +952,7 @@ pub struct Lz4Config { /// so if there was a bad actor, they could upload an extremely large /// `block_size`'ed entry and we'd allocate a large amount of memory /// when retrieving the data. To prevent this from happening, we - /// allow you to specify the maximum that we'll attempt deserialize. + /// allow you to specify the maximum that we'll attempt to deserialize. /// /// Default: value in `block_size`. #[serde(default, deserialize_with = "convert_data_size_with_shellexpand")] @@ -1478,16 +1479,6 @@ pub struct RedisSpec { pub scan_count: usize, /// Retry configuration to use when a network request fails. - /// See the `Retry` struct for more information. - /// - /// ```txt - /// Default: Retry { - /// max_retries: 0, /* unlimited */ - /// delay: 0.1, /* 100ms */ - /// jitter: 0.5, /* 50% */ - /// retry_on_errors: None, /* not used in redis store */ - /// } - /// ``` #[serde(default)] pub retry: Retry, @@ -1508,8 +1499,13 @@ pub struct RedisSpec { #[serde(rename_all = "snake_case")] #[cfg_attr(feature = "dev-schema", derive(JsonSchema))] pub enum RedisMode { + /// Use Redis Cluster. Cluster, + + /// Use Redis Sentinel. Sentinel, + + /// Use a standalone Redis server. #[default] Standard, } diff --git a/nativelink-error/Cargo.toml b/nativelink-error/Cargo.toml index 8f12b1c63..9c0cbde92 100644 --- a/nativelink-error/Cargo.toml +++ b/nativelink-error/Cargo.toml @@ -8,7 +8,7 @@ autoexamples = false autotests = true edition = "2024" name = "nativelink-error" -version = "1.5.1" +version = "1.5.2" [dependencies] nativelink-metric = { path = "../nativelink-metric" } diff --git a/nativelink-macro/Cargo.toml b/nativelink-macro/Cargo.toml index 0a0d1ce69..5b9bb8937 100644 --- a/nativelink-macro/Cargo.toml +++ b/nativelink-macro/Cargo.toml @@ -4,7 +4,7 @@ lints.workspace = true [package] edition = "2024" name = "nativelink-macro" -version = "1.5.1" +version = "1.5.2" [lib] proc-macro = true diff --git a/nativelink-metric/Cargo.toml b/nativelink-metric/Cargo.toml index 87f02080f..15162b7da 100644 --- a/nativelink-metric/Cargo.toml +++ b/nativelink-metric/Cargo.toml @@ -4,7 +4,7 @@ lints.workspace = true [package] edition = "2024" name = "nativelink-metric" -version = "1.5.1" +version = "1.5.2" [dependencies] nativelink-metric-macro-derive = { path = "nativelink-metric-macro-derive" } diff --git a/nativelink-metric/nativelink-metric-macro-derive/Cargo.toml b/nativelink-metric/nativelink-metric-macro-derive/Cargo.toml index 01f355840..37f45137e 100644 --- a/nativelink-metric/nativelink-metric-macro-derive/Cargo.toml +++ b/nativelink-metric/nativelink-metric-macro-derive/Cargo.toml @@ -1,7 +1,7 @@ [package] edition = "2024" name = "nativelink-metric-macro-derive" -version = "1.5.1" +version = "1.5.2" [lib] proc-macro = true diff --git a/nativelink-proto/Cargo.toml b/nativelink-proto/Cargo.toml index 5ce9a9f89..595e93870 100644 --- a/nativelink-proto/Cargo.toml +++ b/nativelink-proto/Cargo.toml @@ -2,7 +2,7 @@ [package] edition = "2024" name = "nativelink-proto" -version = "1.5.1" +version = "1.5.2" [lib] doctest = false # because some of the generated protos have things that look like doctests but break diff --git a/nativelink-redis-tester/Cargo.toml b/nativelink-redis-tester/Cargo.toml index 2b876a101..cbb65e9ea 100644 --- a/nativelink-redis-tester/Cargo.toml +++ b/nativelink-redis-tester/Cargo.toml @@ -4,7 +4,7 @@ lints.workspace = true [package] edition = "2024" name = "nativelink-redis-tester" -version = "1.5.1" +version = "1.5.2" [dependencies] nativelink-util = { path = "../nativelink-util" } diff --git a/nativelink-scheduler/Cargo.toml b/nativelink-scheduler/Cargo.toml index 2dc11eb85..e57468d3c 100644 --- a/nativelink-scheduler/Cargo.toml +++ b/nativelink-scheduler/Cargo.toml @@ -4,7 +4,7 @@ lints.workspace = true [package] edition = "2024" name = "nativelink-scheduler" -version = "1.5.1" +version = "1.5.2" [dependencies] nativelink-config = { path = "../nativelink-config" } diff --git a/nativelink-scheduler/src/api_worker_scheduler.rs b/nativelink-scheduler/src/api_worker_scheduler.rs index ab57ee325..25b727d0b 100644 --- a/nativelink-scheduler/src/api_worker_scheduler.rs +++ b/nativelink-scheduler/src/api_worker_scheduler.rs @@ -680,7 +680,10 @@ impl WorkerScheduler for ApiWorkerScheduler { identity: action_info.origin_metadata.identity, event: Some(event), }; - if let Err(err) = origin_event_tx.try_send(origin_event) { + // Awaited send (not try_send): apply backpressure when the publisher + // queue is full instead of silently dropping the resource-usage event, + // which is what drives action-level resource sizing in the UI. + if let Err(err) = origin_event_tx.send(origin_event).await { warn!(?err, "Failed to publish action resource usage origin event"); } Ok(()) diff --git a/nativelink-scheduler/src/simple_scheduler.rs b/nativelink-scheduler/src/simple_scheduler.rs index 04697d656..0abd6bad7 100644 --- a/nativelink-scheduler/src/simple_scheduler.rs +++ b/nativelink-scheduler/src/simple_scheduler.rs @@ -192,7 +192,7 @@ impl SimpleScheduler { } } - fn publish_scheduler_start_execute( + async fn publish_scheduler_start_execute( maybe_origin_event_tx: Option<&mpsc::Sender>, origin_metadata: &OriginMetadata, event_id: String, @@ -211,7 +211,10 @@ impl SimpleScheduler { event: Some(event), }; - if let Err(err) = origin_event_tx.try_send(origin_event) { + // Awaited send (not try_send): backpressure rather than drop, so the + // start-execute event that later resource-usage events reference as + // their parent isn't silently lost when the queue is full. + if let Err(err) = origin_event_tx.send(origin_event).await { warn!( ?err, "Failed to publish scheduler start execute origin event" @@ -367,7 +370,8 @@ impl SimpleScheduler { &event_origin_metadata, event_id, event, - ); + ) + .await; } Ok(()) diff --git a/nativelink-service/Cargo.toml b/nativelink-service/Cargo.toml index fb8f04d46..2fadad6d2 100644 --- a/nativelink-service/Cargo.toml +++ b/nativelink-service/Cargo.toml @@ -4,7 +4,7 @@ lints.workspace = true [package] edition = "2024" name = "nativelink-service" -version = "1.5.1" +version = "1.5.2" [dependencies] nativelink-config = { path = "../nativelink-config" } diff --git a/nativelink-service/src/bep_server.rs b/nativelink-service/src/bep_server.rs index e2e0289e1..89c4ffeea 100644 --- a/nativelink-service/src/bep_server.rs +++ b/nativelink-service/src/bep_server.rs @@ -13,6 +13,7 @@ // limitations under the License. use core::pin::Pin; +use core::time::Duration; use std::borrow::Cow; use bytes::BytesMut; @@ -33,8 +34,14 @@ use opentelemetry::baggage::BaggageExt; use opentelemetry::context::Context; use opentelemetry_semantic_conventions::attribute::ENDUSER_ID; use prost::Message; +use tokio::time::sleep; use tonic::{Request, Response, Result, Status, Streaming}; -use tracing::{Level, instrument}; +use tracing::{Level, instrument, warn}; + +/// Bounded retries for persisting a BEP event so a transient store failure +/// (e.g. a Redis Sentinel failover) doesn't drop it — BEP events are the +/// authoritative build record, the same durability class as origin events. +const MAX_BEP_UPLOAD_ATTEMPTS: u32 = 5; /// Current version of the BEP event. This might be used in the future if /// there is a breaking change in the BEP event format. @@ -100,10 +107,33 @@ impl BepServer { .encode(&mut buf) .err_tip(|| "Could not encode PublishLifecycleEventRequest proto")?; - self.store - .update_oneshot(store_key.clone(), buf.freeze()) - .await - .err_tip(|| format!("Failed to store PublishLifecycleEventRequest for {store_key}",))?; + let data = buf.freeze(); + for attempt in 1..=MAX_BEP_UPLOAD_ATTEMPTS { + match self + .store + .update_oneshot(store_key.borrow(), data.clone()) + .await + { + Ok(()) => break, + Err(err) if attempt < MAX_BEP_UPLOAD_ATTEMPTS => { + warn!( + attempt, + max = MAX_BEP_UPLOAD_ATTEMPTS, + ?err, + %store_key, + "Failed to store BEP lifecycle event, retrying" + ); + sleep(Duration::from_secs_f32(0.1 * attempt as f32)).await; + } + Err(err) => { + return Err(err).err_tip(|| { + format!( + "Failed to store PublishLifecycleEventRequest for {store_key} after retries" + ) + }); + } + } + } Ok(Response::new(())) } @@ -141,16 +171,31 @@ impl BepServer { .encode(&mut buf) .err_tip(|| "Could not encode PublishBuildToolEventStreamRequest proto")?; - store - .update_oneshot( - StoreKey::Str(Cow::Owned(format!( - "BepEvent:be:{}:{}:{}", - &stream_id.build_id, &stream_id.invocation_id, sequence_number, - ))), - buf.freeze(), - ) - .await - .err_tip(|| "Failed to store PublishBuildToolEventStreamRequest")?; + let store_key = StoreKey::Str(Cow::Owned(format!( + "BepEvent:be:{}:{}:{}", + &stream_id.build_id, &stream_id.invocation_id, sequence_number, + ))); + let data = buf.freeze(); + for attempt in 1..=MAX_BEP_UPLOAD_ATTEMPTS { + match store.update_oneshot(store_key.borrow(), data.clone()).await { + Ok(()) => break, + Err(err) if attempt < MAX_BEP_UPLOAD_ATTEMPTS => { + warn!( + attempt, + max = MAX_BEP_UPLOAD_ATTEMPTS, + ?err, + %store_key, + "Failed to store BEP build-tool event, retrying" + ); + sleep(Duration::from_secs_f32(0.1 * attempt as f32)).await; + } + Err(err) => { + return Err(err).err_tip( + || "Failed to store PublishBuildToolEventStreamRequest after retries", + )?; + } + } + } Ok(PublishBuildToolEventStreamResponse { stream_id: Some(stream_id.clone()), diff --git a/nativelink-service/tests/bep_server_test.rs b/nativelink-service/tests/bep_server_test.rs index d6461875d..ac5b735f9 100644 --- a/nativelink-service/tests/bep_server_test.rs +++ b/nativelink-service/tests/bep_server_test.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use core::pin::Pin; +use core::sync::atomic::{AtomicUsize, Ordering}; use std::borrow::Cow; use std::sync::Arc; @@ -19,8 +21,9 @@ use futures::StreamExt; use hyper::body::Frame; use nativelink_config::cas_server::BepConfig; use nativelink_config::stores::{MemorySpec, StoreSpec}; -use nativelink_error::{Error, ResultExt}; +use nativelink_error::{Code, Error, ResultExt, make_err}; use nativelink_macro::nativelink_test; +use nativelink_metric::MetricsComponent; use nativelink_proto::com::github::trace_machina::nativelink::events::{BepEvent, bep_event}; use nativelink_proto::google::devtools::build::v1::build_event::console_output::Output; use nativelink_proto::google::devtools::build::v1::build_event::{ @@ -37,15 +40,21 @@ use nativelink_proto::google::devtools::build::v1::{ use nativelink_service::bep_server::BepServer; use nativelink_store::default_store_factory::store_factory; use nativelink_store::store_manager::StoreManager; -use nativelink_util::buf_channel::make_buf_channel_pair; +use nativelink_util::buf_channel::{ + DropCloserReadHalf, DropCloserWriteHalf, make_buf_channel_pair, +}; use nativelink_util::channel_body_for_tests::ChannelBody; use nativelink_util::common::encode_stream_proto; -use nativelink_util::store_trait::{Store, StoreKey, StoreLike}; +use nativelink_util::default_health_status_indicator; +use nativelink_util::health_utils::HealthStatusIndicator; +use nativelink_util::store_trait::{ + RemoveItemCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, +}; use pretty_assertions::assert_eq; use prost::Message; use prost_types::Timestamp; use tonic::codec::{Codec, ProstCodec}; -use tonic::{Request, Streaming}; +use tonic::{Request, Streaming, async_trait}; const BEP_STORE_NAME: &str = "main_bep"; @@ -80,6 +89,124 @@ fn get_bep_store(store_manager: &StoreManager) -> Result { .err_tip(|| format!("While retrieving bep_store {BEP_STORE_NAME}")) } +/// A store whose `update` fails the first `fail_until` calls, then succeeds — +/// to verify the BEP server retries a transient upload failure (e.g. a Redis +/// Sentinel failover) instead of dropping the event. +#[derive(Debug, MetricsComponent)] +struct FlakyStore { + fail_until: usize, + attempts: AtomicUsize, +} + +#[async_trait] +#[allow(clippy::todo)] +impl StoreDriver for FlakyStore { + async fn post_init(self: Arc) -> Result<(), Error> { + Ok(()) + } + + async fn has_with_results( + self: Pin<&Self>, + _keys: &[StoreKey<'_>], + _results: &mut [Option], + ) -> Result<(), Error> { + todo!(); + } + + async fn update( + self: Pin<&Self>, + _key: StoreKey<'_>, + mut reader: DropCloserReadHalf, + _upload_size: UploadSizeInfo, + ) -> Result { + // Drain so the writer side completes. + reader.consume(None).await?; + let attempt = self.attempts.fetch_add(1, Ordering::SeqCst) + 1; + if attempt <= self.fail_until { + return Err(make_err!( + Code::Unavailable, + "flaky store failure {attempt}" + )); + } + Ok(0) + } + + async fn get_part( + self: Pin<&Self>, + _key: StoreKey<'_>, + _writer: &mut DropCloserWriteHalf, + _offset: u64, + _length: Option, + ) -> Result<(), Error> { + todo!(); + } + + fn inner_store(&self, _digest: Option) -> &dyn StoreDriver { + self + } + + fn as_any(&self) -> &(dyn core::any::Any + Sync + Send + 'static) { + self + } + + fn as_any_arc(self: Arc) -> Arc { + self + } + + fn register_remove_callback( + self: Arc, + _callback: Arc, + ) -> Result<(), Error> { + todo!(); + } +} + +default_health_status_indicator!(FlakyStore); + +/// A transient store-upload failure (e.g. a Redis Sentinel failover) must NOT +/// drop a BEP lifecycle event — the server retries the upload until it lands. +#[nativelink_test] +async fn publish_lifecycle_event_retries_transient_store_failure() +-> Result<(), Box> { + let flaky = Arc::new(FlakyStore { + fail_until: 2, + attempts: AtomicUsize::new(0), + }); + let store_manager = Arc::new(StoreManager::new()); + store_manager.add_store(BEP_STORE_NAME, Store::new(flaky.clone())); + let bep_server = make_bep_server(&store_manager)?; + + let request = PublishLifecycleEventRequest { + service_level: ServiceLevel::Interactive as i32, + build_event: Some(OrderedBuildEvent { + stream_id: Some(StreamId { + build_id: "some-build-id".to_string(), + invocation_id: "some-invocation-id".to_string(), + component: BuildComponent::Controller as i32, + }), + sequence_number: 1, + event: None, + }), + stream_timeout: None, + notification_keywords: vec![], + project_id: "some-project-id".to_string(), + check_preceding_lifecycle_events_present: false, + }; + + // Must succeed despite the first two transient failures. + bep_server + .publish_lifecycle_event(Request::new(request)) + .await + .err_tip(|| "publish_lifecycle_event should succeed after retrying")?; + + assert_eq!( + flaky.attempts.load(Ordering::SeqCst), + 3, + "the BEP publish must retry past transient store failures (2 failures + 1 success)", + ); + Ok(()) +} + /// Asserts that a gRPC request for a [`PublishLifecycleEventRequest`] is correctly dumped into a [`Store`] #[nativelink_test] async fn publish_lifecycle_event_test() -> Result<(), Box> { diff --git a/nativelink-service/tests/cas_server_test.rs b/nativelink-service/tests/cas_server_test.rs index 20a23907b..dccc90208 100644 --- a/nativelink-service/tests/cas_server_test.rs +++ b/nativelink-service/tests/cas_server_test.rs @@ -681,6 +681,10 @@ struct StallStore { #[async_trait] impl StoreDriver for StallStore { + async fn post_init(self: Arc) -> Result<(), Error> { + Ok(()) + } + async fn has_with_results( self: Pin<&Self>, _digests: &[StoreKey<'_>], diff --git a/nativelink-store/Cargo.toml b/nativelink-store/Cargo.toml index 84c35c543..c9c30a3c8 100644 --- a/nativelink-store/Cargo.toml +++ b/nativelink-store/Cargo.toml @@ -4,7 +4,7 @@ lints.workspace = true [package] edition = "2024" name = "nativelink-store" -version = "1.5.1" +version = "1.5.2" [dependencies] nativelink-config = { path = "../nativelink-config" } @@ -43,7 +43,7 @@ byteorder = { version = "1.5.0", default-features = false } bytes = { version = "1.10.1", default-features = false } const_format = { version = "0.2.34", default-features = false } futures = { version = "0.3.31", default-features = false, features = ["std"] } -gcloud-auth = { version = "1.2", default-features = false, features = [ +gcloud-auth = { version = "=1.2.0", default-features = false, features = [ "jwt-rust-crypto", ] } gcloud-storage = { version = "1", default-features = false, features = [ @@ -91,7 +91,7 @@ redis = { version = "1.0.0", default-features = false, features = [ regex = { version = "1.11.1", default-features = false } reqwest = { version = "0.12", default-features = false } reqwest-middleware = { version = "0.4.2", default-features = false } -rustls = { version = "0.23.27", default-features = false, features = [] } +rustls = { version = "0.23.27", default-features = false, features = ["ring"] } rustls-pki-types = { version = "1.13.1", default-features = false } serde = { version = "1.0.219", default-features = false } serde_json = { version = "1.0.140", default-features = false } diff --git a/nativelink-store/src/azure_blob_store.rs b/nativelink-store/src/azure_blob_store.rs index 3878abf34..689a5e0f5 100644 --- a/nativelink-store/src/azure_blob_store.rs +++ b/nativelink-store/src/azure_blob_store.rs @@ -53,6 +53,7 @@ use tokio::time::sleep; use tracing::{Level, event}; use crate::cas_utils::is_zero_digest; +use crate::common_s3_utils::install_default_rustls_crypto_provider; // Check the below doc for the limits specific to Azure. // https://learn.microsoft.com/en-us/azure/storage/blobs/scalability-targets#scale-targets-for-blob-storage @@ -347,6 +348,8 @@ impl AzureClient { } fn build_connector(config: &ExperimentalAzureSpec) -> HttpsConnector { + install_default_rustls_crypto_provider(); + let builder = HttpsConnectorBuilder::new().with_webpki_roots(); let builder_with_schemes = if config.common.insecure_allow_http { @@ -557,6 +560,10 @@ where I: InstantWrapper, NowFn: Fn() -> I + Send + Sync + Unpin + 'static, { + async fn post_init(self: Arc) -> Result<(), Error> { + Ok(()) + } + async fn has_with_results( self: Pin<&Self>, keys: &[StoreKey<'_>], diff --git a/nativelink-store/src/cache_metrics_store.rs b/nativelink-store/src/cache_metrics_store.rs index dd9456e56..9c3a7933a 100644 --- a/nativelink-store/src/cache_metrics_store.rs +++ b/nativelink-store/src/cache_metrics_store.rs @@ -101,6 +101,11 @@ impl MetricsComponent for CacheMetricsStore { #[async_trait] impl StoreDriver for CacheMetricsStore { + async fn post_init(self: Arc) -> Result<(), Error> { + self.backend.clone().into_inner().post_init().await?; + Ok(()) + } + async fn has_with_results( self: Pin<&Self>, keys: &[StoreKey<'_>], diff --git a/nativelink-store/src/common_s3_utils.rs b/nativelink-store/src/common_s3_utils.rs index 9dec3497a..c1c0bd100 100644 --- a/nativelink-store/src/common_s3_utils.rs +++ b/nativelink-store/src/common_s3_utils.rs @@ -47,6 +47,10 @@ use nativelink_util::fs; use nativelink_util::retry::{Retrier, RetryResult}; use tokio::time::sleep; +pub(crate) fn install_default_rustls_crypto_provider() { + drop(rustls::crypto::ring::default_provider().install_default()); +} + #[derive(Clone)] pub struct TlsClient { client: LegacyClient, SdkBody>, @@ -56,6 +60,8 @@ pub struct TlsClient { impl TlsClient { #[must_use] pub fn new(common: &CommonObjectSpec) -> Self { + install_default_rustls_crypto_provider(); + let connector_with_roots = HttpsConnectorBuilder::new().with_platform_verifier(); let connector_with_schemes = if common.insecure_allow_http { @@ -77,6 +83,8 @@ impl TlsClient { common: &CommonObjectSpec, connector: HttpsConnector, ) -> Self { + install_default_rustls_crypto_provider(); + let client = LegacyClient::builder(TokioExecutor::new()).build(connector); Self { diff --git a/nativelink-store/src/completeness_checking_store.rs b/nativelink-store/src/completeness_checking_store.rs index 8f2982120..a7f81f807 100644 --- a/nativelink-store/src/completeness_checking_store.rs +++ b/nativelink-store/src/completeness_checking_store.rs @@ -18,7 +18,7 @@ use std::sync::Arc; use async_trait::async_trait; use futures::stream::{FuturesUnordered, StreamExt}; -use futures::{FutureExt, TryFutureExt, select}; +use futures::{FutureExt, TryFutureExt, select, try_join}; use nativelink_error::{Code, Error, ResultExt, make_err}; use nativelink_metric::MetricsComponent; use nativelink_proto::build::bazel::remote::execution::v2::{ @@ -341,6 +341,14 @@ impl CompletenessCheckingStore { #[async_trait] impl StoreDriver for CompletenessCheckingStore { + async fn post_init(self: Arc) -> Result<(), Error> { + try_join!( + self.cas_store.clone().into_inner().post_init(), + self.ac_store.clone().into_inner().post_init() + )?; + Ok(()) + } + async fn has_with_results( self: Pin<&Self>, keys: &[StoreKey<'_>], diff --git a/nativelink-store/src/compression_store.rs b/nativelink-store/src/compression_store.rs index 9dfe5dcbb..ab85d320d 100644 --- a/nativelink-store/src/compression_store.rs +++ b/nativelink-store/src/compression_store.rs @@ -278,6 +278,11 @@ impl CompressionStore { #[async_trait] impl StoreDriver for CompressionStore { + async fn post_init(self: Arc) -> Result<(), Error> { + self.inner_store.clone().into_inner().post_init().await?; + Ok(()) + } + async fn has_with_results( self: Pin<&Self>, digests: &[StoreKey<'_>], diff --git a/nativelink-store/src/dedup_store.rs b/nativelink-store/src/dedup_store.rs index 236a18f9f..bde8ed7c5 100644 --- a/nativelink-store/src/dedup_store.rs +++ b/nativelink-store/src/dedup_store.rs @@ -18,6 +18,7 @@ use std::sync::Arc; use async_trait::async_trait; use futures::stream::{self, FuturesOrdered, StreamExt, TryStreamExt}; +use futures::try_join; use nativelink_config::stores::DedupSpec; use nativelink_error::{Code, Error, ResultExt, make_err}; use nativelink_metric::MetricsComponent; @@ -176,6 +177,14 @@ impl DedupStore { #[async_trait] impl StoreDriver for DedupStore { + async fn post_init(self: Arc) -> Result<(), Error> { + try_join!( + self.index_store.clone().into_inner().post_init(), + self.content_store.clone().into_inner().post_init(), + )?; + Ok(()) + } + async fn has_with_results( self: Pin<&Self>, digests: &[StoreKey<'_>], diff --git a/nativelink-store/src/existence_cache_store.rs b/nativelink-store/src/existence_cache_store.rs index a0f4de894..a26b5796e 100644 --- a/nativelink-store/src/existence_cache_store.rs +++ b/nativelink-store/src/existence_cache_store.rs @@ -210,6 +210,11 @@ impl ExistenceCacheStore { #[async_trait] impl StoreDriver for ExistenceCacheStore { + async fn post_init(self: Arc) -> Result<(), Error> { + self.inner_store.clone().into_inner().post_init().await?; + Ok(()) + } + async fn has_with_results( self: Pin<&Self>, digests: &[StoreKey<'_>], diff --git a/nativelink-store/src/fast_slow_store.rs b/nativelink-store/src/fast_slow_store.rs index a734edec4..d7ae49a05 100644 --- a/nativelink-store/src/fast_slow_store.rs +++ b/nativelink-store/src/fast_slow_store.rs @@ -24,7 +24,7 @@ use std::sync::{Arc, Weak}; use async_trait::async_trait; use futures::stream::{FuturesUnordered, StreamExt}; -use futures::{FutureExt, join}; +use futures::{FutureExt, join, try_join}; use nativelink_config::stores::{FastSlowSpec, StoreDirection}; use nativelink_error::{Code, Error, ErrorContext, ResultExt, make_err}; use nativelink_metric::MetricsComponent; @@ -428,6 +428,14 @@ impl FastSlowStore { #[async_trait] impl StoreDriver for FastSlowStore { + async fn post_init(self: Arc) -> Result<(), Error> { + try_join!( + self.fast_store.clone().into_inner().post_init(), + self.slow_store.clone().into_inner().post_init() + )?; + Ok(()) + } + async fn has_with_results( self: Pin<&Self>, key: &[StoreKey<'_>], diff --git a/nativelink-store/src/filesystem_store.rs b/nativelink-store/src/filesystem_store.rs index 19f5d01de..3624e17c7 100644 --- a/nativelink-store/src/filesystem_store.rs +++ b/nativelink-store/src/filesystem_store.rs @@ -1260,6 +1260,10 @@ impl FilesystemStore { #[async_trait] impl StoreDriver for FilesystemStore { + async fn post_init(self: Arc) -> Result<(), Error> { + Ok(()) + } + async fn has_with_results( self: Pin<&Self>, keys: &[StoreKey<'_>], diff --git a/nativelink-store/src/gcs_client/client.rs b/nativelink-store/src/gcs_client/client.rs index d2ee2993f..5e812ea7a 100644 --- a/nativelink-store/src/gcs_client/client.rs +++ b/nativelink-store/src/gcs_client/client.rs @@ -35,6 +35,7 @@ use rand::Rng; use tokio::sync::{OwnedSemaphorePermit, Semaphore}; use tokio::time::sleep; +use crate::common_s3_utils::install_default_rustls_crypto_provider; use crate::gcs_client::types::{ CHUNK_SIZE, DEFAULT_CONCURRENT_UPLOADS, DEFAULT_CONTENT_TYPE, GcsObject, INITIAL_UPLOAD_RETRY_DELAY_MS, MAX_UPLOAD_RETRIES, MAX_UPLOAD_RETRY_DELAY_MS, ObjectPath, @@ -109,6 +110,8 @@ pub struct GcsClient { impl GcsClient { fn create_client_config(spec: &ExperimentalGcsSpec) -> Result { + install_default_rustls_crypto_provider(); + let mut client_config = ClientConfig::default(); let connect_timeout = if spec.connection_timeout_s > 0 { Duration::from_secs(spec.connection_timeout_s) diff --git a/nativelink-store/src/gcs_store.rs b/nativelink-store/src/gcs_store.rs index 14e92a70a..247ced282 100644 --- a/nativelink-store/src/gcs_store.rs +++ b/nativelink-store/src/gcs_store.rs @@ -206,6 +206,10 @@ where Client: GcsOperations + 'static, NowFn: Fn() -> I + Send + Sync + Unpin + 'static, { + async fn post_init(self: Arc) -> Result<(), Error> { + Ok(()) + } + async fn has_with_results( self: Pin<&Self>, keys: &[StoreKey<'_>], diff --git a/nativelink-store/src/grpc_store.rs b/nativelink-store/src/grpc_store.rs index c2ff75ff7..40edc4dc0 100644 --- a/nativelink-store/src/grpc_store.rs +++ b/nativelink-store/src/grpc_store.rs @@ -703,6 +703,10 @@ impl GrpcStore { #[async_trait] impl StoreDriver for GrpcStore { + async fn post_init(self: Arc) -> Result<(), Error> { + Ok(()) + } + // NOTE: This function can only be safely used on CAS stores. AC stores may return a size that // is incorrect. async fn has_with_results( diff --git a/nativelink-store/src/memory_store.rs b/nativelink-store/src/memory_store.rs index b8b5009af..b0d80bf14 100644 --- a/nativelink-store/src/memory_store.rs +++ b/nativelink-store/src/memory_store.rs @@ -92,6 +92,10 @@ impl MemoryStore { #[async_trait] impl StoreDriver for MemoryStore { + async fn post_init(self: Arc) -> Result<(), Error> { + Ok(()) + } + async fn has_with_results( self: Pin<&Self>, keys: &[StoreKey<'_>], diff --git a/nativelink-store/src/mongo_store.rs b/nativelink-store/src/mongo_store.rs index 5db444ac5..3f784c7fd 100644 --- a/nativelink-store/src/mongo_store.rs +++ b/nativelink-store/src/mongo_store.rs @@ -313,6 +313,10 @@ impl ExperimentalMongoStore { #[async_trait] impl StoreDriver for ExperimentalMongoStore { + async fn post_init(self: Arc) -> Result<(), Error> { + Ok(()) + } + async fn has_with_results( self: Pin<&Self>, keys: &[StoreKey<'_>], diff --git a/nativelink-store/src/noop_store.rs b/nativelink-store/src/noop_store.rs index 025ac82e5..b7aaa7a8d 100644 --- a/nativelink-store/src/noop_store.rs +++ b/nativelink-store/src/noop_store.rs @@ -47,6 +47,10 @@ impl NoopStore { #[async_trait] impl StoreDriver for NoopStore { + async fn post_init(self: Arc) -> Result<(), Error> { + Ok(()) + } + async fn has_with_results( self: Pin<&Self>, _keys: &[StoreKey<'_>], diff --git a/nativelink-store/src/ontap_s3_existence_cache_store.rs b/nativelink-store/src/ontap_s3_existence_cache_store.rs index b220f4a94..264e18660 100644 --- a/nativelink-store/src/ontap_s3_existence_cache_store.rs +++ b/nativelink-store/src/ontap_s3_existence_cache_store.rs @@ -442,6 +442,11 @@ where I: InstantWrapper, NowFn: Fn() -> I + Send + Sync + Unpin + Clone + 'static, { + async fn post_init(self: Arc) -> Result<(), Error> { + self.inner_store.clone().into_inner().post_init().await?; + Ok(()) + } + async fn has_with_results( self: Pin<&Self>, keys: &[StoreKey<'_>], diff --git a/nativelink-store/src/ontap_s3_store.rs b/nativelink-store/src/ontap_s3_store.rs index 373398464..42335d0f3 100644 --- a/nativelink-store/src/ontap_s3_store.rs +++ b/nativelink-store/src/ontap_s3_store.rs @@ -57,7 +57,7 @@ use tokio::time::sleep; use tracing::{Level, event, warn}; use crate::cas_utils::is_zero_digest; -use crate::common_s3_utils::TlsClient; +use crate::common_s3_utils::{TlsClient, install_default_rustls_crypto_provider}; // S3 parts cannot be smaller than this number const MIN_MULTIPART_SIZE: u64 = 5 * 1024 * 1024; // 5MB @@ -129,6 +129,8 @@ where NowFn: Fn() -> I + Send + Sync + Unpin + 'static, { pub async fn new(spec: &ExperimentalOntapS3Spec, now_fn: NowFn) -> Result, Error> { + install_default_rustls_crypto_provider(); + // Load custom CA config let ca_config = if let Some(cert_path) = &spec.root_certificates { load_custom_certs(cert_path)? @@ -292,6 +294,10 @@ where I: InstantWrapper, NowFn: Fn() -> I + Send + Sync + Unpin + 'static, { + async fn post_init(self: Arc) -> Result<(), Error> { + Ok(()) + } + async fn has_with_results( self: Pin<&Self>, keys: &[StoreKey<'_>], diff --git a/nativelink-store/src/redis_store.rs b/nativelink-store/src/redis_store.rs index fc3729570..78e29075c 100644 --- a/nativelink-store/src/redis_store.rs +++ b/nativelink-store/src/redis_store.rs @@ -783,6 +783,10 @@ where C: ConnectionLike + Clone + Send + Sync + Unpin + 'static, M: RedisManager + Unpin + Send + Sync + 'static, { + async fn post_init(self: Arc) -> Result<(), Error> { + Ok(()) + } + async fn has_with_results( self: Pin<&Self>, keys: &[StoreKey<'_>], @@ -1019,7 +1023,7 @@ where .await { Ok(_) => {}, Err(err) - if err.kind() == redis::ErrorKind::Server(redis::ServerErrorKind::ReadOnly) => + if is_retryable_redis_error(&err) => { let (mut connection_manager, _connect_id) = self.connection_manager.reconnect(connect_id).await?; connection_manager @@ -1051,7 +1055,7 @@ where let expected_len = usize::try_from(total_len).unwrap_or(usize::MAX); - // The chunk writes above reconnect on ReadOnly, so on a mid-write Redis + // The chunk writes above reconnect on any transient failover error, so on a mid-write Redis // failover the data lands on the *current* master. The length check and // rename below must run against that same master: the connection // captured before the writes may now point at a demoted replica, where @@ -1098,16 +1102,14 @@ where }; // Rename the temp key so that the data appears under the real key. Any data already present in the real key is lost. - // Reconnect once on ReadOnly in case the master moved between the verify and here. + // Reconnect once on a transient failover error in case the master moved between the verify and here. match client .connection_manager .rename::<_, _, ()>(&temp_key, final_key.as_ref()) .await { Ok(()) => {} - Err(err) - if err.kind() == redis::ErrorKind::Server(redis::ServerErrorKind::ReadOnly) => - { + Err(err) if is_retryable_redis_error(&err) => { let (connection_manager, uuid) = self.connection_manager.reconnect(client.uuid).await?; client.connection_manager = connection_manager; @@ -1835,9 +1837,7 @@ where .await { Ok(v) => v, - Err(err) - if err.kind() == redis::ErrorKind::Server(redis::ServerErrorKind::ReadOnly) => - { + Err(err) if is_retryable_redis_error(&err) => { client.reconnect(&self.connection_manager).await?; script_invocation .invoke_async(&mut client.connection_manager) @@ -1922,9 +1922,7 @@ where } } } - Err(err) - if err.kind() == redis::ErrorKind::Server(redis::ServerErrorKind::ReadOnly) => - { + Err(err) if is_retryable_redis_error(&err) => { client.reconnect(&self.connection_manager).await?; client .connection_manager @@ -2048,10 +2046,7 @@ where Err(_) => { let (connection_manager, result) = match run_ft_create(connection_manager.clone()).await { - Err(err) - if err.kind() - == redis::ErrorKind::Server(redis::ServerErrorKind::ReadOnly) => - { + Err(err) if is_retryable_redis_error(&err) => { let (connection_manager, _connect_id) = self.connection_manager.reconnect(connect_id).await?; ( @@ -2194,14 +2189,36 @@ where let key = key.get_key(); let key = self.encode_key(&key); let mut client = self.get_client().await?; - let results: Vec = client - .connection_manager - .hmget::<_, Vec, Vec>( - key.as_ref(), - vec![VERSION_FIELD_NAME.into(), DATA_FIELD_NAME.into()], - ) - .await - .err_tip(|| format!("In RedisStore::get_without_version::notversioned {key}"))?; + // hmget is idempotent, so re-resolve the master and retry on a transient + // failover (matching the read paths in get_part/list) instead of failing + // a scheduler-state read while the master is moving. + let results: Vec = { + let mut attempt: u32 = 0; + loop { + attempt += 1; + match client + .connection_manager + .hmget::<_, Vec, Vec>( + key.as_ref(), + vec![VERSION_FIELD_NAME.into(), DATA_FIELD_NAME.into()], + ) + .await + { + Ok(v) => break v, + Err(err) + if attempt < MAX_REDIS_RETRY_ATTEMPTS && is_retryable_redis_error(&err) => + { + client.reconnect(&self.connection_manager).await?; + sleep(Duration::from_secs_f32(DEFAULT_RETRY_DELAY)).await; + } + Err(err) => { + return Err(Error::from(err).append(format!( + "In RedisStore::get_without_version::notversioned {key}" + ))); + } + } + } + }; let Some(Value::BulkString(data)) = results.get(1) else { return Ok(None); }; diff --git a/nativelink-store/src/ref_store.rs b/nativelink-store/src/ref_store.rs index a764853a2..c6100db8f 100644 --- a/nativelink-store/src/ref_store.rs +++ b/nativelink-store/src/ref_store.rs @@ -26,7 +26,7 @@ use nativelink_util::store_trait::{ RemoveItemCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, }; use parking_lot::Mutex; -use tracing::error; +use tracing::{debug, error}; use crate::store_manager::StoreManager; @@ -64,13 +64,6 @@ impl RefStore { }) } - // This will get the store or populate it if needed. It is designed to be quite fast on the - // common path, but slow on the uncommon path. It does use some unsafe functions because we - // wanted it to be fast. It is technically possible on some platforms for this function to - // create a data race here is the reason I do not believe it is an issue: - // 1. It would only happen on the very first call of the function (after first call we are safe) - // 2. It should only happen on platforms that are < 64 bit address space - // 3. It is likely that the internals of how Option work protect us anyway. #[inline] fn get_store(&self) -> Result<&Store, Error> { let ref_store = self.inner.cell.0.get(); @@ -79,8 +72,23 @@ impl RefStore { return Ok(store); } } - // This should protect us against multiple writers writing the same location at the same - // time. + Err(make_input_err!( + "ref_store cannot get store '{}', was post_init called?", + self.name + )) + } +} + +#[async_trait] +impl StoreDriver for RefStore { + async fn post_init(self: Arc) -> Result<(), Error> { + debug!("Running post_init to get store"); + let ref_store = self.inner.cell.0.get(); + unsafe { + if (*ref_store).is_some() { + return Err(make_input_err!("post_init already called for RefStore")); + } + } let _lock = self.inner.mux.lock(); let store_manager = self .store_manager @@ -89,22 +97,21 @@ impl RefStore { if let Some(store) = store_manager.get_store(&self.name) { let remove_callbacks = self.remove_callbacks.lock().clone(); for callback in remove_callbacks { + debug!(?callback, ?store, "Added callback to store"); store.register_remove_callback(callback)?; } unsafe { *ref_store = Some(store); - return Ok((*ref_store).as_ref().unwrap()); } + Ok(()) + } else { + Err(make_input_err!( + "Failed to find store '{}' in StoreManager in RefStore", + self.name + )) } - Err(make_input_err!( - "Failed to find store '{}' in StoreManager in RefStore", - self.name - )) } -} -#[async_trait] -impl StoreDriver for RefStore { async fn has_with_results( self: Pin<&Self>, keys: &[StoreKey<'_>], @@ -138,7 +145,7 @@ impl StoreDriver for RefStore { match self.get_store() { Ok(store) => store.inner_store(key), Err(err) => { - error!(?key, ?err, "Failed to get store for key",); + error!(?key, ?err, "Failed to get store for key"); self } } @@ -160,7 +167,10 @@ impl StoreDriver for RefStore { let ref_store = self.inner.cell.0.get(); unsafe { if let Some(ref store) = *ref_store { + debug!(?callback, ?store, "New callback"); store.register_remove_callback(callback)?; + } else { + debug!(?callback, "New callback, no store yet"); } } Ok(()) diff --git a/nativelink-store/src/s3_store.rs b/nativelink-store/src/s3_store.rs index 23c5e709b..5c2d3837e 100644 --- a/nativelink-store/src/s3_store.rs +++ b/nativelink-store/src/s3_store.rs @@ -243,6 +243,10 @@ where I: InstantWrapper, NowFn: Fn() -> I + Send + Sync + Unpin + 'static, { + async fn post_init(self: Arc) -> Result<(), Error> { + Ok(()) + } + async fn has_with_results( self: Pin<&Self>, keys: &[StoreKey<'_>], diff --git a/nativelink-store/src/shard_store.rs b/nativelink-store/src/shard_store.rs index 1d1740c8b..6855d3a96 100644 --- a/nativelink-store/src/shard_store.rs +++ b/nativelink-store/src/shard_store.rs @@ -19,6 +19,7 @@ use std::hash::DefaultHasher; use std::sync::Arc; use async_trait::async_trait; +use futures::future::try_join_all; use futures::stream::{FuturesUnordered, TryStreamExt}; use nativelink_config::stores::ShardSpec; use nativelink_error::{Error, ResultExt, error_if}; @@ -146,6 +147,15 @@ impl ShardStore { #[async_trait] impl StoreDriver for ShardStore { + async fn post_init(self: Arc) -> Result<(), Error> { + let mut futures = vec![]; + for store_and_weight in &self.weights_and_stores { + futures.push(store_and_weight.store.clone().into_inner().post_init()); + } + try_join_all(futures).await?; + Ok(()) + } + async fn has_with_results( self: Pin<&Self>, keys: &[StoreKey<'_>], diff --git a/nativelink-store/src/size_partitioning_store.rs b/nativelink-store/src/size_partitioning_store.rs index 39ae34814..883530078 100644 --- a/nativelink-store/src/size_partitioning_store.rs +++ b/nativelink-store/src/size_partitioning_store.rs @@ -16,6 +16,7 @@ use core::pin::Pin; use std::sync::Arc; use async_trait::async_trait; +use futures::try_join; use nativelink_config::stores::SizePartitioningSpec; use nativelink_error::{Error, ResultExt, make_input_err}; use nativelink_metric::MetricsComponent; @@ -48,6 +49,14 @@ impl SizePartitioningStore { #[async_trait] impl StoreDriver for SizePartitioningStore { + async fn post_init(self: Arc) -> Result<(), Error> { + try_join!( + self.upper_store.clone().into_inner().post_init(), + self.lower_store.clone().into_inner().post_init(), + )?; + Ok(()) + } + async fn has_with_results( self: Pin<&Self>, keys: &[StoreKey<'_>], diff --git a/nativelink-store/src/store_manager.rs b/nativelink-store/src/store_manager.rs index 0857e43bc..9920a012c 100644 --- a/nativelink-store/src/store_manager.rs +++ b/nativelink-store/src/store_manager.rs @@ -14,6 +14,7 @@ use std::collections::HashMap; +use nativelink_error::Error; use nativelink_metric::{MetricsComponent, RootMetricsComponent}; use nativelink_util::store_trait::Store; use parking_lot::RwLock; @@ -43,6 +44,17 @@ impl StoreManager { } None } + + pub async fn run_post_init(&self) -> Result<(), Error> { + let stores = { + let lock = self.stores.read(); + lock.values().cloned().collect::>() + }; + for store in stores { + store.into_inner().post_init().await?; + } + Ok(()) + } } impl RootMetricsComponent for StoreManager {} diff --git a/nativelink-store/src/verify_store.rs b/nativelink-store/src/verify_store.rs index 7f645f61a..e3722ff48 100644 --- a/nativelink-store/src/verify_store.rs +++ b/nativelink-store/src/verify_store.rs @@ -146,6 +146,11 @@ impl VerifyStore { #[async_trait] impl StoreDriver for VerifyStore { + async fn post_init(self: Arc) -> Result<(), Error> { + self.inner_store.clone().into_inner().post_init().await?; + Ok(()) + } + async fn has_with_results( self: Pin<&Self>, digests: &[StoreKey<'_>], diff --git a/nativelink-store/tests/fast_slow_store_test.rs b/nativelink-store/tests/fast_slow_store_test.rs index 1a54002f5..769c489e6 100644 --- a/nativelink-store/tests/fast_slow_store_test.rs +++ b/nativelink-store/tests/fast_slow_store_test.rs @@ -254,6 +254,10 @@ async fn drop_on_eof_completes_store_futures() -> Result<(), Error> { #[async_trait] impl StoreDriver for DropCheckStore { + async fn post_init(self: Arc) -> Result<(), Error> { + Ok(()) + } + async fn has_with_results( self: Pin<&Self>, digests: &[StoreKey<'_>], @@ -589,6 +593,10 @@ fn make_stores_with_lazy_slow() -> (Store, Store, Store) { #[async_trait] impl StoreDriver for LazyStore { + async fn post_init(self: Arc) -> Result<(), Error> { + Ok(()) + } + async fn has_with_results( self: Pin<&Self>, digests: &[StoreKey<'_>], @@ -728,6 +736,10 @@ struct InstrumentedSlowStore { #[async_trait] impl StoreDriver for InstrumentedSlowStore { + async fn post_init(self: Arc) -> Result<(), Error> { + Ok(()) + } + async fn has_with_results( self: Pin<&Self>, keys: &[StoreKey<'_>], @@ -983,6 +995,10 @@ async fn has_sees_in_flight_slow_writes() -> Result<(), Error> { #[async_trait] impl StoreDriver for GatedSlowStore { + async fn post_init(self: Arc) -> Result<(), Error> { + Ok(()) + } + async fn has_with_results( self: Pin<&Self>, _keys: &[StoreKey<'_>], @@ -1142,6 +1158,10 @@ async fn has_does_not_consult_fast_store_when_slow_store_hits() -> Result<(), Er #[async_trait] impl StoreDriver for CountingFastStore { + async fn post_init(self: Arc) -> Result<(), Error> { + Ok(()) + } + async fn has_with_results( self: Pin<&Self>, keys: &[StoreKey<'_>], @@ -1248,6 +1268,10 @@ struct GatedSlowStore2 { #[async_trait] impl StoreDriver for GatedSlowStore2 { + async fn post_init(self: Arc) -> Result<(), Error> { + Ok(()) + } + async fn has_with_results( self: Pin<&Self>, _keys: &[StoreKey<'_>], @@ -1405,6 +1429,10 @@ struct MapBackedSlow { } #[async_trait] impl StoreDriver for MapBackedSlow { + async fn post_init(self: Arc) -> Result<(), Error> { + Ok(()) + } + async fn has_with_results( self: Pin<&Self>, keys: &[StoreKey<'_>], @@ -1583,6 +1611,10 @@ struct CountingSlowStore { #[async_trait] impl StoreDriver for CountingSlowStore { + async fn post_init(self: Arc) -> Result<(), Error> { + Ok(()) + } + async fn has_with_results( self: Pin<&Self>, keys: &[StoreKey<'_>], @@ -1831,6 +1863,10 @@ struct StaleFastStore { #[async_trait] impl StoreDriver for StaleFastStore { + async fn post_init(self: Arc) -> Result<(), Error> { + Ok(()) + } + async fn has_with_results( self: Pin<&Self>, _digests: &[StoreKey<'_>], diff --git a/nativelink-store/tests/redis_store_test.rs b/nativelink-store/tests/redis_store_test.rs index c870d02c3..d58340bfd 100644 --- a/nativelink-store/tests/redis_store_test.rs +++ b/nativelink-store/tests/redis_store_test.rs @@ -304,6 +304,55 @@ async fn has_retries_after_transient_error() -> Result<(), Error> { Ok(()) } +// The write path must also ride out a transient Redis error (e.g. a connection +// dropped by a Sentinel failover) by re-resolving the master and retrying the +// chunk write, rather than hard-failing the upload. Previously the chunk write +// only retried a ReadOnly reply (a demoted replica), so a dropped connection — +// the more common failover symptom — failed the upload and dropped the data. +#[nativelink_test] +async fn update_retries_after_transient_write_error() -> Result<(), Error> { + let data = Bytes::from_static(b"14"); + let digest = DigestInfo::try_new(VALID_HASH1, 2)?; + let packed_hash_hex = format!("{digest}"); + let temp_key = make_temp_key(&packed_hash_hex); + let real_key = packed_hash_hex; + + let commands = vec![ + // First chunk write fails with a retryable connection error. + MockCmd::new( + redis::cmd("SETRANGE") + .arg(temp_key.clone()) + .arg(0) + .arg(data.to_vec()), + Err::(RedisError::from(std::io::Error::new( + std::io::ErrorKind::ConnectionReset, + "transient", + ))), + ), + // After re-resolving the master, the retried chunk write succeeds. + MockCmd::new( + redis::cmd("SETRANGE") + .arg(temp_key.clone()) + .arg(0) + .arg(data.to_vec()), + Ok(Value::Int(0)), + ), + MockCmd::new( + redis::cmd("STRLEN").arg(temp_key.clone()), + Ok(Value::Int(data.len() as i64)), + ), + MockCmd::new( + redis::cmd("RENAME").arg(temp_key).arg(real_key), + Ok(Value::Nil), + ), + ]; + + let store = make_mock_store(commands).await; + // Must succeed despite the transient write error. + store.update_oneshot(digest, data).await?; + Ok(()) +} + #[nativelink_test] async fn upload_and_get_data_with_prefix() -> Result<(), Error> { let data = Bytes::from_static(b"14"); @@ -794,13 +843,16 @@ async fn test_health() { struct_name, "nativelink_store::redis_store::RedisStore>" ); + // The write path treats a timeout as retryable (a failover symptom), + // so it re-resolves the master and retries the chunk write once before + // surfacing the error — hence the "(after reconnect)" prefix. assert!( - message.starts_with("Store.update_oneshot() failed: Error { code: DeadlineExceeded, messages: [\"Io: timed out\", \"While appending to temp key ("), + message.starts_with("Store.update_oneshot() failed: Error { code: DeadlineExceeded, messages: [\"Io: timed out\", \"(after reconnect) while appending to temp key ("), "message: '{message}'" ); logs_assert(|logs| { for log in logs { - if log.contains("check_health Store.update_oneshot() failed e=Error { code: DeadlineExceeded, messages: [\"Io: timed out\", \"While appending to temp key (") { + if log.contains("check_health Store.update_oneshot() failed e=Error { code: DeadlineExceeded, messages: [\"Io: timed out\", \"(after reconnect) while appending to temp key (") { return Ok(()) } } diff --git a/nativelink-store/tests/ref_store_test.rs b/nativelink-store/tests/ref_store_test.rs index fb405f891..32cf5c32d 100644 --- a/nativelink-store/tests/ref_store_test.rs +++ b/nativelink-store/tests/ref_store_test.rs @@ -24,10 +24,11 @@ use nativelink_store::store_manager::StoreManager; use nativelink_util::common::DigestInfo; use nativelink_util::store_trait::{Store, StoreDriver, StoreLike}; use pretty_assertions::assert_eq; +use tonic::Code; const VALID_HASH1: &str = "0123456789abcdef000000000000000000010000000000000123456789abcdef"; -fn setup_stores() -> (Arc, Store, Store) { +async fn setup_stores() -> (Arc, Store, Store) { let store_manager = Arc::new(StoreManager::new()); let memory_store = Store::new(MemoryStore::new(&MemorySpec::default())); @@ -40,6 +41,7 @@ fn setup_stores() -> (Arc, Store, Store) { Arc::downgrade(&store_manager), )); store_manager.add_store("bar", ref_store.clone()); + store_manager.run_post_init().await.unwrap(); (store_manager, memory_store, ref_store) } @@ -47,7 +49,7 @@ fn setup_stores() -> (Arc, Store, Store) { async fn has_test() -> Result<(), Error> { const VALUE1: &str = "13"; - let (_store_manager, memory_store, ref_store) = setup_stores(); + let (_store_manager, memory_store, ref_store) = setup_stores().await; { // Insert data into memory store. @@ -77,7 +79,7 @@ async fn has_test() -> Result<(), Error> { async fn get_test() -> Result<(), Error> { const VALUE1: &str = "13"; - let (_store_manager, memory_store, ref_store) = setup_stores(); + let (_store_manager, memory_store, ref_store) = setup_stores().await; { // Insert data into memory store. @@ -108,7 +110,7 @@ async fn get_test() -> Result<(), Error> { async fn update_test() -> Result<(), Error> { const VALUE1: &str = "13"; - let (_store_manager, memory_store, ref_store) = setup_stores(); + let (_store_manager, memory_store, ref_store) = setup_stores().await; { // Insert data into ref_store. @@ -158,6 +160,8 @@ async fn inner_store_test() -> Result<(), Error> { )); store_manager.add_store("ref_store_outer", ref_store_outer.clone()); + store_manager.run_post_init().await.unwrap(); + // Ensure the result of inner_store() points to exact same memory store. assert_eq!( from_ref::(ref_store_outer.inner_store(Option::::None)) @@ -167,3 +171,29 @@ async fn inner_store_test() -> Result<(), Error> { ); Ok(()) } + +#[nativelink_test] +async fn no_post_init_failure() -> Result<(), Error> { + let store_manager = Arc::new(StoreManager::new()); + + let memory_store = Store::new(MemoryStore::new(&MemorySpec::default())); + store_manager.add_store("mem_store", memory_store.clone()); + + let ref_store = Store::new(RefStore::new( + &RefSpec { + name: "mem_store".to_string(), + }, + Arc::downgrade(&store_manager), + )); + store_manager.add_store("ref_store", ref_store.clone()); + + // Should fail because we never called post_init + assert_eq!( + ref_store.has(DigestInfo::try_new(VALID_HASH1, 1)?).await, + Err(Error::new( + Code::InvalidArgument, + "ref_store cannot get store 'mem_store', was post_init called?".into() + )) + ); + Ok(()) +} diff --git a/nativelink-test/fuzz/Cargo.lock b/nativelink-test/fuzz/Cargo.lock new file mode 100644 index 000000000..9096df349 --- /dev/null +++ b/nativelink-test/fuzz/Cargo.lock @@ -0,0 +1,2523 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" + +[[package]] +name = "arcstr" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03918c3dbd7701a85c6b9887732e2921175f26c350b4563841d0958c21d57e6d" + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bson" +version = "2.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969a9ba84b0ff843813e7249eed1678d9b6607ce5a3b8f0a47af3fcf7978e6e" +dependencies = [ + "ahash", + "base64", + "bitvec", + "getrandom 0.2.17", + "getrandom 0.3.4", + "hex", + "indexmap", + "js-sys", + "once_cell", + "rand", + "serde", + "serde_bytes", + "serde_json", + "time", + "uuid", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "byte-unit" +version = "5.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37bcaa4a0975bed4a760af3efe4368825098ce5f9d37a30c5a021d635dc63d8f" +dependencies = [ + "rust_decimal", + "utf8-width", +] + +[[package]] +name = "bytes" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" + +[[package]] +name = "cc" +version = "1.2.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dad887fd958be91b5098c0248def011f4523ab786cd411be668777e55063501f" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "convert_case" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" + +[[package]] +name = "derive-syn-parse" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d65d7ce8132b7c0e54497a4d9a55a1c2a0912a0d786cf894472ba818fba45762" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "derive-where" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d08b3a0bcc0d079199cd476b2cae8435016ec11d1c0986c6901c5ac223041534" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "rustc_version", + "syn", + "unicode-xid", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "either" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "event-listener" +version = "5.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-core", + "futures-io", + "futures-macro", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 5.3.0", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", +] + +[[package]] +name = "h2" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "http" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "humantime" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" + +[[package]] +name = "hyper" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2 0.6.4", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libfuzzer-sys" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9fd2f41a1cba099f79a0b6b6c35656cf7c03351a7bae8ff0f28f25270f929d2" +dependencies = [ + "arbitrary", + "cc", +] + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" + +[[package]] +name = "macro_magic" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc33f9f0351468d26fbc53d9ce00a096c8522ecb42f19b50f34f2c422f76d21d" +dependencies = [ + "macro_magic_core", + "macro_magic_macros", + "quote", + "syn", +] + +[[package]] +name = "macro_magic_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1687dc887e42f352865a393acae7cf79d98fab6351cde1f58e9e057da89bf150" +dependencies = [ + "const-random", + "derive-syn-parse", + "macro_magic_core_macros", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "macro_magic_core_macros" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b02abfe41815b5bd98dbd4260173db2c116dda171dc0fe7838cb206333b83308" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "macro_magic_macros" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ea28ee64b88876bf45277ed9a5817c1817df061a74f2b988971a12570e5869" +dependencies = [ + "macro_magic_core", + "quote", + "syn", +] + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + +[[package]] +name = "memchr" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" + +[[package]] +name = "mio" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "mongocrypt" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8da0cd419a51a5fb44819e290fbdb0665a54f21dead8923446a799c7f4d26ad9" +dependencies = [ + "bson", + "mongocrypt-sys", + "once_cell", + "serde", +] + +[[package]] +name = "mongocrypt-sys" +version = "0.1.5+1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224484c5d09285a7b8cb0a0c117e847ebd14cb6e4470ecf68cdb89c503b0edb9" + +[[package]] +name = "mongodb" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "276ba0cd571553d1f6936c6f180964776ece6ab7507dc8765f8a9c9c49d8cd00" +dependencies = [ + "base64", + "bitflags", + "bson", + "derive-where", + "derive_more", + "futures-core", + "futures-io", + "futures-util", + "hex", + "hmac", + "macro_magic", + "md-5", + "mongocrypt", + "mongodb-internal-macros", + "pbkdf2", + "percent-encoding", + "rand", + "rustc_version_runtime", + "rustls", + "serde", + "serde_bytes", + "serde_with", + "sha1", + "sha2", + "socket2 0.6.4", + "stringprep", + "strsim", + "take_mut", + "thiserror", + "tokio", + "tokio-rustls", + "tokio-util", + "typed-builder", + "uuid", + "webpki-roots", +] + +[[package]] +name = "mongodb-internal-macros" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99ceb1a9a1018e470077ec94cf3a8c2d0e6da542b2c05ea95a59a0a627147375" +dependencies = [ + "macro_magic", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "nativelink-config" +version = "1.5.2" +dependencies = [ + "byte-unit", + "humantime", + "nativelink-error", + "rand", + "serde", + "serde_json", + "serde_json5", + "shellexpand", + "tracing", +] + +[[package]] +name = "nativelink-error" +version = "1.5.2" +dependencies = [ + "mongodb", + "nativelink-metric", + "nativelink-proto", + "prost", + "prost-types", + "redis", + "reqwest", + "rustls-pki-types", + "serde", + "serde_json5", + "tokio", + "tonic", + "url", + "uuid", + "walkdir", + "zip", +] + +[[package]] +name = "nativelink-fuzz" +version = "0.0.0" +dependencies = [ + "libfuzzer-sys", + "nativelink-config", + "serde_json5", +] + +[[package]] +name = "nativelink-metric" +version = "1.5.2" +dependencies = [ + "async-lock", + "nativelink-metric-macro-derive", + "parking_lot", + "tokio", + "tracing", +] + +[[package]] +name = "nativelink-metric-macro-derive" +version = "1.5.2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "nativelink-proto" +version = "1.5.2" +dependencies = [ + "derive_more", + "prost", + "prost-types", + "tonic", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pest" +version = "2.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0848c601009d37dfa3430c4666e147e49cdcf1b92ecd3e63657d8a5f19da662" +dependencies = [ + "memchr", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11f486f1ea21e6c10ed15d5a7c77165d0ee443402f0780849d1768e7d9d6fe77" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8040c4647b13b210a963c1ed407c1ff4fdfa01c31d6d2a098218702e6664f94f" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pest_meta" +version = "2.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89815c69d36021a140146f26659a81d6c2afa33d216d736dd4be5381a7362220" +dependencies = [ + "pest", + "sha2", +] + +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "prost-types" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +dependencies = [ + "prost", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "redis" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9fd510128eda94d1d49b9f81487744d5c451422431cce41238fe2853d29f4cc" +dependencies = [ + "arcstr", + "combine", + "itoa", + "percent-encoding", + "ryu", + "socket2 0.6.4", + "url", + "xxhash-rust", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rust_decimal" +version = "1.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be2a24f50780bc85f09cc6ac299bdf1424302742d77221106859c9d8b102126a" +dependencies = [ + "arrayvec", + "num-traits", +] + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustc_version_runtime" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dd18cd2bae1820af0b6ad5e54f4a51d0f3fcc53b05f845675074efcc7af071d" +dependencies = [ + "rustc_version", + "semver", +] + +[[package]] +name = "rustls" +version = "0.23.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "indexmap", + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_json5" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d34d03f54462862f2a42918391c9526337f53171eaa4d8894562be7f252edd3" +dependencies = [ + "pest", + "pest_derive", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "3.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" +dependencies = [ + "serde_core", + "serde_with_macros", +] + +[[package]] +name = "serde_with_macros" +version = "3.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shellexpand" +version = "3.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32824fab5e16e6c4d86dc1ba84489390419a39f97699852b66480bb87d297ed8" + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "socket2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "take_mut" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" + +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "time" +version = "0.3.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711a53c2d47bbd818258c498c8dbfe186a2526c631495cfe7e078567f86b8469" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71c652a3727a9cbb9a02f707f530b618ce00d0ccd762009c8c23bd191df3c17d" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "socket2 0.6.4", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-io", + "futures-sink", + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tonic" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e581ba15a835f4d9ea06c55ab1bd4dce26fc53752c69a04aac00703bfb49ba9" +dependencies = [ + "async-trait", + "base64", + "bytes", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "prost", + "socket2 0.5.10", + "tokio", + "tokio-rustls", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "indexmap", + "pin-project-lite", + "slab", + "sync_wrapper", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typed-builder" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "398a3a3c918c96de527dc11e6e846cd549d4508030b8a33e1da12789c856b81a" +dependencies = [ + "typed-builder-macro", +] + +[[package]] +name = "typed-builder-macro" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e48cea23f68d1f78eb7bc092881b6bb88d3d6b5b7e6234f6f9c911da1ffb221" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "typed-path" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "ucd-trie" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" + +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8-width" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1292c0d970b54115d14f2492fe0170adf21d68a1de108eebc51c1df4f346a091" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "uuid" +version = "1.23.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "144d6b123cef80b301b8f72a9e2ca4370ddec21950d0a103dd22c437006d2db7" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "serde_core", + "wasm-bindgen", +] + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "503b14d284f2c8dac03b819967e155ea753f573586193b2b2c95990cb5d69280" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6430a72df5eb332242960fe84b3002a241163998241eb596d4f739b9757061d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + +[[package]] +name = "xxhash-rust" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zip" +version = "8.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b" +dependencies = [ + "crc32fast", + "indexmap", + "memchr", + "typed-path", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/nativelink-test/fuzz/Cargo.toml b/nativelink-test/fuzz/Cargo.toml new file mode 100644 index 000000000..7d6ca6bde --- /dev/null +++ b/nativelink-test/fuzz/Cargo.toml @@ -0,0 +1,21 @@ +[package] +edition = "2024" +name = "nativelink-fuzz" +publish = false +version = "0.0.0" + +[package.metadata] +cargo-fuzz = true + +[dependencies] +nativelink-config = { path = "../../nativelink-config" } + +libfuzzer-sys = "0.4" +serde_json5 = { version = "0.2.1", default-features = false } + +[[bin]] +bench = false +doc = false +name = "cas_config" +path = "fuzz_targets/cas_config.rs" +test = false diff --git a/nativelink-test/fuzz/fuzz_targets/cas_config.rs b/nativelink-test/fuzz/fuzz_targets/cas_config.rs new file mode 100644 index 000000000..c31bcfbb1 --- /dev/null +++ b/nativelink-test/fuzz/fuzz_targets/cas_config.rs @@ -0,0 +1,10 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use nativelink_config::cas_server::CasConfig; + +fuzz_target!(|data: &[u8]| { + if let Ok(input) = std::str::from_utf8(data) { + let _ = serde_json5::from_str::(input); + } +}); diff --git a/nativelink-util/Cargo.toml b/nativelink-util/Cargo.toml index 8e634a0d4..5ab543189 100644 --- a/nativelink-util/Cargo.toml +++ b/nativelink-util/Cargo.toml @@ -4,7 +4,7 @@ lints.workspace = true [package] edition = "2024" name = "nativelink-util" -version = "1.5.1" +version = "1.5.2" [dependencies] nativelink-config = { path = "../nativelink-config" } diff --git a/nativelink-util/src/origin_event_publisher.rs b/nativelink-util/src/origin_event_publisher.rs index 8903c001e..e8fd7f98c 100644 --- a/nativelink-util/src/origin_event_publisher.rs +++ b/nativelink-util/src/origin_event_publisher.rs @@ -12,12 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. +use core::time::Duration; + use bytes::BytesMut; use futures::{FutureExt, future}; use nativelink_proto::com::github::trace_machina::nativelink::events::{OriginEvent, OriginEvents}; use prost::Message; use tokio::sync::{broadcast, mpsc}; -use tracing::error; +use tokio::time::sleep; +use tracing::{error, warn}; use uuid::{Timestamp, Uuid}; use crate::shutdown_guard::{Priority, ShutdownGuard}; @@ -92,6 +95,10 @@ impl OriginEventPublisher { } async fn handle_batch(&self, batch: &mut Vec) { + // Bounded so a sustained store outage can't block the publisher (and + // thus the schedulers feeding it) forever; enough to ride out a + // Sentinel failover. + const MAX_UPLOAD_ATTEMPTS: u32 = 5; // UUID v6 requires a timestamp and node ID // Create timestamp from current system time with nanosecond precision let now = std::time::SystemTime::now() @@ -119,16 +126,39 @@ impl OriginEventPublisher { error!("Failed to encode origin events: {}", e); return; } - let update_result = self - .store - .as_store_driver_pin() - .update_oneshot( - format!("OriginEvents:{}", uuid.hyphenated()).into(), - data.freeze(), - ) - .await; - if let Err(err) = update_result { - error!("Failed to upload origin events: {}", err); + // The batch has already been drained out of `batch`, so a failed store + // write would silently lose these events (the source of the recurring + // "No action-level resource sizing records" — origin events dropped + // during a transient Redis disruption such as a Sentinel failover). + // Retry the upload, re-resolving the master each time, so a transient + // failure doesn't drop the events. + let key = format!("OriginEvents:{}", uuid.hyphenated()); + let data = data.freeze(); + for attempt in 1..=MAX_UPLOAD_ATTEMPTS { + match self + .store + .as_store_driver_pin() + .update_oneshot(key.clone().into(), data.clone()) + .await + { + Ok(()) => return, + Err(err) if attempt < MAX_UPLOAD_ATTEMPTS => { + warn!( + attempt, + max = MAX_UPLOAD_ATTEMPTS, + ?err, + "Failed to upload origin events, retrying" + ); + sleep(Duration::from_secs_f32(0.1 * attempt as f32)).await; + } + Err(err) => { + error!( + attempts = MAX_UPLOAD_ATTEMPTS, + ?err, + "Failed to upload origin events after retries" + ); + } + } } } } diff --git a/nativelink-util/src/store_trait.rs b/nativelink-util/src/store_trait.rs index 235b74477..942d3a15c 100644 --- a/nativelink-util/src/store_trait.rs +++ b/nativelink-util/src/store_trait.rs @@ -621,6 +621,10 @@ pub trait StoreLike: Send + Sync + Sized + Unpin + 'static { pub trait StoreDriver: Sync + Send + Unpin + MetricsComponent + HealthStatusIndicator + 'static { + // Do "all the stores are setup" init e.g. if we need access to the store manager + // for ref stores + async fn post_init(self: Arc) -> Result<(), Error>; + /// See: [`StoreLike::has`] for details. #[inline] async fn has(self: Pin<&Self>, key: StoreKey<'_>) -> Result, Error> { diff --git a/nativelink-util/tests/origin_event_test.rs b/nativelink-util/tests/origin_event_test.rs index 93b5a5d3c..5404ad278 100644 --- a/nativelink-util/tests/origin_event_test.rs +++ b/nativelink-util/tests/origin_event_test.rs @@ -12,12 +12,143 @@ // See the License for the specific language governing permissions and // limitations under the License. +use core::pin::Pin; +use core::sync::atomic::{AtomicUsize, Ordering}; +use core::time::Duration; +use std::sync::Arc; + +use nativelink_error::{Code, Error, make_err}; use nativelink_macro::nativelink_test; +use nativelink_metric::MetricsComponent; use nativelink_proto::com::github::trace_machina::nativelink::events::{ - Event, RequestEvent, ResponseEvent, StreamEvent, event, request_event, response_event, - stream_event, + Event, OriginEvent, RequestEvent, ResponseEvent, StreamEvent, event, request_event, + response_event, stream_event, }; +use nativelink_util::buf_channel::{DropCloserReadHalf, DropCloserWriteHalf}; +use nativelink_util::default_health_status_indicator; +use nativelink_util::health_utils::HealthStatusIndicator; use nativelink_util::origin_event::get_id_for_event; +use nativelink_util::origin_event_publisher::OriginEventPublisher; +use nativelink_util::store_trait::{ + RemoveItemCallback, Store, StoreDriver, StoreKey, UploadSizeInfo, +}; +use tokio::sync::{broadcast, mpsc}; +use tokio::time::sleep; +use tonic::async_trait; + +/// A store whose `update` fails the first `fail_until` calls, then succeeds — +/// to verify the origin-event publisher retries a transient upload failure +/// instead of dropping the events (the resource-sizing durability gap). +#[derive(Debug, MetricsComponent)] +struct FlakyStore { + fail_until: usize, + attempts: AtomicUsize, +} + +#[async_trait] +#[allow(clippy::todo)] +impl StoreDriver for FlakyStore { + async fn post_init(self: Arc) -> Result<(), Error> { + Ok(()) + } + + async fn has_with_results( + self: Pin<&Self>, + _keys: &[StoreKey<'_>], + _results: &mut [Option], + ) -> Result<(), Error> { + todo!(); + } + + async fn update( + self: Pin<&Self>, + _key: StoreKey<'_>, + mut reader: DropCloserReadHalf, + _upload_size: UploadSizeInfo, + ) -> Result { + // Drain so the writer side completes. + reader.consume(None).await?; + let attempt = self.attempts.fetch_add(1, Ordering::SeqCst) + 1; + if attempt <= self.fail_until { + return Err(make_err!( + Code::Unavailable, + "flaky store failure {attempt}" + )); + } + Ok(0) + } + + async fn get_part( + self: Pin<&Self>, + _key: StoreKey<'_>, + _writer: &mut DropCloserWriteHalf, + _offset: u64, + _length: Option, + ) -> Result<(), Error> { + todo!(); + } + + fn inner_store(&self, _digest: Option) -> &dyn StoreDriver { + self + } + + fn as_any(&self) -> &(dyn core::any::Any + Sync + Send + 'static) { + self + } + + fn as_any_arc(self: Arc) -> Arc { + self + } + + fn register_remove_callback( + self: Arc, + _callback: Arc, + ) -> Result<(), Error> { + todo!(); + } +} + +default_health_status_indicator!(FlakyStore); + +// Regression test for the action-level resource-sizing durability gap: a +// transient store-upload failure (e.g. a Redis Sentinel failover) must NOT drop +// origin events — the publisher retries until the upload succeeds. +#[nativelink_test] +async fn origin_event_publisher_retries_store_upload() { + let store = Arc::new(FlakyStore { + fail_until: 2, + attempts: AtomicUsize::new(0), + }); + let (tx, rx) = mpsc::channel::(16); + let (shutdown_tx, _shutdown_rx) = broadcast::channel(1); + let publisher = OriginEventPublisher::new(Store::new(store.clone()), rx, shutdown_tx); + let task = tokio::spawn(publisher.run()); + + tx.send(OriginEvent { + version: 0, + event_id: "e1".to_string(), + parent_event_id: String::new(), + bazel_request_metadata: None, + identity: String::new(), + event: None, + }) + .await + .unwrap(); + + // 2 transient failures + 1 success = 3 attempts. + for _ in 0..100 { + if store.attempts.load(Ordering::SeqCst) >= 3 { + break; + } + sleep(Duration::from_millis(20)).await; + } + assert_eq!( + store.attempts.load(Ordering::SeqCst), + 3, + "publisher must retry the upload past transient failures instead of dropping events", + ); + task.abort(); +} macro_rules! event_assert { ($event:ident, $val:expr) => { diff --git a/nativelink-util/tests/store_trait_test.rs b/nativelink-util/tests/store_trait_test.rs index ac5cd5197..c09b03838 100644 --- a/nativelink-util/tests/store_trait_test.rs +++ b/nativelink-util/tests/store_trait_test.rs @@ -18,6 +18,10 @@ struct FakeStore {} #[async_trait] #[allow(clippy::todo)] impl StoreDriver for FakeStore { + async fn post_init(self: Arc) -> Result<(), Error> { + Ok(()) + } + async fn has_with_results( self: Pin<&Self>, _keys: &[StoreKey<'_>], diff --git a/nativelink-worker/Cargo.toml b/nativelink-worker/Cargo.toml index bfbe65aa8..e6aebabe0 100644 --- a/nativelink-worker/Cargo.toml +++ b/nativelink-worker/Cargo.toml @@ -4,7 +4,7 @@ lints.workspace = true [package] edition = "2024" name = "nativelink-worker" -version = "1.5.1" +version = "1.5.2" [features] nix = [] diff --git a/run_integration_tests.sh b/run_integration_tests.sh index fd911d2a5..c1009a909 100755 --- a/run_integration_tests.sh +++ b/run_integration_tests.sh @@ -96,6 +96,23 @@ echo "" # New line. export NATIVELINK_DIR="$CACHE_DIR/nativelink" mkdir -p "$NATIVELINK_DIR" +wait_for_tcp() { + local host="$1" + local port="$2" + local deadline=$((SECONDS + 30)) + + until (true > "/dev/tcp/$host/$port") 2> /dev/null; do + if ((SECONDS >= deadline)); then + echo "TCP endpoint $host:$port did not become ready within 30 seconds." + sudo docker compose logs + exit 1 + fi + sleep 1 + done + + echo "TCP endpoint $host:$port is ready." +} + for pattern in "${TEST_PATTERNS[@]}"; do find "$SELF_DIR/integration_tests/" -name "$pattern" -type f -print0 | while IFS= read -r -d $'\0' fullpath; do # Cleanup. @@ -118,6 +135,9 @@ for pattern in "${TEST_PATTERNS[@]}"; do sudo docker compose logs exit 1 fi + wait_for_tcp 127.0.0.1 50051 + wait_for_tcp 127.0.0.1 50052 + wait_for_tcp 127.0.0.1 50071 set +e bash -euo pipefail "$fullpath" EXIT_CODE="$?" diff --git a/src/bin/nativelink.rs b/src/bin/nativelink.rs index a3ca95cea..a36e77d3f 100644 --- a/src/bin/nativelink.rs +++ b/src/bin/nativelink.rs @@ -95,6 +95,10 @@ const DEFAULT_MAX_QUEUE_EVENTS: usize = 0x0001_0000; /// Note: The actual capacity may be greater than the provided capacity. const BROADCAST_CAPACITY: usize = 1; +fn install_default_rustls_crypto_provider() { + drop(tokio_rustls::rustls::crypto::ring::default_provider().install_default()); +} + /// Bind a [`TcpListener`] with `IP_FREEBIND` set. fn bind_freebind(socket_addr: SocketAddr) -> Result { let socket = match socket_addr { @@ -213,6 +217,7 @@ async fn inner_main( .err_tip(|| format!("Failed to create store '{name}'"))?; store_manager.add_store(&name, store); } + store_manager.run_post_init().await?; } let mut root_futures: Vec>> = Vec::new(); @@ -749,6 +754,8 @@ fn get_config() -> Result { } fn main() -> Result<(), Box> { + install_default_rustls_crypto_provider(); + // Set QoS to USER_INITIATED on the main thread *before* the tokio // runtime is built so the spawned worker threads inherit P-core // scheduling preference via pthread QoS inheritance on Apple @@ -824,13 +831,13 @@ fn main() -> Result<(), Box> { .expect("Failed to listen to SIGTERM") .recv() .await; - warn!("Process terminated via SIGTERM",); + warn!("Process terminated via SIGTERM"); drop(shutdown_tx_clone.send(shutdown_guard.clone())); scheduler_shutdown_rx .await .expect("Failed to receive scheduler shutdown"); let () = shutdown_guard.wait_for(Priority::P0).await; - warn!("Successfully shut down nativelink.",); + warn!("Successfully shut down nativelink."); std::process::exit(143); }); diff --git a/tools/public/create-multi-arch-image.nix b/tools/public/create-multi-arch-image.nix new file mode 100644 index 000000000..b59c19b26 --- /dev/null +++ b/tools/public/create-multi-arch-image.nix @@ -0,0 +1,39 @@ +# FIXME: Really this should be using nix2container or skopeo, but they can't do that +# See https://github.com/podman-container-tools/skopeo/issues/1136 +# * docker manifest create fails with permission errors (and no logging about _what_) +# * Podman breaks in CI because fun with permissions +# So we're down to regctl, but better options welcomed +{ + writeShellScriptBin, + regclient, + trivy-report, +}: +writeShellScriptBin "create-multi-arch-image" '' + set -euo pipefail + + REGISTRY=$1 + shift + IMAGE_TARGET=$(echo $1 | tr '[:upper:]' '[:lower:]') + shift + FULL_IMAGE_TARGET=''${REGISTRY}/''${IMAGE_TARGET} + echo "Creating multi-arch image for ''${FULL_IMAGE_TARGET} from: $@" + set -x + + if [ $REGISTRY == "localhost:5000" ]; then + ${regclient}/bin/regctl registry set --tls disabled localhost:5000 + fi + + IMAGES= + + for image in "$@" + do + IMAGE_TAG=$(nix eval .#$image.imageTag --raw) + IMAGE_DIR=$(mktemp -d) + nix run .#$image.copyTo oci:''${IMAGE_DIR}:''${IMAGE_TAG} + IMAGES="$IMAGES --ref ocidir://''${IMAGE_DIR}:''${IMAGE_TAG}" + done + + ${regclient}/bin/regctl -v info index create ''${FULL_IMAGE_TARGET} ''${IMAGES} + + ${trivy-report}/bin/trivy-report ''${FULL_IMAGE_TARGET} +'' diff --git a/tools/public/default.nix b/tools/public/default.nix index 674551644..952e2542a 100644 --- a/tools/public/default.nix +++ b/tools/public/default.nix @@ -2,12 +2,21 @@ inherit (nix2container.packages.${final.stdenv.hostPlatform.system}) nix2container; # Note: Only put tools here that should be usable from external flakes. - nativelink-tools = { + nativelink-tools = let + trivy-report = final.callPackage ./trivy-report.nix {}; + in { local-image-test = final.callPackage ./local-image-test.nix { skopeo = nix2container.packages.${final.stdenv.hostPlatform.system}.skopeo-nix2container; }; - publish-ghcr = final.callPackage ./publish-ghcr.nix {}; + publish-ghcr = final.callPackage ./publish-ghcr.nix {inherit trivy-report;}; create-local-image = final.callPackage ./create-local-image.nix {}; + create-multi-arch-image = final.callPackage ./create-multi-arch-image.nix { + regclient = final.callPackage ../regclient.nix {}; + inherit trivy-report; + }; + regctl-ghcr-login = final.callPackage ./regctl-ghcr-login.nix { + regclient = final.callPackage ../regclient.nix {}; + }; lib = { createWorker = self: diff --git a/tools/public/local-image-test.nix b/tools/public/local-image-test.nix index 498213607..428c52e73 100644 --- a/tools/public/local-image-test.nix +++ b/tools/public/local-image-test.nix @@ -10,34 +10,36 @@ writeShellScriptBin "local-image-test" '' echo "Testing image: $1" set -x - # Commit hashes would not be a good choice here as they are not - # fully dependent on the inputs to the image. For instance, amending - # nothing would still lead to a new hash. Instead we use the - # derivation hash as the tag so that the tag is reused if the image - # didn't change. - IMAGE_TAG=$(nix eval .#$1.imageTag --raw) - IMAGE_NAME=$(nix eval .#$1.imageName --raw) - IMAGE=$(nix eval .#$1 --raw) + if [[ $1 == *:* ]]; then + # Nix targets generally don't have a :, so we assume this is an existing docker image name + IMAGE_TARGET=$1 + else + # Commit hashes would not be a good choice here as they are not + # fully dependent on the inputs to the image. For instance, amending + # nothing would still lead to a new hash. Instead we use the + # derivation hash as the tag so that the tag is reused if the image + # didn't change. + IMAGE_TAG=$(nix eval .#$1.imageTag --raw) + IMAGE_NAME=$(nix eval .#$1.imageName --raw) + IMAGE=$(nix eval .#$1 --raw) + IMAGE_TARGET=''${IMAGE_NAME}:''${IMAGE_TAG} - # Inlining from https://github.com/nlewo/nix2container/blob/76be9608a7f4d6c985d28b0e7be903ae2547df3e/default.nix#L88 - # so we can add parameters to skopeo - # nix run .#$1.copyTo \ - # docker-daemon:''${IMAGE_NAME}:''${IMAGE_TAG} - nix build .#$1 - # Double-check available disk before we run skopeo - df -h - ${skopeo}/bin/skopeo --insecure-policy copy --dest-daemon-host=''${DOCKER_HOST:-unix:///var/run/docker.sock} nix:''${IMAGE} docker-daemon:''${IMAGE_NAME}:''${IMAGE_TAG} + # Inlining from https://github.com/nlewo/nix2container/blob/76be9608a7f4d6c985d28b0e7be903ae2547df3e/default.nix#L88 + # so we can run --debug on skopeo + # nix run .#$1.copyTo docker-daemon:''${IMAGE_TARGET} + nix build .#$1 + ${skopeo}/bin/skopeo --debug --insecure-policy copy nix:''${IMAGE} docker-daemon:''${IMAGE_TARGET} + fi # Ensure that the image has minimal closure size. # TODO(palfrey): The default allows 10% inefficiency. Since we control all # our images fully we should enforce 0% inefficiency. At # the moment this breaks lre-cc. - CI=1 ${dive}/bin/dive \ - ''${IMAGE_NAME}:''${IMAGE_TAG} + CI=1 ${dive}/bin/dive ''${IMAGE_TARGET} # TODO(palfrey): Keep monitoring this for better solutions to ratelimits: # https://github.com/aquasecurity/trivy-action/issues/389 ${trivy}/bin/trivy image \ - ''${IMAGE_NAME}:''${IMAGE_TAG} \ + ''${IMAGE_TARGET} \ --db-repository public.ecr.aws/aquasecurity/trivy-db:2 '' diff --git a/tools/public/publish-ghcr.nix b/tools/public/publish-ghcr.nix index 18d2b2341..bac88a1e8 100644 --- a/tools/public/publish-ghcr.nix +++ b/tools/public/publish-ghcr.nix @@ -2,7 +2,7 @@ writeShellScriptBin, skopeo, cosign, - trivy, + trivy-report, }: writeShellScriptBin "publish-ghcr" '' set -xeuo pipefail @@ -48,16 +48,7 @@ writeShellScriptBin "publish-ghcr" '' echo "Skipping cosign signing (SKIP_SIGNING=true)" fi - # Skip trivy scan if SKIP_TRIVY is set - if [[ "''${SKIP_TRIVY:-false}" != "true" ]]; then - ${trivy}/bin/trivy \ - image \ - --format sarif \ - ''${TAGGED_IMAGE} \ - > trivy-results.sarif - else - echo "Skipping trivy scan (SKIP_TRIVY=true)" - fi + ${trivy-report}/bin/trivy-report ''${TAGGED_IMAGE} echo "Published: ''${TAGGED_IMAGE}" '' diff --git a/tools/public/regctl-ghcr-login.nix b/tools/public/regctl-ghcr-login.nix new file mode 100644 index 000000000..d72096496 --- /dev/null +++ b/tools/public/regctl-ghcr-login.nix @@ -0,0 +1,11 @@ +{ + writeShellScriptBin, + regclient, +}: +writeShellScriptBin "regctl-ghcr-login" '' + set -xeuo pipefail + echo $GHCR_PASSWORD | ${regclient}/bin/regctl \ + registry login ghcr.io \ + --user=$GHCR_USERNAME \ + --pass-stdin +'' diff --git a/tools/public/trivy-report.nix b/tools/public/trivy-report.nix new file mode 100644 index 000000000..9b76d066d --- /dev/null +++ b/tools/public/trivy-report.nix @@ -0,0 +1,18 @@ +{ + writeShellScriptBin, + trivy, +}: +writeShellScriptBin "trivy-report" '' + set -euo pipefail + + # Skip trivy scan if SKIP_TRIVY is set + if [[ "''${SKIP_TRIVY:-false}" != "true" ]]; then + ${trivy}/bin/trivy \ + image \ + --format sarif \ + $1 \ + > trivy-results.sarif + else + echo "Skipping trivy scan (SKIP_TRIVY=true)" + fi +'' diff --git a/tools/regclient.nix b/tools/regclient.nix new file mode 100644 index 000000000..e839b272d --- /dev/null +++ b/tools/regclient.nix @@ -0,0 +1,89 @@ +# FIXME: Based off of https://github.com/NixOS/nixpkgs/blob/nixos-26.05/pkgs/by-name/re/regclient/package.nix +# plus https://github.com/regclient/regclient/pull/1106 for better "not found" +{ + lib, + buildGoModule, + fetchFromGitHub, + installShellFiles, + lndir, + testers, + regclient, +}: let + bins = [ + "regbot" + "regctl" + "regsync" + ]; +in + buildGoModule rec { + pname = "regclient"; + version = "0.11.5"; + tag = "v${version}"; + + src = fetchFromGitHub { + owner = "regclient"; + repo = "regclient"; + rev = "8a3d428f8503450580f60e71f08ac4cecfdcc01d"; + sha256 = "sha256-Ho5XKxK+xEhnUryZY0KGcu19fH26nPn6DqsDsoW30p4="; + }; + vendorHash = "sha256-5SI1c0yszMTiC20bBXKIrtHVuDWA9BaSC5RY0F8B1Us="; + + outputs = ["out"] ++ bins; + + ldflags = [ + "-s" + "-w" + "-X github.com/regclient/regclient/internal/version.vcsTag=${tag}" + ]; + + nativeBuildInputs = [ + installShellFiles + lndir + ]; + + postInstall = + lib.concatMapStringsSep "\n" (bin: '' + export bin=''$${bin} + export outputBin=bin + + mkdir -p $bin/bin + mv $out/bin/${bin} $bin/bin + + installShellCompletion --cmd ${bin} \ + --bash <($bin/bin/${bin} completion bash) \ + --fish <($bin/bin/${bin} completion fish) \ + --zsh <($bin/bin/${bin} completion zsh) + + lndir -silent $bin $out + + unset bin outputBin + '') + bins; + + doCheck = false; + + checkFlags = [ + # touches network + "-skip=^ExampleNew$" + ]; + + passthru.tests = lib.mergeAttrsList ( + map (bin: { + "${bin}Version" = testers.testVersion { + package = regclient; + command = "${bin} version"; + version = tag; + }; + }) + bins + ); + + __darwinAllowLocalNetworking = true; + + meta = { + description = "Docker and OCI Registry Client in Go and tooling using those libraries"; + homepage = "https://github.com/regclient/regclient"; + license = lib.licenses.asl20; + maintainers = with lib.maintainers; [maxbrunet]; + }; + } diff --git a/web/apps/docs/content/docs/getting-started/other-build-systems.mdx b/web/apps/docs/content/docs/getting-started/other-build-systems.mdx deleted file mode 100644 index 3f769865a..000000000 --- a/web/apps/docs/content/docs/getting-started/other-build-systems.mdx +++ /dev/null @@ -1,18 +0,0 @@ ---- -title: Other build systems -description: Use NativeLink without Bazel — Buck2, Reclient, Pants, and CMake (via recc). ---- - -NativeLink speaks the standard Remote Execution API. If your build tool does -too, you can swap NativeLink in without changing a build file. - -## Supported clients - -- **Buck2** — point `buck2_remote_execution.engine.address` at your cluster. -- **Reclient** — set `RBE_service` to your NativeLink endpoint. -- **Pants** — configure `[remote_caching]` and `[remote_execution]` in - `pants.toml`. -- **CMake via [recc](https://github.com/bloomberg/recc)** — wrap compile - invocations with `recc` and let NativeLink handle the cache layer. - -Content coming. diff --git a/web/apps/docs/content/docs/getting-started/other-build-systems/buck2.mdx b/web/apps/docs/content/docs/getting-started/other-build-systems/buck2.mdx new file mode 100644 index 000000000..f734543ca --- /dev/null +++ b/web/apps/docs/content/docs/getting-started/other-build-systems/buck2.mdx @@ -0,0 +1,93 @@ +--- +title: Buck2 +description: Configure Buck2 to use NativeLink for remote cache and execution. +--- + +Buck2 has a native Remote Execution API client. NativeLink can serve the +CAS, Action Cache, capabilities, and execution services that Buck2 expects. + +The repo includes a runnable Buck2 integration test at +[`integration_tests/buck2/`](https://github.com/TraceMachina/nativelink/tree/main/integration_tests/buck2). +It starts NativeLink with a combined scheduler and local worker, runs +`buck2 build //...`, and checks for `BUILD SUCCEEDED`. + +## Start NativeLink + +Use a config that exposes these services on `localhost:50051`: + +- CAS +- Action Cache +- execution +- capabilities +- ByteStream + +The integration test uses +[`integration_tests/buck2/buck2_cas.json5`](https://github.com/TraceMachina/nativelink/tree/main/integration_tests/buck2/buck2_cas.json5) +and instance name `main`. + +## Configure Buck2 + +In your Buck2 project, add `.buckconfig`: + +```ini +[cells] +root = . + +[buck2_re_client] +engine_address = localhost:50051 +action_cache_address = localhost:50051 +cas_address = localhost:50051 +tls = false +instance_name = main + +[build] +execution_platforms = root//platforms:platforms +``` + +The checked-in integration test uses the same settings in +[`integration_tests/buck2/.buckconfig`](https://github.com/TraceMachina/nativelink/tree/main/integration_tests/buck2/.buckconfig). + +## Register an execution platform + +Buck2 needs an execution platform that allows remote actions. The +integration test registers one with `remote_enabled = true` and +`local_enabled = true`: + +```python +platform = ExecutionPlatformInfo( + label = ctx.label.raw_target(), + configuration = configuration, + executor_config = CommandExecutorConfig( + local_enabled = True, + remote_enabled = True, + use_limited_hybrid = True, + remote_execution_properties = {}, + remote_execution_use_case = "buck2-default", + remote_output_paths = "output_paths", + ), +) +``` + +See +[`integration_tests/buck2/platforms/defs.bzl`](https://github.com/TraceMachina/nativelink/tree/main/integration_tests/buck2/platforms/defs.bzl) +for the complete rule. + +## Run a build + +```bash +buck2 build //... +``` + +The integration test uses a small staged build graph that mixes local-only +and remote-capable actions, then verifies the output with `diff`. That shape +is useful when validating a new cluster because it proves both upload and +download paths are working. + +## Troubleshooting + +- **Buck2 cannot connect.** Confirm NativeLink is listening on `50051` and + Buck2 is not expecting TLS. +- **Everything runs locally.** Check that your selected execution platform + sets `remote_enabled = True`. +- **Instance-name errors.** Make sure Buck2's `instance_name` matches the + NativeLink config. The integration test uses `main`. diff --git a/web/apps/docs/content/docs/getting-started/other-build-systems/buildstream.mdx b/web/apps/docs/content/docs/getting-started/other-build-systems/buildstream.mdx new file mode 100644 index 000000000..da02be24c --- /dev/null +++ b/web/apps/docs/content/docs/getting-started/other-build-systems/buildstream.mdx @@ -0,0 +1,117 @@ +--- +title: BuildStream +description: Configure BuildStream to use NativeLink for artifact storage and remote execution. +--- + +BuildStream can use a Remote Execution API backend for artifact storage, +Action Cache, CAS, and remote execution. NativeLink can serve those +endpoints from the same `localhost:50051` listener used by the other local +examples. + +The repo includes a runnable BuildStream integration test at +[`integration_tests/buildstream/`](https://github.com/TraceMachina/nativelink/tree/main/integration_tests/buildstream). +It starts NativeLink with a local worker, runs `bst build hello.bst`, and +checks for `SUCCESS Build`. + +## Start NativeLink + +Use a config that exposes CAS, Action Cache, execution, capabilities, +ByteStream, fetch, and push services on `localhost:50051`. + +The integration test uses +[`integration_tests/buildstream/buildstream_cas.json5`](https://github.com/TraceMachina/nativelink/tree/main/integration_tests/buildstream/buildstream_cas.json5). +It also starts a private worker API on `localhost:50061` for the local +worker. + +```bash +nativelink -- integration_tests/buildstream/buildstream_cas.json5 +``` + +## Configure BuildStream + +Add a BuildStream config file that points artifact storage and remote +execution services at NativeLink: + +```yaml +cachedir: /tmp/buildstream + +artifacts: + servers: + - url: http://localhost:50051 + push: true + +remote-execution: + execution-service: + url: http://localhost:50051 + action-cache-service: + url: http://localhost:50051 + storage-service: + url: http://localhost:50051 +``` + +The checked-in integration test uses the same settings in +[`integration_tests/buildstream/buildstream.conf`](https://github.com/TraceMachina/nativelink/tree/main/integration_tests/buildstream/buildstream.conf). + +## Configure a project + +A minimal BuildStream project needs a project name, a supported BuildStream +version, and an element directory: + +```yaml +name: first-project +min-version: 2.4 +element-path: elements + +aliases: + alpine: https://bst-integration-test-images.ams3.cdn.digitaloceanspaces.com/ +``` + +The integration test uses this shape in +[`integration_tests/buildstream/project.conf`](https://github.com/TraceMachina/nativelink/tree/main/integration_tests/buildstream/project.conf). + +## Add an element + +This small element stages local source files and runs a build command: + +```yaml +kind: manual +description: | + Building manually + +depends: +- base.bst + +sources: + - kind: local + path: files/src + +config: + build-commands: + - echo hello.c +``` + +See +[`integration_tests/buildstream/elements/hello.bst`](https://github.com/TraceMachina/nativelink/tree/main/integration_tests/buildstream/elements/hello.bst) +for the complete test element. + +## Run a build + +Run BuildStream with the config file: + +```bash +bst -c buildstream.conf build hello.bst +``` + +The integration test checks for `SUCCESS Build` in the BuildStream output +and checks that NativeLink did not log an error. + +## Troubleshooting + +- **BuildStream cannot connect.** Match the `http://localhost:50051` + endpoint in `buildstream.conf` to the NativeLink listener. The local test + uses an insecure HTTP URL. +- **Uploads are missing.** Set `push: true` under `artifacts.servers` so + BuildStream can write artifacts back to NativeLink. +- **Remote actions do not match a worker.** Compare the platform properties + BuildStream requests with the properties your NativeLink worker advertises. + The integration test worker advertises `ISA: x86-64`. diff --git a/web/apps/docs/content/docs/rbe/cmake-recc.mdx b/web/apps/docs/content/docs/getting-started/other-build-systems/cmake-recc.mdx similarity index 99% rename from web/apps/docs/content/docs/rbe/cmake-recc.mdx rename to web/apps/docs/content/docs/getting-started/other-build-systems/cmake-recc.mdx index e6c0a36da..a0b8731c7 100644 --- a/web/apps/docs/content/docs/rbe/cmake-recc.mdx +++ b/web/apps/docs/content/docs/getting-started/other-build-systems/cmake-recc.mdx @@ -1,5 +1,5 @@ --- -title: "CMake with NativeLink (via recc)" +title: "CMake with recc" description: "Accelerate CMake builds with NativeLink as a remote cache, using recc as the bridge." pagefind: true --- diff --git a/web/apps/docs/content/docs/getting-started/other-build-systems/index.mdx b/web/apps/docs/content/docs/getting-started/other-build-systems/index.mdx new file mode 100644 index 000000000..1364bf34b --- /dev/null +++ b/web/apps/docs/content/docs/getting-started/other-build-systems/index.mdx @@ -0,0 +1,37 @@ +--- +title: Other build systems +description: Use NativeLink without Bazel — Buck2, Reclient, Pants, BuildStream, and CMake with recc. +--- + +NativeLink speaks the standard Remote Execution API. If your build tool +speaks the same protocol, you can point it at NativeLink without rewriting +your build files. + +## Options + +- [Buck2](/getting-started/other-build-systems/buck2) — use Buck2's + built-in remote execution client with NativeLink as CAS, Action Cache, + and executor. +- [Reclient](/getting-started/other-build-systems/reclient) — configure + `reproxy` and `rewrapper` for Chromium-style builds. +- [Pants](/getting-started/other-build-systems/pants) — enable remote + cache reads and writes from `pants.toml`. +- [BuildStream](/getting-started/other-build-systems/buildstream) — point + BuildStream artifact storage and remote execution at NativeLink. +- [CMake with recc](/getting-started/other-build-systems/cmake-recc) — + wrap CMake compile actions with `recc` and share compile outputs + through NativeLink. + +## Common NativeLink endpoint + +The examples below assume a NativeLink server exposing CAS, Action Cache, +execution, capabilities, and ByteStream on port `50051` with instance name +`main`. + +The BuildStream page follows the checked-in integration test, which uses +the default empty instance on the same local port. + +For a local Docker server, start with +[Getting Started → Setup](/getting-started/setup). For the Buck2 integration +test shape, see +[`integration_tests/buck2/buck2_cas.json5`](https://github.com/TraceMachina/nativelink/tree/main/integration_tests/buck2/buck2_cas.json5). diff --git a/web/apps/docs/content/docs/getting-started/other-build-systems/meta.json b/web/apps/docs/content/docs/getting-started/other-build-systems/meta.json new file mode 100644 index 000000000..1e4ab9cf8 --- /dev/null +++ b/web/apps/docs/content/docs/getting-started/other-build-systems/meta.json @@ -0,0 +1,10 @@ +{ + "pages": [ + "buck2", + "reclient", + "pants", + "buildstream", + "cmake-recc" + ], + "title": "Other build systems" +} diff --git a/web/apps/docs/content/docs/getting-started/other-build-systems/pants.mdx b/web/apps/docs/content/docs/getting-started/other-build-systems/pants.mdx new file mode 100644 index 000000000..033ae8e69 --- /dev/null +++ b/web/apps/docs/content/docs/getting-started/other-build-systems/pants.mdx @@ -0,0 +1,57 @@ +--- +title: Pants +description: Enable Pants remote caching with NativeLink. +--- + +Pants can read and write remote cache entries through the Remote Execution +API. NativeLink can provide the remote store and Action Cache for Pants +goals without changing BUILD files. + +## Start NativeLink + +Start a NativeLink server that exposes CAS and Action Cache on +`localhost:50051` with instance name `main`. The Docker quickstart in +[Getting Started → Setup](/getting-started/setup) is enough for local +cache validation. + +## Configure Pants + +Add the remote cache settings to `pants.toml`: + +```toml +[GLOBAL] +remote_cache_read = true +remote_cache_write = true +remote_store_address = "grpc://localhost:50051" +remote_instance_name = "main" +``` + +For a shared deployment, replace `localhost:50051` with your NativeLink +endpoint and use the same instance name your server config exposes. + +## Run a goal + +Run any Pants goal as usual: + +```bash +pants test :: +``` + +The first run uploads results. A second run against the same inputs should +read cached process results from NativeLink. + +## Remote execution + +Pants remote execution uses the same Remote Execution API, but it needs a +worker environment that matches the actions Pants sends. Start with remote +caching, then add remote execution once you have worker platform properties +and toolchains aligned. + +## Troubleshooting + +- **No cache hits on the second run.** Check that `remote_cache_write` and + `remote_cache_read` are both enabled. +- **Instance-name errors.** Match `remote_instance_name` to the NativeLink + config. The local examples use `main`. +- **Connection failures.** Pants expects a `grpc://` URL in + `remote_store_address`. diff --git a/web/apps/docs/content/docs/getting-started/other-build-systems/reclient.mdx b/web/apps/docs/content/docs/getting-started/other-build-systems/reclient.mdx new file mode 100644 index 000000000..17db1a152 --- /dev/null +++ b/web/apps/docs/content/docs/getting-started/other-build-systems/reclient.mdx @@ -0,0 +1,82 @@ +--- +title: Reclient +description: Point Reclient at NativeLink for Chromium-style remote caching and execution. +--- + +Reclient speaks the Remote Execution API through a local daemon, +`reproxy`, and command wrapper, `rewrapper`. NativeLink can be the +backend for Reclient clients such as Chromium and other GN/Ninja builds. + +For a production Chromium walkthrough, see +[Deployment → Chromium](/deployment/chromium). + +## Local cache-only setup + +Start NativeLink on `localhost:50051` with instance name `main`, then +export the environment variables Reclient reads: + +```bash +export RBE_service=localhost:50051 +export RBE_instance=main +export RBE_service_no_security=true +``` + +For cache-only compiler wrapping, run `reproxy` and invoke a command with +`rewrapper`: + +```bash +reproxy & +rewrapper -- clang++ -c hello.cc -o hello.o +``` + +For CMake projects, [`recc`](/getting-started/other-build-systems/cmake-recc) +is often the smaller setup. It uses the same Remote Execution API but fits +CMake compiler-launcher workflows directly. + +## Chromium-style setup + +Chromium checks in Reclient configs. Point them at NativeLink by enabling +remote execution in GN: + +```bash +gn gen out/Default --args=' + use_remoteexec=true + use_goma=false + use_siso=false + reclient_cfg_dir="../../buildtools/reclient_cfgs/chromium-browser-clang" +' +``` + +Start the daemon before building: + +```bash +buildtools/reclient/bootstrap --re_proxy=buildtools/reclient/reproxy +autoninja -C out/Default chrome +``` + +## Production TLS settings + +For a secured NativeLink endpoint, set the Reclient TLS variables instead +of `RBE_service_no_security=true`: + +```bash +export RBE_service=cas.nativelink.internal:50051 +export RBE_instance=main +export RBE_service_no_security=false +export RBE_tls_client_auth_cert=/etc/nativelink/tls/client.crt +export RBE_tls_client_auth_key=/etc/nativelink/tls/client.key +export RBE_tls_ca_cert=/etc/nativelink/tls/ca.crt +``` + +## Confirm it is working + +Use Reclient stats after a build: + +```bash +reclient stats +``` + +On a warm cache, cache hits should climb and network errors should stay at +zero. If actions fall back to local execution, compare the platform +properties Reclient requests with the properties your NativeLink workers +advertise. diff --git a/web/apps/docs/content/docs/rbe/meta.json b/web/apps/docs/content/docs/rbe/meta.json index b2c1afbd6..7b27cf714 100644 --- a/web/apps/docs/content/docs/rbe/meta.json +++ b/web/apps/docs/content/docs/rbe/meta.json @@ -1,8 +1,7 @@ { "pages": [ "examples", - "nix-templates", - "cmake-recc" + "nix-templates" ], "title": "Testing Remote Execution" } diff --git a/web/apps/docs/content/docs/reference/nativelink-config/index.mdx b/web/apps/docs/content/docs/reference/nativelink-config/index.mdx index b333fa9d6..62968266e 100644 --- a/web/apps/docs/content/docs/reference/nativelink-config/index.mdx +++ b/web/apps/docs/content/docs/reference/nativelink-config/index.mdx @@ -5,14 +5,14 @@ full: true --- {/* AUTOGENERATED — do not edit by hand. - Source: nativelink-config @ v1.5.1 (3d9d74ab) + Source: nativelink-config @ v1.5.2 (6e63ef9a) Regenerate from web/: bun --filter @nativelink/docs gen:config-reference */} -This is the canonical NativeLink configuration reference for **v1.5.1**. +This is the canonical NativeLink configuration reference for **v1.5.2**. It is autogenerated from the Rust config crate -([`nativelink-config/src`](https://github.com/TraceMachina/nativelink/tree/v1.5.1/nativelink-config/src)) via the `build-schema` binary, so +([`nativelink-config/src`](https://github.com/TraceMachina/nativelink/tree/v1.5.2/nativelink-config/src)) via the `build-schema` binary, so it can never drift from what the binary actually deserializes. ## Top-level fields @@ -698,13 +698,7 @@ Plus exactly one of the following variants (the key selects the variant): ## ExperimentalCloudObjectSpec -One of: - -- `provider`: `"aws"` -- `provider`: `"gcs"` -- `provider`: `"azure"` -- object -- object +See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for details ## OntapS3ExistenceCacheSpec @@ -742,9 +736,9 @@ One of: | --- | --- | --- | --- | --- | | `index_store` | [StoreSpec](#storespec) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | | `content_store` | [StoreSpec](#storespec) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | -| `min_size` | integer (uint32) | — | 65536 (64k) | Minimum size that a chunk will be when slicing up the content. Note: This setting can be increased to improve performance because it will actually not check this number of bytes when deciding where to partition the data. | -| `normal_size` | integer (uint32) | — | 262144 (256k) | A best-effort attempt will be made to keep the average size of the chunks to this number. It is not a guarantee, but a slight attempt will be made. | -| `max_size` | integer (uint32) | — | 524288 (512k) | Maximum size a chunk is allowed to be. | +| `min_size` | integer (uint32) | — | 64k | Minimum size that a chunk will be when slicing up the content. Note: This setting can be increased to improve performance because it will actually not check this number of bytes when deciding where to partition the data. | +| `normal_size` | integer (uint32) | — | 256k | A best-effort attempt will be made to keep the average size of the chunks to this number. It is not a guarantee, but a slight attempt will be made. | +| `max_size` | integer (uint32) | — | 512k | Maximum size a chunk is allowed to be. | | `max_concurrent_fetch_per_get` | integer (uint32) | — | 10 | Due to implementation detail, we want to prefer to download the first chunks of the file so we can stream the content out and free up some of our buffers. This configuration will be used to restrict the number of concurrent chunk downloads at a time per `get()` request. | ## ExistenceCacheSpec @@ -762,7 +756,7 @@ One of: | `fast_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the fast store. This can be useful to set to Get for worker nodes such that results are persisted to the slow store only. | | `slow` | [StoreSpec](#storespec) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | | `slow_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the slow store. This can be useful if creating a diode and you wish to have an upstream read only store. | -| `bypass_dedup_threshold_bytes` | integer (uint64) | — | `0` | Reads of blobs at or above this size skip the leader/follower dedup map and stream straight from the slow store without populating the fast tier. `0` (the default) disables the bypass: every read goes through dedup, matching the prior behaviour. Enable it by setting a threshold — 256 MiB is a reasonable starting point for backends where large-blob dedup is a net loss (followers tend to time out anyway), but the right value is workload-dependent. | +| `bypass_dedup_threshold_bytes` | integer (uint64) | — | disabled (0) | Reads of blobs at or above this size skip the leader/follower dedup map and stream straight from the slow store without populating the fast tier. `0` (the default) disables the bypass: every read goes through dedup, matching the prior behaviour. Enable it by setting a threshold — 256 MiB is a reasonable starting point for backends where large-blob dedup is a net loss (followers tend to time out anyway), but the right value is workload-dependent. | ## ShardSpec @@ -778,8 +772,8 @@ One of: | `temp_path` | string | Yes | — | A temporary location of where files that are being uploaded or deleted will be placed while the content cannot be guaranteed to be accurate. This location must be on the same block device as `content_path` so atomic moves can happen (i.e., move without copy). All files in this folder will be deleted on every startup. | | `read_buffer_size` | integer (uint32) | — | 32k | Buffer size to use when reading files. Generally this should be left to the default value except for testing. | | `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | -| `block_size` | integer (uint64) | — | 4096 | The block size of the filesystem for the running machine value is used to determine an entry's actual size on disk consumed For a 4KB block size filesystem, a 1B file actually consumes 4KB | -| `max_concurrent_writes` | integer (uint) | — | 0 | Maximum number of concurrent write operations allowed. Each write involves streaming data to a temp file and calling `sync_all()`, which can saturate disk I/O when many writes happen simultaneously. Limiting concurrency prevents disk saturation from blocking the async runtime. A value of 0 means unlimited (no concurrency limit). | +| `block_size` | integer (uint64) | — | 4kb | The block size of the filesystem for the running machine value is used to determine an entry's actual size on disk consumed For a 4KB block size filesystem, a 1B file actually consumes 4KB | +| `max_concurrent_writes` | integer (uint) | — | unlimited | Maximum number of concurrent write operations allowed. Each write involves streaming data to a temp file and calling `sync_all()`, which can saturate disk I/O when many writes happen simultaneously. Limiting concurrency prevents disk saturation from blocking the async runtime. A value of 0 means unlimited (no concurrency limit). | ## RefSpec @@ -828,7 +822,7 @@ One of: | `connection_pool_size` | integer (uint) | — | 3 | The number of connections to keep open to the Redis servers. | | `max_chunk_uploads_per_update` | integer (uint) | — | 10 | The maximum number of upload chunks to allow per update. This is used to limit the amount of memory used when uploading large objects to the Redis server. A good rule of thumb is to think of the data as: `AVAIL_MEMORY / (read_chunk_size * max_chunk_uploads_per_update) = THORETICAL_MAX_CONCURRENT_UPLOADS` (note: it is a good idea to divide `AVAIL_MAX_MEMORY` by ~10 to account for other memory usage) | | `scan_count` | integer (uint) | — | 10000 | The COUNT value passed when scanning keys in Redis. This is used to hint the amount of work that should be done per response. | -| `retry` | [Retry](#retry) | — | Retry { | Retry configuration to use when a network request fails. See the `Retry` struct for more information. | +| `retry` | [Retry](#retry) | — | — | Retry configuration to use when a network request fails. | | `max_client_permits` | integer (uint) | — | 500 | Maximum number of permitted actions to the Redis store at any one time This stops problems with timeouts due to many, many inflight actions | | `max_count_per_cursor` | integer (uint64) | — | 1500 | Maximum number of items returned per cursor for the search indexes May reduce thundering herd issues with worker provisioner at higher node counts, | @@ -863,8 +857,8 @@ Configuration for `ExperimentalMongoDB` store. | --- | --- | --- | --- | --- | | `name` | string | — | {Index position in the workers list} | Name of the worker. This is give a more friendly name to a worker for logging and metric publishing. This is also the prefix of the worker id (i.e., "{name}{uuidv6}"). | | `worker_api_endpoint` | [EndpointConfig](#endpointconfig) | Yes | — | Endpoint which the worker will connect to the scheduler's `WorkerApiService`. | -| `max_action_timeout` | integer (uint) | — | 1200 (seconds / 20 minutes) | The maximum time an action is allowed to run. If a task requests for a timeout longer than this time limit, the task will be rejected. Value in seconds. | -| `max_upload_timeout` | integer (uint) | — | 600 (seconds / 10 minutes) | Maximum time allowed for uploading action results to CAS after execution completes. If upload takes longer than this, the action fails with `DeadlineExceeded` and may be retried by the scheduler. Value in seconds. | +| `max_action_timeout` | integer (uint) | — | 20 minutes | The maximum time an action is allowed to run. If a task requests for a timeout longer than this time limit, the task will be rejected. Value in seconds. | +| `max_upload_timeout` | integer (uint) | — | 10 minutes | Maximum time allowed for uploading action results to CAS after execution completes. If upload takes longer than this, the action fails with `DeadlineExceeded` and may be retried by the scheduler. Value in seconds. | | `max_inflight_tasks` | integer (uint64) | — | 0 (infinite tasks) | Maximum number of inflight tasks this worker can cope with. | | `timeout_handled_externally` | boolean | — | false (`NativeLink` fully handles timeouts) | If timeout is handled in `entrypoint` or another wrapper script. If set to true `NativeLink` will not honor the timeout the action requested and instead will always force kill the action after `max_action_timeout` has been reached. If this is set to false, the smaller value of the action's timeout and `max_action_timeout` will be used to which `NativeLink` will kill the action. | | `entrypoint` | string | — | {Use the command from the job request} | The command to execute on every execution request. This will be parsed as a command + arguments (not shell). Example: "run.sh" and a job with command: "sleep 5" will result in a command like: "run.sh sleep 5". | @@ -883,9 +877,9 @@ Configuration for `ExperimentalMongoDB` store. | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | | `supported_platform_properties` | map of string to [PropertyType](#propertytype) | — | — | A list of supported platform properties mapped to how these properties are used when the scheduler looks for worker nodes capable of running the task. | -| `retain_completed_for_s` | integer (uint32) | — | 60 (seconds) | The amount of time to retain completed actions for in case a `WaitExecution` is called after the action has completed. | -| `client_action_timeout_s` | integer (uint64) | — | 60 (seconds) | Mark operations as completed with error if no client has updated them within this duration. | -| `worker_timeout_s` | integer (uint64) | — | 5 (seconds) | Remove workers from pool once the worker has not responded in this amount of time in seconds. | +| `retain_completed_for_s` | integer (uint32) | — | 60 seconds | The amount of time to retain completed actions for in case a `WaitExecution` is called after the action has completed. | +| `client_action_timeout_s` | integer (uint64) | — | 60 seconds | Mark operations as completed with error if no client has updated them within this duration. | +| `worker_timeout_s` | integer (uint64) | — | 5 seconds | Remove workers from pool once the worker has not responded in this amount of time in seconds. | | `max_action_executing_timeout_s` | integer (uint64) | — | 0 (disabled) | Maximum time (seconds) an action can stay in Executing state without any worker update before being timed out and re-queued. This applies regardless of worker keepalive status, catching cases where a worker is alive (sending keepalives) but stuck on a specific action. Set to 0 to disable (relies only on `worker_timeout_s`). | | `max_job_retries` | integer (uint) | — | 3 | If a job returns an internal error or times out this many times when attempting to run on a worker the scheduler will return the last error to the client. Jobs will be retried and this configuration is to help prevent one rogue job from infinitely retrying and taking up a lot of resources when the task itself is the one causing the server to go into a bad state. | | `allocation_strategy` | [WorkerAllocationStrategy](#workerallocationstrategy) | — | `"least_recently_used"` | The strategy used to assign workers jobs. | @@ -903,8 +897,8 @@ build at the main scheduler directly though. | --- | --- | --- | --- | --- | | `endpoint` | [GrpcEndpoint](#grpcendpoint) | Yes | — | The upstream scheduler to forward requests to. | | `retry` | [Retry](#retry) | — | — | Retry configuration to use when a network request fails. | -| `max_concurrent_requests` | integer (uint) | — | `0` | Limit the number of simultaneous upstream requests to this many. A value of zero is treated as unlimited. If the limit is reached the request is queued. | -| `connections_per_endpoint` | integer (uint) | — | `0` | The number of connections to make to each specified endpoint to balance the load over multiple TCP connections. Default 1. | +| `max_concurrent_requests` | integer (uint) | — | unlimited | Limit the number of simultaneous upstream requests to this many. A value of zero is treated as unlimited. If the limit is reached the request is queued. | +| `connections_per_endpoint` | integer (uint) | — | 1 | The number of connections to make to each specified endpoint to balance the load over multiple TCP connections. | ## CacheLookupSpec @@ -925,7 +919,7 @@ build at the main scheduler directly though. | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | | `hints_file` | string | Yes | — | JSON file containing historical resource hints keyed by Bazel `RequestMetadata` `target_id` and/or `action_mnemonic`. | -| `refresh_interval_s` | integer (uint64) | — | 30 (seconds) | Reload interval for `hints_file`. Set to 0 to load once. | +| `refresh_interval_s` | integer (uint64) | — | 30 seconds | Reload interval for `hints_file`. Set to 0 to load once. | | `cpu_property_name` | string | — | `cpu_count` | Platform property name used for CPU minimum values. | | `memory_property_name` | string | — | `memory_kb` | Platform property name used for memory minimum values, expressed in KiB. | | `scheduler` | [SchedulerSpec](#schedulerspec) | Yes | — | The nested scheduler to use after applying resource hints. | @@ -1594,6 +1588,7 @@ will result in: | 6 | 1.2 to 2 s | | 7 | 2.4 to 4 s | | 8 | 4.8 to 8 s | + The total delay is additive, so this example produces 9.525 to 15.875 s of total delay for a single request. | Field | Type | Required | Default | Description | @@ -1660,10 +1655,10 @@ Configuration for an individual shard of the store. | `address` | string | Yes | — | The endpoint address (i.e. grpc://example.com:443 or grpcs://example.com:443). | | `tls_config` | [ClientTlsConfig](#clienttlsconfig) | — | — | The TLS configuration to use to connect to the endpoint (if grpcs). | | `concurrency_limit` | integer (uint) | — | — | The maximum concurrency to allow on this endpoint. | -| `connect_timeout_s` | integer (uint64) | — | 0 (uses 30 seconds) | Timeout for establishing a TCP connection to the endpoint (seconds). | -| `tcp_keepalive_s` | integer (uint64) | — | 0 (uses 30 seconds) | TCP keepalive interval (seconds). Sends TCP keepalive probes at this interval to detect dead connections at the OS level. | -| `http2_keepalive_interval_s` | integer (uint64) | — | 0 (uses 30 seconds) | HTTP/2 keepalive interval (seconds). Sends HTTP/2 PING frames at this interval to detect dead connections at the application level. | -| `http2_keepalive_timeout_s` | integer (uint64) | — | 0 (uses 20 seconds) | HTTP/2 keepalive timeout (seconds). If a PING response is not received within this duration, the connection is considered dead. | +| `connect_timeout_s` | integer (uint64) | — | 30 seconds | Timeout for establishing a TCP connection to the endpoint (seconds). | +| `tcp_keepalive_s` | integer (uint64) | — | 30 seconds | TCP keepalive interval (seconds). Sends TCP keepalive probes at this interval to detect dead connections at the OS level. | +| `http2_keepalive_interval_s` | integer (uint64) | — | 30 seconds | HTTP/2 keepalive interval (seconds). Sends HTTP/2 PING frames at this interval to detect dead connections at the application level. | +| `http2_keepalive_timeout_s` | integer (uint64) | — | 20 seconds | HTTP/2 keepalive timeout (seconds). If a PING response is not received within this duration, the connection is considered dead. | ## StoreType @@ -1687,7 +1682,7 @@ Generic config for an endpoint and associated configs. | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | | `uri` | string | Yes | — | URI of the endpoint. | -| `timeout` | number (float) | — | 5 (seconds) | Timeout in seconds that a request should take. | +| `timeout` | number (float) | — | 5 seconds | Timeout in seconds that a request should take. | | `tls_config` | [ClientTlsConfig](#clienttlsconfig) | — | — | The TLS configuration to use to connect to the endpoint. | ## UploadActionResultConfig @@ -1695,9 +1690,9 @@ Generic config for an endpoint and associated configs. | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | | `ac_store` | string | — | {No uploading is done} | Underlying AC store that the worker will use to publish execution results into. Objects placed in this store should be reachable from the scheduler/client-cas after they have finished updating. | -| `upload_ac_results_strategy` | [UploadCacheResultsStrategy](#uploadcacheresultsstrategy) | — | `UploadCacheResultsStrategy::SuccessOnly` | In which situations should the results be published to the `ac_store`, if set to `SuccessOnly` then only results with an exit code of 0 will be uploaded, if set to Everything all completed results will be uploaded. | +| `upload_ac_results_strategy` | [UploadCacheResultsStrategy](#uploadcacheresultsstrategy) | — | `SuccessOnly` | In which situations should the results be published to the `ac_store`, if set to `SuccessOnly` then only results with an exit code of 0 will be uploaded, if set to Everything all completed results will be uploaded. | | `historical_results_store` | string | — | {CAS store of parent} | Store to upload historical results to. This should be a CAS store if set. | -| `upload_historical_results_strategy` | [UploadCacheResultsStrategy](#uploadcacheresultsstrategy) | — | `UploadCacheResultsStrategy::FailuresOnly` | In which situations should the results be published to the historical CAS. The historical CAS is where failures are published. These messages conform to the CAS key-value lookup format and are always a `HistoricalExecuteResponse` serialized message. | +| `upload_historical_results_strategy` | [UploadCacheResultsStrategy](#uploadcacheresultsstrategy) | — | `FailuresOnly` | In which situations should the results be published to the historical CAS. The historical CAS is where failures are published. These messages conform to the CAS key-value lookup format and are always a `HistoricalExecuteResponse` serialized message. | | `success_message_template` | string | — | "" (no message) | Template to use for the `ExecuteResponse.message` property. This message is attached to the response before it is sent to the client. The following special variables are supported:; `digest_function`: Digest function used to calculate the action digest: `action_digest_hash`: Action digest hash: `action_digest_size`: Action digest size: `historical_results_hash`: `HistoricalExecuteResponse` digest hash: `historical_results_size`: `HistoricalExecuteResponse` digest size. | | `failure_message_template` | string | — | "" (no message) | Same as `success_message_template` but for failure case. | @@ -1749,7 +1744,7 @@ One of: - string - `"minimum"` — Requires the platform property to be a u64 and when the scheduler looks for appropriate worker nodes that are capable of executing the task, the task will not run on a node that has less than this value. - `"exact"` — Requires the platform property to be a string and when the scheduler looks for appropriate worker nodes that are capable of executing the task, the task will not run on a node that does not have this property set to the value with exact string match. -- `"priority"` — Does not restrict on this value and instead will be passed to the worker as an informational piece. TODO(palfrey) In the future this will be used by the scheduler and worker to cause the scheduler to prefer certain workers over others, but not restrict them based on these values. +- `"priority"` — Does not restrict on this value and instead will be passed to the worker as an informational piece. In the future this will be used by the scheduler and worker to cause the scheduler to prefer certain workers over others, but not restrict them based on these values. ## WorkerAllocationStrategy @@ -1830,14 +1825,14 @@ If a property is found, then replace it with another one. | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `instance_name` | string | — | `""` | | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | | `cas_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. | ## ActionCacheServiceConfig | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `instance_name` | string | — | `""` | | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | | `ac_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. | | `read_only` | boolean | — | `false` | Whether the Action Cache store may be written to, this if set to false it is only possible to read from the Action Cache. | @@ -1845,14 +1840,14 @@ If a property is found, then replace it with another one. | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `instance_name` | string | — | `""` | | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | | `remote_execution` | [CapabilitiesRemoteExecutionConfig](#capabilitiesremoteexecutionconfig) | — | — | Configuration for remote execution capabilities. If not set the capabilities service will inform the client that remote execution is not supported. | ## ExecutionServiceConfig | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `instance_name` | string | — | `""` | | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | | `cas_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. This value must be a CAS store reference. | | `scheduler` | string | Yes | — | The scheduler name referenced in the `schedulers` map in the main config. | @@ -1860,23 +1855,23 @@ If a property is found, then replace it with another one. | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `instance_name` | string | — | `""` | | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | | `cas_store` | string | Yes | — | Name of the store in the "stores" configuration. | | `max_bytes_per_stream` | integer (uint) | — | 64KiB | Max number of bytes to send on each grpc stream chunk. According to [https://github.com/grpc/grpc.github.io/issues/371](https://github.com/grpc/grpc.github.io/issues/371) 16KiB - 64KiB is optimal. | -| `persist_stream_on_disconnect_timeout` | integer (uint) | — | 10 (seconds) | In the event a client disconnects while uploading a blob, we will hold the internal stream open for this many seconds before closing it. This allows clients that disconnect to reconnect and continue uploading the same blob. | +| `persist_stream_on_disconnect_timeout` | integer (uint) | — | 10 seconds | In the event a client disconnects while uploading a blob, we will hold the internal stream open for this many seconds before closing it. This allows clients that disconnect to reconnect and continue uploading the same blob. | ## FetchServiceConfig | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `instance_name` | string | — | `""` | | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | | `fetch_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. | ## PushServiceConfig | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `instance_name` | string | — | `""` | | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | | `push_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. | | `read_only` | boolean | — | `false` | Whether the Action Cache store may be written to, this if set to false it is only possible to read from the Action Cache. | @@ -1903,7 +1898,7 @@ If a property is found, then replace it with another one. | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | | `path` | string | — | "/status" | Path to register the health status check. If path is "/status", and your domain is "example.com", you can reach the endpoint with: [http://example.com/status](http://example.com/status). | -| `timeout_seconds` | integer (uint64) | — | `0` | | +| `timeout_seconds` | integer (uint64) | — | 5s | Timeout on health checks. | ## ErrorCode @@ -1933,7 +1928,7 @@ The possible error codes that might occur on an upstream request. | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | | `block_size` | integer (uint32) | — | 65536 (64k) | Size of the blocks to compress. Higher values require more ram, but might yield slightly better compression ratios. | -| `max_decode_block_size` | integer (uint32) | — | value in `block_size` | Maximum size allowed to attempt to deserialize data into. This is needed because the `block_size` is embedded into the data so if there was a bad actor, they could upload an extremely large `block_size`'ed entry and we'd allocate a large amount of memory when retrieving the data. To prevent this from happening, we allow you to specify the maximum that we'll attempt deserialize. | +| `max_decode_block_size` | integer (uint32) | — | value in `block_size` | Maximum size allowed to attempt to deserialize data into. This is needed because the `block_size` is embedded into the data so if there was a bad actor, they could upload an extremely large `block_size`'ed entry and we'd allocate a large amount of memory when retrieving the data. To prevent this from happening, we allow you to specify the maximum that we'll attempt to deserialize. | ## ClientTlsConfig @@ -2041,6 +2036,6 @@ specified. If anything here disagrees with the binary, the source wins: -- [`stores.rs`](https://github.com/TraceMachina/nativelink/tree/v1.5.1/nativelink-config/src/stores.rs) -- [`cas_server.rs`](https://github.com/TraceMachina/nativelink/tree/v1.5.1/nativelink-config/src/cas_server.rs) -- [`schedulers.rs`](https://github.com/TraceMachina/nativelink/tree/v1.5.1/nativelink-config/src/schedulers.rs) +- [`stores.rs`](https://github.com/TraceMachina/nativelink/tree/v1.5.2/nativelink-config/src/stores.rs) +- [`cas_server.rs`](https://github.com/TraceMachina/nativelink/tree/v1.5.2/nativelink-config/src/cas_server.rs) +- [`schedulers.rs`](https://github.com/TraceMachina/nativelink/tree/v1.5.2/nativelink-config/src/schedulers.rs) diff --git a/web/apps/docs/content/docs/reference/nativelink-config/main.mdx b/web/apps/docs/content/docs/reference/nativelink-config/main.mdx index e126a4b21..26044f99b 100644 --- a/web/apps/docs/content/docs/reference/nativelink-config/main.mdx +++ b/web/apps/docs/content/docs/reference/nativelink-config/main.mdx @@ -5,8 +5,8 @@ full: true --- {/* AUTOGENERATED — do not edit by hand. - Source: nativelink-config @ main (a608fc1e) - Regenerate: bun run --filter=@nativelink/docs gen:config-reference */} + Source: nativelink-config @ main (3bca6f7e) + Regenerate from web/: bun --filter @nativelink/docs gen:config-reference */} @@ -698,13 +698,7 @@ Plus exactly one of the following variants (the key selects the variant): ## ExperimentalCloudObjectSpec -One of: - -- `provider`: `"aws"` -- `provider`: `"gcs"` -- `provider`: `"azure"` -- object -- object +See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for details ## OntapS3ExistenceCacheSpec @@ -742,9 +736,9 @@ One of: | --- | --- | --- | --- | --- | | `index_store` | [StoreSpec](#storespec) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | | `content_store` | [StoreSpec](#storespec) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | -| `min_size` | integer (uint32) | — | 65536 (64k) | Minimum size that a chunk will be when slicing up the content. Note: This setting can be increased to improve performance because it will actually not check this number of bytes when deciding where to partition the data. | -| `normal_size` | integer (uint32) | — | 262144 (256k) | A best-effort attempt will be made to keep the average size of the chunks to this number. It is not a guarantee, but a slight attempt will be made. | -| `max_size` | integer (uint32) | — | 524288 (512k) | Maximum size a chunk is allowed to be. | +| `min_size` | integer (uint32) | — | 64k | Minimum size that a chunk will be when slicing up the content. Note: This setting can be increased to improve performance because it will actually not check this number of bytes when deciding where to partition the data. | +| `normal_size` | integer (uint32) | — | 256k | A best-effort attempt will be made to keep the average size of the chunks to this number. It is not a guarantee, but a slight attempt will be made. | +| `max_size` | integer (uint32) | — | 512k | Maximum size a chunk is allowed to be. | | `max_concurrent_fetch_per_get` | integer (uint32) | — | 10 | Due to implementation detail, we want to prefer to download the first chunks of the file so we can stream the content out and free up some of our buffers. This configuration will be used to restrict the number of concurrent chunk downloads at a time per `get()` request. | ## ExistenceCacheSpec @@ -762,7 +756,7 @@ One of: | `fast_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the fast store. This can be useful to set to Get for worker nodes such that results are persisted to the slow store only. | | `slow` | [StoreSpec](#storespec) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | | `slow_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the slow store. This can be useful if creating a diode and you wish to have an upstream read only store. | -| `bypass_dedup_threshold_bytes` | integer (uint64) | — | `0` | Reads of blobs at or above this size skip the leader/follower dedup map and stream straight from the slow store without populating the fast tier. `0` (the default) disables the bypass: every read goes through dedup, matching the prior behaviour. Enable it by setting a threshold — 256 MiB is a reasonable starting point for backends where large-blob dedup is a net loss (followers tend to time out anyway), but the right value is workload-dependent. | +| `bypass_dedup_threshold_bytes` | integer (uint64) | — | disabled (0) | Reads of blobs at or above this size skip the leader/follower dedup map and stream straight from the slow store without populating the fast tier. `0` (the default) disables the bypass: every read goes through dedup, matching the prior behaviour. Enable it by setting a threshold — 256 MiB is a reasonable starting point for backends where large-blob dedup is a net loss (followers tend to time out anyway), but the right value is workload-dependent. | ## ShardSpec @@ -778,8 +772,8 @@ One of: | `temp_path` | string | Yes | — | A temporary location of where files that are being uploaded or deleted will be placed while the content cannot be guaranteed to be accurate. This location must be on the same block device as `content_path` so atomic moves can happen (i.e., move without copy). All files in this folder will be deleted on every startup. | | `read_buffer_size` | integer (uint32) | — | 32k | Buffer size to use when reading files. Generally this should be left to the default value except for testing. | | `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | -| `block_size` | integer (uint64) | — | 4096 | The block size of the filesystem for the running machine value is used to determine an entry's actual size on disk consumed For a 4KB block size filesystem, a 1B file actually consumes 4KB | -| `max_concurrent_writes` | integer (uint) | — | 0 | Maximum number of concurrent write operations allowed. Each write involves streaming data to a temp file and calling `sync_all()`, which can saturate disk I/O when many writes happen simultaneously. Limiting concurrency prevents disk saturation from blocking the async runtime. A value of 0 means unlimited (no concurrency limit). | +| `block_size` | integer (uint64) | — | 4kb | The block size of the filesystem for the running machine value is used to determine an entry's actual size on disk consumed For a 4KB block size filesystem, a 1B file actually consumes 4KB | +| `max_concurrent_writes` | integer (uint) | — | unlimited | Maximum number of concurrent write operations allowed. Each write involves streaming data to a temp file and calling `sync_all()`, which can saturate disk I/O when many writes happen simultaneously. Limiting concurrency prevents disk saturation from blocking the async runtime. A value of 0 means unlimited (no concurrency limit). | ## RefSpec @@ -828,7 +822,7 @@ One of: | `connection_pool_size` | integer (uint) | — | 3 | The number of connections to keep open to the Redis servers. | | `max_chunk_uploads_per_update` | integer (uint) | — | 10 | The maximum number of upload chunks to allow per update. This is used to limit the amount of memory used when uploading large objects to the Redis server. A good rule of thumb is to think of the data as: `AVAIL_MEMORY / (read_chunk_size * max_chunk_uploads_per_update) = THORETICAL_MAX_CONCURRENT_UPLOADS` (note: it is a good idea to divide `AVAIL_MAX_MEMORY` by ~10 to account for other memory usage) | | `scan_count` | integer (uint) | — | 10000 | The COUNT value passed when scanning keys in Redis. This is used to hint the amount of work that should be done per response. | -| `retry` | [Retry](#retry) | — | Retry { | Retry configuration to use when a network request fails. See the `Retry` struct for more information. | +| `retry` | [Retry](#retry) | — | — | Retry configuration to use when a network request fails. | | `max_client_permits` | integer (uint) | — | 500 | Maximum number of permitted actions to the Redis store at any one time This stops problems with timeouts due to many, many inflight actions | | `max_count_per_cursor` | integer (uint64) | — | 1500 | Maximum number of items returned per cursor for the search indexes May reduce thundering herd issues with worker provisioner at higher node counts, | @@ -863,8 +857,8 @@ Configuration for `ExperimentalMongoDB` store. | --- | --- | --- | --- | --- | | `name` | string | — | {Index position in the workers list} | Name of the worker. This is give a more friendly name to a worker for logging and metric publishing. This is also the prefix of the worker id (i.e., "{name}{uuidv6}"). | | `worker_api_endpoint` | [EndpointConfig](#endpointconfig) | Yes | — | Endpoint which the worker will connect to the scheduler's `WorkerApiService`. | -| `max_action_timeout` | integer (uint) | — | 1200 (seconds / 20 minutes) | The maximum time an action is allowed to run. If a task requests for a timeout longer than this time limit, the task will be rejected. Value in seconds. | -| `max_upload_timeout` | integer (uint) | — | 600 (seconds / 10 minutes) | Maximum time allowed for uploading action results to CAS after execution completes. If upload takes longer than this, the action fails with `DeadlineExceeded` and may be retried by the scheduler. Value in seconds. | +| `max_action_timeout` | integer (uint) | — | 20 minutes | The maximum time an action is allowed to run. If a task requests for a timeout longer than this time limit, the task will be rejected. Value in seconds. | +| `max_upload_timeout` | integer (uint) | — | 10 minutes | Maximum time allowed for uploading action results to CAS after execution completes. If upload takes longer than this, the action fails with `DeadlineExceeded` and may be retried by the scheduler. Value in seconds. | | `max_inflight_tasks` | integer (uint64) | — | 0 (infinite tasks) | Maximum number of inflight tasks this worker can cope with. | | `timeout_handled_externally` | boolean | — | false (`NativeLink` fully handles timeouts) | If timeout is handled in `entrypoint` or another wrapper script. If set to true `NativeLink` will not honor the timeout the action requested and instead will always force kill the action after `max_action_timeout` has been reached. If this is set to false, the smaller value of the action's timeout and `max_action_timeout` will be used to which `NativeLink` will kill the action. | | `entrypoint` | string | — | {Use the command from the job request} | The command to execute on every execution request. This will be parsed as a command + arguments (not shell). Example: "run.sh" and a job with command: "sleep 5" will result in a command like: "run.sh sleep 5". | @@ -883,9 +877,9 @@ Configuration for `ExperimentalMongoDB` store. | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | | `supported_platform_properties` | map of string to [PropertyType](#propertytype) | — | — | A list of supported platform properties mapped to how these properties are used when the scheduler looks for worker nodes capable of running the task. | -| `retain_completed_for_s` | integer (uint32) | — | 60 (seconds) | The amount of time to retain completed actions for in case a `WaitExecution` is called after the action has completed. | -| `client_action_timeout_s` | integer (uint64) | — | 60 (seconds) | Mark operations as completed with error if no client has updated them within this duration. | -| `worker_timeout_s` | integer (uint64) | — | 5 (seconds) | Remove workers from pool once the worker has not responded in this amount of time in seconds. | +| `retain_completed_for_s` | integer (uint32) | — | 60 seconds | The amount of time to retain completed actions for in case a `WaitExecution` is called after the action has completed. | +| `client_action_timeout_s` | integer (uint64) | — | 60 seconds | Mark operations as completed with error if no client has updated them within this duration. | +| `worker_timeout_s` | integer (uint64) | — | 5 seconds | Remove workers from pool once the worker has not responded in this amount of time in seconds. | | `max_action_executing_timeout_s` | integer (uint64) | — | 0 (disabled) | Maximum time (seconds) an action can stay in Executing state without any worker update before being timed out and re-queued. This applies regardless of worker keepalive status, catching cases where a worker is alive (sending keepalives) but stuck on a specific action. Set to 0 to disable (relies only on `worker_timeout_s`). | | `max_job_retries` | integer (uint) | — | 3 | If a job returns an internal error or times out this many times when attempting to run on a worker the scheduler will return the last error to the client. Jobs will be retried and this configuration is to help prevent one rogue job from infinitely retrying and taking up a lot of resources when the task itself is the one causing the server to go into a bad state. | | `allocation_strategy` | [WorkerAllocationStrategy](#workerallocationstrategy) | — | `"least_recently_used"` | The strategy used to assign workers jobs. | @@ -903,8 +897,8 @@ build at the main scheduler directly though. | --- | --- | --- | --- | --- | | `endpoint` | [GrpcEndpoint](#grpcendpoint) | Yes | — | The upstream scheduler to forward requests to. | | `retry` | [Retry](#retry) | — | — | Retry configuration to use when a network request fails. | -| `max_concurrent_requests` | integer (uint) | — | `0` | Limit the number of simultaneous upstream requests to this many. A value of zero is treated as unlimited. If the limit is reached the request is queued. | -| `connections_per_endpoint` | integer (uint) | — | `0` | The number of connections to make to each specified endpoint to balance the load over multiple TCP connections. Default 1. | +| `max_concurrent_requests` | integer (uint) | — | unlimited | Limit the number of simultaneous upstream requests to this many. A value of zero is treated as unlimited. If the limit is reached the request is queued. | +| `connections_per_endpoint` | integer (uint) | — | 1 | The number of connections to make to each specified endpoint to balance the load over multiple TCP connections. | ## CacheLookupSpec @@ -925,7 +919,7 @@ build at the main scheduler directly though. | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | | `hints_file` | string | Yes | — | JSON file containing historical resource hints keyed by Bazel `RequestMetadata` `target_id` and/or `action_mnemonic`. | -| `refresh_interval_s` | integer (uint64) | — | 30 (seconds) | Reload interval for `hints_file`. Set to 0 to load once. | +| `refresh_interval_s` | integer (uint64) | — | 30 seconds | Reload interval for `hints_file`. Set to 0 to load once. | | `cpu_property_name` | string | — | `cpu_count` | Platform property name used for CPU minimum values. | | `memory_property_name` | string | — | `memory_kb` | Platform property name used for memory minimum values, expressed in KiB. | | `scheduler` | [SchedulerSpec](#schedulerspec) | Yes | — | The nested scheduler to use after applying resource hints. | @@ -1594,6 +1588,7 @@ will result in: | 6 | 1.2 to 2 s | | 7 | 2.4 to 4 s | | 8 | 4.8 to 8 s | + The total delay is additive, so this example produces 9.525 to 15.875 s of total delay for a single request. | Field | Type | Required | Default | Description | @@ -1660,10 +1655,10 @@ Configuration for an individual shard of the store. | `address` | string | Yes | — | The endpoint address (i.e. grpc://example.com:443 or grpcs://example.com:443). | | `tls_config` | [ClientTlsConfig](#clienttlsconfig) | — | — | The TLS configuration to use to connect to the endpoint (if grpcs). | | `concurrency_limit` | integer (uint) | — | — | The maximum concurrency to allow on this endpoint. | -| `connect_timeout_s` | integer (uint64) | — | 0 (uses 30 seconds) | Timeout for establishing a TCP connection to the endpoint (seconds). | -| `tcp_keepalive_s` | integer (uint64) | — | 0 (uses 30 seconds) | TCP keepalive interval (seconds). Sends TCP keepalive probes at this interval to detect dead connections at the OS level. | -| `http2_keepalive_interval_s` | integer (uint64) | — | 0 (uses 30 seconds) | HTTP/2 keepalive interval (seconds). Sends HTTP/2 PING frames at this interval to detect dead connections at the application level. | -| `http2_keepalive_timeout_s` | integer (uint64) | — | 0 (uses 20 seconds) | HTTP/2 keepalive timeout (seconds). If a PING response is not received within this duration, the connection is considered dead. | +| `connect_timeout_s` | integer (uint64) | — | 30 seconds | Timeout for establishing a TCP connection to the endpoint (seconds). | +| `tcp_keepalive_s` | integer (uint64) | — | 30 seconds | TCP keepalive interval (seconds). Sends TCP keepalive probes at this interval to detect dead connections at the OS level. | +| `http2_keepalive_interval_s` | integer (uint64) | — | 30 seconds | HTTP/2 keepalive interval (seconds). Sends HTTP/2 PING frames at this interval to detect dead connections at the application level. | +| `http2_keepalive_timeout_s` | integer (uint64) | — | 20 seconds | HTTP/2 keepalive timeout (seconds). If a PING response is not received within this duration, the connection is considered dead. | ## StoreType @@ -1687,7 +1682,7 @@ Generic config for an endpoint and associated configs. | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | | `uri` | string | Yes | — | URI of the endpoint. | -| `timeout` | number (float) | — | 5 (seconds) | Timeout in seconds that a request should take. | +| `timeout` | number (float) | — | 5 seconds | Timeout in seconds that a request should take. | | `tls_config` | [ClientTlsConfig](#clienttlsconfig) | — | — | The TLS configuration to use to connect to the endpoint. | ## UploadActionResultConfig @@ -1695,9 +1690,9 @@ Generic config for an endpoint and associated configs. | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | | `ac_store` | string | — | {No uploading is done} | Underlying AC store that the worker will use to publish execution results into. Objects placed in this store should be reachable from the scheduler/client-cas after they have finished updating. | -| `upload_ac_results_strategy` | [UploadCacheResultsStrategy](#uploadcacheresultsstrategy) | — | `UploadCacheResultsStrategy::SuccessOnly` | In which situations should the results be published to the `ac_store`, if set to `SuccessOnly` then only results with an exit code of 0 will be uploaded, if set to Everything all completed results will be uploaded. | +| `upload_ac_results_strategy` | [UploadCacheResultsStrategy](#uploadcacheresultsstrategy) | — | `SuccessOnly` | In which situations should the results be published to the `ac_store`, if set to `SuccessOnly` then only results with an exit code of 0 will be uploaded, if set to Everything all completed results will be uploaded. | | `historical_results_store` | string | — | {CAS store of parent} | Store to upload historical results to. This should be a CAS store if set. | -| `upload_historical_results_strategy` | [UploadCacheResultsStrategy](#uploadcacheresultsstrategy) | — | `UploadCacheResultsStrategy::FailuresOnly` | In which situations should the results be published to the historical CAS. The historical CAS is where failures are published. These messages conform to the CAS key-value lookup format and are always a `HistoricalExecuteResponse` serialized message. | +| `upload_historical_results_strategy` | [UploadCacheResultsStrategy](#uploadcacheresultsstrategy) | — | `FailuresOnly` | In which situations should the results be published to the historical CAS. The historical CAS is where failures are published. These messages conform to the CAS key-value lookup format and are always a `HistoricalExecuteResponse` serialized message. | | `success_message_template` | string | — | "" (no message) | Template to use for the `ExecuteResponse.message` property. This message is attached to the response before it is sent to the client. The following special variables are supported:; `digest_function`: Digest function used to calculate the action digest: `action_digest_hash`: Action digest hash: `action_digest_size`: Action digest size: `historical_results_hash`: `HistoricalExecuteResponse` digest hash: `historical_results_size`: `HistoricalExecuteResponse` digest size. | | `failure_message_template` | string | — | "" (no message) | Same as `success_message_template` but for failure case. | @@ -1749,7 +1744,7 @@ One of: - string - `"minimum"` — Requires the platform property to be a u64 and when the scheduler looks for appropriate worker nodes that are capable of executing the task, the task will not run on a node that has less than this value. - `"exact"` — Requires the platform property to be a string and when the scheduler looks for appropriate worker nodes that are capable of executing the task, the task will not run on a node that does not have this property set to the value with exact string match. -- `"priority"` — Does not restrict on this value and instead will be passed to the worker as an informational piece. TODO(palfrey) In the future this will be used by the scheduler and worker to cause the scheduler to prefer certain workers over others, but not restrict them based on these values. +- `"priority"` — Does not restrict on this value and instead will be passed to the worker as an informational piece. In the future this will be used by the scheduler and worker to cause the scheduler to prefer certain workers over others, but not restrict them based on these values. ## WorkerAllocationStrategy @@ -1830,14 +1825,14 @@ If a property is found, then replace it with another one. | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `instance_name` | string | — | `""` | | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | | `cas_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. | ## ActionCacheServiceConfig | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `instance_name` | string | — | `""` | | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | | `ac_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. | | `read_only` | boolean | — | `false` | Whether the Action Cache store may be written to, this if set to false it is only possible to read from the Action Cache. | @@ -1845,14 +1840,14 @@ If a property is found, then replace it with another one. | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `instance_name` | string | — | `""` | | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | | `remote_execution` | [CapabilitiesRemoteExecutionConfig](#capabilitiesremoteexecutionconfig) | — | — | Configuration for remote execution capabilities. If not set the capabilities service will inform the client that remote execution is not supported. | ## ExecutionServiceConfig | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `instance_name` | string | — | `""` | | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | | `cas_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. This value must be a CAS store reference. | | `scheduler` | string | Yes | — | The scheduler name referenced in the `schedulers` map in the main config. | @@ -1860,23 +1855,23 @@ If a property is found, then replace it with another one. | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `instance_name` | string | — | `""` | | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | | `cas_store` | string | Yes | — | Name of the store in the "stores" configuration. | | `max_bytes_per_stream` | integer (uint) | — | 64KiB | Max number of bytes to send on each grpc stream chunk. According to [https://github.com/grpc/grpc.github.io/issues/371](https://github.com/grpc/grpc.github.io/issues/371) 16KiB - 64KiB is optimal. | -| `persist_stream_on_disconnect_timeout` | integer (uint) | — | 10 (seconds) | In the event a client disconnects while uploading a blob, we will hold the internal stream open for this many seconds before closing it. This allows clients that disconnect to reconnect and continue uploading the same blob. | +| `persist_stream_on_disconnect_timeout` | integer (uint) | — | 10 seconds | In the event a client disconnects while uploading a blob, we will hold the internal stream open for this many seconds before closing it. This allows clients that disconnect to reconnect and continue uploading the same blob. | ## FetchServiceConfig | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `instance_name` | string | — | `""` | | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | | `fetch_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. | ## PushServiceConfig | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `instance_name` | string | — | `""` | | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | | `push_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. | | `read_only` | boolean | — | `false` | Whether the Action Cache store may be written to, this if set to false it is only possible to read from the Action Cache. | @@ -1903,7 +1898,7 @@ If a property is found, then replace it with another one. | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | | `path` | string | — | "/status" | Path to register the health status check. If path is "/status", and your domain is "example.com", you can reach the endpoint with: [http://example.com/status](http://example.com/status). | -| `timeout_seconds` | integer (uint64) | — | `0` | | +| `timeout_seconds` | integer (uint64) | — | 5s | Timeout on health checks. | ## ErrorCode @@ -1933,7 +1928,7 @@ The possible error codes that might occur on an upstream request. | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | | `block_size` | integer (uint32) | — | 65536 (64k) | Size of the blocks to compress. Higher values require more ram, but might yield slightly better compression ratios. | -| `max_decode_block_size` | integer (uint32) | — | value in `block_size` | Maximum size allowed to attempt to deserialize data into. This is needed because the `block_size` is embedded into the data so if there was a bad actor, they could upload an extremely large `block_size`'ed entry and we'd allocate a large amount of memory when retrieving the data. To prevent this from happening, we allow you to specify the maximum that we'll attempt deserialize. | +| `max_decode_block_size` | integer (uint32) | — | value in `block_size` | Maximum size allowed to attempt to deserialize data into. This is needed because the `block_size` is embedded into the data so if there was a bad actor, they could upload an extremely large `block_size`'ed entry and we'd allocate a large amount of memory when retrieving the data. To prevent this from happening, we allow you to specify the maximum that we'll attempt to deserialize. | ## ClientTlsConfig diff --git a/web/apps/docs/content/docs/reference/nativelink-config/v1.0.0.mdx b/web/apps/docs/content/docs/reference/nativelink-config/v1.0.0.mdx index 7d66419d2..cc4e46aca 100644 --- a/web/apps/docs/content/docs/reference/nativelink-config/v1.0.0.mdx +++ b/web/apps/docs/content/docs/reference/nativelink-config/v1.0.0.mdx @@ -6,7 +6,7 @@ full: true {/* AUTOGENERATED — do not edit by hand. Source: nativelink-config @ v1.0.0 (42212c51) - Regenerate: bun run --filter=@nativelink/docs gen:config-reference */} + Regenerate from web/: bun --filter @nativelink/docs gen:config-reference */} @@ -653,12 +653,7 @@ Plus exactly one of the following variants (the key selects the variant): ## ExperimentalCloudObjectSpec -One of: - -- `provider`: `"aws"` -- `provider`: `"gcs"` -- `provider`: `"azure"` -- object +See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for details ## OntapS3ExistenceCacheSpec @@ -949,6 +944,7 @@ will result in: | 6 | 1.2 to 2 s | | 7 | 2.4 to 4 s | | 8 | 4.8 to 8 s | + The total delay is additive, so this example produces 9.525 to 15.875 s of total delay for a single request. | Field | Type | Required | Default | Description | @@ -1562,10 +1558,10 @@ Configuration for an individual shard of the store. | `address` | string | Yes | — | The endpoint address (i.e. grpc://example.com:443 or grpcs://example.com:443). | | `tls_config` | [ClientTlsConfig](#clienttlsconfig) | — | — | The TLS configuration to use to connect to the endpoint (if grpcs). | | `concurrency_limit` | integer (uint) | — | — | The maximum concurrency to allow on this endpoint. | -| `connect_timeout_s` | integer (uint64) | — | 0 (uses 30 seconds) | Timeout for establishing a TCP connection to the endpoint (seconds). | -| `tcp_keepalive_s` | integer (uint64) | — | 0 (uses 30 seconds) | TCP keepalive interval (seconds). Sends TCP keepalive probes at this interval to detect dead connections at the OS level. | -| `http2_keepalive_interval_s` | integer (uint64) | — | 0 (uses 30 seconds) | HTTP/2 keepalive interval (seconds). Sends HTTP/2 PING frames at this interval to detect dead connections at the application level. | -| `http2_keepalive_timeout_s` | integer (uint64) | — | 0 (uses 20 seconds) | HTTP/2 keepalive timeout (seconds). If a PING response is not received within this duration, the connection is considered dead. | +| `connect_timeout_s` | integer (uint64) | — | 30 seconds | Timeout for establishing a TCP connection to the endpoint (seconds). | +| `tcp_keepalive_s` | integer (uint64) | — | 30 seconds | TCP keepalive interval (seconds). Sends TCP keepalive probes at this interval to detect dead connections at the OS level. | +| `http2_keepalive_interval_s` | integer (uint64) | — | 30 seconds | HTTP/2 keepalive interval (seconds). Sends HTTP/2 PING frames at this interval to detect dead connections at the application level. | +| `http2_keepalive_timeout_s` | integer (uint64) | — | 20 seconds | HTTP/2 keepalive timeout (seconds). If a PING response is not received within this duration, the connection is considered dead. | ## StoreType @@ -1651,7 +1647,7 @@ One of: - string - `"minimum"` — Requires the platform property to be a u64 and when the scheduler looks for appropriate worker nodes that are capable of executing the task, the task will not run on a node that has less than this value. - `"exact"` — Requires the platform property to be a string and when the scheduler looks for appropriate worker nodes that are capable of executing the task, the task will not run on a node that does not have this property set to the value with exact string match. -- `"priority"` — Does not restrict on this value and instead will be passed to the worker as an informational piece. TODO(palfrey) In the future this will be used by the scheduler and worker to cause the scheduler to prefer certain workers over others, but not restrict them based on these values. +- `"priority"` — Does not restrict on this value and instead will be passed to the worker as an informational piece. In the future this will be used by the scheduler and worker to cause the scheduler to prefer certain workers over others, but not restrict them based on these values. ## WorkerAllocationStrategy diff --git a/web/apps/docs/content/docs/reference/nativelink-config/v1.1.0.mdx b/web/apps/docs/content/docs/reference/nativelink-config/v1.1.0.mdx index 7523f1b84..410d1a6b1 100644 --- a/web/apps/docs/content/docs/reference/nativelink-config/v1.1.0.mdx +++ b/web/apps/docs/content/docs/reference/nativelink-config/v1.1.0.mdx @@ -6,7 +6,7 @@ full: true {/* AUTOGENERATED — do not edit by hand. Source: nativelink-config @ v1.1.0 (7911affb) - Regenerate: bun run --filter=@nativelink/docs gen:config-reference */} + Regenerate from web/: bun --filter @nativelink/docs gen:config-reference */} @@ -663,12 +663,7 @@ Plus exactly one of the following variants (the key selects the variant): ## ExperimentalCloudObjectSpec -One of: - -- `provider`: `"aws"` -- `provider`: `"gcs"` -- `provider`: `"azure"` -- object +See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for details ## OntapS3ExistenceCacheSpec @@ -965,6 +960,7 @@ will result in: | 6 | 1.2 to 2 s | | 7 | 2.4 to 4 s | | 8 | 4.8 to 8 s | + The total delay is additive, so this example produces 9.525 to 15.875 s of total delay for a single request. | Field | Type | Required | Default | Description | @@ -1588,10 +1584,10 @@ Configuration for an individual shard of the store. | `address` | string | Yes | — | The endpoint address (i.e. grpc://example.com:443 or grpcs://example.com:443). | | `tls_config` | [ClientTlsConfig](#clienttlsconfig) | — | — | The TLS configuration to use to connect to the endpoint (if grpcs). | | `concurrency_limit` | integer (uint) | — | — | The maximum concurrency to allow on this endpoint. | -| `connect_timeout_s` | integer (uint64) | — | 0 (uses 30 seconds) | Timeout for establishing a TCP connection to the endpoint (seconds). | -| `tcp_keepalive_s` | integer (uint64) | — | 0 (uses 30 seconds) | TCP keepalive interval (seconds). Sends TCP keepalive probes at this interval to detect dead connections at the OS level. | -| `http2_keepalive_interval_s` | integer (uint64) | — | 0 (uses 30 seconds) | HTTP/2 keepalive interval (seconds). Sends HTTP/2 PING frames at this interval to detect dead connections at the application level. | -| `http2_keepalive_timeout_s` | integer (uint64) | — | 0 (uses 20 seconds) | HTTP/2 keepalive timeout (seconds). If a PING response is not received within this duration, the connection is considered dead. | +| `connect_timeout_s` | integer (uint64) | — | 30 seconds | Timeout for establishing a TCP connection to the endpoint (seconds). | +| `tcp_keepalive_s` | integer (uint64) | — | 30 seconds | TCP keepalive interval (seconds). Sends TCP keepalive probes at this interval to detect dead connections at the OS level. | +| `http2_keepalive_interval_s` | integer (uint64) | — | 30 seconds | HTTP/2 keepalive interval (seconds). Sends HTTP/2 PING frames at this interval to detect dead connections at the application level. | +| `http2_keepalive_timeout_s` | integer (uint64) | — | 20 seconds | HTTP/2 keepalive timeout (seconds). If a PING response is not received within this duration, the connection is considered dead. | ## StoreType @@ -1677,7 +1673,7 @@ One of: - string - `"minimum"` — Requires the platform property to be a u64 and when the scheduler looks for appropriate worker nodes that are capable of executing the task, the task will not run on a node that has less than this value. - `"exact"` — Requires the platform property to be a string and when the scheduler looks for appropriate worker nodes that are capable of executing the task, the task will not run on a node that does not have this property set to the value with exact string match. -- `"priority"` — Does not restrict on this value and instead will be passed to the worker as an informational piece. TODO(palfrey) In the future this will be used by the scheduler and worker to cause the scheduler to prefer certain workers over others, but not restrict them based on these values. +- `"priority"` — Does not restrict on this value and instead will be passed to the worker as an informational piece. In the future this will be used by the scheduler and worker to cause the scheduler to prefer certain workers over others, but not restrict them based on these values. ## WorkerAllocationStrategy diff --git a/web/apps/docs/content/docs/reference/nativelink-config/v1.2.0.mdx b/web/apps/docs/content/docs/reference/nativelink-config/v1.2.0.mdx index 7f7b41ab1..b25c586b0 100644 --- a/web/apps/docs/content/docs/reference/nativelink-config/v1.2.0.mdx +++ b/web/apps/docs/content/docs/reference/nativelink-config/v1.2.0.mdx @@ -6,7 +6,7 @@ full: true {/* AUTOGENERATED — do not edit by hand. Source: nativelink-config @ v1.2.0 (a5741fc8) - Regenerate: bun run --filter=@nativelink/docs gen:config-reference */} + Regenerate from web/: bun --filter @nativelink/docs gen:config-reference */} @@ -663,12 +663,7 @@ Plus exactly one of the following variants (the key selects the variant): ## ExperimentalCloudObjectSpec -One of: - -- `provider`: `"aws"` -- `provider`: `"gcs"` -- `provider`: `"azure"` -- object +See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for details ## OntapS3ExistenceCacheSpec @@ -965,6 +960,7 @@ will result in: | 6 | 1.2 to 2 s | | 7 | 2.4 to 4 s | | 8 | 4.8 to 8 s | + The total delay is additive, so this example produces 9.525 to 15.875 s of total delay for a single request. | Field | Type | Required | Default | Description | @@ -1588,10 +1584,10 @@ Configuration for an individual shard of the store. | `address` | string | Yes | — | The endpoint address (i.e. grpc://example.com:443 or grpcs://example.com:443). | | `tls_config` | [ClientTlsConfig](#clienttlsconfig) | — | — | The TLS configuration to use to connect to the endpoint (if grpcs). | | `concurrency_limit` | integer (uint) | — | — | The maximum concurrency to allow on this endpoint. | -| `connect_timeout_s` | integer (uint64) | — | 0 (uses 30 seconds) | Timeout for establishing a TCP connection to the endpoint (seconds). | -| `tcp_keepalive_s` | integer (uint64) | — | 0 (uses 30 seconds) | TCP keepalive interval (seconds). Sends TCP keepalive probes at this interval to detect dead connections at the OS level. | -| `http2_keepalive_interval_s` | integer (uint64) | — | 0 (uses 30 seconds) | HTTP/2 keepalive interval (seconds). Sends HTTP/2 PING frames at this interval to detect dead connections at the application level. | -| `http2_keepalive_timeout_s` | integer (uint64) | — | 0 (uses 20 seconds) | HTTP/2 keepalive timeout (seconds). If a PING response is not received within this duration, the connection is considered dead. | +| `connect_timeout_s` | integer (uint64) | — | 30 seconds | Timeout for establishing a TCP connection to the endpoint (seconds). | +| `tcp_keepalive_s` | integer (uint64) | — | 30 seconds | TCP keepalive interval (seconds). Sends TCP keepalive probes at this interval to detect dead connections at the OS level. | +| `http2_keepalive_interval_s` | integer (uint64) | — | 30 seconds | HTTP/2 keepalive interval (seconds). Sends HTTP/2 PING frames at this interval to detect dead connections at the application level. | +| `http2_keepalive_timeout_s` | integer (uint64) | — | 20 seconds | HTTP/2 keepalive timeout (seconds). If a PING response is not received within this duration, the connection is considered dead. | ## StoreType @@ -1677,7 +1673,7 @@ One of: - string - `"minimum"` — Requires the platform property to be a u64 and when the scheduler looks for appropriate worker nodes that are capable of executing the task, the task will not run on a node that has less than this value. - `"exact"` — Requires the platform property to be a string and when the scheduler looks for appropriate worker nodes that are capable of executing the task, the task will not run on a node that does not have this property set to the value with exact string match. -- `"priority"` — Does not restrict on this value and instead will be passed to the worker as an informational piece. TODO(palfrey) In the future this will be used by the scheduler and worker to cause the scheduler to prefer certain workers over others, but not restrict them based on these values. +- `"priority"` — Does not restrict on this value and instead will be passed to the worker as an informational piece. In the future this will be used by the scheduler and worker to cause the scheduler to prefer certain workers over others, but not restrict them based on these values. ## WorkerAllocationStrategy diff --git a/web/apps/docs/content/docs/reference/nativelink-config/v1.3.0.mdx b/web/apps/docs/content/docs/reference/nativelink-config/v1.3.0.mdx index a4892c7d8..7cb96983f 100644 --- a/web/apps/docs/content/docs/reference/nativelink-config/v1.3.0.mdx +++ b/web/apps/docs/content/docs/reference/nativelink-config/v1.3.0.mdx @@ -6,7 +6,7 @@ full: true {/* AUTOGENERATED — do not edit by hand. Source: nativelink-config @ v1.3.0 (f11e340b) - Regenerate: bun run --filter=@nativelink/docs gen:config-reference */} + Regenerate from web/: bun --filter @nativelink/docs gen:config-reference */} @@ -663,13 +663,7 @@ Plus exactly one of the following variants (the key selects the variant): ## ExperimentalCloudObjectSpec -One of: - -- `provider`: `"aws"` -- `provider`: `"gcs"` -- `provider`: `"azure"` -- object -- object +See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for details ## OntapS3ExistenceCacheSpec @@ -967,6 +961,7 @@ will result in: | 6 | 1.2 to 2 s | | 7 | 2.4 to 4 s | | 8 | 4.8 to 8 s | + The total delay is additive, so this example produces 9.525 to 15.875 s of total delay for a single request. | Field | Type | Required | Default | Description | @@ -1590,10 +1585,10 @@ Configuration for an individual shard of the store. | `address` | string | Yes | — | The endpoint address (i.e. grpc://example.com:443 or grpcs://example.com:443). | | `tls_config` | [ClientTlsConfig](#clienttlsconfig) | — | — | The TLS configuration to use to connect to the endpoint (if grpcs). | | `concurrency_limit` | integer (uint) | — | — | The maximum concurrency to allow on this endpoint. | -| `connect_timeout_s` | integer (uint64) | — | 0 (uses 30 seconds) | Timeout for establishing a TCP connection to the endpoint (seconds). | -| `tcp_keepalive_s` | integer (uint64) | — | 0 (uses 30 seconds) | TCP keepalive interval (seconds). Sends TCP keepalive probes at this interval to detect dead connections at the OS level. | -| `http2_keepalive_interval_s` | integer (uint64) | — | 0 (uses 30 seconds) | HTTP/2 keepalive interval (seconds). Sends HTTP/2 PING frames at this interval to detect dead connections at the application level. | -| `http2_keepalive_timeout_s` | integer (uint64) | — | 0 (uses 20 seconds) | HTTP/2 keepalive timeout (seconds). If a PING response is not received within this duration, the connection is considered dead. | +| `connect_timeout_s` | integer (uint64) | — | 30 seconds | Timeout for establishing a TCP connection to the endpoint (seconds). | +| `tcp_keepalive_s` | integer (uint64) | — | 30 seconds | TCP keepalive interval (seconds). Sends TCP keepalive probes at this interval to detect dead connections at the OS level. | +| `http2_keepalive_interval_s` | integer (uint64) | — | 30 seconds | HTTP/2 keepalive interval (seconds). Sends HTTP/2 PING frames at this interval to detect dead connections at the application level. | +| `http2_keepalive_timeout_s` | integer (uint64) | — | 20 seconds | HTTP/2 keepalive timeout (seconds). If a PING response is not received within this duration, the connection is considered dead. | ## StoreType @@ -1679,7 +1674,7 @@ One of: - string - `"minimum"` — Requires the platform property to be a u64 and when the scheduler looks for appropriate worker nodes that are capable of executing the task, the task will not run on a node that has less than this value. - `"exact"` — Requires the platform property to be a string and when the scheduler looks for appropriate worker nodes that are capable of executing the task, the task will not run on a node that does not have this property set to the value with exact string match. -- `"priority"` — Does not restrict on this value and instead will be passed to the worker as an informational piece. TODO(palfrey) In the future this will be used by the scheduler and worker to cause the scheduler to prefer certain workers over others, but not restrict them based on these values. +- `"priority"` — Does not restrict on this value and instead will be passed to the worker as an informational piece. In the future this will be used by the scheduler and worker to cause the scheduler to prefer certain workers over others, but not restrict them based on these values. ## WorkerAllocationStrategy diff --git a/web/apps/docs/content/docs/reference/nativelink-config/v1.3.1.mdx b/web/apps/docs/content/docs/reference/nativelink-config/v1.3.1.mdx index 4b77da64d..24c1f5891 100644 --- a/web/apps/docs/content/docs/reference/nativelink-config/v1.3.1.mdx +++ b/web/apps/docs/content/docs/reference/nativelink-config/v1.3.1.mdx @@ -6,7 +6,7 @@ full: true {/* AUTOGENERATED — do not edit by hand. Source: nativelink-config @ v1.3.1 (4120b19d) - Regenerate: bun run --filter=@nativelink/docs gen:config-reference */} + Regenerate from web/: bun --filter @nativelink/docs gen:config-reference */} @@ -663,13 +663,7 @@ Plus exactly one of the following variants (the key selects the variant): ## ExperimentalCloudObjectSpec -One of: - -- `provider`: `"aws"` -- `provider`: `"gcs"` -- `provider`: `"azure"` -- object -- object +See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for details ## OntapS3ExistenceCacheSpec @@ -967,6 +961,7 @@ will result in: | 6 | 1.2 to 2 s | | 7 | 2.4 to 4 s | | 8 | 4.8 to 8 s | + The total delay is additive, so this example produces 9.525 to 15.875 s of total delay for a single request. | Field | Type | Required | Default | Description | @@ -1590,10 +1585,10 @@ Configuration for an individual shard of the store. | `address` | string | Yes | — | The endpoint address (i.e. grpc://example.com:443 or grpcs://example.com:443). | | `tls_config` | [ClientTlsConfig](#clienttlsconfig) | — | — | The TLS configuration to use to connect to the endpoint (if grpcs). | | `concurrency_limit` | integer (uint) | — | — | The maximum concurrency to allow on this endpoint. | -| `connect_timeout_s` | integer (uint64) | — | 0 (uses 30 seconds) | Timeout for establishing a TCP connection to the endpoint (seconds). | -| `tcp_keepalive_s` | integer (uint64) | — | 0 (uses 30 seconds) | TCP keepalive interval (seconds). Sends TCP keepalive probes at this interval to detect dead connections at the OS level. | -| `http2_keepalive_interval_s` | integer (uint64) | — | 0 (uses 30 seconds) | HTTP/2 keepalive interval (seconds). Sends HTTP/2 PING frames at this interval to detect dead connections at the application level. | -| `http2_keepalive_timeout_s` | integer (uint64) | — | 0 (uses 20 seconds) | HTTP/2 keepalive timeout (seconds). If a PING response is not received within this duration, the connection is considered dead. | +| `connect_timeout_s` | integer (uint64) | — | 30 seconds | Timeout for establishing a TCP connection to the endpoint (seconds). | +| `tcp_keepalive_s` | integer (uint64) | — | 30 seconds | TCP keepalive interval (seconds). Sends TCP keepalive probes at this interval to detect dead connections at the OS level. | +| `http2_keepalive_interval_s` | integer (uint64) | — | 30 seconds | HTTP/2 keepalive interval (seconds). Sends HTTP/2 PING frames at this interval to detect dead connections at the application level. | +| `http2_keepalive_timeout_s` | integer (uint64) | — | 20 seconds | HTTP/2 keepalive timeout (seconds). If a PING response is not received within this duration, the connection is considered dead. | ## StoreType @@ -1679,7 +1674,7 @@ One of: - string - `"minimum"` — Requires the platform property to be a u64 and when the scheduler looks for appropriate worker nodes that are capable of executing the task, the task will not run on a node that has less than this value. - `"exact"` — Requires the platform property to be a string and when the scheduler looks for appropriate worker nodes that are capable of executing the task, the task will not run on a node that does not have this property set to the value with exact string match. -- `"priority"` — Does not restrict on this value and instead will be passed to the worker as an informational piece. TODO(palfrey) In the future this will be used by the scheduler and worker to cause the scheduler to prefer certain workers over others, but not restrict them based on these values. +- `"priority"` — Does not restrict on this value and instead will be passed to the worker as an informational piece. In the future this will be used by the scheduler and worker to cause the scheduler to prefer certain workers over others, but not restrict them based on these values. ## WorkerAllocationStrategy diff --git a/web/apps/docs/content/docs/reference/nativelink-config/v1.3.2.mdx b/web/apps/docs/content/docs/reference/nativelink-config/v1.3.2.mdx index 74e552a4b..27bf375fa 100644 --- a/web/apps/docs/content/docs/reference/nativelink-config/v1.3.2.mdx +++ b/web/apps/docs/content/docs/reference/nativelink-config/v1.3.2.mdx @@ -6,7 +6,7 @@ full: true {/* AUTOGENERATED — do not edit by hand. Source: nativelink-config @ v1.3.2 (b5e1f337) - Regenerate: bun run --filter=@nativelink/docs gen:config-reference */} + Regenerate from web/: bun --filter @nativelink/docs gen:config-reference */} @@ -694,13 +694,7 @@ Plus exactly one of the following variants (the key selects the variant): ## ExperimentalCloudObjectSpec -One of: - -- `provider`: `"aws"` -- `provider`: `"gcs"` -- `provider`: `"azure"` -- object -- object +See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for details ## OntapS3ExistenceCacheSpec @@ -1579,6 +1573,7 @@ will result in: | 6 | 1.2 to 2 s | | 7 | 2.4 to 4 s | | 8 | 4.8 to 8 s | + The total delay is additive, so this example produces 9.525 to 15.875 s of total delay for a single request. | Field | Type | Required | Default | Description | @@ -1645,10 +1640,10 @@ Configuration for an individual shard of the store. | `address` | string | Yes | — | The endpoint address (i.e. grpc://example.com:443 or grpcs://example.com:443). | | `tls_config` | [ClientTlsConfig](#clienttlsconfig) | — | — | The TLS configuration to use to connect to the endpoint (if grpcs). | | `concurrency_limit` | integer (uint) | — | — | The maximum concurrency to allow on this endpoint. | -| `connect_timeout_s` | integer (uint64) | — | 0 (uses 30 seconds) | Timeout for establishing a TCP connection to the endpoint (seconds). | -| `tcp_keepalive_s` | integer (uint64) | — | 0 (uses 30 seconds) | TCP keepalive interval (seconds). Sends TCP keepalive probes at this interval to detect dead connections at the OS level. | -| `http2_keepalive_interval_s` | integer (uint64) | — | 0 (uses 30 seconds) | HTTP/2 keepalive interval (seconds). Sends HTTP/2 PING frames at this interval to detect dead connections at the application level. | -| `http2_keepalive_timeout_s` | integer (uint64) | — | 0 (uses 20 seconds) | HTTP/2 keepalive timeout (seconds). If a PING response is not received within this duration, the connection is considered dead. | +| `connect_timeout_s` | integer (uint64) | — | 30 seconds | Timeout for establishing a TCP connection to the endpoint (seconds). | +| `tcp_keepalive_s` | integer (uint64) | — | 30 seconds | TCP keepalive interval (seconds). Sends TCP keepalive probes at this interval to detect dead connections at the OS level. | +| `http2_keepalive_interval_s` | integer (uint64) | — | 30 seconds | HTTP/2 keepalive interval (seconds). Sends HTTP/2 PING frames at this interval to detect dead connections at the application level. | +| `http2_keepalive_timeout_s` | integer (uint64) | — | 20 seconds | HTTP/2 keepalive timeout (seconds). If a PING response is not received within this duration, the connection is considered dead. | ## StoreType @@ -1734,7 +1729,7 @@ One of: - string - `"minimum"` — Requires the platform property to be a u64 and when the scheduler looks for appropriate worker nodes that are capable of executing the task, the task will not run on a node that has less than this value. - `"exact"` — Requires the platform property to be a string and when the scheduler looks for appropriate worker nodes that are capable of executing the task, the task will not run on a node that does not have this property set to the value with exact string match. -- `"priority"` — Does not restrict on this value and instead will be passed to the worker as an informational piece. TODO(palfrey) In the future this will be used by the scheduler and worker to cause the scheduler to prefer certain workers over others, but not restrict them based on these values. +- `"priority"` — Does not restrict on this value and instead will be passed to the worker as an informational piece. In the future this will be used by the scheduler and worker to cause the scheduler to prefer certain workers over others, but not restrict them based on these values. ## WorkerAllocationStrategy diff --git a/web/apps/docs/content/docs/reference/nativelink-config/v1.4.0.mdx b/web/apps/docs/content/docs/reference/nativelink-config/v1.4.0.mdx index 3e749cf85..789716942 100644 --- a/web/apps/docs/content/docs/reference/nativelink-config/v1.4.0.mdx +++ b/web/apps/docs/content/docs/reference/nativelink-config/v1.4.0.mdx @@ -6,7 +6,7 @@ full: true {/* AUTOGENERATED — do not edit by hand. Source: nativelink-config @ v1.4.0 (c9a7669b) - Regenerate: bun run --filter=@nativelink/docs gen:config-reference */} + Regenerate from web/: bun --filter @nativelink/docs gen:config-reference */} @@ -698,13 +698,7 @@ Plus exactly one of the following variants (the key selects the variant): ## ExperimentalCloudObjectSpec -One of: - -- `provider`: `"aws"` -- `provider`: `"gcs"` -- `provider`: `"azure"` -- object -- object +See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for details ## OntapS3ExistenceCacheSpec @@ -1593,6 +1587,7 @@ will result in: | 6 | 1.2 to 2 s | | 7 | 2.4 to 4 s | | 8 | 4.8 to 8 s | + The total delay is additive, so this example produces 9.525 to 15.875 s of total delay for a single request. | Field | Type | Required | Default | Description | @@ -1659,10 +1654,10 @@ Configuration for an individual shard of the store. | `address` | string | Yes | — | The endpoint address (i.e. grpc://example.com:443 or grpcs://example.com:443). | | `tls_config` | [ClientTlsConfig](#clienttlsconfig) | — | — | The TLS configuration to use to connect to the endpoint (if grpcs). | | `concurrency_limit` | integer (uint) | — | — | The maximum concurrency to allow on this endpoint. | -| `connect_timeout_s` | integer (uint64) | — | 0 (uses 30 seconds) | Timeout for establishing a TCP connection to the endpoint (seconds). | -| `tcp_keepalive_s` | integer (uint64) | — | 0 (uses 30 seconds) | TCP keepalive interval (seconds). Sends TCP keepalive probes at this interval to detect dead connections at the OS level. | -| `http2_keepalive_interval_s` | integer (uint64) | — | 0 (uses 30 seconds) | HTTP/2 keepalive interval (seconds). Sends HTTP/2 PING frames at this interval to detect dead connections at the application level. | -| `http2_keepalive_timeout_s` | integer (uint64) | — | 0 (uses 20 seconds) | HTTP/2 keepalive timeout (seconds). If a PING response is not received within this duration, the connection is considered dead. | +| `connect_timeout_s` | integer (uint64) | — | 30 seconds | Timeout for establishing a TCP connection to the endpoint (seconds). | +| `tcp_keepalive_s` | integer (uint64) | — | 30 seconds | TCP keepalive interval (seconds). Sends TCP keepalive probes at this interval to detect dead connections at the OS level. | +| `http2_keepalive_interval_s` | integer (uint64) | — | 30 seconds | HTTP/2 keepalive interval (seconds). Sends HTTP/2 PING frames at this interval to detect dead connections at the application level. | +| `http2_keepalive_timeout_s` | integer (uint64) | — | 20 seconds | HTTP/2 keepalive timeout (seconds). If a PING response is not received within this duration, the connection is considered dead. | ## StoreType @@ -1748,7 +1743,7 @@ One of: - string - `"minimum"` — Requires the platform property to be a u64 and when the scheduler looks for appropriate worker nodes that are capable of executing the task, the task will not run on a node that has less than this value. - `"exact"` — Requires the platform property to be a string and when the scheduler looks for appropriate worker nodes that are capable of executing the task, the task will not run on a node that does not have this property set to the value with exact string match. -- `"priority"` — Does not restrict on this value and instead will be passed to the worker as an informational piece. TODO(palfrey) In the future this will be used by the scheduler and worker to cause the scheduler to prefer certain workers over others, but not restrict them based on these values. +- `"priority"` — Does not restrict on this value and instead will be passed to the worker as an informational piece. In the future this will be used by the scheduler and worker to cause the scheduler to prefer certain workers over others, but not restrict them based on these values. ## WorkerAllocationStrategy diff --git a/web/apps/docs/content/docs/reference/nativelink-config/v1.5.0.mdx b/web/apps/docs/content/docs/reference/nativelink-config/v1.5.0.mdx index 50132e649..f83656b30 100644 --- a/web/apps/docs/content/docs/reference/nativelink-config/v1.5.0.mdx +++ b/web/apps/docs/content/docs/reference/nativelink-config/v1.5.0.mdx @@ -698,13 +698,7 @@ Plus exactly one of the following variants (the key selects the variant): ## ExperimentalCloudObjectSpec -One of: - -- `provider`: `"aws"` -- `provider`: `"gcs"` -- `provider`: `"azure"` -- object -- object +See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for details ## OntapS3ExistenceCacheSpec @@ -1594,6 +1588,7 @@ will result in: | 6 | 1.2 to 2 s | | 7 | 2.4 to 4 s | | 8 | 4.8 to 8 s | + The total delay is additive, so this example produces 9.525 to 15.875 s of total delay for a single request. | Field | Type | Required | Default | Description | @@ -1660,10 +1655,10 @@ Configuration for an individual shard of the store. | `address` | string | Yes | — | The endpoint address (i.e. grpc://example.com:443 or grpcs://example.com:443). | | `tls_config` | [ClientTlsConfig](#clienttlsconfig) | — | — | The TLS configuration to use to connect to the endpoint (if grpcs). | | `concurrency_limit` | integer (uint) | — | — | The maximum concurrency to allow on this endpoint. | -| `connect_timeout_s` | integer (uint64) | — | 0 (uses 30 seconds) | Timeout for establishing a TCP connection to the endpoint (seconds). | -| `tcp_keepalive_s` | integer (uint64) | — | 0 (uses 30 seconds) | TCP keepalive interval (seconds). Sends TCP keepalive probes at this interval to detect dead connections at the OS level. | -| `http2_keepalive_interval_s` | integer (uint64) | — | 0 (uses 30 seconds) | HTTP/2 keepalive interval (seconds). Sends HTTP/2 PING frames at this interval to detect dead connections at the application level. | -| `http2_keepalive_timeout_s` | integer (uint64) | — | 0 (uses 20 seconds) | HTTP/2 keepalive timeout (seconds). If a PING response is not received within this duration, the connection is considered dead. | +| `connect_timeout_s` | integer (uint64) | — | 30 seconds | Timeout for establishing a TCP connection to the endpoint (seconds). | +| `tcp_keepalive_s` | integer (uint64) | — | 30 seconds | TCP keepalive interval (seconds). Sends TCP keepalive probes at this interval to detect dead connections at the OS level. | +| `http2_keepalive_interval_s` | integer (uint64) | — | 30 seconds | HTTP/2 keepalive interval (seconds). Sends HTTP/2 PING frames at this interval to detect dead connections at the application level. | +| `http2_keepalive_timeout_s` | integer (uint64) | — | 20 seconds | HTTP/2 keepalive timeout (seconds). If a PING response is not received within this duration, the connection is considered dead. | ## StoreType @@ -1749,7 +1744,7 @@ One of: - string - `"minimum"` — Requires the platform property to be a u64 and when the scheduler looks for appropriate worker nodes that are capable of executing the task, the task will not run on a node that has less than this value. - `"exact"` — Requires the platform property to be a string and when the scheduler looks for appropriate worker nodes that are capable of executing the task, the task will not run on a node that does not have this property set to the value with exact string match. -- `"priority"` — Does not restrict on this value and instead will be passed to the worker as an informational piece. TODO(palfrey) In the future this will be used by the scheduler and worker to cause the scheduler to prefer certain workers over others, but not restrict them based on these values. +- `"priority"` — Does not restrict on this value and instead will be passed to the worker as an informational piece. In the future this will be used by the scheduler and worker to cause the scheduler to prefer certain workers over others, but not restrict them based on these values. ## WorkerAllocationStrategy diff --git a/web/apps/docs/content/docs/reference/nativelink-config/v1.5.1.mdx b/web/apps/docs/content/docs/reference/nativelink-config/v1.5.1.mdx new file mode 100644 index 000000000..998dc3d6e --- /dev/null +++ b/web/apps/docs/content/docs/reference/nativelink-config/v1.5.1.mdx @@ -0,0 +1,2041 @@ +--- +title: Configuration reference +description: Every knob in the NativeLink JSON5 configuration — types, defaults, and links to source, autogenerated from the Rust config crate. +full: true +--- + +{/* AUTOGENERATED — do not edit by hand. + Source: nativelink-config @ v1.5.1 (3d9d74ab) + Regenerate from web/: bun --filter @nativelink/docs gen:config-reference */} + + + +This is the canonical NativeLink configuration reference for **v1.5.1**. +It is autogenerated from the Rust config crate +([`nativelink-config/src`](https://github.com/TraceMachina/nativelink/tree/v1.5.1/nativelink-config/src)) via the `build-schema` binary, so +it can never drift from what the binary actually deserializes. + +## Top-level fields + +The root object (`CasConfig`) accepts the following fields: + +| Field | Type | Required | Description | +| --- | --- | --- | --- | +| `stores` | array of [NamedStoreConfig](#namedstoreconfig) | Yes | List of stores available to use in this config. The keys can be used in other configs when needing to reference a store. | +| `workers` | array of [WorkerConfig](#workerconfig) | — | Worker configurations used to execute jobs. | +| `schedulers` | array of [NamedSchedulerConfig](#namedschedulerconfig) | — | List of schedulers available to use in this config. The keys can be used in other configs when needing to reference a scheduler. | +| `servers` | array of [ServerConfig](#serverconfig) | Yes | Servers to setup for this process. | +| `experimental_origin_events` | [OriginEventsSpec](#origineventsspec) | — | Experimental - Origin events configuration. This is the service that will collect and publish nativelink events to a store for processing by an external service. | +| `global` | [GlobalConfig](#globalconfig) | — | Any global configurations that apply to all modules live here. | + +## Configuration types + +Every type reachable from the root configuration, in reading order. + +## NamedStoreConfig + +**Common fields** + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | Yes | — | | + +Plus exactly one of the following variants (the key selects the variant): + +### `cache_metrics` + +Cache metrics store wraps another store and emits low-cardinality +OpenTelemetry cache operation metrics for the wrapped store. + +This wrapper is opt-in. Stores that are not explicitly wrapped by +`cache_metrics` are constructed exactly as they are without this +wrapper and do not pay its hot-path timing or recording cost. + +**Example JSON5 config:** +```json5 +"cache_metrics": { + "cache_type": "cas", + "backend": { + "filesystem": { + "content_path": "~/.cache/nativelink/content_path-cas", + "temp_path": "~/.cache/nativelink/tmp_path-cas" + } + } +} +``` + +**Type:** [CacheMetricsSpec](#cachemetricsspec) + +### `memory` + +Memory store will store all data in a hash map in memory. + +**Example JSON5 config:** +```json5 +"memory": { + "eviction_policy": { + "max_bytes": "10mb", + } +} +``` + +**Type:** [MemorySpec](#memoryspec) + +### `experimental_cloud_object_store` + +A generic blob store that will store files on the cloud +provider. This configuration will never delete files, so you are +responsible for purging old files in other ways. +It supports the following backends: + +1. **Amazon S3:** + S3 store will use Amazon's S3 service as a backend to store + the files. This configuration can be used to share files + across multiple instances. Uses system certificates for TLS + verification via `rustls-platform-verifier`. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "aws", + "region": "eu-north-1", + "bucket": "crossplane-bucket-af79aeca9", + "key_prefix": "test-prefix-index/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +2. **Google Cloud Storage:** + GCS store uses Google's GCS service as a backend to store + the files. This configuration can be used to share files + across multiple instances. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "gcs", + "bucket": "test-bucket", + "key_prefix": "test-prefix-index/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +3. **Azure Blob Store:** + Azure Blob store will use Microsoft's Azure Blob service as a + backend to store the files. This configuration can be used to + share files across multiple instances. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "azure", + "account_name": "cloudshell1393657559", + "container": "simple-test-container", + "key_prefix": "folder/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +4. **`NetApp` ONTAP S3** + `NetApp` ONTAP S3 store will use ONTAP's S3-compatible storage as a backend + to store files. This store is specifically configured for ONTAP's S3 requirements + including custom TLS configuration, credentials management, and proper vserver + configuration. + + This store uses AWS environment variables for credentials: + - `AWS_ACCESS_KEY_ID` + - `AWS_SECRET_ACCESS_KEY` + - `AWS_DEFAULT_REGION` + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "ontap", + "endpoint": "https://ontap-s3-endpoint:443", + "vserver_name": "your-vserver", + "bucket": "your-bucket", + "root_certificates": "/path/to/certs.pem", // Optional + "key_prefix": "test-prefix/", // Optional + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +**Type:** [ExperimentalCloudObjectSpec](#experimentalcloudobjectspec) + +### `ontap_s3_existence_cache` + +ONTAP S3 Existence Cache provides a caching layer on top of the ONTAP S3 store +to optimize repeated existence checks. It maintains an in-memory cache of object +digests and periodically syncs this cache to disk for persistence. + +The cache helps reduce latency for repeated calls to check object existence, +while still ensuring eventual consistency with the underlying ONTAP S3 store. + +Example JSON5 config: +```json5 +"ontap_s3_existence_cache": { + "index_path": "/path/to/cache/index.json", + "sync_interval_seconds": 300, + "backend": { + "endpoint": "https://ontap-s3-endpoint:443", + "vserver_name": "your-vserver", + "bucket": "your-bucket", + "key_prefix": "test-prefix/" + } +} +``` + +**Type:** [OntapS3ExistenceCacheSpec](#ontaps3existencecachespec) + +### `verify` + +Verify store is used to apply verifications to an underlying +store implementation. It is strongly encouraged to validate +as much data as you can before accepting data from a client, +failing to do so may cause the data in the store to be +populated with invalid data causing all kinds of problems. + +The suggested configuration is to have the CAS validate the +hash and size and the AC validate nothing. + +**Example JSON5 config:** +```json5 +"verify": { + "backend": { + "memory": { + "eviction_policy": { + "max_bytes": "500mb" + } + }, + }, + "verify_size": true, + "verify_hash": true +} +``` + +**Type:** [VerifySpec](#verifyspec) + +### `completeness_checking` + +Completeness checking store verifies if the +output files & folders exist in the CAS before forwarding +the request to the underlying store. +Note: This store should only be used on AC stores. + +**Example JSON5 config:** +```json5 +"completeness_checking": { + "backend": { + "filesystem": { + "content_path": "~/.cache/nativelink/content_path-ac", + "temp_path": "~/.cache/nativelink/tmp_path-ac", + "eviction_policy": { + "max_bytes": "500mb", + } + } + }, + "cas_store": { + "ref_store": { + "name": "CAS_MAIN_STORE" + } + } +} +``` + +**Type:** [CompletenessCheckingSpec](#completenesscheckingspec) + +### `compression` + +A compression store that will compress the data inbound and +outbound. There will be a non-trivial cost to compress and +decompress the data, but in many cases if the final store is +a store that requires network transport and/or storage space +is a concern it is often faster and more efficient to use this +store before those stores. + +**Example JSON5 config:** +```json5 +"compression": { + "compression_algorithm": { + "lz4": {} + }, + "backend": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-cas", + "temp_path": "/tmp/nativelink/data/tmp_path-cas", + "eviction_policy": { + "max_bytes": "2gb", + } + } + } +} +``` + +**Type:** [CompressionSpec](#compressionspec) + +### `dedup` + +A dedup store will take the inputs and run a rolling hash +algorithm on them to slice the input into smaller parts then +run a sha256 algorithm on the slice and if the object doesn't +already exist, upload the slice to the `content_store` using +a new digest of just the slice. Once all parts exist, an +Action-Cache-like digest will be built and uploaded to the +`index_store` which will contain a reference to each +chunk/digest of the uploaded file. Downloading a request will +first grab the index from the `index_store`, and forward the +download content of each chunk as if it were one file. + +This store is exceptionally good when the following conditions +are met: +* Content is mostly the same (inserts, updates, deletes are ok) +* Content is not compressed or encrypted +* Uploading or downloading from `content_store` is the bottleneck. + +Note: This store pairs well when used with `CompressionSpec` as +the `content_store`, but never put `DedupSpec` as the backend of +`CompressionSpec` as it will negate all the gains. + +Note: When running `.has()` on this store, it will only check +to see if the entry exists in the `index_store` and not check +if the individual chunks exist in the `content_store`. + +**Example JSON5 config:** +```json5 +"dedup": { + "index_store": { + "memory": { + "eviction_policy": { + "max_bytes": "1GB", + } + } + }, + "content_store": { + "compression": { + "compression_algorithm": { + "lz4": {} + }, + "backend": { + "fast_slow": { + "fast": { + "memory": { + "eviction_policy": { + "max_bytes": "500MB", + } + } + }, + "slow": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-content", + "temp_path": "/tmp/nativelink/data/tmp_path-content", + "eviction_policy": { + "max_bytes": "2gb" + } + } + } + } + } + } + } +} +``` + +**Type:** [DedupSpec](#dedupspec) + +### `existence_cache` + +Existence store will wrap around another store and cache calls +to has so that subsequent `has_with_results` calls will be +faster. This is useful for cases when you have a store that +is slow to respond to has calls. +Note: This store should only be used on CAS stores. + +**Example JSON5 config:** +```json5 +"existence_cache": { + "backend": { + "memory": { + "eviction_policy": { + "max_bytes": "500mb", + } + } + }, + // Note this is the existence store policy, not the backend policy + "eviction_policy": { + "max_seconds": 100, + } +} +``` + +**Type:** [ExistenceCacheSpec](#existencecachespec) + +### `fast_slow` + +`FastSlow` store will first try to fetch the data from the `fast` +store and then if it does not exist try the `slow` store. +When the object does exist in the `slow` store, it will copy +the data to the `fast` store while returning the data. +This store should be thought of as a store that "buffers" +the data to the `fast` store. +On uploads it will mirror data to both `fast` and `slow` stores. + +WARNING: If you need data to always exist in the `slow` store +for something like remote execution, be careful because this +store will never check to see if the objects exist in the +`slow` store if it exists in the `fast` store (i.e., it assumes +that if an object exists in the `fast` store it will exist in +the `slow` store). + +***Example JSON5 config:*** +```json5 +"fast_slow": { + "fast": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-index", + "temp_path": "/tmp/nativelink/data/tmp_path-index", + "eviction_policy": { + "max_bytes": "500mb", + } + } + }, + "slow": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-index", + "temp_path": "/tmp/nativelink/data/tmp_path-index", + "eviction_policy": { + "max_bytes": "500mb", + } + } + } +} +``` + +**Type:** [FastSlowSpec](#fastslowspec) + +### `shard` + +Shards the data to multiple stores. This is useful for cases +when you want to distribute the load across multiple stores. +The digest hash is used to determine which store to send the +data to. + +**Example JSON5 config:** +```json5 +"shard": { + "stores": [ + { + "store": { + "memory": { + "eviction_policy": { + "max_bytes": "10mb" + }, + }, + }, + "weight": 1 + }] +} +``` + +**Type:** [ShardSpec](#shardspec) + +### `filesystem` + +Stores the data on the filesystem. This store is designed for +local persistent storage. Restarts of this program should restore +the previous state, meaning anything uploaded will be persistent +as long as the filesystem integrity holds. + +**Example JSON5 config:** +```json5 +"filesystem": { + "content_path": "/tmp/nativelink/data-worker-test/content_path-cas", + "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas", + "eviction_policy": { + "max_bytes": "10gb", + } +} +``` + +**Type:** [FilesystemSpec](#filesystemspec) + +### `ref_store` + +Store used to reference a store in the root store manager. +This is useful for cases when you want to share a store in different +nested stores. Example, you may want to share the same memory store +used for the action cache, but use a `FastSlowSpec` and have the fast +store also share the memory store for efficiency. + +**Example JSON5 config:** +```json5 +"ref_store": { + "name": "FS_CONTENT_STORE" +} +``` + +**Type:** [RefSpec](#refspec) + +### `size_partitioning` + +Uses the size field of the digest to separate which store to send the +data. This is useful for cases when you'd like to put small objects +in one store and large objects in another store. This should only be +used if the size field is the real size of the content, in other +words, don't use on AC (Action Cache) stores. Any store where you can +safely use `VerifySpec.verify_size = true`, this store should be safe +to use (i.e., CAS stores). + +**Example JSON5 config:** +```json5 +"size_partitioning": { + "size": "128mib", + "lower_store": { + "memory": { + "eviction_policy": { + "max_bytes": "${NATIVELINK_CAS_MEMORY_CONTENT_LIMIT:-100mb}" + } + } + }, + "upper_store": { + /// This store discards data larger than 128mib. + "noop": {} + } +} +``` + +**Type:** [SizePartitioningSpec](#sizepartitioningspec) + +### `grpc` + +This store will pass-through calls to another GRPC store. This store +is not designed to be used as a sub-store of another store, but it +does satisfy the interface and will likely work. + +One major GOTCHA is that some stores use a special function on this +store to get the size of the underlying object, which is only reliable +when this store is serving the a CAS store, not an AC store. If using +this store directly without being a child of any store there are no +side effects and is the most efficient way to use it. + +**Example JSON5 config:** +```json5 +"grpc": { + "instance_name": "main", + "endpoints": [ + {"address": "grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"} + ], + "connections_per_endpoint": "5", + "rpc_timeout_s": "5m", + "store_type": "ac", + // Static headers attached to every outgoing request to the upstream + // remote cache. Useful for fixed service-account credentials. + "headers": { + "authorization": "Bearer my-static-token" + }, + // Header names to copy from the inbound client request and forward to + // the upstream remote cache. Use this to pass through dynamic + // credentials such as a JWT sent by the build client. + "forward_headers": ["authorization", "x-custom-token"] +} +``` + +**Type:** [GrpcSpec](#grpcspec) + +### `redis_store` + +Stores data in any stores compatible with Redis APIs. + +Pairs well with `SizePartitioning` and/or `FastSlow` stores. +Ideal for accepting small object sizes as most Redis store +services have a max file upload of between 256Mb-512Mb. + +**Example JSON5 config:** +```json5 +"redis_store": { + "addresses": [ + "redis://127.0.0.1:6379/", + ], + "max_client_permits": 1000, +} +``` + +**Type:** [RedisSpec](#redisspec) + +### `noop` + +Noop store is a store that sends streams into the void and all data +retrieval will return 404 (`NotFound`). This can be useful for cases +where you may need to partition your data and part of your data needs +to be discarded. + +**Example JSON5 config:** +```json5 +"noop": {} +``` + +**Type:** [NoopSpec](#noopspec) + +### `experimental_mongo` + +Experimental `MongoDB` store implementation. + +This store uses `MongoDB` as a backend for storing data. It supports +both CAS (Content Addressable Storage) and scheduler data with +optional change streams for real-time updates. + +**Example JSON5 config:** +```json5 +"experimental_mongo": { + "connection_string": "mongodb://localhost:27017", + "database": "nativelink", + "cas_collection": "cas", + "key_prefix": "cas:", + "read_chunk_size": 65536, + "max_concurrent_uploads": 10, + "enable_change_streams": false, + "max_requests": "100" +} +``` + +**Type:** [ExperimentalMongoSpec](#experimentalmongospec) + +## WorkerConfig + +Plus exactly one of the following variants (the key selects the variant): + +### `local` + +A worker type that executes jobs locally on this machine. + +**Type:** [LocalWorkerConfig](#localworkerconfig) + +## NamedSchedulerConfig + +**Common fields** + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | Yes | — | | + +Plus exactly one of the following variants (the key selects the variant): + +### `simple` + +**Type:** [SimpleSpec](#simplespec) + +### `grpc` + +**Type:** [SchedulerGrpcSpec](#schedulergrpcspec) + +### `cache_lookup` + +**Type:** [CacheLookupSpec](#cachelookupspec) + +### `property_modifier` + +**Type:** [PropertyModifierSpec](#propertymodifierspec) + +### `historical_resource` + +**Type:** [HistoricalResourceSpec](#historicalresourcespec) + +## ServerConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | — | {index of server in config} | Name of the server. This is used to help identify the service for telemetry and logs. | +| `listener` | [ListenerConfig](#listenerconfig) | Yes | — | Configuration | +| `services` | [ServicesConfig](#servicesconfig) | — | — | Services to attach to server. | +| `experimental_identity_header` | [IdentityHeaderSpec](#identityheaderspec) | — | {see `IdentityHeaderSpec`} | The config related to identifying the client. | + +## OriginEventsSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `publisher` | [OriginEventsPublisherSpec](#origineventspublisherspec) | Yes | — | The publisher configuration for origin events. | +| `max_event_queue_size` | integer (uint) | — | 65536 (zero defaults to this) | The maximum number of events to queue before applying back pressure. IMPORTANT: Backpressure causes all clients to slow down significantly. Zero is default. | + +## GlobalConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `max_open_files` | integer (uint) | Yes | 24576 (= 24 * 1024) | Maximum number of open files that can be opened at one time. This value is not strictly enforced, it is a best effort. Some internal libraries open files or read metadata from a files which do not obey this limit, however the vast majority of cases will have this limit be honored. This value must be larger than `ulimit -n` to have any effect. Any network open file descriptors is not counted in this limit, but is counted in the kernel limit. It is a good idea to set a very large `ulimit -n`. Note: This value must be greater than 10. | +| `default_digest_hash_function` | [ConfigDigestHashFunction](#configdigesthashfunction) | — | `ConfigDigestHashFunction::sha256` | Default hash function to use while uploading blobs to the CAS when not set by client. | +| `default_digest_size_health_check` | integer (uint) | — | 1024*1024 (1MiB) | Default digest size to use for health check when running diagnostics checks. Health checks are expected to use this size for filling a buffer that is used for creation of digest. | + +## CacheMetricsSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `cache_type` | string | Yes | — | Low-cardinality cache type label for metrics, for example `cas` or `ac`. | +| `backend` | [StoreSpec](#storespec) | Yes | — | Store to wrap with cache operation metrics. | + +## MemorySpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | + +## ExperimentalCloudObjectSpec + +See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for details + +## OntapS3ExistenceCacheSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `index_path` | string | Yes | — | | +| `sync_interval_seconds` | integer (uint32) | Yes | — | | +| `backend` | [ExperimentalOntapS3Spec](#experimentalontaps3spec) | Yes | — | | + +## VerifySpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `verify_size` | boolean | — | `false` | If set the store will verify the size of the data before accepting an upload of data. | +| `verify_hash` | boolean | — | `false` | If the data should be hashed and verify that the key matches the computed hash. The hash function is automatically determined based request and if not set will use the global default. | + +## CompletenessCheckingSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store that will have it's results validated before sending to client. | +| `cas_store` | [StoreSpec](#storespec) | Yes | — | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | + +## CompressionSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `compression_algorithm` | [CompressionAlgorithm](#compressionalgorithm) | Yes | — | The compression algorithm to use. | + +## DedupSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `index_store` | [StoreSpec](#storespec) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | +| `content_store` | [StoreSpec](#storespec) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | +| `min_size` | integer (uint32) | — | 65536 (64k) | Minimum size that a chunk will be when slicing up the content. Note: This setting can be increased to improve performance because it will actually not check this number of bytes when deciding where to partition the data. | +| `normal_size` | integer (uint32) | — | 262144 (256k) | A best-effort attempt will be made to keep the average size of the chunks to this number. It is not a guarantee, but a slight attempt will be made. | +| `max_size` | integer (uint32) | — | 524288 (512k) | Maximum size a chunk is allowed to be. | +| `max_concurrent_fetch_per_get` | integer (uint32) | — | 10 | Due to implementation detail, we want to prefer to download the first chunks of the file so we can stream the content out and free up some of our buffers. This configuration will be used to restrict the number of concurrent chunk downloads at a time per `get()` request. | + +## ExistenceCacheSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | + +## FastSlowSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `fast` | [StoreSpec](#storespec) | Yes | — | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | +| `fast_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the fast store. This can be useful to set to Get for worker nodes such that results are persisted to the slow store only. | +| `slow` | [StoreSpec](#storespec) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | +| `slow_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the slow store. This can be useful if creating a diode and you wish to have an upstream read only store. | +| `bypass_dedup_threshold_bytes` | integer (uint64) | — | `0` | Reads of blobs at or above this size skip the leader/follower dedup map and stream straight from the slow store without populating the fast tier. `0` (the default) disables the bypass: every read goes through dedup, matching the prior behaviour. Enable it by setting a threshold — 256 MiB is a reasonable starting point for backends where large-blob dedup is a net loss (followers tend to time out anyway), but the right value is workload-dependent. | + +## ShardSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `stores` | array of [ShardConfig](#shardconfig) | Yes | — | Stores to shard the data to. | + +## FilesystemSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `content_path` | string | Yes | — | Path on the system where to store the actual content. This is where the bulk of the data will be placed. On service startup this folder will be scanned and all files will be added to the cache. In the event one of the files doesn't match the criteria, the file will be deleted. | +| `temp_path` | string | Yes | — | A temporary location of where files that are being uploaded or deleted will be placed while the content cannot be guaranteed to be accurate. This location must be on the same block device as `content_path` so atomic moves can happen (i.e., move without copy). All files in this folder will be deleted on every startup. | +| `read_buffer_size` | integer (uint32) | — | 32k | Buffer size to use when reading files. Generally this should be left to the default value except for testing. | +| `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | +| `block_size` | integer (uint64) | — | 4096 | The block size of the filesystem for the running machine value is used to determine an entry's actual size on disk consumed For a 4KB block size filesystem, a 1B file actually consumes 4KB | +| `max_concurrent_writes` | integer (uint) | — | 0 | Maximum number of concurrent write operations allowed. Each write involves streaming data to a temp file and calling `sync_all()`, which can saturate disk I/O when many writes happen simultaneously. Limiting concurrency prevents disk saturation from blocking the async runtime. A value of 0 means unlimited (no concurrency limit). | + +## RefSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | Yes | — | Name of the store under the root "stores" config object. | + +## SizePartitioningSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `size` | integer (uint64) | Yes | — | Size to partition the data on. | +| `lower_store` | [StoreSpec](#storespec) | Yes | — | Store to send data when object is < (less than) size. | +| `upper_store` | [StoreSpec](#storespec) | Yes | — | Store to send data when object is >= (less than eq) size. | + +## GrpcSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Instance name for GRPC calls. Proxy calls will have the `instance_name` changed to this. | +| `endpoints` | array of [GrpcEndpoint](#grpcendpoint) | Yes | — | The endpoint of the grpc connection. | +| `store_type` | [StoreType](#storetype) | Yes | — | The type of the upstream store, this ensures that the correct server calls are made. | +| `retry` | [Retry](#retry) | — | — | Retry configuration to use when a network request fails. | +| `max_concurrent_requests` | integer (uint) | — | `0` | Limit the number of simultaneous upstream requests to this many. A value of zero is treated as unlimited. If the limit is reached the request is queued. | +| `connections_per_endpoint` | integer (uint) | — | `0` | The number of connections to make to each specified endpoint to balance the load over multiple TCP connections. Default 1. | +| `rpc_timeout_s` | integer (uint64) | — | 0 (disabled) | Maximum time (seconds) allowed for a single RPC request (e.g. a `ByteStream.Write` call) before it is cancelled. | +| `use_legacy_resource_names` | boolean | — | false | Use legacy `ByteStream` resource name format, omitting the digest function component from the path. | +| `headers` | map of string to string | — | — | Static headers to attach to every outgoing gRPC request sent to this store's upstream endpoints. Useful for fixed authentication tokens (e.g. `{"authorization": "Bearer "}`) and other static metadata. | +| `forward_headers` | array of string | — | — | Header names to forward from the incoming client request to every outgoing upstream request. The header value is taken from the client request that triggered this store operation. Use this to pass through dynamic credentials such as JWT tokens sent by build clients. | + +## RedisSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `addresses` | array of string | Yes | — | The hostname or IP address of the Redis server. Ex: `["redis://username:password@redis-server-url:6380/99"]` 99 Represents database ID, 6380 represents the port. | +| `response_timeout_s` | integer (uint64) | — | 10 | DEPRECATED: use `command_timeout_ms` The response timeout for the Redis connection in seconds. | +| `connection_timeout_s` | integer (uint64) | — | 10 | DEPRECATED: use `connection_timeout_ms` | +| `experimental_pub_sub_channel` | string | — | (Empty String / No Channel) | An optional and experimental Redis channel to publish write events to. | +| `key_prefix` | string | — | (Empty String / No Prefix) | An optional prefix to prepend to all keys in this store. | +| `mode` | [RedisMode](#redismode) | — | standard, | Set the mode Redis is operating in. | +| `broadcast_channel_capacity` | integer (uint) | — | `0` | Deprecated as redis-rs doesn't use it | +| `command_timeout_ms` | integer (uint64) | — | 10000 (10 seconds) | The amount of time in milliseconds until the Redis store considers the command to be timed out. This will trigger a retry of the command and potentially a reconnection to the Redis server. | +| `connection_timeout_ms` | integer (uint64) | — | 3000 (3 seconds) | The amount of time in milliseconds until the Redis store considers the connection to unresponsive. This will trigger a reconnection to the Redis server. | +| `health_check_timeout_ms` | integer (uint64) | — | 4000 (4 seconds) | Per-call ceiling for the `check_health` PING in milliseconds. | +| `read_chunk_size` | integer (uint) | — | 64KiB | The amount of data to read from the Redis server at a time. This is used to limit the amount of memory used when reading large objects from the Redis server as well as limiting the amount of time a single read operation can take. | +| `connection_pool_size` | integer (uint) | — | 3 | The number of connections to keep open to the Redis servers. | +| `max_chunk_uploads_per_update` | integer (uint) | — | 10 | The maximum number of upload chunks to allow per update. This is used to limit the amount of memory used when uploading large objects to the Redis server. A good rule of thumb is to think of the data as: `AVAIL_MEMORY / (read_chunk_size * max_chunk_uploads_per_update) = THORETICAL_MAX_CONCURRENT_UPLOADS` (note: it is a good idea to divide `AVAIL_MAX_MEMORY` by ~10 to account for other memory usage) | +| `scan_count` | integer (uint) | — | 10000 | The COUNT value passed when scanning keys in Redis. This is used to hint the amount of work that should be done per response. | +| `retry` | [Retry](#retry) | — | Retry { | Retry configuration to use when a network request fails. See the `Retry` struct for more information. | +| `max_client_permits` | integer (uint) | — | 500 | Maximum number of permitted actions to the Redis store at any one time This stops problems with timeouts due to many, many inflight actions | +| `max_count_per_cursor` | integer (uint64) | — | 1500 | Maximum number of items returned per cursor for the search indexes May reduce thundering herd issues with worker provisioner at higher node counts, | + +## NoopSpec + +_No fields._ + +## ExperimentalMongoSpec + +Configuration for `ExperimentalMongoDB` store. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `connection_string` | string | Yes | — | `ExperimentalMongoDB` connection string. Example: <mongodb://localhost:27017> or <mongodb+srv://cluster.mongodb.net> | +| `database` | string | — | "nativelink" | The database name to use. | +| `cas_collection` | string | — | "cas" | The collection name for CAS data. | +| `scheduler_collection` | string | — | "scheduler" | The collection name for scheduler data. | +| `key_prefix` | string | — | "" | Prefix to prepend to all keys stored in `MongoDB`. | +| `read_chunk_size` | integer (uint) | — | 65536 (64KB) | The maximum amount of data to read from `MongoDB` in a single chunk (in bytes). | +| `max_concurrent_uploads` | integer (uint) | — | 10 | Deprecated, unused Maximum number of concurrent uploads allowed. | +| `connection_timeout_ms` | integer (uint64) | — | 3000 | Connection timeout in milliseconds. | +| `command_timeout_ms` | integer (uint64) | — | 10000 | Command timeout in milliseconds. | +| `enable_change_streams` | boolean | — | false | Enable `MongoDB` change streams for real-time updates. Required for scheduler subscriptions. | +| `write_concern_w` | string | — | — | Write concern 'w' parameter. Can be a number (e.g., 1) or string (e.g., "majority"). | +| `write_concern_j` | boolean | — | — | Write concern 'j' parameter (journal acknowledgment). | +| `write_concern_timeout_ms` | integer (uint32) | — | — | Write concern timeout in milliseconds. | +| `max_requests` | integer (uint) | — | Unlimited | Limits the number of requests at any one time | + +## LocalWorkerConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | — | {Index position in the workers list} | Name of the worker. This is give a more friendly name to a worker for logging and metric publishing. This is also the prefix of the worker id (i.e., "{name}{uuidv6}"). | +| `worker_api_endpoint` | [EndpointConfig](#endpointconfig) | Yes | — | Endpoint which the worker will connect to the scheduler's `WorkerApiService`. | +| `max_action_timeout` | integer (uint) | — | 1200 (seconds / 20 minutes) | The maximum time an action is allowed to run. If a task requests for a timeout longer than this time limit, the task will be rejected. Value in seconds. | +| `max_upload_timeout` | integer (uint) | — | 600 (seconds / 10 minutes) | Maximum time allowed for uploading action results to CAS after execution completes. If upload takes longer than this, the action fails with `DeadlineExceeded` and may be retried by the scheduler. Value in seconds. | +| `max_inflight_tasks` | integer (uint64) | — | 0 (infinite tasks) | Maximum number of inflight tasks this worker can cope with. | +| `timeout_handled_externally` | boolean | — | false (`NativeLink` fully handles timeouts) | If timeout is handled in `entrypoint` or another wrapper script. If set to true `NativeLink` will not honor the timeout the action requested and instead will always force kill the action after `max_action_timeout` has been reached. If this is set to false, the smaller value of the action's timeout and `max_action_timeout` will be used to which `NativeLink` will kill the action. | +| `entrypoint` | string | — | {Use the command from the job request} | The command to execute on every execution request. This will be parsed as a command + arguments (not shell). Example: "run.sh" and a job with command: "sleep 5" will result in a command like: "run.sh sleep 5". | +| `experimental_precondition_script` | string | — | — | An optional script to run before every action is processed on the worker. The value should be the full path to the script to execute and will pause all actions on the worker if it returns an exit code other than 0. If not set, then the worker will never pause and will continue to accept jobs according to the scheduler configuration. This is useful, for example, if the worker should not take any more actions until there is enough resource available on the machine to handle them. | +| `cas_fast_slow_store` | string | Yes | — | Underlying CAS store that the worker will use to download CAS artifacts. This store must be a `FastSlowStore`. The `fast` store must be a `FileSystemStore` because it will use hardlinks when building out the files instead of copying the files. The slow store must eventually resolve to the same store the scheduler/client uses to send job requests. | +| `upload_action_result` | [UploadActionResultConfig](#uploadactionresultconfig) | — | — | Configuration for uploading action results. | +| `work_directory` | string | Yes | — | The directory work jobs will be executed from. This directory will be fully managed by the worker service and will be purged on startup. This directory and the directory referenced in `local_filesystem_store_ref`'s `stores::FilesystemStore::content_path` must be on the same filesystem. Hardlinks will be used when placing files that are accessible to the jobs that are sourced from `local_filesystem_store_ref`'s `content_path`. | +| `platform_properties` | map of string to [WorkerProperty](#workerproperty) | Yes | — | Properties of this worker. This configuration will be sent to the scheduler and used to tell the scheduler to restrict what should be executed on this worker. | +| `additional_environment` | map of string to [EnvironmentSource](#environmentsource) | — | — | An optional mapping of environment names to set for the execution as well as those specified in the action itself. If set, will set each key as an environment variable before executing the job with the value of the environment variable being the value of the property of the action being executed of that name or the fixed value. | +| `directory_cache` | [DirectoryCacheConfig](#directorycacheconfig) | — | — | Optional directory cache configuration for improving performance by caching reconstructed input directories and using hardlinks instead of rebuilding them from CAS for every action. | +| `use_namespaces` | boolean | — | False | Whether to use namespaces to isolate the execution. This is only available on Linux. It is highly recommended as it avoids a number of issues with zombie processes and also provides additional hermeticity. If explicitly set to true and it is not supported the worker will exit with an error. | +| `use_mount_namespace` | boolean | — | False | Whether to use a mount namespace to isolate the worker root. This is only available on Linux and when `use_namespaces` is true. It is highly recommended provides additional hermeticity. If explicitly set to true and it is not supported or `use_namespaces` is not set to true the worker will exit with an error. | + +## SimpleSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `supported_platform_properties` | map of string to [PropertyType](#propertytype) | — | — | A list of supported platform properties mapped to how these properties are used when the scheduler looks for worker nodes capable of running the task. | +| `retain_completed_for_s` | integer (uint32) | — | 60 (seconds) | The amount of time to retain completed actions for in case a `WaitExecution` is called after the action has completed. | +| `client_action_timeout_s` | integer (uint64) | — | 60 (seconds) | Mark operations as completed with error if no client has updated them within this duration. | +| `worker_timeout_s` | integer (uint64) | — | 5 (seconds) | Remove workers from pool once the worker has not responded in this amount of time in seconds. | +| `max_action_executing_timeout_s` | integer (uint64) | — | 0 (disabled) | Maximum time (seconds) an action can stay in Executing state without any worker update before being timed out and re-queued. This applies regardless of worker keepalive status, catching cases where a worker is alive (sending keepalives) but stuck on a specific action. Set to 0 to disable (relies only on `worker_timeout_s`). | +| `max_job_retries` | integer (uint) | — | 3 | If a job returns an internal error or times out this many times when attempting to run on a worker the scheduler will return the last error to the client. Jobs will be retried and this configuration is to help prevent one rogue job from infinitely retrying and taking up a lot of resources when the task itself is the one causing the server to go into a bad state. | +| `allocation_strategy` | [WorkerAllocationStrategy](#workerallocationstrategy) | — | `"least_recently_used"` | The strategy used to assign workers jobs. | +| `experimental_backend` | [ExperimentalSimpleSchedulerBackend](#experimentalsimpleschedulerbackend) | — | memory | The storage backend to use for the scheduler. | +| `worker_match_logging_interval_s` | integer (int64) | — | `10` | Every N seconds, do logging of worker matching e.g. "worker busy", "can't find any worker" Defaults to 10s. Can be set to `-1` to disable | + +## SchedulerGrpcSpec + +A scheduler that forwards requests to an upstream scheduler. This +is useful to use when doing some kind of local action cache or CAS away from +the main cluster of workers. In general, it's more efficient to point the +build at the main scheduler directly though. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `endpoint` | [GrpcEndpoint](#grpcendpoint) | Yes | — | The upstream scheduler to forward requests to. | +| `retry` | [Retry](#retry) | — | — | Retry configuration to use when a network request fails. | +| `max_concurrent_requests` | integer (uint) | — | `0` | Limit the number of simultaneous upstream requests to this many. A value of zero is treated as unlimited. If the limit is reached the request is queued. | +| `connections_per_endpoint` | integer (uint) | — | `0` | The number of connections to make to each specified endpoint to balance the load over multiple TCP connections. Default 1. | + +## CacheLookupSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `ac_store` | string | Yes | — | The reference to the action cache store used to return cached actions from rather than running them again. To prevent unintended issues, this store should probably be a `CompletenessCheckingSpec`. | +| `scheduler` | [SchedulerSpec](#schedulerspec) | Yes | — | The nested scheduler to use if cache lookup fails. | + +## PropertyModifierSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `modifications` | array of [PropertyModification](#propertymodification) | Yes | — | A list of modifications to perform to incoming actions for the nested scheduler. These are performed in order and blindly, so removing a property that doesn't exist is fine and overwriting an existing property is also fine. If adding properties that do not exist in the nested scheduler is not supported and will likely cause unexpected behaviour. | +| `scheduler` | [SchedulerSpec](#schedulerspec) | Yes | — | The nested scheduler to use after modifying the properties. | + +## HistoricalResourceSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `hints_file` | string | Yes | — | JSON file containing historical resource hints keyed by Bazel `RequestMetadata` `target_id` and/or `action_mnemonic`. | +| `refresh_interval_s` | integer (uint64) | — | 30 (seconds) | Reload interval for `hints_file`. Set to 0 to load once. | +| `cpu_property_name` | string | — | `cpu_count` | Platform property name used for CPU minimum values. | +| `memory_property_name` | string | — | `memory_kb` | Platform property name used for memory minimum values, expressed in KiB. | +| `scheduler` | [SchedulerSpec](#schedulerspec) | Yes | — | The nested scheduler to use after applying resource hints. | + +## ListenerConfig + +Plus exactly one of the following variants (the key selects the variant): + +### `http` + +Listener for HTTP/HTTPS/HTTP2 sockets. + +**Type:** [HttpListener](#httplistener) + +## ServicesConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `cas` | array of [CasServiceConfig](#casserviceconfig) | — | — | The Content Addressable Storage (CAS) backend config. The key is the `instance_name` used in the protocol and the value is the underlying CAS store config. | +| `ac` | array of [ActionCacheServiceConfig](#actioncacheserviceconfig) | — | — | The Action Cache (AC) backend config. The key is the `instance_name` used in the protocol and the value is the underlying AC store config. | +| `capabilities` | array of [CapabilitiesServiceConfig](#capabilitiesserviceconfig) | — | — | Capabilities service is required in order to use most of the bazel protocol. This service is used to provide the supported features and versions of this bazel GRPC service. | +| `execution` | array of [ExecutionServiceConfig](#executionserviceconfig) | — | — | The remote execution service configuration. NOTE: This service is under development and is currently just a place holder. | +| `bytestream` | array of [ByteStreamServiceConfig](#bytestreamserviceconfig) | — | — | This is the service used to stream data to and from the CAS. Bazel's protocol strongly encourages users to use this streaming interface to interact with the CAS when the data is large. | +| `fetch` | array of [FetchServiceConfig](#fetchserviceconfig) | — | — | These two are collectively the Remote Asset protocol, but it's defined as two separate services | +| `push` | array of [PushServiceConfig](#pushserviceconfig) | — | — | | +| `worker_api` | [WorkerApiConfig](#workerapiconfig) | — | — | This is the service used for workers to connect and communicate through. NOTE: This service should be served on a different, non-public port. In other words, `worker_api` configuration should not have any other services that are served on the same port. Doing so is a security risk, as workers have a different permission set than a client that makes the remote execution/cache requests. | +| `experimental_bep` | [BepConfig](#bepconfig) | — | — | Experimental - Build Event Protocol (BEP) configuration. This is the service that will consume build events from the client and publish them to a store for processing by an external service. | +| `admin` | [AdminConfig](#adminconfig) | — | — | This is the service for any administrative tasks. It provides a REST API endpoint for administrative purposes. | +| `health` | [HealthConfig](#healthconfig) | — | — | This is the service for health status check. | + +## IdentityHeaderSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `header_name` | string | — | "x-identity" | The name of the header to look for the identity in. | +| `required` | boolean | — | `false` | If the header is required to be set or fail the request. | + +## OriginEventsPublisherSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `store` | string | Yes | — | The store to publish nativelink events to. The store name referenced in the `stores` map in the main config. | + +## ConfigDigestHashFunction + +| Value | Description | +| --- | --- | +| `"sha256"` | Use the sha256 hash function. [https://en.wikipedia.org/wiki/SHA-2](https://en.wikipedia.org/wiki/SHA-2) | +| `"blake3"` | Use the blake3 hash function. [https://en.wikipedia.org/wiki/BLAKE_(hash_function)](https://en.wikipedia.org/wiki/BLAKE_(hash_function)) | + +## StoreSpec + +Plus exactly one of the following variants (the key selects the variant): + +### `cache_metrics` + +Cache metrics store wraps another store and emits low-cardinality +OpenTelemetry cache operation metrics for the wrapped store. + +This wrapper is opt-in. Stores that are not explicitly wrapped by +`cache_metrics` are constructed exactly as they are without this +wrapper and do not pay its hot-path timing or recording cost. + +**Example JSON5 config:** +```json5 +"cache_metrics": { + "cache_type": "cas", + "backend": { + "filesystem": { + "content_path": "~/.cache/nativelink/content_path-cas", + "temp_path": "~/.cache/nativelink/tmp_path-cas" + } + } +} +``` + +**Type:** [CacheMetricsSpec](#cachemetricsspec) + +### `memory` + +Memory store will store all data in a hash map in memory. + +**Example JSON5 config:** +```json5 +"memory": { + "eviction_policy": { + "max_bytes": "10mb", + } +} +``` + +**Type:** [MemorySpec](#memoryspec) + +### `experimental_cloud_object_store` + +A generic blob store that will store files on the cloud +provider. This configuration will never delete files, so you are +responsible for purging old files in other ways. +It supports the following backends: + +1. **Amazon S3:** + S3 store will use Amazon's S3 service as a backend to store + the files. This configuration can be used to share files + across multiple instances. Uses system certificates for TLS + verification via `rustls-platform-verifier`. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "aws", + "region": "eu-north-1", + "bucket": "crossplane-bucket-af79aeca9", + "key_prefix": "test-prefix-index/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +2. **Google Cloud Storage:** + GCS store uses Google's GCS service as a backend to store + the files. This configuration can be used to share files + across multiple instances. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "gcs", + "bucket": "test-bucket", + "key_prefix": "test-prefix-index/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +3. **Azure Blob Store:** + Azure Blob store will use Microsoft's Azure Blob service as a + backend to store the files. This configuration can be used to + share files across multiple instances. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "azure", + "account_name": "cloudshell1393657559", + "container": "simple-test-container", + "key_prefix": "folder/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +4. **`NetApp` ONTAP S3** + `NetApp` ONTAP S3 store will use ONTAP's S3-compatible storage as a backend + to store files. This store is specifically configured for ONTAP's S3 requirements + including custom TLS configuration, credentials management, and proper vserver + configuration. + + This store uses AWS environment variables for credentials: + - `AWS_ACCESS_KEY_ID` + - `AWS_SECRET_ACCESS_KEY` + - `AWS_DEFAULT_REGION` + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "ontap", + "endpoint": "https://ontap-s3-endpoint:443", + "vserver_name": "your-vserver", + "bucket": "your-bucket", + "root_certificates": "/path/to/certs.pem", // Optional + "key_prefix": "test-prefix/", // Optional + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +**Type:** [ExperimentalCloudObjectSpec](#experimentalcloudobjectspec) + +### `ontap_s3_existence_cache` + +ONTAP S3 Existence Cache provides a caching layer on top of the ONTAP S3 store +to optimize repeated existence checks. It maintains an in-memory cache of object +digests and periodically syncs this cache to disk for persistence. + +The cache helps reduce latency for repeated calls to check object existence, +while still ensuring eventual consistency with the underlying ONTAP S3 store. + +Example JSON5 config: +```json5 +"ontap_s3_existence_cache": { + "index_path": "/path/to/cache/index.json", + "sync_interval_seconds": 300, + "backend": { + "endpoint": "https://ontap-s3-endpoint:443", + "vserver_name": "your-vserver", + "bucket": "your-bucket", + "key_prefix": "test-prefix/" + } +} +``` + +**Type:** [OntapS3ExistenceCacheSpec](#ontaps3existencecachespec) + +### `verify` + +Verify store is used to apply verifications to an underlying +store implementation. It is strongly encouraged to validate +as much data as you can before accepting data from a client, +failing to do so may cause the data in the store to be +populated with invalid data causing all kinds of problems. + +The suggested configuration is to have the CAS validate the +hash and size and the AC validate nothing. + +**Example JSON5 config:** +```json5 +"verify": { + "backend": { + "memory": { + "eviction_policy": { + "max_bytes": "500mb" + } + }, + }, + "verify_size": true, + "verify_hash": true +} +``` + +**Type:** [VerifySpec](#verifyspec) + +### `completeness_checking` + +Completeness checking store verifies if the +output files & folders exist in the CAS before forwarding +the request to the underlying store. +Note: This store should only be used on AC stores. + +**Example JSON5 config:** +```json5 +"completeness_checking": { + "backend": { + "filesystem": { + "content_path": "~/.cache/nativelink/content_path-ac", + "temp_path": "~/.cache/nativelink/tmp_path-ac", + "eviction_policy": { + "max_bytes": "500mb", + } + } + }, + "cas_store": { + "ref_store": { + "name": "CAS_MAIN_STORE" + } + } +} +``` + +**Type:** [CompletenessCheckingSpec](#completenesscheckingspec) + +### `compression` + +A compression store that will compress the data inbound and +outbound. There will be a non-trivial cost to compress and +decompress the data, but in many cases if the final store is +a store that requires network transport and/or storage space +is a concern it is often faster and more efficient to use this +store before those stores. + +**Example JSON5 config:** +```json5 +"compression": { + "compression_algorithm": { + "lz4": {} + }, + "backend": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-cas", + "temp_path": "/tmp/nativelink/data/tmp_path-cas", + "eviction_policy": { + "max_bytes": "2gb", + } + } + } +} +``` + +**Type:** [CompressionSpec](#compressionspec) + +### `dedup` + +A dedup store will take the inputs and run a rolling hash +algorithm on them to slice the input into smaller parts then +run a sha256 algorithm on the slice and if the object doesn't +already exist, upload the slice to the `content_store` using +a new digest of just the slice. Once all parts exist, an +Action-Cache-like digest will be built and uploaded to the +`index_store` which will contain a reference to each +chunk/digest of the uploaded file. Downloading a request will +first grab the index from the `index_store`, and forward the +download content of each chunk as if it were one file. + +This store is exceptionally good when the following conditions +are met: +* Content is mostly the same (inserts, updates, deletes are ok) +* Content is not compressed or encrypted +* Uploading or downloading from `content_store` is the bottleneck. + +Note: This store pairs well when used with `CompressionSpec` as +the `content_store`, but never put `DedupSpec` as the backend of +`CompressionSpec` as it will negate all the gains. + +Note: When running `.has()` on this store, it will only check +to see if the entry exists in the `index_store` and not check +if the individual chunks exist in the `content_store`. + +**Example JSON5 config:** +```json5 +"dedup": { + "index_store": { + "memory": { + "eviction_policy": { + "max_bytes": "1GB", + } + } + }, + "content_store": { + "compression": { + "compression_algorithm": { + "lz4": {} + }, + "backend": { + "fast_slow": { + "fast": { + "memory": { + "eviction_policy": { + "max_bytes": "500MB", + } + } + }, + "slow": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-content", + "temp_path": "/tmp/nativelink/data/tmp_path-content", + "eviction_policy": { + "max_bytes": "2gb" + } + } + } + } + } + } + } +} +``` + +**Type:** [DedupSpec](#dedupspec) + +### `existence_cache` + +Existence store will wrap around another store and cache calls +to has so that subsequent `has_with_results` calls will be +faster. This is useful for cases when you have a store that +is slow to respond to has calls. +Note: This store should only be used on CAS stores. + +**Example JSON5 config:** +```json5 +"existence_cache": { + "backend": { + "memory": { + "eviction_policy": { + "max_bytes": "500mb", + } + } + }, + // Note this is the existence store policy, not the backend policy + "eviction_policy": { + "max_seconds": 100, + } +} +``` + +**Type:** [ExistenceCacheSpec](#existencecachespec) + +### `fast_slow` + +`FastSlow` store will first try to fetch the data from the `fast` +store and then if it does not exist try the `slow` store. +When the object does exist in the `slow` store, it will copy +the data to the `fast` store while returning the data. +This store should be thought of as a store that "buffers" +the data to the `fast` store. +On uploads it will mirror data to both `fast` and `slow` stores. + +WARNING: If you need data to always exist in the `slow` store +for something like remote execution, be careful because this +store will never check to see if the objects exist in the +`slow` store if it exists in the `fast` store (i.e., it assumes +that if an object exists in the `fast` store it will exist in +the `slow` store). + +***Example JSON5 config:*** +```json5 +"fast_slow": { + "fast": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-index", + "temp_path": "/tmp/nativelink/data/tmp_path-index", + "eviction_policy": { + "max_bytes": "500mb", + } + } + }, + "slow": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-index", + "temp_path": "/tmp/nativelink/data/tmp_path-index", + "eviction_policy": { + "max_bytes": "500mb", + } + } + } +} +``` + +**Type:** [FastSlowSpec](#fastslowspec) + +### `shard` + +Shards the data to multiple stores. This is useful for cases +when you want to distribute the load across multiple stores. +The digest hash is used to determine which store to send the +data to. + +**Example JSON5 config:** +```json5 +"shard": { + "stores": [ + { + "store": { + "memory": { + "eviction_policy": { + "max_bytes": "10mb" + }, + }, + }, + "weight": 1 + }] +} +``` + +**Type:** [ShardSpec](#shardspec) + +### `filesystem` + +Stores the data on the filesystem. This store is designed for +local persistent storage. Restarts of this program should restore +the previous state, meaning anything uploaded will be persistent +as long as the filesystem integrity holds. + +**Example JSON5 config:** +```json5 +"filesystem": { + "content_path": "/tmp/nativelink/data-worker-test/content_path-cas", + "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas", + "eviction_policy": { + "max_bytes": "10gb", + } +} +``` + +**Type:** [FilesystemSpec](#filesystemspec) + +### `ref_store` + +Store used to reference a store in the root store manager. +This is useful for cases when you want to share a store in different +nested stores. Example, you may want to share the same memory store +used for the action cache, but use a `FastSlowSpec` and have the fast +store also share the memory store for efficiency. + +**Example JSON5 config:** +```json5 +"ref_store": { + "name": "FS_CONTENT_STORE" +} +``` + +**Type:** [RefSpec](#refspec) + +### `size_partitioning` + +Uses the size field of the digest to separate which store to send the +data. This is useful for cases when you'd like to put small objects +in one store and large objects in another store. This should only be +used if the size field is the real size of the content, in other +words, don't use on AC (Action Cache) stores. Any store where you can +safely use `VerifySpec.verify_size = true`, this store should be safe +to use (i.e., CAS stores). + +**Example JSON5 config:** +```json5 +"size_partitioning": { + "size": "128mib", + "lower_store": { + "memory": { + "eviction_policy": { + "max_bytes": "${NATIVELINK_CAS_MEMORY_CONTENT_LIMIT:-100mb}" + } + } + }, + "upper_store": { + /// This store discards data larger than 128mib. + "noop": {} + } +} +``` + +**Type:** [SizePartitioningSpec](#sizepartitioningspec) + +### `grpc` + +This store will pass-through calls to another GRPC store. This store +is not designed to be used as a sub-store of another store, but it +does satisfy the interface and will likely work. + +One major GOTCHA is that some stores use a special function on this +store to get the size of the underlying object, which is only reliable +when this store is serving the a CAS store, not an AC store. If using +this store directly without being a child of any store there are no +side effects and is the most efficient way to use it. + +**Example JSON5 config:** +```json5 +"grpc": { + "instance_name": "main", + "endpoints": [ + {"address": "grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"} + ], + "connections_per_endpoint": "5", + "rpc_timeout_s": "5m", + "store_type": "ac", + // Static headers attached to every outgoing request to the upstream + // remote cache. Useful for fixed service-account credentials. + "headers": { + "authorization": "Bearer my-static-token" + }, + // Header names to copy from the inbound client request and forward to + // the upstream remote cache. Use this to pass through dynamic + // credentials such as a JWT sent by the build client. + "forward_headers": ["authorization", "x-custom-token"] +} +``` + +**Type:** [GrpcSpec](#grpcspec) + +### `redis_store` + +Stores data in any stores compatible with Redis APIs. + +Pairs well with `SizePartitioning` and/or `FastSlow` stores. +Ideal for accepting small object sizes as most Redis store +services have a max file upload of between 256Mb-512Mb. + +**Example JSON5 config:** +```json5 +"redis_store": { + "addresses": [ + "redis://127.0.0.1:6379/", + ], + "max_client_permits": 1000, +} +``` + +**Type:** [RedisSpec](#redisspec) + +### `noop` + +Noop store is a store that sends streams into the void and all data +retrieval will return 404 (`NotFound`). This can be useful for cases +where you may need to partition your data and part of your data needs +to be discarded. + +**Example JSON5 config:** +```json5 +"noop": {} +``` + +**Type:** [NoopSpec](#noopspec) + +### `experimental_mongo` + +Experimental `MongoDB` store implementation. + +This store uses `MongoDB` as a backend for storing data. It supports +both CAS (Content Addressable Storage) and scheduler data with +optional change streams for real-time updates. + +**Example JSON5 config:** +```json5 +"experimental_mongo": { + "connection_string": "mongodb://localhost:27017", + "database": "nativelink", + "cas_collection": "cas", + "key_prefix": "cas:", + "read_chunk_size": 65536, + "max_concurrent_uploads": 10, + "enable_change_streams": false, + "max_requests": "100" +} +``` + +**Type:** [ExperimentalMongoSpec](#experimentalmongospec) + +## EvictionPolicy + +Eviction policy always works on LRU (Least Recently Used). Any time an entry +is touched it updates the timestamp. Inserts and updates will execute the +eviction policy removing any expired entries and/or the oldest entries +until the store size becomes smaller than `max_bytes`. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `max_bytes` | integer (uint) | — | 0 | Maximum number of bytes before eviction takes place. Zero means never evict based on size. | +| `evict_bytes` | integer (uint) | — | 0 | When eviction starts based on hitting `max_bytes`, continue until `max_bytes - evict_bytes` is met to create a low watermark. This stops operations from thrashing when the store is close to the limit. | +| `max_seconds` | integer (uint32) | — | 0 | Maximum number of seconds for an entry to live since it was last accessed before it is evicted. Zero means never evict based on time. | +| `max_count` | integer (uint64) | — | 0 | Maximum size of the store before an eviction takes place. Zero means never evict based on count. | + +## Retry + +Retry configuration. This configuration is exponential and each iteration +a jitter as a percentage is applied of the calculated delay. For example: +```haskell +Retry{ + max_retries: 7, + delay: 0.1, + jitter: 0.5, +} +``` +will result in: + +| Attempt | Delay | +| --- | --- | +| 1 | 0 ms | +| 2 | 75 to 125 ms | +| 3 | 150 to 250 ms | +| 4 | 300 to 500 ms | +| 5 | 600 ms to 1 s | +| 6 | 1.2 to 2 s | +| 7 | 2.4 to 4 s | +| 8 | 4.8 to 8 s | + +The total delay is additive, so this example produces 9.525 to 15.875 s of total delay for a single request. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `max_retries` | integer (uint) | — | `0` | Maximum number of retries until retrying stops. Setting this to zero will always attempt 1 time, but not retry. | +| `delay` | number (float) | — | `0` | Delay in seconds for exponential back off. | +| `jitter` | number (float) | — | `0` | Amount of jitter to add as a percentage in decimal form. This will change the formula like: | +| `retry_on_errors` | array of [ErrorCode](#errorcode) | — | — | A list of error codes to retry on, if this is not set then the default error codes to retry on are used. These default codes are the most likely to be non-permanent: `Unknown`; `Cancelled`; `DeadlineExceeded`; `ResourceExhausted`; `Aborted`; `Internal`; `Unavailable`; `DataLoss` | + +## ExperimentalOntapS3Spec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `endpoint` | string | Yes | — | | +| `vserver_name` | string | Yes | — | | +| `bucket` | string | Yes | — | | +| `root_certificates` | string | — | — | | +| `key_prefix` | string | — | — | If you wish to prefix the location in the bucket. If None, no prefix will be used. | +| `retry` | [Retry](#retry) | — | — | Retry configuration to use when a network request fails. | +| `consider_expired_after_s` | integer (uint32) | — | 0 | If the number of seconds since the `last_modified` time of the object is greater than this value, the object will not be considered "existing". This allows for external tools to delete objects that have not been uploaded in a long time. If a client receives a `NotFound` the client should re-upload the object. | +| `max_retry_buffer_per_request` | integer (uint) | — | 5MB | The maximum buffer size to retain in case of a retryable error during upload. Setting this to zero will disable upload buffering; this means that in the event of a failure during upload, the entire upload will be aborted and the client will likely receive an error. | +| `multipart_max_concurrent_uploads` | integer (uint) | — | 10 | Maximum number of concurrent `UploadPart` requests per `MultipartUpload`. | +| `insecure_allow_http` | boolean | — | false | Allow unencrypted HTTP connections. Only use this for local testing. | +| `disable_http2` | boolean | — | false | Disable HTTP/2 connections and only use HTTP/1.1. Default client configuration will have HTTP/1.1 and HTTP/2 enabled for connection schemes. HTTP/2 should be disabled if environments have poor support or performance related to HTTP/2. Safe to keep default unless underlying network environment, S3, or GCS API servers specify otherwise. | + +## CompressionAlgorithm + +Plus exactly one of the following variants (the key selects the variant): + +### `lz4` + +LZ4 compression algorithm is extremely fast for compression and +decompression, however does not perform very well in compression +ratio. In most cases build artifacts are highly compressible, however +lz4 is quite good at aborting early if the data is not deemed very +compressible. + +see: [https://lz4.github.io/lz4/](https://lz4.github.io/lz4/) + +**Type:** [Lz4Config](#lz4config) + +## StoreDirection + +| Value | Description | +| --- | --- | +| `"both"` | The store operates normally and all get and put operations are handled by it. | +| `"update"` | Update operations will cause persistence to this store, but Get operations will be ignored. This only makes sense on the fast store as the slow store will never get written to on Get anyway. | +| `"get"` | Get operations will cause persistence to this store, but Update operations will be ignored. | +| `"read_only"` | Operate as a read only store, only really makes sense if there's another way to write to it. | + +## ShardConfig + +Configuration for an individual shard of the store. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `store` | [StoreSpec](#storespec) | Yes | — | Store to shard the data to. | +| `weight` | integer (uint32) | — | 1 | The weight of the store. This is used to determine how much data should be sent to the store. The actual percentage is the sum of all the store's weights divided by the individual store's weight. | + +## GrpcEndpoint + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `address` | string | Yes | — | The endpoint address (i.e. grpc://example.com:443 or grpcs://example.com:443). | +| `tls_config` | [ClientTlsConfig](#clienttlsconfig) | — | — | The TLS configuration to use to connect to the endpoint (if grpcs). | +| `concurrency_limit` | integer (uint) | — | — | The maximum concurrency to allow on this endpoint. | +| `connect_timeout_s` | integer (uint64) | — | 30 seconds | Timeout for establishing a TCP connection to the endpoint (seconds). | +| `tcp_keepalive_s` | integer (uint64) | — | 30 seconds | TCP keepalive interval (seconds). Sends TCP keepalive probes at this interval to detect dead connections at the OS level. | +| `http2_keepalive_interval_s` | integer (uint64) | — | 30 seconds | HTTP/2 keepalive interval (seconds). Sends HTTP/2 PING frames at this interval to detect dead connections at the application level. | +| `http2_keepalive_timeout_s` | integer (uint64) | — | 20 seconds | HTTP/2 keepalive timeout (seconds). If a PING response is not received within this duration, the connection is considered dead. | + +## StoreType + +| Value | Description | +| --- | --- | +| `"cas"` | The store is content addressable storage. | +| `"ac"` | The store is an action cache. | + +## RedisMode + +| Value | Description | +| --- | --- | +| `"cluster"` | Use Redis Cluster. | +| `"sentinel"` | Use Redis Sentinel. | +| `"standard"` | Use a standalone Redis server. | + +## EndpointConfig + +Generic config for an endpoint and associated configs. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `uri` | string | Yes | — | URI of the endpoint. | +| `timeout` | number (float) | — | 5 (seconds) | Timeout in seconds that a request should take. | +| `tls_config` | [ClientTlsConfig](#clienttlsconfig) | — | — | The TLS configuration to use to connect to the endpoint. | + +## UploadActionResultConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `ac_store` | string | — | {No uploading is done} | Underlying AC store that the worker will use to publish execution results into. Objects placed in this store should be reachable from the scheduler/client-cas after they have finished updating. | +| `upload_ac_results_strategy` | [UploadCacheResultsStrategy](#uploadcacheresultsstrategy) | — | `UploadCacheResultsStrategy::SuccessOnly` | In which situations should the results be published to the `ac_store`, if set to `SuccessOnly` then only results with an exit code of 0 will be uploaded, if set to Everything all completed results will be uploaded. | +| `historical_results_store` | string | — | {CAS store of parent} | Store to upload historical results to. This should be a CAS store if set. | +| `upload_historical_results_strategy` | [UploadCacheResultsStrategy](#uploadcacheresultsstrategy) | — | `UploadCacheResultsStrategy::FailuresOnly` | In which situations should the results be published to the historical CAS. The historical CAS is where failures are published. These messages conform to the CAS key-value lookup format and are always a `HistoricalExecuteResponse` serialized message. | +| `success_message_template` | string | — | "" (no message) | Template to use for the `ExecuteResponse.message` property. This message is attached to the response before it is sent to the client. The following special variables are supported:; `digest_function`: Digest function used to calculate the action digest: `action_digest_hash`: Action digest hash: `action_digest_size`: Action digest size: `historical_results_hash`: `HistoricalExecuteResponse` digest hash: `historical_results_size`: `HistoricalExecuteResponse` digest size. | +| `failure_message_template` | string | — | "" (no message) | Same as `success_message_template` but for failure case. | + +## WorkerProperty + +Plus exactly one of the following variants (the key selects the variant): + +### `values` + +List of static values. +Note: Generally there should only ever be 1 value, but if the platform +property key is `PropertyType::Priority` it may have more than one value. + +**Type:** array of string + +### `query_cmd` + +A dynamic configuration. The string will be executed as a command +(not shell) and will be split by "\n" (new line character). + +**Type:** string + +## EnvironmentSource + +One of: + +- `property`: string — The name of the platform property in the action to get the value from. +- `value`: string — The raw value to set. +- `"from_environment"` — Take the value from the local environment corresponding to the name key +- `"timeout_millis"` — The max amount of time in milliseconds the command is allowed to run (requested by the client). +- `"side_channel_file"` — A special file path will be provided that can be used to communicate with the parent process about out-of-band information. This file will be read after the command has finished executing. Based on the contents of the file, the behavior of the result may be modified. +- `"action_directory"` — A "root" directory for the action. This directory can be used to store temporary files that are not needed after the action has completed. This directory will be purged after the action has completed. + +## DirectoryCacheConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `max_entries` | integer (uint) | — | 1000 | Maximum number of cached directories. | +| `max_size_bytes` | integer (uint64) | — | 10737418240 (10 GB) | Maximum total size in bytes for all cached directories (0 = unlimited). | +| `cache_root` | string | — | `{work_directory}/../directory_cache` | Base directory for cache storage. This directory will be managed by the worker and should be on the same filesystem as `work_directory`. | + +## PropertyType + +When the scheduler matches tasks to workers that are capable of running +the task, this value will be used to determine how the property is treated. + +One of: + +- string +- `"minimum"` — Requires the platform property to be a u64 and when the scheduler looks for appropriate worker nodes that are capable of executing the task, the task will not run on a node that has less than this value. +- `"exact"` — Requires the platform property to be a string and when the scheduler looks for appropriate worker nodes that are capable of executing the task, the task will not run on a node that does not have this property set to the value with exact string match. +- `"priority"` — Does not restrict on this value and instead will be passed to the worker as an informational piece. In the future this will be used by the scheduler and worker to cause the scheduler to prefer certain workers over others, but not restrict them based on these values. + +## WorkerAllocationStrategy + +When a worker is being searched for to run a job, this will be used +on how to choose which worker should run the job when multiple +workers are able to run the task. + +| Value | Description | +| --- | --- | +| `"least_recently_used"` | Prefer workers that have been least recently used to run a job. | +| `"most_recently_used"` | Prefer workers that have been most recently used to run a job. | + +## ExperimentalSimpleSchedulerBackend + +One of: + +- `"memory"` — Use an in-memory store for the scheduler. +- `redis`: [ExperimentalRedisSchedulerBackend](#experimentalredisschedulerbackend) — Use a Redis store for the scheduler. + +## SchedulerSpec + +Plus exactly one of the following variants (the key selects the variant): + +### `simple` + +**Type:** [SimpleSpec](#simplespec) + +### `grpc` + +**Type:** [SchedulerGrpcSpec](#schedulergrpcspec) + +### `cache_lookup` + +**Type:** [CacheLookupSpec](#cachelookupspec) + +### `property_modifier` + +**Type:** [PropertyModifierSpec](#propertymodifierspec) + +### `historical_resource` + +**Type:** [HistoricalResourceSpec](#historicalresourcespec) + +## PropertyModification + +Plus exactly one of the following variants (the key selects the variant): + +### `add` + +Add a property to the action properties. + +**Type:** [PlatformPropertyAddition](#platformpropertyaddition) + +### `remove` + +Remove a named property from the action. + +**Type:** string + +### `replace` + +If a property is found, then replace it with another one. + +**Type:** [PlatformPropertyReplacement](#platformpropertyreplacement) + +## HttpListener + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `socket_address` | string | Yes | — | Address to listen on. Example: `127.0.0.1:8080` or `:8080` to listen to all IPs. | +| `freebind` | boolean | — | false | Allow binding `socket_address` before it is assigned locally. | +| `compression` | [HttpCompressionConfig](#httpcompressionconfig) | — | — | Data transport compression configuration to use for this service. | +| `advanced_http` | [HttpServerConfig](#httpserverconfig) | — | — | Advanced HTTP server configuration. | +| `max_decoding_message_size` | integer (uint) | — | 4 MiB | Maximum number of bytes to decode on each grpc stream chunk. | +| `tls` | [TlsConfig](#tlsconfig) | — | — | TLS configuration for this server. If not set, the server will not use TLS. | + +## CasServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | | +| `cas_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. | + +## ActionCacheServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | | +| `ac_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. | +| `read_only` | boolean | — | `false` | Whether the Action Cache store may be written to, this if set to false it is only possible to read from the Action Cache. | + +## CapabilitiesServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | | +| `remote_execution` | [CapabilitiesRemoteExecutionConfig](#capabilitiesremoteexecutionconfig) | — | — | Configuration for remote execution capabilities. If not set the capabilities service will inform the client that remote execution is not supported. | + +## ExecutionServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | | +| `cas_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. This value must be a CAS store reference. | +| `scheduler` | string | Yes | — | The scheduler name referenced in the `schedulers` map in the main config. | + +## ByteStreamServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | | +| `cas_store` | string | Yes | — | Name of the store in the "stores" configuration. | +| `max_bytes_per_stream` | integer (uint) | — | 64KiB | Max number of bytes to send on each grpc stream chunk. According to [https://github.com/grpc/grpc.github.io/issues/371](https://github.com/grpc/grpc.github.io/issues/371) 16KiB - 64KiB is optimal. | +| `persist_stream_on_disconnect_timeout` | integer (uint) | — | 10 (seconds) | In the event a client disconnects while uploading a blob, we will hold the internal stream open for this many seconds before closing it. This allows clients that disconnect to reconnect and continue uploading the same blob. | + +## FetchServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | | +| `fetch_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. | + +## PushServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | | +| `push_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. | +| `read_only` | boolean | — | `false` | Whether the Action Cache store may be written to, this if set to false it is only possible to read from the Action Cache. | + +## WorkerApiConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `scheduler` | string | Yes | — | The scheduler name referenced in the `schedulers` map in the main config. | + +## BepConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `store` | string | Yes | — | The store to publish build events to. The store name referenced in the `stores` map in the main config. | + +## AdminConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `path` | string | — | "/admin" | Path to register the admin API. If path is "/admin", and your domain is "example.com", you can reach the endpoint with: [http://example.com/admin](http://example.com/admin). | + +## HealthConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `path` | string | — | "/status" | Path to register the health status check. If path is "/status", and your domain is "example.com", you can reach the endpoint with: [http://example.com/status](http://example.com/status). | +| `timeout_seconds` | integer (uint64) | — | `0` | | + +## ErrorCode + +The possible error codes that might occur on an upstream request. + +| Value | Description | +| --- | --- | +| `"Cancelled"` | | +| `"Unknown"` | | +| `"InvalidArgument"` | | +| `"DeadlineExceeded"` | | +| `"NotFound"` | | +| `"AlreadyExists"` | | +| `"PermissionDenied"` | | +| `"ResourceExhausted"` | | +| `"FailedPrecondition"` | | +| `"Aborted"` | | +| `"OutOfRange"` | | +| `"Unimplemented"` | | +| `"Internal"` | | +| `"Unavailable"` | | +| `"DataLoss"` | | +| `"Unauthenticated"` | | + +## Lz4Config + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `block_size` | integer (uint32) | — | 65536 (64k) | Size of the blocks to compress. Higher values require more ram, but might yield slightly better compression ratios. | +| `max_decode_block_size` | integer (uint32) | — | value in `block_size` | Maximum size allowed to attempt to deserialize data into. This is needed because the `block_size` is embedded into the data so if there was a bad actor, they could upload an extremely large `block_size`'ed entry and we'd allocate a large amount of memory when retrieving the data. To prevent this from happening, we allow you to specify the maximum that we'll attempt deserialize. | + +## ClientTlsConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `ca_file` | string | — | — | Path to the certificate authority to use to validate the remote. | +| `cert_file` | string | — | — | Path to the certificate file for client authentication. | +| `key_file` | string | — | — | Path to the private key file for client authentication. | +| `use_native_roots` | boolean | — | false | If set the client will use the native roots for TLS connections. | + +## UploadCacheResultsStrategy + +| Value | Description | +| --- | --- | +| `"success_only"` | Only upload action results with an exit code of 0. | +| `"never"` | Don't upload any action results. | +| `"everything"` | Upload all action results that complete. | +| `"failures_only"` | Only upload action results that fail. | + +## ExperimentalRedisSchedulerBackend + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `redis_store` | string | Yes | — | A reference to the Redis store to use for the scheduler. Note: This MUST resolve to a `RedisSpec`. | + +## PlatformPropertyAddition + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | Yes | — | The name of the property to add. | +| `value` | string | Yes | — | The value to assign to the property. | + +## PlatformPropertyReplacement + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | Yes | — | The name of the property to replace. | +| `value` | string | — | — | The value to match against, if unset then any instance matches. | +| `new_name` | string | Yes | — | The new name of the property. | +| `new_value` | string | — | — | The value to assign to the property, if unset will remain the same. | + +## HttpCompressionConfig + +Note: Compressing data in the cloud rarely has a benefit, since most +cloud providers have very high bandwidth backplanes. However, for +clients not inside the data center, it might be a good idea to +compress data to and from the cloud. This will however come at a high +CPU and performance cost. If you are making remote execution share the +same CAS/AC servers as client's remote cache, you can create multiple +services with different compression settings that are served on +different ports. Then configure the non-cloud clients to use one port +and cloud-clients to use another. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `send_compression_algorithm` | [HttpCompressionAlgorithm](#httpcompressionalgorithm) | — | `HttpCompressionAlgorithm::None` | The compression algorithm that the server will use when sending responses to clients. Enabling this will likely save a lot of data transfer, but will consume a lot of CPU and add a lot of latency. see: [https://github.com/tracemachina/nativelink/issues/109](https://github.com/tracemachina/nativelink/issues/109) | +| `accepted_compression_algorithms` | array of [HttpCompressionAlgorithm](#httpcompressionalgorithm) | Yes | {no supported compression} | The compression algorithm that the server will accept from clients. The server will broadcast the supported compression algorithms to clients and the client will choose which compression algorithm to use. Enabling this will likely save a lot of data transfer, but will consume a lot of CPU and add a lot of latency. see: [https://github.com/tracemachina/nativelink/issues/109](https://github.com/tracemachina/nativelink/issues/109) | + +## HttpServerConfig + +Advanced HTTP configuration. These generally should not be set. +For documentation on these settings, see the hyper documentation: +See: [hyper HTTP server docs](https://docs.rs/hyper/latest/hyper/server/conn/struct.Http.html) + +Note: All of these default to the default values from hyper unless otherwise +specified. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `http2_keep_alive_interval` | integer (uint32) | — | — | Interval to send keep-alive pings via HTTP2. Note: This is in seconds. | +| `experimental_http2_max_pending_accept_reset_streams` | integer (uint32) | — | — | | +| `experimental_http2_initial_stream_window_size` | integer (uint32) | — | — | | +| `experimental_http2_initial_connection_window_size` | integer (uint32) | — | — | | +| `experimental_http2_adaptive_window` | boolean | — | — | | +| `experimental_http2_max_frame_size` | integer (uint32) | — | — | | +| `experimental_http2_max_concurrent_streams` | integer (uint32) | — | — | | +| `experimental_http2_keep_alive_timeout` | integer (uint32) | — | — | Note: This is in seconds. | +| `experimental_http2_max_send_buf_size` | integer (uint32) | — | — | | +| `experimental_http2_enable_connect_protocol` | boolean | — | — | | +| `experimental_http2_max_header_list_size` | integer (uint32) | — | — | | + +## TlsConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `cert_file` | string | Yes | — | Path to the certificate file. | +| `key_file` | string | Yes | — | Path to the private key file. | +| `client_ca_file` | string | — | — | Path to the certificate authority for mTLS, if client authentication is required for this endpoint. | +| `client_crl_file` | string | — | — | Path to the certificate revocation list for mTLS, if client authentication is required for this endpoint. | + +## CapabilitiesRemoteExecutionConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `scheduler` | string | Yes | — | Scheduler used to configure the capabilities of remote execution. | + +## HttpCompressionAlgorithm + +| Value | Description | +| --- | --- | +| `"none"` | No compression. | +| `"gzip"` | Zlib compression. | + +## Reading the source + +If anything here disagrees with the binary, the source wins: + +- [`stores.rs`](https://github.com/TraceMachina/nativelink/tree/v1.5.1/nativelink-config/src/stores.rs) +- [`cas_server.rs`](https://github.com/TraceMachina/nativelink/tree/v1.5.1/nativelink-config/src/cas_server.rs) +- [`schedulers.rs`](https://github.com/TraceMachina/nativelink/tree/v1.5.1/nativelink-config/src/schedulers.rs) diff --git a/web/apps/docs/lib/config-versions.ts b/web/apps/docs/lib/config-versions.ts index 0da19f04d..624a044b4 100644 --- a/web/apps/docs/lib/config-versions.ts +++ b/web/apps/docs/lib/config-versions.ts @@ -28,13 +28,21 @@ export const CONFIG_VERSIONS: ConfigVersion[] = [ "isDev": true }, { - "version": "v1.5.1", - "label": "v1.5.1 (latest)", + "version": "v1.5.2", + "label": "v1.5.2 (latest)", "href": "/reference/nativelink-config", - "ref": "v1.5.1", + "ref": "v1.5.2", "isLatest": true, "isDev": false }, + { + "version": "v1.5.1", + "label": "v1.5.1", + "href": "/reference/nativelink-config/v1.5.1", + "ref": "v1.5.1", + "isLatest": false, + "isDev": false + }, { "version": "v1.5.0", "label": "v1.5.0", @@ -91,6 +99,14 @@ export const CONFIG_VERSIONS: ConfigVersion[] = [ "isLatest": false, "isDev": false }, + { + "version": "v1.0.1", + "label": "v1.0.1", + "href": "/reference/nativelink-config/v1.0.1", + "ref": "v1.0.1", + "isLatest": false, + "isDev": false + }, { "version": "v1.0.0", "label": "v1.0.0", diff --git a/web/apps/docs/package.json b/web/apps/docs/package.json index 3bf5aa504..d61a9b2bd 100644 --- a/web/apps/docs/package.json +++ b/web/apps/docs/package.json @@ -17,7 +17,7 @@ "@nativelink/config": "workspace:*", "@tailwindcss/postcss": "^4.0.0", "@types/mdx": "^2.0.13", - "@types/node": "^25.0.0", + "@types/node": "^26.0.0", "@types/react": "^19.0.7", "@types/react-dom": "^19.0.3", "tailwindcss": "^4.0.0", diff --git a/web/apps/docs/scripts/lib/schema-to-mdx.mjs b/web/apps/docs/scripts/lib/schema-to-mdx.mjs index 21a076d0f..60caa0e0d 100644 --- a/web/apps/docs/scripts/lib/schema-to-mdx.mjs +++ b/web/apps/docs/scripts/lib/schema-to-mdx.mjs @@ -62,7 +62,7 @@ function displayName(name) { function defaultFromDescription(md) { if (!md) return ""; const zeroMeansDefault = /If not set or 0, defaults to ([^.]+)\./i.exec(md); - if (zeroMeansDefault) return `0 (uses ${zeroMeansDefault[1]})`; + if (zeroMeansDefault) return `${zeroMeansDefault[1]}`; const codeDefault = /Default:\s*(`[^`]+`)/.exec(md); if (codeDefault) return codeDefault[1]; @@ -106,7 +106,7 @@ function normalizeMarkdown(md) { ) .replace( /Remember that to get total results is additive, meaning the above results\nwould mean a single request would have a total delay of 9\.525s - 15\.875s\./g, - "The total delay is additive, so this example produces 9.525 to 15.875 s of total delay for a single request.", + "\nThe total delay is additive, so this example produces 9.525 to 15.875 s of total delay for a single request.", ); } @@ -146,6 +146,7 @@ function normalizeProseText(text) { .replace(/\bA scheduler that simply forwards\b/g, "A scheduler that forwards") .replace(/\bThe the value\b/g, "The value") .replace(/\bset to -1 to disable\b/g, "set to `-1` to disable") + .replace(/TODO\(.+\)/, "") .replace(/([.!?]) {2,}(?=[A-Z`])/g, "$1 "); } @@ -344,6 +345,7 @@ function renderTaggedEnum(name, def, known) { function enumDescription(name, value, description) { if (description) return description; + // Backfill for versions up to 1.5.1 that don't have comments for RedisMode if (name === "RedisMode") { return { cluster: "Use Redis Cluster.", @@ -393,11 +395,15 @@ function unionBranchType(branch, known) { function renderUnion(name, def, known) { const out = [`## ${displayName(name)}`, ""]; - if (def.description) out.push(sanitizeMarkdown(def.description), ""); - out.push("One of:", ""); - for (const branch of def.oneOf) { - const desc = branch.description ? ` — ${cellText(branch.description)}` : ""; - out.push(`- ${unionBranchType(branch, known)}${desc}`); + if (name == "ExperimentalCloudObjectSpec") { + out.push("See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for details") + } else { + if (def.description) out.push(sanitizeMarkdown(def.description), ""); + out.push("One of:", ""); + for (const branch of def.oneOf) { + const desc = branch.description ? ` — ${cellText(branch.description)}` : ""; + out.push(`- ${unionBranchType(branch, known)}${desc}`); + } } out.push(""); return out.join("\n"); diff --git a/web/apps/web/app/company/page.tsx b/web/apps/web/app/company/page.tsx index daf31f611..fe479c5dd 100644 --- a/web/apps/web/app/company/page.tsx +++ b/web/apps/web/app/company/page.tsx @@ -102,13 +102,12 @@ export default function CompanyPage() { - {/* MISSION / 3-PARA */} + {/* MISSION */}
-

- We empower engineers to build the future of technology by making advanced build and - simulation processes that move at machine speed. +

+ Mission: Accelerate the progress of humanity in the machine age.

Our products amplify the rate at which companies can innovate across mission-critical diff --git a/web/apps/web/app/page.tsx b/web/apps/web/app/page.tsx index 039b0e38a..1b70b1fb1 100644 --- a/web/apps/web/app/page.tsx +++ b/web/apps/web/app/page.tsx @@ -1,5 +1,13 @@ import { AnimatedTerminal } from "@/components/animated-terminal"; import { ArchitectureDiagram } from "@/components/architecture-diagram"; +import { + CitrixLogo, + MenloSecurityLogo, + MetaLogo, + SamsungLogo, + TeslaLogo, + ThirdWaveLogo, +} from "@/components/customer-logos"; import { Badge, Button, Eyebrow, HeroVisual, Marquee, Reveal, Section } from "@nativelink/ui"; export const metadata = { @@ -21,13 +29,43 @@ const integrations = [ "Soong", ]; -const customers: { name: string; cls: string }[] = [ - { name: "Menlo Security", cls: "font-semibold tracking-[-0.04em]" }, - { name: "Citrix", cls: "font-extralight uppercase tracking-[0.18em] text-base" }, - { name: "Tesla", cls: "font-mono uppercase tracking-[0.05em] text-base" }, - { name: "Meta", cls: "font-semibold tracking-[-0.05em]" }, - { name: "Samsung", cls: "font-semibold tracking-[-0.04em]" }, - { name: "Third Wave", cls: "italic font-light tracking-tight" }, +const customerLogos = [ + { + name: "Menlo Security", + href: "https://www.menlosecurity.com/", + Logo: MenloSecurityLogo, + className: "h-9 md:h-10", + }, + { + name: "Citrix", + href: "https://www.citrix.com/", + Logo: CitrixLogo, + className: "h-7 md:h-8", + }, + { + name: "Tesla", + href: "https://www.tesla.com/", + Logo: TeslaLogo, + className: "h-6 md:h-7", + }, + { + name: "Meta", + href: "https://www.meta.com/about/", + Logo: MetaLogo, + className: "h-7 md:h-8", + }, + { + name: "Samsung", + href: "https://www.samsung.com/", + Logo: SamsungLogo, + className: "h-6 md:h-7", + }, + { + name: "Third Wave", + href: "https://thirdwave.ai/", + Logo: ThirdWaveLogo, + className: "h-10 md:h-11", + }, ]; const stats = [ @@ -156,20 +194,30 @@ export default function HomePage() { - +

+