From 5050174363c97ad320fe28affd37561731f6f929 Mon Sep 17 00:00:00 2001 From: synoet Date: Tue, 23 Jun 2026 09:42:17 -0400 Subject: [PATCH 1/8] feat: gen check and test workflow --- .github/actionlint.yaml | 2 + ...anged-cloud-storage-doppler-config-bins.sh | 119 ----- .../workflows/check-generated-workflows.yml | 25 + .../workflows/code-check-cloud-storage.yml | 482 +++++++++--------- rust/cloud-storage/.cargo/config.toml | 10 + rust/cloud-storage/Cargo.lock | 126 +++++ rust/cloud-storage/tools/xtask/Cargo.toml | 4 + .../tools/xtask/src/doppler_bins.rs | 82 +++ rust/cloud-storage/tools/xtask/src/main.rs | 10 +- .../xtask/src/workflows/check_generated.rs | 33 ++ .../src/workflows/code_check_cloud_storage.rs | 369 ++++++++++++++ .../tools/xtask/src/workflows/mod.rs | 112 ++++ .../tools/xtask/src/workflows/runners.rs | 17 + .../tools/xtask/src/workflows/steps.rs | 116 +++++ .../tools/xtask/src/workflows/vars.rs | 46 ++ 15 files changed, 1195 insertions(+), 358 deletions(-) delete mode 100755 .github/scripts/changed-cloud-storage-doppler-config-bins.sh create mode 100644 .github/workflows/check-generated-workflows.yml create mode 100644 rust/cloud-storage/.cargo/config.toml create mode 100644 rust/cloud-storage/tools/xtask/src/doppler_bins.rs create mode 100644 rust/cloud-storage/tools/xtask/src/workflows/check_generated.rs create mode 100644 rust/cloud-storage/tools/xtask/src/workflows/code_check_cloud_storage.rs create mode 100644 rust/cloud-storage/tools/xtask/src/workflows/mod.rs create mode 100644 rust/cloud-storage/tools/xtask/src/workflows/runners.rs create mode 100644 rust/cloud-storage/tools/xtask/src/workflows/steps.rs create mode 100644 rust/cloud-storage/tools/xtask/src/workflows/vars.rs diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index 31af77e9de5..2d66f2a34da 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -2,3 +2,5 @@ self-hosted-runner: labels: - linux-extra-beefy - linux-latest-middy + - namespace-profile-linux-mid + - namespace-profile-linux-small diff --git a/.github/scripts/changed-cloud-storage-doppler-config-bins.sh b/.github/scripts/changed-cloud-storage-doppler-config-bins.sh deleted file mode 100755 index ad21ee97da1..00000000000 --- a/.github/scripts/changed-cloud-storage-doppler-config-bins.sh +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -if [[ $# -lt 1 || $# -gt 2 ]]; then - echo "Usage: $0 [repo-root]" >&2 - exit 2 -fi - -changed_files=$1 -if [[ ! -f "$changed_files" ]]; then - echo "Changed files list not found: $changed_files" >&2 - exit 2 -fi - -if [[ $# -eq 2 ]]; then - repo_root=$2 -elif repo_root="$(git rev-parse --show-toplevel 2>/dev/null)"; then - : -else - repo_root=$PWD -fi -repo_root=${repo_root%/} - -if [[ ! -d "$repo_root" ]]; then - echo "Repository root not found: $repo_root" >&2 - exit 2 -fi - -declare -A crate_paths=() -while IFS= read -r changed_file || [[ -n "$changed_file" ]]; do - changed_file=${changed_file%$'\r'} - changed_file=${changed_file#./} - - case "$changed_file" in - rust/cloud-storage/*/src/config.rs) - crate_path=${changed_file#rust/cloud-storage/} - crate_path=${crate_path%/src/config.rs} - ;; - rust/cloud-storage/*/src/doppler_config.rs) - crate_path=${changed_file#rust/cloud-storage/} - crate_path=${crate_path%/src/doppler_config.rs} - ;; - rust/cloud-storage/*/Cargo.toml) - crate_path=${changed_file#rust/cloud-storage/} - crate_path=${crate_path%/Cargo.toml} - ;; - *) - continue - ;; - esac - if [[ -n "$crate_path" ]]; then - crate_paths["$crate_path"]=1 - fi -done < "$changed_files" - -if [[ ${#crate_paths[@]} -eq 0 ]]; then - exit 0 -fi - -bin_names_file=$(mktemp) -trap 'rm -f "$bin_names_file"' EXIT - -for crate_path in "${!crate_paths[@]}"; do - crate_dir="$repo_root/rust/cloud-storage/$crate_path" - manifest="$crate_dir/Cargo.toml" - doppler_config="$crate_dir/src/doppler_config.rs" - - if [[ ! -f "$manifest" || ! -f "$doppler_config" ]]; then - continue - fi - - python3 - "$manifest" "$crate_path" >> "$bin_names_file" <<'PY' -import sys -from pathlib import Path - -try: - import tomllib -except ModuleNotFoundError: - print("python3 with tomllib support is required", file=sys.stderr) - raise SystemExit(1) - -manifest_path = Path(sys.argv[1]) -crate_path = sys.argv[2] -doppler_config_path = "src/doppler_config.rs" - -try: - manifest = tomllib.loads(manifest_path.read_text()) -except (OSError, tomllib.TOMLDecodeError) as error: - print(f"Failed to read Cargo manifest for {crate_path}: {error}", file=sys.stderr) - raise SystemExit(1) - -matching_bins = [ - bin_config - for bin_config in manifest.get("bin", []) - if bin_config.get("path") == doppler_config_path -] - -if not matching_bins: - print( - f"cloud-storage crate '{crate_path}' has {doppler_config_path} " - f"but no [[bin]] with path = \"{doppler_config_path}\" in {manifest_path}", - file=sys.stderr, - ) - raise SystemExit(1) - -for bin_config in matching_bins: - bin_name = bin_config.get("name") - if not bin_name: - print( - f"cloud-storage crate '{crate_path}' has a [[bin]] with " - f"path = \"{doppler_config_path}\" but no name in {manifest_path}", - file=sys.stderr, - ) - raise SystemExit(1) - print(bin_name) -PY -done - -LC_ALL=C sort -u "$bin_names_file" diff --git a/.github/workflows/check-generated-workflows.yml b/.github/workflows/check-generated-workflows.yml new file mode 100644 index 00000000000..bc15470d6f4 --- /dev/null +++ b/.github/workflows/check-generated-workflows.yml @@ -0,0 +1,25 @@ +# DO NOT EDIT — regenerate with `cargo x workflows` (from rust/cloud-storage). +# Source: rust/cloud-storage/tools/xtask/src/workflows/check_generated.rs +name: check generated workflows +'on': + pull_request: + branches: + - main + paths: + - rust/cloud-storage/tools/xtask/** + - '.github/workflows/**' +jobs: + check-workflows: + runs-on: namespace-profile-linux-small + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + clean: 'false' + - name: Setup Rust + uses: './.github/actions/setup-rust' + with: + sccache: 'false' + rust-cache: 'false' + - name: verify workflows are up to date + run: cargo run --manifest-path rust/cloud-storage/tools/xtask/Cargo.toml -- workflows --check diff --git a/.github/workflows/code-check-cloud-storage.yml b/.github/workflows/code-check-cloud-storage.yml index be3ef0fe712..02f325c015c 100644 --- a/.github/workflows/code-check-cloud-storage.yml +++ b/.github/workflows/code-check-cloud-storage.yml @@ -1,292 +1,298 @@ +# DO NOT EDIT — regenerate with `cargo x workflows` (from rust/cloud-storage). +# Source: rust/cloud-storage/tools/xtask/src/workflows/code_check_cloud_storage.rs name: cloud storage code check -on: - pull_request: - branches: [ 'main' ] - types: [opened, synchronize, reopened, ready_for_review] - env: CARGO_INCREMENTAL: '0' CARGO_TERM_COLOR: always CARGO_PROFILE_DEV_DEBUG: limited CARGO_PROFILE_TEST_DEBUG: limited RUST_BACKTRACE: '1' - RUSTFLAGS: "-C link-arg=-fuse-ld=lld" - -concurrency: - group: code-check-cloud-storage-${{ github.ref }} - cancel-in-progress: true - + RUSTFLAGS: '-C link-arg=-fuse-ld=lld' +'on': + pull_request: + types: + - opened + - synchronize + - reopened + - ready_for_review + branches: + - main jobs: path-check: - runs-on: ubuntu-latest + runs-on: namespace-profile-linux-small outputs: should_run: ${{ steps.filter.outputs.should_run }} - nextest_filter: ${{ steps.package-filter.outputs.nextest_filter }} - doppler_config_bins: ${{ steps.package-filter.outputs.doppler_config_bins }} + nextest_filter: ${{ steps.nextest-filter.outputs.nextest_filter }} + doppler_config_bins: ${{ steps.doppler-bins.outputs.doppler_config_bins }} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - with: - clean: false - fetch-depth: 0 - - name: Setup Rust - uses: ./.github/actions/setup-rust - with: - sccache: 'false' - rust-cache: 'false' - - uses: dorny/paths-filter@v3 - id: filter - with: - filters: | - should_run: - - 'rust/cloud-storage/**' - - 'rust/rust-toolchain.toml' - - 'flake.nix' - - 'flake.lock' - - '.github/actions/setup-cachix/**' - - '.github/actions/setup-sccache/**' - - '.github/services-config.json' - - '.github/scripts/build-cloud-storage-lambdas.sh' - - '.github/scripts/build-cloud-storage-lambdas-nix.sh' - - '.github/scripts/changed-cloud-storage-doppler-config-bins.sh' - - .github/workflows/code-check-cloud-storage.yml - - name: compute nextest package filter - id: package-filter - if: steps.filter.outputs.should_run == 'true' - shell: bash - run: | - set -euo pipefail + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + clean: 'false' + fetch-depth: '0' + - name: Setup Rust + uses: './.github/actions/setup-rust' + with: + sccache: 'false' + rust-cache: 'false' + - id: filter + name: Filter changed paths + uses: dorny/paths-filter@v3 + with: + filters: | + should_run: + - 'rust/cloud-storage/**' + - 'rust/rust-toolchain.toml' + - 'flake.nix' + - 'flake.lock' + - '.github/actions/setup-cachix/**' + - '.github/actions/setup-sccache/**' + - '.github/services-config.json' + - '.github/scripts/build-cloud-storage-lambdas.sh' + - '.github/scripts/build-cloud-storage-lambdas-nix.sh' + - .github/workflows/code-check-cloud-storage.yml + - name: compute changed files + if: steps.filter.outputs.should_run == 'true' + run: | + set -euo pipefail - if [ -z "${GITHUB_BASE_REF:-}" ]; then - compare_rev="$(git rev-parse HEAD~1)" - else - git fetch origin "$GITHUB_BASE_REF:refs/remotes/origin/$GITHUB_BASE_REF" - if ! compare_rev="$(git merge-base "origin/${GITHUB_BASE_REF}" HEAD)"; then - echo "Unable to find merge-base for origin/${GITHUB_BASE_REF}; falling back to full test suite" >&2 - echo "nextest_filter=" >> "$GITHUB_OUTPUT" - exit 0 - fi + if [ -z "${GITHUB_BASE_REF:-}" ]; then + compare_rev="$(git rev-parse HEAD~1)" + else + git fetch origin "$GITHUB_BASE_REF:refs/remotes/origin/$GITHUB_BASE_REF" + if ! compare_rev="$(git merge-base "origin/${GITHUB_BASE_REF}" HEAD)"; then + echo "Unable to find merge-base for origin/${GITHUB_BASE_REF}; falling back to full test suite" >&2 + : > /tmp/changed-files + exit 0 fi + fi - changed_files="$(git diff --name-only "$compare_rev" "$GITHUB_SHA")" - printf '%s\n' "$changed_files" > /tmp/changed-files - - doppler_config_bins="$("$GITHUB_WORKSPACE/.github/scripts/changed-cloud-storage-doppler-config-bins.sh" /tmp/changed-files "$GITHUB_WORKSPACE")" - { - echo 'doppler_config_bins<<__DOPPLER_CONFIG_BINS__' - if [ -n "$doppler_config_bins" ]; then - printf '%s\n' "$doppler_config_bins" - fi - echo '__DOPPLER_CONFIG_BINS__' - } >> "$GITHUB_OUTPUT" + git diff --name-only "$compare_rev" "$GITHUB_SHA" > /tmp/changed-files + shell: bash + - id: doppler-bins + name: compute affected Doppler config bins + if: steps.filter.outputs.should_run == 'true' + run: | + set -euo pipefail - # Root cargo/toolchain/CI changes can affect the whole workspace, so run all tests. - if printf '%s\n' "$changed_files" | grep -qE '^(rust/rust-toolchain\.toml|flake\.nix|flake\.lock|rust/cloud-storage/Cargo\.(toml|lock)|rust/cloud-storage/\.cargo/.*|\.github/actions/(setup-rust|setup-cachix|setup-sccache)/.*|\.github/workflows/code-check-cloud-storage\.yml)$'; then - echo "Workspace-level change detected; running all tests" - echo "nextest_filter=" >> "$GITHUB_OUTPUT" - exit 0 + doppler_config_bins="$(cargo run --manifest-path rust/cloud-storage/tools/xtask/Cargo.toml -- doppler-bins /tmp/changed-files)" + { + echo 'doppler_config_bins<<__DOPPLER_CONFIG_BINS__' + if [ -n "$doppler_config_bins" ]; then + printf '%s\n' "$doppler_config_bins" fi + echo '__DOPPLER_CONFIG_BINS__' + } >> "$GITHUB_OUTPUT" + shell: bash + - id: nextest-filter + name: compute nextest package filter + if: steps.filter.outputs.should_run == 'true' + run: | + set -euo pipefail - filterset="$(cargo run --manifest-path rust/cloud-storage/tools/xtask/Cargo.toml -- nextest-filter /tmp/changed-files)" + # Root cargo/toolchain/CI changes can affect the whole workspace, so run all tests. + if grep -qE '^(rust/rust-toolchain\.toml|flake\.nix|flake\.lock|rust/cloud-storage/Cargo\.(toml|lock)|rust/cloud-storage/\.cargo/.*|\.github/actions/(setup-rust|setup-cachix|setup-sccache)/.*|\.github/workflows/code-check-cloud-storage\.yml)$' /tmp/changed-files; then + echo "Workspace-level change detected; running all tests" + echo "nextest_filter=" >> "$GITHUB_OUTPUT" + exit 0 + fi - if [ -z "$filterset" ]; then - echo "No package-specific Rust changes detected; running all tests" - else - echo "nextest filter: $filterset" - fi - echo "nextest_filter=$filterset" >> "$GITHUB_OUTPUT" + filterset="$(cargo run --manifest-path rust/cloud-storage/tools/xtask/Cargo.toml -- nextest-filter /tmp/changed-files)" + if [ -z "$filterset" ]; then + echo "No package-specific Rust changes detected; running all tests" + else + echo "nextest filter: $filterset" + fi + echo "nextest_filter=$filterset" >> "$GITHUB_OUTPUT" + shell: bash check: - needs: path-check + needs: + - path-check if: needs.path-check.outputs.should_run == 'true' && github.event.pull_request.draft == false - runs-on: linux-extra-beefy + runs-on: namespace-profile-linux-mid env: - RUSTFLAGS: "-Dwarnings -Dclippy::disallowed_methods -C link-arg=-fuse-ld=mold" - RUSTDOCFLAGS: "-Dwarnings" + RUSTFLAGS: '-Dwarnings -Dclippy::disallowed_methods -C link-arg=-fuse-ld=mold' + RUSTDOCFLAGS: '-Dwarnings' steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - with: - clean: false - - name: Setup Nix dev shell - uses: ./.github/actions/setup-cachix - with: - cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }} - dev-shell: 'true' - sccache-bucket: ${{ vars.SCCACHE_BUCKET || secrets.SCCACHE_BUCKET }} - - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 - with: - workspaces: rust/cloud-storage - shared-key: cloud-storage-check - cache-on-failure: true - cache-targets: false - - name: validate Doppler configs - if: needs.path-check.outputs.doppler_config_bins != '' - env: - DOPPLER_CONFIG_BINS: ${{ needs.path-check.outputs.doppler_config_bins }} - DOPPLER_TOKEN: ${{ secrets.DOPPLER_TOKEN }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID || secrets.AWS_ACCESS_KEY }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_SESSION_TOKEN: ${{ secrets.AWS_SESSION_TOKEN }} - run: | - set -euo pipefail + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + clean: 'false' + - name: Mount Namespace cache volume + uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0 + with: + cache: rust + path: |- + /sccache + /nix + continue-on-error: true + - name: Setup Nix dev shell + uses: './.github/actions/setup-cachix' + with: + cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }} + dev-shell: 'true' + - name: Point sccache at the cache volume + run: |- + echo "SCCACHE_DIR=/sccache" >> "$GITHUB_ENV" + echo "SCCACHE_CACHE_SIZE=20G" >> "$GITHUB_ENV" + - name: validate Doppler configs + if: needs.path-check.outputs.doppler_config_bins != '' + run: | + set -euo pipefail - if [ -z "${DOPPLER_TOKEN:-}" ]; then - echo "DOPPLER_TOKEN secret is required to validate Doppler configs" >&2 - exit 1 - fi + if [ -z "${DOPPLER_TOKEN:-}" ]; then + echo "DOPPLER_TOKEN secret is required to validate Doppler configs" >&2 + exit 1 + fi - if [ -z "${RUSTC_WRAPPER:-}" ]; then - echo "RUSTC_WRAPPER is required so Doppler config binaries build through sccache" >&2 - exit 1 - fi + if [ -z "${RUSTC_WRAPPER:-}" ]; then + echo "RUSTC_WRAPPER is required so Doppler config binaries build through sccache" >&2 + exit 1 + fi - bins=() - cargo_args=(build --locked --all-features) - while IFS= read -r bin; do - if [ -z "$bin" ]; then - continue - fi - - bins+=("$bin") - cargo_args+=(--bin "$bin") - done <<< "$DOPPLER_CONFIG_BINS" - - if [ "${#bins[@]}" -eq 0 ]; then - echo "No Doppler config binaries to validate" - exit 0 + bins=() + cargo_args=(build --locked --all-features) + while IFS= read -r bin; do + if [ -z "$bin" ]; then + continue fi - echo "Building affected Doppler config binaries with RUSTC_WRAPPER=$RUSTC_WRAPPER" - printf ' %s\n' "${bins[@]}" + bins+=("$bin") + cargo_args+=(--bin "$bin") + done <<< "$DOPPLER_CONFIG_BINS" - ( - cd rust/cloud-storage - cargo "${cargo_args[@]}" - for bin in "${bins[@]}"; do - "./target/debug/$bin" - done - ) - - name: fmt - run: cd rust/cloud-storage && cargo fmt --check - - name: clippy - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID || secrets.AWS_ACCESS_KEY }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - run: cd rust/cloud-storage && cargo clippy --workspace --all-features - - name: show sccache stats - if: always() - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID || secrets.AWS_ACCESS_KEY }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - run: sccache --show-stats || true + if [ "${#bins[@]}" -eq 0 ]; then + echo "No Doppler config binaries to validate" + exit 0 + fi + + echo "Building affected Doppler config binaries with RUSTC_WRAPPER=$RUSTC_WRAPPER" + printf ' %s\n' "${bins[@]}" + ( + cd rust/cloud-storage + cargo "${cargo_args[@]}" + for bin in "${bins[@]}"; do + "./target/debug/$bin" + done + ) + env: + DOPPLER_CONFIG_BINS: ${{ needs.path-check.outputs.doppler_config_bins }} + DOPPLER_TOKEN: ${{ secrets.DOPPLER_TOKEN }} + - name: fmt + run: cd rust/cloud-storage && cargo fmt --check + - name: clippy + run: cd rust/cloud-storage && cargo clippy --workspace --all-features + - name: show sccache stats + if: always() + run: sccache --show-stats || true test: - needs: path-check + needs: + - path-check if: needs.path-check.outputs.should_run == 'true' && github.event.pull_request.draft == false - runs-on: linux-extra-beefy + runs-on: namespace-profile-linux-mid env: NEXTEST_FILTER: ${{ needs.path-check.outputs.nextest_filter }} NEXTEST_TEST_THREADS: '32' - RUSTFLAGS: "-Dwarnings -C link-arg=-fuse-ld=mold" + RUSTFLAGS: '-Dwarnings -C link-arg=-fuse-ld=mold' services: postgres: image: pgvector/pgvector:pg16 env: POSTGRES_USER: user POSTGRES_PASSWORD: password - POSTGRES_DB: macrodb # Start with macrodb, we initialize all others in the cloud-storage/justfile + POSTGRES_DB: macrodb ports: - - 5432:5432 - # needed because the postgres container does not provide a healthcheck - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - --shm-size 1g + - '5432:5432' + options: '--health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 --shm-size 1g' redis: image: redis:7 ports: - - 6379:6379 - options: --health-cmd "redis-cli ping" --health-interval 10s --health-timeout 5s --health-retries 5 + - '6379:6379' + options: '--health-cmd "redis-cli ping" --health-interval 10s --health-timeout 5s --health-retries 5' steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - with: - clean: false - - name: Setup Nix dev shell - uses: ./.github/actions/setup-cachix - with: - cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }} - dev-shell: 'true' - sccache-bucket: ${{ vars.SCCACHE_BUCKET || secrets.SCCACHE_BUCKET }} - - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 - with: - workspaces: rust/cloud-storage - shared-key: cloud-storage-test - cache-on-failure: true - cache-targets: false - - name: configure postgres for concurrent tests - run: | - postgres_container="$(docker ps --format '{{.ID}} {{.Image}}' | awk '$2 == "pgvector/pgvector:pg16" { print $1; exit }')" - if [ -z "$postgres_container" ]; then - echo "pgvector/pgvector:pg16 service container not found" >&2 - docker ps - exit 1 - fi - - docker exec -i "$postgres_container" psql -U user -d macrodb <<'SQL' - ALTER SYSTEM SET fsync = off; - ALTER SYSTEM SET synchronous_commit = off; - ALTER SYSTEM SET full_page_writes = off; - ALTER SYSTEM SET max_wal_size = '4GB'; - ALTER SYSTEM SET checkpoint_timeout = '30min'; - ALTER SYSTEM SET max_locks_per_transaction = 8192; - SQL - docker restart "$postgres_container" - until docker exec "$postgres_container" pg_isready -U user -d macrodb; do - sleep 1 - done - docker exec "$postgres_container" psql -U user -d macrodb -c "SHOW max_locks_per_transaction;" - - name: prepare tests - run: just rust/cloud-storage/setup_test_envs && just rust/cloud-storage/initialize_dbs - - name: run tests - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID || secrets.AWS_ACCESS_KEY }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - run: | - cd rust/cloud-storage - - args=(--all-features --lib --bins --tests --test-threads "$NEXTEST_TEST_THREADS") - if [ -n "$NEXTEST_FILTER" ]; then - args+=(-E "$NEXTEST_FILTER") - fi - cargo nextest run "${args[@]}" - - name: show sccache stats - if: always() - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID || secrets.AWS_ACCESS_KEY }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - run: sccache --show-stats || true + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + clean: 'false' + - name: Mount Namespace cache volume + uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0 + with: + cache: rust + path: |- + /sccache + /nix + continue-on-error: true + - name: Setup Nix dev shell + uses: './.github/actions/setup-cachix' + with: + cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }} + dev-shell: 'true' + - name: Point sccache at the cache volume + run: |- + echo "SCCACHE_DIR=/sccache" >> "$GITHUB_ENV" + echo "SCCACHE_CACHE_SIZE=20G" >> "$GITHUB_ENV" + - name: configure postgres for concurrent tests + run: | + postgres_container="$(docker ps --format '{{.ID}} {{.Image}}' | awk '$2 == "pgvector/pgvector:pg16" { print $1; exit }')" + if [ -z "$postgres_container" ]; then + echo "pgvector/pgvector:pg16 service container not found" >&2 + docker ps + exit 1 + fi + docker exec -i "$postgres_container" psql -U user -d macrodb <<'SQL' + ALTER SYSTEM SET fsync = off; + ALTER SYSTEM SET synchronous_commit = off; + ALTER SYSTEM SET full_page_writes = off; + ALTER SYSTEM SET max_wal_size = '4GB'; + ALTER SYSTEM SET checkpoint_timeout = '30min'; + ALTER SYSTEM SET max_locks_per_transaction = 8192; + SQL + docker restart "$postgres_container" + until docker exec "$postgres_container" pg_isready -U user -d macrodb; do + sleep 1 + done + docker exec "$postgres_container" psql -U user -d macrodb -c "SHOW max_locks_per_transaction;" + - name: prepare tests + run: just rust/cloud-storage/setup_test_envs && just rust/cloud-storage/initialize_dbs + - name: run tests + run: | + cd rust/cloud-storage - # Collector job that always runs - use this as the required status check + args=(--all-features --lib --bins --tests --test-threads "$NEXTEST_TEST_THREADS") + if [ -n "$NEXTEST_FILTER" ]; then + args+=(-E "$NEXTEST_FILTER") + fi + cargo nextest run "${args[@]}" + - name: show sccache stats + if: always() + run: sccache --show-stats || true status-check: - name: Cloud Storage Status Check + needs: + - path-check + - check + - test if: always() - needs: [path-check, check, test] - runs-on: ubuntu-latest + name: Cloud Storage Status Check + runs-on: namespace-profile-linux-small steps: - - name: Check job results - run: | - echo "path-check: ${{ needs.path-check.result }}" - echo "check: ${{ needs.check.result }}" - echo "test: ${{ needs.test.result }}" + - name: Check job results + run: | + echo "path-check: ${{ needs.path-check.result }}" + echo "check: ${{ needs.check.result }}" + echo "test: ${{ needs.test.result }}" - # Fail if any job failed (skipped and success are both OK) - if [[ "${{ needs.path-check.result }}" == "failure" ]] || \ - [[ "${{ needs.check.result }}" == "failure" ]] || \ - [[ "${{ needs.test.result }}" == "failure" ]]; then - echo "❌ One or more jobs failed" - exit 1 - fi + # Fail if any job failed (skipped and success are both OK) + if [[ "${{ needs.path-check.result }}" == "failure" ]] || \ + [[ "${{ needs.check.result }}" == "failure" ]] || \ + [[ "${{ needs.test.result }}" == "failure" ]]; then + echo "❌ One or more jobs failed" + exit 1 + fi - echo "✅ All jobs passed or were skipped" + echo "✅ All jobs passed or were skipped" +concurrency: + group: code-check-cloud-storage-${{ github.ref }} + cancel-in-progress: true diff --git a/rust/cloud-storage/.cargo/config.toml b/rust/cloud-storage/.cargo/config.toml new file mode 100644 index 00000000000..87de5908466 --- /dev/null +++ b/rust/cloud-storage/.cargo/config.toml @@ -0,0 +1,10 @@ +# Cargo aliases for the cloud-storage workspace. +# +# `cargo x ` runs the xtask repo-automation tool from anywhere inside +# rust/cloud-storage: +# cargo x workflows [--check] # regenerate / verify .github/workflows +# cargo x deps [--check] # regenerate / verify workspace-hack + dep closures +# cargo x nextest-filter # nextest rdeps filter for changed files +# cargo x doppler-bins # Doppler config-validation bins affected by changed files +[alias] +x = "run -p xtask --" diff --git a/rust/cloud-storage/Cargo.lock b/rust/cloud-storage/Cargo.lock index 3739506b48b..51464d3788f 100644 --- a/rust/cloud-storage/Cargo.lock +++ b/rust/cloud-storage/Cargo.lock @@ -3077,6 +3077,16 @@ dependencies = [ "darling_macro 0.20.11", ] +[[package]] +name = "darling" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" +dependencies = [ + "darling_core 0.21.3", + "darling_macro 0.21.3", +] + [[package]] name = "darling" version = "0.23.0" @@ -3101,6 +3111,20 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "darling_core" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim 0.11.1", + "syn 2.0.117", +] + [[package]] name = "darling_core" version = "0.23.0" @@ -3125,6 +3149,17 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "darling_macro" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" +dependencies = [ + "darling_core 0.21.3", + "quote", + "syn 2.0.117", +] + [[package]] name = "darling_macro" version = "0.23.0" @@ -3412,6 +3447,18 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "derive_setters" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7e6f6fa1f03c14ae082120b84b3c7fbd7b8588d924cf2d7c3daf9afd49df8b9" +dependencies = [ + "darling 0.21.3", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "device-info" version = "0.1.1" @@ -5037,6 +5084,35 @@ dependencies = [ "wasip3", ] +[[package]] +name = "gh-workflow" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389296a8f9ebed67f6f49f3ee7fb5870d34b1f134d6033727de8caac3a6336a1" +dependencies = [ + "async-trait", + "derive_more 2.1.1", + "derive_setters", + "gh-workflow-macros", + "indexmap 2.14.0", + "merge", + "serde", + "serde_json", + "serde_yml", + "strum_macros", +] + +[[package]] +name = "gh-workflow-macros" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b8281789edecfe1c6dab6312577f5ec0f7f8b860cad70156b8fc70ebedc786d" +dependencies = [ + "heck 0.5.0", + "quote", + "syn 2.0.117", +] + [[package]] name = "ghash" version = "0.5.1" @@ -6583,6 +6659,16 @@ dependencies = [ "glob", ] +[[package]] +name = "libyml" +version = "0.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3302702afa434ffa30847a83305f0a69d6abd74293b6554c18ec85c7ef30c980" +dependencies = [ + "anyhow", + "version_check", +] + [[package]] name = "linux-raw-sys" version = "0.4.15" @@ -7378,6 +7464,28 @@ dependencies = [ "workspace-hack", ] +[[package]] +name = "merge" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e520ba58faea3487f75df198b1d079644ec226ea3b0507d002c6fa4b8cf93a" +dependencies = [ + "merge_derive", + "num-traits", +] + +[[package]] +name = "merge_derive" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c8f8ce6efff81cbc83caf4af0905c46e58cb46892f63ad3835e81b47eaf7968" +dependencies = [ + "proc-macro-error2", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "mime" version = "0.3.17" @@ -9312,6 +9420,7 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", + "syn 2.0.117", ] [[package]] @@ -11124,6 +11233,21 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "serde_yml" +version = "0.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59e2dd588bf1597a252c3b920e0143eb99b0f76e4e082f4c92ce34fbc9e71ddd" +dependencies = [ + "indexmap 2.14.0", + "itoa", + "libyml", + "memchr", + "ryu", + "serde", + "version_check", +] + [[package]] name = "serial_test" version = "3.4.0" @@ -14133,8 +14257,10 @@ name = "xtask" version = "0.1.0" dependencies = [ "anyhow", + "gh-workflow", "guppy", "hakari", + "indoc", "serde_json", ] diff --git a/rust/cloud-storage/tools/xtask/Cargo.toml b/rust/cloud-storage/tools/xtask/Cargo.toml index e3701018519..07447829c58 100644 --- a/rust/cloud-storage/tools/xtask/Cargo.toml +++ b/rust/cloud-storage/tools/xtask/Cargo.toml @@ -14,3 +14,7 @@ serde_json = { workspace = true } # version this hakari release is built against. hakari = { version = "0.17.9", features = ["cli-support"] } guppy = "0.17.24" +# Define GitHub Actions workflows in Rust (`cargo run -p xtask -- workflows`). +# xtask is excluded from hakari traversal, so these stay out of workspace-hack. +gh-workflow = "0.8" +indoc = "2" diff --git a/rust/cloud-storage/tools/xtask/src/doppler_bins.rs b/rust/cloud-storage/tools/xtask/src/doppler_bins.rs new file mode 100644 index 00000000000..f4125df269f --- /dev/null +++ b/rust/cloud-storage/tools/xtask/src/doppler_bins.rs @@ -0,0 +1,82 @@ +//! Computes which "doppler config" validation binaries are affected by a set of +//! changed files, for cloud-storage CI. +//! +//! A handful of services ship a `doppler_config` binary — a `[[bin]]` whose +//! source is `src/doppler_config.rs` — that loads the service's `Config` from +//! Doppler for both the dev and prod environments. It is a config-contract +//! check: if the code expects a field Doppler doesn't have, the binary fails. +//! CI only needs to run it for a service when that service's config could have +//! changed, i.e. when its `src/config.rs`, `src/doppler_config.rs`, or +//! `Cargo.toml` changed. +//! +//! Input is a newline-delimited file of paths relative to the repository root +//! (typically `git diff --name-only ...`). Output is the affected bin names, one +//! per line, sorted and deduplicated. This is the native replacement for the +//! previous bash + python script and reuses the same guppy package graph as +//! [`crate::nextest_filter`]. + +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; +use guppy::graph::{BuildTargetId, PackageGraph}; + +/// Source path (relative to a crate root) of a service's doppler-config binary. +const DOPPLER_CONFIG_SRC: &str = "src/doppler_config.rs"; + +pub fn run(graph: &PackageGraph, changed_files_path: &Path) -> Result<()> { + let workspace = graph.workspace(); + let ws_root = workspace.root(); + let repo_root = ws_root + .ancestors() + .nth(2) + .with_context(|| format!("no repo root two levels above {ws_root}"))?; + + let changed_files = std::fs::read_to_string(changed_files_path).with_context(|| { + format!( + "reading changed files from {}", + changed_files_path.display() + ) + })?; + let changed: BTreeSet = changed_files + .lines() + .filter(|line| !line.is_empty()) + .map(|line| repo_root.join(line).into_std_path_buf()) + .collect(); + + let mut bins = BTreeSet::new(); + for package in workspace.iter() { + // Only services that actually ship a `src/doppler_config.rs` binary. + let Some(bin_name) = package + .build_targets() + .find_map(|target| match target.id() { + BuildTargetId::Binary(name) if target.path().ends_with(DOPPLER_CONFIG_SRC) => { + Some(name) + } + _ => None, + }) + else { + continue; + }; + + let dir = package + .manifest_path() + .parent() + .with_context(|| format!("manifest {} has no parent", package.manifest_path()))?; + + // Re-validate when the config schema, the validator, or the manifest changes. + let triggers = [ + dir.join("src/config.rs").into_std_path_buf(), + dir.join(DOPPLER_CONFIG_SRC).into_std_path_buf(), + dir.join("Cargo.toml").into_std_path_buf(), + ]; + if triggers.iter().any(|trigger| changed.contains(trigger)) { + bins.insert(bin_name.to_owned()); + } + } + + for bin in bins { + println!("{bin}"); + } + Ok(()) +} diff --git a/rust/cloud-storage/tools/xtask/src/main.rs b/rust/cloud-storage/tools/xtask/src/main.rs index 03bd83c950c..673602d88ab 100644 --- a/rust/cloud-storage/tools/xtask/src/main.rs +++ b/rust/cloud-storage/tools/xtask/src/main.rs @@ -16,8 +16,10 @@ //! the regenerate mode; CI runs `--check` and fails on drift. mod closures; +mod doppler_bins; mod hakari_ops; mod nextest_filter; +mod workflows; use std::path::Path; @@ -34,8 +36,14 @@ fn main() -> Result<()> { let graph = build_graph(false)?; nextest_filter::run(&graph, Path::new(changed_files_path)) } + ["doppler-bins", changed_files_path] => { + let graph = build_graph(false)?; + doppler_bins::run(&graph, Path::new(changed_files_path)) + } + ["workflows"] => workflows::generate(), + ["workflows", "--check"] => workflows::check(), _ => bail!( - "usage:\n cargo run -p xtask -- deps [--check]\n cargo run -p xtask -- nextest-filter " + "usage (from rust/cloud-storage):\n cargo x deps [--check]\n cargo x nextest-filter \n cargo x doppler-bins \n cargo x workflows [--check]" ), } } diff --git a/rust/cloud-storage/tools/xtask/src/workflows/check_generated.rs b/rust/cloud-storage/tools/xtask/src/workflows/check_generated.rs new file mode 100644 index 00000000000..ca0809b97a6 --- /dev/null +++ b/rust/cloud-storage/tools/xtask/src/workflows/check_generated.rs @@ -0,0 +1,33 @@ +//! `check generated workflows` — fails a PR if the committed `.github/workflows` +//! YAML has drifted from this Rust source. Generated into +//! `check-generated-workflows.yml`. This is what makes "workflows are defined in +//! Rust" enforceable rather than aspirational. + +use gh_workflow::{Event, Job, PullRequest, Step, Workflow}; + +use crate::workflows::{runners, steps}; + +/// Build the workflow. +pub fn check_generated_workflows() -> Workflow { + Workflow::new("check generated workflows") + .on(Event::default().pull_request( + PullRequest::default() + .add_branch("main") + .add_path("rust/cloud-storage/tools/xtask/**") + .add_path(".github/workflows/**"), + )) + .add_job("check-workflows", check_workflows()) +} + +/// Regenerate in `--check` mode and fail on any difference. +fn check_workflows() -> Job { + Job::default() + .runs_on(runners::LINUX_SMALL) + .add_step(steps::checkout(false)) + .add_step(steps::setup_rust_light()) + .add_step( + Step::new("verify workflows are up to date").run( + "cargo run --manifest-path rust/cloud-storage/tools/xtask/Cargo.toml -- workflows --check", + ), + ) +} diff --git a/rust/cloud-storage/tools/xtask/src/workflows/code_check_cloud_storage.rs b/rust/cloud-storage/tools/xtask/src/workflows/code_check_cloud_storage.rs new file mode 100644 index 00000000000..e10b4777149 --- /dev/null +++ b/rust/cloud-storage/tools/xtask/src/workflows/code_check_cloud_storage.rs @@ -0,0 +1,369 @@ +//! `cloud storage code check` — cargo fmt/clippy/test for the cloud-storage +//! workspace on pull requests. Generated into `code-check-cloud-storage.yml`. +//! +//! Ported from the hand-written workflow, with two infra changes: the runners +//! moved to Namespace profiles, and sccache moved off the S3 bucket onto a +//! persisted Namespace cache volume (so there are no AWS credentials anywhere). + +use gh_workflow::{ + Container, Env, Event, Expression, Job, Port, PullRequest, PullRequestType, Run, Step, Workflow, +}; + +use crate::workflows::{ + runners, + steps::{self, FluentBuilder}, + vars, +}; + +/// Build the workflow. +pub fn code_check_cloud_storage() -> Workflow { + Workflow::new("cloud storage code check") + .on(Event::default().pull_request( + PullRequest::default() + .add_branch("main") + .add_type(PullRequestType::Opened) + .add_type(PullRequestType::Synchronize) + .add_type(PullRequestType::Reopened) + .add_type(PullRequestType::ReadyForReview), + )) + .map(vars::with_global_env) + .concurrency(vars::concurrency("code-check-cloud-storage")) + .add_job("path-check", path_check()) + .add_job("check", check()) + .add_job("test", test()) + .add_job("status-check", status_check()) +} + +/// Decide whether the rest of the workflow runs, and compute the nextest filter. +fn path_check() -> Job { + Job::default() + .runs_on(runners::LINUX_SMALL) + .add_output("should_run", "${{ steps.filter.outputs.should_run }}") + .add_output( + "nextest_filter", + "${{ steps.nextest-filter.outputs.nextest_filter }}", + ) + .add_output( + "doppler_config_bins", + "${{ steps.doppler-bins.outputs.doppler_config_bins }}", + ) + .add_step(steps::checkout(true)) + .add_step(steps::setup_rust_light()) + .add_step(paths_filter()) + .add_step(compute_changed_files()) + .add_step(compute_doppler_bins()) + .add_step(compute_nextest_filter()) +} + +/// fmt + clippy (and Doppler-config validation). +fn check() -> Job { + steps::gated_job() + .runs_on(runners::LINUX_MID) + .add_env(( + "RUSTFLAGS", + "-Dwarnings -Dclippy::disallowed_methods -C link-arg=-fuse-ld=mold", + )) + .add_env(("RUSTDOCFLAGS", "-Dwarnings")) + .add_step(steps::checkout(false)) + .add_step(steps::mount_cache_volume()) + .add_step(steps::setup_dev_shell()) + .add_step(steps::pin_sccache_dir()) + .add_step(validate_doppler_configs()) + .add_step(cargo_fmt()) + .add_step(cargo_clippy()) + .add_step(steps::show_sccache_stats()) +} + +/// cargo nextest against postgres + redis service containers. +fn test() -> Job { + steps::gated_job() + .runs_on(runners::LINUX_MID) + .add_env(( + "NEXTEST_FILTER", + "${{ needs.path-check.outputs.nextest_filter }}", + )) + .add_env(("NEXTEST_TEST_THREADS", vars::NEXTEST_TEST_THREADS)) + .add_env(("RUSTFLAGS", "-Dwarnings -C link-arg=-fuse-ld=mold")) + .add_service("postgres", postgres_service()) + .add_service("redis", redis_service()) + .add_step(steps::checkout(false)) + .add_step(steps::mount_cache_volume()) + .add_step(steps::setup_dev_shell()) + .add_step(steps::pin_sccache_dir()) + .add_step(configure_postgres()) + .add_step(prepare_tests()) + .add_step(run_tests()) + .add_step(steps::show_sccache_stats()) +} + +/// Always-run collector used as the required status check. Its name must stay +/// stable — branch protection references it. +fn status_check() -> Job { + Job::default() + .name("Cloud Storage Status Check") + .runs_on(runners::LINUX_SMALL) + .cond(Expression::new("always()")) + .needs(vec![ + "path-check".to_string(), + "check".to_string(), + "test".to_string(), + ]) + .add_step(check_job_results()) +} + +// --- workflow-specific steps ------------------------------------------------- + +/// Detect whether cloud-storage-relevant paths changed. +fn paths_filter() -> Step { + Step::new("Filter changed paths") + .uses("dorny", "paths-filter", "v3") + .id("filter") + .add_with(( + "filters", + indoc::indoc! {r#" + should_run: + - 'rust/cloud-storage/**' + - 'rust/rust-toolchain.toml' + - 'flake.nix' + - 'flake.lock' + - '.github/actions/setup-cachix/**' + - '.github/actions/setup-sccache/**' + - '.github/services-config.json' + - '.github/scripts/build-cloud-storage-lambdas.sh' + - '.github/scripts/build-cloud-storage-lambdas-nix.sh' + - .github/workflows/code-check-cloud-storage.yml + "#}, + )) +} + +/// Compute the changed-file set once (shared by the nextest filter and the +/// Doppler-bin detection) and write it to `/tmp/changed-files`. On a missing +/// merge-base we leave the list empty, which makes both downstream steps fall +/// back to "everything": run all tests, validate no Doppler bins. +fn compute_changed_files() -> Step { + Step::new("compute changed files") + .run(indoc::indoc! {r#" + set -euo pipefail + + if [ -z "${GITHUB_BASE_REF:-}" ]; then + compare_rev="$(git rev-parse HEAD~1)" + else + git fetch origin "$GITHUB_BASE_REF:refs/remotes/origin/$GITHUB_BASE_REF" + if ! compare_rev="$(git merge-base "origin/${GITHUB_BASE_REF}" HEAD)"; then + echo "Unable to find merge-base for origin/${GITHUB_BASE_REF}; falling back to full test suite" >&2 + : > /tmp/changed-files + exit 0 + fi + fi + + git diff --name-only "$compare_rev" "$GITHUB_SHA" > /tmp/changed-files + "#}) + .if_condition(Expression::new("steps.filter.outputs.should_run == 'true'")) + .shell("bash") +} + +/// Determine which services' Doppler config-validation binaries are affected by +/// the changed files, via the `xtask doppler-bins` subcommand. +fn compute_doppler_bins() -> Step { + Step::new("compute affected Doppler config bins") + .run(indoc::indoc! {r#" + set -euo pipefail + + doppler_config_bins="$(cargo run --manifest-path rust/cloud-storage/tools/xtask/Cargo.toml -- doppler-bins /tmp/changed-files)" + { + echo 'doppler_config_bins<<__DOPPLER_CONFIG_BINS__' + if [ -n "$doppler_config_bins" ]; then + printf '%s\n' "$doppler_config_bins" + fi + echo '__DOPPLER_CONFIG_BINS__' + } >> "$GITHUB_OUTPUT" + "#}) + .id("doppler-bins") + .if_condition(Expression::new("steps.filter.outputs.should_run == 'true'")) + .shell("bash") +} + +/// Compute the cargo-nextest package filter from the changed files, via the +/// `xtask nextest-filter` subcommand. Root cargo/toolchain/CI changes +/// short-circuit to an empty filter (run the whole suite). +fn compute_nextest_filter() -> Step { + Step::new("compute nextest package filter") + .run(indoc::indoc! {r#" + set -euo pipefail + + # Root cargo/toolchain/CI changes can affect the whole workspace, so run all tests. + if grep -qE '^(rust/rust-toolchain\.toml|flake\.nix|flake\.lock|rust/cloud-storage/Cargo\.(toml|lock)|rust/cloud-storage/\.cargo/.*|\.github/actions/(setup-rust|setup-cachix|setup-sccache)/.*|\.github/workflows/code-check-cloud-storage\.yml)$' /tmp/changed-files; then + echo "Workspace-level change detected; running all tests" + echo "nextest_filter=" >> "$GITHUB_OUTPUT" + exit 0 + fi + + filterset="$(cargo run --manifest-path rust/cloud-storage/tools/xtask/Cargo.toml -- nextest-filter /tmp/changed-files)" + + if [ -z "$filterset" ]; then + echo "No package-specific Rust changes detected; running all tests" + else + echo "nextest filter: $filterset" + fi + echo "nextest_filter=$filterset" >> "$GITHUB_OUTPUT" + "#}) + .id("nextest-filter") + .if_condition(Expression::new("steps.filter.outputs.should_run == 'true'")) + .shell("bash") +} + +/// Build and run the Doppler config binaries affected by this PR. (sccache is +/// local now, so this no longer needs AWS credentials — only the assertion that +/// `RUSTC_WRAPPER` is wired stays.) +fn validate_doppler_configs() -> Step { + Step::new("validate Doppler configs") + .run(indoc::indoc! {r#" + set -euo pipefail + + if [ -z "${DOPPLER_TOKEN:-}" ]; then + echo "DOPPLER_TOKEN secret is required to validate Doppler configs" >&2 + exit 1 + fi + + if [ -z "${RUSTC_WRAPPER:-}" ]; then + echo "RUSTC_WRAPPER is required so Doppler config binaries build through sccache" >&2 + exit 1 + fi + + bins=() + cargo_args=(build --locked --all-features) + while IFS= read -r bin; do + if [ -z "$bin" ]; then + continue + fi + + bins+=("$bin") + cargo_args+=(--bin "$bin") + done <<< "$DOPPLER_CONFIG_BINS" + + if [ "${#bins[@]}" -eq 0 ]; then + echo "No Doppler config binaries to validate" + exit 0 + fi + + echo "Building affected Doppler config binaries with RUSTC_WRAPPER=$RUSTC_WRAPPER" + printf ' %s\n' "${bins[@]}" + + ( + cd rust/cloud-storage + cargo "${cargo_args[@]}" + for bin in "${bins[@]}"; do + "./target/debug/$bin" + done + ) + "#}) + .if_condition(Expression::new( + "needs.path-check.outputs.doppler_config_bins != ''", + )) + .add_env(( + "DOPPLER_CONFIG_BINS", + "${{ needs.path-check.outputs.doppler_config_bins }}", + )) + .add_env(("DOPPLER_TOKEN", vars::DOPPLER_TOKEN)) +} + +/// `cargo fmt --check`. +fn cargo_fmt() -> Step { + Step::new("fmt").run("cd rust/cloud-storage && cargo fmt --check") +} + +/// `cargo clippy` (no AWS creds — sccache is local). +fn cargo_clippy() -> Step { + Step::new("clippy").run("cd rust/cloud-storage && cargo clippy --workspace --all-features") +} + +/// pgvector service container, tuned env preserved. +fn postgres_service() -> Container { + Container::default() + .image("pgvector/pgvector:pg16") + .env( + Env::new("POSTGRES_USER", "user") + .add("POSTGRES_PASSWORD", "password") + .add("POSTGRES_DB", "macrodb"), + ) + .ports(vec![Port::Name("5432:5432".to_string())]) + .options( + "--health-cmd pg_isready --health-interval 10s --health-timeout 5s \ + --health-retries 5 --shm-size 1g", + ) +} + +/// redis service container. +fn redis_service() -> Container { + Container::default() + .image("redis:7") + .ports(vec![Port::Name("6379:6379".to_string())]) + .options( + "--health-cmd \"redis-cli ping\" --health-interval 10s \ + --health-timeout 5s --health-retries 5", + ) +} + +/// Tune the postgres service container for fast concurrent tests. +fn configure_postgres() -> Step { + Step::new("configure postgres for concurrent tests").run(indoc::indoc! {r#" + postgres_container="$(docker ps --format '{{.ID}} {{.Image}}' | awk '$2 == "pgvector/pgvector:pg16" { print $1; exit }')" + if [ -z "$postgres_container" ]; then + echo "pgvector/pgvector:pg16 service container not found" >&2 + docker ps + exit 1 + fi + + docker exec -i "$postgres_container" psql -U user -d macrodb <<'SQL' + ALTER SYSTEM SET fsync = off; + ALTER SYSTEM SET synchronous_commit = off; + ALTER SYSTEM SET full_page_writes = off; + ALTER SYSTEM SET max_wal_size = '4GB'; + ALTER SYSTEM SET checkpoint_timeout = '30min'; + ALTER SYSTEM SET max_locks_per_transaction = 8192; + SQL + docker restart "$postgres_container" + until docker exec "$postgres_container" pg_isready -U user -d macrodb; do + sleep 1 + done + docker exec "$postgres_container" psql -U user -d macrodb -c "SHOW max_locks_per_transaction;" + "#}) +} + +/// Set up test env files and databases. +fn prepare_tests() -> Step { + Step::new("prepare tests") + .run("just rust/cloud-storage/setup_test_envs && just rust/cloud-storage/initialize_dbs") +} + +/// Run the test suite (no AWS creds — sccache is local). +fn run_tests() -> Step { + Step::new("run tests").run(indoc::indoc! {r#" + cd rust/cloud-storage + + args=(--all-features --lib --bins --tests --test-threads "$NEXTEST_TEST_THREADS") + if [ -n "$NEXTEST_FILTER" ]; then + args+=(-E "$NEXTEST_FILTER") + fi + cargo nextest run "${args[@]}" + "#}) +} + +/// Aggregate the upstream job results into a single required status check. +fn check_job_results() -> Step { + Step::new("Check job results").run(indoc::indoc! {r#" + echo "path-check: ${{ needs.path-check.result }}" + echo "check: ${{ needs.check.result }}" + echo "test: ${{ needs.test.result }}" + + # Fail if any job failed (skipped and success are both OK) + if [[ "${{ needs.path-check.result }}" == "failure" ]] || \ + [[ "${{ needs.check.result }}" == "failure" ]] || \ + [[ "${{ needs.test.result }}" == "failure" ]]; then + echo "❌ One or more jobs failed" + exit 1 + fi + + echo "✅ All jobs passed or were skipped" + "#}) +} diff --git a/rust/cloud-storage/tools/xtask/src/workflows/mod.rs b/rust/cloud-storage/tools/xtask/src/workflows/mod.rs new file mode 100644 index 00000000000..5fa07c8be05 --- /dev/null +++ b/rust/cloud-storage/tools/xtask/src/workflows/mod.rs @@ -0,0 +1,112 @@ +//! Generate the repo's GitHub Actions workflows from Rust. +//! +//! `cargo run -p xtask -- workflows` writes every entry in [`WORKFLOWS`] to +//! `/.github/workflows/`. The `--check` variant regenerates +//! them in memory and fails if the committed YAML has drifted, so CI can +//! guarantee the checked-in YAML always matches this source. +//! +//! Layout mirrors Zed's xtask: one file per workflow plus three shared "library" +//! files — [`runners`] (runner labels), [`vars`] (env / secrets / concurrency), +//! and [`steps`] (reusable step + job helpers). + +mod check_generated; +mod code_check_cloud_storage; +mod runners; +mod steps; +mod vars; + +use std::fs; +use std::path::{Path, PathBuf}; + +use anyhow::{bail, Context, Result}; +use gh_workflow::Workflow; + +/// A generated workflow. +struct WorkflowFile { + /// File written under `.github/workflows/`. Kept stable so branch + /// protection's required status checks keep matching. + filename: &'static str, + /// Source module (for the "do not edit" header), e.g. `code_check_cloud_storage`. + source: &'static str, + /// Builds the workflow definition. + build: fn() -> Workflow, +} + +/// Every workflow we generate. Add new workflows here. +const WORKFLOWS: &[WorkflowFile] = &[ + WorkflowFile { + filename: "code-check-cloud-storage.yml", + source: "code_check_cloud_storage", + build: code_check_cloud_storage::code_check_cloud_storage, + }, + WorkflowFile { + filename: "check-generated-workflows.yml", + source: "check_generated", + build: check_generated::check_generated_workflows, + }, +]; + +/// Write every workflow to disk. +pub fn generate() -> Result<()> { + let dir = workflows_dir()?; + for workflow in WORKFLOWS { + let path = dir.join(workflow.filename); + fs::write(&path, render(workflow)?) + .with_context(|| format!("writing {}", path.display()))?; + println!("generated {}", path.display()); + } + Ok(()) +} + +/// Fail if any committed workflow differs from what we'd generate now. +pub fn check() -> Result<()> { + let dir = workflows_dir()?; + let mut stale = Vec::new(); + for workflow in WORKFLOWS { + let path = dir.join(workflow.filename); + let on_disk = + fs::read_to_string(&path).with_context(|| format!("reading {}", path.display()))?; + if on_disk != render(workflow)? { + stale.push(workflow.filename); + } + } + if !stale.is_empty() { + bail!( + "generated workflows are stale: {}\nrun `cargo x workflows` from rust/cloud-storage and commit the result", + stale.join(", ") + ); + } + println!("all generated workflows are up to date"); + Ok(()) +} + +/// Serialize a workflow to YAML and prepend the "do not edit" header. +fn render(workflow: &WorkflowFile) -> Result { + let yaml = (workflow.build)() + .to_string() + .map_err(|e| anyhow::anyhow!("serializing {}: {e:?}", workflow.filename))?; + Ok(format!("{}{yaml}", disclaimer(workflow.source))) +} + +/// The header every generated file starts with. +fn disclaimer(source: &str) -> String { + format!( + "# DO NOT EDIT — regenerate with `cargo x workflows` (from rust/cloud-storage).\n\ + # Source: rust/cloud-storage/tools/xtask/src/workflows/{source}.rs\n", + ) +} + +/// `/.github/workflows`, anchored on the crate's manifest dir so the +/// task works from any cwd. This crate lives at +/// `/rust/cloud-storage/tools/xtask`, i.e. four ancestors up. +fn workflows_dir() -> Result { + let repo_root = Path::new(env!("CARGO_MANIFEST_DIR")) + .ancestors() + .nth(4) + .context("xtask manifest dir has no repo root four levels up")?; + let dir = repo_root.join(".github").join("workflows"); + if !dir.is_dir() { + bail!("expected a workflows directory at {}", dir.display()); + } + Ok(dir) +} diff --git a/rust/cloud-storage/tools/xtask/src/workflows/runners.rs b/rust/cloud-storage/tools/xtask/src/workflows/runners.rs new file mode 100644 index 00000000000..e7a145b49b2 --- /dev/null +++ b/rust/cloud-storage/tools/xtask/src/workflows/runners.rs @@ -0,0 +1,17 @@ +//! GitHub Actions runner labels. +//! +//! We run on Namespace (namespace.so) hosted runners, selected by the dashboard +//! *profile* name — the same convention the deploy workflows already use +//! (`namespace-profile-linux-mid`, etc.). Each profile's persisted cache volume +//! is configured in the Namespace dashboard; that volume is what backs the +//! sccache + cargo caches (see [`crate::workflows::steps::mount_cache_volume`]). +//! +//! `runs_on` accepts any `&str`, so these are plain consts — greppable, and the +//! whole runner set lives in this one file. + +/// Small profile for light jobs (path filtering, status aggregation). +pub const LINUX_SMALL: &str = "namespace-profile-linux-small"; + +/// Mid profile for the heavy compile + test jobs. Has a cache volume configured, +/// which is what makes the persisted sccache cache possible. +pub const LINUX_MID: &str = "namespace-profile-linux-mid"; diff --git a/rust/cloud-storage/tools/xtask/src/workflows/steps.rs b/rust/cloud-storage/tools/xtask/src/workflows/steps.rs new file mode 100644 index 00000000000..f9b7a752885 --- /dev/null +++ b/rust/cloud-storage/tools/xtask/src/workflows/steps.rs @@ -0,0 +1,116 @@ +//! Reusable workflow building blocks: a small fluent-builder trait plus typed +//! helpers that return `Step`s and `Job`s, composed by the workflow files. +//! +//! Third-party actions are pinned to a SHA with the human-readable version in a +//! trailing comment, matching the rest of the repo's workflows. + +use gh_workflow::{Expression, Job, Run, Step, Use}; + +use crate::workflows::vars; + +/// `.map` / `.when` combinators for fluent conditional composition +/// ("push ifs up"): centralize branching in the builder chain instead of +/// building values imperatively. +pub trait FluentBuilder: Sized { + /// Apply `f` to `self`. + fn map(self, f: impl FnOnce(Self) -> Self) -> Self { + f(self) + } + /// Apply `f` only when `cond` holds. + fn when(self, cond: bool, f: impl FnOnce(Self) -> Self) -> Self { + if cond { + f(self) + } else { + self + } + } +} + +impl FluentBuilder for gh_workflow::Workflow {} +impl FluentBuilder for Job {} +impl FluentBuilder for Step {} + +/// Reference a repo-local composite action (`uses: ./path`). The base +/// `gh-workflow` `uses()` only builds `owner/repo@version`, so we set the raw +/// `uses` field directly. Kept in one place so the workaround is contained. +fn uses_local(name: &str, path: &str) -> Step { + let mut step = Step::new(name).uses("local", "local", "0"); + step.value.uses = Some(path.to_string()); + step +} + +/// `actions/checkout`, pinned. `full_history` fetches the full history, which the +/// path-filter diff in `path-check` needs. +pub fn checkout(full_history: bool) -> Step { + Step::new("Checkout") + .uses( + "actions", + "checkout", + "de0fac2e4500dabe0009e67214ff5f5447ce83dd", + ) // v4 + .add_with(("clean", false)) + .when(full_history, |step| step.add_with(("fetch-depth", 0))) +} + +/// Install the Rust toolchain only (no sccache, no cache) — for the lightweight +/// `path-check` and workflow-drift jobs. +pub fn setup_rust_light() -> Step { + uses_local("Setup Rust", "./.github/actions/setup-rust") + .add_with(("sccache", "false")) + .add_with(("rust-cache", "false")) +} + +/// Enter the repo's Nix dev shell (toolchain, mold, just, the sccache binary, +/// and `RUSTC_WRAPPER=sccache`). We pass NO `sccache-bucket`, so sccache runs in +/// local-disk mode instead of talking to S3. +pub fn setup_dev_shell() -> Step { + uses_local("Setup Nix dev shell", "./.github/actions/setup-cachix") + .add_with(("cachix-auth-token", vars::CACHIX_AUTH_TOKEN)) + .add_with(("dev-shell", "true")) +} + +/// Mount the Namespace profile's persisted cache volume: `cache: rust` persists +/// the cargo registry/git, and `path:` persists the sccache dir plus the Nix +/// store. `continue-on-error` because the cache is a pure optimization — a +/// missing/failed volume just means a cold build, never a wrong one (mirrors the +/// deploy workflows' `/nix` mounts). +pub fn mount_cache_volume() -> Step { + Step::new("Mount Namespace cache volume") + .uses( + "namespacelabs", + "nscloud-cache-action", + "15799a6b54e5765f85b2aac25b3f0df43ed571c0", // v1.4.3 + ) + .add_with(("cache", "rust")) + .add_with(("path", format!("{}\n/nix", vars::SCCACHE_VOLUME_DIR))) + .continue_on_error(true) +} + +/// Repoint sccache at the persisted volume. Runs AFTER `setup-cachix`, which +/// writes `SCCACHE_DIR=$HOME/.cache/sccache` to `$GITHUB_ENV`; this overrides it +/// for the cargo steps so compilation artifacts land on the sticky volume. +pub fn pin_sccache_dir() -> Step { + Step::new("Point sccache at the cache volume").run(format!( + "echo \"SCCACHE_DIR={dir}\" >> \"$GITHUB_ENV\"\n\ + echo \"SCCACHE_CACHE_SIZE={size}\" >> \"$GITHUB_ENV\"", + dir = vars::SCCACHE_VOLUME_DIR, + size = vars::SCCACHE_CACHE_SIZE, + )) +} + +/// `sccache --show-stats` at the end of a job (never fails the job). +pub fn show_sccache_stats() -> Step { + Step::new("show sccache stats") + .run("sccache --show-stats || true") + .if_condition(Expression::new("always()")) +} + +/// Base for jobs gated behind `path-check`: depends on it and runs only on +/// non-draft PRs where the path filter matched. Shared by `check` and `test`. +pub fn gated_job() -> Job { + Job::default() + .needs(vec!["path-check".to_string()]) + .cond(Expression::new( + "needs.path-check.outputs.should_run == 'true' && github.event.pull_request.draft == false", + )) +} diff --git a/rust/cloud-storage/tools/xtask/src/workflows/vars.rs b/rust/cloud-storage/tools/xtask/src/workflows/vars.rs new file mode 100644 index 00000000000..d4762fa8345 --- /dev/null +++ b/rust/cloud-storage/tools/xtask/src/workflows/vars.rs @@ -0,0 +1,46 @@ +//! Shared workflow environment: secrets, the repo-wide env block, concurrency, +//! and the sccache-on-volume settings. This is the "environment" file. + +use gh_workflow::{Concurrency, Expression, Workflow}; + +/// Declares a `${{ secrets.NAME }}` reference as a `&str` const named `NAME`, so +/// secret usage is greppable and typo-proof. +macro_rules! secret { + ($name:ident) => { + pub const $name: &str = concat!("${{ secrets.", stringify!($name), " }}"); + }; +} + +secret!(CACHIX_AUTH_TOKEN); +secret!(DOPPLER_TOKEN); + +/// Nextest thread count for the test job. Tuned for the previous +/// `linux-extra-beefy` runner; revisit if `namespace-profile-linux-mid` is +/// smaller. +pub const NEXTEST_TEST_THREADS: u32 = 32; + +/// Directory sccache uses for its local-disk cache. Lives on the Namespace cache +/// volume so it persists across runs — this is what replaces the S3 bucket. +pub const SCCACHE_VOLUME_DIR: &str = "/sccache"; + +/// Max on-disk size for the sccache cache. Larger than the setup default since +/// the persisted volume can hold a full-workspace cache. +pub const SCCACHE_CACHE_SIZE: &str = "20G"; + +/// The repo-wide env block (mirrors the original top-level `env:`). Defaults the +/// linker to `lld`; the heavy jobs override `RUSTFLAGS` to use `mold`. +pub fn with_global_env(workflow: Workflow) -> Workflow { + workflow + .add_env(("CARGO_INCREMENTAL", "0")) + .add_env(("CARGO_TERM_COLOR", "always")) + .add_env(("CARGO_PROFILE_DEV_DEBUG", "limited")) + .add_env(("CARGO_PROFILE_TEST_DEBUG", "limited")) + .add_env(("RUST_BACKTRACE", "1")) + .add_env(("RUSTFLAGS", "-C link-arg=-fuse-ld=lld")) +} + +/// Cancel superseded runs of this workflow on the same git ref. +pub fn concurrency(prefix: &str) -> Concurrency { + Concurrency::new(Expression::new(format!("{prefix}-${{{{ github.ref }}}}"))) + .cancel_in_progress(true) +} From a651bf9570559c2fbda405870eb6b026ec2b17cd Mon Sep 17 00:00:00 2001 From: synoet Date: Tue, 23 Jun 2026 14:41:55 -0400 Subject: [PATCH 2/8] chore(ci): address review feedback - extract inline workflow bash to scripts/*.sh via include_str! - model runners as an enum with Display - generalize FluentBuilder::map to T -> U - mark generated workflow YAML linguist-generated (collapse in PR diffs) Co-Authored-By: Claude Opus 4.8 (1M context) --- .gitattributes | 5 + .../xtask/src/workflows/check_generated.rs | 2 +- .../src/workflows/code_check_cloud_storage.rs | 133 ++---------------- .../tools/xtask/src/workflows/runners.rs | 36 +++-- .../scripts/compute_changed_files.sh | 14 ++ .../workflows/scripts/compute_doppler_bins.sh | 10 ++ .../scripts/compute_nextest_filter.sh | 17 +++ .../workflows/scripts/configure_postgres.sh | 20 +++ .../xtask/src/workflows/scripts/run_tests.sh | 7 + .../scripts/validate_doppler_configs.sh | 38 +++++ .../tools/xtask/src/workflows/steps.rs | 2 +- 11 files changed, 148 insertions(+), 136 deletions(-) create mode 100644 rust/cloud-storage/tools/xtask/src/workflows/scripts/compute_changed_files.sh create mode 100644 rust/cloud-storage/tools/xtask/src/workflows/scripts/compute_doppler_bins.sh create mode 100644 rust/cloud-storage/tools/xtask/src/workflows/scripts/compute_nextest_filter.sh create mode 100644 rust/cloud-storage/tools/xtask/src/workflows/scripts/configure_postgres.sh create mode 100644 rust/cloud-storage/tools/xtask/src/workflows/scripts/run_tests.sh create mode 100644 rust/cloud-storage/tools/xtask/src/workflows/scripts/validate_doppler_configs.sh diff --git a/.gitattributes b/.gitattributes index 8d0e6f7110d..7624912aec3 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,7 @@ # Mark .sqlx files as generated so they're excluded from PR diff stats **/.sqlx/** linguist-generated=true + +# Workflows generated by `cargo x workflows` (xtask). Collapse them in PR diffs +# and exclude from language stats; edit the Rust source, not the YAML. +.github/workflows/code-check-cloud-storage.yml linguist-generated=true +.github/workflows/check-generated-workflows.yml linguist-generated=true diff --git a/rust/cloud-storage/tools/xtask/src/workflows/check_generated.rs b/rust/cloud-storage/tools/xtask/src/workflows/check_generated.rs index ca0809b97a6..4cef3d0caa4 100644 --- a/rust/cloud-storage/tools/xtask/src/workflows/check_generated.rs +++ b/rust/cloud-storage/tools/xtask/src/workflows/check_generated.rs @@ -22,7 +22,7 @@ pub fn check_generated_workflows() -> Workflow { /// Regenerate in `--check` mode and fail on any difference. fn check_workflows() -> Job { Job::default() - .runs_on(runners::LINUX_SMALL) + .runs_on(runners::Runner::LinuxSmall.to_string()) .add_step(steps::checkout(false)) .add_step(steps::setup_rust_light()) .add_step( diff --git a/rust/cloud-storage/tools/xtask/src/workflows/code_check_cloud_storage.rs b/rust/cloud-storage/tools/xtask/src/workflows/code_check_cloud_storage.rs index e10b4777149..dd68d2f444b 100644 --- a/rust/cloud-storage/tools/xtask/src/workflows/code_check_cloud_storage.rs +++ b/rust/cloud-storage/tools/xtask/src/workflows/code_check_cloud_storage.rs @@ -37,7 +37,7 @@ pub fn code_check_cloud_storage() -> Workflow { /// Decide whether the rest of the workflow runs, and compute the nextest filter. fn path_check() -> Job { Job::default() - .runs_on(runners::LINUX_SMALL) + .runs_on(runners::Runner::LinuxSmall.to_string()) .add_output("should_run", "${{ steps.filter.outputs.should_run }}") .add_output( "nextest_filter", @@ -58,7 +58,7 @@ fn path_check() -> Job { /// fmt + clippy (and Doppler-config validation). fn check() -> Job { steps::gated_job() - .runs_on(runners::LINUX_MID) + .runs_on(runners::Runner::LinuxMid.to_string()) .add_env(( "RUSTFLAGS", "-Dwarnings -Dclippy::disallowed_methods -C link-arg=-fuse-ld=mold", @@ -77,7 +77,7 @@ fn check() -> Job { /// cargo nextest against postgres + redis service containers. fn test() -> Job { steps::gated_job() - .runs_on(runners::LINUX_MID) + .runs_on(runners::Runner::LinuxMid.to_string()) .add_env(( "NEXTEST_FILTER", "${{ needs.path-check.outputs.nextest_filter }}", @@ -101,7 +101,7 @@ fn test() -> Job { fn status_check() -> Job { Job::default() .name("Cloud Storage Status Check") - .runs_on(runners::LINUX_SMALL) + .runs_on(runners::Runner::LinuxSmall.to_string()) .cond(Expression::new("always()")) .needs(vec![ "path-check".to_string(), @@ -142,22 +142,7 @@ fn paths_filter() -> Step { /// back to "everything": run all tests, validate no Doppler bins. fn compute_changed_files() -> Step { Step::new("compute changed files") - .run(indoc::indoc! {r#" - set -euo pipefail - - if [ -z "${GITHUB_BASE_REF:-}" ]; then - compare_rev="$(git rev-parse HEAD~1)" - else - git fetch origin "$GITHUB_BASE_REF:refs/remotes/origin/$GITHUB_BASE_REF" - if ! compare_rev="$(git merge-base "origin/${GITHUB_BASE_REF}" HEAD)"; then - echo "Unable to find merge-base for origin/${GITHUB_BASE_REF}; falling back to full test suite" >&2 - : > /tmp/changed-files - exit 0 - fi - fi - - git diff --name-only "$compare_rev" "$GITHUB_SHA" > /tmp/changed-files - "#}) + .run(include_str!("scripts/compute_changed_files.sh")) .if_condition(Expression::new("steps.filter.outputs.should_run == 'true'")) .shell("bash") } @@ -166,18 +151,7 @@ fn compute_changed_files() -> Step { /// the changed files, via the `xtask doppler-bins` subcommand. fn compute_doppler_bins() -> Step { Step::new("compute affected Doppler config bins") - .run(indoc::indoc! {r#" - set -euo pipefail - - doppler_config_bins="$(cargo run --manifest-path rust/cloud-storage/tools/xtask/Cargo.toml -- doppler-bins /tmp/changed-files)" - { - echo 'doppler_config_bins<<__DOPPLER_CONFIG_BINS__' - if [ -n "$doppler_config_bins" ]; then - printf '%s\n' "$doppler_config_bins" - fi - echo '__DOPPLER_CONFIG_BINS__' - } >> "$GITHUB_OUTPUT" - "#}) + .run(include_str!("scripts/compute_doppler_bins.sh")) .id("doppler-bins") .if_condition(Expression::new("steps.filter.outputs.should_run == 'true'")) .shell("bash") @@ -188,25 +162,7 @@ fn compute_doppler_bins() -> Step { /// short-circuit to an empty filter (run the whole suite). fn compute_nextest_filter() -> Step { Step::new("compute nextest package filter") - .run(indoc::indoc! {r#" - set -euo pipefail - - # Root cargo/toolchain/CI changes can affect the whole workspace, so run all tests. - if grep -qE '^(rust/rust-toolchain\.toml|flake\.nix|flake\.lock|rust/cloud-storage/Cargo\.(toml|lock)|rust/cloud-storage/\.cargo/.*|\.github/actions/(setup-rust|setup-cachix|setup-sccache)/.*|\.github/workflows/code-check-cloud-storage\.yml)$' /tmp/changed-files; then - echo "Workspace-level change detected; running all tests" - echo "nextest_filter=" >> "$GITHUB_OUTPUT" - exit 0 - fi - - filterset="$(cargo run --manifest-path rust/cloud-storage/tools/xtask/Cargo.toml -- nextest-filter /tmp/changed-files)" - - if [ -z "$filterset" ]; then - echo "No package-specific Rust changes detected; running all tests" - else - echo "nextest filter: $filterset" - fi - echo "nextest_filter=$filterset" >> "$GITHUB_OUTPUT" - "#}) + .run(include_str!("scripts/compute_nextest_filter.sh")) .id("nextest-filter") .if_condition(Expression::new("steps.filter.outputs.should_run == 'true'")) .shell("bash") @@ -217,46 +173,7 @@ fn compute_nextest_filter() -> Step { /// `RUSTC_WRAPPER` is wired stays.) fn validate_doppler_configs() -> Step { Step::new("validate Doppler configs") - .run(indoc::indoc! {r#" - set -euo pipefail - - if [ -z "${DOPPLER_TOKEN:-}" ]; then - echo "DOPPLER_TOKEN secret is required to validate Doppler configs" >&2 - exit 1 - fi - - if [ -z "${RUSTC_WRAPPER:-}" ]; then - echo "RUSTC_WRAPPER is required so Doppler config binaries build through sccache" >&2 - exit 1 - fi - - bins=() - cargo_args=(build --locked --all-features) - while IFS= read -r bin; do - if [ -z "$bin" ]; then - continue - fi - - bins+=("$bin") - cargo_args+=(--bin "$bin") - done <<< "$DOPPLER_CONFIG_BINS" - - if [ "${#bins[@]}" -eq 0 ]; then - echo "No Doppler config binaries to validate" - exit 0 - fi - - echo "Building affected Doppler config binaries with RUSTC_WRAPPER=$RUSTC_WRAPPER" - printf ' %s\n' "${bins[@]}" - - ( - cd rust/cloud-storage - cargo "${cargo_args[@]}" - for bin in "${bins[@]}"; do - "./target/debug/$bin" - done - ) - "#}) + .run(include_str!("scripts/validate_doppler_configs.sh")) .if_condition(Expression::new( "needs.path-check.outputs.doppler_config_bins != ''", )) @@ -306,28 +223,8 @@ fn redis_service() -> Container { /// Tune the postgres service container for fast concurrent tests. fn configure_postgres() -> Step { - Step::new("configure postgres for concurrent tests").run(indoc::indoc! {r#" - postgres_container="$(docker ps --format '{{.ID}} {{.Image}}' | awk '$2 == "pgvector/pgvector:pg16" { print $1; exit }')" - if [ -z "$postgres_container" ]; then - echo "pgvector/pgvector:pg16 service container not found" >&2 - docker ps - exit 1 - fi - - docker exec -i "$postgres_container" psql -U user -d macrodb <<'SQL' - ALTER SYSTEM SET fsync = off; - ALTER SYSTEM SET synchronous_commit = off; - ALTER SYSTEM SET full_page_writes = off; - ALTER SYSTEM SET max_wal_size = '4GB'; - ALTER SYSTEM SET checkpoint_timeout = '30min'; - ALTER SYSTEM SET max_locks_per_transaction = 8192; - SQL - docker restart "$postgres_container" - until docker exec "$postgres_container" pg_isready -U user -d macrodb; do - sleep 1 - done - docker exec "$postgres_container" psql -U user -d macrodb -c "SHOW max_locks_per_transaction;" - "#}) + Step::new("configure postgres for concurrent tests") + .run(include_str!("scripts/configure_postgres.sh")) } /// Set up test env files and databases. @@ -338,15 +235,7 @@ fn prepare_tests() -> Step { /// Run the test suite (no AWS creds — sccache is local). fn run_tests() -> Step { - Step::new("run tests").run(indoc::indoc! {r#" - cd rust/cloud-storage - - args=(--all-features --lib --bins --tests --test-threads "$NEXTEST_TEST_THREADS") - if [ -n "$NEXTEST_FILTER" ]; then - args+=(-E "$NEXTEST_FILTER") - fi - cargo nextest run "${args[@]}" - "#}) + Step::new("run tests").run(include_str!("scripts/run_tests.sh")) } /// Aggregate the upstream job results into a single required status check. diff --git a/rust/cloud-storage/tools/xtask/src/workflows/runners.rs b/rust/cloud-storage/tools/xtask/src/workflows/runners.rs index e7a145b49b2..dcc0f0625e1 100644 --- a/rust/cloud-storage/tools/xtask/src/workflows/runners.rs +++ b/rust/cloud-storage/tools/xtask/src/workflows/runners.rs @@ -1,17 +1,29 @@ //! GitHub Actions runner labels. //! //! We run on Namespace (namespace.so) hosted runners, selected by the dashboard -//! *profile* name — the same convention the deploy workflows already use -//! (`namespace-profile-linux-mid`, etc.). Each profile's persisted cache volume -//! is configured in the Namespace dashboard; that volume is what backs the -//! sccache + cargo caches (see [`crate::workflows::steps::mount_cache_volume`]). -//! -//! `runs_on` accepts any `&str`, so these are plain consts — greppable, and the -//! whole runner set lives in this one file. +//! *profile* name — the same convention the deploy workflows already use. Each +//! profile's persisted cache volume is configured in the Namespace dashboard; +//! that volume backs the sccache + cargo caches (see +//! [`crate::workflows::steps::mount_cache_volume`]). + +use std::fmt; -/// Small profile for light jobs (path filtering, status aggregation). -pub const LINUX_SMALL: &str = "namespace-profile-linux-small"; +/// A Namespace runner profile. The set of profiles we're allowed to target is +/// closed; `Display` renders the `runs-on` label. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Runner { + /// Small profile for light jobs (path filtering, status aggregation). + LinuxSmall, + /// Mid profile for the heavy compile + test jobs. Has a cache volume + /// configured, which is what makes the persisted sccache cache possible. + LinuxMid, +} -/// Mid profile for the heavy compile + test jobs. Has a cache volume configured, -/// which is what makes the persisted sccache cache possible. -pub const LINUX_MID: &str = "namespace-profile-linux-mid"; +impl fmt::Display for Runner { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Runner::LinuxSmall => "namespace-profile-linux-small", + Runner::LinuxMid => "namespace-profile-linux-mid", + }) + } +} diff --git a/rust/cloud-storage/tools/xtask/src/workflows/scripts/compute_changed_files.sh b/rust/cloud-storage/tools/xtask/src/workflows/scripts/compute_changed_files.sh new file mode 100644 index 00000000000..4ad5cb04781 --- /dev/null +++ b/rust/cloud-storage/tools/xtask/src/workflows/scripts/compute_changed_files.sh @@ -0,0 +1,14 @@ +set -euo pipefail + +if [ -z "${GITHUB_BASE_REF:-}" ]; then + compare_rev="$(git rev-parse HEAD~1)" +else + git fetch origin "$GITHUB_BASE_REF:refs/remotes/origin/$GITHUB_BASE_REF" + if ! compare_rev="$(git merge-base "origin/${GITHUB_BASE_REF}" HEAD)"; then + echo "Unable to find merge-base for origin/${GITHUB_BASE_REF}; falling back to full test suite" >&2 + : > /tmp/changed-files + exit 0 + fi +fi + +git diff --name-only "$compare_rev" "$GITHUB_SHA" > /tmp/changed-files diff --git a/rust/cloud-storage/tools/xtask/src/workflows/scripts/compute_doppler_bins.sh b/rust/cloud-storage/tools/xtask/src/workflows/scripts/compute_doppler_bins.sh new file mode 100644 index 00000000000..ab391f4fd46 --- /dev/null +++ b/rust/cloud-storage/tools/xtask/src/workflows/scripts/compute_doppler_bins.sh @@ -0,0 +1,10 @@ +set -euo pipefail + +doppler_config_bins="$(cargo run --manifest-path rust/cloud-storage/tools/xtask/Cargo.toml -- doppler-bins /tmp/changed-files)" +{ + echo 'doppler_config_bins<<__DOPPLER_CONFIG_BINS__' + if [ -n "$doppler_config_bins" ]; then + printf '%s\n' "$doppler_config_bins" + fi + echo '__DOPPLER_CONFIG_BINS__' +} >> "$GITHUB_OUTPUT" diff --git a/rust/cloud-storage/tools/xtask/src/workflows/scripts/compute_nextest_filter.sh b/rust/cloud-storage/tools/xtask/src/workflows/scripts/compute_nextest_filter.sh new file mode 100644 index 00000000000..e9f09e15521 --- /dev/null +++ b/rust/cloud-storage/tools/xtask/src/workflows/scripts/compute_nextest_filter.sh @@ -0,0 +1,17 @@ +set -euo pipefail + +# Root cargo/toolchain/CI changes can affect the whole workspace, so run all tests. +if grep -qE '^(rust/rust-toolchain\.toml|flake\.nix|flake\.lock|rust/cloud-storage/Cargo\.(toml|lock)|rust/cloud-storage/\.cargo/.*|\.github/actions/(setup-rust|setup-cachix|setup-sccache)/.*|\.github/workflows/code-check-cloud-storage\.yml)$' /tmp/changed-files; then + echo "Workspace-level change detected; running all tests" + echo "nextest_filter=" >> "$GITHUB_OUTPUT" + exit 0 +fi + +filterset="$(cargo run --manifest-path rust/cloud-storage/tools/xtask/Cargo.toml -- nextest-filter /tmp/changed-files)" + +if [ -z "$filterset" ]; then + echo "No package-specific Rust changes detected; running all tests" +else + echo "nextest filter: $filterset" +fi +echo "nextest_filter=$filterset" >> "$GITHUB_OUTPUT" diff --git a/rust/cloud-storage/tools/xtask/src/workflows/scripts/configure_postgres.sh b/rust/cloud-storage/tools/xtask/src/workflows/scripts/configure_postgres.sh new file mode 100644 index 00000000000..880e6b53ea2 --- /dev/null +++ b/rust/cloud-storage/tools/xtask/src/workflows/scripts/configure_postgres.sh @@ -0,0 +1,20 @@ +postgres_container="$(docker ps --format '{{.ID}} {{.Image}}' | awk '$2 == "pgvector/pgvector:pg16" { print $1; exit }')" +if [ -z "$postgres_container" ]; then + echo "pgvector/pgvector:pg16 service container not found" >&2 + docker ps + exit 1 +fi + +docker exec -i "$postgres_container" psql -U user -d macrodb <<'SQL' +ALTER SYSTEM SET fsync = off; +ALTER SYSTEM SET synchronous_commit = off; +ALTER SYSTEM SET full_page_writes = off; +ALTER SYSTEM SET max_wal_size = '4GB'; +ALTER SYSTEM SET checkpoint_timeout = '30min'; +ALTER SYSTEM SET max_locks_per_transaction = 8192; +SQL +docker restart "$postgres_container" +until docker exec "$postgres_container" pg_isready -U user -d macrodb; do + sleep 1 +done +docker exec "$postgres_container" psql -U user -d macrodb -c "SHOW max_locks_per_transaction;" diff --git a/rust/cloud-storage/tools/xtask/src/workflows/scripts/run_tests.sh b/rust/cloud-storage/tools/xtask/src/workflows/scripts/run_tests.sh new file mode 100644 index 00000000000..ec93f4fadd3 --- /dev/null +++ b/rust/cloud-storage/tools/xtask/src/workflows/scripts/run_tests.sh @@ -0,0 +1,7 @@ +cd rust/cloud-storage + +args=(--all-features --lib --bins --tests --test-threads "$NEXTEST_TEST_THREADS") +if [ -n "$NEXTEST_FILTER" ]; then + args+=(-E "$NEXTEST_FILTER") +fi +cargo nextest run "${args[@]}" diff --git a/rust/cloud-storage/tools/xtask/src/workflows/scripts/validate_doppler_configs.sh b/rust/cloud-storage/tools/xtask/src/workflows/scripts/validate_doppler_configs.sh new file mode 100644 index 00000000000..25fa695bbc2 --- /dev/null +++ b/rust/cloud-storage/tools/xtask/src/workflows/scripts/validate_doppler_configs.sh @@ -0,0 +1,38 @@ +set -euo pipefail + +if [ -z "${DOPPLER_TOKEN:-}" ]; then + echo "DOPPLER_TOKEN secret is required to validate Doppler configs" >&2 + exit 1 +fi + +if [ -z "${RUSTC_WRAPPER:-}" ]; then + echo "RUSTC_WRAPPER is required so Doppler config binaries build through sccache" >&2 + exit 1 +fi + +bins=() +cargo_args=(build --locked --all-features) +while IFS= read -r bin; do + if [ -z "$bin" ]; then + continue + fi + + bins+=("$bin") + cargo_args+=(--bin "$bin") +done <<< "$DOPPLER_CONFIG_BINS" + +if [ "${#bins[@]}" -eq 0 ]; then + echo "No Doppler config binaries to validate" + exit 0 +fi + +echo "Building affected Doppler config binaries with RUSTC_WRAPPER=$RUSTC_WRAPPER" +printf ' %s\n' "${bins[@]}" + +( + cd rust/cloud-storage + cargo "${cargo_args[@]}" + for bin in "${bins[@]}"; do + "./target/debug/$bin" + done +) diff --git a/rust/cloud-storage/tools/xtask/src/workflows/steps.rs b/rust/cloud-storage/tools/xtask/src/workflows/steps.rs index f9b7a752885..43013461982 100644 --- a/rust/cloud-storage/tools/xtask/src/workflows/steps.rs +++ b/rust/cloud-storage/tools/xtask/src/workflows/steps.rs @@ -13,7 +13,7 @@ use crate::workflows::vars; /// building values imperatively. pub trait FluentBuilder: Sized { /// Apply `f` to `self`. - fn map(self, f: impl FnOnce(Self) -> Self) -> Self { + fn map(self, f: impl FnOnce(Self) -> U) -> U { f(self) } /// Apply `f` only when `cond` holds. From b40155b3cbbeb31561d00411fa39b24d389827cf Mon Sep 17 00:00:00 2001 From: synoet Date: Tue, 23 Jun 2026 15:37:16 -0400 Subject: [PATCH 3/8] chore(ci): TEMP run check/test on draft PR [revert before merge] Drops the `&& github.event.pull_request.draft == false` gate on the check/test jobs so they run on this draft PR while validating the new workflow end-to-end. Restore before merging. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/code-check-cloud-storage.yml | 4 ++-- rust/cloud-storage/tools/xtask/src/workflows/steps.rs | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/code-check-cloud-storage.yml b/.github/workflows/code-check-cloud-storage.yml index 02f325c015c..894cd18421f 100644 --- a/.github/workflows/code-check-cloud-storage.yml +++ b/.github/workflows/code-check-cloud-storage.yml @@ -109,7 +109,7 @@ jobs: check: needs: - path-check - if: needs.path-check.outputs.should_run == 'true' && github.event.pull_request.draft == false + if: needs.path-check.outputs.should_run == 'true' runs-on: namespace-profile-linux-mid env: RUSTFLAGS: '-Dwarnings -Dclippy::disallowed_methods -C link-arg=-fuse-ld=mold' @@ -190,7 +190,7 @@ jobs: test: needs: - path-check - if: needs.path-check.outputs.should_run == 'true' && github.event.pull_request.draft == false + if: needs.path-check.outputs.should_run == 'true' runs-on: namespace-profile-linux-mid env: NEXTEST_FILTER: ${{ needs.path-check.outputs.nextest_filter }} diff --git a/rust/cloud-storage/tools/xtask/src/workflows/steps.rs b/rust/cloud-storage/tools/xtask/src/workflows/steps.rs index 43013461982..87d6cc4602a 100644 --- a/rust/cloud-storage/tools/xtask/src/workflows/steps.rs +++ b/rust/cloud-storage/tools/xtask/src/workflows/steps.rs @@ -110,7 +110,10 @@ pub fn show_sccache_stats() -> Step { pub fn gated_job() -> Job { Job::default() .needs(vec!["path-check".to_string()]) + // TEMP: the draft gate (`&& github.event.pull_request.draft == false`) is + // dropped so check/test run on this draft PR while we validate the + // workflow. Restore it before merging. .cond(Expression::new( - "needs.path-check.outputs.should_run == 'true' && github.event.pull_request.draft == false", - )) + "needs.path-check.outputs.should_run == 'true'", + )) } From 428d90072f9bf30790cba52b8e8e8408ca7a06d0 Mon Sep 17 00:00:00 2001 From: synoet Date: Tue, 23 Jun 2026 15:44:19 -0400 Subject: [PATCH 4/8] fix(ci): install Nix before entering the dev shell on Namespace runners Namespace profiles don't ship Nix (unlike the old self-hosted linux-extra-beefy), so setup-cachix's nix calls failed with 'cannot connect to socket'. Run ./.github/actions/setup-nix before setup-cachix in the check/test jobs, mirroring the deploy workflow. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/code-check-cloud-storage.yml | 4 ++++ .../xtask/src/workflows/code_check_cloud_storage.rs | 2 ++ rust/cloud-storage/tools/xtask/src/workflows/steps.rs | 10 +++++++++- 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/workflows/code-check-cloud-storage.yml b/.github/workflows/code-check-cloud-storage.yml index 894cd18421f..661f614a596 100644 --- a/.github/workflows/code-check-cloud-storage.yml +++ b/.github/workflows/code-check-cloud-storage.yml @@ -127,6 +127,8 @@ jobs: /sccache /nix continue-on-error: true + - name: Setup Nix + uses: './.github/actions/setup-nix' - name: Setup Nix dev shell uses: './.github/actions/setup-cachix' with: @@ -224,6 +226,8 @@ jobs: /sccache /nix continue-on-error: true + - name: Setup Nix + uses: './.github/actions/setup-nix' - name: Setup Nix dev shell uses: './.github/actions/setup-cachix' with: diff --git a/rust/cloud-storage/tools/xtask/src/workflows/code_check_cloud_storage.rs b/rust/cloud-storage/tools/xtask/src/workflows/code_check_cloud_storage.rs index dd68d2f444b..aeafd5068db 100644 --- a/rust/cloud-storage/tools/xtask/src/workflows/code_check_cloud_storage.rs +++ b/rust/cloud-storage/tools/xtask/src/workflows/code_check_cloud_storage.rs @@ -66,6 +66,7 @@ fn check() -> Job { .add_env(("RUSTDOCFLAGS", "-Dwarnings")) .add_step(steps::checkout(false)) .add_step(steps::mount_cache_volume()) + .add_step(steps::setup_nix()) .add_step(steps::setup_dev_shell()) .add_step(steps::pin_sccache_dir()) .add_step(validate_doppler_configs()) @@ -88,6 +89,7 @@ fn test() -> Job { .add_service("redis", redis_service()) .add_step(steps::checkout(false)) .add_step(steps::mount_cache_volume()) + .add_step(steps::setup_nix()) .add_step(steps::setup_dev_shell()) .add_step(steps::pin_sccache_dir()) .add_step(configure_postgres()) diff --git a/rust/cloud-storage/tools/xtask/src/workflows/steps.rs b/rust/cloud-storage/tools/xtask/src/workflows/steps.rs index 87d6cc4602a..941ce750c04 100644 --- a/rust/cloud-storage/tools/xtask/src/workflows/steps.rs +++ b/rust/cloud-storage/tools/xtask/src/workflows/steps.rs @@ -60,9 +60,17 @@ pub fn setup_rust_light() -> Step { .add_with(("rust-cache", "false")) } +/// Install + initialise Nix on the runner. Namespace profiles don't ship Nix, +/// so this must run before [`setup_dev_shell`] (which shells out to `nix`). The +/// `/nix` cache volume mounted by [`mount_cache_volume`] keeps the store warm, +/// so it re-inits the daemon rather than doing a full install. +pub fn setup_nix() -> Step { + uses_local("Setup Nix", "./.github/actions/setup-nix") +} + /// Enter the repo's Nix dev shell (toolchain, mold, just, the sccache binary, /// and `RUSTC_WRAPPER=sccache`). We pass NO `sccache-bucket`, so sccache runs in -/// local-disk mode instead of talking to S3. +/// local-disk mode instead of talking to S3. Requires [`setup_nix`] first. pub fn setup_dev_shell() -> Step { uses_local("Setup Nix dev shell", "./.github/actions/setup-cachix") .add_with(("cachix-auth-token", vars::CACHIX_AUTH_TOKEN)) From 5474c08d004565cd5d89b00cdbb277060ec07c65 Mon Sep 17 00:00:00 2001 From: synoet Date: Tue, 23 Jun 2026 15:59:25 -0400 Subject: [PATCH 5/8] chore(ci): name generated workflows snake_case after their source module Match Zed's one-module-per-workflow naming: the registry carries a single `slug` that names both the source module (.rs) and the output file (.yml), replacing the separate explicit filename. - code-check-cloud-storage.yml -> code_check_cloud_storage.yml - check-generated-workflows.yml -> check_generated.yml The required status check is the job name "Cloud Storage Status Check" (unchanged), so the rename doesn't affect the branch ruleset. Co-Authored-By: Claude Opus 4.8 (1M context) --- .gitattributes | 4 +-- ...ated-workflows.yml => check_generated.yml} | 0 ...orage.yml => code_check_cloud_storage.yml} | 4 +-- .../src/workflows/code_check_cloud_storage.rs | 2 +- .../tools/xtask/src/workflows/mod.rs | 34 +++++++++++-------- .../scripts/compute_nextest_filter.sh | 2 +- 6 files changed, 25 insertions(+), 21 deletions(-) rename .github/workflows/{check-generated-workflows.yml => check_generated.yml} (100%) rename .github/workflows/{code-check-cloud-storage.yml => code_check_cloud_storage.yml} (98%) diff --git a/.gitattributes b/.gitattributes index 7624912aec3..d9751bb11d5 100644 --- a/.gitattributes +++ b/.gitattributes @@ -3,5 +3,5 @@ # Workflows generated by `cargo x workflows` (xtask). Collapse them in PR diffs # and exclude from language stats; edit the Rust source, not the YAML. -.github/workflows/code-check-cloud-storage.yml linguist-generated=true -.github/workflows/check-generated-workflows.yml linguist-generated=true +.github/workflows/code_check_cloud_storage.yml linguist-generated=true +.github/workflows/check_generated.yml linguist-generated=true diff --git a/.github/workflows/check-generated-workflows.yml b/.github/workflows/check_generated.yml similarity index 100% rename from .github/workflows/check-generated-workflows.yml rename to .github/workflows/check_generated.yml diff --git a/.github/workflows/code-check-cloud-storage.yml b/.github/workflows/code_check_cloud_storage.yml similarity index 98% rename from .github/workflows/code-check-cloud-storage.yml rename to .github/workflows/code_check_cloud_storage.yml index 661f614a596..e6df0f16880 100644 --- a/.github/workflows/code-check-cloud-storage.yml +++ b/.github/workflows/code_check_cloud_storage.yml @@ -50,7 +50,7 @@ jobs: - '.github/services-config.json' - '.github/scripts/build-cloud-storage-lambdas.sh' - '.github/scripts/build-cloud-storage-lambdas-nix.sh' - - .github/workflows/code-check-cloud-storage.yml + - .github/workflows/code_check_cloud_storage.yml - name: compute changed files if: steps.filter.outputs.should_run == 'true' run: | @@ -91,7 +91,7 @@ jobs: set -euo pipefail # Root cargo/toolchain/CI changes can affect the whole workspace, so run all tests. - if grep -qE '^(rust/rust-toolchain\.toml|flake\.nix|flake\.lock|rust/cloud-storage/Cargo\.(toml|lock)|rust/cloud-storage/\.cargo/.*|\.github/actions/(setup-rust|setup-cachix|setup-sccache)/.*|\.github/workflows/code-check-cloud-storage\.yml)$' /tmp/changed-files; then + if grep -qE '^(rust/rust-toolchain\.toml|flake\.nix|flake\.lock|rust/cloud-storage/Cargo\.(toml|lock)|rust/cloud-storage/\.cargo/.*|\.github/actions/(setup-rust|setup-cachix|setup-sccache)/.*|\.github/workflows/code_check_cloud_storage\.yml)$' /tmp/changed-files; then echo "Workspace-level change detected; running all tests" echo "nextest_filter=" >> "$GITHUB_OUTPUT" exit 0 diff --git a/rust/cloud-storage/tools/xtask/src/workflows/code_check_cloud_storage.rs b/rust/cloud-storage/tools/xtask/src/workflows/code_check_cloud_storage.rs index aeafd5068db..deb0e510298 100644 --- a/rust/cloud-storage/tools/xtask/src/workflows/code_check_cloud_storage.rs +++ b/rust/cloud-storage/tools/xtask/src/workflows/code_check_cloud_storage.rs @@ -133,7 +133,7 @@ fn paths_filter() -> Step { - '.github/services-config.json' - '.github/scripts/build-cloud-storage-lambdas.sh' - '.github/scripts/build-cloud-storage-lambdas-nix.sh' - - .github/workflows/code-check-cloud-storage.yml + - .github/workflows/code_check_cloud_storage.yml "#}, )) } diff --git a/rust/cloud-storage/tools/xtask/src/workflows/mod.rs b/rust/cloud-storage/tools/xtask/src/workflows/mod.rs index 5fa07c8be05..5b058d45bdf 100644 --- a/rust/cloud-storage/tools/xtask/src/workflows/mod.rs +++ b/rust/cloud-storage/tools/xtask/src/workflows/mod.rs @@ -21,27 +21,31 @@ use std::path::{Path, PathBuf}; use anyhow::{bail, Context, Result}; use gh_workflow::Workflow; -/// A generated workflow. +/// A generated workflow. `slug` is the snake_case identifier shared by the +/// source module (`.rs`) and the output file (`.yml`) — one source +/// of truth, mirroring Zed's one-module-per-workflow naming. (Required status +/// checks key on a job *name*, not the filename, so renaming files is safe.) struct WorkflowFile { - /// File written under `.github/workflows/`. Kept stable so branch - /// protection's required status checks keep matching. - filename: &'static str, - /// Source module (for the "do not edit" header), e.g. `code_check_cloud_storage`. - source: &'static str, + slug: &'static str, /// Builds the workflow definition. build: fn() -> Workflow, } +impl WorkflowFile { + /// Output file under `.github/workflows/`. + fn file_name(&self) -> String { + format!("{}.yml", self.slug) + } +} + /// Every workflow we generate. Add new workflows here. const WORKFLOWS: &[WorkflowFile] = &[ WorkflowFile { - filename: "code-check-cloud-storage.yml", - source: "code_check_cloud_storage", + slug: "code_check_cloud_storage", build: code_check_cloud_storage::code_check_cloud_storage, }, WorkflowFile { - filename: "check-generated-workflows.yml", - source: "check_generated", + slug: "check_generated", build: check_generated::check_generated_workflows, }, ]; @@ -50,7 +54,7 @@ const WORKFLOWS: &[WorkflowFile] = &[ pub fn generate() -> Result<()> { let dir = workflows_dir()?; for workflow in WORKFLOWS { - let path = dir.join(workflow.filename); + let path = dir.join(workflow.file_name()); fs::write(&path, render(workflow)?) .with_context(|| format!("writing {}", path.display()))?; println!("generated {}", path.display()); @@ -63,11 +67,11 @@ pub fn check() -> Result<()> { let dir = workflows_dir()?; let mut stale = Vec::new(); for workflow in WORKFLOWS { - let path = dir.join(workflow.filename); + let path = dir.join(workflow.file_name()); let on_disk = fs::read_to_string(&path).with_context(|| format!("reading {}", path.display()))?; if on_disk != render(workflow)? { - stale.push(workflow.filename); + stale.push(workflow.file_name()); } } if !stale.is_empty() { @@ -84,8 +88,8 @@ pub fn check() -> Result<()> { fn render(workflow: &WorkflowFile) -> Result { let yaml = (workflow.build)() .to_string() - .map_err(|e| anyhow::anyhow!("serializing {}: {e:?}", workflow.filename))?; - Ok(format!("{}{yaml}", disclaimer(workflow.source))) + .map_err(|e| anyhow::anyhow!("serializing {}: {e:?}", workflow.file_name()))?; + Ok(format!("{}{yaml}", disclaimer(workflow.slug))) } /// The header every generated file starts with. diff --git a/rust/cloud-storage/tools/xtask/src/workflows/scripts/compute_nextest_filter.sh b/rust/cloud-storage/tools/xtask/src/workflows/scripts/compute_nextest_filter.sh index e9f09e15521..c46fba55ab6 100644 --- a/rust/cloud-storage/tools/xtask/src/workflows/scripts/compute_nextest_filter.sh +++ b/rust/cloud-storage/tools/xtask/src/workflows/scripts/compute_nextest_filter.sh @@ -1,7 +1,7 @@ set -euo pipefail # Root cargo/toolchain/CI changes can affect the whole workspace, so run all tests. -if grep -qE '^(rust/rust-toolchain\.toml|flake\.nix|flake\.lock|rust/cloud-storage/Cargo\.(toml|lock)|rust/cloud-storage/\.cargo/.*|\.github/actions/(setup-rust|setup-cachix|setup-sccache)/.*|\.github/workflows/code-check-cloud-storage\.yml)$' /tmp/changed-files; then +if grep -qE '^(rust/rust-toolchain\.toml|flake\.nix|flake\.lock|rust/cloud-storage/Cargo\.(toml|lock)|rust/cloud-storage/\.cargo/.*|\.github/actions/(setup-rust|setup-cachix|setup-sccache)/.*|\.github/workflows/code_check_cloud_storage\.yml)$' /tmp/changed-files; then echo "Workspace-level change detected; running all tests" echo "nextest_filter=" >> "$GITHUB_OUTPUT" exit 0 From c62859e8ed510f13d0207b9b773684f397df54db Mon Sep 17 00:00:00 2001 From: synoet Date: Tue, 23 Jun 2026 16:20:11 -0400 Subject: [PATCH 6/8] chore(ci): restore the draft gate on check/test Reverts the temporary b40155b3c. The check/test jobs skip on draft PRs again (`&& github.event.pull_request.draft == false`), which is the intended pre-merge behavior. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/code_check_cloud_storage.yml | 4 ++-- rust/cloud-storage/tools/xtask/src/workflows/steps.rs | 7 ++----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/code_check_cloud_storage.yml b/.github/workflows/code_check_cloud_storage.yml index e6df0f16880..b5eab26a8c8 100644 --- a/.github/workflows/code_check_cloud_storage.yml +++ b/.github/workflows/code_check_cloud_storage.yml @@ -109,7 +109,7 @@ jobs: check: needs: - path-check - if: needs.path-check.outputs.should_run == 'true' + if: needs.path-check.outputs.should_run == 'true' && github.event.pull_request.draft == false runs-on: namespace-profile-linux-mid env: RUSTFLAGS: '-Dwarnings -Dclippy::disallowed_methods -C link-arg=-fuse-ld=mold' @@ -192,7 +192,7 @@ jobs: test: needs: - path-check - if: needs.path-check.outputs.should_run == 'true' + if: needs.path-check.outputs.should_run == 'true' && github.event.pull_request.draft == false runs-on: namespace-profile-linux-mid env: NEXTEST_FILTER: ${{ needs.path-check.outputs.nextest_filter }} diff --git a/rust/cloud-storage/tools/xtask/src/workflows/steps.rs b/rust/cloud-storage/tools/xtask/src/workflows/steps.rs index 941ce750c04..c73f88ec526 100644 --- a/rust/cloud-storage/tools/xtask/src/workflows/steps.rs +++ b/rust/cloud-storage/tools/xtask/src/workflows/steps.rs @@ -118,10 +118,7 @@ pub fn show_sccache_stats() -> Step { pub fn gated_job() -> Job { Job::default() .needs(vec!["path-check".to_string()]) - // TEMP: the draft gate (`&& github.event.pull_request.draft == false`) is - // dropped so check/test run on this draft PR while we validate the - // workflow. Restore it before merging. .cond(Expression::new( - "needs.path-check.outputs.should_run == 'true'", - )) + "needs.path-check.outputs.should_run == 'true' && github.event.pull_request.draft == false", + )) } From a333d35793958822cff7e57729fcd6bf96e4527d Mon Sep 17 00:00:00 2001 From: synoet Date: Tue, 23 Jun 2026 16:33:18 -0400 Subject: [PATCH 7/8] fix(ci): pin dorny/paths-filter to a commit SHA Tag refs like @v3 can be repointed to malicious commits; pin to the SHA v3 currently resolves to (v3.0.3), matching the repo's other SHA-pinned actions. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/code_check_cloud_storage.yml | 2 +- .../tools/xtask/src/workflows/code_check_cloud_storage.rs | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/code_check_cloud_storage.yml b/.github/workflows/code_check_cloud_storage.yml index b5eab26a8c8..5c25d307d18 100644 --- a/.github/workflows/code_check_cloud_storage.yml +++ b/.github/workflows/code_check_cloud_storage.yml @@ -37,7 +37,7 @@ jobs: rust-cache: 'false' - id: filter name: Filter changed paths - uses: dorny/paths-filter@v3 + uses: dorny/paths-filter@d1c1ffe0248fe513906c8e24db8ea791d46f8590 with: filters: | should_run: diff --git a/rust/cloud-storage/tools/xtask/src/workflows/code_check_cloud_storage.rs b/rust/cloud-storage/tools/xtask/src/workflows/code_check_cloud_storage.rs index deb0e510298..38e99fd0cf7 100644 --- a/rust/cloud-storage/tools/xtask/src/workflows/code_check_cloud_storage.rs +++ b/rust/cloud-storage/tools/xtask/src/workflows/code_check_cloud_storage.rs @@ -118,7 +118,11 @@ fn status_check() -> Job { /// Detect whether cloud-storage-relevant paths changed. fn paths_filter() -> Step { Step::new("Filter changed paths") - .uses("dorny", "paths-filter", "v3") + .uses( + "dorny", + "paths-filter", + "d1c1ffe0248fe513906c8e24db8ea791d46f8590", + ) // v3.0.3 .id("filter") .add_with(( "filters", From 4e79cca9543971a03b6f4cabe7e9a4456edea57a Mon Sep 17 00:00:00 2001 From: synoet Date: Tue, 23 Jun 2026 16:55:39 -0400 Subject: [PATCH 8/8] fix(ci): run check/test on a dedicated linux-rust-ci profile check/test shared namespace-profile-linux-mid with the deploy workflow, whose cache-volume churn evicted the CI sccache caches between runs (the larger test cache lost first -> cold). Move them to a dedicated CI profile with its own cache volume. path-check/status-check stay on linux-small (they don't use the sccache volume). Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/actionlint.yaml | 1 + .github/workflows/code_check_cloud_storage.yml | 4 ++-- .../xtask/src/workflows/code_check_cloud_storage.rs | 4 ++-- rust/cloud-storage/tools/xtask/src/workflows/runners.rs | 9 +++++---- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index 2d66f2a34da..feca2a453bd 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -4,3 +4,4 @@ self-hosted-runner: - linux-latest-middy - namespace-profile-linux-mid - namespace-profile-linux-small + - namespace-profile-linux-rust-ci diff --git a/.github/workflows/code_check_cloud_storage.yml b/.github/workflows/code_check_cloud_storage.yml index 5c25d307d18..5e0aa965dc6 100644 --- a/.github/workflows/code_check_cloud_storage.yml +++ b/.github/workflows/code_check_cloud_storage.yml @@ -110,7 +110,7 @@ jobs: needs: - path-check if: needs.path-check.outputs.should_run == 'true' && github.event.pull_request.draft == false - runs-on: namespace-profile-linux-mid + runs-on: namespace-profile-linux-rust-ci env: RUSTFLAGS: '-Dwarnings -Dclippy::disallowed_methods -C link-arg=-fuse-ld=mold' RUSTDOCFLAGS: '-Dwarnings' @@ -193,7 +193,7 @@ jobs: needs: - path-check if: needs.path-check.outputs.should_run == 'true' && github.event.pull_request.draft == false - runs-on: namespace-profile-linux-mid + runs-on: namespace-profile-linux-rust-ci env: NEXTEST_FILTER: ${{ needs.path-check.outputs.nextest_filter }} NEXTEST_TEST_THREADS: '32' diff --git a/rust/cloud-storage/tools/xtask/src/workflows/code_check_cloud_storage.rs b/rust/cloud-storage/tools/xtask/src/workflows/code_check_cloud_storage.rs index 38e99fd0cf7..739e7f4e0d7 100644 --- a/rust/cloud-storage/tools/xtask/src/workflows/code_check_cloud_storage.rs +++ b/rust/cloud-storage/tools/xtask/src/workflows/code_check_cloud_storage.rs @@ -58,7 +58,7 @@ fn path_check() -> Job { /// fmt + clippy (and Doppler-config validation). fn check() -> Job { steps::gated_job() - .runs_on(runners::Runner::LinuxMid.to_string()) + .runs_on(runners::Runner::LinuxRustCi.to_string()) .add_env(( "RUSTFLAGS", "-Dwarnings -Dclippy::disallowed_methods -C link-arg=-fuse-ld=mold", @@ -78,7 +78,7 @@ fn check() -> Job { /// cargo nextest against postgres + redis service containers. fn test() -> Job { steps::gated_job() - .runs_on(runners::Runner::LinuxMid.to_string()) + .runs_on(runners::Runner::LinuxRustCi.to_string()) .add_env(( "NEXTEST_FILTER", "${{ needs.path-check.outputs.nextest_filter }}", diff --git a/rust/cloud-storage/tools/xtask/src/workflows/runners.rs b/rust/cloud-storage/tools/xtask/src/workflows/runners.rs index dcc0f0625e1..419012fc2bf 100644 --- a/rust/cloud-storage/tools/xtask/src/workflows/runners.rs +++ b/rust/cloud-storage/tools/xtask/src/workflows/runners.rs @@ -14,16 +14,17 @@ use std::fmt; pub enum Runner { /// Small profile for light jobs (path filtering, status aggregation). LinuxSmall, - /// Mid profile for the heavy compile + test jobs. Has a cache volume - /// configured, which is what makes the persisted sccache cache possible. - LinuxMid, + /// Dedicated CI profile for the heavy compile + test jobs. Has its own + /// cache volume, isolated from the deploy profiles so deploy's churn can't + /// evict the CI sccache/cargo caches. + LinuxRustCi, } impl fmt::Display for Runner { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str(match self { Runner::LinuxSmall => "namespace-profile-linux-small", - Runner::LinuxMid => "namespace-profile-linux-mid", + Runner::LinuxRustCi => "namespace-profile-linux-rust-ci", }) } }