ci: better DX

.github/workflows/publish-preview.yml

@@ -35,6 +35,7 @@ on:
       - 'Cargo.toml'
       - 'Cargo.lock'
       - 'rust-toolchain.toml'
+      - 'justfile'
       - 'src/**'
       - 'packages/forge/**'
       - '!packages/forge/test/**'
@@ -47,6 +48,7 @@ on:
       - 'Cargo.toml'
       - 'Cargo.lock'
       - 'rust-toolchain.toml'
+      - 'justfile'
       - 'src/**'
       - 'packages/forge/**'
       - '!packages/forge/test/**'
@@ -70,10 +72,8 @@ jobs:
   # platforms.mjs; scripts/archive.mjs refuses to pack a package that misses
   # a platform.
   #
-  # Each job uploads its binary as forge-<platform>. upload-artifact keeps the
-  # path from the first wildcard on, so target/*/release/forge keeps
-  # <target>/release/, where scripts/archive.mjs looks. A job builds only its
-  # own target, so the wildcard matches one file.
+  # Each job builds with `just build <target>`, the recipe local builds use,
+  # and uploads target/binaries as forge-<platform>.
   build:
     name: Build (${{ matrix.platform }})
     runs-on: ${{ matrix.runner }}
@@ -102,19 +102,16 @@ jobs:
             runner: ubuntu-24.04
             exe: forge
             runs: true
-          # Link the C runtime in, or the binary needs VCRUNTIME140.dll from
-          # the Visual C++ Redistributable, which many machines don't have.
+          # `just build` links the C runtime into both Windows binaries.
           - platform: win32-arm64
             target: aarch64-pc-windows-msvc
             runner: windows-latest
             exe: forge.exe
-            rustflags: -C target-feature=+crt-static
             runs: false
           - platform: win32-x64
             target: x86_64-pc-windows-msvc
             runner: windows-latest
             exe: forge.exe
-            rustflags: -C target-feature=+crt-static
             runs: true
     defaults:
       run:
@@ -133,6 +130,7 @@ jobs:
         uses: moonrepo/setup-rust@abb2d32350334249b178c401e5ec5836e0cd88d3 # v1.3.0
         with:
           targets: ${{ matrix.target }}
+          bins: just@1.58.0
           cache-target: ${{ matrix.target }}/release
           cache-base: main
         env:
@@ -145,27 +143,28 @@ jobs:
           sudo apt-get update
           sudo apt-get install -y --no-install-recommends musl-tools
 
-      # `wasm` gives `--target wasm://`, which runs plugins.
+      # The same recipe builds locally; it puts the binary in
+      # target/binaries/<target>/release/.
       - name: Build
         env:
-          RUSTFLAGS: ${{ matrix.rustflags }}
           TARGET: ${{ matrix.target }}
-        run: cargo build --release --locked --features wasm --target "$TARGET"
+        run: just build "$TARGET"
 
       # A binary the runner can execute gets a quick run, which catches a
       # broken link before the package is published.
       - name: Smoke test
         if: matrix.runs
         env:
-          BIN: target/${{ matrix.target }}/release/${{ matrix.exe }}
+          BIN: target/binaries/${{ matrix.target }}/release/${{ matrix.exe }}
         run: '"$BIN" --version'
 
+      # Uploading target/binaries keeps <target>/release/ inside the artifact.
       # Artifacts drop the executable bit, which scripts/archive.mjs sets
       # again when it packs.
       - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
         with:
           name: forge-${{ matrix.platform }}
-          path: target/*/release/${{ matrix.exe }}
+          path: target/binaries
           if-no-files-found: error
           retention-days: 7
 
@@ -182,19 +181,20 @@ jobs:
 
       - uses: ./.github/actions/setup-workspace
 
-      # Merges the build jobs' artifacts into binaries/<target>/release/,
-      # where scripts/archive.mjs picks them up.
+      - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4.0.0
+        with:
+          just-version: 1.58.0
+
+      # Merges the build jobs' artifacts into target/binaries/<target>/release/,
+      # where `just pack` picks them up, as it does locally.
       - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
         with:
           pattern: forge-*
           merge-multiple: true
-          path: ${{ runner.temp }}/binaries
+          path: target/binaries
 
       - name: Pack
-        working-directory: packages/forge
-        env:
-          BINARIES: ${{ runner.temp }}/binaries
-        run: node scripts/archive.mjs --artifacts "$BINARIES"
+        run: just pack
 
       - name: Publish
         id: publish

docker/certs/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

docker/release.Dockerfile

@@ -0,0 +1,34 @@
+# Images for building forge's release binaries locally, for the targets a
+# machine can't build natively. `just build-docker <target>` builds and runs
+# them; see the justfile. CI builds on native runners instead.
+#
+# The justfile sets RUST_VERSION to the channel in rust-toolchain.toml, so the
+# containers build with the same Rust as CI. The default, the latest 1.x,
+# only keeps the image names valid for a plain `docker build`.
+#
+# The images also trust any .crt files in docker/certs/, which is gitignored.
+
+ARG RUST_VERSION=1
+
+# Linux: musl, so the binary is static and runs on any distro. Alpine is
+# musl-based, so its C compiler targets musl too, which the cc crate uses
+# for wasmtime's C code. Run with --platform linux/arm64 or linux/amd64 to
+# build for that architecture.
+FROM rust:${RUST_VERSION}-alpine AS linux
+COPY certs/ /tmp/certs/
+RUN find /tmp/certs -name '*.crt' -exec cp {} /usr/local/share/ca-certificates/ \; \
+    && update-ca-certificates \
+    && apk add --no-cache musl-dev
+
+# Windows: MSVC, cross-compiled with cargo-xwin, which downloads Microsoft's
+# CRT and Windows SDK on first use. clang, lld and llvm are its toolchain.
+FROM rust:${RUST_VERSION}-slim-bookworm AS windows
+ARG CARGO_XWIN_VERSION=0.23.1
+COPY certs/ /tmp/certs/
+RUN find /tmp/certs -name '*.crt' -exec cp {} /usr/local/share/ca-certificates/ \; \
+    && update-ca-certificates
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends clang lld llvm \
+    && rm -rf /var/lib/apt/lists/*
+RUN cargo install --locked cargo-xwin --version "${CARGO_XWIN_VERSION}" --root /usr/local \
+    && rustup target add aarch64-pc-windows-msvc x86_64-pc-windows-msvc

justfile

@@ -1,5 +1,7 @@
 # Project commands. Run `just --list` to see them all. CI runs these same
 # recipes, so a recipe that passes here passes there.
+_default:
+  just --list -u
 
 alias f := fmt
 alias l := lint
@@ -24,3 +26,87 @@ fmt-check:
 lint:
     cargo clippy --locked --all-targets --all-features -- -D warnings
     cargo clippy --locked --all-targets --no-default-features -- -D warnings
+
+# Release builds
+#
+# Every way of getting release binaries puts them in target/binaries/<target>/
+# release/, and `just pack` packs whatever is there. CI builds each target
+# with `just build` on a native runner. Locally, `just build-all` builds them
+# all: macOS targets natively, which needs a Mac, and the rest in Docker.
+# `just fetch` downloads a CI run's binaries instead.
+
+# The Rust targets forge ships, as in packages/forge/scripts/platforms.mjs.
+mac_targets := "aarch64-apple-darwin x86_64-apple-darwin"
+docker_targets := "aarch64-unknown-linux-musl x86_64-unknown-linux-musl aarch64-pc-windows-msvc x86_64-pc-windows-msvc"
+binaries := "target/binaries"
+
+# Link the C runtime into the Windows binaries, or they need VCRUNTIME140.dll
+# from the Visual C++ Redistributable, which many machines don't have. Both
+# `just build` (CI's Windows runners) and `just build-docker` (cargo-xwin, which
+# keeps these flags) build with them. Other targets ignore them.
+export CARGO_TARGET_AARCH64_PC_WINDOWS_MSVC_RUSTFLAGS := "-C target-feature=+crt-static"
+export CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_RUSTFLAGS := "-C target-feature=+crt-static"
+
+# Build the release binary for <target> on this machine
+build target:
+    rustup target add {{ target }}
+    cargo build --release --locked --features wasm --target {{ target }}
+    mkdir -p {{ binaries }}/{{ target }}/release
+    cp target/{{ target }}/release/{{ if target =~ 'windows' { "forge.exe" } else { "forge" } }} {{ binaries }}/{{ target }}/release/
+
+# Build the release binary for a Linux or Windows <target> in Docker
+build-docker target:
+    #!/usr/bin/env bash
+    set -euo pipefail
+    target='{{ target }}'
+    rust=$(sed -n 's/^channel = "\(.*\)"$/\1/p' rust-toolchain.toml)
+    case "$target" in
+      aarch64-unknown-linux-musl) stage=linux platform=linux/arm64 exe=forge cargo=(cargo build) ;;
+      x86_64-unknown-linux-musl) stage=linux platform=linux/amd64 exe=forge cargo=(cargo build) ;;
+      *-pc-windows-msvc) stage=windows platform= exe=forge.exe cargo=(cargo xwin build) ;;
+      *) echo "error: build-docker builds Linux and Windows targets; build $target with: just build $target" >&2; exit 1 ;;
+    esac
+    image="forge-release-$stage:$rust${platform:+-${platform#linux/}}"
+    echo "==> $target in $image"
+    docker build --quiet ${platform:+--platform "$platform"} --target "$stage" --build-arg "RUST_VERSION=$rust" \
+      --tag "$image" --file docker/release.Dockerfile docker >/dev/null
+    # Runs as you, so everything it writes under target/ is yours. Each target
+    # builds into a directory of its own: the host's target/ holds build
+    # scripts compiled for the host, which can't run in the container.
+    # RUSTUP_TOOLCHAIN pins the image's Rust, so rustup doesn't try to install
+    # the components rust-toolchain.toml lists.
+    docker run --rm ${platform:+--platform "$platform"} --user "$(id -u):$(id -g)" \
+      --volume "$PWD:/work" --workdir /work \
+      --env HOME=/tmp \
+      --env CARGO_HOME=/work/target/docker/cargo-home \
+      --env CARGO_TARGET_DIR="/work/target/docker/$target" \
+      --env XWIN_CACHE_DIR=/work/target/docker/xwin \
+      --env RUSTUP_TOOLCHAIN="$rust" \
+      --env CARGO_TARGET_AARCH64_PC_WINDOWS_MSVC_RUSTFLAGS \
+      --env CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_RUSTFLAGS \
+      "$image" "${cargo[@]}" --release --locked --features wasm --target "$target"
+    mkdir -p "{{ binaries }}/$target/release"
+    cp "target/docker/$target/$target/release/$exe" "{{ binaries }}/$target/release/"
+
+# Build every release binary this machine can: macOS ones need a Mac
+build-all:
+    #!/usr/bin/env bash
+    set -euo pipefail
+    if [ "$(uname -s)" = Darwin ]; then
+      for target in {{ mac_targets }}; do just build "$target"; done
+    else
+      echo "warning: skipping {{ mac_targets }}, which build only on a Mac; just fetch gets them from CI" >&2
+    fi
+    for target in {{ docker_targets }}; do just build-docker "$target"; done
+
+# Download the release binaries a publish-preview run built, instead of building them
+fetch run:
+    rm -rf target/fetch {{ binaries }}
+    gh run download {{ run }} --pattern 'forge-*' --dir target/fetch
+    mkdir -p {{ binaries }}
+    cp -R target/fetch/forge-*/. {{ binaries }}/
+    rm -rf target/fetch
+
+# Pack the binaries in target/binaries into packages/forge/dist/; takes archive.mjs options
+pack *args:
+    node packages/forge/scripts/archive.mjs --artifacts {{ binaries }} {{ args }}

packages/forge/README.md

@@ -276,18 +276,20 @@ or list it in `package.json`. A path or URL to the tarball both work:
 
 ## Releasing
 
-From `packages/forge` in the forge repository, on a Mac with Docker running:
+From the root of the forge repository, with [just](https://just.systems) installed:
 
 ```sh
-pnpm install                # the dev dependencies, which build the plugin library
-node scripts/version.mjs    # after bumping the version in Cargo.toml
-node scripts/wit.mjs        # after changing the WIT in src/plugin/wasm.rs
-node scripts/compile.mjs    # builds all six binaries into target/
-node scripts/archive.mjs    # packs them into dist/cloudflare-forge-<version>.tgz
-node scripts/publish.mjs    # publishes that tarball; takes npm publish options
+pnpm install                            # the dev dependencies, which build the plugin library
+node packages/forge/scripts/version.mjs # after bumping the version in Cargo.toml
+node packages/forge/scripts/wit.mjs     # after changing the WIT in src/plugin/wasm.rs
+just build-all                          # builds all six binaries into target/binaries/ (or: just fetch <run-id>)
+just pack                               # packs them into packages/forge/dist/cloudflare-forge-<version>.tgz
 ```
 
-The tarball that `archive.mjs` makes is the one that gets published, so a
+To vendor the package in another repository, copy the tarball from
+`packages/forge/dist/` and depend on it with a `file:` specifier.
+
+The tarball that `just pack` makes is the one that gets published, so a
 tarball you hand around and the package on the registry are the same.
 `npm pack` and `npm publish` refuse to run in this directory, which has no
 binaries of its own.
@@ -315,35 +317,38 @@ runs its tests in Node: the tests of `forge::ts` in
 
 ### Building the binaries
 
-`scripts/compile.mjs` builds each platform with the tool that can:
+Every way of getting the binaries puts them in `target/binaries/<target>/release/`,
+and `just pack` packs whatever is there:
 
-| Platform | Tool                                                           | Needs                                   |
-| -------- | -------------------------------------------------------------- | --------------------------------------- |
-| macOS    | `cargo build`                                                  | a Mac, Xcode's command line tools       |
-| Linux    | [`cross build`](https://github.com/cross-rs/cross)             | Docker                                  |
-| Windows  | [`cargo xwin build`](https://github.com/rust-cross/cargo-xwin) | nothing; it downloads the MSVC CRT once |
+| Recipe                       | Builds                                      | Needs                    |
+| ---------------------------- | ------------------------------------------- | ------------------------ |
+| `just build <target>`        | one target, with this machine's `cargo`     | a host that can build it |
+| `just build-docker <target>` | a Linux (musl) or Windows (MSVC) target     | Docker                   |
+| `just build-all`             | all six: macOS natively, the rest in Docker | a Mac and Docker         |
+| `just fetch <run-id>`        | nothing; downloads a publish-preview run's  | the GitHub CLI, `gh`     |
 
-One-time setup:
+macOS binaries build only on a Mac. On Linux, `just build-all` builds the
+other four, and `just fetch` gets all six from CI. The Docker images are in
+`docker/release.Dockerfile` and build with the Rust in `rust-toolchain.toml`;
+on an Apple silicon Mac, the x64 Linux build runs emulated, so it's slow.
 
-```sh
-rustup target add x86_64-apple-darwin aarch64-apple-darwin \
-  aarch64-pc-windows-msvc x86_64-pc-windows-msvc
-cargo install cross --git https://github.com/cross-rs/cross
-cargo install --locked cargo-xwin
-```
+The images also trust any `.crt` files in `docker/certs/`, which is gitignored.
+
+CI builds each target with `just build` on a runner that builds it natively,
+in the publish-preview workflow, then packs them with `just pack`.
 
-cross can't build macOS or Windows MSVC binaries, because it has no prebuilt
-images for them. The Windows binaries link the C runtime statically.
+The Windows binaries link the C runtime statically (see the `justfile`), so
+they don't need the Visual C++ Redistributable.
 
 ### Trying it out locally
 
-`compile.mjs` and `archive.mjs` take `--platform <os>-<cpu>`, repeated, to
-build or pack only some platforms. For a package that runs on this machine:
+`just pack` takes `--platform <os>-<cpu>`, repeated, to pack only some
+platforms. For a package that runs on this machine:
 
 ```sh
-cargo build --release --features wasm
-node scripts/archive.mjs --platform darwin-arm64
+just build aarch64-apple-darwin
+just pack --platform darwin-arm64
 ```
 
 A tarball that leaves platforms out lists the ones it has in its name, as in
-`cloudflare-forge-0.1.0-darwin-arm64.tgz`, and `publish.mjs` won't publish it.
+`cloudflare-forge-0.1.0-darwin-arm64.tgz`, so it can't pass for the full package.

packages/forge/package.json

@@ -41,14 +41,12 @@
     "access": "public"
   },
   "scripts": {
-    "compile": "node scripts/compile.mjs",
     "wit": "node scripts/wit.mjs",
     "build": "node scripts/build.mjs",
     "test": "node scripts/build.mjs && tsc -p test && node --test \"test/*.test.js\"",
     "archive": "node scripts/archive.mjs",
-    "release": "node scripts/publish.mjs",
     "sync-version": "node scripts/version.mjs",
-    "prepack": "node -e \"console.error('error: this directory has no binaries; pack with npm run archive, publish with npm run release'); process.exit(1)\""
+    "prepack": "node -e \"console.error('error: this directory has no binaries; pack the package with just pack from the repository root'); process.exit(1)\""
   },
   "devDependencies": {
     "@bytecodealliance/jco": "1.37.0",

packages/forge/scripts/archive.mjs

@@ -1,20 +1,21 @@
 // Packs @cloudflare/forge, with the forge binary for every platform, into a
-// tarball. That tarball is the package: scripts/publish.mjs publishes it, and
-// another package can install it straight from the file:
+// tarball. That tarball is the package: the publish-preview workflow publishes
+// it, and another package can install it straight from the file:
 //
 //     node scripts/archive.mjs [--artifacts <dir>] [--platform <os-cpu>]... [--out <dir>]
 //
 //     npm install /path/to/cloudflare-forge-0.1.0.tgz
 //     # or, in package.json: "@cloudflare/forge": "file:../cloudflare-forge-0.1.0.tgz"
 //
 // The binaries come from <dir>, the crate's target directory unless you say
-// otherwise (see findBinary in platforms.mjs); scripts/compile.mjs builds
-// them all. Every platform needs one.
+// otherwise (see findBinary in platforms.mjs). Every platform needs one.
+// `just pack` runs this script on target/binaries, which `just build-all`
+// fills, or `just fetch` from a CI run; see the justfile.
 //
 // --platform packs only the platforms it names, such as darwin-arm64, which
 // is enough to try the package out locally; `cargo build --release
 // --features wasm` builds the binary for this machine. The tarball's name
-// then lists them, and scripts/publish.mjs won't publish it.
+// then lists them, so it can't pass for the full package.
 //
 // It builds @cloudflare/forge/plugin into the package, as scripts/build.mjs
 // does, which needs this directory's dev dependencies: pnpm install here.
@@ -77,7 +78,8 @@ function archive(artifacts, only, out) {
   if (missing.length > 0) {
     throw new Error(
       `no forge binary in ${artifacts} for ${missing.join(', ')}; ` +
-        `run node scripts/compile.mjs${only ? '' : ', or pack only some platforms with --platform'}`,
+        `build them with just build-all, or fetch a CI run's with just fetch <run-id>` +
+        (only ? '' : ', or pack only some platforms with --platform'),
     );
   }