diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index db3745bb..f87920b1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -87,30 +87,3 @@ jobs: env: CODSPEED_SKIP_UPLOAD: true CODSPEED_DEBUG: true - - walltime-macos-test: - runs-on: macos-latest - steps: - - uses: "actions/checkout@v4" - with: - fetch-depth: 0 - submodules: true - - uses: pnpm/action-setup@v2 - - uses: actions/setup-node@v3 - with: - cache: pnpm - node-version-file: .nvmrc - - name: Restore turbo cache - uses: ./.github/actions/turbo-cache - - run: pnpm install --frozen-lockfile --prefer-offline - - run: pnpm turbo run build - - - name: Run benchmarks - uses: CodSpeedHQ/action@main - env: - CODSPEED_SKIP_UPLOAD: "true" - # Samply fails to profile pnpm targets for now - CODSPEED_PROFILER_ENABLED: "false" - with: - run: pnpm turbo run bench --filter=@codspeed/vitest-plugin - mode: walltime diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index b28c2645..16666328 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -91,6 +91,40 @@ jobs: pnpm --workspace-concurrency 1 -r bench-tinybench pnpm --workspace-concurrency 1 -r bench-vitest + codspeed-walltime-macos: + runs-on: macos-26 + steps: + - uses: "actions/checkout@v4" + with: + fetch-depth: 0 + submodules: true + - uses: pnpm/action-setup@v2 + - uses: actions/setup-node@v3 + with: + cache: pnpm + node-version-file: .nvmrc + - name: Restore turbo cache + uses: ./.github/actions/turbo-cache + - run: pnpm install --frozen-lockfile --prefer-offline + - run: pnpm turbo run build + + - name: Diagnose resign+exec on this runner + run: | + sw_vers; uname -m; sysctl -n hw.memsize hw.ncpu + cp /bin/sh /tmp/sh_copy + /usr/bin/codesign -s - -f /tmp/sh_copy + lipo -info /tmp/sh_copy + /tmp/sh_copy -c 'echo RESIGNED_ARM64E_OK; uname -m' + echo "exit=$?" + + - name: Run macOS-only benchmarks + uses: CodSpeedHQ/action@main + with: + working-directory: packages/vitest-plugin + run: pnpm turbo run bench --env-mode=loose --filter=@codspeed/vitest-plugin + mode: walltime + runner-version: branch:sip-resign-exec-redirect + electron-e2e: name: Run electron inbox e2e runs-on: codspeed-macro diff --git a/packages/core/src/native_core/instruments/hooks b/packages/core/src/native_core/instruments/hooks index d83209f9..b9ddb5bc 160000 --- a/packages/core/src/native_core/instruments/hooks +++ b/packages/core/src/native_core/instruments/hooks @@ -1 +1 @@ -Subproject commit d83209f91683cf4c1677bdde28e0e43ca201f6ed +Subproject commit b9ddb5bc654b2e6fa13eb18efcd3a45e7ecda0bb diff --git a/packages/tinybench-plugin/src/walltime.ts b/packages/tinybench-plugin/src/walltime.ts index 95ccd27d..c709afa5 100644 --- a/packages/tinybench-plugin/src/walltime.ts +++ b/packages/tinybench-plugin/src/walltime.ts @@ -1,5 +1,8 @@ import { calculateQuantiles, + InstrumentHooks, + MARKER_TYPE_BENCHMARK_END, + MARKER_TYPE_BENCHMARK_START, mongoMeasurement, msToNs, msToS, @@ -64,17 +67,31 @@ class WalltimeBenchRunner extends BaseBenchRunner { private wrapTaskFunction(task: Task, isAsync: boolean): void { const { fn } = task as unknown as { fn: Fn }; + const pid = process.pid; + + // Emit per-round markers so the walltime instrument can bracket the actual + // benchmark body (excluding the harness) when attributing perf samples. + const finishRound = (roundStart: bigint): void => { + const roundEnd = InstrumentHooks.currentTimestamp(); + InstrumentHooks.addMarker(pid, MARKER_TYPE_BENCHMARK_START, roundStart); + InstrumentHooks.addMarker(pid, MARKER_TYPE_BENCHMARK_END, roundEnd); + }; + if (isAsync) { // eslint-disable-next-line no-inner-declarations async function __codspeed_root_frame__() { + const roundStart = InstrumentHooks.currentTimestamp(); await fn(); + finishRound(roundStart); } // eslint-disable-next-line @typescript-eslint/no-explicit-any (task as any).fn = __codspeed_root_frame__; } else { // eslint-disable-next-line no-inner-declarations function __codspeed_root_frame__() { + const roundStart = InstrumentHooks.currentTimestamp(); fn(); + finishRound(roundStart); } // eslint-disable-next-line @typescript-eslint/no-explicit-any (task as any).fn = __codspeed_root_frame__; diff --git a/packages/vitest-plugin/benches/macos.bench.ts b/packages/vitest-plugin/benches/macos.bench.ts new file mode 100644 index 00000000..b1a9c276 --- /dev/null +++ b/packages/vitest-plugin/benches/macos.bench.ts @@ -0,0 +1,16 @@ +import { bench, describe } from "vitest"; + +const isMacOS = process.platform === "darwin"; + +function fibo(n: number): number { + if (n < 2) return 1; + return fibo(n - 1) + fibo(n - 2); +} + +// macOS-only benchmark: skipped on every other platform, so it only runs on +// the `walltime-macos-test` CI job (see .github/workflows/ci.yml). +describe.skipIf(!isMacOS)("macos only", () => { + bench("fibo darwin", () => { + fibo(30); + }); +}); diff --git a/packages/vitest-plugin/src/walltime/index.ts b/packages/vitest-plugin/src/walltime/index.ts index 0d068941..ef5a6bf7 100644 --- a/packages/vitest-plugin/src/walltime/index.ts +++ b/packages/vitest-plugin/src/walltime/index.ts @@ -1,5 +1,7 @@ import { InstrumentHooks, + MARKER_TYPE_BENCHMARK_END, + MARKER_TYPE_BENCHMARK_START, setupCore, writeWalltimeResults, } from "@codspeed/core"; @@ -66,6 +68,7 @@ export class WalltimeRunner extends NodeBenchmarkRunner { this.isTinybenchHookedWithCodspeed = true; const originalRun = tinybench.Task.prototype.run; + const pid = process.pid; const getSuiteUri = (): string => { if (this.currentSuiteId === null) { @@ -78,10 +81,29 @@ export class WalltimeRunner extends NodeBenchmarkRunner { const { fn } = this as { fn: Fn }; const suiteUri = getSuiteUri(); - function __codspeed_root_frame__() { - return fn(); + const finishRound = (roundStart: bigint): void => { + const roundEnd = InstrumentHooks.currentTimestamp(); + InstrumentHooks.addMarker(pid, MARKER_TYPE_BENCHMARK_START, roundStart); + InstrumentHooks.addMarker(pid, MARKER_TYPE_BENCHMARK_END, roundEnd); + }; + + function __codspeed_root_frame__(): unknown { + const roundStart = InstrumentHooks.currentTimestamp(); + const result = fn(); + if ( + result !== null && + typeof result === "object" && + typeof (result as PromiseLike).then === "function" + ) { + return (result as PromiseLike).then((value) => { + finishRound(roundStart); + return value; + }); + } + finishRound(roundStart); + return result; } - (this as { fn: Fn }).fn = __codspeed_root_frame__; + (this as { fn: Fn }).fn = __codspeed_root_frame__ as Fn; InstrumentHooks.startBenchmark(); await originalRun.call(this); @@ -89,7 +111,7 @@ export class WalltimeRunner extends NodeBenchmarkRunner { // Look up the URI by task name const uri = `${suiteUri}::${this.name}`; - InstrumentHooks.setExecutedBenchmark(process.pid, uri); + InstrumentHooks.setExecutedBenchmark(pid, uri); return this; };