feat: the baseline measured on the demo at fe2d9ed — sizes from builds whose build step runs offline, and cold starts

The release APK grows by 578 KB and the simulator app by 3.7 MB; the cold start grows by 76 ms on the emulator and by nothing measurable on the simulator.
The with variant keeps its build step, HOTCODEPUSH_OFFLINE set by the size script and the workflows, so no build asks the API.
The first-paint marker is logged at the error level, which an iOS release build keeps, and the first launch after the install is not counted.
The React Native demo passes the lifecycle contract on both platforms in the device test.

Closes #1

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

Author: robingenz

.github/workflows/baseline.yml

@@ -9,6 +9,8 @@ permissions:
   contents: read
 
 env:
+  # The demo's build step asks no API: these builds are never shipped.
+  HOTCODEPUSH_OFFLINE: '1'
   JAVA_VERSION: 21
 
 jobs:
@@ -41,6 +43,6 @@ jobs:
         working-directory: package
         run: npm ci && npm run build && npm pack --pack-destination "$RUNNER_TEMP"
       - name: Prepare the demo with the package
-        run: node package/benchmarks/prepare-demo.mjs demo "$RUNNER_TEMP/with" with "$(ls "$RUNNER_TEMP"/hotcodepush-react-native-code-push-*.tgz)" --no-binary-create
+        run: node package/benchmarks/prepare-demo.mjs demo "$RUNNER_TEMP/with" with "$(ls "$RUNNER_TEMP"/hotcodepush-react-native-code-push-*.tgz)"
       - name: Measure the sizes and compare them with the baseline
         run: node package/benchmarks/measure-size.mjs "$RUNNER_TEMP/with" > "$RUNNER_TEMP/sizes.json" && node package/benchmarks/guard-size.mjs package/benchmarks/baseline.json "$RUNNER_TEMP/sizes.json" >> "$GITHUB_STEP_SUMMARY"

.github/workflows/ci.yml

@@ -10,10 +10,12 @@ permissions:
   contents: read
 
 env:
+  # The demo's build step asks no API: these builds are never shipped.
+  HOTCODEPUSH_OFFLINE: '1'
   JAVA_VERSION: 21
 
 # The native code compiles inside an app: the platform jobs install the package from this commit into the demo at
-# the commit the baseline names and build it. Debug builds, which bundle no JavaScript, with the build step unwired.
+# the commit the baseline names and build it. Debug builds, which bundle no JavaScript and so create no binary.
 jobs:
   javascript:
     name: Lint, typecheck, test and build the JavaScript
@@ -56,7 +58,7 @@ jobs:
         working-directory: package
         run: npm ci && npm run build && npm pack --pack-destination "$RUNNER_TEMP"
       - name: Prepare the demo with the package
-        run: node package/benchmarks/prepare-demo.mjs demo "$RUNNER_TEMP/with" with "$(ls "$RUNNER_TEMP"/hotcodepush-react-native-code-push-*.tgz)" --no-binary-create
+        run: node package/benchmarks/prepare-demo.mjs demo "$RUNNER_TEMP/with" with "$(ls "$RUNNER_TEMP"/hotcodepush-react-native-code-push-*.tgz)"
       - run: ./gradlew assembleDebug --console=plain
         working-directory: ${{ runner.temp }}/with/android
   ios:
@@ -87,7 +89,7 @@ jobs:
         working-directory: package
         run: npm ci && npm run build && npm pack --pack-destination "$RUNNER_TEMP"
       - name: Prepare the demo with the package
-        run: node package/benchmarks/prepare-demo.mjs demo "$RUNNER_TEMP/with" with "$(ls "$RUNNER_TEMP"/hotcodepush-react-native-code-push-*.tgz)" --no-binary-create
+        run: node package/benchmarks/prepare-demo.mjs demo "$RUNNER_TEMP/with" with "$(ls "$RUNNER_TEMP"/hotcodepush-react-native-code-push-*.tgz)"
       - run: xcodebuild -workspace HotCodePushDemo.xcworkspace -scheme HotCodePushDemo -configuration Debug -destination 'generic/platform=iOS Simulator' CODE_SIGNING_ALLOWED=NO -quiet build
         working-directory: ${{ runner.temp }}/with/ios
   preview:

benchmarks/README.md

@@ -11,30 +11,31 @@ What the package adds to an app, measured on the demo app and guarded from then
 | Cold start added, Android  | from the activity's start (`ActivityTaskManager: START` in logcat) to the first paint, on the Pixel_9_Pro emulator; the median of five cold launches with the package minus the median without |
 | Cold start added, iOS      | from the launch call to the first paint, on an iPhone simulator; the same medians                                                                                                              |
 
-The first paint is the `[baseline] first paint` line both variants log on their first animation frame, read from the JavaScript console: logcat on Android, the unified log on iOS.
+The first paint is the `[baseline] first paint` line both variants log on their first animation frame, at the error level, since an iOS release build drops what the console logs below it; it is read from logcat on Android and from the unified log on iOS.
+The first launch after the install is not counted, since the system verifies and compiles the app on it.
 Both numbers come from release builds, since a debug build asks Metro for its JavaScript and the SDK stays off in it.
 The "without" variant is the demo with the package and every line `init` wired removed — the build step in Xcode and Gradle, the bundle the two apps ask the SDK for, the pinned pod — and the screen swapped for the same screen with nothing behind it.
 
-The sizes are measured with the build step unwired in the "with" variant too, `--no-binary-create`: the step needs a login, the resource file it writes weighs a few kilobytes against a binary of tens of megabytes, and the guard in CI holds no credential.
-The cold starts are measured with the build step in place, against a stack the CLI is logged in to, since an app without its resource file starts no SDK.
+The sizes are measured on builds whose build step runs offline, `HOTCODEPUSH_OFFLINE=1`, which the size script sets: `binary create` writes the resource file, creates no binary and asks no account, so the guard in CI holds no credential.
+The cold starts are measured on builds made against a stack the CLI is logged in to, since a build without a channel checks for nothing at its start.
 
 ## Where the bytes sit
 
 On the current baseline the Android release APK grows by about 578 KB.
-About 310 KB of it is `libappmodules.so`, the Turbo Module's generated C++ once for each of the four ABIs the demo ships, of which an app bundle delivers one; about 270 KB is `classes2.dex`, the package's code and the shared core's, with OkHttp 5 and Okio in place of the OkHttp 4 React Native brings; the rest is the package's JavaScript in the bundle.
-The simulator app grows by about 3.6 MB, nearly all of it in the `HotCodePushDemo` binary: the module and the core are linked statically, and a simulator build is a two-slice fat binary, so a device build carries about half of that; the rest is the core's privacy-manifest bundle and about 12 KB of JavaScript.
+About 310 KB of it is `libappmodules.so`, the Turbo Module's generated C++ once for each of the four ABIs the demo ships, of which an app bundle delivers one; about 270 KB is `classes2.dex`, the package's code and the shared core's, with OkHttp 5 and Okio in place of the OkHttp 4 React Native brings; the rest is the package's JavaScript in the bundle and the resource file.
+The simulator app grows by about 3.7 MB, nearly all of it in the `HotCodePushDemo` binary: the module and the core are linked statically, and a simulator build is a two-slice fat binary, so a device build carries about half of that; the rest is the core's privacy-manifest bundle, about 12 KB of JavaScript and the resource file.
 
 ## Running it
 
 ```sh
 npm run build && npm pack --pack-destination /tmp
-node benchmarks/prepare-demo.mjs ../react-native-code-push-demo /tmp/baseline/with with /tmp/hotcodepush-react-native-code-push-0.0.0.tgz --no-binary-create
+node benchmarks/prepare-demo.mjs ../react-native-code-push-demo /tmp/baseline/with with /tmp/hotcodepush-react-native-code-push-0.0.0.tgz
 node benchmarks/prepare-demo.mjs ../react-native-code-push-demo /tmp/baseline/without without
 node benchmarks/measure-size.mjs /tmp/baseline/with
 node benchmarks/measure-size.mjs /tmp/baseline/without
 ```
 
-For the cold starts, prepare the "with" variant again without `--no-binary-create`, with `HOTCODEPUSH_TOKEN` set and `hotcodepush.json` naming an app of that account, then:
+For the cold starts, set `HOTCODEPUSH_TOKEN` and let the "with" variant's `hotcodepush.json` name an app of that account, then:
 
 ```sh
 node benchmarks/measure-cold-start.mjs /tmp/baseline/with --android emulator-5554 --ios <udid>

benchmarks/baseline.json

@@ -1,23 +1,43 @@
 {
-  "measuredAt": "2026-10-03",
+  "measuredAt": "2026-10-04",
   "package": {
-    "commit": "86af7a05ebf8bfac615f21af8e683f3b8f340342"
+    "commit": "432fc160bb6bc957db884b8009ba46114baac6d0"
   },
   "demo": {
-    "commit": "f81de286cfe01f65d67f020ff408078b686cba86"
+    "commit": "fe2d9ed064e2bd912f0c1a77731353944aab6c50"
   },
   "android": {
     "releaseApkBytes": {
-      "with": 47543773,
+      "with": 47544434,
       "without": 46966076,
-      "added": 577697
+      "added": 578358
+    },
+    "coldStartMs": {
+      "with": 663,
+      "without": 587,
+      "added": 76,
+      "device": "Pixel_9_Pro emulator, headless with the host GPU, Android 17 (API 37), 16 KB pages",
+      "runsMs": {
+        "with": [643, 690, 663, 644, 678],
+        "without": [595, 589, 587, 567, 532]
+      }
     }
   },
   "ios": {
     "simulatorAppBytes": {
-      "with": 40884443,
-      "without": 37248622,
-      "added": 3635821
+      "with": 40904188,
+      "without": 37248618,
+      "added": 3655570
+    },
+    "coldStartMs": {
+      "with": 463,
+      "without": 463,
+      "added": 0,
+      "device": "iPhone 17 Pro simulator, iOS 26.5",
+      "runsMs": {
+        "with": [463, 467, 463, 473, 454],
+        "without": [463, 456, 447, 483, 473]
+      }
     }
   }
 }

benchmarks/measure-cold-start.mjs

@@ -2,7 +2,8 @@
 // Cold-start milliseconds of a prepared demo variant: from the launch to the first paint — the
 // `[baseline] first paint` console line the demo logs on its first animation frame — median of N
 // cold launches on the Pixel_9_Pro emulator and the iPhone simulator, both on release builds, since
-// a debug build asks Metro for its JavaScript. Prints JSON with every run.
+// a debug build asks Metro for its JavaScript. The first launch after the install is not counted:
+// the system verifies and compiles the app on it. Prints JSON with every counted run.
 import { execFileSync, spawn } from 'node:child_process';
 import { join } from 'node:path';
 
@@ -41,7 +42,7 @@ async function measureAndroid() {
   });
   adb('install', '-r', apk);
   const samples = [];
-  for (let run = 0; run < runs; run++) {
+  for (let run = 0; run <= runs; run++) {
     adb('shell', 'am', 'force-stop', bundleId);
     await waitFor(() => adb('shell', 'pidof', bundleId).trim() === '');
     adb('logcat', '-c');
@@ -61,7 +62,7 @@ async function measureAndroid() {
     const painted = epochOf(lines.find(line => line.includes(marker)));
     samples.push(Math.round(painted - started));
   }
-  return summarize(samples);
+  return summarize(samples.slice(1));
 }
 
 async function measureIos() {
@@ -93,7 +94,7 @@ async function measureIos() {
     join(derivedData, `Build/Products/Release-iphonesimulator/${appName}.app`),
   );
   const samples = [];
-  for (let run = 0; run < runs; run++) {
+  for (let run = 0; run <= runs; run++) {
     simctl('terminate', iosUdid, bundleId);
     await sleep(1500);
     // React Native writes the JavaScript console to the unified log, which the stream is listening to before the launch.
@@ -110,7 +111,7 @@ async function measureIos() {
     ]);
     let paintedAt = null;
     log.stdout.on('data', chunk => {
-      if (paintedAt === null && String(chunk).includes(marker)) {
+      if (paintedAt === null && hasAppLine(String(chunk))) {
         paintedAt = Date.now();
       }
     });
@@ -122,7 +123,14 @@ async function measureIos() {
     samples.push(paintedAt - launchedAt);
   }
   simctl('terminate', iosUdid, bundleId);
-  return summarize(samples);
+  return summarize(samples.slice(1));
+}
+
+// The stream opens with a line that repeats its own predicate, the marker in it; the app's lines name the process.
+function hasAppLine(output) {
+  return output
+    .split('\n')
+    .some(line => line.includes(marker) && line.includes(`${appName}[`));
 }
 
 function summarize(samples) {

benchmarks/measure-size.mjs

@@ -11,9 +11,12 @@ if (!target) {
   process.exit(2);
 }
 
+// The build step runs offline: it writes the resource file, creates no binary and asks no account.
+const env = { ...process.env, HOTCODEPUSH_OFFLINE: '1' };
 const sizes = {};
 execFileSync('./gradlew', ['assembleRelease', '--console=plain', '-q'], {
   cwd: join(target, 'android'),
+  env,
   stdio: ['ignore', 'ignore', 'inherit'],
 });
 sizes.android = {
@@ -42,7 +45,7 @@ if (!flags.includes('--android-only')) {
       '-quiet',
       'build',
     ],
-    { stdio: ['ignore', 'ignore', 'inherit'] },
+    { env, stdio: ['ignore', 'ignore', 'inherit'] },
   );
   sizes.ios = {
     simulatorAppBytes: directorySize(

benchmarks/prepare-demo.mjs

@@ -2,24 +2,19 @@
 // Copies the demo app into a scratch directory as one of the two variants the baseline compares:
 // `with` installs the package tarball, `without` removes the package and every line `init` wired
 // and swaps the screen for the same screen with nothing behind it. Both log the first paint.
-// `--no-binary-create` unwires the build step of the `with` variant, so its release build needs no login:
-// the resource file it would write weighs nothing against the binary, and the sizes are measured without it.
 import { execFileSync } from 'node:child_process';
 import { cpSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
 import { join } from 'node:path';
 
-const [source, target, variant, ...rest] = process.argv.slice(2);
-const tarball = rest.find(argument => !argument.startsWith('--'));
-const isBinaryCreateUnwired =
-  variant === 'without' || rest.includes('--no-binary-create');
+const [source, target, variant, tarball] = process.argv.slice(2);
 if (
   !source ||
   !target ||
   !['with', 'without'].includes(variant) ||
   (variant === 'with' && !tarball)
 ) {
   console.error(
-    'usage: prepare-demo.mjs <demo> <target> with <package.tgz> [--no-binary-create] | without',
+    'usage: prepare-demo.mjs <demo> <target> with <package.tgz> | without',
   );
   process.exit(2);
 }
@@ -39,17 +34,9 @@ cpSync(source, target, {
   filter: path => !excluded.has(path.split('/').pop()),
 });
 
+// At the error level: an iOS release build drops what the console logs below it.
 const firstPaintMarker =
-  "requestAnimationFrame(() => console.log('[baseline] first paint'));";
-if (isBinaryCreateUnwired) {
-  editFile('android/app/build.gradle', text =>
-    removeLines(text, 'hotcodepush.gradle'),
-  );
-  editFile(
-    'ios/HotCodePushDemo.xcodeproj/project.pbxproj',
-    removeBinaryCreatePhase,
-  );
-}
+  "requestAnimationFrame(() => console.error('[baseline] first paint'));";
 if (variant === 'with') {
   editFile('App.tsx', text =>
     text.replace(
@@ -60,6 +47,13 @@ if (variant === 'with') {
   run('npm', ['install', tarball, '--no-audit', '--no-fund']);
 } else {
   run('npm', ['uninstall', PACKAGE_NAME, '--no-audit', '--no-fund']);
+  editFile('android/app/build.gradle', text =>
+    removeLines(text, 'hotcodepush.gradle'),
+  );
+  editFile(
+    'ios/HotCodePushDemo.xcodeproj/project.pbxproj',
+    removeBinaryCreatePhase,
+  );
   editFile('ios/HotCodePushDemo/AppDelegate.swift', text =>
     removeLines(text, 'import HotcodepushReactNativeCodePush').replace(
       'HotCodePush.bundleURL()',