deploy-native.yml

# Reusable native deploy: cross-build the binary for inputs.arch (arm64 or amd64),
# then ship it over SSH and (re)start the systemd services. Called by
# deploy-candidate.yml (rc-* tags -> staging), which supplies the environment,
# runner labels and architecture, and on a manual run pins an explicit ref so a
# rollback can name an earlier tag. See deploy/native/README.md.
#
# Split into two jobs on purpose:
#
#   build  runs on inputs.build_runs_on — normally a GitHub-hosted runner whose
#          architecture matches inputs.arch, so the compile is native rather than
#          emulated. It needs Docker (build.sh compiles in a container) and does
#          the expensive Rust build. Nothing here touches the target host, so it
#          needs no network access to it.
#   ship   runs wherever the target host is reachable. It only needs ssh + tar, so a
#          self-hosted runner on a private network does NOT need Docker or Rust.
#
# That split is what makes a private-IP target workable: the heavy build stays on
# hosted infrastructure, and only the ~20MB binary crosses into the private network.
name: Deploy (native)

on:
  workflow_call:
    outputs:
      sha:
        # The commit the binary was actually built from. A caller must use this
        # rather than github.sha to describe what was deployed: on a
        # workflow_dispatch rollback github.sha is the default branch, while this
        # is the ref the run was asked to deploy.
        description: Commit the deployed binary was built from.
        value: ${{ jobs.build.outputs.sha }}
    inputs:
      environment:
        description: GitHub Environment for the ship job (carries the approval gate).
        required: true
        type: string
      ship_runs_on:
        description: >-
          JSON runner spec for the ship job — a bare name for an ARC runner scale
          set ('"dind-small"'), or a label list for classic runners
          ('["self-hosted","linux"]'). Scale sets carry no default labels, so a list
          will not select one. Must be somewhere the target host is reachable.
          Parsed with fromJSON, so it must be valid JSON.
        required: true
        type: string
      arch:
        description: >-
          Target CPU architecture: arm64 (Graviton) or amd64 (x86_64). Must match
          the deploy target's `uname -m`; deploy.sh aborts on a mismatch.
        required: true
        type: string
      build_runs_on:
        description: >-
          JSON runner spec for the build job. Build natively for the target arch
          where possible — emulating the other arch under QEMU is several times
          slower. Parsed with fromJSON.
        required: true
        type: string
      ref:
        description: Git ref to build and deploy. Defaults to the triggering ref.
        required: false
        type: string
        default: ''
    secrets:
      DEPLOY_SSH_KEY:
        description: Private SSH key for the sudo-capable host user.
        required: true
      DEPLOY_HOST:
        description: >-
          user@host for the target's PRIVATE address, e.g. ec2-user@<private-ip>.
          Must NOT be the public FQDN from DEPLOY_DOMAIN: that resolves to the
          public-facing interface, so the deploy leaves over the internet rather
          than the VPN, and that interface's security group does not permit 22.
          Staging was set this way and failed with a connection timeout.
        required: true
      DEPLOY_DOMAIN:
        description: Public FQDN served over HTTPS.
        required: true
      DEPLOY_ACME_EMAIL:
        description: Email for Let's Encrypt / ACME.
        required: true
      DEPLOY_KNOWN_HOSTS:
        description: Optional pinned host keys (ssh-keyscan output). TOFU if unset.
        required: false
      OPENAI_APPS_CHALLENGE_TOKEN:
        description: >-
          OpenAI Apps domain-verification token (ChatGPT-directory submission),
          served verbatim at /.well-known/openai-apps-challenge. It is
          world-readable at that endpoint by design; it is handled as a secret
          for consistency with the other deploy inputs. Unset -> the endpoint
          404s. Not host-specific, so both callers pass the same
          repository-level secret.
        required: false

jobs:
  build:
    # Pick a runner matching inputs.arch so build.sh compiles natively rather than
    # under QEMU. The bullseye build container in build.sh pins glibc 2.31 for
    # Amazon Linux 2023 compatibility regardless of architecture.
    runs-on: ${{ fromJSON(inputs.build_runs_on) }}
    steps:
      # Every third-party action below is pinned to a commit SHA against tag
      # movement (supply-chain hardening), and the trailing comment records the
      # readable version.
      - name: Checkout
        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        with:
          ref: ${{ inputs.ref }}

      # Gate the deploy on the status dashboard's unit tests. They are fast and
      # network-free, and the dashboard is shipped (and run as a systemd service)
      # by deploy.sh, so a regression here should stop the rollout.
      - name: Set up Node.js
        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
        with:
          node-version: 20

      - name: Test status dashboard
        run: npm test --prefix monitoring/mcp-status

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0

      # Resolve the commit from the checkout, not from github.sha: on a
      # workflow_dispatch rollback the event SHA is the default branch, while the
      # checkout is the ref we were asked to deploy. The build bakes this in and
      # the ship job asserts the host reports it back, so both must agree.
      - name: Resolve commit
        id: built
        run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"

      - name: Cross-build ${{ inputs.arch }} binary
        env:
          # Baked into the binary and surfaced at GET /version, so a live host can
          # always be traced back to an exact commit.
          GIT_SHA: ${{ steps.built.outputs.sha }}
          ARCH: ${{ inputs.arch }}
        run: deploy/native/build.sh

      - name: Upload binary
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        with:
          name: imcp2-binary
          path: build-out/imcp2
          retention-days: 7
          if-no-files-found: error

    outputs:
      sha: ${{ steps.built.outputs.sha }}

  ship:
    needs: build
    runs-on: ${{ fromJSON(inputs.ship_runs_on) }}
    # Approval gate: this job only runs once the named environment's protection
    # rules are satisfied. Configure required reviewers under
    # Settings -> Environments -> <name> for the gate to take effect.
    environment: ${{ inputs.environment }}
    steps:
      # Check out the exact commit the build job resolved, not inputs.ref. The
      # binary arrives as an artifact from that commit, but deploy.sh ships
      # monitoring/ straight from this checkout — so if inputs.ref is a branch
      # (or a retagged tag) that moved while the build ran or while this job waited
      # on the environment's approval gate, the host would get assets from one
      # commit and a binary from another. The /version assertion below would not
      # catch it either: it compares against the build's SHA, which the binary
      # reports correctly no matter how stale the assets are.
      - name: Checkout
        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        with:
          ref: ${{ needs.build.outputs.sha }}

      - name: Download binary
        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
        with:
          name: imcp2-binary
          path: build-out

      # The artifact format does not preserve the executable bit, and deploy.sh
      # refuses to run unless build-out/imcp2 is executable.
      - name: Restore executable bit
        run: chmod +x build-out/imcp2

      - name: Configure SSH
        env:
          DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }}
          DEPLOY_KNOWN_HOSTS: ${{ secrets.DEPLOY_KNOWN_HOSTS }}
          HOST: ${{ secrets.DEPLOY_HOST }}
        run: |
          install -d -m 700 ~/.ssh
          printf '%s\n' "$DEPLOY_SSH_KEY" > ~/.ssh/id_ed25519
          chmod 600 ~/.ssh/id_ed25519
          if [ -n "$DEPLOY_KNOWN_HOSTS" ]; then
            printf '%s\n' "$DEPLOY_KNOWN_HOSTS" > ~/.ssh/known_hosts
          else
            # Trust-on-first-use: no pinned key, so we accept whatever the host
            # presents now. This is susceptible to MITM / DNS hijack — set the
            # DEPLOY_KNOWN_HOSTS secret (output of `ssh-keyscan <host>`) to pin it.
            echo "::warning::DEPLOY_KNOWN_HOSTS not set; trusting host key on first use (ssh-keyscan). Set DEPLOY_KNOWN_HOSTS to pin it."
            ssh-keyscan -T 20 -H "${HOST##*@}" > ~/.ssh/known_hosts 2>/dev/null || true
            if [ ! -s ~/.ssh/known_hosts ]; then
              echo "::error::ssh-keyscan captured no host keys for ${HOST##*@}; aborting." >&2
              exit 1
            fi
          fi
          chmod 644 ~/.ssh/known_hosts

      # Record the address the HOST sees this runner coming from. That is the only
      # thing a 22/tcp security-group rule can match on, and it is not knowable from
      # the runner: `curl ifconfig.me` reports public internet egress, while this path
      # goes over the VPN to an RFC1918 address, so the two are unrelated.
      #
      # Worth having on every run because narrowing 22/tcp away from 0.0.0.0/0 needs
      # it, and a single observation is not enough to write a rule from — the ship
      # pools are ephemeral pods spread over more than one cluster, so consecutive
      # deploys can present different sources. Assuming this address was inside
      # 10.0.0.0/8, which the host already permits, is what broke two deploys.
      #
      # Never fails the job: this is diagnostics, and if SSH is broken the Deploy step
      # below reports it properly.
      - name: Report source address as seen by the host
        env:
          HOST: ${{ secrets.DEPLOY_HOST }}
        run: |
          # Keep ssh's stderr: "Connection timed out", "Permission denied" and a host
          # key mismatch are entirely different diagnoses, and surfacing which one it
          # was is most of the value of a step that exists to explain SSH problems.
          # Collapsed to one line because ::warning:: annotations are line-oriented.
          err="${RUNNER_TEMP:-/tmp}/ssh-source.err"
          if src="$(ssh -o BatchMode=yes -o ConnectTimeout=20 "$HOST" \
                     'echo "${SSH_CLIENT%% *}"' 2>"$err")" && [ -n "$src" ]; then
            echo "ship runner reaches the host from: $src"
          else
            echo "::warning::could not read SSH_CLIENT from the host: $(tr '\n' ' ' < "$err" 2>/dev/null)"
          fi

      - name: Deploy
        env:
          HOST: ${{ secrets.DEPLOY_HOST }}
          DOMAIN: ${{ secrets.DEPLOY_DOMAIN }}
          ACME_EMAIL: ${{ secrets.DEPLOY_ACME_EMAIL }}
          # Expose the beta II instance at /mcp-beta on staging only; production
          # serves /mcp (production II) alone. Empty string reads as off.
          MCP_SERVE_BETA: ${{ inputs.environment == 'staging' && '1' || '' }}
          # Publish the status dashboard at /status/ on staging only. Production
          # keeps running the imcp-status service (its Statuspage pusher feeds
          # status.internetcomputer.org) but does not expose it at its origin.
          SERVE_STATUS: ${{ inputs.environment == 'staging' && '1' || '' }}
          # OpenAI Apps domain verification (ChatGPT-directory submission).
          # Unset -> endpoint 404s. An optional secret that neither caller
          # passes arrives here as the empty string.
          OPENAI_APPS_CHALLENGE_TOKEN: ${{ secrets.OPENAI_APPS_CHALLENGE_TOKEN }}
          # Client ID Metadata Documents: on, unless this GitHub Environment
          # defines the OAUTH_CIMD_ENABLED variable as `0` (this job runs in that
          # environment, so its variables resolve here). Unset arrives as the
          # empty string, which the server reads as on. A roll-out kill switch
          # only, to be removed.
          OAUTH_CIMD_ENABLED: ${{ vars.OAUTH_CIMD_ENABLED }}
        run: deploy/native/deploy.sh

      # Confirm the host is actually running the commit we just shipped, rather
      # than trusting that `systemctl restart` picked up the new binary. Read over
      # SSH from the host itself so this works even when $DOMAIN is not resolvable
      # from the runner (private-network deploys).
      - name: Verify deployed version
        env:
          HOST: ${{ secrets.DEPLOY_HOST }}
          EXPECTED: ${{ needs.build.outputs.sha }}
        run: |
          set -euo pipefail
          # --max-time bounds each attempt (per transfer, so --retry keeps its own
          # budget); without it a stalled response hangs the step until the job
          # timeout and the assertion below is never reached.
          ssh -o BatchMode=yes -o ConnectTimeout=20 "$HOST" \
            'curl -fsS --max-time 10 --retry 5 --retry-delay 2 --retry-connrefused http://127.0.0.1:8000/version' > version.json
          cat version.json
          got="$(sed -n 's/.*"commit"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' version.json)"
          if [ -z "$got" ] || [ "$got" = unknown ]; then
            echo "::warning::/version reported no usable commit; skipping assertion."
          elif [ "$got" != "$EXPECTED" ]; then
            echo "::error::host reports commit=$got, expected $EXPECTED" >&2
            exit 1
          else
            echo "verified: host is running $got"
          fi