diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 3f266b1c..adcc23c7 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -1,20 +1,31 @@ #******************************************************************************* # gh-pages.yml # -# Github Workflow to deploy Interlisp.org. +# Github Workflow to deploy Interlisp.org. # -# Interlisp.org is a Hugo based static website that contains a -# detailed bibliography maintained using Zotero (https://www.zotero.org/groups/2914042/interlispwww.zotero.org/). +# Interlisp.org is a Hugo based static website that contains a +# detailed bibliography maintained using Zotero (https://www.zotero.org/groups/2914042/interlispwww.zotero.org/). # -# This workflow consists of two jobs, one to ensure that we have the latest -# version of the Zotero bibliography and a second job to deploy the website. +# This workflow consists of several jobs: # -# The workflow is executed either on a push or via scheduled run times. When -# started at a scheduled run time we only do a deploy if the cached bibliography -# is no longer current. On a push, we always verifty the the current -# bibliography is loaded and deploy a new version of the website. +# validate-docs - ensure README.md references the correct Hugo version +# hugo-version - expose the Hugo version as a job output +# build - build the website and run the full test suite using the +# org-level reusable workflow; also checks the Zotero +# bibliography version (skipping the build on scheduled +# runs when it is unchanged) +# (Interlisp/shared-workflows/.github/workflows/build-site.yml) +# preview - trigger a per-PR staging preview in +# Interlisp/Interlisp.staging (skips on fork PRs or when +# the STAGING_APP_ID variable is unset) +# deploy - deploy the built site to GitHub Pages # -# 2023-10-20 Bill Stumbo +# The workflow is executed either on a push or via scheduled run times. When +# started at a scheduled run time we only do a deploy if the cached bibliography +# is no longer current. On a push, we always verify the the current +# bibliography is loaded and deploy a new version of the website. +# +# 2023-10-20 Bill Stumbo # # Copyright 2023 by Interlisp.org # @@ -49,47 +60,9 @@ defaults: shell: bash env: - # ---------------------------------------------------------------------------- - # Specify the deployment environment: staging or production - HUGO_ENVIRONMENT: ${{ vars.HUGO_ENVIRONMENT || 'staging' }} HUGO_VERSION: 0.155.3 jobs: - # ---------------------------------------------------------------------------- - # Use the Zotero REST API to get the current version of the Zotero Bibliography - # Compare against a cached version of the bibliography. - # - check: - outputs: - zoteroVersion: ${{ steps.zoteroVersion.outputs.version }} - cacheHit: ${{ steps.cache-zotero.outputs.cache-hit }} - - runs-on: ubuntu-latest - steps: - - - name: Get Zotero Version Information - id: zoteroVersion - run: | - set -euo pipefail - VERSION=$(curl -fsSLI "https://api.zotero.org/groups/2914042/items?format=versions" \ - | { grep -i "last-modified-version" \ - | cut -d: -f2 \ - | tr -d $'\r ' || true; }) - if [[ -z "${VERSION:-}" ]]; then - echo "Error: Failed to determine Zotero Last-Modified-Version from API response." >&2 - exit 1 - fi - echo "version=$VERSION" >> "$GITHUB_OUTPUT" - - - name: Cache Zotero Bibliography - id: cache-zotero - uses: actions/cache/restore@v5 - with: - lookup-only: true - path: | - content/en/history/bibliography - key: bib-${{ steps.zoteroVersion.outputs.version }} - # ---------------------------------------------------------------------------- # Validate that README.md references the correct Hugo version. # Only runs on push/pull_request, not scheduled runs. @@ -124,149 +97,88 @@ jobs: echo "README.md Hugo version references are consistent" # ---------------------------------------------------------------------------- - # Run the bibliography JSON-LD test suite against a test-environment Hugo - # build that mounts tests/fixtures/bibliography/ alongside the real content. - # This job must pass before changes can be merged or deployed. + # Expose the Hugo version from the workflow environment as a job output. + # The env context is not available in a reusable workflow's `with:` block, + # so the version is surfaced here instead. Keeping it in the workflow `env` + # means version bumps are ordinary, reviewed pull requests. # - test: - needs: [check, validate-docs] - if: always() && (github.event_name == 'push' || github.event_name == 'pull_request' || needs.check.outputs.cacheHit != 'true') && (needs.validate-docs.result == 'success' || needs.validate-docs.result == 'skipped') && (needs.check.result == 'success' || needs.check.result == 'skipped') + hugo-version: runs-on: ubuntu-latest + outputs: + version: ${{ env.HUGO_VERSION }} steps: - - uses: actions/checkout@v6 - with: - submodules: recursive - fetch-depth: 0 - - - name: Cache Zotero Bibliography - id: cache-bib - uses: actions/cache@v5 - with: - path: | - static/data/bibliography.json - static/data/bibItems - content/en/history/bibliography - key: bib-${{ needs.check.outputs.zoteroVersion }} - - - name: Install Bibliography - if: steps.cache-bib.outputs.cache-hit != 'true' - run: | - echo "Retrieve bibliography" - cd scripts - chmod +x ./update_bibliography.sh - chmod +x ./bibSplit.pl - ./update_bibliography.sh - - - name: Install Hugo CLI - run: | - wget -O ${{ runner.temp }}/hugo.deb https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}/hugo_extended_${HUGO_VERSION}_linux-amd64.deb \ - && sudo dpkg -i ${{ runner.temp }}/hugo.deb - - - name: Setup Node - uses: actions/setup-node@v6 - with: - node-version: 24 - cache: 'npm' - - - name: Install Node dependencies - run: npm ci - - - name: Build test site + - name: Verify Hugo version is set env: - HUGO_CACHEDIR: ${{ runner.temp }}/hugo_cache - TZ: America/New_York - run: hugo --environment testing --destination tests/public_test - - - name: Install pytest - run: pip install pytest pyyaml pytest-cov - - - name: Run content-integrity tests - run: pytest tests/test_content_integrity.py -v --cov-report=term-missing - - - name: Run JSON-LD tests - run: pytest tests/test_bibliography_jsonld.py -v + VERSION: ${{ env.HUGO_VERSION }} + run: | + [[ -n "$VERSION" ]] || { echo "::error::HUGO_VERSION is empty"; exit 1; } + echo "Hugo version: $VERSION" # ---------------------------------------------------------------------------- - # Build the website. This job is conditional, we will always run it on a - # push or if on a scheduled run the cache was determined to be out of date. - # + # Build the website using the org-level reusable workflow. This job is + # conditional, we will always run it on a push or if on a scheduled run the + # cache was determined to be out of date. + # build: - needs: [check, validate-docs, test] - if: always() && (github.event_name == 'push' || github.event_name == 'pull_request' || needs.check.outputs.cacheHit != 'true') && (needs.validate-docs.result == 'success' || needs.validate-docs.result == 'skipped') && (needs.check.result == 'success' || needs.check.result == 'skipped') && (needs.test.result == 'success') + needs: [validate-docs, hugo-version] + if: always() && (needs.validate-docs.result == 'success' || needs.validate-docs.result == 'skipped') && (needs.hugo-version.result == 'success') + uses: Interlisp/shared-workflows/.github/workflows/build-site.yml@main + with: + hugo-version: ${{ needs.hugo-version.outputs.version }} + hugo-environment: ${{ vars.HUGO_ENVIRONMENT || 'production' }} + skip-if-fresh: ${{ github.event_name == 'schedule' }} + + # ---------------------------------------------------------------------------- + # Trigger a per-PR staging preview in Interlisp/Interlisp.staging. Skips on + # fork pull requests and when the GitHub App credentials are not configured, + # so the workflow remains green during rollout. The Hugo version is passed + # as a dispatch input (a step can read the env context, unlike a reusable + # workflow call), keeping gh-pages.yml's env the single source of truth. + # + preview: + if: github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name == github.repository && + vars.STAGING_APP_ID != '' runs-on: ubuntu-latest + needs: [build] steps: - - uses: actions/checkout@v6 + - name: Mint GitHub App token + id: app-token + uses: actions/create-github-app-token@v3 with: - submodules: recursive - fetch-depth: 0 - - - name: Cache Zotero Bibliography - id: cache-bib - uses: actions/cache@v5 + client-id: ${{ vars.STAGING_APP_ID }} + private-key: ${{ secrets.STAGING_APP_PRIVATE_KEY }} + owner: Interlisp + repositories: | + Interlisp.staging + + - name: Trigger staging preview + uses: benc-uk/workflow-dispatch@v1 with: - path: | - static/data/bibliography.json - static/data/bibItems - content/en/history/bibliography - key: bib-${{ needs.check.outputs.zoteroVersion }} - - - name: Install Bibliography - if: steps.cache-bib.outputs.cache-hit != 'true' - run: | - echo "Retrieve bibliography" - cd scripts - chmod +x ./update_bibliography.sh - chmod +x ./bibSplit.pl - ./update_bibliography.sh - - # Install Hugo Extended - # - - name: Install Hugo CLI - run: | - wget -O ${{ runner.temp }}/hugo.deb https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}/hugo_extended_${HUGO_VERSION}_linux-amd64.deb \ - && sudo dpkg -i ${{ runner.temp }}/hugo.deb - - - name: Setup Pages - id: pages - uses: actions/configure-pages@v6.0.0 - - - name: Setup Node - uses: actions/setup-node@v6 - with: - node-version: 24 - cache: 'npm' - - - name: Install dependencies - run: | - npm ci - - - name: Build - env: - HUGO_CACHEDIR: ${{ runner.temp }}/hugo_cache - TZ: America/New_York - run: hugo --cleanDestinationDir -e $HUGO_ENVIRONMENT - - - name: Install pytest - run: pip install pytest pytest-cov - - - name: Run build-integrity tests - run: pytest tests/test_hugo_build.py -v - - - name: Upload artifact - uses: actions/upload-pages-artifact@v5.0.0 - with: - path: ./public + workflow: deploy-preview.yml + repo: Interlisp/Interlisp.staging + ref: main + token: ${{ steps.app-token.outputs.token }} + inputs: '{ + "pr_number": "${{ github.event.pull_request.number }}", + "pr_sha": "${{ github.event.pull_request.head.sha }}", + "hugo_version": "${{ env.HUGO_VERSION }}" + }' + # ---------------------------------------------------------------------------- + # Deploy the built site to GitHub Pages. + # Only runs on push or scheduled runs, never on pull requests, and never + # when the build was skipped because the bibliography was already current. + # deploy: - if: github.event_name != 'pull_request' + if: github.event_name != 'pull_request' && needs.build.outputs.skipped != 'true' environment: name: github-pages url: ${{ steps.deployment.outputs.page_url }} runs-on: ubuntu-latest - needs: [build, test] + needs: [build] steps: - name: Deploy to GitHub Pages id: deployment uses: actions/deploy-pages@v5 - diff --git a/.github/workflows/preview-cleanup.yml b/.github/workflows/preview-cleanup.yml new file mode 100644 index 00000000..fac87f73 --- /dev/null +++ b/.github/workflows/preview-cleanup.yml @@ -0,0 +1,46 @@ +#******************************************************************************* +# preview-cleanup.yml +# +# GitHub Workflow that removes the staging preview for a pull request once +# the pull request is closed or merged. +# +# Triggers the remove-preview workflow in the Interlisp/Interlisp.staging +# repository, which deletes the pr-/ subdirectory from the staging +# Pages site. +# +# Copyright 2026 by Interlisp.org +# +# ****************************************************************************** +name: Preview Cleanup + +on: + pull_request: + branches: + - main + types: [closed] + +jobs: + remove-preview: + if: vars.STAGING_APP_ID != '' + runs-on: ubuntu-latest + steps: + - name: Mint GitHub App token + id: app-token + uses: actions/create-github-app-token@v3 + with: + client-id: ${{ vars.STAGING_APP_ID }} + private-key: ${{ secrets.STAGING_APP_PRIVATE_KEY }} + owner: Interlisp + repositories: | + Interlisp.staging + + - name: Trigger staging preview removal + uses: benc-uk/workflow-dispatch@v1 + with: + workflow: remove-preview.yml + repo: Interlisp/Interlisp.staging + ref: main + token: ${{ steps.app-token.outputs.token }} + inputs: '{ + "pr_number": "${{ github.event.pull_request.number }}" + }' diff --git a/.gitignore b/.gitignore index 2a1a5fd2..651b401b 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,9 @@ tests/public_test/ node_modules/ _gen/ .hugo_build.lock +.coverage +tests/__pycache__/ +_vendor/ .idea data/bibliography.json static/data/bibliography.json diff --git a/README.md b/README.md index ddb01006..24ef364a 100644 --- a/README.md +++ b/README.md @@ -241,17 +241,7 @@ Building the website is driven by a GitHub workflow (`.github/workflows/gh-pages The workflow consists of four jobs: -**1. `check` — Verify Bibliography is Current** - -Uses Zotero's REST interface to query for the latest version of the group bibliography. A `GET` call is made to: - -``` -https://api.zotero.org/groups/2914042/items -``` - -This returns metadata including the `Last-Modified-Version` header, which is incremented every time the Zotero Interlisp catalog is updated. This value is used as a cache key for the bibliography. If the cache key matches one in the current GitHub Action cache, we reuse the saved bibliography and skip rebuilding. - -**2. `validate-docs` — Verify Documentation Consistency** +**1. `validate-docs` — Verify Documentation Consistency** Runs on `push` and `pull_request` events to ensure that README.md references the correct Hugo version. Checks that: - The Hugo badge displays the version defined in `HUGO_VERSION` @@ -259,32 +249,49 @@ Runs on `push` and `pull_request` events to ensure that README.md references the This job prevents documentation drift from the actual build configuration. -**3. `build` — Build the Website** - -- Determines if a build is needed: - - On `push` or `pull_request`: Always builds - - On schedule: Skips build if Zotero cache is current -- Checks out the repository -- If the Zotero cache is valid, copies its contents into the `content/en/history/bibliography` directory -- If the cache is invalid, runs `update_bibliography.sh` to download and process a new copy -- Runs Hugo Extended (version defined by `HUGO_VERSION` environment variable) with flags: - - `-e $HUGO_ENVIRONMENT` — specifies production or staging build - - `--cleanDestinationDir` — clears `./public` directory to avoid stale artifacts +**2. `hugo-version` — Expose the Hugo Version** + +Reads the `HUGO_VERSION` environment variable and exposes it as a job +output. The `env` context is not available in a reusable workflow's +`with:` block, so the version is surfaced here instead. Keeping it in +the workflow `env` means version bumps are ordinary, reviewed pull +requests. + +**3. `build` — Build the Website and Run the Tests** + +Delegates to the org-level reusable workflow +(`Interlisp/shared-workflows/.github/workflows/build-site.yml`), which: +- Queries the Zotero REST API for the bibliography version and caches the + bibliography, running `update_bibliography.sh` to download and process a + new copy whenever the version has changed (a cache miss) +- On scheduled runs, skips the build when the bibliography is unchanged + (exposed via the workflow's `skipped` output) +- Runs Hugo Extended with the build environment passed as an input + (`-e production` for this repository) +- Runs the full test suite: + - Content-integrity and bibliography JSON-LD suites against a + `testing`-environment build (which mounts bibliography test fixtures) + - Build-integrity tests (`test_hugo_build.py`) against the production + output - Uses the GitHub `upload-pages-artifact` action to package and store the `./public` directory contents for deployment **4. `deploy` — Deploy to GitHub Pages** -Takes the output of the build step and deploys it to GitHub Pages using the GitHub `deploy-pages` action. Skipped on pull requests. +Takes the output of the build step and deploys it to GitHub Pages using the GitHub `deploy-pages` action. Skipped on pull requests and when the build was skipped because the bibliography was already current. ### Environment Variables -The following environment variables control the build and deployment process: +The following repository variables control the build and deployment process: | Variable | Description | Values | Default | |----------|-------------|--------|---------| -| `HUGO_ENVIRONMENT` | Specifies the build environment | `development`, `staging`, `production` | `staging` | +| `HUGO_ENVIRONMENT` | Build environment used by the production workflow | `development`, `staging`, `production` | `production` | | `HUGO_VERSION` | Hugo version used in CI/CD | Semantic version (e.g., `0.155.3`) | Set in workflow | +`HUGO_ENVIRONMENT` is resolved in `.github/workflows/gh-pages.yml` and passed +as the `hugo-environment` input to the shared build workflow. The staging +repository always builds with the `staging` environment. + **Environment-specific behavior:** | Environment | Analytics | Crawlers | Use Case | @@ -293,42 +300,35 @@ The following environment variables control the build and deployment process: | `staging` | Disabled | Blocked | PR previews and testing | | `development` | Disabled | N/A | Local development | -These variables are set in `.github/workflows/gh-pages.yml` and can be overridden via GitHub repository variables. - ### Deploying a Staging Site -To deploy a personal staging site for testing: - -**1. Initial Setup:** - -1. Fork/clone the Interlisp.github.io repository to your GitHub account -2. In your repository, go to **Settings → Pages** -3. Under **Build and deployment**, set Source to **Deploy from GitHub Actions** - -**2. Configure Your Fork:** - -Create a branch and make these required changes: - -1. Update `baseURL` in `config/staging/hugo.yaml` to match your repository: - -```yaml -baseURL: https://YOUR_USERNAME.github.io/YOUR_REPO_NAME/ - -languageCode: en-us -title: 'Staging Environment' -``` - -> **Important:** The `baseURL` must reflect the complete path of your repository. Incorrect URLs will cause deployment failures or broken links. - -**3. Deploy:** - -1. Commit and push your branch -2. Create a Pull Request to merge into your repository's main branch -3. Merge the PR — GitHub Actions will build and deploy your staging site - -**4. Develop Features:** - -Create feature branches for new work. Once tested on your staging site, create a PR to merge content into the main Interlisp repository. +Every pull request to `main` is automatically deployed to a per-PR staging +preview. A dedicated repository, `Interlisp/Interlisp.staging`, acts as the +deployment target. Each PR is served from a unique subdirectory of that +repository's GitHub Pages site: + +| Deployment | URL | +|------------|-----| +| Staging root | `https://interlisp.github.io/Interlisp.staging/` | +| PR #123 preview | `https://interlisp.github.io/Interlisp.staging/pr-123/` | + +When a PR is opened or updated, the production workflow (`gh-pages.yml`) +triggers the `deploy-preview` workflow in the staging repository, which: + +1. Checks out the PR's head commit +2. Builds the site using the shared org-level workflow + (`Interlisp/shared-workflows/.github/workflows/build-site.yml`) +3. Deploys the result to the `pr-/` subdirectory +4. Posts the preview URL as a comment on the PR + +When a PR is closed or merged, `preview-cleanup.yml` triggers the +`remove-preview` workflow in the staging repository to delete the +subdirectory. + +Because the build logic lives in the org-level reusable workflow, production +and staging are always built the same way. The staging environment uses +`config/staging/hugo.yaml`, which sets a `baseURL` for the staging site and +disables production-only behavior such as search-engine crawling. --- diff --git a/config/staging/hugo.yaml b/config/staging/hugo.yaml index 904e9b61..d6576ecb 100644 --- a/config/staging/hugo.yaml +++ b/config/staging/hugo.yaml @@ -1,4 +1,4 @@ -baseURL: https://stumbo.github.io/InterlispDraft.github.io/ +baseURL: https://interlisp.github.io/Interlisp.staging/ languageCode: en-us diff --git a/staging-site-design.md b/staging-site-design.md new file mode 100644 index 00000000..91abf999 --- /dev/null +++ b/staging-site-design.md @@ -0,0 +1,257 @@ +# Interlisp.org Staging Preview System: Design and Operations + +This document describes the automated per-PR staging preview system for +[`Interlisp/Interlisp.github.io`](https://github.com/Interlisp/Interlisp.github.io), +which is the repository for the Interlisp.org website. + +Every pull request to the production repository is automatically deployed to a +temporary staging site where it can be viewed and discussed before merging. +Previews are served from a separate repository's GitHub Pages *project site*, +each from a unique subdirectory URL, so multiple pull requests can be reviewed +simultaneously. The system stays entirely within GitHub — no third-party +services — and removes each preview automatically when its pull request is +closed or merged. + +--- + +## Architecture + +```mermaid +flowchart TD + subgraph Prod["Interlisp/Interlisp.github.io (production org site)"] + GH[gh-pages.yml
build + preview dispatch] + CLEANUP[preview-cleanup.yml
on PR close] + end + + subgraph Staging["Interlisp/Interlisp.staging (deployment target)"] + DEPLOY[deploy-preview.yml
build + deploy to pr-N] + REMOVE[remove-preview.yml
delete pr-N] + PRUNE[prune-previews.yml
daily safety net] + end + + subgraph Shared["Interlisp/shared-workflows"] + BUILD[build-site.yml
reusable build pipeline] + end + + GH -- "workflow_dispatch (app token)" --> DEPLOY + CLEANUP -- "workflow_dispatch (app token)" --> REMOVE + GH -- "workflow_call (build)" --> BUILD + DEPLOY -- "workflow_call (build)" --> BUILD + DEPLOY -- "comment preview URL (app token)" --> GH +``` + +## Components + +| Component | Purpose | Location | Key triggers | +|-----------|---------|----------|--------------| +| `Interlisp/Interlisp.github.io` | Production org site. Owns PR events: builds the site, triggers the staging preview, and triggers preview removal on PR close. | `.github/workflows/gh-pages.yml`, `.github/workflows/preview-cleanup.yml` | `push` / `pull_request` / `schedule` / `workflow_dispatch`; `pull_request: [closed]` | +| `Interlisp/Interlisp.staging` | Pure **deployment target** — never holds source code. Serves previews from its GitHub Pages project site. | `.github/workflows/deploy-preview.yml`, `.github/workflows/remove-preview.yml`, `.github/workflows/prune-previews.yml` | `workflow_dispatch`; daily `schedule` | +| `Interlisp/shared-workflows` | Org-level repo hosting the single reusable build workflow shared by production and staging, so build logic never drifts. | `.github/workflows/build-site.yml` | `workflow_call` | +| `Interlisp-staging-bot` | GitHub App used for cross-repo authentication (dispatch and comment steps). | — (registered in org developer settings) | — | + +### Shared build pipeline (`build-site.yml`) + +`build-site.yml` is a reusable workflow (`on: workflow_call`) that encapsulates +the entire build pipeline: + +1. Check out the source repository at the given ref. +2. Query the Zotero API for the bibliography version and restore the cached + bibliography (or rebuild it on a cache miss). +3. Install Hugo Extended, configure Pages, set up Node (v24), and `npm ci`. +4. Build with Hugo using the requested environment and an optional `--baseURL` + override. +5. Optionally run the test suite (content-integrity, JSON-LD, and + build-integrity tests). +6. Optionally upload the `github-pages` artifact. + +Its inputs are `repository`, `ref`, `hugo-version` (required), +`hugo-environment` (required), `base-url`, `upload-artifact` (default `true`), +`run-build-tests` (default `true`), and `skip-if-fresh` (default `false`). +It exposes a `skipped` output, true when the build was skipped because +`skip-if-fresh` was set and the Zotero bibliography cache was already fresh. + +Both callers pin the workflow at `@main` of `Interlisp/shared-workflows`. + +## Preview lifecycle + +### Deploy (PR open/update) + +```mermaid +sequenceDiagram + participant Prod as Interlisp.github.io (gh-pages.yml) + participant Staging as Interlisp.staging (deploy-preview.yml) + participant Shared as shared-workflows (build-site.yml) + participant Pages as gh-pages branch + + Prod->>Prod: build job runs (production env, full tests) + Prod->>Staging: preview job dispatches deploy-preview.yml
(app token, pr_number + pr_sha + hugo_version) + Staging->>Shared: build job calls build-site.yml
(staging env, per-PR baseURL, tests off) + Shared-->>Staging: uploads github-pages artifact + Staging->>Staging: extract artifact.tar + Staging->>Pages: deploy to pr-/ (keep_files: true) + Staging->>Prod: comment preview URL on PR (app token) +``` + +On every PR open/update to `main`, `gh-pages.yml` in the production repo: + +1. The `build` job calls the shared reusable workflow + (`build-site.yml@main`) with the production environment and + `skip-if-fresh` (true only on scheduled runs). +2. The `preview` job — guarded to run only for pull requests from the same + repository and only when the `STAGING_APP_ID` variable is set — mints a + GitHub App token scoped to `Interlisp.staging` and dispatches + `deploy-preview.yml` on the staging repo's `main` branch, passing + `pr_number`, `pr_sha`, and `hugo_version`. + +In the staging repo, `deploy-preview.yml`: + +1. The `build` job calls the same shared reusable workflow against the PR head + commit with `hugo-environment: staging`, the per-PR baseURL + (`https://interlisp.github.io/Interlisp.staging/pr-/`), and + `run-build-tests: false`. +2. The `deploy` job downloads the `github-pages` artifact, extracts + `artifact.tar`, and deploys the contents to the `pr-/` subdirectory of + the `gh-pages` branch via `peaceiris/actions-gh-pages@v4` with + `keep_files: true` (so other previews are preserved). The deploy commit + message and the run's job summary both state the exact deployed path. +3. Mints a second GitHub App token (scoped to `Interlisp.github.io`) and posts + a comment on the source PR with the preview URL. + +The `hugo_version` dispatch input keeps the production workflow's `HUGO_VERSION` +environment variable the single source of truth for the Hugo version across +both production and staging builds. + +### Teardown (PR close/merge) + +Teardown is two-tiered: + +1. **Fast path:** `preview-cleanup.yml` in the production repo triggers on + `pull_request: [closed]`, mints a token scoped to `Interlisp.staging`, and + dispatches `remove-preview.yml` with the PR number. That workflow checks out + the `gh-pages` branch, `git rm -r pr-/`, commits, and pushes. It is + hardened to no-op cleanly when the branch or subdirectory does not exist, + and a `concurrency` guard prevents overlapping removals. +2. **Safety net:** `prune-previews.yml` runs **daily** (and is manually + triggerable). It lists open PRs in `Interlisp/Interlisp.github.io`, then + checks out `gh-pages` and removes any `pr-*` directory whose PR is no longer + open. This guarantees no stale preview survives even if the fast path is + missed or fails. + +Both teardown workflows are guarded by `if: vars.STAGING_APP_ID != ''` so they +skip silently before app credentials exist. + +## URLs + +| Deployment | URL | +|------------|-----| +| Production | `https://interlisp.org` | +| Staging root | `https://interlisp.org/Interlisp.staging/` | +| PR #123 preview | `https://interlisp.org/Interlisp.staging/pr-123/` | +| PR #456 preview | `https://interlisp.org/Interlisp.staging/pr-456/` | + +The org-site `CNAME` (`interlisp.org`) makes the staging project site reachable +under the custom domain at `https://interlisp.org/Interlisp.staging/`. + +Previews are removed automatically when a PR is closed or merged. + +## Cross-repo authentication + +The default repo-scoped `GITHUB_TOKEN` cannot trigger workflows in another +repository, so a GitHub App identity is used. A short-lived installation token +is minted at runtime by `actions/create-github-app-token@v3`, scoped to exactly +the repository needed via the `repositories:` input: + +```yaml +- uses: actions/create-github-app-token@v3 + id: app-token + with: + client-id: ${{ vars.STAGING_APP_ID }} + private-key: ${{ secrets.STAGING_APP_PRIVATE_KEY }} + owner: Interlisp + repositories: | + Interlisp.staging +``` + +Credentials are stored at the org level: + +- `STAGING_APP_ID` — the app's **Client ID**, stored as an org **variable**. +- `STAGING_APP_PRIVATE_KEY` — the entire `.pem` file contents, stored as an org + **secret**. + +The `GITHUB_TOKEN` is used for the deploy and cleanup pushes, which stay inside +the staging repo. + +## Configuration + +- **GitHub App** (`Interlisp-staging-bot`), installed on both repos: + - Actions: Read and write (trigger `workflow_dispatch` on the staging repo) + - Pull requests: Read and write (post the preview URL comment) + - Issues: Read and write (also accepted for the comment endpoint) + - No webhook configured. +- **Pages:** staging repo serves from the `gh-pages` branch ("Deploy from a + branch", not "GitHub Actions"). +- **Staging config:** `config/staging/hugo.yaml` sets + `baseURL: https://interlisp.github.io/Interlisp.staging/` and titles pages + "Staging Environment". +- **Repo pins:** both callers pin the shared workflow at `@main` of + `Interlisp/shared-workflows`. +- **`Interlisp/shared-workflows` `main`:** protected — all updates land via PR + with one approving review; Dependabot opens monthly `github-actions` update + PRs through the same path. + +## Operational notes + +- **Per-PR `baseURL` is passed at build time** via `--baseURL`, so each + preview's absolute links resolve under its own `/pr-/` path. +- **Fails safe:** the preview and cleanup jobs skip (rather than fail) for fork + PRs or when `STAGING_APP_ID` is unset. +- **App permission changes do not retro-apply to existing installations.** + After adding repository permissions to the app, an org owner must approve the + change at the org's GitHub Apps settings; until then the comment step fails + with HTTP 403 (`Resource not accessible by integration`). +- **Previews count against the 1 GB Pages limit** — each `pr-*` directory + accumulates on the `gh-pages` branch. +- **Staging previews build with tests disabled** (`run-build-tests: false`) + because the test suite's fixtures rebuild `public/` with the production + environment and would clobber the staged artifact. + +## Code References + +| Component | File | Key actions | +|-----------|------|-------------| +| Production build + preview dispatch | `Interlisp/Interlisp.github.io/.github/workflows/gh-pages.yml` | `build` (calls `build-site.yml@main`), `preview` (mints app token, dispatches `deploy-preview.yml`), `deploy` (Pages deploy) | +| Preview removal trigger | `Interlisp/Interlisp.github.io/.github/workflows/preview-cleanup.yml` | Mints app token, dispatches `remove-preview.yml` on PR close | +| Preview deploy | `Interlisp/Interlisp.staging/.github/workflows/deploy-preview.yml` | `build` (calls `build-site.yml@main` with staging env + per-PR baseURL), `deploy` (extract artifact, push to `pr-/`), comment URL on PR | +| Preview removal | `Interlisp/Interlisp.staging/.github/workflows/remove-preview.yml` | Checks out `gh-pages`, `git rm -r pr-/`, commits, pushes | +| Stale preview prune | `Interlisp/Interlisp.staging/.github/workflows/prune-previews.yml` | Lists open PRs, removes any `pr-*` dir whose PR is not open | +| Shared build pipeline | `Interlisp/shared-workflows/.github/workflows/build-site.yml` | Zotero bibliography check/cache, Hugo Extended build, test suite, artifact upload | + +## Glossary + +| Term | Definition | +|------|------------| +| `pr-/` | The subdirectory on the `gh-pages` branch (and URL path) that holds one PR's preview. | +| `github-pages` artifact / `artifact.tar` | The artifact uploaded by the shared build workflow; `artifact.tar` is extracted before deploying to a preview directory. | +| `workflow_dispatch` | A GitHub event used to trigger a workflow in another repository with inputs. | +| GitHub App installation token | A short-lived token minted at runtime from the app's credentials, scoped to specific repositories. | +| Org site vs. project site | An org site (`.github.io`) is the account-level site; a project site (`.github.io/`) lives under a repo name path. | +| `keep_files` | The `peaceiris/actions-gh-pages` option that preserves existing files in the destination directory (other previews) instead of replacing the branch. | +| `baseURL` override | The `--baseURL` passed to Hugo at build time so a preview's absolute links resolve under its own `/pr-/` path. | +| `skipped` output | The shared build workflow's output indicating the build was skipped because the bibliography cache was fresh. | + +## Future Work + +- **Exclude the large static documentation tree from staging previews.** The + `static/documentation` directory (~175 MB) is not needed for preview + evaluation; excluding it would keep preview builds fast and reduce storage + against the Pages limit. +- **Add a dedicated staging custom domain** (e.g., `staging.interlisp.org`) + via the staging repo's own `CNAME` file for cleaner preview URLs. +- **Support previews for fork PRs.** The `preview` job currently skips PRs + whose head repo differs from the base repo; enabling fork previews would let + outside contributors get staging sites too. +- **Monitor preview storage.** Subdirectory previews accumulate on the + `gh-pages` branch; consider retention tuning or alerting as PR volume grows. +- **Confirm teardown behavior in production.** The on-close fast path and the + daily prune have been exercised; a final end-to-end observation on a real + closed PR would fully validate cleanup. diff --git a/tests/test_hugo_build.py b/tests/test_hugo_build.py index a69049cc..24596afc 100644 --- a/tests/test_hugo_build.py +++ b/tests/test_hugo_build.py @@ -36,8 +36,8 @@ def _get_baseurl_path() -> str: """Return the path component of the Hugo baseURL for the active environment. When ``baseURL`` contains a path prefix (e.g. - ``https://stumbo.github.io/InterlispDraft.github.io/``), Hugo prepends - that path (``/InterlispDraft.github.io``) to every site-root-relative + ``https://interlisp.github.io/interlisp-staging/``), Hugo prepends + that path (``/interlisp-staging``) to every site-root-relative ``href``. This helper extracts just the path component so link checks can strip it before resolving to the filesystem. """ @@ -174,7 +174,7 @@ def test_no_broken_internal_links(self) -> None: content = html_file.read_text(encoding="utf-8", errors="ignore") # Match href values that start with / (site-root-relative) for href in re.findall(r'href="(/[^"#?]*?)"', content): - # If baseURL has a path component (e.g., /InterlispDraft.github.io), + # If baseURL has a path component (e.g., /interlisp-staging), # Hugo prepends it to site-root-relative links. Strip it before # resolving to the filesystem. resolved = href