ci.yml

name: CI

on:
  push:
    branches:
      - main
    paths-ignore:
      - "conductor-clients/**"
  pull_request:
    paths-ignore:
      - "conductor-clients/**"
  workflow_dispatch:
    inputs:
      e2e_profiles:
        description: 'E2E profiles to run in addition to redis-es8 (which always runs): "all", "none", or a comma separated subset of postgres, mysql, redis-os3'
        required: false
        default: all
        type: string
  # GitHub Actions cron expressions are UTC.
  schedule:
    - cron: "0 2 * * *"

# A new push to a PR supersedes that PR's in-flight run - cancel it instead of
# burning runners on a stale sha (same pattern as orkes-conductor's workflows).
# Groups are separated by event type so nightly/dispatch/push runs never
# cross-cancel each other or PR runs.
concurrency:
  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
  detect-changes:
    runs-on: ubuntu-latest
    outputs:
      persistence: ${{ steps.filter.outputs.persistence }}
      ui: ${{ steps.filter.outputs.ui }}
    steps:
      - uses: actions/checkout@v7
        with:
          ref: ${{ github.event.pull_request.head.sha }}
          fetch-depth: 0
      - uses: dorny/paths-filter@v3
        id: filter
        with:
          filters: |
            persistence:
              - 'cassandra-persistence/**'
              - 'common-persistence/**'
              - 'es6-persistence/**'
              - 'es7-persistence/**'
              - 'es8-persistence/**'
              - 'mysql-persistence/**'
              - 'os-persistence/**'
              - 'os-persistence-v2/**'
              - 'os-persistence-v3/**'
              - 'postgres-persistence/**'
              - 'redis-persistence/**'
              - 'scheduler/cassandra-persistence/**'
              - 'scheduler/mysql-persistence/**'
              - 'scheduler/postgres-persistence/**'
              - 'scheduler/redis-persistence/**'
              - 'scheduler/sqlite-persistence/**'
              - 'sqlite-persistence/**'
            ui:
              - 'ui/**'
              - 'ui-next/**'

  build:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v7
        with:
          ref: ${{ github.event.pull_request.head.sha }}
          fetch-depth: 0
      - name: Free disk space
        run: |
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /usr/local/lib/android
          sudo rm -rf /opt/ghc
      - name: Gradle wrapper validation
        uses: gradle/wrapper-validation-action@v3
      - name: Set up Zulu JDK 21
        uses: actions/setup-java@v5
        with:
          distribution: "zulu"
          java-version: "21"
      - name: Cache SonarCloud packages
        uses: actions/cache@v5
        with:
          path: ~/.sonar/cache
          key: ${{ runner.os }}-sonar
          restore-keys: ${{ runner.os }}-sonar
      - name: Cache Gradle packages
        uses: actions/cache@v5
        with:
          path: |
            ~/.gradle/caches
            ~/.gradle/wrapper
          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
          restore-keys: ${{ runner.os }}-gradle-
      - name: Build with Gradle
        if: github.event_name != 'push' || github.ref != 'refs/heads/main'
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
        run: |
          ./gradlew build -x :conductor-test-harness:test -x test
      - name: Build and Publish snapshot
        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
        run: |
          echo "Running build for commit ${{ github.sha }}"
          ./gradlew build -x :conductor-test-harness:test -x test
      - name: Generate aggregated coverage report
        if: always()
        run: ./gradlew jacocoAggregatedReport -x test || true
      - name: Publish Test Report
        uses: mikepenz/action-junit-report@v6
        if: always()
        with:
          report_paths: "**/build/test-results/test/TEST-*.xml"
      - name: Upload build artifacts
        uses: actions/upload-artifact@v7
        with:
          name: build-artifacts
          path: "**/build/reports"
      - name: Upload coverage report
        uses: actions/upload-artifact@v7
        if: always()
        with:
          name: coverage-report
          path: build/reports/jacoco/aggregated
  unit-test:
    needs: detect-changes
    runs-on: ubuntu-latest
    permissions:
      contents: read
      checks: write
    steps:
      - uses: actions/checkout@v7
        with:
          ref: ${{ github.event.pull_request.head.sha }}
          fetch-depth: 0
      - name: Free disk space
        run: |
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /usr/local/lib/android
          sudo rm -rf /opt/ghc
      - name: Set up Zulu JDK 21
        uses: actions/setup-java@v5
        with:
          distribution: "zulu"
          java-version: "21"
      - name: Cache Gradle packages
        uses: actions/cache@v5
        with:
          path: |
            ~/.gradle/caches
            ~/.gradle/wrapper
          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
          restore-keys: ${{ runner.os }}-gradle-
      - name: Force Docker API Version
        run: echo 'api.version=1.44' > ~/.docker-java.properties
      - name: Run unit tests
        run: |
          # Manual dispatch and the nightly build are the full validation paths.
          if [ "${{ github.event_name }}" = "workflow_dispatch" ] || [ "${{ github.event_name }}" = "schedule" ]; then
            ./gradlew test -x :conductor-test-harness:test
            exit 0
          fi
          # Push and PR CI run persistence tests only when one of the persistence modules changed.
          # Direct Gradle invocations (for example, :conductor-postgres-persistence:test) remain
          # unaffected; this gate applies only to the aggregate CI command.
          if [ "${{ needs.detect-changes.outputs.persistence }}" == "true" ]; then
            ./gradlew test -x :conductor-test-harness:test
          else
            ./gradlew test \
              -x :conductor-test-harness:test \
              -x :conductor-cassandra-persistence:test \
              -x :conductor-common-persistence:test \
              -x :conductor-scheduler-cassandra-persistence:test \
              -x :conductor-es6-persistence:test \
              -x :conductor-es7-persistence:test \
              -x :conductor-es8-persistence:test \
              -x :conductor-mysql-persistence:test \
              -x :conductor-scheduler-mysql-persistence:test \
              -x :conductor-os-persistence:test \
              -x :conductor-os-persistence-v2:test \
              -x :conductor-os-persistence-v3:test \
              -x :conductor-postgres-persistence:test \
              -x :conductor-redis-persistence:test \
              -x :conductor-scheduler-postgres-persistence:test \
              -x :conductor-scheduler-redis-persistence:test \
              -x :conductor-scheduler-sqlite-persistence:test \
              -x :conductor-sqlite-persistence:test
          fi
      - name: Publish Test Report
        uses: mikepenz/action-junit-report@v6
        if: always()
        with:
          check_name: "Unit Test results"
          check_title_template: "{{SUITE_NAME}} | {{TEST_NAME}} | {{CLASS_NAME}}"
          report_paths: "**/build/test-results/test/TEST-*.xml"
          check_retries: true
          detailed_summary: true
          include_passed: false
          flaky_summary: true
      - name: Upload unit-test reports
        uses: actions/upload-artifact@v7
        if: always()
        with:
          name: unit-test-reports
          path: "**/build/reports/tests"
  test-harness:
    runs-on: ubuntu-latest
    # checks: write is required for the test-result check below; without it the
    # default read-only token fails with "Resource not accessible by integration".
    permissions:
      contents: read
      checks: write
    steps:
      - uses: actions/checkout@v7
        with:
          ref: ${{ github.event.pull_request.head.sha }}
          fetch-depth: 0
      - name: Free disk space
        run: |
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /usr/local/lib/android
          sudo rm -rf /opt/ghc
      - name: Set up Zulu JDK 21
        uses: actions/setup-java@v5
        with:
          distribution: "zulu"
          java-version: "21"
      - name: Cache Gradle packages
        uses: actions/cache@v5
        with:
          path: |
            ~/.gradle/caches
            ~/.gradle/wrapper
          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
          restore-keys: ${{ runner.os }}-gradle-
      - name: Force Docker API Version
        run: echo 'api.version=1.44' > ~/.docker-java.properties
      - name: Cache Docker images
        uses: actions/cache@v5
        id: docker-cache
        with:
          path: /tmp/docker-images-test-harness.tar
          key: docker-test-harness-v2
      - name: Load cached Docker images
        if: steps.docker-cache.outputs.cache-hit == 'true'
        run: docker load -i /tmp/docker-images-test-harness.tar || true
      - name: Run test-harness tests
        run: |
          ./gradlew :conductor-test-harness:test
      - name: Save Docker images for cache
        if: steps.docker-cache.outputs.cache-hit != 'true'
        run: |
          set -euo pipefail
          docker image prune -f
          mapfile -t images < <((docker images --format '{{.Repository}}:{{.Tag}}' \
            | grep -v '<none>' \
            | grep -E '(^|/)(elasticsearch|redis|postgres|mysql|mongo|cassandra)(:|/)|mockserver/mockserver|opensearchproject/opensearch|testcontainers/|orkesio/') || true)
          if [ "${#images[@]}" -eq 0 ]; then
            echo "No Testcontainers-related images to cache; writing empty tar for cache action."
            tar -cf /tmp/docker-images-test-harness.tar --files-from /dev/null
            exit 0
          fi
          printf '%s\n' "${images[@]}"
          docker save -o /tmp/docker-images-test-harness.tar "${images[@]}"
      - name: Publish Test Report
        uses: mikepenz/action-junit-report@v6
        if: always()
        with:
          check_name: "Test-Harness results"
          check_title_template: "{{SUITE_NAME}} | {{TEST_NAME}} | {{CLASS_NAME}}"
          report_paths: "test-harness/build/test-results/test/TEST-*.xml"
          check_retries: true
          detailed_summary: true
          include_passed: false
          flaky_summary: true
      - name: Upload test-harness reports
        uses: actions/upload-artifact@v7
        if: always()
        with:
          name: test-harness-reports
          path: "test-harness/build/reports"
      - name: Upload test-harness coverage report
        uses: actions/upload-artifact@v7
        if: always()
        with:
          name: test-harness-coverage-report
          path: "test-harness/build/reports/jacoco"
  agentspan-e2e:
    if: github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'
    runs-on: ubuntu-latest
    # checks: write is required for the test-result check below; without it the
    # default read-only token fails with "Resource not accessible by integration".
    permissions:
      contents: read
      checks: write
    steps:
      - uses: actions/checkout@v7
        with:
          ref: ${{ github.event.pull_request.head.sha }}
          fetch-depth: 0
      - name: Free disk space
        run: |
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /usr/local/lib/android
          sudo rm -rf /opt/ghc
      - name: Set up Zulu JDK 21
        uses: actions/setup-java@v5
        with:
          distribution: "zulu"
          java-version: "21"
      - name: Cache Gradle packages
        uses: actions/cache@v5
        with:
          path: |
            ~/.gradle/caches
            ~/.gradle/wrapper
          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
          restore-keys: ${{ runner.os }}-gradle-
      - name: Force Docker API Version
        run: echo 'api.version=1.44' > ~/.docker-java.properties
      - name: Run deterministic AgentSpan E2E tests
        run: ./gradlew :conductor-test-harness:agentspanDeterministicE2E
      - name: Run live-provider AgentSpan E2E coverage
        if: github.ref == 'refs/heads/main'
        env:
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          ANTHROPIC_MODEL: ${{ vars.ANTHROPIC_MODEL }}
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          OPENAI_MODEL: ${{ vars.OPENAI_MODEL }}
        run: ./gradlew :conductor-agentspan:agentspanEndToEndCoverage
      - name: Publish AgentSpan E2E test report
        uses: mikepenz/action-junit-report@v6
        if: always()
        with:
          report_paths: |
            test-harness/build/test-results/agentspanDeterministicE2E/TEST-*.xml
            test-harness/build/test-results/agentspanRealE2E/TEST-*.xml
          check_name: AgentSpan E2E Test Report
      - name: Upload AgentSpan E2E reports
        uses: actions/upload-artifact@v7
        if: always()
        with:
          name: agentspan-e2e-reports
          path: |
            test-harness/build/reports/tests/agentspanDeterministicE2E
            test-harness/build/reports/tests/agentspanRealE2E
          if-no-files-found: warn
      - name: Upload AgentSpan coverage report
        uses: actions/upload-artifact@v7
        if: github.ref == 'refs/heads/main'
        with:
          name: agentspan-e2e-coverage-report
          path: agentspan/build/reports/jacoco/agentspanEndToEndCoverage
          if-no-files-found: error
  generate-e2e-matrix:
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
    steps:
      - id: set-matrix
        shell: bash
        env:
          EVENT_NAME: ${{ github.event_name }}
          E2E_PROFILES: ${{ inputs.e2e_profiles }}
        run: |
          set -euo pipefail

          # redis-es8 runs on every trigger, so every push and pull request gets one full
          # end-to-end run. It is listed first in the matrix.
          # redis-es7 and cassandra-es7 are disabled: ES7 coverage is superseded by the
          # redis-es8 flavor and cassandra support is partial. Re-enable by adding them back
          # to the optional list (scripts remain in e2e/).
          always='[{"name":"redis-es8","script":"./e2e/run_tests-es8.sh"}]'
          optional='[{"name":"postgres","script":"./e2e/run_tests-postgres.sh"},
                     {"name":"mysql","script":"./e2e/run_tests-mysql.sh"},
                     {"name":"redis-os3","script":"./e2e/run_tests-redis-os3.sh"}]'

          case "$EVENT_NAME" in
            workflow_dispatch) selection="${E2E_PROFILES:-all}" ;;
            schedule)          selection="all" ;;
            *)                 selection="none" ;;
          esac

          case "$selection" in
            all)  extra="$optional" ;;
            none) extra='[]' ;;
            *)
              # Comma separated subset, e.g. "mysql,postgres". redis-es8 is always added.
              unknown=$(jq -rn --argjson optional "$optional" --arg sel "$selection" '
                ($sel | split(",") | map(gsub("^\\s+|\\s+$";"")) | map(select(length > 0))) as $want
                | (($optional | map(.name)) + ["redis-es8"]) as $known
                | [$want[] | select(. as $n | ($known | any(. == $n)) | not)] | join(", ")')
              if [ -n "$unknown" ]; then
                echo "::error::Unknown e2e profile(s): $unknown. Valid values: all, none, redis-es8, $(jq -r --argjson o "$optional" -n '$o | map(.name) | join(", ")')"
                exit 1
              fi
              extra=$(jq -c --arg sel "$selection" '
                ($sel | split(",") | map(gsub("^\\s+|\\s+$";""))) as $want
                | map(select(.name as $n | $want | any(. == $n)))' <<< "$optional")
              ;;
          esac

          include=$(jq -cn --argjson always "$always" --argjson extra "$extra" '$always + $extra')
          echo "Selected e2e profiles: $(jq -r 'map(.name) | join(", ")' <<< "$include")"
          echo "matrix={\"include\":$include}" >> "$GITHUB_OUTPUT"

  e2e:
    needs: generate-e2e-matrix
    if: ${{ needs.generate-e2e-matrix.outputs.matrix != '{"include":[]}' }}
    runs-on: ubuntu-latest
    permissions:
      contents: read
      checks: write
    strategy:
      fail-fast: false
      # All backends in parallel — each matrix job gets its own runner VM, so a full
      # 6-flavor matrix completes in ~15 minutes instead of ~90 sequential.
      matrix: ${{ fromJson(needs.generate-e2e-matrix.outputs.matrix) }}
    steps:
      - uses: actions/checkout@v7
        with:
          ref: ${{ github.event.pull_request.head.sha }}
          fetch-depth: 0
      - name: Free disk space
        run: |
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /usr/local/lib/android
          sudo rm -rf /opt/ghc
      - name: Set up Zulu JDK 21
        uses: actions/setup-java@v5
        with:
          distribution: "zulu"
          java-version: "21"
      - name: Cache Gradle packages
        uses: actions/cache@v5
        with:
          path: |
            ~/.gradle/caches
            ~/.gradle/wrapper
          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
          restore-keys: ${{ runner.os }}-gradle-
      - name: Run E2E tests (${{ matrix.name }})
        run: ${{ matrix.script }}
      - name: Publish Test Report
        uses: mikepenz/action-junit-report@v6
        if: always()
        with:
          check_name: "E2E results (${{ matrix.name }})"
          check_title_template: "{{SUITE_NAME}} | {{TEST_NAME}} | {{CLASS_NAME}}"
          report_paths: "e2e/build/test-results/test/TEST-*.xml"
          check_retries: true
          detailed_summary: true
          include_passed: false
          flaky_summary: true
      - name: Generate test summary table
        if: always()
        shell: python3 {0}
        run: |
          import os, glob, xml.etree.ElementTree as ET

          xml_files = glob.glob("e2e/build/test-results/test/TEST-*.xml")
          rows = []
          totals = {"passed": 0, "failed": 0, "skipped": 0}

          for f in sorted(xml_files):
              try:
                  root = ET.parse(f).getroot()
              except ET.ParseError:
                  continue
              for tc in root.iter("testcase"):
                  name = tc.get("name", "?")
                  classname = tc.get("classname", "").split(".")[-1]
                  duration = float(tc.get("time", 0))
                  if tc.find("skipped") is not None:
                      status, totals["skipped"] = "⏭ skip", totals["skipped"] + 1
                  elif tc.find("failure") is not None or tc.find("error") is not None:
                      node = tc.find("failure") if tc.find("failure") is not None else tc.find("error")
                      msg = (node.get("message") or "")[:120]
                      status, totals["failed"] = f"❌ `{msg}`", totals["failed"] + 1
                  else:
                      status, totals["passed"] = "✅", totals["passed"] + 1
                  rows.append((classname, name, f"{duration:.1f}s", status))

          backend = "${{ matrix.name }}"
          lines = [
              f"## E2E results — {backend}",
              f"**✅ {totals['passed']} passed · ❌ {totals['failed']} failed · ⏭ {totals['skipped']} skipped**",
              "",
              "| Class | Test | Duration | Result |",
              "|-------|------|----------|--------|",
          ]
          for cls, name, dur, status in rows:
              lines.append(f"| {cls} | {name} | {dur} | {status} |")

          summary = os.environ.get("GITHUB_STEP_SUMMARY", "/dev/null")
          with open(summary, "a") as fh:
              fh.write("\n".join(lines) + "\n")
      - name: Upload E2E reports
        uses: actions/upload-artifact@v7
        if: always()
        with:
          name: e2e-reports-${{ matrix.name }}
          path: "e2e/build/reports"

  build-ui:
    # ui-next is the default UI: it is what docker/server/Dockerfile bundles
    # into the conductor:server image, so it is what CI builds and tests.
    needs: detect-changes
    if: >-
      github.event_name == 'workflow_dispatch' ||
      github.event_name == 'schedule' ||
      needs.detect-changes.outputs.ui == 'true'
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: ui-next
    steps:
      - uses: actions/checkout@v7

      - name: Set up pnpm
        uses: pnpm/action-setup@v4
        with:
          package_json_file: ui-next/package.json

      - name: Set up Node
        uses: actions/setup-node@v6
        with:
          node-version: "22"
          cache: "pnpm"
          cache-dependency-path: ui-next/pnpm-lock.yaml

      - name: Install Dependencies
        run: pnpm install --frozen-lockfile

      - name: Typecheck
        run: pnpm typecheck

      - name: Unit tests (vitest)
        run: pnpm test

      - name: Build UI
        run: NODE_OPTIONS=--max-old-space-size=4096 pnpm build