add test against oss on push of pr-to-main

.github/workflows/ci.yml

@@ -8,11 +8,20 @@ on:
     branches:
       - main
   workflow_dispatch:
+    inputs:
+      oss_conductor_version:
+        description: 'OSS Conductor image tag (falls back to E2E_TEST_OSS_CONDUCTOR_VERSION org var)'
+        required: false
+        type: string
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+permissions:
+  contents: read
+  checks: write
+
 jobs:
   documentation-validation:
     runs-on: ubuntu-latest
@@ -152,4 +161,60 @@ jobs:
 
       - name: Check Tests Status
         if: steps.tests.outcome == 'failure'
-        run: exit 1
+        run: |
+          echo "::error::Tests failed. See the 'Run Tests' step above for the Gradle/test output, and the 'Publish Test Report' step's JUnit summary for which test(s) failed."
+          exit 1
+
+  integration-tests-oss:
+    runs-on: ubuntu-latest
+    name: Integration Tests (OSS)
+    timeout-minutes: 30
+    env:
+      CONDUCTOR_SERVER_URL: http://localhost:8080/api
+      CONDUCTOR_SERVER_TYPE: oss
+      OSS_CONDUCTOR_VERSION: ${{ inputs.oss_conductor_version || vars.E2E_TEST_OSS_CONDUCTOR_VERSION }}
+
+    steps:
+      - name: Verify OSS Conductor version is set
+        run: |
+          if [ -z "$OSS_CONDUCTOR_VERSION" ]; then
+            echo "::error::No Conductor OSS image tag resolved. Set the E2E_TEST_OSS_CONDUCTOR_VERSION organization variable (and ensure its repository access policy includes this repo), or pass the oss_conductor_version input via workflow_dispatch."
+            exit 1
+          fi
+          echo "Using conductoross/conductor:$OSS_CONDUCTOR_VERSION"
+
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Set up Zulu JDK 21
+        uses: actions/setup-java@v5
+        with:
+          distribution: "zulu"
+          java-version: "21"
+
+      - name: Start Conductor OSS stack
+        run: docker compose -f scripts/docker-compose-oss.yaml up -d
+
+      - name: Wait for Conductor to be healthy
+        run: timeout 120 bash -c 'until curl -sf http://localhost:8080/health; do sleep 5; done'
+
+      - name: Run integration tests (OSS)
+        id: integration_tests
+        continue-on-error: true
+        run: ./gradlew :tests:test -PIntegrationTests
+
+      - name: Dump Conductor logs
+        if: failure() || steps.integration_tests.outcome == 'failure'
+        run: docker compose -f scripts/docker-compose-oss.yaml logs conductor-server
+
+      - name: Publish Test Report
+        if: always()
+        uses: mikepenz/action-junit-report@v6
+        with:
+          report_paths: '**/tests/build/test-results/test/TEST-*.xml'
+
+      - name: Check Integration Tests Status
+        if: steps.integration_tests.outcome == 'failure'
+        run: |
+          echo "::error::Integration tests (OSS) failed. See the 'Run integration tests (OSS)' step above for the Gradle/test output, the 'Dump Conductor logs' step for server-side logs, and the 'Publish Test Report' step's JUnit summary for which test(s) failed."
+          exit 1

CONTRIBUTING.md

@@ -23,6 +23,47 @@ Run the SDK test suite:
 ./gradlew test jacocoTestReport
 ```
 
+### Running the OSS integration suite locally
+
+The `tests` module also has an integration suite (`-PIntegrationTests`) that runs against a
+real Conductor server, separate from the unit suite above. `scripts/run-integration-oss.sh`
+mirrors the `integration-tests-oss` job in `ci.yml`: it starts a local Conductor OSS +
+Postgres stack (defined in `scripts/docker-compose-oss.yaml`), waits for `/health`, runs the
+integration suite, and tears the stack down on exit.
+
+```shell
+scripts/run-integration-oss.sh                        # against `latest`
+scripts/run-integration-oss.sh --version 3.32.0-rc18
+scripts/run-integration-oss.sh --keep-up               # leave the stack running afterwards
+scripts/run-integration-oss.sh --include-gated         # also run tests normally skipped as Orkes-only
+```
+
+The script always prints the resolved `conductoross/conductor` tag and pulls it before
+starting the stack, since `latest` (the local default) is a mutable tag — without an
+explicit pull, `docker compose up` would silently reuse a stale cached image instead of
+fetching the current one. It also always runs Gradle with `--rerun-tasks`, since the `test`
+task's up-to-date check doesn't account for env vars like `CONDUCTOR_SERVER_TYPE` or the
+state of the live server underneath — without it, a rerun after changing gating or switching
+server versions could silently report a stale cached result instead of executing anything.
+
+The script doesn't pin a JDK itself, but CI runs on Zulu 21. If your local default JDK is
+newer (e.g. 23) you may hit `Unsupported class file major version` errors compiling tests —
+set `JAVA_HOME` explicitly to match CI:
+
+```shell
+JAVA_HOME=/Library/Java/JavaVirtualMachines/zulu-21.jdk/Contents/Home ./scripts/run-integration-oss.sh
+```
+
+Tests annotated `@DisabledIfEnvironmentVariable(named = "CONDUCTOR_SERVER_TYPE", matches =
+"oss")` skip themselves against plain OSS because they exercise Orkes-managed-only features
+(e.g. the Service Registry, Authorization, Prompts/Integrations, Environment Variables, and
+Secrets APIs, plus a handful of task/workflow endpoints OSS doesn't implement or that hit
+known Postgres-persistence bugs). Each annotation's `disabledReason` documents the specific,
+empirically-confirmed gap — treat those as the source of truth rather than a list here, since
+they can drift as OSS gains features. If you add or remove that annotation, re-verify against
+a freshly-pulled image first: a test that fails against a stale local image may pass against
+current OSS, and vice versa.
+
 Compile the maintained agent examples when changing their APIs or documentation:
 
 ```shell

conductor-client/src/main/java/com/netflix/conductor/sdk/workflow/executor/WorkflowExecutor.java

@@ -174,10 +174,16 @@ private void initMonitor() {
                     for (Map.Entry<String, CompletableFuture<Workflow>> entry : runningWorkflowFutures.entrySet()) {
                         String workflowId = entry.getKey();
                         CompletableFuture<Workflow> future = entry.getValue();
-                        Workflow workflow = workflowClient.getWorkflow(workflowId, true);
-                        if (workflow.getStatus().isTerminal()) {
-                            future.complete(workflow);
-                            runningWorkflowFutures.remove(workflowId);
+                        try {
+                            Workflow workflow = workflowClient.getWorkflow(workflowId, true);
+                            if (workflow.getStatus().isTerminal()) {
+                                future.complete(workflow);
+                                runningWorkflowFutures.remove(workflowId);
+                            }
+                        } catch (Exception e) {
+                            // scheduleAtFixedRate silently kills all future ticks on any uncaught exception, so catch here instead of letting one transient error stop completion-tracking forever.
+                            LOGGER.warn("Error polling workflow {} for completion; will retry on "
+                                    + "the next tick", workflowId, e);
                         }
                     }
                 },

conductor-client/src/main/java/com/netflix/conductor/sdk/workflow/executor/task/AnnotatedWorkerExecutor.java

@@ -76,8 +76,9 @@ public AnnotatedWorkerExecutor(TaskClient taskClient, WorkerConfiguration worker
      *     implementation
      */
     public synchronized void initWorkers(String... basePackages) {
+        // scanWorkers() -> initWorkersFromClasses() -> initWorkersFromInstances() already calls startPolling();
+        // an extra call here used to race with that one and could drop a task polled by the runner it replaces.
         scanWorkers(basePackages);
-        startPolling();
     }
 
     public synchronized void initWorkersFromInstances(List<Object> workerInstances) {
@@ -157,7 +158,10 @@ private void scanWorkers(String... basePackages) {
             initWorkersFromClasses(classes);
 
         } catch (Exception e) {
-            LOGGER.error("Error while scanning for workers: ", e);
+            // Rethrow (unchecked) rather than swallow: initWorkers() no longer has its own startPolling()
+            // fallback, so a swallowed failure here would otherwise leave the caller believing workers are
+            // running when none were ever started.
+            throw new RuntimeException("Error while scanning for workers", e);
         }
     }
 

scripts/docker-compose-oss.yaml

@@ -0,0 +1,28 @@
+services:
+  conductor-server:
+    image: conductoross/conductor:${OSS_CONDUCTOR_VERSION:-latest}
+    environment:
+      - CONFIG_PROP=config-postgres.properties
+    ports:
+      - "8080:8080"
+    healthcheck:
+      test: ["CMD", "curl", "-I", "-XGET", "http://localhost:8080/health"]
+      interval: 10s
+      timeout: 10s
+      retries: 20
+    links:
+      - conductor-postgres:postgresdb
+    depends_on:
+      conductor-postgres:
+        condition: service_healthy
+
+  conductor-postgres:
+    image: postgres:16
+    environment:
+      - POSTGRES_USER=conductor
+      - POSTGRES_PASSWORD=conductor
+    healthcheck:
+      test: timeout 5 bash -c 'cat < /dev/null > /dev/tcp/localhost/5432'
+      interval: 5s
+      timeout: 5s
+      retries: 12

scripts/run-integration-oss.sh

@@ -0,0 +1,106 @@
+#!/usr/bin/env bash
+#
+# Spin up a local Conductor OSS stack and run the `tests` module's
+# integration suite against it, mirroring the `integration-tests-oss` job in
+# .github/workflows/integration-tests-oss.yml. Orkes-Enterprise-only test
+# classes are annotated with
+# @DisabledIfEnvironmentVariable(named = "CONDUCTOR_SERVER_TYPE", matches = "oss")
+# so they skip themselves when it's set (see the individual test files for
+# the empirically-confirmed gaps).
+#
+# The stack (Conductor OSS + Postgres) is defined in
+# scripts/docker-compose-oss.yaml and is torn down automatically on exit. The
+# image is always pulled before starting, since `latest` (the local default)
+# is a mutable tag and a cached copy would otherwise go stale silently.
+#
+# Usage:
+#   scripts/run-integration-oss.sh [--keep-up] [--version <tag>] [--include-gated] [-- gradle args]
+# Examples:
+#   scripts/run-integration-oss.sh
+#   scripts/run-integration-oss.sh --version 3.32.0-rc18
+#   scripts/run-integration-oss.sh --keep-up
+#   scripts/run-integration-oss.sh --include-gated       # also run tests normally skipped as Orkes-only
+#   scripts/run-integration-oss.sh -- --tests "*WorkflowClientTests"
+set -euo pipefail
+
+KEEP_UP=0
+INCLUDE_GATED=0
+extra=()
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --keep-up) KEEP_UP=1; shift ;;
+    --version) OSS_CONDUCTOR_VERSION="${2:?--version needs a tag}"; shift 2 ;;
+    --include-gated) INCLUDE_GATED=1; shift ;;
+    -h|--help)
+      echo "Usage: $0 [--keep-up] [--version <tag>] [--include-gated] [-- gradle args]"
+      exit 0
+      ;;
+    --) shift; extra=("$@"); break ;;
+    *) echo "Unknown argument: $1" >&2; exit 1 ;;
+  esac
+done
+
+export OSS_CONDUCTOR_VERSION="${OSS_CONDUCTOR_VERSION:-latest}"
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
+COMPOSE_FILE="${SCRIPT_DIR}/docker-compose-oss.yaml"
+cd "${REPO_ROOT}"
+
+compose() { docker compose -f "${COMPOSE_FILE}" "$@"; }
+
+cleanup() {
+  if [[ "${KEEP_UP}" == "1" ]]; then
+    echo "--keep-up set: leaving the OSS stack running. Tear down with:"
+    echo "  docker compose -f ${COMPOSE_FILE} down -v"
+    return
+  fi
+  echo "Tearing down Conductor OSS stack..."
+  compose down -v || true
+}
+trap cleanup EXIT
+
+echo "Using conductoross/conductor:${OSS_CONDUCTOR_VERSION}"
+
+# `docker compose up` only pulls an image when it is missing locally, so a
+# previously-cached `latest` (or any other mutable tag) would silently be
+# reused instead of getting the current version. Pull unconditionally so the
+# stack always reflects the tag we just printed.
+echo "Pulling conductoross/conductor:${OSS_CONDUCTOR_VERSION} to ensure it's current..."
+compose pull conductor-server
+
+echo "Starting Conductor OSS stack..."
+compose up -d
+
+echo "Waiting for Conductor to be healthy..."
+HEALTH_TIMEOUT="${HEALTH_TIMEOUT:-180}"
+deadline=$(( SECONDS + HEALTH_TIMEOUT ))
+until curl -sf http://localhost:8080/health >/dev/null 2>&1; do
+  if (( SECONDS >= deadline )); then
+    echo "Error: Conductor did not become healthy within ${HEALTH_TIMEOUT}s." >&2
+    compose logs conductor-server || true
+    exit 1
+  fi
+  sleep 5
+done
+echo "Conductor is up."
+
+export CONDUCTOR_SERVER_URL="http://localhost:8080/api"
+
+if [[ "${INCLUDE_GATED}" == "1" ]]; then
+  echo "--include-gated set: leaving CONDUCTOR_SERVER_TYPE unset, so tests normally" \
+       "skipped as Orkes-only will run against OSS too."
+  unset CONDUCTOR_SERVER_TYPE || true
+else
+  export CONDUCTOR_SERVER_TYPE="oss"
+fi
+
+
+# --rerun-tasks: the `test` task's up-to-date check only considers the compiled
+# test classpath, not env vars like CONDUCTOR_SERVER_URL/CONDUCTOR_SERVER_TYPE
+# or the state of the live server underneath. Without this, Gradle can report
+# BUILD SUCCESSFUL while silently reusing a stale cached result from a
+# previous run against a different server/tag/gating state instead of
+# actually executing anything.
+./gradlew :tests:test -PIntegrationTests --rerun-tasks ${extra[@]+"${extra[@]}"}

tests/src/test/java/io/orkes/conductor/client/ServiceRegistryClientTest.java

@@ -19,6 +19,7 @@
 
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.DisabledIfEnvironmentVariable;
 
 import com.netflix.conductor.common.model.OrkesCircuitBreakerConfig;
 import com.netflix.conductor.common.model.ServiceMethod;
@@ -30,6 +31,8 @@
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
+@DisabledIfEnvironmentVariable(named = "CONDUCTOR_SERVER_TYPE", matches = "oss",
+        disabledReason = "the Service Registry API (/registry/service) is not implemented by plain OSS Conductor, confirmed empirically (404 'No static resource api/registry/service')")
 public class ServiceRegistryClientTest {
 
     private static final String PROTO_FILENAME = "compiled.bin";

tests/src/test/java/io/orkes/conductor/client/WorkflowRetryTest.java

@@ -18,6 +18,7 @@
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.DisplayName;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.DisabledIfEnvironmentVariable;
 
 import com.netflix.conductor.common.metadata.tasks.TaskDef;
 import com.netflix.conductor.common.metadata.tasks.TaskResult;
@@ -36,6 +37,8 @@
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
+@DisabledIfEnvironmentVariable(named = "CONDUCTOR_SERVER_TYPE", matches = "oss",
+        disabledReason = "workflowClient.uploadCompletedWorkflows() (/workflow/document-store/upload) is not implemented by plain OSS Conductor, confirmed empirically (404 'No static resource api/workflow/document-store/upload')")
 public class WorkflowRetryTest {
     private final OrkesMetadataClient metadataClient;
     private final OrkesWorkflowClient workflowClient;

tests/src/test/java/io/orkes/conductor/client/http/AuthorizationClientTests.java

@@ -25,6 +25,7 @@
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.DisplayName;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.DisabledIfEnvironmentVariable;
 
 import com.netflix.conductor.client.exception.ConductorClientException;
 import com.netflix.conductor.common.metadata.workflow.WorkflowDef;
@@ -49,6 +50,8 @@
 import io.orkes.conductor.client.util.ClientTestUtil;
 import io.orkes.conductor.client.util.Commons;
 
+@DisabledIfEnvironmentVariable(named = "CONDUCTOR_SERVER_TYPE", matches = "oss",
+        disabledReason = "the Authorization APIs (applications/users/groups/roles/permissions) are not implemented by plain OSS Conductor, confirmed empirically (404 'No static resource api/applications|users|groups|...')")
 public class AuthorizationClientTests {
     private static AuthorizationClient authorizationClient;
     private static String applicationId;

tests/src/test/java/io/orkes/conductor/client/http/EnvironmentClientTests.java

@@ -19,11 +19,14 @@
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.DisabledIfEnvironmentVariable;
 
 import java.util.List;
 import java.util.Optional;
 import java.util.UUID;
 
+@DisabledIfEnvironmentVariable(named = "CONDUCTOR_SERVER_TYPE", matches = "oss",
+        disabledReason = "environment variable writes are not supported by plain OSS Conductor, confirmed empirically: OSS added read-only GET /environment in 3.32.0-rc.9 but PUT /environment/{key} still 405s ('Request method 'PUT' is not supported')")
 public class EnvironmentClientTests {
 
     private static EnvironmentClient envClient;

tests/src/test/java/io/orkes/conductor/client/http/EventClientTests.java

@@ -24,6 +24,7 @@
 
 import io.orkes.conductor.client.util.ClientTestUtil;
 import io.orkes.conductor.client.util.Commons;
+import io.orkes.conductor.client.util.TestUtil;
 
 public class EventClientTests {
     private static final String EVENT_NAME = "test_sdk_java_event_name";
@@ -35,9 +36,9 @@ void testEventHandler() {
         try {
             eventClient.unregisterEventHandler(EVENT_NAME);
         } catch (ConductorClientException e) {
-            if (e.getStatus() != 404) {
-                throw e;
-            }
+            // Best-effort cleanup: tolerate "doesn't exist" in whichever shape the server
+            // we're running against actually reports it.
+            TestUtil.assertNotFoundOrRethrow(e, "not found");
         }
         EventHandler eventHandler = getEventHandler();
         eventClient.registerEventHandler(eventHandler);