From 71dd1ade95a1a65a8f021cd79be1419a8c4fa2e0 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 26 Feb 2026 16:49:16 -0500 Subject: [PATCH 1/5] docker compose dev / prod, better tests, support custom mongo --- .github/workflows/tests.yml | 42 +++++ Dockerfile | 1 + README.md | 32 +++- bbot_server/modules/server/server_cli.py | 17 +- helm/Chart.yaml | 1 + helm/templates/server-deployment.yaml | 19 +- helm/templates/worker-deployment.yaml | 19 +- helm/values.yaml | 9 + pyproject.toml | 2 +- tests/test_docker_compose.py | 217 +++++++++++------------ 10 files changed, 237 insertions(+), 122 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4f50b4c9..a5620f73 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -47,3 +47,45 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} files: ./cov.xml verbose: true + + publish_pypi: + needs: test + runs-on: ubuntu-latest + if: github.event_name == 'push' && github.ref == 'refs/heads/stable' + continue-on-error: true + steps: + - uses: actions/checkout@v6 + - name: Install uv + uses: astral-sh/setup-uv@v7 + - name: Get version + id: calc_version + run: | + VERSION=$(grep '^version = ' pyproject.toml | sed 's/version = "\(.*\)"/\1/') + echo "VERSION=$VERSION" >> $GITHUB_OUTPUT + echo "Version: $VERSION" + - name: Build PyPI package + run: uv build + - name: Publish PyPI package + uses: pypa/gh-action-pypi-publish@release/v1.13 + with: + password: ${{ secrets.PYPI_API_TOKEN }} + outputs: + VERSION: ${{ steps.calc_version.outputs.VERSION }} + + tag_commit: + needs: publish_pypi + runs-on: ubuntu-latest + if: github.event_name == 'push' && github.ref == 'refs/heads/stable' + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + - name: Configure git + run: | + git config --local user.email "info@blacklanternsecurity.com" + git config --local user.name "GitHub Actions" + - name: Tag commit + run: | + VERSION="v${{ needs.publish_pypi.outputs.VERSION }}" + git tag -a "$VERSION" -m "Stable Release $VERSION" + git push origin "$VERSION" diff --git a/Dockerfile b/Dockerfile index 9301d714..58feef49 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,7 @@ WORKDIR /app COPY pyproject.toml uv.lock ./ RUN uv sync --frozen --no-install-project COPY . . +RUN cp bbot_server/defaults_docker.yml bbot_server/defaults.yml RUN uv sync --frozen RUN useradd -u 1000 -m bbot && chown -R bbot:bbot /app USER bbot diff --git a/README.md b/README.md index 4d0d6b48..7e6c3b63 100644 --- a/README.md +++ b/README.md @@ -24,28 +24,46 @@ BBOT Server is a database and multiplayer hub for all your [BBOT](https://github ## Installation ```bash -# clone the repo and cd into it -git clone git@github.com:blacklanternsecurity/bbot-server.git && cd bbot-server +# install with uv (recommended) +uv tool install bbot-server + +# or with pipx +pipx install bbot-server +``` + +To install `uv`, see the [uv installation docs](https://docs.astral.sh/uv/getting-started/installation/). + +### From source (for development) -# Install dependencies and create virtual environment +```bash +git clone git@github.com:blacklanternsecurity/bbot-server.git && cd bbot-server uv sync ``` Note: to update to the latest version, run `git pull && uv sync` in the `bbot-server` directory. -To install `uv`, see the [uv installation docs](https://docs.astral.sh/uv/getting-started/installation/). - -Commands shown below as `bbctl` can be run with `uv run bbctl` from the project directory, or you can activate the virtual environment first with `source .venv/bin/activate`. +When installed from source, commands shown below as `bbctl` can be run with `uv run bbctl` from the project directory, or you can activate the virtual environment first with `source .venv/bin/activate`. ## Start the server Note: this requires Docker and Docker Compose to be installed. ```bash -# Start BBOT server using Docker Compose +# Start BBOT server using Docker Compose (pulls from Docker Hub) bbctl server start ``` +### Development mode + +When working on the bbot-server source code, use `--dev` to build from source and mount your code for live reload: + +```bash +# From the bbot-server repo root +bbctl server --dev start +``` + +The `--dev` flag works with all server subcommands (`start`, `stop`, `down`, `logs`, `status`, `compose`). + ## Deploy with Helm (Kubernetes) BBOT Server can be deployed to Kubernetes using its official Helm chart. The chart deploys the API server, worker, MongoDB, and Redis. diff --git a/bbot_server/modules/server/server_cli.py b/bbot_server/modules/server/server_cli.py index 8c704c72..ba3763ad 100644 --- a/bbot_server/modules/server/server_cli.py +++ b/bbot_server/modules/server/server_cli.py @@ -7,7 +7,7 @@ from subprocess import run from contextlib import suppress -from bbot_server.config import BBOT_SERVER_CONFIG as bbcfg +from bbot_server.config import BBOT_SERVER_CONFIG as bbcfg, BBOT_SERVER_DIR from bbot_server.cli.base import BaseBBCTL, subcommand, Option, Annotated @@ -17,11 +17,24 @@ class ServerCTL(BaseBBCTL): short_help = "Start/stop BBOT server and manage API keys" attach_to = "bbctl" + _invoke_without_command = True + _no_args_is_help = True + def setup(self): self._docker_command = None - self.docker_compose_dir = Path(__file__).parent.parent + self.docker_compose_dir = BBOT_SERVER_DIR self.docker_compose_file = self.docker_compose_dir / "compose.yml" + def main( + self, + dev: Annotated[ + bool, Option("--dev", "-d", help="Use dev compose file (builds from source, mounts code for live reload)") + ] = False, + ): + if dev: + self.docker_compose_dir = BBOT_SERVER_DIR.parent + self.docker_compose_file = self.docker_compose_dir / "compose.yml" + @subcommand(help="Start BBOT server") def start( self, diff --git a/helm/Chart.yaml b/helm/Chart.yaml index 42742ca6..17b13efa 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -8,6 +8,7 @@ dependencies: - name: mongodb version: "18.x.x" repository: "https://charts.bitnami.com/bitnami" + condition: mongodb.enabled - name: redis version: "23.x.x" repository: "https://charts.bitnami.com/bitnami" diff --git a/helm/templates/server-deployment.yaml b/helm/templates/server-deployment.yaml index 7d14183c..14a3f82c 100644 --- a/helm/templates/server-deployment.yaml +++ b/helm/templates/server-deployment.yaml @@ -36,12 +36,14 @@ spec: name: http protocol: TCP env: + {{- if .Values.mongodb.enabled }} # Internal password references (used for URI interpolation below) - name: MONGO_PASSWORD valueFrom: secretKeyRef: name: {{ .Values.mongodb.auth.existingSecret | default (printf "%s-mongodb" .Release.Name) | quote }} key: mongodb-root-password + {{- end }} - name: REDIS_PASSWORD valueFrom: secretKeyRef: @@ -57,17 +59,32 @@ spec: secretKeyRef: name: {{ .Values.secrets.existingApiKeySecret | default (printf "%s-api-key" .Release.Name) | quote }} key: api-key - # MongoDB URIs (credentials injected via $(VAR) interpolation) + # MongoDB URIs + {{- if .Values.mongodb.enabled }} - name: BBOT_SERVER_EVENT_STORE__URI value: "mongodb://root:$(MONGO_PASSWORD)@{{ .Release.Name }}-mongodb:27017/bbot?authSource=admin" + {{- else }} + - name: BBOT_SERVER_EVENT_STORE__URI + value: {{ .Values.externalMongodb.uri | quote }} + {{- end }} - name: BBOT_SERVER_EVENT_STORE__COLLECTION_PREFIX value: "eventstore__" + {{- if .Values.mongodb.enabled }} - name: BBOT_SERVER_ASSET_STORE__URI value: "mongodb://root:$(MONGO_PASSWORD)@{{ .Release.Name }}-mongodb:27017/bbot?authSource=admin" + {{- else }} + - name: BBOT_SERVER_ASSET_STORE__URI + value: {{ .Values.externalMongodb.uri | quote }} + {{- end }} - name: BBOT_SERVER_ASSET_STORE__COLLECTION_PREFIX value: "assetstore__" + {{- if .Values.mongodb.enabled }} - name: BBOT_SERVER_USER_STORE__URI value: "mongodb://root:$(MONGO_PASSWORD)@{{ .Release.Name }}-mongodb:27017/bbot?authSource=admin" + {{- else }} + - name: BBOT_SERVER_USER_STORE__URI + value: {{ .Values.externalMongodb.uri | quote }} + {{- end }} - name: BBOT_SERVER_USER_STORE__COLLECTION_PREFIX value: "userstore__" # Redis URI diff --git a/helm/templates/worker-deployment.yaml b/helm/templates/worker-deployment.yaml index 83314ae0..50e90dec 100644 --- a/helm/templates/worker-deployment.yaml +++ b/helm/templates/worker-deployment.yaml @@ -49,12 +49,14 @@ spec: imagePullPolicy: {{ .Values.image.pullPolicy }} command: ["bbctl", "server", "start", "--worker-only"] env: + {{- if .Values.mongodb.enabled }} # Internal password references (used for URI interpolation below) - name: MONGO_PASSWORD valueFrom: secretKeyRef: name: {{ .Values.mongodb.auth.existingSecret | default (printf "%s-mongodb" .Release.Name) | quote }} key: mongodb-root-password + {{- end }} - name: REDIS_PASSWORD valueFrom: secretKeyRef: @@ -70,17 +72,32 @@ spec: secretKeyRef: name: {{ .Values.secrets.existingApiKeySecret | default (printf "%s-api-key" .Release.Name) | quote }} key: api-key - # MongoDB URIs (credentials injected via $(VAR) interpolation) + # MongoDB URIs + {{- if .Values.mongodb.enabled }} - name: BBOT_SERVER_EVENT_STORE__URI value: "mongodb://root:$(MONGO_PASSWORD)@{{ .Release.Name }}-mongodb:27017/bbot?authSource=admin" + {{- else }} + - name: BBOT_SERVER_EVENT_STORE__URI + value: {{ .Values.externalMongodb.uri | quote }} + {{- end }} - name: BBOT_SERVER_EVENT_STORE__COLLECTION_PREFIX value: "eventstore__" + {{- if .Values.mongodb.enabled }} - name: BBOT_SERVER_ASSET_STORE__URI value: "mongodb://root:$(MONGO_PASSWORD)@{{ .Release.Name }}-mongodb:27017/bbot?authSource=admin" + {{- else }} + - name: BBOT_SERVER_ASSET_STORE__URI + value: {{ .Values.externalMongodb.uri | quote }} + {{- end }} - name: BBOT_SERVER_ASSET_STORE__COLLECTION_PREFIX value: "assetstore__" + {{- if .Values.mongodb.enabled }} - name: BBOT_SERVER_USER_STORE__URI value: "mongodb://root:$(MONGO_PASSWORD)@{{ .Release.Name }}-mongodb:27017/bbot?authSource=admin" + {{- else }} + - name: BBOT_SERVER_USER_STORE__URI + value: {{ .Values.externalMongodb.uri | quote }} + {{- end }} - name: BBOT_SERVER_USER_STORE__COLLECTION_PREFIX value: "userstore__" # Redis URI diff --git a/helm/values.yaml b/helm/values.yaml index 8a572f94..c63488be 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -63,6 +63,8 @@ secrets: # MongoDB (Bitnami subchart) # ============================================================================= mongodb: + # -- Set to false to disable the built-in MongoDB and use an external instance + enabled: true architecture: standalone auth: enabled: true @@ -72,6 +74,13 @@ mongodb: size: 50Gi resourcePolicy: keep +# ============================================================================= +# External MongoDB (used when mongodb.enabled is false) +# ============================================================================= +externalMongodb: + # -- Full MongoDB connection URI (e.g. "mongodb://user:pass@host:27017/bbot?authSource=admin") + uri: "" + # ============================================================================= # Redis (Bitnami subchart) # ============================================================================= diff --git a/pyproject.toml b/pyproject.toml index 08fc8a3a..d1ffc9f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "bbot-server" -version = "0.1.2" +version = "0.1.3" description = "" authors = [{name = "TheTechromancer"}] license = "AGPL-3.0" diff --git a/tests/test_docker_compose.py b/tests/test_docker_compose.py index 130ca845..6f7b364a 100644 --- a/tests/test_docker_compose.py +++ b/tests/test_docker_compose.py @@ -23,37 +23,53 @@ def reset_config_file(): copyfile(Path(__file__).parent / "test_config_docker.yml", custom_config_file) +def bbctl_command(config_file=None): + cmd = ["uv", "run", "bbctl"] + if config_file: + cmd += ["--config", str(config_file)] + return cmd + + +def server_args(dev=True): + """Return the server subcommand args with or without --dev.""" + if dev: + return ["server", "--dev"] + return ["server"] + + @contextmanager -def docker_test_env(reset_config=True, docker_down_first=True, cleanup_config=True): +def docker_test_env(dev=True, reset_config=True, docker_down_first=True, cleanup_config=True): """ Context manager for docker compose tests. Args: + dev: Use dev compose (build from source) vs production compose (pull from Docker Hub) reset_config: Reset config file at start docker_down_first: Run docker compose down before test cleanup_config: Delete custom config file after test """ + srv = server_args(dev) + # Setup if reset_config: reset_config_file() if docker_down_first: - result = subprocess.run( - ["docker", "compose", "down"], + subprocess.run( + bbctl_command(custom_config_file) + srv + ["down"], cwd=project_root, ) - assert result.returncode == 0 try: yield finally: # Cleanup - print logs if test failed if sys.exc_info()[0] is not None: - print_docker_logs() + print_docker_logs(dev=dev) # Always stop docker compose subprocess.run( - ["docker", "compose", "down"], + bbctl_command(custom_config_file) + srv + ["down"], cwd=project_root, ) @@ -62,21 +78,13 @@ def docker_test_env(reset_config=True, docker_down_first=True, cleanup_config=Tr custom_config_file.unlink(missing_ok=True) -# we typically only want to run this on CI -# to avoid messing with the user's existing bbot server data / api keys -pytestmark = pytest.mark.skipif( - os.environ.get("BBOT_SERVER_TEST_DOCKER_COMPOSE", "false").lower() != "true", - reason="BBOT_SERVER_TEST_DOCKER_COMPOSE is not set to true", -) - - -def print_docker_logs(): +def print_docker_logs(dev=True): """Helper to print docker compose logs""" print("\n" + "=" * 80) print("TEST FAILED - Docker Compose Logs:") print("=" * 80) result = subprocess.run( - ["docker", "compose", "logs", "--no-color"], + bbctl_command(custom_config_file) + server_args(dev) + ["logs"], cwd=project_root, capture_output=True, text=True, @@ -91,18 +99,47 @@ def print_docker_logs(): print("=" * 80 + "\n") +def start_server(bbctl_cmd, dev=True, extra_args=None): + """Start the server and return the result. Asserts success.""" + cmd = bbctl_cmd + server_args(dev) + ["start"] + (extra_args or []) + result = subprocess.run( + cmd, + cwd=project_root, + capture_output=True, + text=True, + ) + assert result.returncode == 0, f"Failed to start server: {result.stderr}" + return result + + +def wait_for_server(bbctl_cmd, timeout_seconds=60): + """Wait for the server to become healthy (asset stats returns 200).""" + for _ in range(timeout_seconds * 2): + result = subprocess.run( + bbctl_cmd + ["asset", "stats"], + cwd=project_root, + capture_output=True, + text=True, + ) + if result.returncode == 0: + return result + time.sleep(0.5) + assert False, f"Server did not become healthy within {timeout_seconds}s, stdout: {result.stdout}, stderr: {result.stderr}" + + +# we typically only want to run this on CI +# to avoid messing with the user's existing bbot server data / api keys +pytestmark = pytest.mark.skipif( + os.environ.get("BBOT_SERVER_TEST_DOCKER_COMPOSE", "false").lower() != "true", + reason="BBOT_SERVER_TEST_DOCKER_COMPOSE is not set to true", +) + + def test_docker_compose_userexperience(): """ A basic up/down test to make sure bbot server is working with docker compose """ - with docker_test_env(reset_config=False, docker_down_first=True): - # build it - # result = subprocess.run( - # ["docker", "compose", "build"], - # cwd=project_root, - # ) - # assert result.returncode == 0 - + with docker_test_env(dev=True, reset_config=False, docker_down_first=True): docker_compose_file = project_root / "compose.yml" assert docker_compose_file.exists() @@ -110,7 +147,7 @@ def test_docker_compose_userexperience(): custom_config_file.unlink(missing_ok=True) custom_config_file.write_text("testasdf: testasdf") - BBCTL_COMMAND = ["uv", "run", "bbctl", "--config", str(custom_config_file)] + BBCTL_COMMAND = bbctl_command(custom_config_file) # current config should have no api key or valid api keys result = subprocess.run( @@ -162,13 +199,7 @@ def test_docker_compose_userexperience(): assert not config.get("api_keys", []) # start docker compose - result = subprocess.run( - BBCTL_COMMAND + ["server", "start"], - cwd=project_root, - capture_output=True, - text=True, - ) - assert result.returncode == 0 + result = start_server(BBCTL_COMMAND, dev=True) assert "First run detected" in result.stderr # is api key listed now? @@ -191,23 +222,7 @@ def test_docker_compose_userexperience(): our_api_key = our_api_key[0] assert docker_api_key[:20] == our_api_key[:20] - for _ in range(120): - # we should be able to list assets now - command = BBCTL_COMMAND + ["asset", "stats"] - print(f"command: {' '.join(command)}") - result = subprocess.run( - command, - cwd=project_root, - capture_output=True, - text=True, - ) - print(f"result: {result.returncode}, stdout: {result.stdout}, stderr: {result.stderr}") - assert not "Invalid API key" in result.stderr - if result.returncode == 0 and json.loads(result.stdout) == {}: - break - time.sleep(0.5) - else: - assert False, f"Failed to list assets, stdout: {result.stdout}, stderr: {result.stderr}" + wait_for_server(BBCTL_COMMAND) def test_docker_compose_custom_config(): @@ -226,7 +241,7 @@ def test_docker_compose_custom_config(): custom_config_file.write_text("test1234: test4321") # can we read it outside of docker compose? - BBCTL_COMMAND = ["uv", "run", "bbctl", "--config", str(custom_config_file)] + BBCTL_COMMAND = bbctl_command(custom_config_file) result = subprocess.run( BBCTL_COMMAND + ["--current-config"], cwd=project_root, @@ -239,7 +254,7 @@ def test_docker_compose_custom_config(): # if we don't pass --config, the docker container should use the default config result = subprocess.run( - ["uv", "run", "bbctl", "server", "compose", "run", "server", "bbctl", "--current-config"], + ["uv", "run", "bbctl", "server", "--dev", "compose", "run", "server", "bbctl", "--current-config"], cwd=project_root, capture_output=True, text=True, @@ -250,7 +265,7 @@ def test_docker_compose_custom_config(): # if we do pass --config, it should use the custom config result = subprocess.run( - BBCTL_COMMAND + ["server", "compose", "run", "server", "bbctl", "--current-config"], + BBCTL_COMMAND + ["server", "--dev", "compose", "run", "server", "bbctl", "--current-config"], cwd=project_root, capture_output=True, text=True, @@ -263,35 +278,18 @@ def test_docker_compose_custom_config(): # test the --listen flag and --port flags def test_docker_compose_listening_interface(): - with docker_test_env(reset_config=True, docker_down_first=True): + with docker_test_env(dev=True, reset_config=True, docker_down_first=True): # create a blank config file just for this test with open(custom_config_file, "a") as f: f.write("\nurl: http://127.0.0.1:8807/v1/\n") - BBCTL_COMMAND = ["uv", "run", "bbctl", "--config", str(custom_config_file)] + BBCTL_COMMAND = bbctl_command(custom_config_file) # start docker compose - result = subprocess.run( - BBCTL_COMMAND + ["server", "start"], - cwd=project_root, - capture_output=True, - text=True, - ) - assert result.returncode == 0 + start_server(BBCTL_COMMAND, dev=True) # we should be able to list assets - for _ in range(120): - result = subprocess.run( - BBCTL_COMMAND + ["asset", "stats"], - cwd=project_root, - capture_output=True, - text=True, - ) - if result.returncode == 0 and json.loads(result.stdout) == {}: - break - time.sleep(0.5) - else: - assert False, f"Failed to list assets, stdout: {result.stdout}, stderr: {result.stderr}" + wait_for_server(BBCTL_COMMAND) # replace the url key with 127.0.0.2 config_content = custom_config_file.read_text() @@ -308,34 +306,16 @@ def test_docker_compose_listening_interface(): assert "Error making GET request" in result.stderr # docker compose down - result = subprocess.run( - ["docker", "compose", "down"], + subprocess.run( + BBCTL_COMMAND + server_args(dev=True) + ["down"], cwd=project_root, ) - assert result.returncode == 0 # this time, we change up the interface - result = subprocess.run( - BBCTL_COMMAND + ["server", "start", "--listen", "127.0.0.2"], - cwd=project_root, - capture_output=True, - text=True, - ) - assert result.returncode == 0 + start_server(BBCTL_COMMAND, dev=True, extra_args=["--listen", "127.0.0.2"]) # we should be able to list assets - for _ in range(120): - result = subprocess.run( - BBCTL_COMMAND + ["asset", "stats"], - cwd=project_root, - capture_output=True, - text=True, - ) - if result.returncode == 0 and json.loads(result.stdout) == {}: - break - time.sleep(0.5) - else: - assert False, f"Failed to list assets, stdout: {result.stdout}, stderr: {result.stderr}" + wait_for_server(BBCTL_COMMAND) # replace the url key with 127.0.0.1 config_content = custom_config_file.read_text() @@ -358,15 +338,11 @@ def test_docker_compose_authentication(): custom_config_file.unlink(missing_ok=True) custom_config_file.write_text("") - BBCTL_COMMAND = ["uv", "run", "bbctl", "--config", str(custom_config_file)] + BBCTL_COMMAND = bbctl_command(custom_config_file) - with docker_test_env(reset_config=True, docker_down_first=True): + with docker_test_env(dev=True, reset_config=True, docker_down_first=True): # start docker compose - result = subprocess.run( - BBCTL_COMMAND + ["server", "start"], - cwd=project_root, - ) - assert result.returncode == 0 + start_server(BBCTL_COMMAND, dev=True) for _ in range(120): # docker should detect first run and write a new api key to the custom config file @@ -377,7 +353,7 @@ def test_docker_compose_authentication(): break time.sleep(0.5) else: - assert False, f"Failed to get api key, stdout: {result.stdout}, stderr: {result.stderr}" + assert False, f"Failed to get api key" # without the API key, auth should fail for _ in range(120): @@ -412,14 +388,10 @@ def test_docker_compose_no_authentication(): """ custom_config_file.unlink(missing_ok=True) - BBCTL_COMMAND = ["uv", "run", "bbctl", "--config", str(custom_config_file)] + BBCTL_COMMAND = bbctl_command(custom_config_file) - with docker_test_env(reset_config=True, docker_down_first=True): - result = subprocess.run( - BBCTL_COMMAND + ["server", "start", "--no-authentication"], - cwd=project_root, - ) - assert result.returncode == 0 + with docker_test_env(dev=True, reset_config=True, docker_down_first=True): + start_server(BBCTL_COMMAND, dev=True, extra_args=["--no-authentication"]) for _ in range(120): try: @@ -433,3 +405,28 @@ def test_docker_compose_no_authentication(): assert False, ( f"Expected 200 OK without auth, got: {response.status_code if 'response' in locals() else 'no response'}" ) + + +def test_docker_compose_production(): + """ + Smoke test for the production compose (pulls from Docker Hub). + Starts the server, waits for healthy, queries asset stats. + """ + with docker_test_env(dev=False, reset_config=True, docker_down_first=True): + BBCTL_COMMAND = bbctl_command(custom_config_file) + + # start production compose (no --dev, pulls from Docker Hub) + result = start_server(BBCTL_COMMAND, dev=False) + assert "First run detected" in result.stderr + + # wait for server to be healthy + wait_for_server(BBCTL_COMMAND) + + # verify asset stats returns successfully + result = subprocess.run( + BBCTL_COMMAND + ["asset", "stats"], + cwd=project_root, + capture_output=True, + text=True, + ) + assert result.returncode == 0 From 8d47d49b2e7850fce75acdd8af082f7f5f55d25a Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 26 Feb 2026 16:49:25 -0500 Subject: [PATCH 2/5] compose file --- bbot_server/compose.yml | 43 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 bbot_server/compose.yml diff --git a/bbot_server/compose.yml b/bbot_server/compose.yml new file mode 100644 index 00000000..fdeb93f7 --- /dev/null +++ b/bbot_server/compose.yml @@ -0,0 +1,43 @@ +x-bbot-server-base: &bbot-server-base + image: blacklanternsecurity/bbot-server:stable + restart: unless-stopped + volumes: + - "${BBOT_SERVER_CONFIG:-~/.config/bbot_server/config.yml}:/home/bbot/.config/bbot_server/config.yml" + +services: + server: + <<: *bbot-server-base + ports: + - "${BBOT_LISTEN_ADDRESS:-127.0.0.1}:${BBOT_PORT:-8807}:8807" + command: ["bbctl", "server", "start", "--api-only", "--listen", "0.0.0.0"] + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8807/v1/docs"] + interval: 10s + timeout: 5s + retries: 3 + start_period: 30s + environment: + - BBOT_SERVER_AUTH_ENABLED=${BBOT_SERVER_AUTH_ENABLED:-true} + depends_on: + - mongodb + - redis + + worker: + <<: *bbot-server-base + command: ["bbctl", "server", "start", "--worker-only"] + depends_on: + server: + condition: service_healthy + + mongodb: + image: mongo:latest + restart: unless-stopped + volumes: + - bbot-mongodb:/data/db + + redis: + image: redis:latest + restart: unless-stopped + +volumes: + bbot-mongodb: From 2dc4f7a7e75a71af9d51e5ba33a54eb331c4fc78 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 26 Feb 2026 16:52:36 -0500 Subject: [PATCH 3/5] ruffed --- bbot_server/modules/server/server_cli.py | 1 - tests/test_docker_compose.py | 5 +++-- uv.lock | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bbot_server/modules/server/server_cli.py b/bbot_server/modules/server/server_cli.py index ba3763ad..02d092d8 100644 --- a/bbot_server/modules/server/server_cli.py +++ b/bbot_server/modules/server/server_cli.py @@ -3,7 +3,6 @@ import typer import asyncio import subprocess -from pathlib import Path from subprocess import run from contextlib import suppress diff --git a/tests/test_docker_compose.py b/tests/test_docker_compose.py index 6f7b364a..4b170be2 100644 --- a/tests/test_docker_compose.py +++ b/tests/test_docker_compose.py @@ -2,7 +2,6 @@ import os import sys -import json import yaml import time import httpx @@ -124,7 +123,9 @@ def wait_for_server(bbctl_cmd, timeout_seconds=60): if result.returncode == 0: return result time.sleep(0.5) - assert False, f"Server did not become healthy within {timeout_seconds}s, stdout: {result.stdout}, stderr: {result.stderr}" + assert False, ( + f"Server did not become healthy within {timeout_seconds}s, stdout: {result.stdout}, stderr: {result.stderr}" + ) # we typically only want to run this on CI diff --git a/uv.lock b/uv.lock index ae9c91b1..7e816e02 100644 --- a/uv.lock +++ b/uv.lock @@ -178,7 +178,7 @@ dependencies = [ [[package]] name = "bbot-server" -version = "0.1.1" +version = "0.1.3" source = { editable = "." } dependencies = [ { name = "bbot" }, From 854e1d675d99f6048c2f6c003abb17b9a8b1ba9d Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Fri, 27 Feb 2026 10:32:58 -0500 Subject: [PATCH 4/5] unify docker and helm tests --- .github/workflows/docker-tests.yml | 92 +++++++++++++++++++++++++++- .github/workflows/helm-tests.yml | 97 ------------------------------ tests/test_docker_compose.py | 45 ++++++++++---- 3 files changed, 125 insertions(+), 109 deletions(-) delete mode 100644 .github/workflows/helm-tests.yml diff --git a/.github/workflows/docker-tests.yml b/.github/workflows/docker-tests.yml index cf616c28..3f391892 100644 --- a/.github/workflows/docker-tests.yml +++ b/.github/workflows/docker-tests.yml @@ -25,9 +25,42 @@ jobs: run: | BBOT_SERVER_TEST_DOCKER_COMPOSE=true uv run pytest --disable-warnings --log-cli-level=INFO -k test_docker_compose + helm-test: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.x' + + - name: Install Python dependencies + run: pip install requests + + - name: Install kubectl + uses: azure/setup-kubectl@v4 + + - name: Install Helm + uses: azure/setup-helm@v4 + + - name: Install minikube + uses: medyagh/setup-minikube@latest + + - name: Run helm deployment tests + run: python helm_deployment_test.py + + - name: Cleanup minikube + if: always() + run: minikube delete + docker-publish: runs-on: ubuntu-latest - needs: docker-test + needs: + - helm-test + - docker-test if: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/stable' || github.ref == 'refs/heads/dev') }} steps: @@ -104,3 +137,60 @@ jobs: done echo "Cleanup completed for bbot-server." + + + helm-publish: + runs-on: ubuntu-latest + needs: + - helm-test + - docker-test + if: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/stable' || github.ref == 'refs/heads/dev') }} + + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Install uv + uses: astral-sh/setup-uv@v7 + + - name: Get version from pyproject.toml + id: version + run: | + VERSION=$(uv version --short) + echo "version=$VERSION" >> $GITHUB_OUTPUT + + - name: Install Helm + uses: azure/setup-helm@v4 + + - name: Update Chart.yaml version and appVersion + run: | + VERSION="${{ steps.version.outputs.version }}" + CHART_VERSION="$VERSION" + IMAGE_TAG="stable" + if [ "${{ github.ref }}" = "refs/heads/dev" ]; then + CHART_VERSION="${VERSION}-dev" + IMAGE_TAG="dev" + fi + sed -i "s/^version:.*/version: ${CHART_VERSION}/" helm/Chart.yaml + sed -i "s/^appVersion:.*/appVersion: \"${VERSION}\"/" helm/Chart.yaml + # ensure the packaged chart defaults to the correct docker image tag + sed -i "s/^ tag:.*/ tag: \"${IMAGE_TAG}\"/" helm/values.yaml + echo "Chart version: $CHART_VERSION" + echo "App version: $VERSION" + echo "Image tag: $IMAGE_TAG" + cat helm/Chart.yaml + + - name: Add Helm repositories + run: helm repo add bitnami https://charts.bitnami.com/bitnami + + - name: Build Helm dependencies + run: helm dependency build helm/ + + - name: Package Helm chart + run: helm package helm/ + + - name: Login to Docker Hub OCI registry + run: echo "${{ secrets.DOCKER_TOKEN }}" | helm registry login registry-1.docker.io -u blacklanternsecurity --password-stdin + + - name: Push Helm chart to Docker Hub + run: helm push bbot-server-*.tgz oci://registry-1.docker.io/blacklanternsecurity diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml deleted file mode 100644 index 7da5df53..00000000 --- a/.github/workflows/helm-tests.yml +++ /dev/null @@ -1,97 +0,0 @@ -name: helm tests -on: - push: - branches: - - stable - - dev - pull_request: - -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - helm-test: - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v6 - - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: '3.x' - - - name: Install Python dependencies - run: pip install requests - - - name: Install kubectl - uses: azure/setup-kubectl@v4 - - - name: Install Helm - uses: azure/setup-helm@v4 - - - name: Install minikube - uses: medyagh/setup-minikube@latest - - - name: Run helm deployment tests - run: python helm_deployment_test.py - - - name: Cleanup minikube - if: always() - run: minikube delete - - helm-publish: - runs-on: ubuntu-latest - needs: helm-test - if: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/stable' || github.ref == 'refs/heads/dev') }} - - steps: - - name: Checkout code - uses: actions/checkout@v6 - - - name: Install uv - uses: astral-sh/setup-uv@v7 - - - name: Get version from pyproject.toml - id: version - run: | - VERSION=$(uv version --short) - echo "version=$VERSION" >> $GITHUB_OUTPUT - - - name: Install Helm - uses: azure/setup-helm@v4 - - - name: Update Chart.yaml version and appVersion - run: | - VERSION="${{ steps.version.outputs.version }}" - CHART_VERSION="$VERSION" - IMAGE_TAG="stable" - if [ "${{ github.ref }}" = "refs/heads/dev" ]; then - CHART_VERSION="${VERSION}-dev" - IMAGE_TAG="dev" - fi - sed -i "s/^version:.*/version: ${CHART_VERSION}/" helm/Chart.yaml - sed -i "s/^appVersion:.*/appVersion: \"${VERSION}\"/" helm/Chart.yaml - # ensure the packaged chart defaults to the correct docker image tag - sed -i "s/^ tag:.*/ tag: \"${IMAGE_TAG}\"/" helm/values.yaml - echo "Chart version: $CHART_VERSION" - echo "App version: $VERSION" - echo "Image tag: $IMAGE_TAG" - cat helm/Chart.yaml - - - name: Add Helm repositories - run: helm repo add bitnami https://charts.bitnami.com/bitnami - - - name: Build Helm dependencies - run: helm dependency build helm/ - - - name: Package Helm chart - run: helm package helm/ - - - name: Login to Docker Hub OCI registry - run: echo "${{ secrets.DOCKER_TOKEN }}" | helm registry login registry-1.docker.io -u blacklanternsecurity --password-stdin - - - name: Push Helm chart to Docker Hub - run: helm push bbot-server-*.tgz oci://registry-1.docker.io/blacklanternsecurity diff --git a/tests/test_docker_compose.py b/tests/test_docker_compose.py index 4b170be2..36c6feb5 100644 --- a/tests/test_docker_compose.py +++ b/tests/test_docker_compose.py @@ -78,23 +78,46 @@ def docker_test_env(dev=True, reset_config=True, docker_down_first=True, cleanup def print_docker_logs(dev=True): - """Helper to print docker compose logs""" + """Helper to print docker compose logs and container state on failure""" print("\n" + "=" * 80) - print("TEST FAILED - Docker Compose Logs:") + print("TEST FAILED - Diagnostics:") print("=" * 80) + + # Show all docker containers (running and stopped) + print("\n--- docker ps -a ---") + result = subprocess.run( + ["docker", "ps", "-a"], + capture_output=True, + text=True, + ) + print(result.stdout or "(empty)") + if result.stderr: + print(f"stderr: {result.stderr}") + + # Show docker compose ps from the relevant compose directory + print("\n--- docker compose ps (from compose dir) ---") + result = subprocess.run( + bbctl_command(custom_config_file) + server_args(dev) + ["status"], + cwd=project_root, + capture_output=True, + text=True, + ) + print(result.stdout or "(empty)") + if result.stderr: + print(f"stderr: {result.stderr}") + + # Show docker compose logs + print("\n--- docker compose logs ---") result = subprocess.run( bbctl_command(custom_config_file) + server_args(dev) + ["logs"], cwd=project_root, capture_output=True, text=True, ) - if result.returncode == 0: - print(result.stdout) - if result.stderr: - print("\nSTDERR:") - print(result.stderr) - else: - print(f"Failed to get docker compose logs (exit code {result.returncode}): {result.stderr}") + print(result.stdout or "(empty)") + if result.stderr: + print(f"stderr: {result.stderr}") + print("=" * 80 + "\n") @@ -420,8 +443,8 @@ def test_docker_compose_production(): result = start_server(BBCTL_COMMAND, dev=False) assert "First run detected" in result.stderr - # wait for server to be healthy - wait_for_server(BBCTL_COMMAND) + # wait for server to be healthy (longer timeout for image pulls) + wait_for_server(BBCTL_COMMAND, timeout_seconds=120) # verify asset stats returns successfully result = subprocess.run( From d479478c5cf85f86f624d78af3508547925a89cf Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Fri, 27 Feb 2026 11:17:14 -0500 Subject: [PATCH 5/5] fix prod image --- tests/test_docker_compose.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_docker_compose.py b/tests/test_docker_compose.py index 36c6feb5..5325f79e 100644 --- a/tests/test_docker_compose.py +++ b/tests/test_docker_compose.py @@ -436,6 +436,16 @@ def test_docker_compose_production(): Smoke test for the production compose (pulls from Docker Hub). Starts the server, waits for healthy, queries asset stats. """ + # Build the image locally and tag it so the production compose can use it + # without pulling from Docker Hub (the tag may not exist yet). + result = subprocess.run( + ["docker", "build", "-t", "blacklanternsecurity/bbot-server:stable", "."], + cwd=project_root, + capture_output=True, + text=True, + ) + assert result.returncode == 0, f"Failed to build production image: {result.stderr}" + with docker_test_env(dev=False, reset_config=True, docker_down_first=True): BBCTL_COMMAND = bbctl_command(custom_config_file)