diff --git a/.github/workflows/docker-tests.yml b/.github/workflows/docker-tests.yml index d83a0776..3f391892 100644 --- a/.github/workflows/docker-tests.yml +++ b/.github/workflows/docker-tests.yml @@ -25,10 +25,43 @@ jobs: run: | BBOT_SERVER_TEST_DOCKER_COMPOSE=true uv run pytest --disable-warnings --log-cli-level=INFO -k test_docker_compose + helm-test: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.x' + + - name: Install Python dependencies + run: pip install requests + + - name: Install kubectl + uses: azure/setup-kubectl@v4 + + - name: Install Helm + uses: azure/setup-helm@v4 + + - name: Install minikube + uses: medyagh/setup-minikube@latest + + - name: Run helm deployment tests + run: python helm_deployment_test.py + + - name: Cleanup minikube + if: always() + run: minikube delete + docker-publish: runs-on: ubuntu-latest - needs: docker-test - if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/stable' }} + needs: + - helm-test + - docker-test + if: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/stable' || github.ref == 'refs/heads/dev') }} steps: - name: Checkout code @@ -61,39 +94,103 @@ jobs: images: blacklanternsecurity/bbot-server tags: | type=ref,event=branch - type=ref,event=pr type=sha,prefix=sha- - type=raw,value=latest,enable={{is_default_branch}} - type=raw,value=v${{ steps.version.outputs.major }} - type=raw,value=v${{ steps.version.outputs.minor }} - type=raw,value=v${{ steps.version.outputs.version }} + type=raw,value=latest,enable=${{ github.ref == 'refs/heads/stable' }} + type=raw,value=v${{ steps.version.outputs.major }},enable=${{ github.ref == 'refs/heads/stable' }} + type=raw,value=v${{ steps.version.outputs.minor }},enable=${{ github.ref == 'refs/heads/stable' }} + type=raw,value=v${{ steps.version.outputs.version }},enable=${{ github.ref == 'refs/heads/stable' }} - name: Build and push Docker image uses: docker/build-push-action@v6 with: + context: . push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max - - name: Clean up old Docker Hub tags (up to 50 most recent tags plus 'latest') + - name: Clean up old Docker Hub tags (keep 50 most recent plus protected tags) run: | # Install jq for JSON processing sudo apt-get update && sudo apt-get install -y jq echo "Cleaning up bbot-server tags..." + # Protected tags that should never be deleted + PROTECTED="latest stable dev v${{ steps.version.outputs.major }} v${{ steps.version.outputs.minor }} v${{ steps.version.outputs.version }}" + tags_response=$(curl -s -H "Authorization: Bearer ${{ secrets.DOCKER_TOKEN }}" \ - "https://hub.docker.com/v2/repositories/bbot-server}/tags/?page_size=100") + "https://hub.docker.com/v2/repositories/blacklanternsecurity/bbot-server/tags/?page_size=100") - tags_to_delete=$(echo "$tags_response" | jq -r '.results[] | select(.name != "latest") | [.last_updated, .name] | @tsv' | \ - sort -r | tail -n +51 | cut -f2) + tags_to_delete=$(echo "$tags_response" | jq -r '.results[].name' | sort -r | tail -n +51) for tag in $tags_to_delete; do + # Skip protected tags + if echo "$PROTECTED" | grep -qw "$tag"; then + echo "Skipping protected tag: $tag" + continue + fi echo "Deleting bbot-server tag: $tag" curl -X DELETE -H "Authorization: Bearer ${{ secrets.DOCKER_TOKEN }}" \ - "https://hub.docker.com/v2/repositories/bbot-server/tags/$tag/" + "https://hub.docker.com/v2/repositories/blacklanternsecurity/bbot-server/tags/$tag/" done - echo "Cleanup completed for bbot-server. Kept 50 most recent tags plus 'latest'." + echo "Cleanup completed for bbot-server." + + + helm-publish: + runs-on: ubuntu-latest + needs: + - helm-test + - docker-test + if: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/stable' || github.ref == 'refs/heads/dev') }} + + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Install uv + uses: astral-sh/setup-uv@v7 + + - name: Get version from pyproject.toml + id: version + run: | + VERSION=$(uv version --short) + echo "version=$VERSION" >> $GITHUB_OUTPUT + + - name: Install Helm + uses: azure/setup-helm@v4 + + - name: Update Chart.yaml version and appVersion + run: | + VERSION="${{ steps.version.outputs.version }}" + CHART_VERSION="$VERSION" + IMAGE_TAG="stable" + if [ "${{ github.ref }}" = "refs/heads/dev" ]; then + CHART_VERSION="${VERSION}-dev" + IMAGE_TAG="dev" + fi + sed -i "s/^version:.*/version: ${CHART_VERSION}/" helm/Chart.yaml + sed -i "s/^appVersion:.*/appVersion: \"${VERSION}\"/" helm/Chart.yaml + # ensure the packaged chart defaults to the correct docker image tag + sed -i "s/^ tag:.*/ tag: \"${IMAGE_TAG}\"/" helm/values.yaml + echo "Chart version: $CHART_VERSION" + echo "App version: $VERSION" + echo "Image tag: $IMAGE_TAG" + cat helm/Chart.yaml + + - name: Add Helm repositories + run: helm repo add bitnami https://charts.bitnami.com/bitnami + + - name: Build Helm dependencies + run: helm dependency build helm/ + + - name: Package Helm chart + run: helm package helm/ + + - name: Login to Docker Hub OCI registry + run: echo "${{ secrets.DOCKER_TOKEN }}" | helm registry login registry-1.docker.io -u blacklanternsecurity --password-stdin + + - name: Push Helm chart to Docker Hub + run: helm push bbot-server-*.tgz oci://registry-1.docker.io/blacklanternsecurity diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index abd186c4..a5620f73 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -37,7 +37,7 @@ jobs: - name: Lint run: | uv run ruff check - uv run ruff format --check + uv run ruff format --diff - name: Run tests run: | NO_COLOR=true uv run pytest --reruns 2 --disable-warnings --log-cli-level=INFO --cov-report xml:cov.xml --cov=bbot_server . @@ -47,3 +47,45 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} files: ./cov.xml verbose: true + + publish_pypi: + needs: test + runs-on: ubuntu-latest + if: github.event_name == 'push' && github.ref == 'refs/heads/stable' + continue-on-error: true + steps: + - uses: actions/checkout@v6 + - name: Install uv + uses: astral-sh/setup-uv@v7 + - name: Get version + id: calc_version + run: | + VERSION=$(grep '^version = ' pyproject.toml | sed 's/version = "\(.*\)"/\1/') + echo "VERSION=$VERSION" >> $GITHUB_OUTPUT + echo "Version: $VERSION" + - name: Build PyPI package + run: uv build + - name: Publish PyPI package + uses: pypa/gh-action-pypi-publish@release/v1.13 + with: + password: ${{ secrets.PYPI_API_TOKEN }} + outputs: + VERSION: ${{ steps.calc_version.outputs.VERSION }} + + tag_commit: + needs: publish_pypi + runs-on: ubuntu-latest + if: github.event_name == 'push' && github.ref == 'refs/heads/stable' + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + - name: Configure git + run: | + git config --local user.email "info@blacklanternsecurity.com" + git config --local user.name "GitHub Actions" + - name: Tag commit + run: | + VERSION="v${{ needs.publish_pypi.outputs.VERSION }}" + git tag -a "$VERSION" -m "Stable Release $VERSION" + git push origin "$VERSION" diff --git a/Dockerfile b/Dockerfile index e19e905a..58feef49 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,13 @@ FROM ghcr.io/astral-sh/uv:python3.11-trixie RUN apt-get -y update && apt-get -y install curl jq -COPY . /app WORKDIR /app +COPY pyproject.toml uv.lock ./ +RUN uv sync --frozen --no-install-project +COPY . . +RUN cp bbot_server/defaults_docker.yml bbot_server/defaults.yml RUN uv sync --frozen +RUN useradd -u 1000 -m bbot && chown -R bbot:bbot /app +USER bbot ENV PATH="/app/.venv/bin:$PATH" EXPOSE 8807 CMD ["uv", "run", "bbctl", "server", "start", "--api-only"] diff --git a/README.md b/README.md index 2cf107f7..7e6c3b63 100644 --- a/README.md +++ b/README.md @@ -24,28 +24,121 @@ BBOT Server is a database and multiplayer hub for all your [BBOT](https://github ## Installation ```bash -# clone the repo and cd into it -git clone git@github.com:blacklanternsecurity/bbot-server.git && cd bbot-server +# install with uv (recommended) +uv tool install bbot-server + +# or with pipx +pipx install bbot-server +``` + +To install `uv`, see the [uv installation docs](https://docs.astral.sh/uv/getting-started/installation/). + +### From source (for development) -# Install dependencies and create virtual environment +```bash +git clone git@github.com:blacklanternsecurity/bbot-server.git && cd bbot-server uv sync ``` Note: to update to the latest version, run `git pull && uv sync` in the `bbot-server` directory. -To install `uv`, see the [uv installation docs](https://docs.astral.sh/uv/getting-started/installation/). - -Commands shown below as `bbctl` can be run with `uv run bbctl` from the project directory, or you can activate the virtual environment first with `source .venv/bin/activate`. +When installed from source, commands shown below as `bbctl` can be run with `uv run bbctl` from the project directory, or you can activate the virtual environment first with `source .venv/bin/activate`. ## Start the server Note: this requires Docker and Docker Compose to be installed. ```bash -# Start BBOT server using Docker Compose +# Start BBOT server using Docker Compose (pulls from Docker Hub) bbctl server start ``` +### Development mode + +When working on the bbot-server source code, use `--dev` to build from source and mount your code for live reload: + +```bash +# From the bbot-server repo root +bbctl server --dev start +``` + +The `--dev` flag works with all server subcommands (`start`, `stop`, `down`, `logs`, `status`, `compose`). + +## Deploy with Helm (Kubernetes) + +BBOT Server can be deployed to Kubernetes using its official Helm chart. The chart deploys the API server, worker, MongoDB, and Redis. + +### Quick Start + +```bash +# Add the Helm repo +helm repo add blacklanternsecurity https://blacklanternsecurity.github.io/bbot-server + +# Install +helm install bbot blacklanternsecurity/bbot-server +``` + +Or install directly from the OCI registry: + +```bash +helm install bbot oci://registry-1.docker.io/blacklanternsecurity/bbot-server +``` + +### Configuration + +Key values can be overridden with `--set` or a custom values file: + +```bash +helm install bbot blacklanternsecurity/bbot-server \ + --set ingress.enabled=true \ + --set ingress.hosts[0].host=bbot.example.com \ + --set ingress.hosts[0].paths[0].path=/ \ + --set ingress.hosts[0].paths[0].pathType=Prefix +``` + +Or with a values file: + +```yaml +# custom-values.yaml +ingress: + enabled: true + className: nginx + annotations: + cert-manager.io/cluster-issuer: letsencrypt + hosts: + - host: bbot.example.com + paths: + - path: / + pathType: Prefix + tls: + - secretName: bbot-tls + hosts: + - bbot.example.com + +secrets: + # provide your own API key, or leave empty to auto-generate + apiKey: "" +``` + +```bash +helm install bbot blacklanternsecurity/bbot-server -f custom-values.yaml +``` + +### Retrieving the API Key + +If you didn't provide an API key, one is auto-generated. Retrieve it with: + +```bash +kubectl get secret bbot-api-key -o jsonpath='{.data.api-key}' | base64 -d +``` + +### Upgrading + +```bash +helm repo update +helm upgrade bbot blacklanternsecurity/bbot-server +``` + ## Interacting with BBOT Server Remotely (Multiplayer) By default, BBOT Server listens on localhost. Use `--listen` to expose it to the network: diff --git a/bbot_server/applets/base.py b/bbot_server/applets/base.py index c4ac8f35..17b71c3a 100644 --- a/bbot_server/applets/base.py +++ b/bbot_server/applets/base.py @@ -49,14 +49,14 @@ def decorator(fn): return decorator -def watchdog_task(**kwargs): +def worker_task(**kwargs): """ - Decorate your applet method with this to make it a watchdog task + Decorate your applet method with this to make it a worker task """ def decorator(fn): fn._kwargs = kwargs - fn._watchdog_task = True + fn._worker_task = True return fn return decorator @@ -223,7 +223,9 @@ async def _native_setup(self): self.collection = self.parent.collection self.strict_collection = self.parent.strict_collection else: - self.collection = self.db[self.table_name] + store = getattr(self, f"{self.store_type}_store") + prefix = getattr(store, "collection_prefix", "") + self.collection = self.db[f"{prefix}{self.table_name}"] # WriteConcern options: # w=1: Acknowledges the write operation only after it has been written to the primary. (the default) # j=True: Ensures the write operation is committed to the journal. (default is False) @@ -238,8 +240,8 @@ async def _native_setup(self): self.task_broker = await self.message_queue.make_taskiq_broker() await self.task_broker.startup() - # register watchdog tasks - await self.register_watchdog_tasks(self.task_broker) + # register worker tasks + await self.register_worker_tasks(self.task_broker) if self.name != "Root Applet": try: @@ -288,15 +290,15 @@ async def reconcile_all_indexes(self): diff = compute_index_diff(desired, desired_text, existing, existing_text) await apply_index_diff(collection, diff, existing) - async def register_watchdog_tasks(self, broker): - # register watchdog tasks + async def register_worker_tasks(self, broker): + # register worker tasks methods = {name: member for name, member in getmembers(self) if callable(member)} for method_name, method in methods.items(): # handle case where tasks have already been registered method = getattr(method, "original_func", method) - _watchdog_task = getattr(method, "_watchdog_task", None) - if _watchdog_task is None: + _worker_task = getattr(method, "_worker_task", None) + if _worker_task is None: continue kwargs = getattr(method, "_kwargs", {}) # crontab handling diff --git a/bbot_server/compose.yml b/bbot_server/compose.yml new file mode 100644 index 00000000..fdeb93f7 --- /dev/null +++ b/bbot_server/compose.yml @@ -0,0 +1,43 @@ +x-bbot-server-base: &bbot-server-base + image: blacklanternsecurity/bbot-server:stable + restart: unless-stopped + volumes: + - "${BBOT_SERVER_CONFIG:-~/.config/bbot_server/config.yml}:/home/bbot/.config/bbot_server/config.yml" + +services: + server: + <<: *bbot-server-base + ports: + - "${BBOT_LISTEN_ADDRESS:-127.0.0.1}:${BBOT_PORT:-8807}:8807" + command: ["bbctl", "server", "start", "--api-only", "--listen", "0.0.0.0"] + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8807/v1/docs"] + interval: 10s + timeout: 5s + retries: 3 + start_period: 30s + environment: + - BBOT_SERVER_AUTH_ENABLED=${BBOT_SERVER_AUTH_ENABLED:-true} + depends_on: + - mongodb + - redis + + worker: + <<: *bbot-server-base + command: ["bbctl", "server", "start", "--worker-only"] + depends_on: + server: + condition: service_healthy + + mongodb: + image: mongo:latest + restart: unless-stopped + volumes: + - bbot-mongodb:/data/db + + redis: + image: redis:latest + restart: unless-stopped + +volumes: + bbot-mongodb: diff --git a/bbot_server/config.py b/bbot_server/config.py index 20b7ea1a..f3c9b872 100644 --- a/bbot_server/config.py +++ b/bbot_server/config.py @@ -26,7 +26,10 @@ if not BBOT_SERVER_CONFIG_PATH.exists(): try: BBOT_SERVER_CONFIG_PATH.parent.mkdir(parents=True, exist_ok=True) - BBOT_SERVER_CONFIG_PATH.touch(mode=0o600) + # 644 permissions are necessary because we cannot predict the UID of the user running `bbctl`. + # since the config is mapped into the Docker container, BBOT server must have permissions to read it + # the alternative is running the BBOT server container as root + BBOT_SERVER_CONFIG_PATH.touch(mode=0o644) # fill with commented defaults with open(BBOT_SERVER_CONFIG_PATH, "w") as f: with open(BBOT_SERVER_DEFAULTS_PATH, "r") as defaults_file: @@ -39,6 +42,7 @@ class StoreConfig(BaseModel): uri: str + collection_prefix: str = "" class MessageQueueConfig(BaseModel): diff --git a/bbot_server/defaults.yml b/bbot_server/defaults.yml index 0f6a4a23..82c0c369 100644 --- a/bbot_server/defaults.yml +++ b/bbot_server/defaults.yml @@ -10,16 +10,19 @@ api_key: null # event store is just a big bucket of BBOT scan events that act as a read-only time machine event_store: - uri: mongodb://localhost:27017/bbot_eventstore + uri: mongodb://localhost:27017/bbot + collection_prefix: "eventstore__" # assets are derived from the event store, and can be recreated at any time asset_store: - uri: mongodb://localhost:27017/bbot_assetstore + uri: mongodb://localhost:27017/bbot + collection_prefix: "assetstore__" # user_store holds any user-specific data, overrides, etc. which are not derived from events # examples include targets, scans, etc. user_store: - uri: mongodb://localhost:27017/bbot_userstore + uri: mongodb://localhost:27017/bbot + collection_prefix: "userstore__" message_queue: uri: redis://localhost:6379/0 diff --git a/bbot_server/defaults_docker.yml b/bbot_server/defaults_docker.yml index a4a5015d..f62deae8 100644 --- a/bbot_server/defaults_docker.yml +++ b/bbot_server/defaults_docker.yml @@ -3,16 +3,19 @@ url: http://server:8807/v1/ # event store is just a big bucket of BBOT scan events event_store: - uri: mongodb://mongodb:27017/bbot_eventstore + uri: mongodb://mongodb:27017/bbot + collection_prefix: "eventstore__" # assets are derived in real time from the event store, and can be recreated at any time asset_store: - uri: mongodb://mongodb:27017/bbot_assetstore + uri: mongodb://mongodb:27017/bbot + collection_prefix: "assetstore__" # user_store holds any user-specific data, overrides, etc. which are not derived from events # examples include targets, scans, etc. user_store: - uri: mongodb://mongodb:27017/bbot_userstore + uri: mongodb://mongodb:27017/bbot + collection_prefix: "userstore__" message_queue: uri: redis://redis:6379/0 diff --git a/bbot_server/modules/emails/emails_api.py b/bbot_server/modules/emails/emails_api.py index c15af4f9..987c0d9f 100644 --- a/bbot_server/modules/emails/emails_api.py +++ b/bbot_server/modules/emails/emails_api.py @@ -1,4 +1,4 @@ -# from bbot_server.watchdogs.emails import EmailWatchdog +# from bbot_server.workers.emails import EmailWorker from bbot_server.applets.base import BaseApplet, api_endpoint, BaseModel, Field @@ -7,7 +7,7 @@ class EmailsApplet(BaseApplet): description = "emails discovered during scans" # watched_events = ["EMAIL_ADDRESS"] route_prefix = "" - # watchdogs = [EmailWatchdog] + # workers = [EmailWorker] attach_to = "assets" class AssetFields(BaseModel): diff --git a/bbot_server/modules/events/events_api.py b/bbot_server/modules/events/events_api.py index 1dd1cf13..d8270583 100644 --- a/bbot_server/modules/events/events_api.py +++ b/bbot_server/modules/events/events_api.py @@ -29,7 +29,7 @@ async def insert_event(self, event: Event): Insert a BBOT event into the asset database """ # publish event to the message queue - # it will be picked up by the watchdog and ingested + # it will be picked up by the worker and ingested await self.root.message_queue.publish_event(event) @api_endpoint("/get/{uuid}", methods=["GET"], summary="Get an event by its UUID") diff --git a/bbot_server/modules/server/server_cli.py b/bbot_server/modules/server/server_cli.py index fa175593..02d092d8 100644 --- a/bbot_server/modules/server/server_cli.py +++ b/bbot_server/modules/server/server_cli.py @@ -3,11 +3,10 @@ import typer import asyncio import subprocess -from pathlib import Path from subprocess import run from contextlib import suppress -from bbot_server.config import BBOT_SERVER_CONFIG as bbcfg +from bbot_server.config import BBOT_SERVER_CONFIG as bbcfg, BBOT_SERVER_DIR from bbot_server.cli.base import BaseBBCTL, subcommand, Option, Annotated @@ -17,19 +16,32 @@ class ServerCTL(BaseBBCTL): short_help = "Start/stop BBOT server and manage API keys" attach_to = "bbctl" + _invoke_without_command = True + _no_args_is_help = True + def setup(self): self._docker_command = None - self.docker_compose_dir = Path(__file__).parent.parent + self.docker_compose_dir = BBOT_SERVER_DIR self.docker_compose_file = self.docker_compose_dir / "compose.yml" + def main( + self, + dev: Annotated[ + bool, Option("--dev", "-d", help="Use dev compose file (builds from source, mounts code for live reload)") + ] = False, + ): + if dev: + self.docker_compose_dir = BBOT_SERVER_DIR.parent + self.docker_compose_file = self.docker_compose_dir / "compose.yml" + @subcommand(help="Start BBOT server") def start( self, api_only: Annotated[ bool, Option("--api-only", "-a", help="Only start the REST API, without Docker Compose") ] = False, - watchdog_only: Annotated[ - bool, Option("--watchdog-only", "-w", help="Only start the watchdog, without Docker Compose") + worker_only: Annotated[ + bool, Option("--worker-only", "-w", help="Only start the worker, without Docker Compose") ] = False, listen: Annotated[str, Option("--listen", "-l", help="Listen address", metavar="IP_ADDRESS")] = "127.0.0.1", port: Annotated[int, Option("--port", "-p", help="Port to run the server on", metavar="PORT")] = 8807, @@ -60,20 +72,20 @@ def start( workers=1, ) - elif watchdog_only: - print("Starting watchdog") + elif worker_only: + print("Starting worker") - async def run_watchdog(): + async def run_worker(): try: from bbot_server import BBOTServer - from bbot_server.watchdog import BBOTWatchdog + from bbot_server.worker import BBOTWorker bbot_server = BBOTServer() await bbot_server.setup() - watchdog = BBOTWatchdog(bbot_server) - await watchdog.start() - print("Watchdog successfully started") + worker = BBOTWorker(bbot_server) + await worker.start() + print("Worker successfully started") # sleep for infinity event = asyncio.Event() @@ -81,9 +93,9 @@ async def run_watchdog(): except KeyboardInterrupt: with suppress(Exception): - await watchdog.stop() + await worker.stop() - asyncio.run(run_watchdog()) + asyncio.run(run_worker()) else: # initialize the config if not already @@ -128,49 +140,36 @@ def logs( @subcommand(help="Clear the database (drop Mongodb collections).") def cleardb( self, - event_store: Annotated[bool, Option("--event-store", "-e", help="Clear the event store database")] = False, - asset_store: Annotated[bool, Option("--asset-store", "-a", help="Clear the asset store database")] = False, - user_store: Annotated[bool, Option("--user-store", "-u", help="Clear the user store database")] = False, + event_store: Annotated[bool, Option("--event-store", "-e", help="Clear the event store collections")] = False, + asset_store: Annotated[bool, Option("--asset-store", "-a", help="Clear the asset store collections")] = False, + user_store: Annotated[bool, Option("--user-store", "-u", help="Clear the user store collections")] = False, ): if not event_store and not asset_store and not user_store: - raise self.BBOTServerError(f"Must specify at least one database to clear") + raise self.BBOTServerError(f"Must specify at least one store to clear") + stores_to_clear = [] if event_store: - event_store_db = self.config.event_store.uri.split("/")[-1] - if not event_store_db: - raise self.BBOTServerError("Event store database not found in config") - response = input( - f"Are you sure you want to clear the event store database: {event_store_db}? This will permanently delete all BBOT scan events! (y/N) " - ) - if response.lower() != "y": - raise self.BBOTServerError("Aborting") - - self._run_docker_compose(["exec", "mongodb", "mongosh", "--eval", "db.dropDatabase()", event_store_db]) - self.log.info(f"Successfully cleared event store database: {event_store_db}") - + stores_to_clear.append(("event store", self.config.event_store, "all BBOT scan events")) if asset_store: - asset_store_db = self.config.asset_store.uri.split("/")[-1] - if not asset_store_db: - raise self.BBOTServerError("Asset store database not found in config") - response = input( - f"Are you sure you want to clear the asset store database: {asset_store_db}? This will permanently delete all BBOT asset data! (y/N) " + stores_to_clear.append(("asset store", self.config.asset_store, "all BBOT asset data")) + if user_store: + stores_to_clear.append( + ("user store", self.config.user_store, "all BBOT user data, including presets and targets") ) - if response.lower() != "y": - raise self.BBOTServerError("Aborting") - self._run_docker_compose(["exec", "mongodb", "mongosh", "--eval", "db.dropDatabase()", asset_store_db]) - self.log.info(f"Successfully cleared asset store database: {asset_store_db}") - if user_store: - user_store_db = self.config.user_store.uri.split("/")[-1] - if not user_store_db: - raise self.BBOTServerError("User store database not found in config") + for store_name, store_config, data_desc in stores_to_clear: + db_name = store_config.uri.split("/")[-1] + prefix = store_config.collection_prefix + if not db_name: + raise self.BBOTServerError(f"{store_name.title()} database not found in config") response = input( - f"Are you sure you want to clear the user store database: {user_store_db}? This will permanently delete all BBOT user data, including presets and targets! (y/N) " + f"Are you sure you want to clear the {store_name} (prefix: {prefix})? This will permanently delete {data_desc}! (y/N) " ) if response.lower() != "y": raise self.BBOTServerError("Aborting") - self._run_docker_compose(["exec", "mongodb", "mongosh", "--eval", "db.dropDatabase()", user_store_db]) - self.log.info(f"Successfully cleared user store database: {user_store_db}") + drop_eval = f"db.getCollectionNames().filter(c => c.startsWith('{prefix}')).forEach(c => db[c].drop())" + self._run_docker_compose(["exec", "mongodb", "mongosh", "--eval", drop_eval, db_name]) + self.log.info(f"Successfully cleared {store_name} collections (prefix: {prefix})") def _run_docker_compose(self, args, **kwargs): kwargs["cwd"] = self.docker_compose_dir diff --git a/bbot_server/store.py b/bbot_server/store.py index 989017cb..3a81e3cf 100644 --- a/bbot_server/store.py +++ b/bbot_server/store.py @@ -8,7 +8,9 @@ class BaseMongoStore(BaseDB): async def setup(self): self.client = AsyncMongoClient(self.uri) self.db = self.client.get_database(self.db_name) - self.fs = AsyncGridFSBucket(self.db) + self.collection_prefix = getattr(self.db_config, "collection_prefix", "") + bucket_name = f"{self.collection_prefix}fs" if self.collection_prefix else "fs" + self.fs = AsyncGridFSBucket(self.db, bucket_name=bucket_name) async def cleanup(self): await self.client.close() diff --git a/bbot_server/watchdog/__init__.py b/bbot_server/watchdog/__init__.py deleted file mode 100644 index b5ba850c..00000000 --- a/bbot_server/watchdog/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .worker import BBOTWatchdog - -__all__ = ["BBOTWatchdog"] diff --git a/bbot_server/worker/__init__.py b/bbot_server/worker/__init__.py new file mode 100644 index 00000000..df91ad11 --- /dev/null +++ b/bbot_server/worker/__init__.py @@ -0,0 +1,3 @@ +from .worker import BBOTWorker + +__all__ = ["BBOTWorker"] diff --git a/bbot_server/watchdog/worker.py b/bbot_server/worker/worker.py similarity index 97% rename from bbot_server/watchdog/worker.py rename to bbot_server/worker/worker.py index 19c505a1..3385f33d 100644 --- a/bbot_server/watchdog/worker.py +++ b/bbot_server/worker/worker.py @@ -12,7 +12,7 @@ from bbot_server.modules.activity.activity_models import Activity -class BBOTWatchdog: +class BBOTWorker: """ Contains: - taskiq worker @@ -40,9 +40,9 @@ async def startup(state: TaskiqState) -> None: await self.broker.startup() - # register watchdog tasks + # register worker tasks for app in self.bbot_server.all_child_applets(include_self=True): - await app.register_watchdog_tasks(self.broker) + await app.register_worker_tasks(self.broker) # taskiq worker tasks self.taskiq_worker_task = asyncio.create_task(run_receiver_task(self.broker)) @@ -157,7 +157,7 @@ async def _get_or_create_asset(self, host: str, event: Event = None, parent_acti return asset, activities async def stop(self) -> None: - self.log.info("Stopping watchdog") + self.log.info("Stopping worker") await self.bbot_server.message_queue.unsubscribe(self.event_listener) self.taskiq_worker_task.cancel() self.taskiq_scheduler_task.cancel() diff --git a/compose.yml b/compose.yml index 708f3f75..f998eb7f 100644 --- a/compose.yml +++ b/compose.yml @@ -3,7 +3,7 @@ x-bbot-server-base: &bbot-server-base context: . restart: unless-stopped volumes: - - "${BBOT_SERVER_CONFIG:-~/.config/bbot_server/config.yml}:/root/.config/bbot_server/config.yml" + - "${BBOT_SERVER_CONFIG:-~/.config/bbot_server/config.yml}:/home/bbot/.config/bbot_server/config.yml" - ./bbot_server:/app/bbot_server - ./bbot_server/defaults_docker.yml:/app/bbot_server/defaults.yml @@ -25,17 +25,9 @@ services: - mongodb - redis - watchdog: + worker: <<: *bbot-server-base - command: ["bbctl", "server", "start", "--watchdog-only"] - depends_on: - server: - condition: service_healthy - - agent: - <<: *bbot-server-base - command: ["bbctl", "agent", "start"] - entrypoint: ["bash", "/app/bbot_server/default_agent.sh"] + command: ["bbctl", "server", "start", "--worker-only"] depends_on: server: condition: service_healthy diff --git a/helm/Chart.lock b/helm/Chart.lock new file mode 100644 index 00000000..20f94099 --- /dev/null +++ b/helm/Chart.lock @@ -0,0 +1,9 @@ +dependencies: +- name: mongodb + repository: https://charts.bitnami.com/bitnami + version: 18.6.1 +- name: redis + repository: https://charts.bitnami.com/bitnami + version: 23.2.12 +digest: sha256:e7950fb219d8931af304b0372fbcd200c46bb182c5d9a2157d8a59190b0c5996 +generated: "2026-02-27T11:45:51.327023189-05:00" diff --git a/helm/Chart.yaml b/helm/Chart.yaml new file mode 100644 index 00000000..17b13efa --- /dev/null +++ b/helm/Chart.yaml @@ -0,0 +1,14 @@ +apiVersion: v2 +name: bbot-server +description: Helm chart for BBOT Server (API + Worker) +type: application +version: 0.1.0 +appVersion: "0.1.0" +dependencies: +- name: mongodb + version: "18.x.x" + repository: "https://charts.bitnami.com/bitnami" + condition: mongodb.enabled +- name: redis + version: "23.x.x" + repository: "https://charts.bitnami.com/bitnami" diff --git a/helm/templates/ingress.yaml b/helm/templates/ingress.yaml new file mode 100644 index 00000000..62baa173 --- /dev/null +++ b/helm/templates/ingress.yaml @@ -0,0 +1,42 @@ +{{- if .Values.ingress.enabled }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ .Release.Name }}-ingress + labels: + app: bbot-server + app.kubernetes.io/instance: {{ .Release.Name | quote }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if .Values.ingress.className }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + pathType: {{ .pathType }} + backend: + service: + name: {{ $.Release.Name }}-server + port: + number: {{ $.Values.server.service.port }} + {{- end }} + {{- end }} +{{- end }} diff --git a/helm/templates/secrets.yaml b/helm/templates/secrets.yaml new file mode 100644 index 00000000..bdc4a4eb --- /dev/null +++ b/helm/templates/secrets.yaml @@ -0,0 +1,20 @@ +# BBOT Server API key (auto-generated on first install, preserved on upgrade) +{{- $apiKeySecretName := .Values.secrets.existingApiKeySecret | default (printf "%s-api-key" .Release.Name) -}} +{{- if not (lookup "v1" "Secret" .Release.Namespace $apiKeySecretName) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ $apiKeySecretName }} + labels: + app.kubernetes.io/managed-by: "Helm" + app.kubernetes.io/instance: {{ .Release.Name | quote }} + annotations: + helm.sh/resource-policy: keep +type: Opaque +data: + {{- if .Values.secrets.apiKey }} + api-key: {{ .Values.secrets.apiKey | b64enc }} + {{- else }} + api-key: {{ uuidv4 | b64enc }} + {{- end }} +{{- end }} diff --git a/helm/templates/server-deployment.yaml b/helm/templates/server-deployment.yaml new file mode 100644 index 00000000..14a3f82c --- /dev/null +++ b/helm/templates/server-deployment.yaml @@ -0,0 +1,130 @@ +{{- if .Values.server.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-server + labels: + app: bbot-server + app.kubernetes.io/instance: {{ .Release.Name | quote }} +spec: + replicas: {{ .Values.server.replicas }} + selector: + matchLabels: + app: bbot-server + template: + metadata: + labels: + app: bbot-server + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml .Values.imagePullSecrets | nindent 8 }} + {{- end }} + containers: + - name: server + securityContext: + allowPrivilegeEscalation: false + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: ["bbctl", "server", "start", "--api-only", "--listen", "0.0.0.0"] + ports: + - containerPort: 8807 + name: http + protocol: TCP + env: + {{- if .Values.mongodb.enabled }} + # Internal password references (used for URI interpolation below) + - name: MONGO_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.mongodb.auth.existingSecret | default (printf "%s-mongodb" .Release.Name) | quote }} + key: mongodb-root-password + {{- end }} + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.redis.auth.existingSecret | default (printf "%s-redis" .Release.Name) | quote }} + key: redis-password + # BBOT Server configuration + - name: BBOT_SERVER_URL + value: {{ .Values.config.url | default (printf "http://%s-server:%d/v1/" .Release.Name (.Values.server.service.port | int)) | quote }} + - name: BBOT_SERVER_AUTH_ENABLED + value: {{ .Values.config.authEnabled | quote }} + - name: BBOT_SERVER_API_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.existingApiKeySecret | default (printf "%s-api-key" .Release.Name) | quote }} + key: api-key + # MongoDB URIs + {{- if .Values.mongodb.enabled }} + - name: BBOT_SERVER_EVENT_STORE__URI + value: "mongodb://root:$(MONGO_PASSWORD)@{{ .Release.Name }}-mongodb:27017/bbot?authSource=admin" + {{- else }} + - name: BBOT_SERVER_EVENT_STORE__URI + value: {{ .Values.externalMongodb.uri | quote }} + {{- end }} + - name: BBOT_SERVER_EVENT_STORE__COLLECTION_PREFIX + value: "eventstore__" + {{- if .Values.mongodb.enabled }} + - name: BBOT_SERVER_ASSET_STORE__URI + value: "mongodb://root:$(MONGO_PASSWORD)@{{ .Release.Name }}-mongodb:27017/bbot?authSource=admin" + {{- else }} + - name: BBOT_SERVER_ASSET_STORE__URI + value: {{ .Values.externalMongodb.uri | quote }} + {{- end }} + - name: BBOT_SERVER_ASSET_STORE__COLLECTION_PREFIX + value: "assetstore__" + {{- if .Values.mongodb.enabled }} + - name: BBOT_SERVER_USER_STORE__URI + value: "mongodb://root:$(MONGO_PASSWORD)@{{ .Release.Name }}-mongodb:27017/bbot?authSource=admin" + {{- else }} + - name: BBOT_SERVER_USER_STORE__URI + value: {{ .Values.externalMongodb.uri | quote }} + {{- end }} + - name: BBOT_SERVER_USER_STORE__COLLECTION_PREFIX + value: "userstore__" + # Redis URI + - name: BBOT_SERVER_MESSAGE_QUEUE__URI + value: "redis://:$(REDIS_PASSWORD)@{{ .Release.Name }}-redis-master:6379/0" + {{- with .Values.server.resources }} + resources: + {{- toYaml . | nindent 10 }} + {{- end }} + readinessProbe: + httpGet: + path: /v1/docs + port: 8807 + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + livenessProbe: + httpGet: + path: /v1/docs + port: 8807 + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-server + labels: + app: bbot-server + app.kubernetes.io/instance: {{ .Release.Name | quote }} +spec: + type: {{ .Values.server.service.type }} + selector: + app: bbot-server + ports: + - port: {{ .Values.server.service.port }} + targetPort: {{ .Values.server.service.targetPort }} + protocol: TCP + name: http +{{- end }} diff --git a/helm/templates/worker-deployment.yaml b/helm/templates/worker-deployment.yaml new file mode 100644 index 00000000..50e90dec --- /dev/null +++ b/helm/templates/worker-deployment.yaml @@ -0,0 +1,109 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-worker + labels: + app: bbot-worker + app.kubernetes.io/instance: {{ .Release.Name | quote }} +spec: + replicas: {{ .Values.worker.replicas }} + selector: + matchLabels: + app: bbot-worker + template: + metadata: + labels: + app: bbot-worker + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml .Values.imagePullSecrets | nindent 8 }} + {{- end }} + {{- if .Values.server.enabled }} + # Wait for the API server to become healthy before starting the worker + initContainers: + - name: wait-for-server + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + allowPrivilegeEscalation: false + command: ["sh", "-c"] + args: + - | + echo "Waiting for BBOT server to become healthy..." + until curl -sf http://{{ .Release.Name }}-server:{{ .Values.server.service.port }}/v1/docs > /dev/null 2>&1; do + echo "Server not ready yet, retrying in 5s..." + sleep 5 + done + echo "Server is healthy, starting worker." + {{- end }} + containers: + - name: worker + securityContext: + allowPrivilegeEscalation: false + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: ["bbctl", "server", "start", "--worker-only"] + env: + {{- if .Values.mongodb.enabled }} + # Internal password references (used for URI interpolation below) + - name: MONGO_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.mongodb.auth.existingSecret | default (printf "%s-mongodb" .Release.Name) | quote }} + key: mongodb-root-password + {{- end }} + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.redis.auth.existingSecret | default (printf "%s-redis" .Release.Name) | quote }} + key: redis-password + # BBOT Server configuration + - name: BBOT_SERVER_URL + value: "http://{{ .Release.Name }}-server:{{ .Values.server.service.port }}/v1/" + - name: BBOT_SERVER_AUTH_ENABLED + value: {{ .Values.config.authEnabled | quote }} + - name: BBOT_SERVER_API_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.existingApiKeySecret | default (printf "%s-api-key" .Release.Name) | quote }} + key: api-key + # MongoDB URIs + {{- if .Values.mongodb.enabled }} + - name: BBOT_SERVER_EVENT_STORE__URI + value: "mongodb://root:$(MONGO_PASSWORD)@{{ .Release.Name }}-mongodb:27017/bbot?authSource=admin" + {{- else }} + - name: BBOT_SERVER_EVENT_STORE__URI + value: {{ .Values.externalMongodb.uri | quote }} + {{- end }} + - name: BBOT_SERVER_EVENT_STORE__COLLECTION_PREFIX + value: "eventstore__" + {{- if .Values.mongodb.enabled }} + - name: BBOT_SERVER_ASSET_STORE__URI + value: "mongodb://root:$(MONGO_PASSWORD)@{{ .Release.Name }}-mongodb:27017/bbot?authSource=admin" + {{- else }} + - name: BBOT_SERVER_ASSET_STORE__URI + value: {{ .Values.externalMongodb.uri | quote }} + {{- end }} + - name: BBOT_SERVER_ASSET_STORE__COLLECTION_PREFIX + value: "assetstore__" + {{- if .Values.mongodb.enabled }} + - name: BBOT_SERVER_USER_STORE__URI + value: "mongodb://root:$(MONGO_PASSWORD)@{{ .Release.Name }}-mongodb:27017/bbot?authSource=admin" + {{- else }} + - name: BBOT_SERVER_USER_STORE__URI + value: {{ .Values.externalMongodb.uri | quote }} + {{- end }} + - name: BBOT_SERVER_USER_STORE__COLLECTION_PREFIX + value: "userstore__" + # Redis URI + - name: BBOT_SERVER_MESSAGE_QUEUE__URI + value: "redis://:$(REDIS_PASSWORD)@{{ .Release.Name }}-redis-master:6379/0" + {{- with .Values.worker.resources }} + resources: + {{- toYaml . | nindent 10 }} + {{- end }} diff --git a/helm/values.yaml b/helm/values.yaml new file mode 100644 index 00000000..c63488be --- /dev/null +++ b/helm/values.yaml @@ -0,0 +1,115 @@ +# ============================================================================= +# BBOT Server Helm Chart +# ============================================================================= + +# -- Docker image configuration +image: + repository: blacklanternsecurity/bbot-server + tag: "stable" + pullPolicy: IfNotPresent + +# -- Image pull secrets for private registries +imagePullSecrets: [] + # - name: docker-registry-secret + +# ============================================================================= +# Server (API) +# ============================================================================= +server: + # -- Set to false to disable the API server (only deploy DBs + worker) + enabled: true + replicas: 1 + resources: {} + # requests: + # cpu: 250m + # memory: 512Mi + # limits: + # cpu: "1" + # memory: 1Gi + service: + type: ClusterIP + port: 8807 + targetPort: 8807 + +# ============================================================================= +# Worker (event processor) +# ============================================================================= +worker: + replicas: 1 + resources: {} + +# ============================================================================= +# BBOT Server Application Configuration +# ============================================================================= +config: + # -- Public URL of the BBOT server API + # If empty, auto-derived as http://{release}-server:8807/v1/ + url: "" + # -- Enable API key authentication + authEnabled: true + +# ============================================================================= +# Secrets +# ============================================================================= +secrets: + # -- Pre-existing secret name for the API key + # If empty, the chart auto-generates one on first install + existingApiKeySecret: "" + # -- API key value to store in the secret + # If empty, a random 32-character key is generated on first install + apiKey: "" + +# ============================================================================= +# MongoDB (Bitnami subchart) +# ============================================================================= +mongodb: + # -- Set to false to disable the built-in MongoDB and use an external instance + enabled: true + architecture: standalone + auth: + enabled: true + # -- Existing secret name (created by this chart if empty) + existingSecret: "" + persistence: + size: 50Gi + resourcePolicy: keep + +# ============================================================================= +# External MongoDB (used when mongodb.enabled is false) +# ============================================================================= +externalMongodb: + # -- Full MongoDB connection URI (e.g. "mongodb://user:pass@host:27017/bbot?authSource=admin") + uri: "" + +# ============================================================================= +# Redis (Bitnami subchart) +# ============================================================================= +redis: + architecture: standalone + auth: + enabled: true + # -- Existing secret name (created by this chart if empty) + existingSecret: "" + existingSecretPasswordKey: redis-password + master: + persistence: + enabled: false + +# ============================================================================= +# Ingress (optional) +# ============================================================================= +ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # cert-manager.io/cluster-issuer: letsencrypt + hosts: [] + # - host: bbot.example.com + # paths: + # - path: / + # pathType: Prefix + tls: [] + # - secretName: bbot-tls + # hosts: + # - bbot.example.com diff --git a/helm_deployment_test.py b/helm_deployment_test.py new file mode 100644 index 00000000..4a23a64d --- /dev/null +++ b/helm_deployment_test.py @@ -0,0 +1,380 @@ +#!/usr/bin/env python3 +import unittest +import subprocess +import time +import json +import base64 +import requests +import atexit +from contextlib import contextmanager + +""" +End-to-end integration tests for the BBOT Server Helm chart running on minikube. + +Run it like so: + +# all tests +uv run python test_helm_deployment.py + +# specific test +uv run python -m unittest test_helm_deployment.TestHelmDeployment.test_swagger_ui + +To bring up the test environment manually, run: + +minikube start +docker build -t blacklanternsecurity/bbot-server:test . +minikube image load blacklanternsecurity/bbot-server:test +helm dependency update helm/ +helm install bbot helm/ --set image.tag=test --set image.pullPolicy=Never +""" + + +class TestHelmDeployment(unittest.TestCase): + release_name = "bbot" + ctx = ["--context", "minikube"] + kube_ctx = ["--kube-context", "minikube"] + _cleanup_registered = False + + @classmethod + def run_command(cls, command, timeout=120, **kwargs): + print(f"Running: {' '.join(command)}") + if "check" not in kwargs: + kwargs["check"] = True + return subprocess.run(command, text=True, timeout=timeout, **kwargs) + + @classmethod + def kubectl(cls, *command, **kwargs): + return cls.run_command(["kubectl", *cls.ctx, *command], **kwargs) + + @classmethod + def helm(cls, *command, **kwargs): + return cls.run_command(["helm", *cls.kube_ctx, *command], **kwargs) + + @classmethod + @contextmanager + def port_forward(cls, service, local_port, remote_port): + """Context manager for kubectl port-forward""" + process = subprocess.Popen( + ["kubectl", *cls.ctx, "port-forward", f"svc/{service}", f"{local_port}:{remote_port}"], + ) + try: + # Wait for port-forward to establish + time.sleep(3) + yield f"http://localhost:{local_port}" + finally: + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + + @classmethod + def cleanup_resources(cls): + """Clean up Helm charts and PVCs - ALWAYS runs""" + print("\nCLEANING UP RESOURCES...") + print("-" * 40) + + # Uninstall Helm chart + print("Uninstalling Helm chart...") + cls.helm("uninstall", cls.release_name, check=False) + + # Delete any existing PVCs to ensure clean state + print("Deleting any existing PVCs...") + cls.kubectl("delete", "pvc", "--all", check=False) + + # Delete leftover secrets with keep policy + for suffix in ("mongodb", "redis", "api-key"): + cls.kubectl("delete", "secret", f"{cls.release_name}-{suffix}", check=False) + + print("Cleanup completed") + + @classmethod + def register_cleanup(cls): + """Register cleanup to run on exit""" + if not cls._cleanup_registered: + atexit.register(cls.cleanup_resources) + cls._cleanup_registered = True + + @classmethod + def dump_cluster_debug_info(cls): + """Print pods, logs, services, and events for debugging""" + print("\n" + "=" * 80) + print("ALL PODS STATUS:") + print("-" * 40) + try: + result = cls.kubectl("get", "pods", "-o", "wide", capture_output=True, check=False) + print(result.stdout) + except Exception as e: + print(f"Failed to get pods: {e}") + + print("\nPOD LOGS:") + print("-" * 40) + try: + result = cls.kubectl( + "get", + "pods", + "-o", + "jsonpath='{.items[*].metadata.name}'", + capture_output=True, + check=False, + ) + if result.returncode == 0 and result.stdout.strip(): + pod_names = result.stdout.strip().strip("'").split() + for pod_name in pod_names: + print(f"\nLogs for pod: {pod_name}") + print("-" * 30) + try: + log_result = cls.kubectl( + "logs", + pod_name, + "--all-containers", + "--tail=200", + capture_output=True, + check=False, + ) + if log_result.returncode == 0: + print(log_result.stdout) + else: + print(f"Failed to get logs: {log_result.stderr}") + except Exception as e: + print(f"Error getting logs for {pod_name}: {e}") + else: + print("Failed to list pods or no pods found") + except Exception as e: + print(f"Failed to get pod names: {e}") + + print("\nSERVICES STATUS:") + print("-" * 40) + try: + result = cls.kubectl("get", "services", "-o", "wide", capture_output=True, check=False) + print(result.stdout) + except Exception as e: + print(f"Failed to get services: {e}") + + print("\nRECENT EVENTS:") + print("-" * 40) + try: + result = cls.kubectl( + "get", + "events", + "--sort-by=.metadata.creationTimestamp", + capture_output=True, + check=False, + ) + print(result.stdout) + except Exception as e: + print(f"Failed to get events: {e}") + print("\n" + "=" * 80) + + @classmethod + def setUpClass(cls): + """Build image, deploy to minikube, wait for readiness""" + + # Ensure minikube is running + print("Checking if minikube is running...") + result = subprocess.run( + ["minikube", "status"], + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0 or "Running" not in result.stdout: + print("Starting minikube...") + subprocess.run(["minikube", "start"], check=True) + print("Minikube started") + else: + print("Minikube is already running") + + # Register cleanup to ALWAYS run, even on test failure + cls.register_cleanup() + + # Clean up any previous deployments + cls.cleanup_resources() + + # Build image locally and load into minikube + print("Building Docker image...") + cls.run_command( + [ + "docker", + "build", + "-t", + "blacklanternsecurity/bbot-server:test", + ".", + ], + timeout=300, + ) + print("Loading image into minikube...") + cls.run_command( + [ + "minikube", + "image", + "load", + "blacklanternsecurity/bbot-server:test", + ], + timeout=120, + ) + print("Image loaded successfully") + + # Add Helm repositories and update dependencies + print("Adding Helm repositories...") + cls.helm("repo", "add", "bitnami", "https://charts.bitnami.com/bitnami", timeout=30) + print("Building Helm dependencies (will fail if Chart.lock is out of sync)...") + cls.helm("dependency", "build", "helm/", timeout=60) + + # Deploy the helm chart + print("Deploying helm chart...") + cls.helm( + "install", + cls.release_name, + "helm/", + "--set", + "image.tag=test", + "--set", + "image.pullPolicy=Never", + timeout=60, + ) + + # Wait for server pod to be ready + print("Waiting for server pod to be ready...") + result = cls.kubectl( + "wait", + "--for=condition=ready", + "pod", + "-l", + "app=bbot-server", + "--timeout=90s", + check=False, + capture_output=True, + timeout=100, + ) + if result.returncode == 0: + print("Server pod is ready") + return + + print("\n" + "=" * 80) + print("TIMEOUT: Server pod failed to become ready") + print("=" * 80) + cls.dump_cluster_debug_info() + raise RuntimeError("Timed out waiting for server pod to be ready") + + def test_swagger_ui(self): + """Test that the swagger-ui loads properly at /v1/docs""" + with self.port_forward(f"{self.release_name}-server", 8807, 8807) as base_url: + response = requests.get(f"{base_url}/v1/docs", timeout=10) + print(f"GET /v1/docs status: {response.status_code}") + print(f"Content length: {len(response.text)}") + + self.assertEqual(response.status_code, 200, "Swagger UI should return 200") + self.assertIn("swagger-ui", response.text.lower(), "Response should contain swagger-ui") + + def test_ingest_and_query_assets(self): + """Test ingesting events from evilcorp.json.gz and querying assets via bbctl""" + # Get the server pod name + result = self.kubectl( + "get", + "pods", + "-l", + "app=bbot-server", + "-o", + "jsonpath={.items[0].metadata.name}", + capture_output=True, + ) + pod_name = result.stdout.strip() + self.assertTrue(pod_name, "Should find a server pod") + print(f"Server pod: {pod_name}") + + # Get the API key from the kubernetes secret + result = self.kubectl( + "get", + "secret", + f"{self.release_name}-api-key", + "-o", + "jsonpath={.data.api-key}", + capture_output=True, + ) + api_key = base64.b64decode(result.stdout).decode() + self.assertTrue(api_key, "Should retrieve API key from secret") + + # Write a bbctl config file inside the pod + server_url = f"http://{self.release_name}-server:8807/v1/" + config_content = f'url: "{server_url}"\napi_keys:\n - "{api_key}"' + self.kubectl( + "exec", + pod_name, + "--", + "sh", + "-c", + f"printf '%s\\n' '{config_content}' > /tmp/bbctl.yaml", + ) + bbctl = "bbctl --no-color --config /tmp/bbctl.yaml" + + # Copy test data into the pod + self.kubectl("cp", "tests/evilcorp.json.gz", f"{pod_name}:/tmp/evilcorp.json.gz") + + # Ingest events using bbctl + print("Ingesting events...") + result = self.kubectl( + "exec", + pod_name, + "--", + "sh", + "-c", + f"gunzip -c /tmp/evilcorp.json.gz | {bbctl} event ingest", + capture_output=True, + timeout=120, + ) + print(f"Ingest stdout: {result.stdout}") + print(f"Ingest stderr: {result.stderr}") + self.assertEqual(result.returncode, 0, f"Event ingest failed: {result.stderr}") + + # Wait for the worker to process events into assets + print("Waiting for worker to process events into assets...") + assets = [] + for i in range(30): + result = self.kubectl( + "exec", + pod_name, + "--", + "sh", + "-c", + f"{bbctl} asset list --json", + capture_output=True, + check=False, + timeout=30, + ) + if result.returncode == 0 and result.stdout.strip(): + assets = [json.loads(line) for line in result.stdout.strip().splitlines() if line.strip()] + if assets: + break + print(f" Attempt {i + 1}/30: {len(assets)} assets so far...") + time.sleep(2) + + print(f"Found {len(assets)} assets") + self.assertGreater(len(assets), 0, "Should have ingested some assets from evilcorp.json.gz") + + def tearDown(self): + """If a test fails, dump cluster debug info""" + failed = False + outcome = getattr(self, "_outcome", None) + try: + if outcome is not None: + if hasattr(outcome, "errors"): + errors = outcome.errors + elif hasattr(outcome, "result"): + result = outcome.result + errors = list(getattr(result, "errors", [])) + list(getattr(result, "failures", [])) + else: + errors = [] + failed = any(err for _, err in errors) + except Exception: + failed = False + if failed: + print("\n" + "=" * 80) + print(f"Test failed: {self.id()} - dumping cluster debug info") + print("=" * 80) + self.__class__.dump_cluster_debug_info() + + +if __name__ == "__main__": + unittest.main() diff --git a/pyproject.toml b/pyproject.toml index 4b3e7947..d1ffc9f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "bbot-server" -version = "0.1.0" +version = "0.1.3" description = "" authors = [{name = "TheTechromancer"}] license = "AGPL-3.0" diff --git a/tests/conftest.py b/tests/conftest.py index c8c9388b..8ae982a2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -41,7 +41,7 @@ bbcfg.refresh(config_path=str(TEST_CONFIG_PATH)) -assert bbcfg.asset_store.uri == "mongodb://localhost:27017/test_bbot_server_assets" +assert bbcfg.asset_store.uri == "mongodb://localhost:27017/test_bbot" if not bbcfg.get_api_keys(): # create a new api key if we don't have one yet @@ -58,7 +58,7 @@ async def bbot_server(request, mongo_cleanup, redis_cleanup): message_queue = None async def _make_bbot_server( - config_overrides=None, needs_agent=False, needs_api=False, needs_watchdog=True, **kwargs + config_overrides=None, needs_agent=False, needs_api=False, needs_worker=True, **kwargs ): nonlocal bbot_server @@ -77,9 +77,9 @@ async def _make_bbot_server( await message_queue.setup() await message_queue.clear() - # watchdog - if needs_watchdog: - request.getfixturevalue("bbot_watchdog") + # worker + if needs_worker: + request.getfixturevalue("bbot_worker") # http server if needs_api or kwargs["interface"] == "http": @@ -101,16 +101,16 @@ async def _make_bbot_server( @pytest.fixture -def bbot_watchdog(mongo_cleanup, redis_cleanup): - command = [*BBCTL_COMMAND, "server", "start", "--watchdog-only"] - watchdog_process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, bufsize=1) +def bbot_worker(mongo_cleanup, redis_cleanup): + command = [*BBCTL_COMMAND, "server", "start", "--worker-only"] + worker_process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, bufsize=1) try: - # Wait for watchdog to be ready by monitoring stderr + # Wait for worker to be ready by monitoring stderr ready = False - while watchdog_process.poll() is None: # 10 second timeout (50 * 0.2) - line = watchdog_process.stderr.readline() - log.critical(f"Watchdog: {line.strip()}") - if "Watchdog started" in line: + while worker_process.poll() is None: # 10 second timeout (50 * 0.2) + line = worker_process.stderr.readline() + log.critical(f"Worker: {line.strip()}") + if "Worker started" in line: ready = True break if "[INFO] Subscribed to bbot:stream:events" in line: @@ -118,29 +118,29 @@ def bbot_watchdog(mongo_cleanup, redis_cleanup): break if not ready: - raise Exception("Watchdog failed to start and subscribe to events") + raise Exception("Worker failed to start and subscribe to events") - # here, start a thread to tail the watchdog's stderr + # here, start a thread to tail the worker's stderr def tail_stderr(): - while watchdog_process.poll() is None: - line = watchdog_process.stderr.readline() + while worker_process.poll() is None: + line = worker_process.stderr.readline() if line: - log.critical(f"Watchdog: {line.strip()}") + log.critical(f"Worker: {line.strip()}") import threading stderr_thread = threading.Thread(target=tail_stderr, daemon=True) stderr_thread.start() - yield watchdog_process + yield worker_process - watchdog_process.send_signal(signal.SIGINT) + worker_process.send_signal(signal.SIGINT) finally: try: - watchdog_process.wait(timeout=5) + worker_process.wait(timeout=5) except subprocess.TimeoutExpired: - log.error("Watchdog process timed out, killing forcefully") - watchdog_process.kill() + log.error("Worker process timed out, killing forcefully") + worker_process.kill() @pytest.fixture @@ -263,9 +263,7 @@ async def mongo_cleanup(): client = AsyncMongoClient(bbcfg.event_store.uri) async def clear_everything(): - await client.drop_database("test_bbot_server_events") - await client.drop_database("test_bbot_server_assets") - await client.drop_database("test_bbot_server_userdata") + await client.drop_database("test_bbot") try: # Clear before test diff --git a/tests/evilcorp.json.gz b/tests/evilcorp.json.gz new file mode 100644 index 00000000..4bdb5d98 Binary files /dev/null and b/tests/evilcorp.json.gz differ diff --git a/tests/test_applets/base.py b/tests/test_applets/base.py index 9c983b40..d59862fe 100644 --- a/tests/test_applets/base.py +++ b/tests/test_applets/base.py @@ -11,7 +11,7 @@ class BaseAppletTest: needs_api = False needs_agent = False - needs_watchdog = False + needs_worker = False # if True, don't test the http interface native_only = False @@ -55,7 +55,7 @@ async def test_applet_run(self, bbot_server, bbot_events): config_overrides=self.config_overrides, needs_api=self.needs_api, needs_agent=self.needs_agent, - needs_watchdog=self.needs_watchdog, + needs_worker=self.needs_worker, ) if self.native_only and not self.bbot_server.is_native: diff --git a/tests/test_applets/test_applet_activity.py b/tests/test_applets/test_applet_activity.py index fe026423..fa682f03 100644 --- a/tests/test_applets/test_applet_activity.py +++ b/tests/test_applets/test_applet_activity.py @@ -3,7 +3,7 @@ class TestAppletActivity(BaseAppletTest): - needs_watchdog = True + needs_worker = True async def setup(self): # at the beginning, everything should be empty diff --git a/tests/test_applets/test_applet_assets.py b/tests/test_applets/test_applet_assets.py index ffdd34d8..cd9124eb 100644 --- a/tests/test_applets/test_applet_assets.py +++ b/tests/test_applets/test_applet_assets.py @@ -10,7 +10,7 @@ class TestAppletAssets(BaseAppletTest): - needs_watchdog = True + needs_worker = True async def setup(self): # # make sure all asset fields have annotations @@ -27,7 +27,7 @@ async def setup(self): assert [a async for a in self.bbot_server.query_assets()] == [] async def after_scan_1(self): - # since this is our first test, and runners are dog slow, it can take a while for the watchdog etc. to get ready + # since this is our first test, and runners are dog slow, it can take a while for the worker etc. to get ready # we loop for a while to give them time to start up expected_hosts = { "1.2.3.4", @@ -260,7 +260,7 @@ async def after_archive(self): # test to make sure you can filter assets by target async def test_applet_target_filter(bbot_server, bbot_events): - bbot_server = await bbot_server(needs_watchdog=True) + bbot_server = await bbot_server(needs_worker=True) target1 = CreateTarget( target=["evilcorp.com", "127.0.0.0/30"], diff --git a/tests/test_applets/test_applet_cloud.py b/tests/test_applets/test_applet_cloud.py index b1f7c67e..4d893fc0 100644 --- a/tests/test_applets/test_applet_cloud.py +++ b/tests/test_applets/test_applet_cloud.py @@ -2,7 +2,7 @@ class TestAppletCloud(BaseAppletTest): - needs_watchdog = True + needs_worker = True async def after_scan_1(self): cloudcheck_result = await self.bbot_server.cloudcheck("www.evilcorp.azure.com") diff --git a/tests/test_applets/test_applet_dns_links.py b/tests/test_applets/test_applet_dns_links.py index 207bdec4..450adba2 100644 --- a/tests/test_applets/test_applet_dns_links.py +++ b/tests/test_applets/test_applet_dns_links.py @@ -2,7 +2,7 @@ class TestAppletDNSLinks(BaseAppletTest): - needs_watchdog = True + needs_worker = True async def after_scan_1(self): activities = [a for a in self.asset_messages if a.type in ["NEW_DNS_LINK", "DELETED_DNS_LINK"]] diff --git a/tests/test_applets/test_applet_events.py b/tests/test_applets/test_applet_events.py index 5300dece..29702a40 100644 --- a/tests/test_applets/test_applet_events.py +++ b/tests/test_applets/test_applet_events.py @@ -48,7 +48,7 @@ async def test_events_http_ingest(bbot_server, bbot_events): class TestAppletEvents(BaseAppletTest): - needs_watchdog = True + needs_worker = True async def setup(self): route = self.bbot_server.route_maps["tail_events"] diff --git a/tests/test_applets/test_applet_findings.py b/tests/test_applets/test_applet_findings.py index d667510a..8c9c0330 100644 --- a/tests/test_applets/test_applet_findings.py +++ b/tests/test_applets/test_applet_findings.py @@ -5,7 +5,7 @@ class TestAppletFindings(BaseAppletTest): - needs_watchdog = True + needs_worker = True async def setup(self): # at the beginning, everything should be empty diff --git a/tests/test_applets/test_applet_open_ports.py b/tests/test_applets/test_applet_open_ports.py index 0870cce9..2f76da9b 100644 --- a/tests/test_applets/test_applet_open_ports.py +++ b/tests/test_applets/test_applet_open_ports.py @@ -2,7 +2,7 @@ class TestAppletOpenPorts(BaseAppletTest): - needs_watchdog = True + needs_worker = True async def setup(self): # at the beginning, everything should be empty diff --git a/tests/test_applets/test_applet_scans.py b/tests/test_applets/test_applet_scans.py index 4686b651..d3725c23 100644 --- a/tests/test_applets/test_applet_scans.py +++ b/tests/test_applets/test_applet_scans.py @@ -13,7 +13,7 @@ # make sure ad-hoc ingestion of a BBOT scan creates an associated scan run in the database async def test_scan_run_adhoc(bbot_server, bbot_events): - bbot_server = await bbot_server(needs_watchdog=True) + bbot_server = await bbot_server(needs_worker=True) activities = [] @@ -77,7 +77,7 @@ async def test_scan_with_invalid_preset(bbot_server, bbot_agent): """ Test that a scan with an invalid preset surfaces the error to the user """ - bbot_server = await bbot_server(needs_agent=True, needs_watchdog=True) + bbot_server = await bbot_server(needs_agent=True, needs_worker=True) preset = await bbot_server.create_preset( preset={"name": "preset1", "description": "preset1 description", "modules": ["invalid"]} @@ -99,7 +99,7 @@ async def test_basic_scan_run(bbot_server): """ A basic scan run, with an agent. Makes sure the scan runs start to finish and reports its statuses correctly """ - bbot_server = await bbot_server(needs_agent=True, needs_watchdog=True) + bbot_server = await bbot_server(needs_agent=True, needs_worker=True) events = [] @@ -220,7 +220,7 @@ async def test_queued_scan_cancellation(bbot_server): assert scans[0].status == "ABORTED" -async def test_running_scan_cancellation(bbot_agent, bbot_watchdog): +async def test_running_scan_cancellation(bbot_agent, bbot_worker): """ Here we start a scan with an agent, so we have a running scan diff --git a/tests/test_applets/test_applet_stats.py b/tests/test_applets/test_applet_stats.py index 2864e99f..7f8df4c0 100644 --- a/tests/test_applets/test_applet_stats.py +++ b/tests/test_applets/test_applet_stats.py @@ -3,7 +3,7 @@ async def test_applet_stats(bbot_server, bbot_events): - bbot_server = await bbot_server(needs_watchdog=True) + bbot_server = await bbot_server(needs_worker=True) target1 = CreateTarget( target=["evilcorp.com"], diff --git a/tests/test_applets/test_applet_targets.py b/tests/test_applets/test_applet_targets.py index fcb2fecd..236c0066 100644 --- a/tests/test_applets/test_applet_targets.py +++ b/tests/test_applets/test_applet_targets.py @@ -387,7 +387,7 @@ async def test_target_default_uniqueness(bbot_server): class TestTargetScopeMaintenance(BaseAppletTest): - needs_watchdog = True + needs_worker = True async def setup(self): assert await self.bbot_server.get_hosts() == [] @@ -501,7 +501,7 @@ class TestTargetUpdateRemovesTargetFromAssets(BaseAppletTest): Also tests that deleting a target doesn't break count_assets(target_id="DEFAULT") afterward """ - needs_watchdog = True + needs_worker = True async def setup(self): assert await self.bbot_server.get_hosts() == [] @@ -543,7 +543,7 @@ async def after_scan_1(self): self.target.target = ["somethingelse.net"] await self.bbot_server.update_target(self.target.id, self.target) - # Give watchdog time to process the TARGET_UPDATED activity + # Give worker time to process the TARGET_UPDATED activity await asyncio.sleep(1.0) # Check that assets no longer have this target in their scope diff --git a/tests/test_applets/test_applet_technologies.py b/tests/test_applets/test_applet_technologies.py index 0358a935..55c234e8 100644 --- a/tests/test_applets/test_applet_technologies.py +++ b/tests/test_applets/test_applet_technologies.py @@ -4,7 +4,7 @@ class TestAppletTechnologies(BaseAppletTest): - needs_watchdog = True + needs_worker = True async def setup(self): # at the beginning, everything should be empty diff --git a/tests/test_archival.py b/tests/test_archival.py index cdb90c68..f615a805 100644 --- a/tests/test_archival.py +++ b/tests/test_archival.py @@ -6,7 +6,7 @@ class TestArchival(BaseAppletTest): This is a basic sanity check to make sure the event archiving process works as intended """ - needs_watchdog = True + needs_worker = True async def setup(self): events = [e async for e in self.bbot_server.list_events()] diff --git a/tests/test_cli/test_cli_assetctl.py b/tests/test_cli/test_cli_assetctl.py index 5e9a3279..9f53ff20 100644 --- a/tests/test_cli/test_cli_assetctl.py +++ b/tests/test_cli/test_cli_assetctl.py @@ -45,7 +45,7 @@ } -def test_cli_assetctl(bbot_server_http, bbot_watchdog, bbot_out_file, bbot_events): +def test_cli_assetctl(bbot_server_http, bbot_worker, bbot_out_file, bbot_events): scan1_out_file, scan2_out_file = bbot_out_file scan1_events, scan2_events = bbot_events diff --git a/tests/test_cli/test_cli_basic.py b/tests/test_cli/test_cli_basic.py index 51a4daf7..e14e5e0e 100644 --- a/tests/test_cli/test_cli_basic.py +++ b/tests/test_cli/test_cli_basic.py @@ -32,7 +32,7 @@ def test_cli_config(): result = subprocess.run(BBCTL_COMMAND + ["--current-config"], capture_output=True, text=True) assert result.returncode == 0 config = yaml.safe_load(result.stdout) - assert config["event_store"]["uri"] == "mongodb://localhost:27017/test_bbot_server_events" + assert config["event_store"]["uri"] == "mongodb://localhost:27017/test_bbot" yaml_config_str = """ event_store: diff --git a/tests/test_cli/test_cli_eventctl.py b/tests/test_cli/test_cli_eventctl.py index 0946e6e0..bb4b2c37 100644 --- a/tests/test_cli/test_cli_eventctl.py +++ b/tests/test_cli/test_cli_eventctl.py @@ -8,7 +8,7 @@ from tests.conftest import BBCTL_COMMAND, INGEST_PROCESSING_DELAY -def test_cli_events(bbot_server_http, bbot_watchdog, bbot_out_file, bbot_events): +def test_cli_events(bbot_server_http, bbot_worker, bbot_out_file, bbot_events): scan1_out_file, scan2_out_file = bbot_out_file scan1_events, scan2_events = bbot_events diff --git a/tests/test_cli/test_cli_findingctl.py b/tests/test_cli/test_cli_findingctl.py index 9bdcc832..dabc057c 100644 --- a/tests/test_cli/test_cli_findingctl.py +++ b/tests/test_cli/test_cli_findingctl.py @@ -6,7 +6,7 @@ from bbot_server.modules.findings.findings_models import Finding -def test_cli_findingctl(bbot_server_http, bbot_watchdog, bbot_out_file): +def test_cli_findingctl(bbot_server_http, bbot_worker, bbot_out_file): # we shouldn't have any findings yet command = BBCTL_COMMAND + ["finding", "list", "--json"] process = subprocess.run(command, capture_output=True, text=True) diff --git a/tests/test_cli/test_cli_scanctl.py b/tests/test_cli/test_cli_scanctl.py index 91c0e857..67a2d41d 100644 --- a/tests/test_cli/test_cli_scanctl.py +++ b/tests/test_cli/test_cli_scanctl.py @@ -8,7 +8,7 @@ from bbot_server.modules.scans.scans_models import Scan -def test_cli_scan_start(bbot_server_http, bbot_watchdog, bbot_agent): +def test_cli_scan_start(bbot_server_http, bbot_worker, bbot_agent): # we shouldn't have any scans yet command = BBCTL_COMMAND + ["scan", "list", "--json"] process = subprocess.run(command, capture_output=True, text=True) @@ -121,7 +121,7 @@ def test_cli_scan_start(bbot_server_http, bbot_watchdog, bbot_agent): assert "Scan with name 'demonic_jimmy' already exists" in process.stderr -def test_cli_scan_ingest(bbot_server_http, bbot_watchdog, bbot_out_file, bbot_events): +def test_cli_scan_ingest(bbot_server_http, bbot_worker, bbot_out_file, bbot_events): scan1_out_file, scan2_out_file = bbot_out_file scan1_events, scan2_events = bbot_events scan1_name = scan1_events[0].data_json["name"] diff --git a/tests/test_cli/test_cli_technologyctl.py b/tests/test_cli/test_cli_technologyctl.py index bd661d88..aeb663db 100644 --- a/tests/test_cli/test_cli_technologyctl.py +++ b/tests/test_cli/test_cli_technologyctl.py @@ -8,7 +8,7 @@ from tests.conftest import BBCTL_COMMAND, INGEST_PROCESSING_DELAY, BBOT_SERVER_TEST_DIR -def test_cli_technologyctl(bbot_server_http, bbot_watchdog, bbot_out_file): +def test_cli_technologyctl(bbot_server_http, bbot_worker, bbot_out_file): # we shouldn't have any technologies yet command = BBCTL_COMMAND + ["technology", "list", "--json"] process = subprocess.run(command, capture_output=True, text=True) diff --git a/tests/test_config.py b/tests/test_config.py index 530b51b1..cb82ad4c 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -8,12 +8,12 @@ def test_config(): os.environ["BBOT_SERVER_URL"] = "http://asdf:8000" bbcfg.refresh() assert bbcfg.url == "http://asdf:8000" - assert bbcfg.event_store.uri == "mongodb://localhost:27017/test_bbot_server_events" + assert bbcfg.event_store.uri == "mongodb://localhost:27017/test_bbot" os.environ["BBOT_SERVER_URL"] = "http://fdsa:8000" bbcfg.refresh() assert bbcfg.url == "http://fdsa:8000" - assert bbcfg.event_store.uri == "mongodb://localhost:27017/test_bbot_server_events" + assert bbcfg.event_store.uri == "mongodb://localhost:27017/test_bbot" tmp_config_file = NamedTemporaryFile(suffix=".yml") with open(tmp_config_file.name, "w") as f: @@ -29,16 +29,16 @@ def test_config(): # asset store uri should be overridden assert bbcfg.asset_store.uri == "mongodb://localhost:27017/asdf" # others should be untouched - assert bbcfg.event_store.uri == "mongodb://localhost:27017/bbot_eventstore" + assert bbcfg.event_store.uri == "mongodb://localhost:27017/bbot" # everything should be the same after a refresh bbcfg.refresh() assert bbcfg.url == "http://fdsa:8000" assert bbcfg.asset_store.uri == "mongodb://localhost:27017/asdf" - assert bbcfg.event_store.uri == "mongodb://localhost:27017/bbot_eventstore" + assert bbcfg.event_store.uri == "mongodb://localhost:27017/bbot" # reset back to testing defaults os.environ.pop("BBOT_SERVER_URL", None) bbcfg.refresh(config_path=TEST_CONFIG_PATH) assert bbcfg.url == "http://localhost:8807/v1/" - assert bbcfg.event_store.uri == "mongodb://localhost:27017/test_bbot_server_events" + assert bbcfg.event_store.uri == "mongodb://localhost:27017/test_bbot" diff --git a/tests/test_config.yml b/tests/test_config.yml index a0f87df8..b057ae42 100644 --- a/tests/test_config.yml +++ b/tests/test_config.yml @@ -1,8 +1,11 @@ event_store: - uri: mongodb://localhost:27017/test_bbot_server_events + uri: mongodb://localhost:27017/test_bbot + collection_prefix: "test_eventstore__" asset_store: - uri: mongodb://localhost:27017/test_bbot_server_assets + uri: mongodb://localhost:27017/test_bbot + collection_prefix: "test_assetstore__" user_store: - uri: mongodb://localhost:27017/test_bbot_server_userdata + uri: mongodb://localhost:27017/test_bbot + collection_prefix: "test_userstore__" message_queue: uri: redis://localhost:6379/15 diff --git a/tests/test_config_docker.yml b/tests/test_config_docker.yml index f2acb3d6..0979b64c 100644 --- a/tests/test_config_docker.yml +++ b/tests/test_config_docker.yml @@ -1,8 +1,11 @@ event_store: - uri: mongodb://mongodb:27017/test_bbot_server_events + uri: mongodb://mongodb:27017/test_bbot + collection_prefix: "eventstore__" asset_store: - uri: mongodb://mongodb:27017/test_bbot_server_assets + uri: mongodb://mongodb:27017/test_bbot + collection_prefix: "assetstore__" user_store: - uri: mongodb://mongodb:27017/test_bbot_server_userdata + uri: mongodb://mongodb:27017/test_bbot + collection_prefix: "userstore__" message_queue: uri: redis://redis:6379/15 diff --git a/tests/test_docker_compose.py b/tests/test_docker_compose.py index a9ae612d..5325f79e 100644 --- a/tests/test_docker_compose.py +++ b/tests/test_docker_compose.py @@ -2,7 +2,6 @@ import os import sys -import json import yaml import time import httpx @@ -23,37 +22,53 @@ def reset_config_file(): copyfile(Path(__file__).parent / "test_config_docker.yml", custom_config_file) +def bbctl_command(config_file=None): + cmd = ["uv", "run", "bbctl"] + if config_file: + cmd += ["--config", str(config_file)] + return cmd + + +def server_args(dev=True): + """Return the server subcommand args with or without --dev.""" + if dev: + return ["server", "--dev"] + return ["server"] + + @contextmanager -def docker_test_env(reset_config=True, docker_down_first=True, cleanup_config=True): +def docker_test_env(dev=True, reset_config=True, docker_down_first=True, cleanup_config=True): """ Context manager for docker compose tests. Args: + dev: Use dev compose (build from source) vs production compose (pull from Docker Hub) reset_config: Reset config file at start docker_down_first: Run docker compose down before test cleanup_config: Delete custom config file after test """ + srv = server_args(dev) + # Setup if reset_config: reset_config_file() if docker_down_first: - result = subprocess.run( - ["docker", "compose", "down"], + subprocess.run( + bbctl_command(custom_config_file) + srv + ["down"], cwd=project_root, ) - assert result.returncode == 0 try: yield finally: # Cleanup - print logs if test failed if sys.exc_info()[0] is not None: - print_docker_logs() + print_docker_logs(dev=dev) # Always stop docker compose subprocess.run( - ["docker", "compose", "down"], + bbctl_command(custom_config_file) + srv + ["down"], cwd=project_root, ) @@ -62,47 +77,93 @@ def docker_test_env(reset_config=True, docker_down_first=True, cleanup_config=Tr custom_config_file.unlink(missing_ok=True) -# we typically only want to run this on CI -# to avoid messing with the user's existing bbot server data / api keys -pytestmark = pytest.mark.skipif( - os.environ.get("BBOT_SERVER_TEST_DOCKER_COMPOSE", "false").lower() != "true", - reason="BBOT_SERVER_TEST_DOCKER_COMPOSE is not set to true", -) - - -def print_docker_logs(): - """Helper to print docker compose logs""" +def print_docker_logs(dev=True): + """Helper to print docker compose logs and container state on failure""" print("\n" + "=" * 80) - print("TEST FAILED - Docker Compose Logs:") + print("TEST FAILED - Diagnostics:") print("=" * 80) + + # Show all docker containers (running and stopped) + print("\n--- docker ps -a ---") result = subprocess.run( - ["docker", "compose", "logs", "--no-color"], + ["docker", "ps", "-a"], + capture_output=True, + text=True, + ) + print(result.stdout or "(empty)") + if result.stderr: + print(f"stderr: {result.stderr}") + + # Show docker compose ps from the relevant compose directory + print("\n--- docker compose ps (from compose dir) ---") + result = subprocess.run( + bbctl_command(custom_config_file) + server_args(dev) + ["status"], + cwd=project_root, + capture_output=True, + text=True, + ) + print(result.stdout or "(empty)") + if result.stderr: + print(f"stderr: {result.stderr}") + + # Show docker compose logs + print("\n--- docker compose logs ---") + result = subprocess.run( + bbctl_command(custom_config_file) + server_args(dev) + ["logs"], cwd=project_root, capture_output=True, text=True, ) - if result.returncode == 0: - print(result.stdout) - if result.stderr: - print("\nSTDERR:") - print(result.stderr) - else: - print(f"Failed to get docker compose logs (exit code {result.returncode}): {result.stderr}") + print(result.stdout or "(empty)") + if result.stderr: + print(f"stderr: {result.stderr}") + print("=" * 80 + "\n") +def start_server(bbctl_cmd, dev=True, extra_args=None): + """Start the server and return the result. Asserts success.""" + cmd = bbctl_cmd + server_args(dev) + ["start"] + (extra_args or []) + result = subprocess.run( + cmd, + cwd=project_root, + capture_output=True, + text=True, + ) + assert result.returncode == 0, f"Failed to start server: {result.stderr}" + return result + + +def wait_for_server(bbctl_cmd, timeout_seconds=60): + """Wait for the server to become healthy (asset stats returns 200).""" + for _ in range(timeout_seconds * 2): + result = subprocess.run( + bbctl_cmd + ["asset", "stats"], + cwd=project_root, + capture_output=True, + text=True, + ) + if result.returncode == 0: + return result + time.sleep(0.5) + assert False, ( + f"Server did not become healthy within {timeout_seconds}s, stdout: {result.stdout}, stderr: {result.stderr}" + ) + + +# we typically only want to run this on CI +# to avoid messing with the user's existing bbot server data / api keys +pytestmark = pytest.mark.skipif( + os.environ.get("BBOT_SERVER_TEST_DOCKER_COMPOSE", "false").lower() != "true", + reason="BBOT_SERVER_TEST_DOCKER_COMPOSE is not set to true", +) + + def test_docker_compose_userexperience(): """ A basic up/down test to make sure bbot server is working with docker compose """ - with docker_test_env(reset_config=False, docker_down_first=True): - # build it - # result = subprocess.run( - # ["docker", "compose", "build"], - # cwd=project_root, - # ) - # assert result.returncode == 0 - + with docker_test_env(dev=True, reset_config=False, docker_down_first=True): docker_compose_file = project_root / "compose.yml" assert docker_compose_file.exists() @@ -110,7 +171,7 @@ def test_docker_compose_userexperience(): custom_config_file.unlink(missing_ok=True) custom_config_file.write_text("testasdf: testasdf") - BBCTL_COMMAND = ["uv", "run", "bbctl", "--config", str(custom_config_file)] + BBCTL_COMMAND = bbctl_command(custom_config_file) # current config should have no api key or valid api keys result = subprocess.run( @@ -162,13 +223,7 @@ def test_docker_compose_userexperience(): assert not config.get("api_keys", []) # start docker compose - result = subprocess.run( - BBCTL_COMMAND + ["server", "start"], - cwd=project_root, - capture_output=True, - text=True, - ) - assert result.returncode == 0 + result = start_server(BBCTL_COMMAND, dev=True) assert "First run detected" in result.stderr # is api key listed now? @@ -191,35 +246,26 @@ def test_docker_compose_userexperience(): our_api_key = our_api_key[0] assert docker_api_key[:20] == our_api_key[:20] - for _ in range(120): - # we should be able to list assets now - command = BBCTL_COMMAND + ["asset", "stats"] - print(f"command: {' '.join(command)}") - result = subprocess.run( - command, - cwd=project_root, - capture_output=True, - text=True, - ) - print(f"result: {result.returncode}, stdout: {result.stdout}, stderr: {result.stderr}") - assert not "Invalid API key" in result.stderr - if result.returncode == 0 and json.loads(result.stdout) == {}: - break - time.sleep(0.5) - else: - assert False, f"Failed to list assets, stdout: {result.stdout}, stderr: {result.stderr}" + wait_for_server(BBCTL_COMMAND) def test_docker_compose_custom_config(): # delete config env var for this test os.environ.pop("BBOT_SERVER_CONFIG", None) + # Ensure the default host config path exists so Docker bind-mounts a file + # (not a root-owned directory) when BBOT_SERVER_CONFIG is unset + default_config = Path.home() / ".config" / "bbot_server" / "config.yml" + default_config.parent.mkdir(parents=True, exist_ok=True) + if not default_config.exists(): + default_config.touch() + # create a blank config file just for this test custom_config_file.unlink(missing_ok=True) custom_config_file.write_text("test1234: test4321") # can we read it outside of docker compose? - BBCTL_COMMAND = ["uv", "run", "bbctl", "--config", str(custom_config_file)] + BBCTL_COMMAND = bbctl_command(custom_config_file) result = subprocess.run( BBCTL_COMMAND + ["--current-config"], cwd=project_root, @@ -232,18 +278,18 @@ def test_docker_compose_custom_config(): # if we don't pass --config, the docker container should use the default config result = subprocess.run( - ["uv", "run", "bbctl", "server", "compose", "run", "server", "bbctl", "--current-config"], + ["uv", "run", "bbctl", "server", "--dev", "compose", "run", "server", "bbctl", "--current-config"], cwd=project_root, capture_output=True, text=True, ) assert result.returncode == 0 - assert "uri: mongodb://mongodb:27017/bbot_eventstore" in result.stdout + assert "uri: mongodb://mongodb:27017/bbot" in result.stdout assert not "test1234: test4321" in result.stdout # if we do pass --config, it should use the custom config result = subprocess.run( - BBCTL_COMMAND + ["server", "compose", "run", "server", "bbctl", "--current-config"], + BBCTL_COMMAND + ["server", "--dev", "compose", "run", "server", "bbctl", "--current-config"], cwd=project_root, capture_output=True, text=True, @@ -256,35 +302,18 @@ def test_docker_compose_custom_config(): # test the --listen flag and --port flags def test_docker_compose_listening_interface(): - with docker_test_env(reset_config=True, docker_down_first=True): + with docker_test_env(dev=True, reset_config=True, docker_down_first=True): # create a blank config file just for this test with open(custom_config_file, "a") as f: f.write("\nurl: http://127.0.0.1:8807/v1/\n") - BBCTL_COMMAND = ["uv", "run", "bbctl", "--config", str(custom_config_file)] + BBCTL_COMMAND = bbctl_command(custom_config_file) # start docker compose - result = subprocess.run( - BBCTL_COMMAND + ["server", "start"], - cwd=project_root, - capture_output=True, - text=True, - ) - assert result.returncode == 0 + start_server(BBCTL_COMMAND, dev=True) # we should be able to list assets - for _ in range(120): - result = subprocess.run( - BBCTL_COMMAND + ["asset", "stats"], - cwd=project_root, - capture_output=True, - text=True, - ) - if result.returncode == 0 and json.loads(result.stdout) == {}: - break - time.sleep(0.5) - else: - assert False, f"Failed to list assets, stdout: {result.stdout}, stderr: {result.stderr}" + wait_for_server(BBCTL_COMMAND) # replace the url key with 127.0.0.2 config_content = custom_config_file.read_text() @@ -301,34 +330,16 @@ def test_docker_compose_listening_interface(): assert "Error making GET request" in result.stderr # docker compose down - result = subprocess.run( - ["docker", "compose", "down"], + subprocess.run( + BBCTL_COMMAND + server_args(dev=True) + ["down"], cwd=project_root, ) - assert result.returncode == 0 # this time, we change up the interface - result = subprocess.run( - BBCTL_COMMAND + ["server", "start", "--listen", "127.0.0.2"], - cwd=project_root, - capture_output=True, - text=True, - ) - assert result.returncode == 0 + start_server(BBCTL_COMMAND, dev=True, extra_args=["--listen", "127.0.0.2"]) # we should be able to list assets - for _ in range(120): - result = subprocess.run( - BBCTL_COMMAND + ["asset", "stats"], - cwd=project_root, - capture_output=True, - text=True, - ) - if result.returncode == 0 and json.loads(result.stdout) == {}: - break - time.sleep(0.5) - else: - assert False, f"Failed to list assets, stdout: {result.stdout}, stderr: {result.stderr}" + wait_for_server(BBCTL_COMMAND) # replace the url key with 127.0.0.1 config_content = custom_config_file.read_text() @@ -351,15 +362,11 @@ def test_docker_compose_authentication(): custom_config_file.unlink(missing_ok=True) custom_config_file.write_text("") - BBCTL_COMMAND = ["uv", "run", "bbctl", "--config", str(custom_config_file)] + BBCTL_COMMAND = bbctl_command(custom_config_file) - with docker_test_env(reset_config=True, docker_down_first=True): + with docker_test_env(dev=True, reset_config=True, docker_down_first=True): # start docker compose - result = subprocess.run( - BBCTL_COMMAND + ["server", "start"], - cwd=project_root, - ) - assert result.returncode == 0 + start_server(BBCTL_COMMAND, dev=True) for _ in range(120): # docker should detect first run and write a new api key to the custom config file @@ -370,7 +377,7 @@ def test_docker_compose_authentication(): break time.sleep(0.5) else: - assert False, f"Failed to get api key, stdout: {result.stdout}, stderr: {result.stderr}" + assert False, f"Failed to get api key" # without the API key, auth should fail for _ in range(120): @@ -405,14 +412,10 @@ def test_docker_compose_no_authentication(): """ custom_config_file.unlink(missing_ok=True) - BBCTL_COMMAND = ["uv", "run", "bbctl", "--config", str(custom_config_file)] + BBCTL_COMMAND = bbctl_command(custom_config_file) - with docker_test_env(reset_config=True, docker_down_first=True): - result = subprocess.run( - BBCTL_COMMAND + ["server", "start", "--no-authentication"], - cwd=project_root, - ) - assert result.returncode == 0 + with docker_test_env(dev=True, reset_config=True, docker_down_first=True): + start_server(BBCTL_COMMAND, dev=True, extra_args=["--no-authentication"]) for _ in range(120): try: @@ -426,3 +429,38 @@ def test_docker_compose_no_authentication(): assert False, ( f"Expected 200 OK without auth, got: {response.status_code if 'response' in locals() else 'no response'}" ) + + +def test_docker_compose_production(): + """ + Smoke test for the production compose (pulls from Docker Hub). + Starts the server, waits for healthy, queries asset stats. + """ + # Build the image locally and tag it so the production compose can use it + # without pulling from Docker Hub (the tag may not exist yet). + result = subprocess.run( + ["docker", "build", "-t", "blacklanternsecurity/bbot-server:stable", "."], + cwd=project_root, + capture_output=True, + text=True, + ) + assert result.returncode == 0, f"Failed to build production image: {result.stderr}" + + with docker_test_env(dev=False, reset_config=True, docker_down_first=True): + BBCTL_COMMAND = bbctl_command(custom_config_file) + + # start production compose (no --dev, pulls from Docker Hub) + result = start_server(BBCTL_COMMAND, dev=False) + assert "First run detected" in result.stderr + + # wait for server to be healthy (longer timeout for image pulls) + wait_for_server(BBCTL_COMMAND, timeout_seconds=120) + + # verify asset stats returns successfully + result = subprocess.run( + BBCTL_COMMAND + ["asset", "stats"], + cwd=project_root, + capture_output=True, + text=True, + ) + assert result.returncode == 0 diff --git a/tests/test_message_queues.py b/tests/test_message_queues.py index 2de2151e..e2349437 100644 --- a/tests/test_message_queues.py +++ b/tests/test_message_queues.py @@ -166,7 +166,7 @@ async def test_queues_redis(): class TestMessageQueuesRedis(BaseAppletTest): - needs_watchdog = True + needs_worker = True expected_message_queue_uri = "redis://localhost:6379/15" diff --git a/tests/test_python_api.py b/tests/test_python_api.py index 7560fd83..89eea6d6 100644 --- a/tests/test_python_api.py +++ b/tests/test_python_api.py @@ -123,9 +123,9 @@ def _test_synchronous_api(interface, bbot_events): bbot_server.cleanup() -def test_synchronous_api_python(bbot_server_http, bbot_events, mongo_cleanup, bbot_watchdog): +def test_synchronous_api_python(bbot_server_http, bbot_events, mongo_cleanup, bbot_worker): _test_synchronous_api("python", bbot_events) -def test_synchronous_api_http(bbot_server_http, bbot_events, mongo_cleanup, bbot_watchdog): +def test_synchronous_api_http(bbot_server_http, bbot_events, mongo_cleanup, bbot_worker): _test_synchronous_api("http", bbot_events) diff --git a/tests/test_scheduled_tasks.py b/tests/test_scheduled_tasks.py index b2d03e2f..abfb6a2b 100644 --- a/tests/test_scheduled_tasks.py +++ b/tests/test_scheduled_tasks.py @@ -12,8 +12,8 @@ # async def test_scheduled_tasks(mongo_cleanup): # from bbot_server import BBOTServer -# from bbot_server.watchdog import BBOTWatchdog -# from bbot_server.applets.base import BaseApplet, watchdog_task +# from bbot_server.worker import BBOTWorker +# from bbot_server.applets.base import BaseApplet, worker_task # class ScheduledTaskApplet(BaseApplet): # name = "Scheduled Tasks" @@ -24,15 +24,15 @@ # self.cron_task_2_ran = False # self.cron_task_3_ran = False -# @watchdog_task(cron="* * * * *") +# @worker_task(cron="* * * * *") # async def cron_task(self): # self.cron_task_ran = True -# @watchdog_task(cron="* * * * *", cron_config_key="test.cron_task_2") +# @worker_task(cron="* * * * *", cron_config_key="test.cron_task_2") # async def cron_task_2(self): # self.cron_task_2_ran = True -# @watchdog_task(cron="* * * * *", cron_config_key="test.cron_task_3") +# @worker_task(cron="* * * * *", cron_config_key="test.cron_task_3") # async def cron_task_3(self): # self.cron_task_3_ran = True @@ -40,21 +40,21 @@ # bbot_server = BBOTServer(config=config) # await bbot_server.setup() -# watchdog = BBOTWatchdog(bbot_server) -# await watchdog.start() +# worker = BBOTWorker(bbot_server) +# await worker.start() # try: # app = bbot_server.include_app(ScheduledTaskApplet) # # register tasks on the bbot server side # await app._setup() -# # register tasks on the watchdog side -# await app.register_watchdog_tasks(watchdog.broker) +# # register tasks on the worker side +# await app.register_worker_tasks(worker.broker) # assert bbot_server.scheduled_tasks.cron_task_ran is False, "cron_task ran before setup" # assert bbot_server.scheduled_tasks.cron_task_2_ran is False, "cron_task_2 ran before setup" # assert bbot_server.scheduled_tasks.cron_task_3_ran is False, "cron_task_3 ran before setup" -# all_tasks = watchdog.broker.get_all_tasks() +# all_tasks = worker.broker.get_all_tasks() # assert "tests.test_scheduled_tasks:cron_task" in all_tasks, "cron_task is not registered" # assert "tests.test_scheduled_tasks:cron_task_2" in all_tasks, "cron_task_2 is not registered" @@ -84,4 +84,4 @@ # with suppress(Exception): # await bbot_server.stop() # with suppress(Exception): -# await watchdog.stop() +# await worker.stop() diff --git a/tests/test_watchdog.py b/tests/test_worker.py similarity index 83% rename from tests/test_watchdog.py rename to tests/test_worker.py index 7f545acb..eae2ba0d 100644 --- a/tests/test_watchdog.py +++ b/tests/test_worker.py @@ -6,21 +6,21 @@ from bbot_server import BBOTServer from bbot.models.pydantic import Event -from bbot_server.watchdog import BBOTWatchdog +from bbot_server.worker import BBOTWorker from .conftest import INGEST_PROCESSING_DELAY @pytest.mark.asyncio -async def test_watchdog(bbot_events): +async def test_worker(bbot_events): bbot_server = BBOTServer() await bbot_server.setup() - watchdog = BBOTWatchdog(bbot_server) - await watchdog.start() + worker = BBOTWorker(bbot_server) + await worker.start() try: - @watchdog.broker.task + @worker.broker.task async def insert_event( event: Event, context: Annotated[Context, TaskiqDepends()], @@ -42,5 +42,5 @@ async def insert_event( assert db_events assert len(db_events) == len(scan1_events) finally: - await watchdog.stop() + await worker.stop() await bbot_server.cleanup() diff --git a/uv.lock b/uv.lock index 4fee0be2..7e816e02 100644 --- a/uv.lock +++ b/uv.lock @@ -178,7 +178,7 @@ dependencies = [ [[package]] name = "bbot-server" -version = "0.1.0" +version = "0.1.3" source = { editable = "." } dependencies = [ { name = "bbot" },