Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ bbctl server start

## Deploy with Helm (Kubernetes)

BBOT Server can be deployed to Kubernetes using its official Helm chart. The chart deploys the API server, watchdog, MongoDB, and Redis.
BBOT Server can be deployed to Kubernetes using its official Helm chart. The chart deploys the API server, worker, MongoDB, and Redis.

### Quick Start

Expand Down
18 changes: 9 additions & 9 deletions bbot_server/applets/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,14 +49,14 @@ def decorator(fn):
return decorator


def watchdog_task(**kwargs):
def worker_task(**kwargs):
"""
Decorate your applet method with this to make it a watchdog task
Decorate your applet method with this to make it a worker task
"""

def decorator(fn):
fn._kwargs = kwargs
fn._watchdog_task = True
fn._worker_task = True
return fn

return decorator
Expand Down Expand Up @@ -240,8 +240,8 @@ async def _native_setup(self):
self.task_broker = await self.message_queue.make_taskiq_broker()
await self.task_broker.startup()

# register watchdog tasks
await self.register_watchdog_tasks(self.task_broker)
# register worker tasks
await self.register_worker_tasks(self.task_broker)

if self.name != "Root Applet":
try:
Expand Down Expand Up @@ -290,15 +290,15 @@ async def reconcile_all_indexes(self):
diff = compute_index_diff(desired, desired_text, existing, existing_text)
await apply_index_diff(collection, diff, existing)

async def register_watchdog_tasks(self, broker):
# register watchdog tasks
async def register_worker_tasks(self, broker):
# register worker tasks
methods = {name: member for name, member in getmembers(self) if callable(member)}
for method_name, method in methods.items():
# handle case where tasks have already been registered
method = getattr(method, "original_func", method)

_watchdog_task = getattr(method, "_watchdog_task", None)
if _watchdog_task is None:
_worker_task = getattr(method, "_worker_task", None)
if _worker_task is None:
continue
kwargs = getattr(method, "_kwargs", {})
# crontab handling
Expand Down
4 changes: 2 additions & 2 deletions bbot_server/modules/emails/emails_api.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# from bbot_server.watchdogs.emails import EmailWatchdog
# from bbot_server.workers.emails import EmailWorker
from bbot_server.applets.base import BaseApplet, api_endpoint, BaseModel, Field


Expand All @@ -7,7 +7,7 @@ class EmailsApplet(BaseApplet):
description = "emails discovered during scans"
# watched_events = ["EMAIL_ADDRESS"]
route_prefix = ""
# watchdogs = [EmailWatchdog]
# workers = [EmailWorker]
attach_to = "assets"

class AssetFields(BaseModel):
Expand Down
2 changes: 1 addition & 1 deletion bbot_server/modules/events/events_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ async def insert_event(self, event: Event):
Insert a BBOT event into the asset database
"""
# publish event to the message queue
# it will be picked up by the watchdog and ingested
# it will be picked up by the worker and ingested
await self.root.message_queue.publish_event(event)

@api_endpoint("/get/{uuid}", methods=["GET"], summary="Get an event by its UUID")
Expand Down
22 changes: 11 additions & 11 deletions bbot_server/modules/server/server_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@ def start(
api_only: Annotated[
bool, Option("--api-only", "-a", help="Only start the REST API, without Docker Compose")
] = False,
watchdog_only: Annotated[
bool, Option("--watchdog-only", "-w", help="Only start the watchdog, without Docker Compose")
worker_only: Annotated[
bool, Option("--worker-only", "-w", help="Only start the worker, without Docker Compose")
] = False,
listen: Annotated[str, Option("--listen", "-l", help="Listen address", metavar="IP_ADDRESS")] = "127.0.0.1",
port: Annotated[int, Option("--port", "-p", help="Port to run the server on", metavar="PORT")] = 8807,
Expand Down Expand Up @@ -60,30 +60,30 @@ def start(
workers=1,
)

elif watchdog_only:
print("Starting watchdog")
elif worker_only:
print("Starting worker")

async def run_watchdog():
async def run_worker():
try:
from bbot_server import BBOTServer
from bbot_server.watchdog import BBOTWatchdog
from bbot_server.worker import BBOTWorker

bbot_server = BBOTServer()
await bbot_server.setup()

watchdog = BBOTWatchdog(bbot_server)
await watchdog.start()
print("Watchdog successfully started")
worker = BBOTWorker(bbot_server)
await worker.start()
print("Worker successfully started")

# sleep for infinity
event = asyncio.Event()
await event.wait()

except KeyboardInterrupt:
with suppress(Exception):
await watchdog.stop()
await worker.stop()

asyncio.run(run_watchdog())
asyncio.run(run_worker())

else:
# initialize the config if not already
Expand Down
3 changes: 0 additions & 3 deletions bbot_server/watchdog/__init__.py

This file was deleted.

3 changes: 3 additions & 0 deletions bbot_server/worker/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
from .worker import BBOTWorker

__all__ = ["BBOTWorker"]
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
from bbot_server.modules.activity.activity_models import Activity


class BBOTWatchdog:
class BBOTWorker:
"""
Contains:
- taskiq worker
Expand Down Expand Up @@ -40,9 +40,9 @@ async def startup(state: TaskiqState) -> None:

await self.broker.startup()

# register watchdog tasks
# register worker tasks
for app in self.bbot_server.all_child_applets(include_self=True):
await app.register_watchdog_tasks(self.broker)
await app.register_worker_tasks(self.broker)

# taskiq worker tasks
self.taskiq_worker_task = asyncio.create_task(run_receiver_task(self.broker))
Expand Down Expand Up @@ -157,7 +157,7 @@ async def _get_or_create_asset(self, host: str, event: Event = None, parent_acti
return asset, activities

async def stop(self) -> None:
self.log.info("Stopping watchdog")
self.log.info("Stopping worker")
await self.bbot_server.message_queue.unsubscribe(self.event_listener)
self.taskiq_worker_task.cancel()
self.taskiq_scheduler_task.cancel()
Expand Down
4 changes: 2 additions & 2 deletions compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,9 @@ services:
- mongodb
- redis

watchdog:
worker:
<<: *bbot-server-base
command: ["bbctl", "server", "start", "--watchdog-only"]
command: ["bbctl", "server", "start", "--worker-only"]
depends_on:
server:
condition: service_healthy
Expand Down
2 changes: 1 addition & 1 deletion helm/Chart.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
apiVersion: v2
name: bbot-server
description: Helm chart for BBOT Server (API + Watchdog)
description: Helm chart for BBOT Server (API + Worker)
type: application
version: 0.1.0
appVersion: "0.1.0"
Expand Down
Original file line number Diff line number Diff line change
@@ -1,19 +1,19 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ .Release.Name }}-watchdog
name: {{ .Release.Name }}-worker
labels:
app: bbot-watchdog
app: bbot-worker
app.kubernetes.io/instance: {{ .Release.Name | quote }}
spec:
replicas: {{ .Values.watchdog.replicas }}
replicas: {{ .Values.worker.replicas }}
selector:
matchLabels:
app: bbot-watchdog
app: bbot-worker
template:
metadata:
labels:
app: bbot-watchdog
app: bbot-worker
spec:
securityContext:
runAsNonRoot: true
Expand All @@ -24,7 +24,7 @@ spec:
{{- toYaml .Values.imagePullSecrets | nindent 8 }}
{{- end }}
{{- if .Values.server.enabled }}
# Wait for the API server to become healthy before starting the watchdog
# Wait for the API server to become healthy before starting the worker
initContainers:
- name: wait-for-server
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
Expand All @@ -39,15 +39,15 @@ spec:
echo "Server not ready yet, retrying in 5s..."
sleep 5
done
echo "Server is healthy, starting watchdog."
echo "Server is healthy, starting worker."
{{- end }}
containers:
- name: watchdog
- name: worker
securityContext:
allowPrivilegeEscalation: false
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
command: ["bbctl", "server", "start", "--watchdog-only"]
command: ["bbctl", "server", "start", "--worker-only"]
env:
# Internal password references (used for URI interpolation below)
- name: MONGO_PASSWORD
Expand Down Expand Up @@ -86,7 +86,7 @@ spec:
# Redis URI
- name: BBOT_SERVER_MESSAGE_QUEUE__URI
value: "redis://:$(REDIS_PASSWORD)@{{ .Release.Name }}-redis-master:6379/0"
{{- with .Values.watchdog.resources }}
{{- with .Values.worker.resources }}
resources:
{{- toYaml . | nindent 10 }}
{{- end }}
6 changes: 3 additions & 3 deletions helm/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ imagePullSecrets: []
# Server (API)
# =============================================================================
server:
# -- Set to false to disable the API server (only deploy DBs + watchdog)
# -- Set to false to disable the API server (only deploy DBs + worker)
enabled: true
replicas: 1
resources: {}
Expand All @@ -32,9 +32,9 @@ server:
targetPort: 8807

# =============================================================================
# Watchdog (event processor)
# Worker (event processor)
# =============================================================================
watchdog:
worker:
replicas: 1
resources: {}

Expand Down
4 changes: 2 additions & 2 deletions helm_deployment_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -328,8 +328,8 @@ def test_ingest_and_query_assets(self):
print(f"Ingest stderr: {result.stderr}")
self.assertEqual(result.returncode, 0, f"Event ingest failed: {result.stderr}")

# Wait for the watchdog to process events into assets
print("Waiting for watchdog to process events into assets...")
# Wait for the worker to process events into assets
print("Waiting for worker to process events into assets...")
assets = []
for i in range(30):
result = self.kubectl(
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[project]
name = "bbot-server"
version = "0.1.1"
version = "0.1.2"
description = ""
authors = [{name = "TheTechromancer"}]
license = "AGPL-3.0"
Expand Down
44 changes: 22 additions & 22 deletions tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ async def bbot_server(request, mongo_cleanup, redis_cleanup):
message_queue = None

async def _make_bbot_server(
config_overrides=None, needs_agent=False, needs_api=False, needs_watchdog=True, **kwargs
config_overrides=None, needs_agent=False, needs_api=False, needs_worker=True, **kwargs
):
nonlocal bbot_server

Expand All @@ -77,9 +77,9 @@ async def _make_bbot_server(
await message_queue.setup()
await message_queue.clear()

# watchdog
if needs_watchdog:
request.getfixturevalue("bbot_watchdog")
# worker
if needs_worker:
request.getfixturevalue("bbot_worker")

# http server
if needs_api or kwargs["interface"] == "http":
Expand All @@ -101,46 +101,46 @@ async def _make_bbot_server(


@pytest.fixture
def bbot_watchdog(mongo_cleanup, redis_cleanup):
command = [*BBCTL_COMMAND, "server", "start", "--watchdog-only"]
watchdog_process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, bufsize=1)
def bbot_worker(mongo_cleanup, redis_cleanup):
command = [*BBCTL_COMMAND, "server", "start", "--worker-only"]
worker_process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, bufsize=1)
try:
# Wait for watchdog to be ready by monitoring stderr
# Wait for worker to be ready by monitoring stderr
ready = False
while watchdog_process.poll() is None: # 10 second timeout (50 * 0.2)
line = watchdog_process.stderr.readline()
log.critical(f"Watchdog: {line.strip()}")
if "Watchdog started" in line:
while worker_process.poll() is None: # 10 second timeout (50 * 0.2)
line = worker_process.stderr.readline()
log.critical(f"Worker: {line.strip()}")
if "Worker started" in line:
ready = True
break
if "[INFO] Subscribed to bbot:stream:events" in line:
ready = True
break

if not ready:
raise Exception("Watchdog failed to start and subscribe to events")
raise Exception("Worker failed to start and subscribe to events")

# here, start a thread to tail the watchdog's stderr
# here, start a thread to tail the worker's stderr
def tail_stderr():
while watchdog_process.poll() is None:
line = watchdog_process.stderr.readline()
while worker_process.poll() is None:
line = worker_process.stderr.readline()
if line:
log.critical(f"Watchdog: {line.strip()}")
log.critical(f"Worker: {line.strip()}")

import threading

stderr_thread = threading.Thread(target=tail_stderr, daemon=True)
stderr_thread.start()

yield watchdog_process
yield worker_process

watchdog_process.send_signal(signal.SIGINT)
worker_process.send_signal(signal.SIGINT)
finally:
try:
watchdog_process.wait(timeout=5)
worker_process.wait(timeout=5)
except subprocess.TimeoutExpired:
log.error("Watchdog process timed out, killing forcefully")
watchdog_process.kill()
log.error("Worker process timed out, killing forcefully")
worker_process.kill()


@pytest.fixture
Expand Down
4 changes: 2 additions & 2 deletions tests/test_applets/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ class BaseAppletTest:

needs_api = False
needs_agent = False
needs_watchdog = False
needs_worker = False

# if True, don't test the http interface
native_only = False
Expand Down Expand Up @@ -55,7 +55,7 @@ async def test_applet_run(self, bbot_server, bbot_events):
config_overrides=self.config_overrides,
needs_api=self.needs_api,
needs_agent=self.needs_agent,
needs_watchdog=self.needs_watchdog,
needs_worker=self.needs_worker,
)

if self.native_only and not self.bbot_server.is_native:
Expand Down
Loading
Loading