Feat: Add celery beat for scheduling new cronjobs

README.md

@@ -190,9 +190,14 @@ For updating the index
 ```(bash)
 docker-compose exec serve bash python manage.py update_index
 ```
+## Cronjobs
+
+For more info checkout [Cronjobs](./docs/cronjobs.md)
+
 ## Sentry
 
-For updating the cron monitored tasks
+For updating the cron monitored tasks (legacy k8s CronJobs only — celery beat
+cronjobs are registered automatically)
 ```(bash)
 docker-compose exec serve bash ./manage.py cron_job_monitor
 ```

api/management/commands/run_celery_dev.py

@@ -4,10 +4,7 @@
 from django.core.management.base import BaseCommand
 from django.utils.autoreload import run_with_reloader
 
-from main.celery import Queues
-
-all_queues = ",".join([q for q in Queues.DEV_QUEUES])
-CMD = f"celery -A main worker -Q {all_queues} --concurrency=2 -l info"
+CMD = "celery -A main worker -E --concurrency=2 -l info"
 
 
 def restart_celery():

api/management/commands/run_celery_prod.py

@@ -5,9 +5,9 @@
 
 from django.core.management.base import BaseCommand
 
-from main.celery import Queues
+from main.cronjobs import CeleryQueue
 
-all_queues = ",".join([q for q in Queues.DEV_QUEUES])
+all_queues = ",".join([q.name for q in CeleryQueue.ALL_QUEUE])
 
 # NOTE: Use a fixed concurrency to prevent the pod from being OOMKilled,
 # as Celery defaults to one worker per available CPU.

api/tasks.py

@@ -2,10 +2,12 @@
 
 from celery import shared_task
 from django.contrib.auth.models import User
+from django.core import management
 from django.utils import timezone
 from rest_framework.authtoken.models import Token
 
 from api.playwright import render_pdf_from_url
+from main.lock import RedisLockKey, redis_lock
 from main.utils import logger_context
 
 from .logger import logger
@@ -59,3 +61,19 @@ def generate_export_pdf(export_id, title, set_user_language="en"):
         export.status = Export.ExportStatus.ERRORED
         export.save(update_fields=["status"])
     logger.info(f"End export: {export.pk}")
+
+
+@shared_task
+def clear_expired_django_sessions():
+    """Purge expired django_session rows -- nothing else prunes them."""
+    with redis_lock(RedisLockKey.CLEAR_EXPIRED_DJANGO_SESSIONS) as acquired:
+        if not acquired:
+            logger.warning("clear_expired_django_sessions: already running, skipping")
+            return
+        management.call_command("clearsessions", verbosity=0)
+
+
+@shared_task
+def celery_queue_uptime_check(queue: str) -> None:
+    """No-op heartbeat proving beat dispatches and that `queue` has a consumer."""
+    logger.info("Celery queue '%s' is taking tasks", queue)

api/views.py

@@ -558,7 +558,7 @@ def get(cls, request):
             "cronjob_err": c,
             "maintenance_mode": settings.DJANGO_READ_ONLY,
             "git_last_tag": settings.LAST_GIT_TAG,
-            "git_last_commit": settings.SENTRY_CONFIG["release"][0:8],
+            "git_last_commit": settings.SENTRY_CONFIG.release[0:8],
         }
         return JsonResponse(res, safe=False)
 

deploy/helm/snapshots/alpha.yaml

@@ -42,6 +42,8 @@ data:
   ELASTIC_SEARCH_HOST: "http://go-api-elasticsearch.example.com:9200"
   FRONTEND_URL: "https://go.alpha.example.com"
   GO_ENVIRONMENT: "ALPHA"
+  SENTRY_DEBUG: "false"
+  SENTRY_MONITOR_CELERY_BEAT_TASKS: "true"
   SENTRY_SAMPLE_RATE: "0.2"
   SESSION_COOKIE_DOMAIN: ".example.com"
 
@@ -107,7 +109,7 @@ spec:
     metadata:
       annotations:
         checksum/secret: dd5e71c25a35366086506b396869b06a501d1ec22e83806a15738e9f3ac3a03d
-        checksum/configmap: 0313dd0020b57c2897db8d61fed2b2611c7bcb7840d7dbc1f37fe90f32881f92
+        checksum/configmap: a69dd58e3dc31298d8e3f2ee16870920ad8ef2c9ad466f5e1ac7d802a9b00449
       labels:
         app: go-api
         component: api
@@ -169,6 +171,95 @@ spec:
               sleep:
                 seconds: 8
 
+---
+# Source: ifrcgo-helm/charts/app/templates/worker-addons/deployment.yaml
+# Addon: beat
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: go-api-worker-beat
+  annotations:
+    argocd.argoproj.io/sync-wave: "30"
+    reloader.stakater.com/auto: "true"
+  labels:
+    app: go-api
+    component: worker-addon
+    addon: beat
+    environment: ALPHA
+    release: release-name
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  revisionHistoryLimit: 1
+  selector:
+    matchLabels:
+      app: go-api
+      component: worker-addon
+      addon: beat
+  template:
+    metadata:
+      annotations:
+        checksum/secret: dd5e71c25a35366086506b396869b06a501d1ec22e83806a15738e9f3ac3a03d
+        checksum/configmap: a69dd58e3dc31298d8e3f2ee16870920ad8ef2c9ad466f5e1ac7d802a9b00449
+      labels:
+        app: go-api
+        component: worker-addon
+        addon: beat
+    spec:
+      volumes:
+        - emptyDir:
+            medium: Memory
+            sizeLimit: 1Mi
+          name: celery-heartbeat
+      containers:
+        - name: beat
+          image: "local.example.com/ifrcgo/go-api:develop.xxxxxxxx"
+          imagePullPolicy: IfNotPresent
+          command:
+            - celery
+            - -A
+            - main
+            - beat
+            - -l
+            - INFO
+            - --scheduler=banjo_utils.celery_health.database.HeartbeatDatabaseScheduler
+          livenessProbe:
+            exec:
+              command:
+              - banjo-celery-probe
+              - --max-age
+              - "60"
+            failureThreshold: 3
+            periodSeconds: 60
+            timeoutSeconds: 5
+          resources:
+            limits:
+              memory: 512Mi
+            requests:
+              cpu: "0.1"
+              memory: 256Mi
+          envFrom:
+            - secretRef:
+                name: go-api-secret
+            - configMapRef:
+                name: go-api-env-name
+          env:
+            - name: DJANGO_APP_TYPE
+              value: "worker"
+            - name: CACHE_REDIS_URL
+              value: redis://go-api-dragonfly:6379/1
+            - name: CELERY_REDIS_URL
+              value: redis://go-api-dragonfly:6379/0
+            - name: PLAYWRIGHT_SERVER_URL
+              value: ws://go-api-playwright:3000/
+
+            - name: BANJO_CELERY_HEARTBEAT_FILE
+              value: "/var/run/celery/beat_heartbeat"
+          volumeMounts:
+            - mountPath: /var/run/celery
+              name: celery-heartbeat
+
 ---
 # Source: ifrcgo-helm/charts/app/templates/worker/deployment.yaml
 # Queue: default
@@ -196,7 +287,7 @@ spec:
     metadata:
       annotations:
         checksum/secret: dd5e71c25a35366086506b396869b06a501d1ec22e83806a15738e9f3ac3a03d
-        checksum/configmap: 0313dd0020b57c2897db8d61fed2b2611c7bcb7840d7dbc1f37fe90f32881f92
+        checksum/configmap: a69dd58e3dc31298d8e3f2ee16870920ad8ef2c9ad466f5e1ac7d802a9b00449
       labels:
         app: go-api
         component: worker
@@ -326,7 +417,7 @@ spec:
     metadata:
       annotations:
         checksum/secret: dd5e71c25a35366086506b396869b06a501d1ec22e83806a15738e9f3ac3a03d
-        checksum/configmap: 0313dd0020b57c2897db8d61fed2b2611c7bcb7840d7dbc1f37fe90f32881f92
+        checksum/configmap: a69dd58e3dc31298d8e3f2ee16870920ad8ef2c9ad466f5e1ac7d802a9b00449
       labels:
         app: go-api
         component: hook
@@ -384,7 +475,7 @@ spec:
     metadata:
       annotations:
         checksum/secret: dd5e71c25a35366086506b396869b06a501d1ec22e83806a15738e9f3ac3a03d
-        checksum/configmap: 0313dd0020b57c2897db8d61fed2b2611c7bcb7840d7dbc1f37fe90f32881f92
+        checksum/configmap: a69dd58e3dc31298d8e3f2ee16870920ad8ef2c9ad466f5e1ac7d802a9b00449
       labels:
         app: go-api
         component: hook
@@ -440,7 +531,7 @@ spec:
     metadata:
       annotations:
         checksum/secret: dd5e71c25a35366086506b396869b06a501d1ec22e83806a15738e9f3ac3a03d
-        checksum/configmap: 0313dd0020b57c2897db8d61fed2b2611c7bcb7840d7dbc1f37fe90f32881f92
+        checksum/configmap: a69dd58e3dc31298d8e3f2ee16870920ad8ef2c9ad466f5e1ac7d802a9b00449
       labels:
         app: go-api
         component: hook