0.1.0

wolai-backend/app/workers/celery_app.py

@@ -1,15 +1,53 @@
+import os
+
 from celery import Celery
+from kombu import Queue
 
 from app.config import settings
 
 
+def _env_flag(name: str, default: bool = False) -> bool:
+    value = os.getenv(name)
+    if value is None:
+        return default
+    return value.strip().lower() in {"1", "true", "yes", "on"}
+
+
 def create_celery_app() -> Celery:
     app = Celery("wolai-backend")
     app.conf.broker_url = settings.redis_url
     app.conf.result_backend = settings.redis_url
-    app.conf.task_routes = {"app.workers.tasks.*": {"queue": "wolai-tasks"}}
+    app.conf.task_default_queue = "wolai-index"
+    app.conf.task_queues = (
+        Queue("wolai-ocr", routing_key="tasks.ocr", max_priority=10),
+        Queue("wolai-index", routing_key="tasks.index", max_priority=10),
+    )
+    app.conf.task_routes = {
+        "app.workers.tasks.ocr_pipeline": {
+            "queue": "wolai-ocr",
+            "routing_key": "tasks.ocr",
+            "priority": 0,
+        },
+        "app.workers.tasks.media_ocr_pipeline": {
+            "queue": "wolai-ocr",
+            "routing_key": "tasks.ocr",
+            "priority": 1,
+        },
+        "app.workers.tasks.lightrag_index_pipeline": {
+            "queue": "wolai-index",
+            "routing_key": "tasks.index",
+            "priority": 5,
+        },
+    }
+    app.conf.broker_transport_options = {"priority_steps": list(range(10))}
+    app.conf.worker_prefetch_multiplier = 1
+    app.conf.task_acks_late = True
+    app.conf.worker_concurrency = 3
+    app.conf.task_always_eager = _env_flag("CELERY_TASK_ALWAYS_EAGER", False)
+    app.conf.task_eager_propagates = _env_flag("CELERY_TASK_EAGER_PROPAGATES", True)
     app.autodiscover_tasks(["app.workers"])
     return app
 
 
 celery_app = create_celery_app()
+celery_app.set_default()