feat: initial Skeen-CRM AI Agent architecture
- FastAPI + Python 3.12 backend - Meta WhatsApp Business API client (official) - OpenAI GPT-4o with function calling - RAG vector store with pgvector - ERPNext Frappe REST client - Celery + Redis async task queue - PostgreSQL with migrations (Alembic) - Docker Compose full stack - Enterprise logging, metrics, health checks
This commit is contained in:
0
src/workers/__init__.py
Normal file
0
src/workers/__init__.py
Normal file
61
src/workers/celery_app.py
Normal file
61
src/workers/celery_app.py
Normal file
@@ -0,0 +1,61 @@
|
||||
"""Celery configuration for background task processing."""
|
||||
|
||||
import os
|
||||
|
||||
from celery import Celery
|
||||
from celery.signals import setup_logging
|
||||
|
||||
from src.config import settings
|
||||
|
||||
os.environ.setdefault("CELERY_CONFIG_MODULE", "src.workers.celery_app")
|
||||
|
||||
celery_app = Celery(
|
||||
"skeen_crm",
|
||||
broker=str(settings.CELERY_BROKER_URL),
|
||||
backend=str(settings.CELERY_RESULT_BACKEND),
|
||||
include=["src.workers.tasks"],
|
||||
)
|
||||
|
||||
celery_app.conf.update(
|
||||
task_serializer="json",
|
||||
accept_content=["json"],
|
||||
result_serializer="json",
|
||||
timezone="America/Tijuana",
|
||||
enable_utc=True,
|
||||
task_track_started=True,
|
||||
task_time_limit=300,
|
||||
worker_prefetch_multiplier=1,
|
||||
worker_concurrency=settings.CELERY_WORKER_CONCURRENCY,
|
||||
task_routes={
|
||||
"src.workers.tasks.process_whatsapp_message_task": {"queue": "whatsapp"},
|
||||
"src.workers.tasks.sync_erpnext_task": {"queue": "erpnext"},
|
||||
"src.workers.tasks.generate_embedding_task": {"queue": "ai"},
|
||||
},
|
||||
task_default_queue="default",
|
||||
)
|
||||
|
||||
|
||||
@setup_logging.connect
|
||||
def config_loggers(*args, **kwargs) -> None:
|
||||
"""Configure structlog for Celery workers."""
|
||||
import logging
|
||||
import structlog
|
||||
|
||||
structlog.configure(
|
||||
processors=[
|
||||
structlog.stdlib.filter_by_level,
|
||||
structlog.stdlib.add_logger_name,
|
||||
structlog.stdlib.add_log_level,
|
||||
structlog.stdlib.PositionalArgumentsFormatter(),
|
||||
structlog.processors.TimeStamper(fmt="iso"),
|
||||
structlog.processors.StackInfoRenderer(),
|
||||
structlog.processors.format_exc_info,
|
||||
structlog.processors.UnicodeDecoder(),
|
||||
structlog.processors.JSONRenderer(),
|
||||
],
|
||||
context_class=dict,
|
||||
logger_factory=structlog.stdlib.LoggerFactory(),
|
||||
wrapper_class=structlog.stdlib.BoundLogger,
|
||||
cache_logger_on_first_use=True,
|
||||
)
|
||||
logging.basicConfig(format="%(message)s", level=logging.INFO)
|
||||
67
src/workers/tasks.py
Normal file
67
src/workers/tasks.py
Normal file
@@ -0,0 +1,67 @@
|
||||
"""Celery background tasks."""
|
||||
|
||||
import structlog
|
||||
|
||||
from src.infrastructure.db import AsyncSessionLocal
|
||||
from src.use_cases.handle_incoming_message import process_incoming_message
|
||||
from src.workers.celery_app import celery_app
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
|
||||
@celery_app.task(bind=True, max_retries=3, default_retry_delay=10)
|
||||
def process_whatsapp_message_task(self, webhook_data: dict, client_ip: str = "unknown") -> dict:
|
||||
"""Process WhatsApp message asynchronously.
|
||||
|
||||
This task runs in a Celery worker and handles the full AI pipeline
|
||||
so the webhook endpoint can return 200 immediately to Meta.
|
||||
"""
|
||||
import asyncio
|
||||
|
||||
async def _process() -> dict:
|
||||
async with AsyncSessionLocal() as db:
|
||||
try:
|
||||
result = await process_incoming_message(
|
||||
db=db,
|
||||
webhook_data=webhook_data,
|
||||
client_ip=client_ip,
|
||||
)
|
||||
return result
|
||||
except Exception as exc:
|
||||
logger.error("whatsapp_task_failed", error=str(exc), attempt=self.request.retries)
|
||||
raise self.retry(exc=exc)
|
||||
|
||||
return asyncio.run(_process())
|
||||
|
||||
|
||||
@celery_app.task
|
||||
def sync_erpnext_patient_task(patient_data: dict) -> dict:
|
||||
"""Sync patient data to ERPNext in background."""
|
||||
logger.info("syncing_patient_to_erpnext", patient=patient_data.get("name"))
|
||||
# TODO: Implement ERPNext sync logic
|
||||
return {"status": "synced", "patient": patient_data.get("name")}
|
||||
|
||||
|
||||
@celery_app.task
|
||||
def generate_embedding_task(document_id: str, content: str, category: str) -> dict:
|
||||
"""Generate embedding for a document chunk asynchronously."""
|
||||
import asyncio
|
||||
|
||||
async def _generate() -> dict:
|
||||
from src.infrastructure.ai.openai_client import get_openai_client
|
||||
from src.infrastructure.db import AsyncSessionLocal
|
||||
from src.infrastructure.ai.rag import RAGStore
|
||||
|
||||
async with AsyncSessionLocal() as db:
|
||||
client = await get_openai_client()
|
||||
embedding = await client.create_embedding(content)
|
||||
|
||||
store = RAGStore(db)
|
||||
await store.add_document(
|
||||
content=content,
|
||||
category=category,
|
||||
doc_id=document_id,
|
||||
)
|
||||
return {"document_id": document_id, "embedding_dims": len(embedding)}
|
||||
|
||||
return asyncio.run(_generate())
|
||||
Reference in New Issue
Block a user