Autoparts-DB/manager/services/portfolio_service.py

"""Portfolio operations service: per-tenant business metrics rollup, health
scores and cross-tenant anomaly detection for the manager.

Design notes
------------
* The daily rollup lives in the **master** database table
  ``tenant_metrics_daily`` (one row per tenant per day). It is created
  idempotently here and by ``scripts/rollup_tenant_metrics.py`` — the tenant
  migration runner is intentionally not involved.
* Point-in-time counts (inventory, customers, employees, db size, fitment)
  are snapshots taken at collection time; daily activity metrics (sales,
  new customers, audit events, API calls) are computed for the requested day.
* Data sources are guarded: optional tables (``audit_log``,
  ``api_request_logs``, ``inventory_vehicle_compat``) may be absent in an older
  tenant and simply yield 0 / NULL instead of failing the rollup.
* DSNs are resolved from environment exactly like the manager config
  (``MASTER_DB_URL`` / ``TENANT_DB_URL_TEMPLATE``) with a local fallback, so the
  script can run standalone without the manager's required secrets.
"""

import os
from datetime import date as date_cls
from datetime import datetime, time, timedelta
from zoneinfo import ZoneInfo

import psycopg2
from psycopg2.extras import Json

from services.business_metrics import (
    DEFAULT_Z,
    MIN_COHORT,
    compute_health,
    detect_portfolio_anomalies,
)

BUSINESS_TZ = os.environ.get("PORTFOLIO_TZ", "America/Mexico_City")

_PGUSER = os.environ.get("PGUSER", "postgres")
MASTER_DB_URL = (
    os.environ.get("MASTER_DB_URL")
    or os.environ.get("DATABASE_URL")
    or f"postgresql://{_PGUSER}@localhost/nexus_autoparts"
)
TENANT_DB_URL_TEMPLATE = (
    os.environ.get("TENANT_DB_URL_TEMPLATE")
    or f"postgresql://{_PGUSER}@localhost/{{db_name}}"
)

DDL = """
CREATE TABLE IF NOT EXISTS tenant_metrics_daily (
    tenant_id           INTEGER NOT NULL REFERENCES tenants(id) ON DELETE CASCADE,
    date                DATE NOT NULL,
    ventas_count        INTEGER NOT NULL DEFAULT 0,
    ventas_total        NUMERIC(14,2) NOT NULL DEFAULT 0,
    ticket_promedio     NUMERIC(12,2) NOT NULL DEFAULT 0,
    ventas_canceladas   INTEGER NOT NULL DEFAULT 0,
    clientes_nuevos     INTEGER NOT NULL DEFAULT 0,
    items_activos       INTEGER NOT NULL DEFAULT 0,
    skus                INTEGER NOT NULL DEFAULT 0,
    clientes            INTEGER NOT NULL DEFAULT 0,
    empleados_activos   INTEGER NOT NULL DEFAULT 0,
    fitment_pct         NUMERIC(5,2),
    uso_modulos         JSONB NOT NULL DEFAULT '{}'::jsonb,
    ai_calls            INTEGER NOT NULL DEFAULT 0,
    api_calls           INTEGER NOT NULL DEFAULT 0,
    errores_5xx         INTEGER NOT NULL DEFAULT 0,
    actividad_count     INTEGER NOT NULL DEFAULT 0,
    ultima_venta_at     TIMESTAMPTZ,
    ultima_actividad_at TIMESTAMPTZ,
    db_size_mb          NUMERIC(12,2),
    collected_at        TIMESTAMPTZ NOT NULL DEFAULT NOW(),
    PRIMARY KEY (tenant_id, date)
);
CREATE INDEX IF NOT EXISTS idx_tenant_metrics_daily_date
    ON tenant_metrics_daily (date);
"""

UPSERT = """
INSERT INTO tenant_metrics_daily (
    tenant_id, date, ventas_count, ventas_total, ticket_promedio,
    ventas_canceladas, clientes_nuevos, items_activos, skus, clientes,
    empleados_activos, fitment_pct, uso_modulos, ai_calls, api_calls,
    errores_5xx, actividad_count, ultima_venta_at, ultima_actividad_at,
    db_size_mb, collected_at
) VALUES (
    %(tenant_id)s, %(date)s, %(ventas_count)s, %(ventas_total)s,
    %(ticket_promedio)s, %(ventas_canceladas)s, %(clientes_nuevos)s,
    %(items_activos)s, %(skus)s, %(clientes)s, %(empleados_activos)s,
    %(fitment_pct)s, %(uso_modulos)s, %(ai_calls)s, %(api_calls)s,
    %(errores_5xx)s, %(actividad_count)s, %(ultima_venta_at)s,
    %(ultima_actividad_at)s, %(db_size_mb)s, NOW()
)
ON CONFLICT (tenant_id, date) DO UPDATE SET
    ventas_count = EXCLUDED.ventas_count,
    ventas_total = EXCLUDED.ventas_total,
    ticket_promedio = EXCLUDED.ticket_promedio,
    ventas_canceladas = EXCLUDED.ventas_canceladas,
    clientes_nuevos = EXCLUDED.clientes_nuevos,
    items_activos = EXCLUDED.items_activos,
    skus = EXCLUDED.skus,
    clientes = EXCLUDED.clientes,
    empleados_activos = EXCLUDED.empleados_activos,
    fitment_pct = EXCLUDED.fitment_pct,
    uso_modulos = EXCLUDED.uso_modulos,
    ai_calls = EXCLUDED.ai_calls,
    api_calls = EXCLUDED.api_calls,
    errores_5xx = EXCLUDED.errores_5xx,
    actividad_count = EXCLUDED.actividad_count,
    ultima_venta_at = EXCLUDED.ultima_venta_at,
    ultima_actividad_at = EXCLUDED.ultima_actividad_at,
    db_size_mb = EXCLUDED.db_size_mb,
    collected_at = NOW()
"""


def get_master_conn():
    return psycopg2.connect(MASTER_DB_URL, connect_timeout=10)


def tenant_dsn(db_name):
    return TENANT_DB_URL_TEMPLATE.format(db_name=db_name)


def ensure_metrics_table(conn=None):
    """Create ``tenant_metrics_daily`` if missing. Idempotent."""
    own = conn is None
    conn = conn or get_master_conn()
    try:
        cur = conn.cursor()
        cur.execute(DDL)
        conn.commit()
        cur.close()
    finally:
        if own:
            conn.close()


def list_tenants(conn=None, include_inactive=True):
    own = conn is None
    conn = conn or get_master_conn()
    try:
        cur = conn.cursor()
        where = "" if include_inactive else "WHERE t.is_active = true"
        cur.execute(f"""
            SELECT t.id, t.name, t.db_name, t.plan, t.is_active, t.created_at,
                   s.expires_at
            FROM tenants t
            LEFT JOIN subscriptions s ON s.tenant_id = t.id
            {where}
            ORDER BY t.id
        """)
        cols = [d[0] for d in cur.description]
        rows = [dict(zip(cols, r)) for r in cur.fetchall()]
        cur.close()
        return rows
    finally:
        if own:
            conn.close()


# ─── Collection ────────────────────────────────────────────────────────────

def _q(cur, sql, params=None, default=None):
    """Run a scalar query, returning ``default`` if the object is missing."""
    try:
        cur.execute(sql, params or ())
        row = cur.fetchone()
        return row[0] if row and row[0] is not None else default
    except Exception:
        cur.connection.rollback()
        return default


def _load_modules(cur):
    cfg = {}
    try:
        cur.execute("SELECT key, value FROM tenant_config")
        cfg = dict(cur.fetchall())
    except Exception:
        cur.connection.rollback()

    def flag(key):
        return (cfg.get(key) or "").lower() == "true"

    return {
        "module_whatsapp": flag("module_whatsapp"),
        "module_marketplace": flag("module_marketplace"),
        "module_meli": flag("module_meli"),
        "module_catalog": flag("module_catalog"),
        "whatsapp_enabled": flag("whatsapp_enabled"),
        "vehicle_compat_source": cfg.get("vehicle_compat_source"),
        "facturapi": bool(cfg.get("cfdi_facturapi_key")),
    }


def _collect_tenant(tenant, start_dt, end_dt, dates):
    """Collect one tenant's rollup rows for ``dates`` inside [start, end)."""
    conn = psycopg2.connect(tenant_dsn(tenant["db_name"]), connect_timeout=10)
    cur = conn.cursor()
    try:
        items_activos = _q(cur, "SELECT COUNT(*) FROM inventory WHERE is_active", default=0)
        skus = _q(cur, "SELECT COUNT(DISTINCT part_number) FROM inventory "
                       "WHERE is_active AND part_number IS NOT NULL", default=0)
        clientes = _q(cur, "SELECT COUNT(*) FROM customers WHERE is_active", default=0)
        empleados = _q(cur, "SELECT COUNT(*) FROM employees WHERE is_active", default=0)
        db_size = _q(cur, "SELECT pg_database_size(current_database()) / 1048576.0")
        ultima_venta = _q(cur, "SELECT MAX(created_at) FROM sales "
                               "WHERE status = 'completed'")
        ultima_actividad = _q(cur, "SELECT MAX(created_at) FROM audit_log")
        modules = _load_modules(cur)

        icv = _q(cur, "SELECT to_regclass('public.inventory_vehicle_compat') IS NOT NULL",
                 default=False)
        if icv:
            fit = _q(cur, "SELECT COUNT(*) FROM inventory i WHERE i.is_active AND ("
                          "i.vehicle_compatibility IS NOT NULL OR EXISTS ("
                          "SELECT 1 FROM inventory_vehicle_compat c "
                          "WHERE c.inventory_id = i.id))", default=0)
        else:
            fit = _q(cur, "SELECT COUNT(*) FROM inventory WHERE is_active "
                          "AND vehicle_compatibility IS NOT NULL", default=0)
        fitment_pct = round(100.0 * fit / items_activos, 2) if items_activos else None

        # Daily sales (completed / cancelled).
        sales = {}
        try:
            cur.execute("""
                SELECT (created_at AT TIME ZONE %s)::date,
                       COUNT(*) FILTER (WHERE status = 'completed'),
                       COALESCE(SUM(total) FILTER (WHERE status = 'completed'), 0),
                       COUNT(*) FILTER (WHERE status = 'cancelled')
                FROM sales
                WHERE created_at >= %s AND created_at < %s
                GROUP BY 1
            """, (BUSINESS_TZ, start_dt, end_dt))
            for d, vc, vt, vx in cur.fetchall():
                sales[d] = (vc or 0, float(vt or 0), vx or 0)
        except Exception:
            conn.rollback()

        nuevos = {}
        try:
            cur.execute("""
                SELECT (created_at AT TIME ZONE %s)::date, COUNT(*)
                FROM customers
                WHERE created_at >= %s AND created_at < %s
                GROUP BY 1
            """, (BUSINESS_TZ, start_dt, end_dt))
            nuevos = dict(cur.fetchall())
        except Exception:
            conn.rollback()

        actividad = {}
        try:
            cur.execute("""
                SELECT (created_at AT TIME ZONE %s)::date, COUNT(*)
                FROM audit_log
                WHERE created_at >= %s AND created_at < %s
                GROUP BY 1
            """, (BUSINESS_TZ, start_dt, end_dt))
            actividad = dict(cur.fetchall())
        except Exception:
            conn.rollback()

        apis = {}
        has_api = _q(cur, "SELECT to_regclass('public.api_request_logs') IS NOT NULL",
                     default=False)
        if has_api:
            try:
                cur.execute("""
                    SELECT (created_at AT TIME ZONE %s)::date,
                           COUNT(*),
                           COUNT(*) FILTER (WHERE status_code >= 500),
                           COUNT(*) FILTER (WHERE path ~*
                               '(/ai/|/chat|vision|semantic|classif|categoriz|/ml/)')
                    FROM api_request_logs
                    WHERE created_at >= %s AND created_at < %s
                    GROUP BY 1
                """, (BUSINESS_TZ, start_dt, end_dt))
                for d, calls, err5, ai in cur.fetchall():
                    apis[d] = (calls or 0, err5 or 0, ai or 0)
            except Exception:
                conn.rollback()
    finally:
        cur.close()
        conn.close()

    rows = []
    for d in dates:
        vc, vt, vx = sales.get(d, (0, 0.0, 0))
        api_calls, err5, ai = apis.get(d, (0, 0, 0))
        rows.append({
            "tenant_id": tenant["id"],
            "date": d,
            "ventas_count": vc,
            "ventas_total": round(vt, 2),
            "ticket_promedio": round(vt / vc, 2) if vc else 0,
            "ventas_canceladas": vx,
            "clientes_nuevos": nuevos.get(d, 0),
            "items_activos": items_activos or 0,
            "skus": skus or 0,
            "clientes": clientes or 0,
            "empleados_activos": empleados or 0,
            "fitment_pct": fitment_pct,
            "uso_modulos": Json(modules),
            "ai_calls": ai,
            "api_calls": api_calls,
            "errores_5xx": err5,
            "actividad_count": actividad.get(d, 0),
            "ultima_venta_at": ultima_venta,
            "ultima_actividad_at": ultima_actividad,
            "db_size_mb": round(db_size, 2) if db_size is not None else None,
        })
    return rows


def run_rollup(date=None, days=1, tenant_ids=None, include_inactive=True):
    """Collect and upsert rollup rows. Idempotent on (tenant_id, date).

    ``date`` is the last day of the range (defaults to today in BUSINESS_TZ);
    ``days`` extends the range backwards (1 == a single day).
    """
    tz = ZoneInfo(BUSINESS_TZ)
    end_date = date or datetime.now(tz).date()
    if isinstance(end_date, str):
        end_date = date_cls.fromisoformat(end_date)
    dates = [end_date - timedelta(days=i) for i in range(days - 1, -1, -1)]

    start_dt = datetime.combine(dates[0], time.min, tzinfo=tz)
    end_dt = datetime.combine(dates[-1] + timedelta(days=1), time.min, tzinfo=tz)

    conn = get_master_conn()
    try:
        ensure_metrics_table(conn)
        tenants = list_tenants(conn, include_inactive=include_inactive)
        if tenant_ids:
            tenants = [t for t in tenants
                       if t["id"] in tenant_ids or t["db_name"] in tenant_ids]

        written = 0
        errors = []
        per_tenant = []
        for t in tenants:
            try:
                rows = _collect_tenant(t, start_dt, end_dt, dates)
            except Exception as e:  # noqa: BLE001 — report and continue
                errors.append({"tenant_id": t["id"], "db_name": t["db_name"],
                               "error": str(e)})
                continue
            cur = conn.cursor()
            try:
                for row in rows:
                    cur.execute(UPSERT, row)
                conn.commit()
                written += len(rows)
                per_tenant.append({
                    "tenant_id": t["id"], "tenant_name": t["name"],
                    "db_name": t["db_name"], "rows": len(rows),
                })
            except Exception as e:  # noqa: BLE001
                conn.rollback()
                errors.append({"tenant_id": t["id"], "db_name": t["db_name"],
                               "error": str(e)})
            finally:
                cur.close()
    finally:
        conn.close()

    return {
        "date_from": str(dates[0]),
        "date_to": str(dates[-1]),
        "days": days,
        "rows_written": written,
        "tenants": per_tenant,
        "errors": errors,
    }


# ─── Readers ───────────────────────────────────────────────────────────────

METRIC_FIELDS = [
    "ventas_count", "ventas_total", "ticket_promedio", "ventas_canceladas",
    "clientes_nuevos", "items_activos", "skus", "clientes",
    "empleados_activos", "fitment_pct", "ai_calls", "api_calls",
    "errores_5xx", "actividad_count", "db_size_mb",
]


def _serialize_row(row):
    out = {}
    for k, v in row.items():
        if isinstance(v, (datetime, date_cls)):
            out[k] = v.isoformat()
        elif hasattr(v, "quantize"):  # Decimal
            out[k] = float(v)
        else:
            out[k] = v
    return out


def _latest_date(conn):
    cur = conn.cursor()
    try:
        cur.execute("SELECT MAX(date) FROM tenant_metrics_daily")
        row = cur.fetchone()
        return row[0] if row else None
    finally:
        cur.close()


def get_tenant_metrics(days=14, include_inactive=True):
    """Rollup window + health score for every tenant (manager API payload)."""
    days = max(1, min(int(days), 90))
    conn = get_master_conn()
    try:
        ensure_metrics_table(conn)
        latest = _latest_date(conn)
        tenants = list_tenants(conn, include_inactive=include_inactive)
        if latest is None:
            return {
                "rollup_available": False,
                "date": None,
                "window_days": days,
                "notes": {
                    "hint": "Sin datos; ejecuta scripts/rollup_tenant_metrics.py",
                    "activity_proxy": "audit_log (no se persisten logins individuales)",
                },
                "tenants": [
                    {"tenant_id": t["id"], "name": t["name"],
                     "db_name": t["db_name"], "plan": t["plan"],
                     "is_active": t["is_active"],
                     "health": compute_health(t, []), "metrics": None}
                    for t in tenants
                ],
            }

        start = latest - timedelta(days=days - 1)
        cur = conn.cursor()
        cur.execute("""
            SELECT tenant_id, date, ventas_count, ventas_total, ticket_promedio,
                   ventas_canceladas, clientes_nuevos, items_activos, skus,
                   clientes, empleados_activos, fitment_pct, uso_modulos,
                   ai_calls, api_calls, errores_5xx, actividad_count,
                   ultima_venta_at, ultima_actividad_at, db_size_mb
            FROM tenant_metrics_daily
            WHERE date >= %s AND date <= %s
            ORDER BY tenant_id, date
        """, (start, latest))
        cols = [d[0] for d in cur.description]
        by_tenant = {}
        for r in cur.fetchall():
            row = dict(zip(cols, r))  # raw: tz-aware datetimes for health
            by_tenant.setdefault(row["tenant_id"], []).append(row)
        cur.close()

        out = []
        for t in tenants:
            history = by_tenant.get(t["id"], [])
            health = compute_health(t, history)
            out.append({
                "tenant_id": t["id"],
                "name": t["name"],
                "db_name": t["db_name"],
                "plan": t["plan"],
                "is_active": t["is_active"],
                "health": health,
                "metrics": _serialize_row(history[-1]) if history else None,
            })
    finally:
        conn.close()

    return {
        "rollup_available": True,
        "date": str(latest),
        "window_days": days,
        "notes": {
            "activity_proxy": "audit_log (no se persisten logins individuales)",
            "point_in_time": "items/skus/clientes/empleados/fitment son snapshot al momento del rollup",
        },
        "tenants": out,
    }


def get_portfolio_anomalies(date=None, z_threshold=DEFAULT_Z, min_tenants=MIN_COHORT,
                            include_inactive=False):
    """Cross-tenant anomaly report built from the daily rollup."""
    conn = get_master_conn()
    try:
        ensure_metrics_table(conn)
        target = date or _latest_date(conn)
        if target is None:
            return detect_portfolio_anomalies([], date=None,
                                              min_tenants=min_tenants,
                                              z_threshold=z_threshold)

        where_active = "" if include_inactive else "AND t.is_active = true"
        cur = conn.cursor()
        cur.execute(f"""
            SELECT m.tenant_id, t.name, m.date,
                   m.ventas_total, m.ventas_count, m.ticket_promedio,
                   m.clientes_nuevos, m.actividad_count, m.errores_5xx,
                   m.items_activos, m.skus, m.clientes, m.fitment_pct
            FROM tenant_metrics_daily m
            JOIN tenants t ON t.id = m.tenant_id
            WHERE m.date = %s {where_active}
            ORDER BY m.tenant_id
        """, (target,))
        cols = [d[0] for d in cur.description]
        rows = []
        for r in cur.fetchall():
            row = _serialize_row(dict(zip(cols, r)))
            row["tenant_name"] = row.pop("name")
            rows.append(row)
        cur.close()
    finally:
        conn.close()

    report = detect_portfolio_anomalies(rows, date=target,
                                        min_tenants=min_tenants,
                                        z_threshold=z_threshold)
    report["tenants_scanned"] = len(rows)
    return report