Mejoras de sesion: estabilizacion, IA, Nexus Yonke y marketplace B2B
- F0 blindaje: Redis/Celery, pg_hba sin trust LAN, secretos a env 600, backups, webhook auth, jobs muertos - F1 caja: bugs POS, reembolsos al corte, apartados, CFDI, topes de descuento en cotizacion/apartado - F2/F7 IA de catalogo: autocategorizacion, dedup de SKUs con embeddings, OCR de facturas, descripciones, claves SAT, limpieza de fitment - F3 IA en el flujo: vision Yonke, preguntas MeLi, venta cruzada, anomalias, sinonimos, busqueda semantica (rerank/embeddings) - F4/F8 escala y cartera: /metrics+Prometheus+Grafana, pool, score de credito, churn, metricas multi-tenant - Nexus Yonke (M1-M6): modelo de datos 1-de-1, alta por donador, vitrina publica, MeLi usado, consignacion, garantia por dias, reportes, etiquetas - Marketplace B2B entre yonkes (catalogo + pedidos cross-tenant) - Seguridad: auth en /api/admin/* (35 endpoints), alertas por tenant - Migraciones v4.27/v4.28; timers expire-demos y rollup-metrics
This commit is contained in:
235
scripts/assign_sat_keys.py
Normal file
235
scripts/assign_sat_keys.py
Normal file
@@ -0,0 +1,235 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Assign SAT ``clave_prod_serv`` / ``clave_unidad`` per product (CFDI 4.0).
|
||||
|
||||
Two modes:
|
||||
|
||||
sale-items (default, writes): backfill the existing ``sale_items`` columns
|
||||
for a tenant. Historically they are 100% NULL, so every CFDI fell back
|
||||
to the hard-coded default (25174800 / H87). Consumers that read
|
||||
``sale_items`` directly (``services/global_invoice.py``,
|
||||
``blueprints/invoicing_bp.py``) keep the default themselves, so the only
|
||||
way to give them a real per-product key is to persist it here.
|
||||
|
||||
inventory (report only): show the key distribution that *would* be assigned
|
||||
to ``inventory`` rows. ``inventory`` has no clave columns today, so this
|
||||
mode never writes — see the module note in the summary for the proposed
|
||||
``inventory.clave_prod_serv`` / ``inventory.clave_unidad varchar(10)``
|
||||
columns.
|
||||
|
||||
Resolution order per item: ``services.sat_keys`` keyword rules over the name,
|
||||
then the canonical category (from ``product_categorizer`` or the tenant
|
||||
``categories`` row), then optionally the ``ai_chat`` LLM fallback, then the SAT
|
||||
default. Every AI call degrades silently when the provider is down.
|
||||
|
||||
Usage (as root, from /home/Autopartes):
|
||||
python3 scripts/assign_sat_keys.py --tenant 45 --limit 200 --dry-run
|
||||
python3 scripts/assign_sat_keys.py --tenant 45 --limit 500 --use-llm
|
||||
python3 scripts/assign_sat_keys.py --tenant 45 --mode inventory --dry-run
|
||||
python3 scripts/assign_sat_keys.py --all --batch 500
|
||||
|
||||
Nothing is written unless ``--apply`` (mode sale-items); ``--dry-run`` is the
|
||||
default to avoid surprises.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'pos'))
|
||||
|
||||
from tenant_db import get_tenant_conn, get_master_conn # noqa: E402
|
||||
from services import sat_keys # noqa: E402
|
||||
from services.product_categorizer import classify_name # noqa: E402
|
||||
|
||||
DEFAULT_CHECKPOINT = os.path.join(
|
||||
os.path.dirname(os.path.abspath(__file__)), '..', 'backups',
|
||||
'sat_keys_checkpoint.json'
|
||||
)
|
||||
|
||||
|
||||
def list_tenants(only_id=None):
|
||||
conn = get_master_conn()
|
||||
cur = conn.cursor()
|
||||
try:
|
||||
if only_id:
|
||||
cur.execute(
|
||||
"SELECT id, name, db_name FROM tenants "
|
||||
"WHERE is_active = true AND id = %s", (only_id,))
|
||||
else:
|
||||
cur.execute(
|
||||
"SELECT id, name, db_name FROM tenants "
|
||||
"WHERE is_active = true ORDER BY id")
|
||||
return cur.fetchall()
|
||||
finally:
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
||||
|
||||
def _load_checkpoint(path):
|
||||
try:
|
||||
with open(path) as f:
|
||||
return json.load(f)
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def _save_checkpoint(path, data):
|
||||
try:
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
with open(path, 'w') as f:
|
||||
json.dump(data, f, indent=2)
|
||||
except Exception as e:
|
||||
print(f" ! checkpoint no guardado: {e}")
|
||||
|
||||
|
||||
def _resolve(prod_name, brand, category_name, part_number, use_llm):
|
||||
canonical = None
|
||||
try:
|
||||
canonical = classify_name(prod_name or part_number or '', brand)[0]
|
||||
except Exception:
|
||||
canonical = None
|
||||
return sat_keys.resolve_key(
|
||||
name=prod_name, brand=brand, category=category_name,
|
||||
canonical=canonical, part_number=part_number, use_llm=use_llm,
|
||||
)
|
||||
|
||||
|
||||
def run_sale_items(tenant_id, name, conn, args, state):
|
||||
after_id = None if args.reset else state.get('after_id')
|
||||
t_key = str(tenant_id)
|
||||
totals = {'scanned': 0, 'updated': 0, 'by_source': {}}
|
||||
while True:
|
||||
cur = conn.cursor()
|
||||
where = ["(si.clave_prod_serv IS NULL OR si.clave_prod_serv = '')"]
|
||||
params = []
|
||||
if after_id is not None:
|
||||
where.append("si.id > %s")
|
||||
params.append(after_id)
|
||||
sql = f"""
|
||||
SELECT si.id, si.part_number, si.name, si.inventory_id,
|
||||
i.name, i.brand, c.name
|
||||
FROM sale_items si
|
||||
LEFT JOIN inventory i ON i.id = si.inventory_id
|
||||
LEFT JOIN categories c ON c.id = i.category_id
|
||||
WHERE {' AND '.join(where)}
|
||||
ORDER BY si.id
|
||||
LIMIT %s
|
||||
"""
|
||||
params.append(args.batch)
|
||||
cur.execute(sql, params)
|
||||
rows = cur.fetchall()
|
||||
cur.close()
|
||||
if not rows:
|
||||
break
|
||||
|
||||
for (si_id, si_pn, si_name, inv_id, inv_name, inv_brand, cat_name) in rows:
|
||||
prod_name = inv_name or si_name
|
||||
clave, unidad, source = _resolve(
|
||||
prod_name, inv_brand, cat_name, si_pn, args.use_llm)
|
||||
totals['by_source'][source] = totals['by_source'].get(source, 0) + 1
|
||||
totals['scanned'] += 1
|
||||
after_id = si_id
|
||||
if args.apply:
|
||||
c2 = conn.cursor()
|
||||
try:
|
||||
c2.execute(
|
||||
"UPDATE sale_items SET clave_prod_serv = %s, "
|
||||
"clave_unidad = %s WHERE id = %s",
|
||||
(clave, unidad, si_id))
|
||||
totals['updated'] += c2.rowcount
|
||||
conn.commit()
|
||||
finally:
|
||||
c2.close()
|
||||
if totals['scanned'] <= args.show:
|
||||
print(f" {si_id:<8} {str(prod_name)[:40]:<40} -> {clave} / {unidad} ({source})")
|
||||
|
||||
state[t_key] = {'after_id': after_id, 'updated': totals['updated'],
|
||||
'updated_at': time.strftime('%Y-%m-%dT%H:%M:%S')}
|
||||
if args.apply and not args.dry_run:
|
||||
_save_checkpoint(args.checkpoint, state)
|
||||
if len(rows) < args.batch:
|
||||
break
|
||||
if args.limit and totals['scanned'] >= args.limit:
|
||||
break
|
||||
return totals
|
||||
|
||||
|
||||
def run_inventory(tenant_id, conn, args):
|
||||
"""Report-only: distribution of keys the rules would assign to inventory."""
|
||||
cur = conn.cursor()
|
||||
cur.execute("""
|
||||
SELECT i.id, i.part_number, i.name, i.brand, c.name
|
||||
FROM inventory i
|
||||
LEFT JOIN categories c ON c.id = i.category_id
|
||||
WHERE i.is_active = true
|
||||
ORDER BY i.id
|
||||
LIMIT %s
|
||||
""", (args.batch,))
|
||||
rows = cur.fetchall()
|
||||
cur.close()
|
||||
by_source, by_key, sample = {}, {}, []
|
||||
for inv_id, pn, nm, brand, cat in rows:
|
||||
clave, unidad, source = _resolve(nm, brand, cat, pn, args.use_llm)
|
||||
by_source[source] = by_source.get(source, 0) + 1
|
||||
by_key[clave] = by_key.get(clave, 0) + 1
|
||||
if len(sample) < args.show:
|
||||
sample.append((inv_id, nm, clave, unidad, source))
|
||||
print(f" inventory scanned={len(rows)} by_source={by_source}")
|
||||
print(f" keys: {by_key}")
|
||||
for inv_id, nm, clave, unidad, source in sample:
|
||||
print(f" {inv_id:<8} {str(nm)[:40]:<40} -> {clave} / {unidad} ({source})")
|
||||
return {'scanned': len(rows), 'by_source': by_source, 'by_key': by_key}
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description='Assign SAT product/unit keys.')
|
||||
p.add_argument('--tenant', type=int)
|
||||
p.add_argument('--all', action='store_true')
|
||||
p.add_argument('--mode', choices=['sale-items', 'inventory'], default='sale-items')
|
||||
p.add_argument('--limit', type=int, default=0, help='max items per tenant (0=all)')
|
||||
p.add_argument('--batch', type=int, default=500)
|
||||
p.add_argument('--apply', action='store_true', help='persist (sale-items only)')
|
||||
p.add_argument('--dry-run', action='store_true', default=True)
|
||||
p.add_argument('--use-llm', action='store_true', help='enable ai_chat fallback')
|
||||
p.add_argument('--reset', action='store_true', help='ignore checkpoint')
|
||||
p.add_argument('--show', type=int, default=8)
|
||||
p.add_argument('--checkpoint', default=DEFAULT_CHECKPOINT)
|
||||
args = p.parse_args()
|
||||
|
||||
# --apply is the only thing that turns off dry-run; keep it explicit.
|
||||
if args.apply:
|
||||
args.dry_run = False
|
||||
if not args.all and not args.tenant:
|
||||
p.error('--tenant ID o --all requerido')
|
||||
|
||||
state = _load_checkpoint(args.checkpoint)
|
||||
tenants = list_tenants(None if args.all else args.tenant)
|
||||
if not tenants:
|
||||
print('sin tenants para procesar')
|
||||
return 1
|
||||
|
||||
print(f"mode={args.mode} apply={not args.dry_run} use_llm={args.use_llm}")
|
||||
grand = {}
|
||||
for tid, name, db_name in tenants:
|
||||
print(f"\n=== tenant {tid} — {name} ({db_name}) ===")
|
||||
conn = get_tenant_conn(tid)
|
||||
try:
|
||||
if args.mode == 'inventory':
|
||||
run_inventory(tid, conn, args)
|
||||
else:
|
||||
totals = run_sale_items(tid, name, conn, args, state)
|
||||
print(f" total: scanned={totals['scanned']} updated={totals['updated']} "
|
||||
f"by_source={totals['by_source']}")
|
||||
for k, v in totals['by_source'].items():
|
||||
grand[k] = grand.get(k, 0) + v
|
||||
finally:
|
||||
conn.close()
|
||||
if args.mode == 'sale-items':
|
||||
print(f"\nTOTAL by_source={grand}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
180
scripts/b2b_marketplace_schema.py
Normal file
180
scripts/b2b_marketplace_schema.py
Normal file
@@ -0,0 +1,180 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Idempotent master-database schema for the B2B yonke marketplace.
|
||||
|
||||
Creates/ensures, in the master DB (nexus_autoparts):
|
||||
- b2b_listings (used-part wholesale catalog published by sellers)
|
||||
- marketplace_orders (cross-tenant order header; from v1.6_marketplace.sql)
|
||||
- marketplace_order_items (order lines; + listing_id link column)
|
||||
- tenants.is_seller (seller opt-in flag)
|
||||
and activates is_seller=true for the demo sellers (47 Yonke Demo, 45 Capi Autopartes).
|
||||
|
||||
Safe to run repeatedly: every statement is CREATE ... IF NOT EXISTS / ADD COLUMN
|
||||
IF NOT EXISTS / UPDATE (idempotent). It never drops or rewrites data.
|
||||
|
||||
Run:
|
||||
cd /home/Autopartes && python3 scripts/b2b_marketplace_schema.py
|
||||
# or point at another master DB:
|
||||
MASTER_DB_URL=postgresql://user@host/nexus_autoparts python3 scripts/b2b_marketplace_schema.py
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
import psycopg2
|
||||
|
||||
MASTER_DB_URL = (
|
||||
os.environ.get('MASTER_DB_URL')
|
||||
or 'postgresql://postgres@localhost/nexus_autoparts'
|
||||
)
|
||||
|
||||
# Demo sellers that must appear in the marketplace so it is never empty.
|
||||
DEMO_SELLER_TENANT_IDS = (47, 45)
|
||||
|
||||
B2B_LISTINGS_DDL = """
|
||||
CREATE TABLE IF NOT EXISTS b2b_listings (
|
||||
id SERIAL PRIMARY KEY,
|
||||
seller_tenant_id INTEGER NOT NULL,
|
||||
seller_inventory_id INTEGER NOT NULL,
|
||||
part_number VARCHAR(100),
|
||||
name VARCHAR(300),
|
||||
condition VARCHAR(20) DEFAULT 'usado',
|
||||
condition_grade VARCHAR(2),
|
||||
price_wholesale NUMERIC(12,2) NOT NULL DEFAULT 0,
|
||||
donor_make VARCHAR(60),
|
||||
donor_model VARCHAR(60),
|
||||
donor_year INTEGER,
|
||||
image_url VARCHAR(500),
|
||||
status VARCHAR(20) NOT NULL DEFAULT 'available',
|
||||
notes TEXT,
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
UNIQUE (seller_tenant_id, seller_inventory_id)
|
||||
)
|
||||
"""
|
||||
|
||||
# Mirror of pos/migrations/v1.6_marketplace.sql (master DB), kept here so this
|
||||
# script can repair a master DB where the migration was skipped.
|
||||
MARKETPLACE_ORDERS_DDL = """
|
||||
CREATE TABLE IF NOT EXISTS marketplace_orders (
|
||||
id SERIAL PRIMARY KEY,
|
||||
buyer_tenant_id INTEGER NOT NULL,
|
||||
seller_tenant_id INTEGER NOT NULL,
|
||||
buyer_name VARCHAR(200),
|
||||
seller_name VARCHAR(200),
|
||||
total NUMERIC(12,2),
|
||||
status VARCHAR(20) DEFAULT 'pending',
|
||||
notes TEXT,
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ DEFAULT NOW()
|
||||
)
|
||||
"""
|
||||
|
||||
MARKETPLACE_ORDER_ITEMS_DDL = """
|
||||
CREATE TABLE IF NOT EXISTS marketplace_order_items (
|
||||
id SERIAL PRIMARY KEY,
|
||||
order_id INTEGER REFERENCES marketplace_orders(id),
|
||||
part_number VARCHAR(100),
|
||||
part_name VARCHAR(300),
|
||||
quantity INTEGER,
|
||||
unit_price NUMERIC(12,2),
|
||||
subtotal NUMERIC(12,2)
|
||||
)
|
||||
"""
|
||||
|
||||
STATEMENTS = [
|
||||
('b2b_listings', B2B_LISTINGS_DDL),
|
||||
('marketplace_orders', MARKETPLACE_ORDERS_DDL),
|
||||
('marketplace_order_items', MARKETPLACE_ORDER_ITEMS_DDL),
|
||||
|
||||
# Extensions to tables that may already exist.
|
||||
('marketplace_order_items.listing_id',
|
||||
"ALTER TABLE marketplace_order_items ADD COLUMN IF NOT EXISTS listing_id INTEGER"),
|
||||
('tenants.is_seller',
|
||||
"ALTER TABLE tenants ADD COLUMN IF NOT EXISTS is_seller BOOLEAN DEFAULT FALSE"),
|
||||
|
||||
# Indexes.
|
||||
('idx_b2b_listings_seller',
|
||||
"CREATE INDEX IF NOT EXISTS idx_b2b_listings_seller ON b2b_listings(seller_tenant_id)"),
|
||||
('idx_b2b_listings_status',
|
||||
"CREATE INDEX IF NOT EXISTS idx_b2b_listings_status ON b2b_listings(status)"),
|
||||
('idx_b2b_listings_catalog',
|
||||
"CREATE INDEX IF NOT EXISTS idx_b2b_listings_catalog ON b2b_listings(status, condition_grade)"),
|
||||
('idx_mp_orders_buyer',
|
||||
"CREATE INDEX IF NOT EXISTS idx_mp_orders_buyer ON marketplace_orders(buyer_tenant_id)"),
|
||||
('idx_mp_orders_seller',
|
||||
"CREATE INDEX IF NOT EXISTS idx_mp_orders_seller ON marketplace_orders(seller_tenant_id)"),
|
||||
('idx_mp_orders_status',
|
||||
"CREATE INDEX IF NOT EXISTS idx_mp_orders_status ON marketplace_orders(status)"),
|
||||
('idx_mp_order_items_order',
|
||||
"CREATE INDEX IF NOT EXISTS idx_mp_order_items_order ON marketplace_order_items(order_id)"),
|
||||
('idx_mp_order_items_listing',
|
||||
"CREATE INDEX IF NOT EXISTS idx_mp_order_items_listing ON marketplace_order_items(listing_id)"),
|
||||
]
|
||||
|
||||
|
||||
def log(msg):
|
||||
print(f" -> {msg}")
|
||||
|
||||
|
||||
def ensure_schema(conn):
|
||||
cur = conn.cursor()
|
||||
for label, sql in STATEMENTS:
|
||||
cur.execute(sql)
|
||||
log(f"ok: {label}")
|
||||
conn.commit()
|
||||
cur.close()
|
||||
|
||||
|
||||
def verify(conn):
|
||||
cur = conn.cursor()
|
||||
cur.execute("""
|
||||
SELECT to_regclass('b2b_listings'),
|
||||
to_regclass('marketplace_orders'),
|
||||
to_regclass('marketplace_order_items')
|
||||
""")
|
||||
row = cur.fetchone()
|
||||
cur.close()
|
||||
return row
|
||||
|
||||
|
||||
def activate_demo_sellers(conn):
|
||||
cur = conn.cursor()
|
||||
cur.execute(
|
||||
"SELECT id, name, COALESCE(is_seller, false) FROM tenants WHERE id = ANY(%s) ORDER BY id",
|
||||
(list(DEMO_SELLER_TENANT_IDS),),
|
||||
)
|
||||
found = cur.fetchall()
|
||||
if not found:
|
||||
log("no demo tenants found (47, 45) — skipping is_seller activation")
|
||||
cur.close()
|
||||
return
|
||||
cur.execute(
|
||||
"UPDATE tenants SET is_seller = TRUE WHERE id = ANY(%s)",
|
||||
(list(DEMO_SELLER_TENANT_IDS),),
|
||||
)
|
||||
conn.commit()
|
||||
for tenant_id, name, was_seller in found:
|
||||
state = "already seller" if was_seller else "activated"
|
||||
log(f"tenant {tenant_id} ({name}): {state}")
|
||||
cur.close()
|
||||
|
||||
|
||||
def main():
|
||||
print(f"B2B marketplace master schema -> {MASTER_DB_URL}")
|
||||
conn = psycopg2.connect(MASTER_DB_URL)
|
||||
try:
|
||||
ensure_schema(conn)
|
||||
activate_demo_sellers(conn)
|
||||
b2b, orders, items = verify(conn)
|
||||
print("Verification:")
|
||||
log(f"b2b_listings={b2b}")
|
||||
log(f"marketplace_orders={orders}")
|
||||
log(f"marketplace_order_items={items}")
|
||||
finally:
|
||||
conn.close()
|
||||
print("Done.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
104
scripts/backfill_meli_listings.py
Normal file
104
scripts/backfill_meli_listings.py
Normal file
@@ -0,0 +1,104 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Backfill marketplace_listings.inventory_id for a tenant (MercadoLibre).
|
||||
|
||||
For every listing already imported from MercadoLibre, fetch the ML item and
|
||||
match it against local inventory by:
|
||||
1. seller_custom_field / seller_sku
|
||||
2. SELLER_SKU / PART_NUMBER / SELLER_PART_NUMBER attributes
|
||||
(item level and per variation)
|
||||
3. inventory.part_number, then inventory_sku_aliases.sku
|
||||
(exact match first, then normalized: spaces/dashes/dots stripped)
|
||||
4. the ML item id itself, as a last resort (some tenants store it as alias)
|
||||
|
||||
Dry-run by default: it reports what it *would* map without writing anything.
|
||||
Pass --apply to persist inventory_id + inventory_match_source + matched_at.
|
||||
|
||||
Usage (from /home/Autopartes):
|
||||
python3 scripts/backfill_meli_listings.py --tenant 31 # dry-run
|
||||
python3 scripts/backfill_meli_listings.py --tenant 31 --apply
|
||||
python3 scripts/backfill_meli_listings.py --tenant 31 --apply --limit 10
|
||||
python3 scripts/backfill_meli_listings.py --tenant 31 --all # re-check mapped too
|
||||
python3 scripts/backfill_meli_listings.py --tenant 31 --json
|
||||
|
||||
Requires MASTER_DB_URL + TENANT_DB_URL_TEMPLATE in the environment; if missing
|
||||
they are loaded from /etc/nexus/pos.env.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'pos'))
|
||||
|
||||
|
||||
def _load_env():
|
||||
env_file = '/etc/nexus/pos.env'
|
||||
if os.environ.get('MASTER_DB_URL') or not os.path.exists(env_file):
|
||||
return
|
||||
for line in open(env_file):
|
||||
line = line.strip()
|
||||
if not line or line.startswith('#') or '=' not in line:
|
||||
continue
|
||||
key, value = line.split('=', 1)
|
||||
os.environ.setdefault(key.strip(), value.strip().strip('"').strip("'"))
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument('--tenant', type=int, required=True, help='tenant id (e.g. 31 = Rached)')
|
||||
ap.add_argument('--apply', action='store_true',
|
||||
help='write the matches (default: dry-run, read-only)')
|
||||
ap.add_argument('--limit', type=int, default=None, help='max listings to process')
|
||||
ap.add_argument('--all', action='store_true',
|
||||
help='also re-check listings that already have an inventory_id')
|
||||
ap.add_argument('--json', action='store_true', help='print the full summary as JSON')
|
||||
args = ap.parse_args()
|
||||
|
||||
_load_env()
|
||||
|
||||
from tenant_db import get_tenant_conn
|
||||
from services import marketplace_external_service as meli
|
||||
|
||||
conn = get_tenant_conn(args.tenant)
|
||||
try:
|
||||
summary = meli.backfill_listing_inventory(
|
||||
conn,
|
||||
limit=args.limit,
|
||||
only_unmapped=not args.all,
|
||||
dry_run=not args.apply,
|
||||
)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
if args.json:
|
||||
print(json.dumps(summary, indent=2, ensure_ascii=False, default=str))
|
||||
return 0
|
||||
|
||||
mode = 'DRY-RUN (no writes)' if summary['dry_run'] else 'APPLIED'
|
||||
print(f"[tenant {args.tenant}] backfill inventory_id — {mode}")
|
||||
print(f" scanned: {summary['scanned']}")
|
||||
print(f" matched: {summary['matched']}")
|
||||
print(f" unmatched: {summary['unmatched']}")
|
||||
print(f" errors: {summary['errors']}")
|
||||
|
||||
unmatched = [d for d in summary['details'] if d.get('status') == 'unmatched']
|
||||
if unmatched:
|
||||
print("\n Unmatched listings (ML item has no local part number/SKU):")
|
||||
for d in unmatched:
|
||||
cands = d.get('candidates') or []
|
||||
print(f" listing {d['listing_id']} ({d['external_item_id']}) "
|
||||
f"{(d.get('title') or '')[:45]!r} candidates={cands}")
|
||||
if summary['errors']:
|
||||
print("\n Errors:")
|
||||
for d in summary['details']:
|
||||
if d.get('status') == 'error':
|
||||
print(f" listing {d['listing_id']}: {d.get('error')}")
|
||||
if summary['dry_run'] and summary['matched']:
|
||||
print("\n Re-run with --apply to persist these matches.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
187
scripts/backfill_vehicle_compat.py
Normal file
187
scripts/backfill_vehicle_compat.py
Normal file
@@ -0,0 +1,187 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Backfill vehicle compatibility for ALL tenant inventory items.
|
||||
|
||||
Matches tenant inventory part numbers (and SKU aliases) against the master
|
||||
supplier_catalog / supplier_catalog_interchange / supplier_catalog_compat
|
||||
tables, then inserts model_year_engine_id compat rows with
|
||||
source='supplier_catalog'.
|
||||
|
||||
Loads the whole mapping in memory (~1M small rows) so it runs in minutes,
|
||||
not days. Safe to re-run: ON CONFLICT DO NOTHING skips existing rows.
|
||||
|
||||
Usage (as postgres):
|
||||
MASTER_DB_URL=... python3 scripts/backfill_vehicle_compat.py [db_name|all]
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'pos'))
|
||||
|
||||
import psycopg2 # noqa: E402
|
||||
from psycopg2.extras import execute_values # noqa: E402
|
||||
|
||||
MASTER_DB = os.environ.get('MASTER_DB_NAME', 'nexus_autoparts')
|
||||
BATCH_COMMIT = 2000
|
||||
|
||||
PREFIX_RE = re.compile(r'^[A-Z]{2,5}[-_ ]') # tenant prefixes like OEM-, INJ-, WAL-
|
||||
|
||||
|
||||
def norm(pn):
|
||||
if not pn:
|
||||
return ''
|
||||
return re.sub(r'[\s\-/._]', '', str(pn)).upper()
|
||||
|
||||
|
||||
def pn_variants(part_number, aliases):
|
||||
"""Candidate normalized part numbers for an inventory item."""
|
||||
out = set()
|
||||
for raw in [part_number] + list(aliases):
|
||||
n = norm(raw)
|
||||
if n:
|
||||
out.add(n)
|
||||
# strip tenant prefixes: OEM-96355 -> 96355
|
||||
stripped = PREFIX_RE.sub('', str(raw or '').upper())
|
||||
n2 = norm(stripped)
|
||||
if n2:
|
||||
out.add(n2)
|
||||
return out
|
||||
|
||||
|
||||
def load_master_mapping(cur):
|
||||
cur.execute("SELECT id, sku FROM supplier_catalog")
|
||||
sku_map = {}
|
||||
for cid, sku in cur.fetchall():
|
||||
n = norm(sku)
|
||||
if n:
|
||||
sku_map.setdefault(n, set()).add(cid)
|
||||
|
||||
ix_map = {}
|
||||
cur.execute("SELECT catalog_id, part_number FROM supplier_catalog_interchange")
|
||||
for cid, pn in cur.fetchall():
|
||||
n = norm(pn)
|
||||
if n:
|
||||
ix_map.setdefault(n, set()).add(cid)
|
||||
|
||||
mye_map = {}
|
||||
cur.execute("""
|
||||
SELECT catalog_id, model_year_engine_id FROM supplier_catalog_compat
|
||||
WHERE model_year_engine_id IS NOT NULL
|
||||
""")
|
||||
for cid, mye in cur.fetchall():
|
||||
mye_map.setdefault(cid, set()).add(mye)
|
||||
|
||||
total = len(sku_map) + len(ix_map) + len(mye_map)
|
||||
print(f'[master] skus={len(sku_map)} interchange={len(ix_map)} mye_entries={len(mye_map)} '
|
||||
f'(total {total})', flush=True)
|
||||
return sku_map, ix_map, mye_map
|
||||
|
||||
|
||||
def backfill_tenant(db_name, sku_map, ix_map, mye_map):
|
||||
conn = psycopg2.connect(dbname=db_name)
|
||||
cur = conn.cursor()
|
||||
|
||||
# Idempotency index (also fixes batch_add_compatibilities' ON CONFLICT)
|
||||
cur.execute("""
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS ux_compat_inv_mye
|
||||
ON inventory_vehicle_compat (inventory_id, model_year_engine_id)
|
||||
""")
|
||||
conn.commit()
|
||||
|
||||
# Items without any compat row yet
|
||||
cur.execute("""
|
||||
SELECT id, part_number FROM inventory
|
||||
WHERE is_active = true
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM inventory_vehicle_compat c WHERE c.inventory_id = inventory.id)
|
||||
ORDER BY id
|
||||
""")
|
||||
items = cur.fetchall()
|
||||
|
||||
# SKU aliases by inventory_id
|
||||
cur.execute("SELECT inventory_id, sku FROM inventory_sku_aliases")
|
||||
aliases = {}
|
||||
for inv_id, sku in cur.fetchall():
|
||||
aliases.setdefault(inv_id, []).append(sku)
|
||||
|
||||
print(f'[{db_name}] items sin compat: {len(items)}', flush=True)
|
||||
|
||||
matched = inserted = 0
|
||||
pending = []
|
||||
for i, (inv_id, pn) in enumerate(items, start=1):
|
||||
mye_ids = set()
|
||||
for v in pn_variants(pn, aliases.get(inv_id, [])):
|
||||
for cid in (sku_map.get(v) or set()) | (ix_map.get(v) or set()):
|
||||
mye_ids |= mye_map.get(cid) or set()
|
||||
if len(mye_ids) > 500:
|
||||
mye_ids = set(list(mye_ids)[:500])
|
||||
break
|
||||
if mye_ids:
|
||||
matched += 1
|
||||
for mye in mye_ids:
|
||||
pending.append((inv_id, mye, 'supplier_catalog', 0.95))
|
||||
|
||||
if len(pending) >= BATCH_COMMIT:
|
||||
execute_values(cur, """
|
||||
INSERT INTO inventory_vehicle_compat
|
||||
(inventory_id, model_year_engine_id, source, confidence)
|
||||
VALUES %s ON CONFLICT (inventory_id, model_year_engine_id) DO NOTHING
|
||||
""", pending)
|
||||
conn.commit()
|
||||
inserted += len(pending)
|
||||
pending = []
|
||||
|
||||
if i % 5000 == 0:
|
||||
print(f' {i}/{len(items)} (matcheados={matched})', flush=True)
|
||||
|
||||
if pending:
|
||||
execute_values(cur, """
|
||||
INSERT INTO inventory_vehicle_compat
|
||||
(inventory_id, model_year_engine_id, source, confidence)
|
||||
VALUES %s ON CONFLICT (inventory_id, model_year_engine_id) DO NOTHING
|
||||
""", pending)
|
||||
conn.commit()
|
||||
inserted += len(pending)
|
||||
|
||||
cur.execute("SELECT count(*) FROM inventory_vehicle_compat")
|
||||
total = cur.fetchone()[0]
|
||||
cur.execute("""
|
||||
SELECT count(*) FROM inventory i
|
||||
WHERE i.is_active AND EXISTS (
|
||||
SELECT 1 FROM inventory_vehicle_compat c WHERE c.inventory_id = i.id)
|
||||
""")
|
||||
covered = cur.fetchone()[0]
|
||||
cur.execute("SELECT count(*) FROM inventory WHERE is_active")
|
||||
active = cur.fetchone()[0]
|
||||
|
||||
print(f'[{db_name}] matcheados={matched} filas_insertadas={inserted} | '
|
||||
f'cobertura total: {covered}/{active} ({100 * covered // max(active, 1)}%)',
|
||||
flush=True)
|
||||
conn.close()
|
||||
|
||||
|
||||
def main():
|
||||
target = sys.argv[1] if len(sys.argv) > 1 else 'all'
|
||||
mconn = psycopg2.connect(dbname=MASTER_DB)
|
||||
mcur = mconn.cursor()
|
||||
sku_map, ix_map, mye_map = load_master_mapping(mcur)
|
||||
|
||||
if target == 'all':
|
||||
mcur.execute("SELECT db_name FROM tenants WHERE is_active = true ORDER BY id")
|
||||
dbs = [r[0] for r in mcur.fetchall()]
|
||||
else:
|
||||
dbs = [target]
|
||||
|
||||
for db in dbs:
|
||||
try:
|
||||
backfill_tenant(db, sku_map, ix_map, mye_map)
|
||||
except Exception as e:
|
||||
print(f'[{db}] ERROR: {e}', flush=True)
|
||||
mcur.close()
|
||||
mconn.close()
|
||||
print('Backfill completado.', flush=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -31,6 +31,18 @@ log() {
|
||||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"
|
||||
}
|
||||
|
||||
# ─── Cleanup old backups (FIRST, so a full disk cannot block it) ───
|
||||
log "Cleaning up backups older than $RETENTION_DAYS days ..."
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
log "DRY-RUN: would delete backups older than $RETENTION_DAYS days"
|
||||
else
|
||||
find "$BACKUP_DIR" -name "nexus_backup_*.tar.gz" -type f -mtime +$RETENTION_DAYS -delete
|
||||
find "$BACKUP_DIR" -name "nexus_backup_*.sql" -type f -mtime +$RETENTION_DAYS -delete
|
||||
# Remove empty leftovers from failed runs (e.g. 0-byte dumps when disk was full)
|
||||
find "$BACKUP_DIR" -name "nexus_backup_*" -type f -empty -delete
|
||||
log "Cleanup complete. Disk usage: $(df -h / | awk 'NR==2{print $3"/"$2" ("$5")"}')"
|
||||
fi
|
||||
|
||||
# ─── PostgreSQL dump ───
|
||||
log "Dumping PostgreSQL database: $DB_NAME ..."
|
||||
DUMP_FILE="${BACKUP_DIR}/${BACKUP_NAME}.sql"
|
||||
@@ -90,14 +102,6 @@ else
|
||||
log "INFO: S3_BUCKET not set. Skipping cloud upload."
|
||||
fi
|
||||
|
||||
# ─── Cleanup old backups ───
|
||||
log "Cleaning up backups older than $RETENTION_DAYS days ..."
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
log "DRY-RUN: would delete backups older than $RETENTION_DAYS days"
|
||||
else
|
||||
find "$BACKUP_DIR" -name "nexus_backup_*.tar.gz" -type f -mtime +$RETENTION_DAYS -delete
|
||||
find "$BACKUP_DIR" -name "nexus_backup_*.sql" -type f -mtime +$RETENTION_DAYS -delete
|
||||
log "Cleanup complete."
|
||||
fi
|
||||
|
||||
# ─── Cleanup ───
|
||||
log "Post-backup cleanup (already done at start)."
|
||||
log "Done."
|
||||
|
||||
@@ -25,9 +25,10 @@ fatal() { err "$*"; exit 1; }
|
||||
|
||||
# ─── Configuration ─────────────────────────────────────────────────────────
|
||||
BACKUP_DIR="${BACKUP_DIR:-${PROJECT_DIR}/backups}"
|
||||
RETENTION_DAYS="${RETENTION_DAYS:-30}"
|
||||
RETENTION_DAYS="${RETENTION_DAYS:-21}"
|
||||
MASTER_DB="${MASTER_DB:-nexus_autoparts}"
|
||||
DB_USER="${DB_USER:-nexus}"
|
||||
AWS_CLI="${PROJECT_DIR}/tools/bin/aws"
|
||||
|
||||
# Load .env if exists
|
||||
if [[ -f "${PROJECT_DIR}/.env" ]]; then
|
||||
@@ -128,8 +129,15 @@ backup_master_data() {
|
||||
rm -f "$tables_file"
|
||||
}
|
||||
|
||||
# ─── Backup vehicle_parts schema only ──────────────────────────────────────
|
||||
# ─── Backup vehicle_parts schema only (if table exists) ────────────────────
|
||||
backup_vehicle_parts_schema() {
|
||||
local exists=$(sudo -u postgres psql "$MASTER_DB" -Atc \
|
||||
"SELECT 1 FROM pg_tables WHERE schemaname='public' AND tablename='vehicle_parts'" 2>/dev/null)
|
||||
if [[ "$exists" != "1" ]]; then
|
||||
info "vehicle_parts table does not exist — skipping (legacy step)"
|
||||
return 0
|
||||
fi
|
||||
|
||||
info "Backing up vehicle_parts schema only (structure, no data)..."
|
||||
|
||||
local output="${BACKUP_PATH}/03_vehicle_parts_schema.sql"
|
||||
@@ -146,6 +154,34 @@ backup_vehicle_parts_schema() {
|
||||
}
|
||||
|
||||
# ─── Backup all tenant databases ───────────────────────────────────────────
|
||||
# ─── Backup WA CRM (Supabase self-hosted, docker) ─────────────────────────
|
||||
backup_wacrm() {
|
||||
local container="supabase-db"
|
||||
if ! docker ps --format '{{.Names}}' 2>/dev/null | grep -qx "$container"; then
|
||||
info "WA CRM (supabase-db) no está corriendo — se omite"
|
||||
return 0
|
||||
fi
|
||||
local output="${BACKUP_PATH}/wacrm_supabase.sql"
|
||||
info "Backing up WA CRM (Supabase postgres)..."
|
||||
if docker exec "$container" pg_dump -U postgres -d postgres --clean --if-exists > "$output" 2>/dev/null; then
|
||||
gzip -f "$output"
|
||||
ok "WA CRM: $(du -h "${output}.gz" | cut -f1)"
|
||||
else
|
||||
warn "WA CRM: pg_dump falló (se omite)"
|
||||
rm -f "$output"
|
||||
fi
|
||||
|
||||
# Volumen de storage (archivos multimedia de los chats) — solo si existe
|
||||
local media_tar="${BACKUP_PATH}/wacrm_storage.tar.gz"
|
||||
if docker volume inspect supabase_storage >/dev/null 2>&1; then
|
||||
if docker run --rm -v supabase_storage:/data:ro -v "${BACKUP_PATH}":/backup alpine \
|
||||
tar czf /backup/wacrm_storage.tar.gz -C /data . >/dev/null 2>&1; then
|
||||
ok "WA CRM storage: $(du -h "$media_tar" 2>/dev/null | cut -f1)"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
backup_tenants() {
|
||||
info "Discovering tenant databases..."
|
||||
|
||||
@@ -205,6 +241,8 @@ CONTENTS:
|
||||
02_master_data.sql.gz — Data for all tables EXCEPT vehicle_parts
|
||||
03_vehicle_parts_schema.sql — Structure of vehicle_parts only (no data)
|
||||
tenant_*.sql.gz — Full backup of each active tenant database
|
||||
wacrm_supabase.sql.gz — Base de datos de WA CRM (Supabase self-hosted)
|
||||
wacrm_storage.tar.gz — Archivos multimedia de los chats de WA CRM (si existe)
|
||||
|
||||
RESTORE INSTRUCTIONS:
|
||||
---------------------
|
||||
@@ -263,9 +301,12 @@ cleanup_old_backups() {
|
||||
local deleted=0
|
||||
while IFS= read -r file; do
|
||||
rm -f "$file"
|
||||
((deleted++))
|
||||
deleted=$((deleted + 1))
|
||||
done < <(find "$BACKUP_DIR" -name "nexus_backup_*.tar.gz" -mtime +$RETENTION_DAYS)
|
||||
|
||||
# Remove empty leftovers from failed runs
|
||||
find "$BACKUP_DIR" -maxdepth 1 -name "nexus_backup_*" -type f -empty -delete
|
||||
|
||||
if [[ "$deleted" -gt 0 ]]; then
|
||||
ok "Deleted ${deleted} old backup(s)"
|
||||
else
|
||||
@@ -273,6 +314,38 @@ cleanup_old_backups() {
|
||||
fi
|
||||
}
|
||||
|
||||
# ─── Verify archive integrity ──────────────────────────────────────────────
|
||||
verify_archive() {
|
||||
local archive="${BACKUP_DIR}/${BACKUP_NAME}.tar.gz"
|
||||
info "Verifying archive integrity..."
|
||||
if ! gzip -t "$archive"; then
|
||||
fatal "Archive failed integrity check: ${archive}"
|
||||
fi
|
||||
if [[ $(stat -c%s "$archive") -lt 1000000 ]]; then
|
||||
fatal "Archive suspiciously small (<1MB): ${archive}"
|
||||
fi
|
||||
ok "Archive integrity verified"
|
||||
}
|
||||
|
||||
# ─── Upload offsite (optional, needs BACKUP_S3_BUCKET) ─────────────────────
|
||||
upload_offsite() {
|
||||
local archive="${BACKUP_DIR}/${BACKUP_NAME}.tar.gz"
|
||||
if [[ -z "${BACKUP_S3_BUCKET:-}" ]]; then
|
||||
info "BACKUP_S3_BUCKET not set. Skipping offsite upload (backups are local-only)."
|
||||
return 0
|
||||
fi
|
||||
if [[ ! -x "$AWS_CLI" ]]; then
|
||||
warn "AWS CLI not found at ${AWS_CLI}. Skipping offsite upload."
|
||||
return 0
|
||||
fi
|
||||
info "Uploading to s3://${BACKUP_S3_BUCKET}/nexus-backups/ ..."
|
||||
if "$AWS_CLI" s3 cp "$archive" "s3://${BACKUP_S3_BUCKET}/nexus-backups/" --storage-class STANDARD_IA; then
|
||||
ok "Offsite upload complete"
|
||||
else
|
||||
warn "Offsite upload failed (local backup is still valid)"
|
||||
fi
|
||||
}
|
||||
|
||||
# ─── Main ──────────────────────────────────────────────────────────────────
|
||||
main() {
|
||||
echo ""
|
||||
@@ -283,14 +356,20 @@ main() {
|
||||
echo -e "${NC}"
|
||||
echo ""
|
||||
|
||||
# Cleanup FIRST so a nearly-full disk cannot turn one failed backup
|
||||
# into permanent retention (the backup.sh vicious cycle).
|
||||
cleanup_old_backups
|
||||
|
||||
check_prerequisites
|
||||
backup_master_schema
|
||||
backup_master_data
|
||||
backup_vehicle_parts_schema
|
||||
backup_tenants
|
||||
backup_wacrm
|
||||
create_manifest
|
||||
create_archive
|
||||
cleanup_old_backups
|
||||
verify_archive
|
||||
upload_offsite
|
||||
|
||||
echo -e "${BOLD}${GREEN}"
|
||||
echo " Backup completed successfully!"
|
||||
|
||||
166
scripts/categorize_inventory.py
Executable file
166
scripts/categorize_inventory.py
Executable file
@@ -0,0 +1,166 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Backfill inventory categories (A1).
|
||||
|
||||
Assigns ``inventory.category_id`` per tenant using the rule engine over the
|
||||
Nexpart taxonomy (+ optional LLM fallback via ``ai_chat.classify_part``).
|
||||
|
||||
Usage (as root/postgres, from /home/Autopartes):
|
||||
python3 scripts/categorize_inventory.py --tenant 31 --limit 200
|
||||
python3 scripts/categorize_inventory.py --tenant 45 --limit 50 --only-uncategorized
|
||||
python3 scripts/categorize_inventory.py --all --batch 500
|
||||
python3 scripts/categorize_inventory.py --tenant 31 --no-llm --dry-run
|
||||
|
||||
Checkpoints progress (last processed inventory id per tenant) in a JSON file so
|
||||
long backfills can be resumed.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'pos'))
|
||||
|
||||
from tenant_db import get_tenant_conn, get_master_conn # noqa: E402
|
||||
from services.product_categorizer import categorize_inventory # noqa: E402
|
||||
|
||||
DEFAULT_CHECKPOINT = os.path.join(
|
||||
os.path.dirname(os.path.abspath(__file__)), '..', 'backups',
|
||||
'categorize_checkpoint.json'
|
||||
)
|
||||
|
||||
|
||||
def list_tenants(only_id=None):
|
||||
conn = get_master_conn()
|
||||
cur = conn.cursor()
|
||||
if only_id:
|
||||
cur.execute(
|
||||
"SELECT id, name, db_name FROM tenants WHERE is_active = true AND id = %s",
|
||||
(only_id,),
|
||||
)
|
||||
else:
|
||||
cur.execute(
|
||||
"SELECT id, name, db_name FROM tenants WHERE is_active = true ORDER BY id"
|
||||
)
|
||||
rows = cur.fetchall()
|
||||
cur.close()
|
||||
conn.close()
|
||||
return rows
|
||||
|
||||
|
||||
def load_checkpoint(path):
|
||||
try:
|
||||
with open(path) as f:
|
||||
return json.load(f)
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def save_checkpoint(path, data):
|
||||
try:
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
with open(path, 'w') as f:
|
||||
json.dump(data, f, indent=2)
|
||||
except Exception as e:
|
||||
print(f" ! no se pudo guardar checkpoint: {e}")
|
||||
|
||||
|
||||
def run_tenant(tenant_id, name, db_name, args, checkpoint):
|
||||
print(f"\n=== tenant {tenant_id} — {name} ({db_name}) ===")
|
||||
conn = get_tenant_conn(tenant_id)
|
||||
key = str(tenant_id)
|
||||
state = checkpoint.get(key, {})
|
||||
after_id = None if args.reset else state.get('after_id')
|
||||
total_scanned = 0
|
||||
total_persisted = 0
|
||||
total_by_source = {}
|
||||
|
||||
try:
|
||||
while True:
|
||||
res = categorize_inventory(
|
||||
conn,
|
||||
limit=args.batch,
|
||||
only_uncategorized=args.only_uncategorized,
|
||||
use_llm=not args.no_llm,
|
||||
persist=not args.dry_run,
|
||||
create_categories=not args.no_create,
|
||||
after_id=after_id,
|
||||
sample_limit=args.show,
|
||||
)
|
||||
total_scanned += res['scanned']
|
||||
total_persisted += res['persisted']
|
||||
for k, v in res['by_source'].items():
|
||||
total_by_source[k] = total_by_source.get(k, 0) + v
|
||||
|
||||
if args.show and res['sample']:
|
||||
for s in res['sample']:
|
||||
print(f" {s['part_number']:<22} {str(s['name'])[:34]:<34} "
|
||||
f"-> {str(s['category_name']):<22} "
|
||||
f"({s['source']}, {s['confidence']})")
|
||||
|
||||
after_id = res['last_id']
|
||||
checkpoint[key] = {
|
||||
'after_id': after_id,
|
||||
'persisted': state.get('persisted', 0) + total_persisted,
|
||||
'updated_at': time.strftime('%Y-%m-%dT%H:%M:%S'),
|
||||
}
|
||||
if not args.dry_run:
|
||||
save_checkpoint(args.checkpoint, checkpoint)
|
||||
|
||||
print(f" batch: scanned={res['scanned']} persisted={res['persisted']} "
|
||||
f"by_source={res['by_source']} last_id={after_id}")
|
||||
|
||||
if res['scanned'] < args.batch:
|
||||
break
|
||||
if args.limit and total_scanned >= args.limit:
|
||||
break
|
||||
print(f" total: scanned={total_scanned} persisted={total_persisted} "
|
||||
f"by_source={total_by_source}")
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
return total_scanned, total_persisted
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description='Backfill inventory categories (A1).')
|
||||
p.add_argument('--tenant', type=int, help='tenant id (omit with --all)')
|
||||
p.add_argument('--all', action='store_true', help='process every active tenant')
|
||||
p.add_argument('--limit', type=int, default=0, help='max items per tenant (0 = all)')
|
||||
p.add_argument('--batch', type=int, default=500, help='rows per batch')
|
||||
p.add_argument('--only-uncategorized', action='store_true', default=True,
|
||||
help='only fill items with category_id IS NULL (default)')
|
||||
p.add_argument('--all-items', dest='only_uncategorized', action='store_false',
|
||||
help='consider every active item')
|
||||
p.add_argument('--no-llm', action='store_true', help='disable LLM fallback')
|
||||
p.add_argument('--no-create', action='store_true',
|
||||
help='do not create missing categories')
|
||||
p.add_argument('--dry-run', action='store_true', help='do not persist')
|
||||
p.add_argument('--reset', action='store_true', help='ignore saved checkpoint')
|
||||
p.add_argument('--show', type=int, default=5, help='sample rows to print (0 = none)')
|
||||
p.add_argument('--checkpoint', default=DEFAULT_CHECKPOINT)
|
||||
args = p.parse_args()
|
||||
|
||||
if not args.all and not args.tenant:
|
||||
p.error('--tenant ID o --all requerido')
|
||||
|
||||
checkpoint = load_checkpoint(args.checkpoint)
|
||||
tenants = list_tenants(None if args.all else args.tenant)
|
||||
if not tenants:
|
||||
print('sin tenants para procesar')
|
||||
return 1
|
||||
|
||||
grand_scanned = grand_persisted = 0
|
||||
for tid, name, db_name in tenants:
|
||||
scanned, persisted = run_tenant(tid, name, db_name, args, checkpoint)
|
||||
grand_scanned += scanned
|
||||
grand_persisted += persisted
|
||||
|
||||
print(f"\nTOTAL: tenants={len(tenants)} scanned={grand_scanned} "
|
||||
f"persisted={grand_persisted}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
238
scripts/clean_fitment_qwen.py
Normal file
238
scripts/clean_fitment_qwen.py
Normal file
@@ -0,0 +1,238 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Flag/remove improbable QWEN fitment rows from ``inventory_vehicle_compat``.
|
||||
|
||||
The ``qwen_ai`` fitment backfill produced 20.7M rows on Estrada: 127,041
|
||||
inventory items, average 162 vehicle links each and a hard cap of 200 — i.e.
|
||||
almost every part was declared compatible with hundreds of vehicles. This
|
||||
script finds those improbable rows and can **mark** them (reversible: set
|
||||
``source='qwen_rejected'``, ``confidence=0``) or **delete** them.
|
||||
|
||||
Detectors (source = ``qwen_ai`` by default):
|
||||
|
||||
1. volume — items with >= ``--min-rows`` fitment rows (a single part rarely
|
||||
fits hundreds of vehicles; the cap betrays completeness padding).
|
||||
2. universal name — a normalized name shared by >= ``--name-min-items``
|
||||
distinct inventory items (e.g. "BALATA DISCO") that still got a full
|
||||
vehicle list.
|
||||
3. semantic (opt-in, ``--semantic``) — QWEN embeddings cluster *name
|
||||
variants* (cosine >= ``--similarity``) so "FILTRO AIRE" and "FILTRO DE
|
||||
AIRE" are treated as one universal name. Degrades silently when the
|
||||
provider is unavailable.
|
||||
|
||||
Safety: dry-run by default; nothing is written without ``--apply``.
|
||||
Delete requires ``--apply --mode delete``; ``--mode mark`` is preferred.
|
||||
|
||||
Usage (as root, from /home/Autopartes):
|
||||
python3 scripts/clean_fitment_qwen.py --tenant 28
|
||||
python3 scripts/clean_fitment_qwen.py --tenant 28 --min-rows 120 --semantic
|
||||
python3 scripts/clean_fitment_qwen.py --tenant 28 --apply --mode mark
|
||||
python3 scripts/clean_fitment_qwen.py --tenant 28 --apply --mode delete --min-rows 150
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
import unicodedata
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'pos'))
|
||||
|
||||
from tenant_db import get_tenant_conn # noqa: E402
|
||||
|
||||
|
||||
def _norm_name(s):
|
||||
s = unicodedata.normalize("NFD", str(s or ""))
|
||||
s = s.encode("ascii", "ignore").decode("ascii")
|
||||
out = []
|
||||
for ch in s.upper():
|
||||
out.append(ch if ch.isalnum() else " ")
|
||||
return " ".join("".join(out).split())
|
||||
|
||||
|
||||
def _collect_counts(cur, source, limit=None):
|
||||
"""[(inventory_id, name, row_count)] for the given source."""
|
||||
sql = """
|
||||
SELECT c.inventory_id, i.name, count(*) AS n
|
||||
FROM inventory_vehicle_compat c
|
||||
JOIN inventory i ON i.id = c.inventory_id
|
||||
WHERE c.source = %s
|
||||
GROUP BY c.inventory_id, i.name
|
||||
"""
|
||||
if limit:
|
||||
sql += " LIMIT %s"
|
||||
cur.execute(sql, (source, limit))
|
||||
else:
|
||||
cur.execute(sql, (source,))
|
||||
return cur.fetchall()
|
||||
|
||||
|
||||
def _semantic_clusters(names, threshold=0.92, batch=64, timeout=30):
|
||||
"""Union-find clusters of near-duplicate names via QWEN embeddings.
|
||||
|
||||
Returns {name: cluster_id} or {} when embeddings are unavailable.
|
||||
"""
|
||||
uniq = list(dict.fromkeys(n for n in names if n))
|
||||
if len(uniq) < 2:
|
||||
return {}
|
||||
try:
|
||||
from services import semantic_search as ss
|
||||
if hasattr(ss, 'is_configured') and not ss.is_configured():
|
||||
return {}
|
||||
vectors = []
|
||||
for i in range(0, len(uniq), batch):
|
||||
out = ss.embed(uniq[i:i + batch], timeout=timeout)
|
||||
if not out or len(out) != len(uniq[i:i + batch]):
|
||||
return {}
|
||||
vectors.extend(out)
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
parent = list(range(len(uniq)))
|
||||
|
||||
def find(x):
|
||||
while parent[x] != x:
|
||||
parent[x] = parent[parent[x]]
|
||||
x = parent[x]
|
||||
return x
|
||||
|
||||
def union(a, b):
|
||||
ra, rb = find(a), find(b)
|
||||
if ra != rb:
|
||||
parent[rb] = ra
|
||||
|
||||
for i in range(len(uniq)):
|
||||
for j in range(i + 1, len(uniq)):
|
||||
sim = ss.cosine_similarity(vectors[i], vectors[j])
|
||||
if sim is not None and sim >= threshold:
|
||||
union(i, j)
|
||||
|
||||
return {uniq[i]: find(i) for i in range(len(uniq))}
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description='Clean improbable QWEN fitment rows.')
|
||||
p.add_argument('--tenant', type=int, required=True)
|
||||
p.add_argument('--source', default='qwen_ai')
|
||||
p.add_argument('--min-rows', type=int, default=120,
|
||||
help='flag items with >= N fitment rows')
|
||||
p.add_argument('--name-min-items', type=int, default=8,
|
||||
help='flag a normalized name shared by >= N inventory items')
|
||||
p.add_argument('--min-name-vol', type=int, default=100,
|
||||
help='only cluster names whose item count is >= this')
|
||||
p.add_argument('--semantic', action='store_true',
|
||||
help='merge name variants with embeddings (opt-in)')
|
||||
p.add_argument('--similarity', type=float, default=0.92)
|
||||
p.add_argument('--semantic-max', type=int, default=150,
|
||||
help='max distinct names sent to the embedding provider')
|
||||
p.add_argument('--mode', choices=['mark', 'delete'], default='mark')
|
||||
p.add_argument('--apply', action='store_true', help='persist (default dry-run)')
|
||||
p.add_argument('--limit', type=int, default=0, help='cap inventory ids scanned')
|
||||
p.add_argument('--inventory-ids', default=None,
|
||||
help='comma-separated inventory ids to force as suspects (targeted runs)')
|
||||
p.add_argument('--show', type=int, default=12)
|
||||
args = p.parse_args()
|
||||
|
||||
conn = get_tenant_conn(args.tenant)
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
print(f"tenant={args.tenant} source={args.source} min_rows={args.min_rows} "
|
||||
f"name_min_items={args.name_min_items} semantic={args.semantic} "
|
||||
f"mode={args.mode} apply={args.apply}")
|
||||
|
||||
if args.inventory_ids:
|
||||
# Targeted run: skip the heavy detection scans, use given ids only.
|
||||
rows = []
|
||||
name_to_ids = {}
|
||||
by_volume = by_name = by_semantic = set()
|
||||
print(" targeted mode: --inventory-ids provided, detection skipped")
|
||||
else:
|
||||
rows = _collect_counts(cur, args.source, args.limit or None)
|
||||
print(f" inventory items with source={args.source}: {len(rows)}")
|
||||
|
||||
by_volume = {iid for iid, _n, n in rows if n >= args.min_rows}
|
||||
|
||||
name_to_ids = {}
|
||||
for iid, name, _n in rows:
|
||||
name_to_ids.setdefault(_norm_name(name), set()).add(iid)
|
||||
by_name = set()
|
||||
for name, ids in name_to_ids.items():
|
||||
if len(ids) >= args.name_min_items:
|
||||
by_name.update(ids)
|
||||
|
||||
by_semantic = set()
|
||||
if args.semantic:
|
||||
vol_names = [name for name, ids in name_to_ids.items()
|
||||
if len(ids) >= args.min_name_vol]
|
||||
vol_names = vol_names[:args.semantic_max]
|
||||
clusters = _semantic_clusters(vol_names, threshold=args.similarity)
|
||||
if not clusters:
|
||||
print(" [semantic] sin embeddings (degradado) — solo reglas")
|
||||
else:
|
||||
cluster_ids = {}
|
||||
for name, cid in clusters.items():
|
||||
cluster_ids.setdefault(cid, set()).update(name_to_ids.get(name, set()))
|
||||
for cid, ids in cluster_ids.items():
|
||||
if len(ids) >= args.name_min_items:
|
||||
by_semantic.update(ids)
|
||||
print(f" [semantic] names clustered={len(clusters)} "
|
||||
f"extra_suspect_items={len(by_semantic - by_volume - by_name)}")
|
||||
|
||||
suspects = by_volume | by_name | by_semantic
|
||||
if args.inventory_ids:
|
||||
forced = {int(x) for x in args.inventory_ids.split(',') if x.strip()}
|
||||
suspects |= forced
|
||||
print(f" suspect inventory items: volume={len(by_volume)} "
|
||||
f"universal_name={len(by_name)} semantic={len(by_semantic)} "
|
||||
f"union={len(suspects)}")
|
||||
|
||||
if args.show:
|
||||
print(" top inflated names:")
|
||||
ranked = sorted(name_to_ids.items(), key=lambda kv: -len(kv[1]))[:args.show]
|
||||
for name, ids in ranked:
|
||||
print(f" {name[:48]:<48} items={len(ids):<5}")
|
||||
|
||||
# Count rows that would be affected
|
||||
affected = 0
|
||||
suspect_list = sorted(suspects)
|
||||
for i in range(0, len(suspect_list), 1000):
|
||||
chunk = suspect_list[i:i + 1000]
|
||||
if not chunk:
|
||||
continue
|
||||
cur.execute(
|
||||
f"SELECT count(*) FROM inventory_vehicle_compat "
|
||||
f"WHERE source = %s AND inventory_id = ANY(%s)",
|
||||
(args.source, chunk))
|
||||
affected += cur.fetchone()[0]
|
||||
total_src = sum(n for _i, _n, n in rows)
|
||||
pct = (affected / total_src * 100) if total_src else 0.0
|
||||
print(f" rows affected: {affected} / {total_src} ({pct:.1f}% of source={args.source})")
|
||||
|
||||
if not args.apply:
|
||||
print(" DRY-RUN: nothing written. Re-run with --apply to persist.")
|
||||
return 0
|
||||
|
||||
done = 0
|
||||
for i in range(0, len(suspect_list), 1000):
|
||||
chunk = suspect_list[i:i + 1000]
|
||||
if not chunk:
|
||||
continue
|
||||
if args.mode == 'delete':
|
||||
cur.execute(
|
||||
"DELETE FROM inventory_vehicle_compat "
|
||||
"WHERE source = %s AND inventory_id = ANY(%s)",
|
||||
(args.source, chunk))
|
||||
else:
|
||||
cur.execute(
|
||||
"UPDATE inventory_vehicle_compat "
|
||||
"SET source = 'qwen_rejected', confidence = 0 "
|
||||
"WHERE source = %s AND inventory_id = ANY(%s)",
|
||||
(args.source, chunk))
|
||||
done += cur.rowcount
|
||||
conn.commit()
|
||||
print(f" APPLIED mode={args.mode}: rows={done}")
|
||||
return 0
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
123
scripts/credit_score_report.py
Normal file
123
scripts/credit_score_report.py
Normal file
@@ -0,0 +1,123 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Credit risk score report (A9).
|
||||
|
||||
Prints rule-based credit scores for a tenant's customers using
|
||||
``services.credit_scoring.score_customer``. Read-only: never writes to the DB.
|
||||
|
||||
Usage (as root/postgres, from /home/Autopartes):
|
||||
python3 scripts/credit_score_report.py --tenant 31
|
||||
python3 scripts/credit_score_report.py --tenant 45 --only-credit
|
||||
python3 scripts/credit_score_report.py --all --tier risk
|
||||
python3 scripts/credit_score_report.py --tenant 31 --json
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'pos'))
|
||||
|
||||
from tenant_db import get_tenant_conn, get_master_conn # noqa: E402
|
||||
from services.credit_scoring import score_customer # noqa: E402
|
||||
|
||||
|
||||
def list_tenants(only_id=None):
|
||||
conn = get_master_conn()
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
if only_id:
|
||||
cur.execute(
|
||||
"SELECT id, name, db_name FROM tenants "
|
||||
"WHERE is_active = true AND id = %s",
|
||||
(only_id,),
|
||||
)
|
||||
else:
|
||||
cur.execute(
|
||||
"SELECT id, name, db_name FROM tenants WHERE is_active = true ORDER BY id"
|
||||
)
|
||||
rows = cur.fetchall()
|
||||
cur.close()
|
||||
return rows
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def customer_ids(conn, only_credit=False, limit=None):
|
||||
cur = conn.cursor()
|
||||
where = "WHERE c.is_active = true"
|
||||
if only_credit:
|
||||
where += " AND (c.credit_limit > 0 OR c.credit_balance > 0)"
|
||||
sql = f"SELECT c.id FROM customers c {where} ORDER BY c.id"
|
||||
if limit:
|
||||
sql += " LIMIT %s"
|
||||
cur.execute(sql, (limit,))
|
||||
else:
|
||||
cur.execute(sql)
|
||||
ids = [r[0] for r in cur.fetchall()]
|
||||
cur.close()
|
||||
return ids
|
||||
|
||||
|
||||
def report_tenant(tenant_id, tenant_name, only_credit, limit, tier_filter, as_json):
|
||||
conn = get_tenant_conn(tenant_id)
|
||||
try:
|
||||
ids = customer_ids(conn, only_credit=only_credit, limit=limit)
|
||||
rows = []
|
||||
for cid in ids:
|
||||
try:
|
||||
rows.append(score_customer(conn, cid))
|
||||
except Exception as exc:
|
||||
rows.append({'customer_id': cid, 'error': str(exc)})
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
if tier_filter:
|
||||
rows = [r for r in rows if r.get('tier') == tier_filter]
|
||||
|
||||
if as_json:
|
||||
print(json.dumps({'tenant_id': tenant_id, 'tenant': tenant_name, 'scores': rows},
|
||||
ensure_ascii=False, indent=2, default=str))
|
||||
return
|
||||
|
||||
print(f"\n=== Tenant {tenant_id} — {tenant_name} ({len(rows)} clientes) ===")
|
||||
print(f"{'ID':>6} {'score':>5} {'tier':<6} {'lim_sug':>12} {'datos':>5} señal principal")
|
||||
for r in sorted(rows, key=lambda x: x.get('score', 0)):
|
||||
if 'error' in r:
|
||||
print(f"{r['customer_id']:>6} ERROR: {r['error']}")
|
||||
continue
|
||||
reasons = r.get('reasons') or []
|
||||
headline = next(
|
||||
(x for x in reasons if 'Mora' in x or 'atraso' in x or 'excedido' in x),
|
||||
reasons[-1] if reasons else '',
|
||||
)
|
||||
print(f"{r['customer_id']:>6} {r['score']:>5} {r['tier']:<6} "
|
||||
f"{r['recommended_limit']:>12,.2f} {r['data_points']:>5} {headline}")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Credit risk score report (A9)")
|
||||
group = parser.add_mutually_exclusive_group(required=True)
|
||||
group.add_argument('--tenant', type=int, help='Tenant id')
|
||||
group.add_argument('--all', action='store_true', help='All active tenants')
|
||||
parser.add_argument('--only-credit', action='store_true',
|
||||
help='Only customers with a credit line or balance')
|
||||
parser.add_argument('--tier', choices=['good', 'medium', 'risk'],
|
||||
help='Only show customers in this tier')
|
||||
parser.add_argument('--limit', type=int, help='Max customers per tenant')
|
||||
parser.add_argument('--json', action='store_true', help='Emit JSON')
|
||||
args = parser.parse_args()
|
||||
|
||||
tenants = list_tenants(None if args.all else args.tenant)
|
||||
if not tenants:
|
||||
print("No tenants found", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
for tenant_id, name, _db_name in tenants:
|
||||
report_tenant(tenant_id, name, args.only_credit, args.limit,
|
||||
args.tier, args.json)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
303
scripts/enrich_descriptions.py
Normal file
303
scripts/enrich_descriptions.py
Normal file
@@ -0,0 +1,303 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate short Spanish descriptions for empty catalog descriptions.
|
||||
|
||||
Two targets:
|
||||
|
||||
inventory (tenant DB) — 95% of rows have ``description`` empty
|
||||
(Rached: 6851/6851; Capi: 3766/14568).
|
||||
supplier_catalog (master) — 17,246/17,246 have no description at all.
|
||||
|
||||
Description format (documented, enforced by the prompt):
|
||||
|
||||
* one short sentence, 8–20 words, plain Mexican Spanish;
|
||||
* starts with the part type, then the brand/compatibility qualifier already
|
||||
present in the name (never invent a vehicle, price, stock or warranty);
|
||||
* no emojis, no markdown, no quotes, no trailing period duplication.
|
||||
|
||||
The LLM call batches ``--batch-llm`` items per request through the same
|
||||
``ai_chat`` backends (QWEN first, OpenRouter free models as fallback). When
|
||||
the provider does not answer, the item is skipped — nothing is written and no
|
||||
exception is raised.
|
||||
|
||||
Quality gate: nothing is written unless ``--apply``. Pass ``--review-out
|
||||
FILE`` to dump every generated candidate as JSONL so a human can review the
|
||||
batch before applying. A checkpoint (last processed id per target) lets long
|
||||
runs resume.
|
||||
|
||||
Usage (as root, from /home/Autopartes):
|
||||
python3 scripts/enrich_descriptions.py --tenant 31 --target inventory --limit 50
|
||||
python3 scripts/enrich_descriptions.py --tenant 31 --target inventory \
|
||||
--limit 200 --review-out /tmp/desc_rached.jsonl
|
||||
python3 scripts/enrich_descriptions.py --target supplier_catalog --limit 100 --dry-run
|
||||
python3 scripts/enrich_descriptions.py --tenant 31 --target inventory --apply --limit 200
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'pos'))
|
||||
|
||||
from tenant_db import get_tenant_conn, get_master_conn # noqa: E402
|
||||
|
||||
DEFAULT_CHECKPOINT = os.path.join(
|
||||
os.path.dirname(os.path.abspath(__file__)), '..', 'backups',
|
||||
'enrich_descriptions_checkpoint.json'
|
||||
)
|
||||
|
||||
SYSTEM_PROMPT = (
|
||||
"Eres redactor de catálogo de una refaccionaria mexicana. Escribes "
|
||||
"descripciones de autopartes claras y breves en español. NUNCA inventas "
|
||||
"compatibilidad de vehículos, precios, garantías ni stock: solo describes "
|
||||
"lo que se te da. Responde SOLO con JSON válido, sin texto adicional."
|
||||
)
|
||||
|
||||
USER_PROMPT = (
|
||||
"Para cada artículo genera una descripción de 8 a 20 palabras en español "
|
||||
"mexicano que empiece con el tipo de pieza. Prohibido: emojis, markdown, "
|
||||
"precios, garantías, años de vehículo que no aparezcan en el nombre.\n"
|
||||
"Devuelve un arreglo JSON: "
|
||||
'[{"id": <id>, "descripcion": "<texto>"}, ...]\n'
|
||||
"Artículos:\n__ITEMS__"
|
||||
)
|
||||
|
||||
|
||||
def _load_checkpoint(path):
|
||||
try:
|
||||
with open(path) as f:
|
||||
return json.load(f)
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def _save_checkpoint(path, data):
|
||||
try:
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
with open(path, 'w') as f:
|
||||
json.dump(data, f, indent=2)
|
||||
except Exception as e:
|
||||
print(f" ! checkpoint no guardado: {e}")
|
||||
|
||||
|
||||
def _llm_descriptions(items, timeout=60):
|
||||
"""Return {id: description} via ai_chat backends; {} on any failure."""
|
||||
try:
|
||||
from services import ai_chat
|
||||
except Exception:
|
||||
return {}
|
||||
payload = json.dumps(
|
||||
[{"id": it["id"], "nombre": it["label"]} for it in items],
|
||||
ensure_ascii=False,
|
||||
)
|
||||
messages = [
|
||||
{"role": "system", "content": SYSTEM_PROMPT},
|
||||
{"role": "user", "content": USER_PROMPT.replace("__ITEMS__", payload)},
|
||||
]
|
||||
backends = []
|
||||
if getattr(ai_chat, "QWEN_ENABLED", False):
|
||||
backends.append((ai_chat.QWEN_CHAT_URL, ai_chat.QWEN_API_KEY, ai_chat.QWEN_MODEL))
|
||||
if getattr(ai_chat, "OPENROUTER_API_KEY", None):
|
||||
backends.append((ai_chat.OPENROUTER_URL, ai_chat.OPENROUTER_API_KEY, ai_chat.MODEL))
|
||||
for url, key, model_id in backends:
|
||||
try:
|
||||
ai_chat._validate_model(model_id)
|
||||
except Exception:
|
||||
continue
|
||||
try:
|
||||
res = ai_chat._post_chat_completion(
|
||||
url, key, model_id, messages,
|
||||
max_tokens=1400, temperature=0.2, timeout=timeout)
|
||||
except Exception:
|
||||
res = None
|
||||
if not res:
|
||||
continue
|
||||
content = (res.get("content") or "").strip()
|
||||
if content.startswith("```"):
|
||||
content = "\n".join(content.split("\n")[1:-1])
|
||||
try:
|
||||
parsed = json.loads(content)
|
||||
except Exception:
|
||||
continue
|
||||
if isinstance(parsed, dict):
|
||||
parsed = parsed.get("items") or parsed.get("data") or []
|
||||
out = {}
|
||||
for row in parsed or []:
|
||||
if not isinstance(row, dict):
|
||||
continue
|
||||
rid = row.get("id")
|
||||
text = (row.get("descripcion") or row.get("description") or "").strip()
|
||||
if rid is not None and text:
|
||||
out[int(rid)] = text[:300]
|
||||
if out:
|
||||
return out
|
||||
return {}
|
||||
|
||||
|
||||
def _iter_inventory(conn, after_id, limit, batch):
|
||||
cur = conn.cursor()
|
||||
where = ["i.is_active = true",
|
||||
"(i.description IS NULL OR i.description = '')"]
|
||||
params = []
|
||||
if after_id is not None:
|
||||
where.append("i.id > %s")
|
||||
params.append(after_id)
|
||||
cur.execute(f"""
|
||||
SELECT i.id, i.name, i.brand, c.name
|
||||
FROM inventory i
|
||||
LEFT JOIN categories c ON c.id = i.category_id
|
||||
WHERE {' AND '.join(where)}
|
||||
ORDER BY i.id
|
||||
LIMIT %s
|
||||
""", params + [batch])
|
||||
rows = cur.fetchall()
|
||||
cur.close()
|
||||
return rows
|
||||
|
||||
|
||||
def _iter_supplier(conn, after_id, batch):
|
||||
cur = conn.cursor()
|
||||
where = ["is_active = true", "(description IS NULL OR description = '')"]
|
||||
params = []
|
||||
if after_id is not None:
|
||||
where.append("id > %s")
|
||||
params.append(after_id)
|
||||
cur.execute(f"""
|
||||
SELECT id, supplier_name, sku, name, category
|
||||
FROM supplier_catalog
|
||||
WHERE {' AND '.join(where)}
|
||||
ORDER BY id
|
||||
LIMIT %s
|
||||
""", params + [batch])
|
||||
rows = cur.fetchall()
|
||||
cur.close()
|
||||
return rows
|
||||
|
||||
|
||||
def run(args, state):
|
||||
master = get_master_conn() if args.target in ('supplier_catalog', 'both') else None
|
||||
tenant = get_tenant_conn(args.tenant) if args.target in ('inventory', 'both') else None
|
||||
review = open(args.review_out, 'a') if args.review_out else None
|
||||
try:
|
||||
total = {'scanned': 0, 'generated': 0, 'written': 0, 'llm_fail': 0}
|
||||
targets = []
|
||||
if args.target in ('inventory', 'both'):
|
||||
targets.append('inventory')
|
||||
if args.target in ('supplier_catalog', 'both'):
|
||||
targets.append('supplier_catalog')
|
||||
|
||||
for target in targets:
|
||||
key = f"{target}:{args.tenant or 'master'}"
|
||||
after_id = None if args.reset else state.get(key, {}).get('after_id')
|
||||
conn = tenant if target == 'inventory' else master
|
||||
if conn is None:
|
||||
continue
|
||||
print(f"\n=== {target} (tenant={args.tenant}) after_id={after_id} ===")
|
||||
while True:
|
||||
if target == 'inventory':
|
||||
rows = _iter_inventory(conn, after_id, args.limit, args.batch)
|
||||
else:
|
||||
rows = _iter_supplier(conn, after_id, args.batch)
|
||||
if not rows:
|
||||
break
|
||||
|
||||
# Build LLM request payload
|
||||
items = []
|
||||
for r in rows:
|
||||
if target == 'inventory':
|
||||
rid, name, brand, cat = r
|
||||
label = ", ".join(x for x in [name, brand, cat] if x)
|
||||
else:
|
||||
rid, sup, sku, name, cat = r
|
||||
label = ", ".join(x for x in [name, cat, sup,
|
||||
f"SKU {sku}"] if x)
|
||||
items.append({'id': rid, 'label': label})
|
||||
|
||||
got = _llm_descriptions(items)
|
||||
if not got:
|
||||
total['llm_fail'] += 1
|
||||
print(" LLM sin respuesta — se detiene el lote (degradado, "
|
||||
"checkpoint no avanza)")
|
||||
break
|
||||
|
||||
for it in items:
|
||||
rid = it['id']
|
||||
text = got.get(rid)
|
||||
total['scanned'] += 1
|
||||
if not text:
|
||||
continue
|
||||
total['generated'] += 1
|
||||
if review:
|
||||
review.write(json.dumps(
|
||||
{'target': target, 'id': rid, 'text': text,
|
||||
'source_label': it['label']},
|
||||
ensure_ascii=False) + "\n")
|
||||
review.flush()
|
||||
if args.apply:
|
||||
c = conn.cursor()
|
||||
try:
|
||||
if target == 'inventory':
|
||||
c.execute(
|
||||
"UPDATE inventory SET description = %s "
|
||||
"WHERE id = %s AND (description IS NULL OR description = '')",
|
||||
(text, rid))
|
||||
else:
|
||||
c.execute(
|
||||
"UPDATE supplier_catalog SET description = %s "
|
||||
"WHERE id = %s AND (description IS NULL OR description = '')",
|
||||
(text, rid))
|
||||
total['written'] += c.rowcount
|
||||
conn.commit()
|
||||
finally:
|
||||
c.close()
|
||||
if total['generated'] <= args.show:
|
||||
print(f" {rid:<8} {it['label'][:44]:<44} -> {text[:70]}")
|
||||
|
||||
after_id = rows[-1][0]
|
||||
state[key] = {'after_id': after_id, 'written': total['written'],
|
||||
'updated_at': time.strftime('%Y-%m-%dT%H:%M:%S')}
|
||||
if args.apply:
|
||||
_save_checkpoint(args.checkpoint, state)
|
||||
if len(rows) < args.batch:
|
||||
break
|
||||
if args.limit and total['scanned'] >= args.limit:
|
||||
break
|
||||
|
||||
print(f"\nresumen: {total} apply={args.apply}")
|
||||
return 0
|
||||
finally:
|
||||
if review:
|
||||
review.close()
|
||||
if tenant is not None:
|
||||
tenant.close()
|
||||
if master is not None:
|
||||
master.close()
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description='Generate short Spanish descriptions.')
|
||||
p.add_argument('--tenant', type=int)
|
||||
p.add_argument('--target', choices=['inventory', 'supplier_catalog', 'both'],
|
||||
default='inventory')
|
||||
p.add_argument('--limit', type=int, default=0)
|
||||
p.add_argument('--batch', type=int, default=10, help='items per LLM request')
|
||||
p.add_argument('--apply', action='store_true', help='write descriptions')
|
||||
p.add_argument('--dry-run', action='store_true', default=True)
|
||||
p.add_argument('--review-out', default=None, help='JSONL file with candidates')
|
||||
p.add_argument('--reset', action='store_true')
|
||||
p.add_argument('--show', type=int, default=5)
|
||||
p.add_argument('--checkpoint', default=DEFAULT_CHECKPOINT)
|
||||
args = p.parse_args()
|
||||
|
||||
if args.apply:
|
||||
args.dry_run = False
|
||||
if args.target in ('inventory', 'both') and not args.tenant:
|
||||
p.error('--tenant requerido para target inventory')
|
||||
|
||||
state = _load_checkpoint(args.checkpoint)
|
||||
return run(args, state)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
139
scripts/eval_sku_matcher.py
Normal file
139
scripts/eval_sku_matcher.py
Normal file
@@ -0,0 +1,139 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Measure the SKU name-matcher: difflib (baseline) vs QWEN embeddings.
|
||||
|
||||
Runs ``sku_matcher.match_inventory`` in dry-run twice over the same inventory
|
||||
slice — once with the difflib name fallback only, once with the semantic
|
||||
(embedding) fallback enabled — and compares:
|
||||
|
||||
* coverage: how many items got matched at all
|
||||
* precision: among items that already carry a ``catalog_part_id`` (ground
|
||||
truth), how often the matcher agrees with the stored id
|
||||
|
||||
The second pass reuses the ``CatalogIndex`` built by the first, so catalog-name
|
||||
embeddings are computed once (in-memory cache).
|
||||
|
||||
Usage (as root, from /home/Autopartes):
|
||||
python3 scripts/eval_sku_matcher.py --tenant 45 --limit 300
|
||||
python3 scripts/eval_sku_matcher.py --tenant 45 --limit 300 --prefer-below 0.9
|
||||
python3 scripts/eval_sku_matcher.py --tenant 31 --limit 200 --threshold 0.88
|
||||
|
||||
Nothing is written: ``apply=False`` and the UPDATE guard never overwrites an
|
||||
existing ``catalog_part_id`` anyway.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'pos'))
|
||||
|
||||
from tenant_db import get_tenant_conn, get_master_conn # noqa: E402
|
||||
from services.sku_matcher import load_catalog_index, match_inventory # noqa: E402
|
||||
|
||||
|
||||
def _precision(res):
|
||||
"""(attempts_on_linked, correct) using catalog_part_id as ground truth."""
|
||||
attempts = correct = 0
|
||||
for r in res.get('results', []):
|
||||
truth = r.get('existing_catalog_id')
|
||||
if not truth:
|
||||
continue
|
||||
attempts += 1
|
||||
if r.get('matched_catalog_id') == truth:
|
||||
correct += 1
|
||||
return attempts, correct
|
||||
|
||||
|
||||
def _report(label, res):
|
||||
attempts, correct = _precision(res)
|
||||
prec = (correct / attempts * 100) if attempts else 0.0
|
||||
print(f" {label:<12} matched={res['matched']:<4} "
|
||||
f"by_method={res['by_method']}")
|
||||
print(f" {'':<12} precision_on_linked={correct}/{attempts} ({prec:.1f}%)")
|
||||
if res.get('semantic'):
|
||||
s = res['semantic']
|
||||
print(f" {'':<12} catalog_emb_cache={s['catalog_embeddings_cached']} "
|
||||
f"embed_requests={s['embed_requests']}")
|
||||
return res['matched'], attempts, correct
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description='Eval SKU name matching (difflib vs embeddings).')
|
||||
p.add_argument('--tenant', type=int, required=True)
|
||||
p.add_argument('--limit', type=int, default=300, help='items to scan')
|
||||
p.add_argument('--offset', type=int, default=0)
|
||||
p.add_argument('--threshold', type=float, default=0.86,
|
||||
help='semantic cosine threshold')
|
||||
p.add_argument('--prefer-below', type=float, default=None,
|
||||
help='also run semantic when fuzzy score < this value')
|
||||
p.add_argument('--name-threshold', type=float, default=0.82)
|
||||
p.add_argument('--top-k', type=int, default=64,
|
||||
help='candidate names embedded per item (bounded cost)')
|
||||
p.add_argument('--batch', type=int, default=64)
|
||||
p.add_argument('--supplier', default=None)
|
||||
p.add_argument('--sample', type=int, default=8, help='examples to print')
|
||||
p.add_argument('--only-linked', action='store_true',
|
||||
help='restrict to items with a catalog_part_id (ground truth)')
|
||||
p.add_argument('--show-examples', action='store_true')
|
||||
args = p.parse_args()
|
||||
|
||||
tenant = get_tenant_conn(args.tenant)
|
||||
master = get_master_conn()
|
||||
try:
|
||||
print(f"Building catalog index (supplier={args.supplier or 'ALL'})...")
|
||||
idx = load_catalog_index(master, supplier=args.supplier)
|
||||
print(f" catalog items={idx.total} sku_keys={len(idx.by_sku)} "
|
||||
f"xref_keys={len(idx.by_xref)}")
|
||||
|
||||
common = dict(
|
||||
limit=args.limit, offset=args.offset,
|
||||
only_unmatched=not args.only_linked, only_linked=args.only_linked,
|
||||
apply=False, create_aliases=False, supplier=args.supplier,
|
||||
name_threshold=args.name_threshold, sample_limit=args.sample,
|
||||
collect=True,
|
||||
)
|
||||
|
||||
print(f"\nInventory slice: tenant={args.tenant} limit={args.limit} "
|
||||
f"offset={args.offset} (dry-run)")
|
||||
base = match_inventory(tenant, master, use_semantic=False, index=idx, **common)
|
||||
print("\n[1] difflib baseline")
|
||||
_report('baseline', base)
|
||||
|
||||
sem = match_inventory(
|
||||
tenant, master, use_semantic=True, index=idx,
|
||||
semantic_threshold=args.threshold,
|
||||
semantic_top_k=args.top_k,
|
||||
semantic_batch=args.batch,
|
||||
semantic_prefer_below=args.prefer_below,
|
||||
**common,
|
||||
)
|
||||
print("\n[2] + semantic (embeddings)")
|
||||
b_m, b_a, b_c = _report('semantic', sem)
|
||||
|
||||
# Deltas
|
||||
print("\n--- delta ---")
|
||||
print(f" matched: {base['matched']} -> {sem['matched']} "
|
||||
f"(+{sem['matched'] - base['matched']})")
|
||||
b_att, b_cor = _precision(base)
|
||||
s_att, s_cor = _precision(sem)
|
||||
bp = (b_cor / b_att * 100) if b_att else 0.0
|
||||
sp = (s_cor / s_att * 100) if s_att else 0.0
|
||||
print(f" precision on linked: {bp:.1f}% -> {sp:.1f}%")
|
||||
|
||||
if args.show_examples:
|
||||
print("\nExamples (semantic pass):")
|
||||
for r in sem.get('results', [])[:args.sample]:
|
||||
mark = ''
|
||||
if r.get('existing_catalog_id'):
|
||||
mark = ' OK' if r['existing_catalog_id'] == r['matched_catalog_id'] else ' MISMATCH'
|
||||
print(f" [{r['method']:<14} {r['score']:.3f}] "
|
||||
f"{str(r['part_number'])[:16]:<16} {str(r['name'])[:38]:<38} "
|
||||
f"-> cat {r['matched_catalog_id']}{mark}")
|
||||
finally:
|
||||
tenant.close()
|
||||
master.close()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
66
scripts/expire_demos.py
Normal file
66
scripts/expire_demos.py
Normal file
@@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Expire demo tenants whose trial has ended.
|
||||
|
||||
A tenant on plan='demo' is a time-limited full trial (no resource limits).
|
||||
When its active subscription's expires_at passes, this job downgrades it to
|
||||
'free' and marks the subscription as expired.
|
||||
|
||||
Safe by default: runs in DRY-RUN and only reports. Pass --apply to enforce.
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
|
||||
import psycopg2
|
||||
|
||||
MASTER_DB_URL = os.environ.get(
|
||||
'MASTER_DB_URL', 'postgresql://postgres@localhost/nexus_autoparts'
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description='Expire ended demo tenants.')
|
||||
ap.add_argument('--apply', action='store_true',
|
||||
help='Actually downgrade expired demos (default: dry-run)')
|
||||
args = ap.parse_args()
|
||||
|
||||
conn = psycopg2.connect(MASTER_DB_URL)
|
||||
cur = conn.cursor()
|
||||
cur.execute("""
|
||||
SELECT t.id, t.name, s.expires_at
|
||||
FROM tenants t
|
||||
JOIN subscriptions s ON s.tenant_id = t.id
|
||||
WHERE t.plan = 'demo'
|
||||
AND t.is_active = true
|
||||
AND s.status = 'active'
|
||||
AND s.expires_at IS NOT NULL
|
||||
AND s.expires_at < NOW()
|
||||
ORDER BY t.id
|
||||
""")
|
||||
rows = cur.fetchall()
|
||||
|
||||
if not rows:
|
||||
print('No expired demo tenants.')
|
||||
else:
|
||||
print(f'{len(rows)} demo tenant(s) with an expired subscription:')
|
||||
for tid, name, exp in rows:
|
||||
print(f' #{tid} {name} (expiró {exp:%Y-%m-%d})')
|
||||
|
||||
if args.apply:
|
||||
for tid, _name, _exp in rows:
|
||||
cur.execute("UPDATE tenants SET plan = 'free' WHERE id = %s", (tid,))
|
||||
cur.execute(
|
||||
"UPDATE subscriptions SET status = 'expired' "
|
||||
"WHERE tenant_id = %s AND status = 'active'",
|
||||
(tid,),
|
||||
)
|
||||
conn.commit()
|
||||
print(f'Applied: {len(rows)} tenant(s) downgraded to free.')
|
||||
else:
|
||||
print('DRY-RUN: nothing changed. Re-run with --apply to enforce.')
|
||||
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
111
scripts/grade_piece_vision.py
Normal file
111
scripts/grade_piece_vision.py
Normal file
@@ -0,0 +1,111 @@
|
||||
#!/usr/bin/env python3
|
||||
"""A4 CLI — suggest a condition grade for a used (yonke) piece from a photo.
|
||||
|
||||
Uses ``services.part_vision`` (QWEN -> Hermes -> OpenRouter). When no vision
|
||||
backend answers it prints the returned error and exits non-zero.
|
||||
|
||||
Usage (from /home/Autopartes):
|
||||
python3 scripts/grade_piece_vision.py --tenant 28 --piece 300537
|
||||
python3 scripts/grade_piece_vision.py --tenant 28 --piece 300537 --image /ruta/foto.jpg
|
||||
python3 scripts/grade_piece_vision.py --image /ruta/foto.jpg --name "Faro" --dry-run
|
||||
python3 scripts/grade_piece_vision.py --tenant 28 --piece 300537 --apply
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'pos'))
|
||||
|
||||
|
||||
def _load_env():
|
||||
env_file = '/etc/nexus/pos.env'
|
||||
if os.environ.get('MASTER_DB_URL') or not os.path.exists(env_file):
|
||||
return
|
||||
for line in open(env_file):
|
||||
line = line.strip()
|
||||
if not line or line.startswith('#') or '=' not in line:
|
||||
continue
|
||||
k, v = line.split('=', 1)
|
||||
os.environ.setdefault(k.strip(), v.strip().strip('"').strip("'"))
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument('--tenant', type=int, help='tenant id')
|
||||
ap.add_argument('--piece', type=int, help='inventory id of the piece')
|
||||
ap.add_argument('--image', help='image file path (overrides the stored photo)')
|
||||
ap.add_argument('--name', help='piece name for the prompt')
|
||||
ap.add_argument('--category', help='piece category for the prompt')
|
||||
ap.add_argument('--condition', default='usado')
|
||||
ap.add_argument('--apply', action='store_true',
|
||||
help='persist the suggested grade to inventory.condition_grade')
|
||||
args = ap.parse_args()
|
||||
|
||||
_load_env()
|
||||
|
||||
from services import part_vision
|
||||
from services.image_service import DATA_DIR, STATIC_DIR
|
||||
|
||||
image_bytes = None
|
||||
piece = None
|
||||
conn = None
|
||||
|
||||
try:
|
||||
if args.tenant and args.piece:
|
||||
from tenant_db import get_tenant_conn
|
||||
from services import yonke_service
|
||||
conn = get_tenant_conn(args.tenant)
|
||||
piece = yonke_service.get_piece(conn, args.piece)
|
||||
if not piece:
|
||||
print(json.dumps({'error': 'Piece not found'}))
|
||||
return 1
|
||||
|
||||
if args.image:
|
||||
with open(args.image, 'rb') as fh:
|
||||
image_bytes = fh.read()
|
||||
elif piece:
|
||||
url = piece.get('image_url')
|
||||
for img in piece.get('images') or []:
|
||||
if img.get('is_primary'):
|
||||
url = img.get('url') or url
|
||||
break
|
||||
idx = (url or '').find('/images/parts/')
|
||||
if idx != -1:
|
||||
rel = url[idx + len('/images/parts/'):]
|
||||
for base in (DATA_DIR, STATIC_DIR):
|
||||
path = os.path.join(base, rel)
|
||||
if os.path.exists(path):
|
||||
with open(path, 'rb') as fh:
|
||||
image_bytes = fh.read()
|
||||
break
|
||||
|
||||
if not image_bytes:
|
||||
print(json.dumps({'error': 'No image available (use --image or a piece with a photo)'}))
|
||||
return 1
|
||||
|
||||
result = part_vision.suggest_grade(
|
||||
image_bytes=image_bytes,
|
||||
name=args.name or (piece or {}).get('name'),
|
||||
category=args.category,
|
||||
condition=args.condition,
|
||||
)
|
||||
|
||||
if args.apply and result.get('condition_grade_suggested') and conn and piece:
|
||||
cur = conn.cursor()
|
||||
cur.execute('UPDATE inventory SET condition_grade = %s WHERE id = %s',
|
||||
(result['condition_grade_suggested'], piece['id']))
|
||||
conn.commit()
|
||||
cur.close()
|
||||
result['applied'] = True
|
||||
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
return 0 if result.get('condition_grade_suggested') else 2
|
||||
finally:
|
||||
if conn:
|
||||
conn.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
156
scripts/import_capi_inventory.py
Normal file
156
scripts/import_capi_inventory.py
Normal file
@@ -0,0 +1,156 @@
|
||||
#!/usr/bin/env python3
|
||||
"""One-off: import Capi Autopartes inventory from CSV into tenant_capi_autopartes.
|
||||
|
||||
Replicates the semantics of pos/blueprints/inventory_bp.py::bulk_import_items:
|
||||
- creates missing categories
|
||||
- INSERT new items (with generated barcode) or UPDATE existing by part_number
|
||||
- records initial stock on the main branch via inventory_engine.record_initial
|
||||
- registers sku_secundario aliases
|
||||
|
||||
Usage (as postgres):
|
||||
MASTER_DB_URL=postgresql://postgres@/nexus_autoparts \
|
||||
TENANT_DB_URL_TEMPLATE='postgresql://postgres@/{db_name}' \
|
||||
POS_JWT_SECRET=... REDIS_ENABLED=false \
|
||||
python3 scripts/import_capi_inventory.py
|
||||
"""
|
||||
|
||||
import csv
|
||||
import io
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'pos'))
|
||||
|
||||
import psycopg2 # noqa: E402
|
||||
from services.barcode_generator import generate_barcode # noqa: E402
|
||||
from services.inventory_engine import record_initial # noqa: E402
|
||||
|
||||
CSV_PATH = '/home/Autopartes/data/Inventario capi.csv'
|
||||
TENANT_DB = 'tenant_capi_autopartes'
|
||||
BRANCH_ID = 1
|
||||
COMMIT_EVERY = 500
|
||||
|
||||
|
||||
def parse_rows(path):
|
||||
raw = open(path, 'rb').read()
|
||||
text = None
|
||||
for enc in ('utf-8-sig', 'cp1252', 'latin-1'):
|
||||
try:
|
||||
text = raw.decode(enc)
|
||||
break
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
if text is None:
|
||||
raise RuntimeError('Could not decode CSV')
|
||||
return list(csv.DictReader(io.StringIO(text)))
|
||||
|
||||
|
||||
def to_decimal(val, default=0.0):
|
||||
try:
|
||||
v = float(str(val).replace(',', '').strip())
|
||||
return v
|
||||
except (ValueError, TypeError):
|
||||
return default
|
||||
|
||||
|
||||
def main():
|
||||
rows = parse_rows(CSV_PATH)
|
||||
print(f'Filas en CSV: {len(rows)}')
|
||||
|
||||
conn = psycopg2.connect(dbname=TENANT_DB)
|
||||
cur = conn.cursor()
|
||||
|
||||
# 1. Create categories
|
||||
cat_names = sorted({r['categoria'].strip() for r in rows if r['categoria'].strip()})
|
||||
for name in cat_names:
|
||||
cur.execute("INSERT INTO categories (name) VALUES (%s) ON CONFLICT DO NOTHING", (name,))
|
||||
conn.commit()
|
||||
cur.execute("SELECT id, name FROM categories")
|
||||
category_map = {r[1].strip().lower(): r[0] for r in cur.fetchall()}
|
||||
print(f'Categorías creadas/existentes: {len(category_map)}')
|
||||
|
||||
# 2. Import items
|
||||
created = updated = skipped = 0
|
||||
warnings = []
|
||||
|
||||
for i, row in enumerate(rows, start=1):
|
||||
part_number = row['numero_de_parte'].strip()
|
||||
name = row['nombre'].strip()
|
||||
if not part_number or not name:
|
||||
skipped += 1
|
||||
warnings.append(f'Fila {i}: sin numero_de_parte o nombre')
|
||||
continue
|
||||
|
||||
brand = row['marca'].strip()
|
||||
price_1 = to_decimal(row['precio'])
|
||||
stock = int(to_decimal(row['cantidad']))
|
||||
cost = to_decimal(row['costo'])
|
||||
description = (row.get('descripcion') or '').strip()
|
||||
category_name = row['categoria'].strip()
|
||||
category_id = category_map.get(category_name.lower()) if category_name else None
|
||||
|
||||
cur.execute("SELECT id FROM inventory WHERE part_number = %s", (part_number,))
|
||||
existing = cur.fetchone()
|
||||
|
||||
if existing:
|
||||
item_id = existing[0]
|
||||
cur.execute("""
|
||||
UPDATE inventory SET
|
||||
name = %s,
|
||||
brand = COALESCE(NULLIF(%s,''), brand),
|
||||
cost = CASE WHEN %s > 0 THEN %s ELSE cost END,
|
||||
price_1 = CASE WHEN %s > 0 THEN %s ELSE price_1 END,
|
||||
description = COALESCE(NULLIF(%s,''), description),
|
||||
category_id = COALESCE(%s, category_id)
|
||||
WHERE id = %s
|
||||
""", (name, brand, cost, cost, price_1, price_1, description, category_id, item_id))
|
||||
updated += 1
|
||||
if stock > 0:
|
||||
record_initial(conn, item_id, BRANCH_ID, stock, cost if cost > 0 else None)
|
||||
else:
|
||||
barcode = generate_barcode(conn, TENANT_DB)
|
||||
cur.execute("""
|
||||
INSERT INTO inventory
|
||||
(part_number, barcode, name, brand, cost, price_1, location, description, category_id, unit)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
RETURNING id
|
||||
""", (part_number, barcode, name, brand, cost, price_1, '', description, category_id, 'PZA'))
|
||||
item_id = cur.fetchone()[0]
|
||||
created += 1
|
||||
if stock > 0:
|
||||
record_initial(conn, item_id, BRANCH_ID, stock, cost if cost > 0 else None)
|
||||
|
||||
sku_secondary = (row.get('sku_secundario') or '').strip()
|
||||
if sku_secondary:
|
||||
cur.execute("""
|
||||
INSERT INTO inventory_sku_aliases (inventory_id, sku, label)
|
||||
VALUES (%s, %s, %s) ON CONFLICT DO NOTHING
|
||||
""", (item_id, sku_secondary, 'Secundario'))
|
||||
|
||||
if i % COMMIT_EVERY == 0:
|
||||
conn.commit()
|
||||
print(f' {i}/{len(rows)} procesadas (nuevos={created}, act={updated})', flush=True)
|
||||
|
||||
conn.commit()
|
||||
|
||||
cur.execute("SELECT count(*) FROM inventory")
|
||||
total_inv = cur.fetchone()[0]
|
||||
cur.execute("SELECT COALESCE(SUM(stock),0) FROM inventory_stock WHERE branch_id = %s", (BRANCH_ID,))
|
||||
total_stock = cur.fetchone()[0]
|
||||
cur.execute("SELECT count(*) FROM inventory_sku_aliases")
|
||||
total_alias = cur.fetchone()[0]
|
||||
|
||||
print('\n=== RESUMEN ===')
|
||||
print(f'Nuevos: {created} | Actualizados: {updated} | Saltados: {skipped}')
|
||||
print(f'Total inventario: {total_inv} | Stock total sucursal {BRANCH_ID}: {total_stock} | Alias: {total_alias}')
|
||||
if warnings:
|
||||
print(f'Advertencias ({len(warnings)}):')
|
||||
for w in warnings[:20]:
|
||||
print(' -', w)
|
||||
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
171
scripts/index_meilisearch.py
Normal file
171
scripts/index_meilisearch.py
Normal file
@@ -0,0 +1,171 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Index catalog data into Meilisearch for typo-tolerant search.
|
||||
|
||||
Indexes:
|
||||
- nexus_supplier_catalog (master DB) — supplier SKUs with cross-references
|
||||
- nexus_inventory_{db_name} — per-tenant inventory (optional refresh)
|
||||
|
||||
Both indexes are configured with:
|
||||
- searchableAttributes including `aliases` (inventory_sku_aliases / supplier
|
||||
interchange numbers → cross-reference search, A6)
|
||||
- a Spanish/English `synonyms` map (curated + services.translations)
|
||||
|
||||
Usage (as postgres):
|
||||
MASTER_DB_NAME=nexus_autoparts python3 scripts/index_meilisearch.py all
|
||||
python3 scripts/index_meilisearch.py catalog
|
||||
python3 scripts/index_meilisearch.py tenant tenant_capi_autopartes
|
||||
python3 scripts/index_meilisearch.py synonyms # settings only, no reindex
|
||||
|
||||
Designed to run from a systemd timer after inventory imports/syncs.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'pos'))
|
||||
|
||||
import psycopg2 # noqa: E402
|
||||
import meilisearch # noqa: E402
|
||||
from services.meili_search import ( # noqa: E402
|
||||
MEILI_URL, MEILI_API_KEY,
|
||||
inventory_index_settings, catalog_index_settings,
|
||||
apply_synonyms_to_all, get_synonyms,
|
||||
)
|
||||
|
||||
MASTER_DB = os.environ.get('MASTER_DB_NAME', 'nexus_autoparts')
|
||||
CATALOG_INDEX = 'nexus_supplier_catalog'
|
||||
|
||||
client = meilisearch.Client(MEILI_URL, MEILI_API_KEY)
|
||||
|
||||
|
||||
def _catalog_aliases(cur):
|
||||
"""catalog_id -> [interchange part numbers] (A6 cross-refs)."""
|
||||
aliases = {}
|
||||
try:
|
||||
cur.execute("SELECT catalog_id, part_number FROM supplier_catalog_interchange")
|
||||
for cid, pn in cur.fetchall():
|
||||
if cid is not None and pn:
|
||||
aliases.setdefault(cid, []).append(str(pn))
|
||||
except Exception:
|
||||
cur.connection.rollback()
|
||||
return aliases
|
||||
|
||||
|
||||
def index_supplier_catalog(cur):
|
||||
cur.execute("""
|
||||
SELECT sc.id, sc.sku, sc.name, sc.category, sc.supplier_name, sc.image_url,
|
||||
COALESCE(scp.price, NULL) AS supplier_price
|
||||
FROM supplier_catalog sc
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT price FROM supplier_catalog_prices p
|
||||
WHERE p.catalog_id = sc.id ORDER BY p.id DESC LIMIT 1
|
||||
) scp ON true
|
||||
WHERE sc.is_active
|
||||
""")
|
||||
rows = cur.fetchall()
|
||||
aliases = _catalog_aliases(cur)
|
||||
docs = [
|
||||
{
|
||||
'id': r[0], 'sku': r[1], 'name': r[2], 'category': r[3] or '',
|
||||
'supplier_name': r[4] or '', 'image_url': r[5] or '',
|
||||
'supplier_price': float(r[6]) if r[6] else None,
|
||||
'aliases': aliases.get(r[0], []),
|
||||
}
|
||||
for r in rows
|
||||
]
|
||||
idx = client.index(CATALOG_INDEX)
|
||||
idx.update_settings(catalog_index_settings())
|
||||
task = idx.add_documents(docs, primary_key='id')
|
||||
client.wait_for_task(task.task_uid)
|
||||
n_alias = sum(1 for d in docs if d['aliases'])
|
||||
print(f'[catalog] {len(docs)} documentos indexados en {CATALOG_INDEX} '
|
||||
f'({n_alias} con cross-refs)', flush=True)
|
||||
|
||||
|
||||
def index_tenant(db_name):
|
||||
conn = psycopg2.connect(dbname=db_name)
|
||||
cur = conn.cursor()
|
||||
cur.execute("""
|
||||
SELECT i.id, i.part_number, i.name, i.brand, i.price_1, i.image_url,
|
||||
COALESCE(s.stock, 0)
|
||||
FROM inventory i
|
||||
LEFT JOIN (
|
||||
SELECT inventory_id, SUM(stock) AS stock FROM inventory_stock
|
||||
GROUP BY inventory_id
|
||||
) s ON s.inventory_id = i.id
|
||||
WHERE i.is_active
|
||||
""")
|
||||
rows = cur.fetchall()
|
||||
|
||||
aliases = {}
|
||||
try:
|
||||
cur.execute("""
|
||||
SELECT inventory_id, sku FROM inventory_sku_aliases
|
||||
WHERE is_active AND sku IS NOT NULL
|
||||
""")
|
||||
for inv_id, sku in cur.fetchall():
|
||||
aliases.setdefault(inv_id, []).append(str(sku))
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
|
||||
docs = [
|
||||
{
|
||||
'id': r[0], 'part_number': r[1], 'name': r[2], 'brand': r[3] or '',
|
||||
'price_1': float(r[4]) if r[4] else None, 'image_url': r[5] or '',
|
||||
'stock': int(r[6] or 0),
|
||||
'aliases': aliases.get(r[0], []),
|
||||
}
|
||||
for r in rows
|
||||
]
|
||||
index_name = f"nexus_inventory_{db_name}"
|
||||
idx = client.index(index_name)
|
||||
idx.update_settings(inventory_index_settings())
|
||||
task = idx.add_documents(docs, primary_key='id')
|
||||
client.wait_for_task(task.task_uid)
|
||||
n_alias = sum(1 for d in docs if d['aliases'])
|
||||
print(f'[{db_name}] {len(docs)} documentos indexados en {index_name} '
|
||||
f'({n_alias} con aliases)', flush=True)
|
||||
conn.close()
|
||||
|
||||
|
||||
def apply_synonyms():
|
||||
"""Push the synonym map to existing indexes without reindexing."""
|
||||
results = apply_synonyms_to_all()
|
||||
for uid, ok in results.items():
|
||||
print(f'[synonyms] {uid}: {"ok" if ok is True else ok}', flush=True)
|
||||
print(f'[synonyms] {len(get_synonyms())} términos', flush=True)
|
||||
|
||||
|
||||
def main():
|
||||
mode = sys.argv[1] if len(sys.argv) > 1 else 'all'
|
||||
|
||||
if mode == 'synonyms':
|
||||
apply_synonyms()
|
||||
return
|
||||
|
||||
if mode in ('all', 'catalog'):
|
||||
conn = psycopg2.connect(dbname=MASTER_DB)
|
||||
cur = conn.cursor()
|
||||
index_supplier_catalog(cur)
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
||||
if mode in ('all', 'tenants'):
|
||||
mconn = psycopg2.connect(dbname=MASTER_DB)
|
||||
mcur = mconn.cursor()
|
||||
if len(sys.argv) > 2:
|
||||
dbs = [sys.argv[2]]
|
||||
else:
|
||||
mcur.execute("SELECT db_name FROM tenants WHERE is_active = true")
|
||||
dbs = [r[0] for r in mcur.fetchall()]
|
||||
for db in dbs:
|
||||
try:
|
||||
index_tenant(db)
|
||||
except Exception as e:
|
||||
print(f'[{db}] ERROR: {e}', flush=True)
|
||||
mcur.close()
|
||||
mconn.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
173
scripts/match_inventory_catalog.py
Executable file
173
scripts/match_inventory_catalog.py
Executable file
@@ -0,0 +1,173 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Inventory <-> supplier-catalog SKU matching (A2).
|
||||
|
||||
Populates ``inventory.catalog_part_id`` and registers matched supplier SKUs as
|
||||
``inventory_sku_aliases`` by matching ``inventory.part_number`` against the
|
||||
master ``supplier_catalog.sku`` / ``supplier_catalog_interchange`` (normalized)
|
||||
and, as a fallback, fuzzy name similarity.
|
||||
|
||||
Usage (as root/postgres, from /home/Autopartes):
|
||||
python3 scripts/match_inventory_catalog.py --tenant 45 --limit 500
|
||||
python3 scripts/match_inventory_catalog.py --tenant 45 --dry-run
|
||||
python3 scripts/match_inventory_catalog.py --all --batch 2000
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'pos'))
|
||||
|
||||
from tenant_db import get_tenant_conn, get_master_conn # noqa: E402
|
||||
from services.sku_matcher import match_inventory, count_links # noqa: E402
|
||||
|
||||
DEFAULT_CHECKPOINT = os.path.join(
|
||||
os.path.dirname(os.path.abspath(__file__)), '..', 'backups',
|
||||
'match_catalog_checkpoint.json'
|
||||
)
|
||||
|
||||
|
||||
def list_tenants(only_id=None):
|
||||
conn = get_master_conn()
|
||||
cur = conn.cursor()
|
||||
if only_id:
|
||||
cur.execute(
|
||||
"SELECT id, name, db_name FROM tenants WHERE is_active = true AND id = %s",
|
||||
(only_id,),
|
||||
)
|
||||
else:
|
||||
cur.execute(
|
||||
"SELECT id, name, db_name FROM tenants WHERE is_active = true ORDER BY id"
|
||||
)
|
||||
rows = cur.fetchall()
|
||||
cur.close()
|
||||
conn.close()
|
||||
return rows
|
||||
|
||||
|
||||
def load_checkpoint(path):
|
||||
try:
|
||||
with open(path) as f:
|
||||
return json.load(f)
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def save_checkpoint(path, data):
|
||||
try:
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
with open(path, 'w') as f:
|
||||
json.dump(data, f, indent=2)
|
||||
except Exception as e:
|
||||
print(f" ! no se pudo guardar checkpoint: {e}")
|
||||
|
||||
|
||||
def run_tenant(tenant_id, name, db_name, args, checkpoint):
|
||||
print(f"\n=== tenant {tenant_id} — {name} ({db_name}) ===")
|
||||
tenant_conn = get_tenant_conn(tenant_id)
|
||||
master_conn = get_master_conn()
|
||||
key = str(tenant_id)
|
||||
state = checkpoint.get(key, {})
|
||||
after_id = None if args.reset else state.get('after_id')
|
||||
|
||||
before_total, before_linked = count_links(tenant_conn)
|
||||
print(f" antes: total={before_total} linked={before_linked} "
|
||||
f"({(before_linked * 100.0 / before_total) if before_total else 0:.2f}%)")
|
||||
|
||||
total_scanned = total_matched = total_updated = total_aliases = 0
|
||||
total_by_method = {}
|
||||
try:
|
||||
while True:
|
||||
call_limit = args.batch
|
||||
if args.limit:
|
||||
call_limit = min(args.batch, max(args.limit - total_scanned, 1))
|
||||
res = match_inventory(
|
||||
tenant_conn, master_conn,
|
||||
limit=call_limit,
|
||||
offset=0,
|
||||
only_unmatched=args.only_unmatched,
|
||||
create_aliases=not args.no_aliases,
|
||||
use_names=not args.no_names,
|
||||
name_threshold=args.name_threshold,
|
||||
apply=not args.dry_run,
|
||||
supplier=args.supplier,
|
||||
after_id=after_id,
|
||||
sample_limit=args.show,
|
||||
)
|
||||
total_scanned += res['scanned']
|
||||
total_matched += res['matched']
|
||||
total_updated += res['updated']
|
||||
total_aliases += res['aliases_created']
|
||||
for k, v in res['by_method'].items():
|
||||
total_by_method[k] = total_by_method.get(k, 0) + v
|
||||
|
||||
if args.show and res['sample']:
|
||||
for s in res['sample']:
|
||||
print(f" {s['part_number']:<20} -> cat#{s['matched_catalog_id']} "
|
||||
f"{s['matched_sku']:<14} ({s['method']}, {s['score']})")
|
||||
|
||||
after_id = res['last_id']
|
||||
checkpoint[key] = {
|
||||
'after_id': after_id,
|
||||
'matched': state.get('matched', 0) + total_matched,
|
||||
'updated_at': time.strftime('%Y-%m-%dT%H:%M:%S'),
|
||||
}
|
||||
if not args.dry_run:
|
||||
save_checkpoint(args.checkpoint, checkpoint)
|
||||
|
||||
print(f" batch: scanned={res['scanned']} matched={res['matched']} "
|
||||
f"updated={res['updated']} aliases={res['aliases_created']} "
|
||||
f"methods={res['by_method']}")
|
||||
if res['scanned'] < args.batch or res['scanned'] == 0:
|
||||
break
|
||||
if args.limit and total_scanned >= args.limit:
|
||||
break
|
||||
finally:
|
||||
after_total, after_linked = count_links(tenant_conn)
|
||||
print(f" despues: total={after_total} linked={after_linked} "
|
||||
f"({(after_linked * 100.0 / after_total) if after_total else 0:.2f}%)")
|
||||
print(f" total: scanned={total_scanned} matched={total_matched} "
|
||||
f"updated={total_updated} aliases={total_aliases} "
|
||||
f"methods={total_by_method}")
|
||||
tenant_conn.close()
|
||||
master_conn.close()
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description='Match inventory to supplier catalog (A2).')
|
||||
p.add_argument('--tenant', type=int, help='tenant id (omit with --all)')
|
||||
p.add_argument('--all', action='store_true', help='process every active tenant')
|
||||
p.add_argument('--batch', type=int, default=5000, help='rows per batch')
|
||||
p.add_argument('--limit', type=int, default=0, help='max rows per tenant (0 = all)')
|
||||
p.add_argument('--only-unmatched', action='store_true', default=True,
|
||||
help='only items with catalog_part_id NULL/0 (default)')
|
||||
p.add_argument('--all-items', dest='only_unmatched', action='store_false',
|
||||
help='consider every active item')
|
||||
p.add_argument('--no-aliases', action='store_true', help='do not create aliases')
|
||||
p.add_argument('--no-names', action='store_true', help='disable fuzzy-name fallback')
|
||||
p.add_argument('--name-threshold', type=float, default=0.82)
|
||||
p.add_argument('--supplier', help='restrict matching to one supplier name')
|
||||
p.add_argument('--dry-run', action='store_true', help='do not persist')
|
||||
p.add_argument('--reset', action='store_true', help='ignore saved checkpoint')
|
||||
p.add_argument('--show', type=int, default=5, help='sample rows to print (0 = none)')
|
||||
p.add_argument('--checkpoint', default=DEFAULT_CHECKPOINT)
|
||||
args = p.parse_args()
|
||||
|
||||
if not args.all and not args.tenant:
|
||||
p.error('--tenant ID o --all requerido')
|
||||
|
||||
checkpoint = load_checkpoint(args.checkpoint)
|
||||
tenants = list_tenants(None if args.all else args.tenant)
|
||||
if not tenants:
|
||||
print('sin tenants para procesar')
|
||||
return 1
|
||||
|
||||
for tid, name, db_name in tenants:
|
||||
run_tenant(tid, name, db_name, args, checkpoint)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
303
scripts/meli_attach_images.py
Normal file
303
scripts/meli_attach_images.py
Normal file
@@ -0,0 +1,303 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Attach images to inventory items that are (or will be) published on MeLi.
|
||||
|
||||
Fixes publishing block #1: ML requires at least one picture, and Rached (and
|
||||
most tenants) have zero rows in inventory_images, so publish_items rejects
|
||||
every item with "El producto no tiene imagen".
|
||||
|
||||
Image sources, in priority order:
|
||||
1. local — files already on disk for this tenant:
|
||||
/home/Autopartes/pos/static/images/parts/<tenant>_<inventory_id>.jpg
|
||||
/home/Autopartes/data/images/parts/<tenant>/<inventory_id>_full.webp
|
||||
2. catalog — master supplier_catalog.image_url, joined by part number / SKU
|
||||
3. ml — pictures of the MercadoLibre listing already linked to the item
|
||||
(downloaded and re-hosted locally with services.image_service, or, with
|
||||
--remote, registered as the ML secure_url directly)
|
||||
|
||||
NOTE on the data/ Excel catalogs (VAZLO, YOKOMITSU, KNADIAN, LUK, RAYBESTOS,
|
||||
KG, PRODUCTOS_RACHED_2026): they were checked and contain NO image columns
|
||||
(marca / sku / INTERCAMBIO / nombre / carro only), so they cannot supply
|
||||
images. Also, master supplier_catalog.image_url is empty for all 17k rows
|
||||
today, so the catalog source is currently a no-op.
|
||||
|
||||
Registered images go to inventory_images (sort_order + is_primary; the first
|
||||
becomes primary) and inventory.image_url is set to the primary one.
|
||||
|
||||
Usage (from /home/Autopartes):
|
||||
python3 scripts/meli_attach_images.py --tenant 31 # dry-run
|
||||
python3 scripts/meli_attach_images.py --tenant 31 --apply
|
||||
python3 scripts/meli_attach_images.py --tenant 31 --apply --remote
|
||||
python3 scripts/meli_attach_images.py --tenant 31 --apply --limit 5
|
||||
python3 scripts/meli_attach_images.py --tenant 31 --apply --inventory-id 4752
|
||||
python3 scripts/meli_attach_images.py --tenant 31 --json
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'pos'))
|
||||
|
||||
STATIC_PARTS_DIR = '/home/Autopartes/pos/static/images/parts'
|
||||
DATA_PARTS_DIR = '/home/Autopartes/data/images/parts'
|
||||
MELI_MAX_PICTURES = 10
|
||||
|
||||
|
||||
def _load_env():
|
||||
env_file = '/etc/nexus/pos.env'
|
||||
if os.environ.get('MASTER_DB_URL') or not os.path.exists(env_file):
|
||||
return
|
||||
for line in open(env_file):
|
||||
line = line.strip()
|
||||
if not line or line.startswith('#') or '=' not in line:
|
||||
continue
|
||||
key, value = line.split('=', 1)
|
||||
os.environ.setdefault(key.strip(), value.strip().strip('"').strip("'"))
|
||||
|
||||
|
||||
def _local_urls(tenant_id, inventory_id):
|
||||
"""Images already on disk for this tenant/item (relative, servable URLs)."""
|
||||
urls = []
|
||||
scraper = os.path.join(STATIC_PARTS_DIR, f'{tenant_id}_{inventory_id}.jpg')
|
||||
if os.path.exists(scraper):
|
||||
urls.append(f'/pos/static/images/parts/{tenant_id}_{inventory_id}.jpg')
|
||||
processed = os.path.join(STATIC_PARTS_DIR, str(tenant_id), f'{inventory_id}_full.webp')
|
||||
if os.path.exists(processed):
|
||||
urls.append(f'/pos/static/images/parts/{tenant_id}/{inventory_id}_full.webp')
|
||||
return urls
|
||||
|
||||
|
||||
def _catalog_urls(master_cur, skus):
|
||||
"""supplier_catalog.image_url for any of the given SKUs (non-empty only)."""
|
||||
keys = [s for s in (skus or []) if s]
|
||||
if not keys:
|
||||
return []
|
||||
master_cur.execute(
|
||||
"""
|
||||
SELECT DISTINCT image_url FROM supplier_catalog
|
||||
WHERE image_url IS NOT NULL AND image_url <> ''
|
||||
AND UPPER(sku) = ANY(%s)
|
||||
LIMIT %s
|
||||
""",
|
||||
([str(k).strip().upper() for k in keys], MELI_MAX_PICTURES),
|
||||
)
|
||||
return [r[0] for r in master_cur.fetchall()]
|
||||
|
||||
|
||||
def _ml_urls(svc, tenant_id, inventory_id, external_item_id, remote, apply=True):
|
||||
"""Pictures of the ML listing. Returns candidate URLs for inventory_images.
|
||||
|
||||
The first picture is downloaded and re-hosted locally with
|
||||
services.image_service (that service stores one file per item, so the
|
||||
primary is self-contained); the remaining pictures keep their ML-hosted
|
||||
secure_url, which is public and valid for ML itself.
|
||||
|
||||
In dry-run (apply=False) nothing is downloaded: secure_urls are returned
|
||||
as candidates so the caller can report without side effects.
|
||||
"""
|
||||
from services import image_service
|
||||
pictures = (svc.get_item(str(external_item_id)).get('pictures') or [])
|
||||
urls = []
|
||||
for index, pic in enumerate(pictures[:MELI_MAX_PICTURES]):
|
||||
remote_url = pic.get('secure_url') or pic.get('url')
|
||||
if not remote_url:
|
||||
continue
|
||||
if remote or not apply or index > 0:
|
||||
urls.append(remote_url)
|
||||
continue
|
||||
try:
|
||||
saved = image_service.save_image(tenant_id, inventory_id, image_url=remote_url)
|
||||
urls.append(saved['image_url'])
|
||||
except Exception as e:
|
||||
print(f" ! could not download {remote_url}: {e}", flush=True)
|
||||
urls.append(remote_url) # fall back to the ML-hosted URL
|
||||
return urls
|
||||
|
||||
|
||||
def _register_images(cur, inventory_id, urls, apply):
|
||||
"""Insert new inventory_images rows (first becomes primary) + set image_url."""
|
||||
urls = [u for u in dict.fromkeys(urls) if u] # dedupe, keep order
|
||||
if not urls:
|
||||
return 0
|
||||
|
||||
cur.execute(
|
||||
"SELECT url, is_primary FROM inventory_images WHERE inventory_id = %s "
|
||||
"ORDER BY is_primary DESC, sort_order, id",
|
||||
(inventory_id,),
|
||||
)
|
||||
existing = cur.fetchall()
|
||||
have = {r[0] for r in existing}
|
||||
has_primary = any(r[1] for r in existing)
|
||||
order = len(existing)
|
||||
|
||||
added = 0
|
||||
for url in urls:
|
||||
if url in have:
|
||||
continue
|
||||
is_primary = (not has_primary and added == 0)
|
||||
if apply:
|
||||
cur.execute(
|
||||
"INSERT INTO inventory_images (inventory_id, url, sort_order, is_primary) "
|
||||
"VALUES (%s, %s, %s, %s)",
|
||||
(inventory_id, url, order, is_primary),
|
||||
)
|
||||
have.add(url)
|
||||
added += 1
|
||||
order += 1
|
||||
if is_primary:
|
||||
has_primary = True
|
||||
|
||||
if apply and added:
|
||||
cur.execute(
|
||||
"SELECT url FROM inventory_images WHERE inventory_id = %s "
|
||||
"ORDER BY is_primary DESC, sort_order, id LIMIT 1",
|
||||
(inventory_id,),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if row:
|
||||
cur.execute("UPDATE inventory SET image_url = %s WHERE id = %s", (row[0], inventory_id))
|
||||
return added
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument('--tenant', type=int, required=True, help='tenant id (e.g. 31 = Rached)')
|
||||
ap.add_argument('--apply', action='store_true', help='write to inventory_images (default: dry-run)')
|
||||
ap.add_argument('--source', choices=['auto', 'local', 'catalog', 'ml'], default='auto',
|
||||
help='restrict to one source (default: auto = first available)')
|
||||
ap.add_argument('--remote', action='store_true',
|
||||
help='register the MeLi secure_url directly instead of downloading it')
|
||||
ap.add_argument('--limit', type=int, default=None, help='max items to process')
|
||||
ap.add_argument('--inventory-id', type=int, action='append', dest='inventory_ids',
|
||||
help='only this inventory id (repeatable)')
|
||||
ap.add_argument('--json', action='store_true', help='print the summary as JSON')
|
||||
args = ap.parse_args()
|
||||
|
||||
_load_env()
|
||||
|
||||
from tenant_db import get_tenant_conn, get_master_conn
|
||||
from services import marketplace_external_service as meli
|
||||
|
||||
conn = get_tenant_conn(args.tenant)
|
||||
master = get_master_conn()
|
||||
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
cfg = meli.get_meli_config(conn)
|
||||
svc = meli._get_meli_service(cfg, conn)
|
||||
|
||||
# Targets: explicit inventory ids, or every mapped ML listing.
|
||||
if args.inventory_ids:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT inventory_id, MAX(external_item_id)
|
||||
FROM marketplace_listings
|
||||
WHERE channel = 'mercadolibre' AND inventory_id = ANY(%s)
|
||||
GROUP BY inventory_id
|
||||
ORDER BY inventory_id
|
||||
""",
|
||||
(args.inventory_ids,),
|
||||
)
|
||||
targets = cur.fetchall()
|
||||
# Inventory ids with no listing yet still deserve an image.
|
||||
seen = {t[0] for t in targets}
|
||||
targets += [(iid, None) for iid in args.inventory_ids if iid not in seen]
|
||||
else:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT inventory_id, MAX(external_item_id)
|
||||
FROM marketplace_listings
|
||||
WHERE channel = 'mercadolibre' AND inventory_id IS NOT NULL
|
||||
GROUP BY inventory_id
|
||||
ORDER BY inventory_id
|
||||
"""
|
||||
)
|
||||
targets = cur.fetchall()
|
||||
|
||||
if args.limit:
|
||||
targets = targets[:args.limit]
|
||||
|
||||
master_cur = master.cursor()
|
||||
summary = {"tenant": args.tenant, "dry_run": not args.apply,
|
||||
"targets": len(targets), "with_images": 0, "no_image": 0,
|
||||
"images_added": 0, "sources": {"local": 0, "catalog": 0, "ml": 0},
|
||||
"no_image_items": [], "details": []}
|
||||
|
||||
if not targets:
|
||||
print("No targets: no marketplace listing is linked to inventory yet.")
|
||||
return 0
|
||||
|
||||
for inventory_id, external_item_id in targets:
|
||||
cur.execute(
|
||||
"SELECT part_number FROM inventory WHERE id = %s", (inventory_id,)
|
||||
)
|
||||
row = cur.fetchone()
|
||||
part_number = row[0] if row else None
|
||||
|
||||
urls, used = [], None
|
||||
want = lambda s: args.source in ('auto', s) # noqa: E731
|
||||
|
||||
if want('local'):
|
||||
local = _local_urls(args.tenant, inventory_id)
|
||||
if local:
|
||||
urls, used = local, 'local'
|
||||
if not urls and want('catalog'):
|
||||
catalog = _catalog_urls(master_cur, [part_number])
|
||||
if catalog:
|
||||
urls, used = catalog, 'catalog'
|
||||
if not urls and want('ml') and external_item_id and svc:
|
||||
try:
|
||||
ml = _ml_urls(svc, args.tenant, inventory_id, external_item_id,
|
||||
args.remote, apply=args.apply)
|
||||
if ml:
|
||||
urls, used = ml, 'ml'
|
||||
except Exception as e:
|
||||
print(f" [{inventory_id}] ML image fetch failed: {e}", flush=True)
|
||||
|
||||
if not urls:
|
||||
summary['no_image'] += 1
|
||||
summary['no_image_items'].append(inventory_id)
|
||||
summary['details'].append({
|
||||
"inventory_id": inventory_id, "part_number": part_number,
|
||||
"status": "no_image",
|
||||
})
|
||||
continue
|
||||
|
||||
added = _register_images(cur, inventory_id, urls, args.apply)
|
||||
if args.apply:
|
||||
conn.commit()
|
||||
summary['with_images'] += 1
|
||||
summary['images_added'] += added
|
||||
summary['sources'][used] = summary['sources'].get(used, 0) + 1
|
||||
summary['details'].append({
|
||||
"inventory_id": inventory_id, "part_number": part_number,
|
||||
"status": "ok", "source": used, "found": len(urls), "added": added,
|
||||
})
|
||||
|
||||
master_cur.close()
|
||||
cur.close()
|
||||
finally:
|
||||
master.close()
|
||||
conn.close()
|
||||
|
||||
if args.json:
|
||||
print(json.dumps(summary, indent=2, ensure_ascii=False, default=str))
|
||||
return 0
|
||||
|
||||
mode = 'DRY-RUN (no writes)' if summary['dry_run'] else 'APPLIED'
|
||||
print(f"[tenant {args.tenant}] attach images — {mode}")
|
||||
print(f" targets: {summary['targets']}")
|
||||
print(f" with images: {summary['with_images']} (by source: {summary['sources']})")
|
||||
print(f" images added: {summary['images_added']}")
|
||||
print(f" no image found: {summary['no_image']}")
|
||||
if summary['no_image_items']:
|
||||
print(f" inventory ids without a usable image: {summary['no_image_items']}")
|
||||
if summary['dry_run'] and summary['images_added'] >= 0:
|
||||
print("\n Re-run with --apply to persist the images in inventory_images.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
117
scripts/meli_questions_sla.py
Normal file
117
scripts/meli_questions_sla.py
Normal file
@@ -0,0 +1,117 @@
|
||||
#!/usr/bin/env python3
|
||||
"""MeLi questions SLA sweep for a tenant.
|
||||
|
||||
Lists unanswered MercadoLibre buyer questions older than N hours (the ones
|
||||
that hurt the seller's response score / reputation), oldest first, so the
|
||||
operator can answer them before the SLA is blown.
|
||||
|
||||
Auto-answering is separate and OPT-IN:
|
||||
* It only runs when the tenant has `meli_auto_answer_questions = true`.
|
||||
* `--auto-answer` sends the high-confidence, template-based answers built by
|
||||
services/question_ai (never free-form LLM text) through
|
||||
marketplace_external_service.auto_answer_questions().
|
||||
* Without `--auto-answer` the script only reports; it never sends anything.
|
||||
|
||||
Note: MercadoLibre returns the question status as UNANSWERED / ANSWERED. The
|
||||
service now normalizes it to lowercase on upsert, and the queries here are
|
||||
case-insensitive either way.
|
||||
|
||||
Usage (from /home/Autopartes):
|
||||
python3 scripts/meli_questions_sla.py --tenant 31
|
||||
python3 scripts/meli_questions_sla.py --tenant 31 --max-age-hours 48
|
||||
python3 scripts/meli_questions_sla.py --tenant 31 --auto-answer
|
||||
python3 scripts/meli_questions_sla.py --tenant 31 --json
|
||||
|
||||
Suggested systemd timer (per tenant), once an hour:
|
||||
ExecStart=/usr/bin/python3 /home/Autopartes/scripts/meli_questions_sla.py \
|
||||
--tenant 31 --max-age-hours 24
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'pos'))
|
||||
|
||||
|
||||
def _load_env():
|
||||
env_file = '/etc/nexus/pos.env'
|
||||
if os.environ.get('MASTER_DB_URL') or not os.path.exists(env_file):
|
||||
return
|
||||
for line in open(env_file):
|
||||
line = line.strip()
|
||||
if not line or line.startswith('#') or '=' not in line:
|
||||
continue
|
||||
key, value = line.split('=', 1)
|
||||
os.environ.setdefault(key.strip(), value.strip().strip('"').strip("'"))
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument('--tenant', type=int, required=True, help='tenant id (e.g. 31 = Rached)')
|
||||
ap.add_argument('--max-age-hours', type=float, default=24,
|
||||
help='age threshold in hours (default 24)')
|
||||
ap.add_argument('--limit', type=int, default=20,
|
||||
help='max questions to auto-answer when --auto-answer is used')
|
||||
ap.add_argument('--auto-answer', action='store_true',
|
||||
help='send high-confidence answers (requires meli_auto_answer_questions = true)')
|
||||
ap.add_argument('--json', action='store_true', help='print the full result as JSON')
|
||||
args = ap.parse_args()
|
||||
|
||||
_load_env()
|
||||
|
||||
from tenant_db import get_tenant_conn
|
||||
from services import marketplace_external_service as meli
|
||||
|
||||
conn = get_tenant_conn(args.tenant)
|
||||
try:
|
||||
stale = meli.list_stale_questions(conn, max_age_hours=args.max_age_hours)
|
||||
cfg = meli.get_meli_config(conn)
|
||||
result = {
|
||||
"tenant": args.tenant,
|
||||
"max_age_hours": args.max_age_hours,
|
||||
"auto_answer_enabled": bool(cfg.get("meli_auto_answer_questions")),
|
||||
"stale_count": len(stale),
|
||||
"stale": stale,
|
||||
}
|
||||
if args.auto_answer:
|
||||
if not cfg.get("meli_auto_answer_questions"):
|
||||
result["auto_answer"] = {
|
||||
"enabled": False,
|
||||
"reason": "meli_auto_answer_questions is not enabled for this tenant",
|
||||
}
|
||||
else:
|
||||
result["auto_answer"] = meli.auto_answer_questions(conn, limit=args.limit)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
if args.json:
|
||||
print(json.dumps(result, indent=2, ensure_ascii=False, default=str))
|
||||
return 0
|
||||
|
||||
print(f"[tenant {args.tenant}] questions SLA — unanswered older than "
|
||||
f"{args.max_age_hours}h: {result['stale_count']}")
|
||||
if not stale:
|
||||
print(" Nothing overdue.")
|
||||
for q in stale:
|
||||
age = q.get('age_hours')
|
||||
print(f" q#{q['id']} ({age}h) {q['buyer_nickname'] or 'comprador'}: "
|
||||
f"{(q['question_text'] or '')[:70]!r} — item {q['external_item_id']}")
|
||||
|
||||
if args.auto_answer:
|
||||
aa = result.get("auto_answer") or {}
|
||||
if aa.get("enabled") is False:
|
||||
print(f"\n auto-answer not run: {aa.get('reason')}")
|
||||
else:
|
||||
print(f"\n auto-answer: answered={aa.get('answered')} "
|
||||
f"skipped={aa.get('skipped')}")
|
||||
else:
|
||||
print("\n (report-only; re-run with --auto-answer to send high-confidence "
|
||||
"answers when meli_auto_answer_questions = true)")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
48
scripts/meli_sync_all.py
Normal file
48
scripts/meli_sync_all.py
Normal file
@@ -0,0 +1,48 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run MercadoLibre sync (stock/price + orders) for all enabled tenants.
|
||||
|
||||
Called by systemd timer every 15 minutes. Runs the Celery task bodies
|
||||
synchronously in-process (no broker needed from the timer context).
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'pos'))
|
||||
|
||||
import psycopg2 # noqa: E402
|
||||
|
||||
MASTER_DB = os.environ.get('MASTER_DB_NAME', 'nexus_autoparts')
|
||||
|
||||
|
||||
def main():
|
||||
conn = psycopg2.connect(dbname=MASTER_DB)
|
||||
cur = conn.cursor()
|
||||
cur.execute("""
|
||||
SELECT t.id FROM tenants t
|
||||
JOIN tenant_schema_version v ON v.tenant_id = t.id
|
||||
WHERE t.is_active
|
||||
ORDER BY t.id
|
||||
""")
|
||||
tenant_ids = [r[0] for r in cur.fetchall()]
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
||||
from tasks import sync_meli_stock_price_task, sync_meli_orders_task
|
||||
|
||||
for tid in tenant_ids:
|
||||
try:
|
||||
r1 = sync_meli_stock_price_task.run(tid)
|
||||
r2 = sync_meli_orders_task.run(tid)
|
||||
print(f'[tenant {tid}] stock={r1} orders={r2}', flush=True)
|
||||
except Exception as e:
|
||||
# Meli disabled / not connected for this tenant is the common case
|
||||
msg = str(e)
|
||||
if 'not configured' in msg or 'not connected' in msg:
|
||||
continue
|
||||
print(f'[tenant {tid}] ERROR: {msg[:200]}', flush=True)
|
||||
print('MeLi sync round complete.', flush=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
72
scripts/process_cfdi_queues.py
Normal file
72
scripts/process_cfdi_queues.py
Normal file
@@ -0,0 +1,72 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Process pending CFDI stamping queues for all tenants (Facturapi).
|
||||
|
||||
Called by systemd timer every 2 minutes so pending invoices stamp without
|
||||
anyone pressing a button. Each round first enqueues the missing payment
|
||||
complements (REP) of PPD invoices and the credit notes of returns.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'pos'))
|
||||
|
||||
import psycopg2 # noqa: E402
|
||||
|
||||
MASTER_DB = os.environ.get('MASTER_DB_NAME', 'nexus_autoparts')
|
||||
|
||||
|
||||
def main():
|
||||
from services.cfdi_queue import enqueue_pending_complements, process_queue
|
||||
|
||||
master = psycopg2.connect(dbname=MASTER_DB)
|
||||
mcur = master.cursor()
|
||||
mcur.execute("SELECT id, db_name FROM tenants WHERE is_active = true ORDER BY id")
|
||||
tenants = mcur.fetchall()
|
||||
mcur.close()
|
||||
master.close()
|
||||
|
||||
for tenant_id, db_name in tenants:
|
||||
try:
|
||||
conn = psycopg2.connect(dbname=db_name)
|
||||
cur = conn.cursor()
|
||||
cur.execute("""
|
||||
SELECT key, value FROM tenant_config
|
||||
WHERE key IN ('cfdi_facturapi_key', 'cfdi_facturapi_org_id',
|
||||
'facturapi_secret_key', 'facturapi_org_id')
|
||||
""")
|
||||
tcfg = {k: v for k, v in cur.fetchall()}
|
||||
cur.close()
|
||||
|
||||
if not tcfg.get('cfdi_facturapi_key') and not tcfg.get('facturapi_secret_key'):
|
||||
conn.close()
|
||||
continue # tenant has no CFDI configured
|
||||
|
||||
# Enqueue missing payment complements (REP) and credit notes first;
|
||||
# they only need the tenant DB, not the Facturapi API.
|
||||
try:
|
||||
enqueue_pending_complements(conn)
|
||||
except Exception as e:
|
||||
conn.rollback()
|
||||
print(f'[{db_name}] complement sync ERROR: {str(e)[:200]}', flush=True)
|
||||
|
||||
# Quick check: are there pending items?
|
||||
cur = conn.cursor()
|
||||
cur.execute("SELECT count(*) FROM cfdi_queue WHERE status IN ('pending','failed')")
|
||||
pending = cur.fetchone()[0]
|
||||
cur.close()
|
||||
if not pending:
|
||||
conn.close()
|
||||
continue
|
||||
|
||||
result = process_queue(conn, tcfg)
|
||||
conn.close()
|
||||
print(f'[{db_name}] processed={result.get("processed")} '
|
||||
f'stamped={result.get("stamped")} failed={result.get("failed")}', flush=True)
|
||||
except Exception as e:
|
||||
print(f'[{db_name}] ERROR: {str(e)[:200]}', flush=True)
|
||||
print('CFDI queue round complete.', flush=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
101
scripts/rollup_tenant_metrics.py
Normal file
101
scripts/rollup_tenant_metrics.py
Normal file
@@ -0,0 +1,101 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Daily business-metrics rollup for every tenant, into the master table
|
||||
``tenant_metrics_daily``.
|
||||
|
||||
Creates the table idempotently (no tenant migration runner involved) and
|
||||
upserts one row per tenant per day, so it is safe to re-run for the same date.
|
||||
|
||||
Usage:
|
||||
python3 scripts/rollup_tenant_metrics.py # today, active tenants
|
||||
python3 scripts/rollup_tenant_metrics.py --date 2026-10-03
|
||||
python3 scripts/rollup_tenant_metrics.py --all # include inactive tenants
|
||||
python3 scripts/rollup_tenant_metrics.py --days 30 # backfill last 30 days
|
||||
python3 scripts/rollup_tenant_metrics.py --tenant tenant_capi_autopartes
|
||||
python3 scripts/rollup_tenant_metrics.py --json
|
||||
|
||||
Exit code 0 on success, 2 when one or more tenants failed to roll up.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from datetime import date as date_cls
|
||||
|
||||
MANAGER_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "manager")
|
||||
if MANAGER_DIR not in sys.path:
|
||||
sys.path.insert(0, MANAGER_DIR)
|
||||
|
||||
from services import portfolio_service # noqa: E402
|
||||
|
||||
|
||||
def _print_report(result, health_by_tenant):
|
||||
print(f"Rollup {result['date_from']} → {result['date_to']} "
|
||||
f"({result['days']} día(s)) — {result['rows_written']} filas, "
|
||||
f"{len(result['tenants'])} tenants")
|
||||
print(f"{'tenant':<34} {'rows':>5} {'ventas(hoy)':>11} {'MX$ hoy':>10} "
|
||||
f"{'items':>8} {'fitment%':>8} {'health':>6}")
|
||||
for t in result["tenants"]:
|
||||
h = health_by_tenant.get(t["tenant_id"], {})
|
||||
last = t.get("last") or {}
|
||||
print(f"{t['db_name']:<34} {t['rows']:>5} "
|
||||
f"{last.get('ventas_count', ''):>11} "
|
||||
f"{last.get('ventas_total', ''):>10} "
|
||||
f"{last.get('items_activos', ''):>8} "
|
||||
f"{str(last.get('fitment_pct', '')):>8} "
|
||||
f"{str(h.get('health_score', '')):>6}")
|
||||
for e in result["errors"]:
|
||||
print(f" ERROR {e['db_name']}: {e['error']}", file=sys.stderr)
|
||||
print("\nHealth (último día):")
|
||||
for t in result["tenants"]:
|
||||
h = health_by_tenant.get(t["tenant_id"], {})
|
||||
codes = ", ".join(s["code"] for s in h.get("signals", [])) or "-"
|
||||
print(f" {t['db_name']:<34} {str(h.get('health_score')):>5} "
|
||||
f"{h.get('band', ''):<9} [{codes}]")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument("--date", help="last day of the range (YYYY-MM-DD, default today)")
|
||||
parser.add_argument("--days", type=int, default=1,
|
||||
help="number of days to roll up ending at --date (default 1)")
|
||||
parser.add_argument("--all", action="store_true",
|
||||
help="include inactive tenants (default: active only)")
|
||||
parser.add_argument("--tenant", action="append", default=None,
|
||||
help="restrict to a db_name or tenant id (repeatable)")
|
||||
parser.add_argument("--json", action="store_true", help="print raw JSON")
|
||||
args = parser.parse_args()
|
||||
|
||||
target = None
|
||||
if args.date:
|
||||
try:
|
||||
target = date_cls.fromisoformat(args.date)
|
||||
except ValueError:
|
||||
parser.error(f"fecha inválida: {args.date} (usa YYYY-MM-DD)")
|
||||
|
||||
result = portfolio_service.run_rollup(
|
||||
date=target,
|
||||
days=max(1, args.days),
|
||||
tenant_ids=args.tenant,
|
||||
include_inactive=args.all,
|
||||
)
|
||||
|
||||
# Attach the latest collected row + health per tenant for the report/JSON.
|
||||
metrics = portfolio_service.get_tenant_metrics(days=14)
|
||||
health_by_tenant = {t["tenant_id"]: t["health"] for t in metrics["tenants"]}
|
||||
latest_metrics = {t["tenant_id"]: t.get("metrics") for t in metrics["tenants"]}
|
||||
for t in result["tenants"]:
|
||||
t["last"] = latest_metrics.get(t["tenant_id"])
|
||||
t["health"] = health_by_tenant.get(t["tenant_id"])
|
||||
|
||||
if args.json:
|
||||
print(json.dumps(result, indent=2, default=str))
|
||||
else:
|
||||
_print_report(result, health_by_tenant)
|
||||
|
||||
return 2 if result["errors"] else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
105
scripts/run_anomaly_scan.py
Normal file
105
scripts/run_anomaly_scan.py
Normal file
@@ -0,0 +1,105 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run the A5 business anomaly scan for one or all tenants.
|
||||
|
||||
Reports, per tenant, the anomalies detected by services.anomaly_detector:
|
||||
atypical discounts (per cajero / per SKU), sales below cost, repeated
|
||||
cancellations and off-hours tickets.
|
||||
|
||||
Usage (as postgres):
|
||||
MASTER_DB_NAME=nexus_autopartes python3 scripts/run_anomaly_scan.py [db_name|all] [days]
|
||||
python3 scripts/run_anomaly_scan.py tenant_refaccionaria_atlas 30
|
||||
python3 scripts/run_anomaly_scan.py all 90 --json
|
||||
|
||||
Exit code is 1 when at least one critical/high finding is reported, so it can
|
||||
be wired into cron/Alertmanager; 0 otherwise.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'pos'))
|
||||
|
||||
import psycopg2 # noqa: E402
|
||||
from services.anomaly_detector import detect_anomalies # noqa: E402
|
||||
|
||||
MASTER_DB = os.environ.get('MASTER_DB_NAME', 'nexus_autoparts')
|
||||
|
||||
SEVERITY_ORDER = {'critical': 0, 'high': 1, 'medium': 2, 'low': 3}
|
||||
|
||||
|
||||
def _tenant_dbs():
|
||||
conn = psycopg2.connect(dbname=MASTER_DB)
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
cur.execute("SELECT db_name FROM tenants WHERE is_active = true ORDER BY db_name")
|
||||
dbs = [r[0] for r in cur.fetchall()]
|
||||
cur.close()
|
||||
finally:
|
||||
conn.close()
|
||||
return dbs
|
||||
|
||||
|
||||
def _print_report(db_name, report):
|
||||
sample = report['sample']
|
||||
cold = ' (COLD START: poca historia)' if report['cold_start'] else ''
|
||||
print(f"\n=== {db_name}{cold} ===")
|
||||
print(f" ventana: {report['window_days']}d | sale_items={sample['sale_items']} "
|
||||
f"sales={sample['sales']} empleados={sample['employees']}")
|
||||
if not report['employees'] and not report['skus']:
|
||||
print(" sin anomalías")
|
||||
return
|
||||
|
||||
for entry in report['employees']:
|
||||
print(f" [empleado {entry['employee_id']} {entry['employee_name']}] "
|
||||
f"score={entry['score']}")
|
||||
for a in sorted(entry['anomalies'], key=lambda x: SEVERITY_ORDER.get(x['severity'], 9)):
|
||||
print(f" - {a['severity']:8s} {a['type']:20s} {a['detail']}")
|
||||
for entry in report['skus']:
|
||||
types = ', '.join(sorted({a['type'] for a in entry['anomalies']}))
|
||||
print(f" [sku {entry['inventory_id']} {entry['name']}] score={entry['score']} -> {types}")
|
||||
|
||||
|
||||
def main():
|
||||
args = [a for a in sys.argv[1:] if not a.startswith('--')]
|
||||
as_json = '--json' in sys.argv
|
||||
target = args[0] if args else 'all'
|
||||
days = int(args[1]) if len(args) > 1 else 90
|
||||
|
||||
dbs = _tenant_dbs() if target == 'all' else [target]
|
||||
reports = {}
|
||||
exit_code = 0
|
||||
|
||||
for db in dbs:
|
||||
try:
|
||||
conn = psycopg2.connect(dbname=db)
|
||||
except Exception as e:
|
||||
print(f"[{db}] ERROR de conexión: {e}", file=sys.stderr)
|
||||
exit_code = 2
|
||||
continue
|
||||
try:
|
||||
report = detect_anomalies(conn, days=days)
|
||||
except Exception as e:
|
||||
print(f"[{db}] ERROR: {e}", file=sys.stderr)
|
||||
exit_code = 2
|
||||
continue
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
reports[db] = report
|
||||
if not as_json:
|
||||
_print_report(db, report)
|
||||
if report['summary']['by_type'] and any(
|
||||
e['anomalies'] and min(
|
||||
SEVERITY_ORDER.get(a['severity'], 9) for a in e['anomalies']) <= 1
|
||||
for e in report['employees'] + report['skus']):
|
||||
exit_code = max(exit_code, 1)
|
||||
|
||||
if as_json:
|
||||
print(json.dumps(reports, indent=2, default=str))
|
||||
|
||||
return exit_code
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
82
scripts/suggest_meli_answer.py
Normal file
82
scripts/suggest_meli_answer.py
Normal file
@@ -0,0 +1,82 @@
|
||||
#!/usr/bin/env python3
|
||||
"""A7 CLI — intent + suggested answer for MercadoLibre buyer questions.
|
||||
|
||||
Uses ``services.question_ai``. Never sends anything unless ``--auto-answer`` is
|
||||
given AND the tenant has ``meli_auto_answer_questions = true``.
|
||||
|
||||
Usage (from /home/Autopartes):
|
||||
python3 scripts/suggest_meli_answer.py --tenant 28 --question 123
|
||||
python3 scripts/suggest_meli_answer.py --tenant 28 --pending --no-llm
|
||||
python3 scripts/suggest_meli_answer.py --tenant 28 --auto-answer
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'pos'))
|
||||
|
||||
|
||||
def _load_env():
|
||||
env_file = '/etc/nexus/pos.env'
|
||||
if os.environ.get('MASTER_DB_URL') or not os.path.exists(env_file):
|
||||
return
|
||||
for line in open(env_file):
|
||||
line = line.strip()
|
||||
if not line or line.startswith('#') or '=' not in line:
|
||||
continue
|
||||
k, v = line.split('=', 1)
|
||||
os.environ.setdefault(k.strip(), v.strip().strip('"').strip("'"))
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument('--tenant', type=int, required=True)
|
||||
ap.add_argument('--question', type=int, help='local marketplace_questions.id')
|
||||
ap.add_argument('--pending', action='store_true',
|
||||
help='suggest for every unanswered question (limit --limit)')
|
||||
ap.add_argument('--limit', type=int, default=20)
|
||||
ap.add_argument('--no-llm', action='store_true', help='templates only, no LLM fallback')
|
||||
ap.add_argument('--auto-answer', action='store_true',
|
||||
help='send high-confidence template answers (requires tenant flag)')
|
||||
args = ap.parse_args()
|
||||
|
||||
_load_env()
|
||||
|
||||
from tenant_db import get_tenant_conn
|
||||
from services import question_ai, marketplace_external_service as meli
|
||||
|
||||
conn = get_tenant_conn(args.tenant)
|
||||
try:
|
||||
if args.auto_answer:
|
||||
print(json.dumps(meli.auto_answer_questions(conn), ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
use_llm = not args.no_llm
|
||||
if args.question:
|
||||
result = question_ai.get_question_suggestion(conn, args.question, use_llm=use_llm)
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
if args.pending:
|
||||
cur = conn.cursor()
|
||||
cur.execute("""
|
||||
SELECT id FROM marketplace_questions
|
||||
WHERE (status IS NULL OR status = 'unanswered')
|
||||
ORDER BY question_date ASC NULLS LAST LIMIT %s
|
||||
""", (args.limit,))
|
||||
ids = [r[0] for r in cur.fetchall()]
|
||||
cur.close()
|
||||
for qid in ids:
|
||||
result = question_ai.get_question_suggestion(conn, qid, use_llm=use_llm)
|
||||
print(json.dumps({'question_id': qid, **result}, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
ap.error('use --question, --pending or --auto-answer')
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user