feat(catalog): supplier catalog cleanup, fuzzy matching, and navigation fixes
- Cleaned 137+ fake engine-displacement models from supplier imports (v3/v4 scripts: Chevrolet, Ford, Chrysler, Dodge, Jeep, Nissan, etc.) - Removed 1,251+ corrupted models (INT. prefixes, year-suffix, torque specs, empty names, trailing-year variants) - Migrated supplier tables to master DB (supplier_catalog, supplier_catalog_compat, supplier_catalog_interchange) - Fixed _get_mye_ids_with_parts() to query supplier_catalog_compat from master DB so supplier-only vehicles appear for all tenants - Added fuzzy model matcher with parenthesis stripping, noise suffix removal, compact matching, prefix/substring fallback, model aliases, and ±3 year proximity - Matched compat rows: KEEP GREEN +14,152, KNADIAN +3,021, VAZLO +127,500, LUK +477, RAYBESTOS +1,743 - Added KNADIAN catalog importer with year-range expansion and future-year filtering - Added VAZLO catalog importer with position parsing and SKU-in-model cleanup - Added Keep Green, LUK, Yokomitsu, Raybestos catalog importers - Cache clearing after cleanups (_classify_cache_*, nexus:mye_ids:*, nexus:brand_mye_counts:*) Final match rates: - KEEP GREEN: 90.3% - VAZLO: 93.6% - YOKOMITSU: 100.0% - KNADIAN: 57.4% - LUK: 51.0% - RAYBESTOS: 55.9%
This commit is contained in:
157
pos/services/catalog_import_service.py
Normal file
157
pos/services/catalog_import_service.py
Normal file
@@ -0,0 +1,157 @@
|
||||
"""Bulk catalog import service.
|
||||
|
||||
Imports products into inventory with optional vehicle compatibilities
|
||||
and SKU aliases. Can auto-generate vehicle fitment via QWEN AI if
|
||||
compatibilities are not provided.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def import_products(
|
||||
tenant_conn,
|
||||
products: list[dict],
|
||||
branch_id: int,
|
||||
auto_generate_compat: bool = False,
|
||||
employee_id: Optional[int] = None,
|
||||
):
|
||||
"""Import a list of products into inventory.
|
||||
|
||||
Each product dict may contain:
|
||||
- sku (str) *required
|
||||
- name (str) *required
|
||||
- brand (str)
|
||||
- description (str)
|
||||
- cost (float)
|
||||
- price (float)
|
||||
- stock (int)
|
||||
- location (str)
|
||||
- sku_aliases (list[dict]) [{"sku": str, "label": str}]
|
||||
- vehicles (list[dict]) [{"make", "model", "year", "engine", "engine_code"}]
|
||||
|
||||
Returns {"imported": N, "failed": [{"sku": ..., "error": ...}], "compat_generated": M}
|
||||
"""
|
||||
cur = tenant_conn.cursor()
|
||||
imported = 0
|
||||
failed = []
|
||||
compat_generated = 0
|
||||
|
||||
for idx, p in enumerate(products):
|
||||
sku = (p.get("sku") or "").strip()
|
||||
name = (p.get("name") or "").strip()
|
||||
if not sku or not name:
|
||||
failed.append({"index": idx, "sku": sku, "error": "sku and name are required"})
|
||||
continue
|
||||
|
||||
brand = (p.get("brand") or "").strip() or None
|
||||
description = (p.get("description") or "").strip() or None
|
||||
cost = float(p.get("cost") or 0)
|
||||
price = float(p.get("price") or 0)
|
||||
stock = int(p.get("stock") or 0)
|
||||
location = (p.get("location") or "").strip() or None
|
||||
barcode = (p.get("barcode") or "").strip() or None
|
||||
|
||||
try:
|
||||
# Check for duplicate SKU in same branch
|
||||
cur.execute(
|
||||
"SELECT id FROM inventory WHERE part_number = %s AND branch_id = %s AND is_active = true",
|
||||
(sku, branch_id),
|
||||
)
|
||||
if cur.fetchone():
|
||||
# Update existing item instead of creating new
|
||||
cur.execute(
|
||||
"""
|
||||
UPDATE inventory
|
||||
SET name = %s, brand = %s, description = %s, cost = %s, price_1 = %s,
|
||||
location = %s, barcode = COALESCE(%s, barcode), updated_at = NOW()
|
||||
WHERE part_number = %s AND branch_id = %s AND is_active = true
|
||||
RETURNING id
|
||||
""",
|
||||
(name, brand, description, cost, price, location, barcode, sku, branch_id),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
item_id = row[0]
|
||||
else:
|
||||
# Insert new item
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO inventory
|
||||
(branch_id, part_number, barcode, name, description, brand,
|
||||
unit, cost, price_1, price_2, price_3, tax_rate,
|
||||
min_stock, max_stock, location, is_active)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, true)
|
||||
RETURNING id
|
||||
""",
|
||||
(
|
||||
branch_id, sku, barcode, name, description, brand,
|
||||
"PZA", cost, price, price, price, 0.16,
|
||||
0, 0, location,
|
||||
),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
item_id = row[0]
|
||||
|
||||
# Record initial stock if provided
|
||||
if stock > 0:
|
||||
from services.inventory_engine import record_initial
|
||||
record_initial(tenant_conn, item_id, branch_id, stock, cost)
|
||||
|
||||
# Insert SKU aliases
|
||||
aliases = p.get("sku_aliases") or []
|
||||
for alias in aliases:
|
||||
alias_sku = (alias.get("sku") or "").strip()
|
||||
label = (alias.get("label") or "").strip() or None
|
||||
if alias_sku:
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO inventory_sku_aliases (inventory_id, sku, label)
|
||||
VALUES (%s, %s, %s)
|
||||
ON CONFLICT (inventory_id, sku) DO UPDATE SET
|
||||
is_active = true, label = EXCLUDED.label
|
||||
""",
|
||||
(item_id, alias_sku, label),
|
||||
)
|
||||
|
||||
# Insert manual vehicle compatibilities
|
||||
vehicles = p.get("vehicles") or []
|
||||
for v in vehicles:
|
||||
make = (v.get("make") or "").strip()
|
||||
model = (v.get("model") or "").strip()
|
||||
year = v.get("year")
|
||||
engine = (v.get("engine") or "").strip() or None
|
||||
engine_code = (v.get("engine_code") or "").strip() or None
|
||||
if make and model and year:
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO inventory_vehicle_compat
|
||||
(inventory_id, make, model, year, engine, engine_code, source, model_year_engine_id)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, 'manual', NULL)
|
||||
ON CONFLICT DO NOTHING
|
||||
""",
|
||||
(item_id, make, model, year, engine, engine_code),
|
||||
)
|
||||
|
||||
tenant_conn.commit()
|
||||
imported += 1
|
||||
|
||||
# Auto-generate compat via QWEN if requested and no vehicles provided
|
||||
if auto_generate_compat and not vehicles:
|
||||
try:
|
||||
from services.qwen_fitment import get_vehicle_fitment
|
||||
from services.inventory_vehicle_compat import save_qwen_fitment
|
||||
fitment = get_vehicle_fitment(sku, name, brand or "")
|
||||
inserted = save_qwen_fitment(tenant_conn, item_id, fitment)
|
||||
compat_generated += inserted
|
||||
except Exception as qe:
|
||||
logger.warning("QWEN auto-match failed for %s: %s", sku, qe)
|
||||
|
||||
except Exception as e:
|
||||
tenant_conn.rollback()
|
||||
logger.warning("Import failed for sku=%s: %s", sku, e)
|
||||
failed.append({"index": idx, "sku": sku, "error": str(e)})
|
||||
|
||||
cur.close()
|
||||
return {"imported": imported, "failed": failed, "compat_generated": compat_generated}
|
||||
Reference in New Issue
Block a user