feat(catalog): supplier catalog cleanup, fuzzy matching, and navigation fixes

- Cleaned 137+ fake engine-displacement models from supplier imports
  (v3/v4 scripts: Chevrolet, Ford, Chrysler, Dodge, Jeep, Nissan, etc.)
- Removed 1,251+ corrupted models (INT. prefixes, year-suffix, torque specs,
  empty names, trailing-year variants)
- Migrated supplier tables to master DB (supplier_catalog,
  supplier_catalog_compat, supplier_catalog_interchange)
- Fixed _get_mye_ids_with_parts() to query supplier_catalog_compat from
  master DB so supplier-only vehicles appear for all tenants
- Added fuzzy model matcher with parenthesis stripping, noise suffix removal,
  compact matching, prefix/substring fallback, model aliases, and ±3 year
  proximity
- Matched compat rows: KEEP GREEN +14,152, KNADIAN +3,021, VAZLO +127,500,
  LUK +477, RAYBESTOS +1,743
- Added KNADIAN catalog importer with year-range expansion and future-year
  filtering
- Added VAZLO catalog importer with position parsing and SKU-in-model cleanup
- Added Keep Green, LUK, Yokomitsu, Raybestos catalog importers
- Cache clearing after cleanups (_classify_cache_*, nexus:mye_ids:*,
  nexus:brand_mye_counts:*)

Final match rates:
- KEEP GREEN: 90.3%
- VAZLO: 93.6%
- YOKOMITSU: 100.0%
- KNADIAN: 57.4%
- LUK: 51.0%
- RAYBESTOS: 55.9%
This commit is contained in:
2026-06-09 07:47:42 +00:00
parent 5ea667b80e
commit ea29cc31c0
53 changed files with 7727 additions and 548 deletions

View File

@@ -4,6 +4,7 @@
import io
import json
import os
import csv
from datetime import datetime, timedelta
from flask import Blueprint, request, jsonify, g
from middleware import require_auth, has_permission
@@ -46,6 +47,24 @@ def _apply_tier_discounts(price_1, discounts):
return p2, p3
def _to_decimal(val, default=0):
if val is None or val == '':
return default
try:
return float(str(val).replace(',', ''))
except (ValueError, TypeError):
return default
def _to_int(val, default=0):
if val is None or val == '':
return default
try:
return int(float(str(val).replace(',', '')))
except (ValueError, TypeError):
return default
# ─── AI Classification ───────────────────────────
@inventory_bp.route('/classify/<part_number>', methods=['GET'])
@@ -203,9 +222,10 @@ def get_item(item_id):
conn = get_tenant_conn(g.tenant_id)
cur = conn.cursor()
cur.execute("""
SELECT i.*, b.name as branch_name
SELECT i.*, b.name as branch_name, c.name as category_name
FROM inventory i
LEFT JOIN branches b ON i.branch_id = b.id
LEFT JOIN categories c ON i.category_id = c.id
WHERE i.id = %s
""", (item_id,))
row = cur.fetchone()
@@ -309,6 +329,23 @@ def create_item():
if initial_stock > 0:
record_initial(conn, item_id, branch_id, initial_stock, data.get('cost'))
# Insert SKU aliases if provided
sku_aliases = data.get('sku_aliases', [])
if sku_aliases:
for alias in sku_aliases:
sku = (alias.get('sku') or '').strip()
label = (alias.get('label') or '').strip()
if sku:
cur.execute(
"""
INSERT INTO inventory_sku_aliases (inventory_id, sku, label)
VALUES (%s, %s, %s)
ON CONFLICT (inventory_id, sku) DO UPDATE SET
is_active = true, label = EXCLUDED.label
""",
(item_id, sku, label or None),
)
log_action(conn, 'INVENTORY_CREATE', 'inventory', item_id,
new_value={'part_number': data['part_number'], 'name': data['name'], 'initial_stock': initial_stock})
@@ -333,13 +370,19 @@ def create_item():
if not compat_background:
# Fallback: synchronous processing
if compat_source in ('tecdoc', 'both'):
master = None
try:
master = get_master_conn()
auto_match_vehicle_compatibility(master, conn, item_id, data['part_number'],
brand=data.get('brand'), name=data.get('name'))
master.close()
except Exception as am_err:
print(f"[auto_match] Error for item {item_id}: {am_err}")
finally:
if master:
try:
master.close()
except Exception:
pass
if compat_source in ('qwen', 'both'):
try:
@@ -371,6 +414,291 @@ def create_item():
return jsonify({'error': str(e)}), 500
@inventory_bp.route('/items/bulk-import', methods=['POST'])
@require_auth('inventory.edit')
def bulk_import_items():
"""
Bulk import inventory items with optional vehicle compatibility.
Expects multipart/form-data with a 'file' (CSV/Excel) or JSON body.
Headers:
X-Import-Mode: 'strict' (default) aborts on first error; 'lenient' skips bad rows.
X-Import-Strategy: 'qwen' (default) auto-generates missing compat via QWEN;
'skip' ignores missing compat; 'reject' requires all compat.
Expected CSV columns (case-insensitive):
sku/part_number, name, brand, price, stock, cost,
location, description, category, make, model, year, engine, engine_code
Optional compat columns: make, model, year, engine, engine_code
"""
from services.qwen_fitment import get_vehicle_fitment
from services.inventory_vehicle_compat import save_qwen_fitment
import services.inventory_vehicle_compat as ivc_service
mode = request.headers.get('X-Import-Mode', 'strict').lower()
strategy = request.headers.get('X-Import-Strategy', 'qwen').lower()
errors = []
warnings = []
created_ids = []
skipped = 0
created = 0
# ---------- 1. Parse input ----------
rows = []
if request.content_type and 'multipart/form-data' in request.content_type:
file = request.files.get('file')
if not file:
return jsonify({'error': 'No file uploaded'}), 400
try:
ext = os.path.splitext(file.filename)[1].lower()
if ext == '.csv':
stream = io.TextIOWrapper(file.stream, encoding='utf-8-sig')
reader = csv.DictReader(stream)
rows = list(reader)
elif ext in ('.xls', '.xlsx', '.xlsm'):
try:
import openpyxl
except ImportError:
return jsonify({'error': 'Excel support requires openpyxl. Please convert to CSV or install openpyxl.'}), 400
wb = openpyxl.load_workbook(file.stream, data_only=True)
ws = wb.active
headers = [str(c).strip().lower().replace(' ', '_') if c else '' for c in next(ws.iter_rows(values_only=True))]
for raw in ws.iter_rows(min_row=2, values_only=True):
rows.append({headers[i]: (str(v) if v is not None else '') for i, v in enumerate(raw) if i < len(headers)})
else:
return jsonify({'error': 'Unsupported file type. Use CSV or Excel.'}), 400
except Exception as e:
return jsonify({'error': f'Failed to parse file: {e}'}), 400
else:
body = request.get_json() or {}
rows = body.get('items')
if not rows or not isinstance(rows, list):
return jsonify({'error': 'Expected JSON body with an "items" array'}), 400
if not rows:
return jsonify({'error': 'No data rows found'}), 400
# Normalise column names on first row
if rows:
first = rows[0]
normalised_keys = {}
for k in list(first.keys()):
nk = str(k).strip().lower().replace(' ', '_')
normalised_keys[k] = nk
for r in rows:
for old_k, new_k in normalised_keys.items():
if old_k in r:
r[new_k] = r.pop(old_k)
# Map common synonyms
col_map = {
'sku': 'part_number', 'numero_de_parte': 'part_number', 'parte': 'part_number',
'nombre': 'name', 'producto': 'name', 'descripcion': 'name',
'marca': 'brand', 'precio': 'price', 'costo': 'cost',
'cantidad': 'stock', 'existencia': 'stock', 'inventario': 'stock',
'ubicacion': 'location', 'categoria': 'category',
'fabricante': 'make', 'vehiculo': 'make', 'auto': 'make',
'modelo': 'model', 'anio': 'year', 'ano': 'year',
'motor': 'engine', 'codigo_motor': 'engine_code',
}
for r in rows:
for old_k, new_k in col_map.items():
if old_k in r and new_k not in r:
r[new_k] = r.pop(old_k)
required = ['part_number', 'name']
first_keys = set(rows[0].keys()) if rows else set()
missing_required = [c for c in required if c not in first_keys]
if missing_required:
return jsonify({'error': f'Missing required columns: {missing_required}'}), 400
conn = get_tenant_conn(g.tenant_id)
try:
cur = conn.cursor()
from services.barcode_generator import generate_barcode
from tenant_db import get_master_conn
from services.inventory_engine import record_initial
# Pre-fetch tenant db_name for barcode generation
mconn = get_master_conn()
mcur = mconn.cursor()
mcur.execute("SELECT db_name FROM tenants WHERE id = %s", (g.tenant_id,))
db_name_row = mcur.fetchone()
db_name = db_name_row[0] if db_name_row else None
mcur.close(); mconn.close()
for row_num, row in enumerate(rows, start=1):
part_number = str(row.get('part_number', '')).strip()
name = str(row.get('name', '')).strip()
if not part_number or not name:
msg = f'Row {row_num}: part_number and name are required'
if mode == 'strict':
conn.rollback(); cur.close(); conn.close()
return jsonify({'error': msg}), 400
warnings.append(msg)
skipped += 1
continue
branch_id = _to_int(row.get('branch_id'), g.branch_id)
if not branch_id:
msg = f'Row {row_num}: branch_id required (not set in row or session)'
if mode == 'strict':
conn.rollback(); cur.close(); conn.close()
return jsonify({'error': msg}), 400
warnings.append(msg)
skipped += 1
continue
brand = str(row.get('brand', '')).strip()
price_1 = _to_decimal(row.get('price'), 0)
stock = _to_int(row.get('stock'), 0)
cost = _to_decimal(row.get('cost'), 0)
location = str(row.get('location', '')).strip()
description = str(row.get('description', '')).strip()
category = str(row.get('category', '')).strip()
# Check if item already exists for this branch
cur.execute("SELECT id FROM inventory WHERE branch_id = %s AND part_number = %s", (branch_id, part_number))
existing = cur.fetchone()
if existing:
item_id = existing[0]
# Update existing item — add stock if provided
cur.execute(
"""
UPDATE inventory SET
name = %s,
brand = COALESCE(NULLIF(%s,''), brand),
cost = CASE WHEN %s > 0 THEN %s ELSE cost END,
price_1 = CASE WHEN %s > 0 THEN %s ELSE price_1 END,
stock = stock + %s,
location = COALESCE(NULLIF(%s,''), location),
description = COALESCE(NULLIF(%s,''), description),
category = COALESCE(NULLIF(%s,''), category)
WHERE id = %s
""",
(name, brand, cost, cost, price_1, price_1, stock, location, description, category, item_id)
)
was_inserted = False
# Record stock adjustment for existing item if stock > 0
if stock > 0:
record_initial(conn, item_id, branch_id, stock, cost if cost > 0 else None)
else:
# Generate barcode for new item
barcode = generate_barcode(conn, db_name)
cur.execute(
"""
INSERT INTO inventory
(branch_id, part_number, barcode, name, brand, cost, price_1, location, description, category, unit)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
RETURNING id
""",
(branch_id, part_number, barcode, name, brand, cost, price_1, location, description, category, 'PZA')
)
item_id = cur.fetchone()[0]
was_inserted = True
# Record initial stock if provided and new item
if was_inserted and stock > 0:
record_initial(conn, item_id, branch_id, stock, cost if cost > 0 else None)
conn.commit()
created_ids.append(item_id)
created += 1
# ---------- 2. Vehicle compatibility ----------
make = str(row.get('make', '')).strip()
model = str(row.get('model', '')).strip()
year_str = str(row.get('year', '')).strip()
engine = str(row.get('engine', '')).strip()
engine_code = str(row.get('engine_code', '')).strip()
has_compat = any([make, model, year_str, engine, engine_code])
if has_compat:
# Validate / resolve against vehicle tables
year = _to_int(year_str, None)
mye_id = None
if make and model and year:
# Try exact match against model_year_engine
cur.execute(
"""
SELECT mye.id FROM model_year_engine mye
JOIN models m ON m.id = mye.model_id
JOIN brands b ON b.id = m.brand_id
JOIN years y ON y.id = mye.year_id
WHERE LOWER(b.name) = LOWER(%s)
AND LOWER(m.name) = LOWER(%s)
AND y.year = %s
LIMIT 1
""",
(make, model, year)
)
r = cur.fetchone()
if r:
mye_id = r[0]
else:
warnings.append(
f'Row {row_num}: vehicle "{make} {model} {year}" not found in catalog; '
'saving as text-only compatibility.'
)
if mye_id:
cur.execute(
"""
INSERT INTO inventory_vehicle_compat
(inventory_id, model_year_engine_id, make, model, year, engine, engine_code)
VALUES (%s, %s, %s, %s, %s, %s, %s)
ON CONFLICT DO NOTHING
""",
(item_id, mye_id, make, model, year_str, engine, engine_code)
)
else:
cur.execute(
"""
INSERT INTO inventory_vehicle_compat
(inventory_id, make, model, year, engine, engine_code)
VALUES (%s, %s, %s, %s, %s, %s)
ON CONFLICT DO NOTHING
""",
(item_id, make, model, year_str, engine, engine_code)
)
conn.commit()
else:
# No compatibility provided
if strategy == 'reject':
msg = f'Row {row_num}: missing vehicle compatibility (strategy=reject)'
if mode == 'strict':
conn.rollback(); cur.close(); conn.close()
return jsonify({'error': msg}), 400
warnings.append(msg)
elif strategy == 'qwen':
try:
fitment = get_vehicle_fitment(part_number, name, brand)
save_qwen_fitment(conn, item_id, fitment)
conn.commit()
except Exception as qe:
warnings.append(f'Row {row_num}: QWEN fitment failed: {qe}')
# strategy == 'skip' → do nothing
cur.close()
conn.close()
return jsonify({
'created': created,
'skipped': skipped,
'item_ids': created_ids,
'warnings': warnings,
'errors': errors,
}), 200
except Exception as e:
conn.rollback()
try: cur.close()
except Exception: pass
conn.close()
return jsonify({'error': str(e)}), 500
@inventory_bp.route('/items/<int:item_id>', methods=['PUT'])
@require_auth('inventory.edit')
def update_item(item_id):
@@ -1371,22 +1699,6 @@ def report_branch_comparison():
# ─── Categories and Brands ─────────────────────
@inventory_bp.route('/categories', methods=['GET'])
@require_auth('inventory.view')
def list_categories():
"""Get distinct categories from inventory."""
conn = get_tenant_conn(g.tenant_id)
cur = conn.cursor()
cur.execute("""
SELECT DISTINCT category_id FROM inventory
WHERE is_active = true AND category_id IS NOT NULL
ORDER BY category_id
""")
categories = [r[0] for r in cur.fetchall()]
cur.close(); conn.close()
return jsonify({'data': categories})
@inventory_bp.route('/brands', methods=['GET'])
@require_auth('inventory.view')
def list_brands():
@@ -1718,6 +2030,89 @@ def auto_match_item_vehicles(item_id):
return jsonify({'error': 'No compatibility source configured'}), 400
# ─── SKU Aliases (multiple part numbers per item) ───────────────────────
@inventory_bp.route('/items/<int:item_id>/skus', methods=['GET'])
@require_auth('inventory.view')
def get_item_sku_aliases(item_id):
"""Return active SKU aliases for an inventory item."""
conn = get_tenant_conn(g.tenant_id)
try:
cur = conn.cursor()
cur.execute(
"""
SELECT id, sku, label, created_at
FROM inventory_sku_aliases
WHERE inventory_id = %s AND is_active = true
ORDER BY created_at
""",
(item_id,),
)
rows = cur.fetchall()
cur.close()
return jsonify({
'aliases': [
{'id': r[0], 'sku': r[1], 'label': r[2], 'created_at': r[3]}
for r in rows
]
})
finally:
conn.close()
@inventory_bp.route('/items/<int:item_id>/skus', methods=['POST'])
@require_auth('inventory.edit')
def add_item_sku_alias(item_id):
"""Add an SKU alias to an inventory item."""
data = request.get_json() or {}
sku = (data.get('sku') or '').strip()
label = (data.get('label') or '').strip()
if not sku:
return jsonify({'error': 'sku is required'}), 400
conn = get_tenant_conn(g.tenant_id)
try:
cur = conn.cursor()
cur.execute(
"""
INSERT INTO inventory_sku_aliases (inventory_id, sku, label)
VALUES (%s, %s, %s)
ON CONFLICT (inventory_id, sku) DO UPDATE SET
is_active = true, label = EXCLUDED.label
RETURNING id
""",
(item_id, sku, label or None),
)
row = cur.fetchone()
conn.commit()
cur.close()
return jsonify({'id': row[0], 'message': 'SKU alias added'}), 201
except Exception as e:
conn.rollback()
return jsonify({'error': str(e)}), 500
finally:
conn.close()
@inventory_bp.route('/items/<int:item_id>/skus/<int:alias_id>', methods=['DELETE'])
@require_auth('inventory.edit')
def delete_item_sku_alias(item_id, alias_id):
"""Soft-delete an SKU alias."""
conn = get_tenant_conn(g.tenant_id)
try:
cur = conn.cursor()
cur.execute(
"""
UPDATE inventory_sku_aliases SET is_active = false WHERE id = %s AND inventory_id = %s
""",
(alias_id, item_id),
)
conn.commit()
cur.close()
return jsonify({'message': 'SKU alias removed'})
finally:
conn.close()
@inventory_bp.route('/mye/search', methods=['GET'])
@require_auth()
def search_mye_endpoint():
@@ -1735,6 +2130,178 @@ def search_mye_endpoint():
master.close()
# ─── Manual Vehicle Compatibility (text-based) ────────────────────────────
@inventory_bp.route('/items/<int:item_id>/vehicles/manual', methods=['POST'])
@require_auth('inventory.edit')
def add_manual_vehicle_compat(item_id):
"""Add a manual vehicle compatibility using free-text fields."""
data = request.get_json() or {}
make = (data.get('make') or '').strip()
model = (data.get('model') or '').strip()
year = data.get('year')
engine = (data.get('engine') or '').strip()
engine_code = (data.get('engine_code') or '').strip()
if not make or not model or not year:
return jsonify({'error': 'make, model and year are required'}), 400
conn = get_tenant_conn(g.tenant_id)
try:
cur = conn.cursor()
cur.execute(
"""
INSERT INTO inventory_vehicle_compat
(inventory_id, make, model, year, engine, engine_code, source, model_year_engine_id)
VALUES (%s, %s, %s, %s, %s, %s, 'manual', NULL)
ON CONFLICT DO NOTHING
RETURNING id
""",
(item_id, make, model, year, engine or None, engine_code or None),
)
row = cur.fetchone()
conn.commit()
cur.close()
if not row:
return jsonify({'error': 'Compatibility already exists or item not found'}), 409
return jsonify({'id': row[0], 'message': 'Compatibility added'}), 201
finally:
conn.close()
@inventory_bp.route('/vehicles/makes', methods=['GET'])
@require_auth()
def get_vehicle_makes():
"""Return distinct vehicle makes from master DB."""
master = get_master_conn()
try:
cur = master.cursor()
cur.execute("SELECT id_brand, name_brand FROM brands ORDER BY name_brand")
rows = cur.fetchall()
cur.close()
return jsonify({'makes': [{'id': r[0], 'name': r[1]} for r in rows]})
finally:
master.close()
@inventory_bp.route('/vehicles/models', methods=['GET'])
@require_auth()
def get_vehicle_models():
"""Return models for a given brand."""
brand_id = request.args.get('brand_id', type=int)
if not brand_id:
return jsonify({'error': 'brand_id required'}), 400
master = get_master_conn()
try:
cur = master.cursor()
cur.execute(
"SELECT id_model, name_model FROM models WHERE brand_id = %s ORDER BY name_model",
(brand_id,),
)
rows = cur.fetchall()
cur.close()
return jsonify({'models': [{'id': r[0], 'name': r[1]} for r in rows]})
finally:
master.close()
@inventory_bp.route('/vehicles/years', methods=['GET'])
@require_auth()
def get_vehicle_years():
"""Return distinct years available for a model."""
model_id = request.args.get('model_id', type=int)
if not model_id:
return jsonify({'error': 'model_id required'}), 400
master = get_master_conn()
try:
cur = master.cursor()
cur.execute(
"""
SELECT DISTINCT y.id_year, y.year_car
FROM model_year_engine mye
JOIN years y ON y.id_year = mye.year_id
WHERE mye.model_id = %s
ORDER BY y.year_car DESC
""",
(model_id,),
)
rows = cur.fetchall()
cur.close()
return jsonify({'years': [{'id': r[0], 'year': r[1]} for r in rows]})
finally:
master.close()
@inventory_bp.route('/vehicles/engines', methods=['GET'])
@require_auth()
def get_vehicle_engines():
"""Return engines available for a model+year."""
model_id = request.args.get('model_id', type=int)
year_id = request.args.get('year_id', type=int)
if not model_id or not year_id:
return jsonify({'error': 'model_id and year_id required'}), 400
master = get_master_conn()
try:
cur = master.cursor()
cur.execute(
"""
SELECT DISTINCT e.id_engine, e.name_engine, e.engine_code
FROM model_year_engine mye
JOIN engines e ON e.id_engine = mye.engine_id
WHERE mye.model_id = %s AND mye.year_id = %s
ORDER BY e.name_engine
""",
(model_id, year_id),
)
rows = cur.fetchall()
cur.close()
return jsonify({'engines': [{'id': r[0], 'name': r[1], 'code': r[2]} for r in rows]})
finally:
master.close()
# ─── Categories ──────────────────────────────────
@inventory_bp.route('/categories', methods=['GET'])
@require_auth()
def list_inventory_categories():
"""Return active categories (root only). Optional ?parent_id= for subcategories."""
parent_id = request.args.get('parent_id')
conn = get_tenant_conn(g.tenant_id)
try:
cur = conn.cursor()
if parent_id:
cur.execute(
"SELECT id, name FROM categories WHERE parent_id = %s AND is_active = true ORDER BY name",
(parent_id,)
)
else:
cur.execute(
"SELECT id, name FROM categories WHERE parent_id IS NULL AND is_active = true ORDER BY name"
)
rows = cur.fetchall()
return jsonify({'categories': [{'id': r[0], 'name': r[1]} for r in rows]})
finally:
conn.close()
@inventory_bp.route('/categories/<int:category_id>/subcategories', methods=['GET'])
@require_auth()
def list_inventory_subcategories(category_id):
"""Return subcategories for a given category."""
conn = get_tenant_conn(g.tenant_id)
try:
cur = conn.cursor()
cur.execute(
"SELECT id, name FROM categories WHERE parent_id = %s AND is_active = true ORDER BY name",
(category_id,)
)
rows = cur.fetchall()
return jsonify({'subcategories': [{'id': r[0], 'name': r[1]} for r in rows]})
finally:
conn.close()
# ─── Global Tier Discounts ───────────────────────
@inventory_bp.route('/tier-discounts', methods=['GET'])