Autoparts-DB/scripts/eval_sku_matcher.py

#!/usr/bin/env python3
"""Measure the SKU name-matcher: difflib (baseline) vs QWEN embeddings.

Runs ``sku_matcher.match_inventory`` in dry-run twice over the same inventory
slice — once with the difflib name fallback only, once with the semantic
(embedding) fallback enabled — and compares:

  * coverage: how many items got matched at all
  * precision: among items that already carry a ``catalog_part_id`` (ground
    truth), how often the matcher agrees with the stored id

The second pass reuses the ``CatalogIndex`` built by the first, so catalog-name
embeddings are computed once (in-memory cache).

Usage (as root, from /home/Autopartes):
    python3 scripts/eval_sku_matcher.py --tenant 45 --limit 300
    python3 scripts/eval_sku_matcher.py --tenant 45 --limit 300 --prefer-below 0.9
    python3 scripts/eval_sku_matcher.py --tenant 31 --limit 200 --threshold 0.88

Nothing is written: ``apply=False`` and the UPDATE guard never overwrites an
existing ``catalog_part_id`` anyway.
"""

import argparse
import os
import sys

sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'pos'))

from tenant_db import get_tenant_conn, get_master_conn  # noqa: E402
from services.sku_matcher import load_catalog_index, match_inventory  # noqa: E402


def _precision(res):
    """(attempts_on_linked, correct) using catalog_part_id as ground truth."""
    attempts = correct = 0
    for r in res.get('results', []):
        truth = r.get('existing_catalog_id')
        if not truth:
            continue
        attempts += 1
        if r.get('matched_catalog_id') == truth:
            correct += 1
    return attempts, correct


def _report(label, res):
    attempts, correct = _precision(res)
    prec = (correct / attempts * 100) if attempts else 0.0
    print(f"  {label:<12} matched={res['matched']:<4} "
          f"by_method={res['by_method']}")
    print(f"  {'':<12} precision_on_linked={correct}/{attempts} ({prec:.1f}%)")
    if res.get('semantic'):
        s = res['semantic']
        print(f"  {'':<12} catalog_emb_cache={s['catalog_embeddings_cached']} "
              f"embed_requests={s['embed_requests']}")
    return res['matched'], attempts, correct


def main():
    p = argparse.ArgumentParser(description='Eval SKU name matching (difflib vs embeddings).')
    p.add_argument('--tenant', type=int, required=True)
    p.add_argument('--limit', type=int, default=300, help='items to scan')
    p.add_argument('--offset', type=int, default=0)
    p.add_argument('--threshold', type=float, default=0.86,
                   help='semantic cosine threshold')
    p.add_argument('--prefer-below', type=float, default=None,
                   help='also run semantic when fuzzy score < this value')
    p.add_argument('--name-threshold', type=float, default=0.82)
    p.add_argument('--top-k', type=int, default=64,
                   help='candidate names embedded per item (bounded cost)')
    p.add_argument('--batch', type=int, default=64)
    p.add_argument('--supplier', default=None)
    p.add_argument('--sample', type=int, default=8, help='examples to print')
    p.add_argument('--only-linked', action='store_true',
                   help='restrict to items with a catalog_part_id (ground truth)')
    p.add_argument('--show-examples', action='store_true')
    args = p.parse_args()

    tenant = get_tenant_conn(args.tenant)
    master = get_master_conn()
    try:
        print(f"Building catalog index (supplier={args.supplier or 'ALL'})...")
        idx = load_catalog_index(master, supplier=args.supplier)
        print(f"  catalog items={idx.total} sku_keys={len(idx.by_sku)} "
              f"xref_keys={len(idx.by_xref)}")

        common = dict(
            limit=args.limit, offset=args.offset,
            only_unmatched=not args.only_linked, only_linked=args.only_linked,
            apply=False, create_aliases=False, supplier=args.supplier,
            name_threshold=args.name_threshold, sample_limit=args.sample,
            collect=True,
        )

        print(f"\nInventory slice: tenant={args.tenant} limit={args.limit} "
              f"offset={args.offset} (dry-run)")
        base = match_inventory(tenant, master, use_semantic=False, index=idx, **common)
        print("\n[1] difflib baseline")
        _report('baseline', base)

        sem = match_inventory(
            tenant, master, use_semantic=True, index=idx,
            semantic_threshold=args.threshold,
            semantic_top_k=args.top_k,
            semantic_batch=args.batch,
            semantic_prefer_below=args.prefer_below,
            **common,
        )
        print("\n[2] + semantic (embeddings)")
        b_m, b_a, b_c = _report('semantic', sem)

        # Deltas
        print("\n--- delta ---")
        print(f"  matched: {base['matched']} -> {sem['matched']} "
              f"(+{sem['matched'] - base['matched']})")
        b_att, b_cor = _precision(base)
        s_att, s_cor = _precision(sem)
        bp = (b_cor / b_att * 100) if b_att else 0.0
        sp = (s_cor / s_att * 100) if s_att else 0.0
        print(f"  precision on linked: {bp:.1f}% -> {sp:.1f}%")

        if args.show_examples:
            print("\nExamples (semantic pass):")
            for r in sem.get('results', [])[:args.sample]:
                mark = ''
                if r.get('existing_catalog_id'):
                    mark = ' OK' if r['existing_catalog_id'] == r['matched_catalog_id'] else ' MISMATCH'
                print(f"  [{r['method']:<14} {r['score']:.3f}] "
                      f"{str(r['part_number'])[:16]:<16} {str(r['name'])[:38]:<38} "
                      f"-> cat {r['matched_catalog_id']}{mark}")
    finally:
        tenant.close()
        master.close()
    return 0


if __name__ == '__main__':
    sys.exit(main())