79 lines
2.4 KiB
Python
Executable File
79 lines
2.4 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Warm Redis cache for vehicle info (part_vehicle_preview alternative).
|
|
|
|
Runs in batches over all parts in the catalog, populating
|
|
nexus:vehicle:{part_id} keys in Redis. This eliminates the
|
|
DISTINCT ON + 4 JOINs query on vehicle_parts (2B rows) for
|
|
cached parts.
|
|
|
|
Usage:
|
|
export MASTER_DB_URL="postgresql://..."
|
|
export REDIS_URL="redis://localhost:6379/0"
|
|
python3 warm_vehicle_cache.py
|
|
"""
|
|
|
|
import os, sys, json, time
|
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'pos'))
|
|
|
|
import psycopg2
|
|
import redis
|
|
|
|
MASTER_DB_URL = os.environ.get('MASTER_DB_URL', 'postgresql://postgres@/nexus_autoparts')
|
|
REDIS_URL = os.environ.get('REDIS_URL', 'redis://localhost:6379/0')
|
|
BATCH_SIZE = 5000
|
|
TTL_SECONDS = 3600
|
|
|
|
|
|
def main():
|
|
print("Connecting to master DB and Redis...")
|
|
conn = psycopg2.connect(MASTER_DB_URL)
|
|
cur = conn.cursor()
|
|
r = redis.from_url(REDIS_URL, decode_responses=True)
|
|
r.ping()
|
|
|
|
# Get all part_ids
|
|
cur.execute("SELECT id_part FROM parts WHERE oem_part_number IS NOT NULL ORDER BY id_part")
|
|
all_ids = [r[0] for r in cur.fetchall()]
|
|
total = len(all_ids)
|
|
print(f"Total parts to warm: {total}")
|
|
|
|
processed = 0
|
|
cached = 0
|
|
start = time.time()
|
|
|
|
for i in range(0, total, BATCH_SIZE):
|
|
batch = all_ids[i:i + BATCH_SIZE]
|
|
cur.execute("""
|
|
SELECT DISTINCT ON (vp.part_id)
|
|
vp.part_id, b.name_brand, m.name_model, y.year_car
|
|
FROM vehicle_parts vp
|
|
JOIN model_year_engine mye ON mye.id_mye = vp.model_year_engine_id
|
|
JOIN models m ON m.id_model = mye.model_id
|
|
JOIN brands b ON b.id_brand = m.brand_id
|
|
JOIN years y ON y.id_year = mye.year_id
|
|
WHERE vp.part_id = ANY(%s)
|
|
ORDER BY vp.part_id, y.year_car DESC
|
|
""", (batch,))
|
|
|
|
pipe = r.pipeline()
|
|
batch_cached = 0
|
|
for row in cur.fetchall():
|
|
info = f"{row[1]} {row[2]} {row[3]}"
|
|
pipe.setex(f'nexus:vehicle:{row[0]}', TTL_SECONDS, info)
|
|
batch_cached += 1
|
|
pipe.execute()
|
|
|
|
processed += len(batch)
|
|
cached += batch_cached
|
|
elapsed = time.time() - start
|
|
rate = processed / elapsed if elapsed > 0 else 0
|
|
print(f" [{processed}/{total}] cached={batch_cached} ({rate:.0f}/s)")
|
|
|
|
cur.close()
|
|
conn.close()
|
|
print(f"\nDone. Cached {cached} vehicle entries in {elapsed:.0f}s")
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|