feat: add TecDoc import pipeline scripts
- import_tecdoc.py: 2-phase TecDoc download + import (brands, models, vehicles) - import_live.py: real-time streaming importer for part details - run_all_brands.sh: automated sequential brand processing pipeline Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
43
scripts/run_all_brands.sh
Executable file
43
scripts/run_all_brands.sh
Executable file
@@ -0,0 +1,43 @@
|
||||
#!/bin/bash
|
||||
# Sequential download + import for all target brands
|
||||
|
||||
LOG="/tmp/tecdoc_all_brands.log"
|
||||
SCRIPTS="/home/Autopartes/scripts"
|
||||
|
||||
BRANDS=("RENAULT" "NISSAN")
|
||||
|
||||
for BRAND in "${BRANDS[@]}"; do
|
||||
echo "" | tee -a "$LOG"
|
||||
echo "$(date): ========== Starting $BRAND ==========" | tee -a "$LOG"
|
||||
|
||||
# Start download
|
||||
BRAND_LOG="/tmp/tecdoc_parts_$(echo $BRAND | tr ' ' '_').log"
|
||||
python3 "$SCRIPTS/import_tecdoc_parts.py" download --brand "$BRAND" >> "$BRAND_LOG" 2>&1 &
|
||||
DL_PID=$!
|
||||
echo "$(date): Download started (PID $DL_PID)" | tee -a "$LOG"
|
||||
|
||||
# Start live importer
|
||||
python3 "$SCRIPTS/import_live.py" >> /tmp/tecdoc_import_live.log 2>&1 &
|
||||
LI_PID=$!
|
||||
echo "$(date): Live importer started (PID $LI_PID)" | tee -a "$LOG"
|
||||
|
||||
# Wait for download to finish
|
||||
wait $DL_PID
|
||||
echo "$(date): Download for $BRAND complete!" | tee -a "$LOG"
|
||||
|
||||
# Give live importer time to catch up, then stop it
|
||||
sleep 60
|
||||
kill $LI_PID 2>/dev/null
|
||||
wait $LI_PID 2>/dev/null
|
||||
echo "$(date): Live importer stopped" | tee -a "$LOG"
|
||||
|
||||
# Run vehicle linker
|
||||
echo "$(date): Starting vehicle linker for $BRAND..." | tee -a "$LOG"
|
||||
python3 "$SCRIPTS/link_vehicle_parts.py" >> /tmp/tecdoc_linker.log 2>&1
|
||||
echo "$(date): Linker for $BRAND complete!" | tee -a "$LOG"
|
||||
|
||||
echo "$(date): ========== $BRAND DONE ==========" | tee -a "$LOG"
|
||||
done
|
||||
|
||||
echo "" | tee -a "$LOG"
|
||||
echo "$(date): ALL BRANDS COMPLETE!" | tee -a "$LOG"
|
||||
Reference in New Issue
Block a user