Autoparts-DB/scripts/scrape_rached_workshop.py

#!/usr/bin/env python3
"""Scrape service orders from Rached legacy web app using their internal API."""
import json
import os
import sys
import time
import requests
from pathlib import Path

BASE_APP = 'http://app.flechasyventiladores-rached.com'
BASE_API = 'http://appapi.flechasyventiladores-rached.com'
OUT_DIR = Path(os.environ.get('OUT_DIR', Path(__file__).resolve().parent.parent / 'data' / 'rached_import'))
USER = os.environ.get('RACHED_USER', 'IVAN')
PASS = os.environ.get('RACHED_PASS', 'Nexus01')
LIMIT = int(os.environ.get('LIMIT', '0'))

HEADERS = {
    'Accept': 'application/json, text/plain, */*',
    'X-Requested-With': 'XMLHttpRequest',
    'Origin': BASE_APP,
    'Referer': f'{BASE_APP}/',
    'Content-Type': 'application/json',
    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
}


def login():
    url = f'{BASE_API}/api/seguridad/login'
    payload = {
        'grant_type': 'password',
        'client_secret': 'JiRE9iL3pqRnqcFp6wDeYH0tYu97QSpkrwVKAvEP',
        'client_id': 2,
        'username': 'IVAN@flechasyventiladores-rached.com',
        'password': PASS,
    }
    resp = requests.post(url, headers=HEADERS, json=payload)
    resp.raise_for_status()
    data = resp.json()
    if data.get('mensaje'):
        raise RuntimeError(f'Login failed: {data["mensaje"]}')
    return data['datos']['token']


def api_headers(token):
    return {**HEADERS, 'Authorization': f'Bearer {token}'}


def fetch_list(token, page):
    url = f'{BASE_API}/api/servicios/ordenes/listado?page={page}'
    resp = requests.post(url, headers=api_headers(token))
    resp.raise_for_status()
    return resp.json()


def fetch_detail(token, order_id):
    url = f'{BASE_API}/api/servicios/ordenes/{order_id}'
    resp = requests.get(url, headers=api_headers(token))
    resp.raise_for_status()
    return resp.json()


def main():
    OUT_DIR.mkdir(parents=True, exist_ok=True)
    log_path = OUT_DIR / 'scrape_py.log'

    def log(msg):
        print(msg)
        with open(log_path, 'a', encoding='utf-8') as f:
            f.write(msg + '\n')

    log('Logging in...')
    token = login()
    log('Logged in.')

    # ── List ──
    list_path = OUT_DIR / 'orders_list.json'
    if list_path.exists():
        all_orders = json.loads(list_path.read_text(encoding='utf-8'))
        log(f'Loaded {len(all_orders)} orders from existing list.')
    else:
        all_orders = []
        page = 1
        while True:
            for attempt in range(3):
                try:
                    data = fetch_list(token, page)
                    break
                except Exception as e:
                    log(f'List page {page} attempt {attempt+1} error: {e}')
                    time.sleep(2)
            else:
                log(f'Failed to fetch list page {page} after 3 attempts.')
                break

            datos = data.get('datos', {})
            orders = datos.get('data', [])
            all_orders.extend(orders)
            (OUT_DIR / f'orders_page_{page}.json').write_text(
                json.dumps(data, indent=2, ensure_ascii=False),
                encoding='utf-8'
            )
            log(f'Page {page}: {len(orders)} orders (total {len(all_orders)}/{datos.get("total", "?")})')
            if datos.get('current_page', 1) >= datos.get('last_page', 1):
                break
            page += 1
            time.sleep(1)

        list_path.write_text(json.dumps(all_orders, indent=2, ensure_ascii=False), encoding='utf-8')
        log(f'Collected {len(all_orders)} orders summary.')

    if not all_orders:
        log('No orders to fetch details for.')
        return

    # ── Details ──
    detail_dir = OUT_DIR / 'details'
    detail_dir.mkdir(exist_ok=True)

    target_orders = all_orders[:LIMIT] if LIMIT > 0 else all_orders
    total = len(target_orders)
    for i, o in enumerate(target_orders, start=1):
        oid = o['id']
        detail_path = detail_dir / f'order_{oid}.json'
        if detail_path.exists():
            log(f'[{i}/{total}] {oid} already saved')
            continue
        for attempt in range(3):
            try:
                data = fetch_detail(token, oid)
                detail_path.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding='utf-8')
                log(f'[{i}/{total}] {oid} saved')
                break
            except Exception as e:
                log(f'[{i}/{total}] {oid} attempt {attempt+1} error: {e}')
                time.sleep(2)
        else:
            log(f'[{i}/{total}] {oid} FAILED after 3 attempts')
        time.sleep(1)

    log('Done.')


if __name__ == '__main__':
    main()