Autoparts-DB/scripts/scrape_rached_catalogs.py

#!/usr/bin/env python3
"""Scrape Rached legacy catalog endpoints discovered via UI."""
import json
import os
import time
import requests
from pathlib import Path

BASE_API = 'http://appapi.flechasyventiladores-rached.com'
OUT_DIR = Path(os.environ.get('OUT_DIR', Path(__file__).resolve().parent.parent / 'data' / 'rached_import'))
USER = 'IVAN@flechasyventiladores-rached.com'
PASS = 'Nexus01'

HEADERS = {
    'Accept': 'application/json, text/plain, */*',
    'Content-Type': 'application/json',
    'Origin': 'http://app.flechasyventiladores-rached.com',
    'Referer': 'http://app.flechasyventiladores-rached.com/',
    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
}


def login():
    url = f'{BASE_API}/api/seguridad/login'
    payload = {
        'grant_type': 'password',
        'client_secret': 'JiRE9iL3pqRnqcFp6wDeYH0tYu97QSpkrwVKAvEP',
        'client_id': 2,
        'username': USER,
        'password': PASS,
    }
    resp = requests.post(url, headers=HEADERS, json=payload)
    resp.raise_for_status()
    data = resp.json()
    if data.get('mensaje'):
        raise RuntimeError(f'Login failed: {data["mensaje"]}')
    return data['datos']['token']


def api_headers(token):
    return {**HEADERS, 'Authorization': f'Bearer {token}'}


def fetch_get(token, path, params=None):
    url = f'{BASE_API}{path}'
    resp = requests.get(url, headers=api_headers(token), params=params, timeout=30)
    resp.raise_for_status()
    return resp.json()


def fetch_post(token, path, payload=None, params=None):
    url = f'{BASE_API}{path}'
    resp = requests.post(url, headers=api_headers(token), json=payload or {}, params=params, timeout=30)
    resp.raise_for_status()
    return resp.json()


def save(name, data):
    path = OUT_DIR / f'{name}.json'
    path.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding='utf-8')
    return path


def fetch_list_all(token, path, method='GET', payload=None):
    """Fetch a list endpoint that may return either a list or Laravel pagination."""
    all_items = []
    page = 1
    while True:
        try:
            if method == 'GET':
                data = fetch_get(token, f'{path}?page={page}')
            else:
                data = fetch_post(token, f'{path}?page={page}', payload)
        except Exception as e:
            print(f'  {path} page {page} error: {e}')
            break
        datos = data.get('datos', {})
        if isinstance(datos, list):
            # Non-paginated full list
            all_items.extend(datos)
            print(f'  {path}: +{len(datos)} (total {len(all_items)})')
            break
        items = datos.get('data', [])
        all_items.extend(items)
        print(f'  {path} page {page}: +{len(items)} (total {len(all_items)}/{datos.get("total", "?")})')
        if datos.get('current_page', 1) >= datos.get('last_page', 1):
            break
        page += 1
        time.sleep(0.6)
    return all_items


def main():
    OUT_DIR.mkdir(parents=True, exist_ok=True)
    log_path = OUT_DIR / 'scrape_catalogs.log'

    def log(msg):
        print(msg)
        with open(log_path, 'a', encoding='utf-8') as f:
            f.write(msg + '\n')

    log('Logging in...')
    token = login()
    log('Logged in.')

    # Simple catalog endpoints (GET, single page or paginated)
    simple_gets = {
        'catalog_sucursales': '/api/catalogos/sucursales',
        'catalog_viasentrega': '/api/catalogos/viasentrega/listado',
        'catalog_order_statuses': '/api/catalogos/ordenesservicioestatus/listado',
        'catalog_item_statuses': '/api/catalogos/ordenesserviciodetalleestatus/listado',
        'catalog_directas': '/api/catalogos/ordenesDirectas',
        'catalog_telefonostipos': '/api/catalogos/telefonostipos/listado',
        'catalog_direccionestipos': '/api/catalogos/direccionestipos/listado',
    }
    for name, path in simple_gets.items():
        try:
            data = fetch_get(token, path)
            save(name, data)
            count = len(data.get('datos', []) if isinstance(data.get('datos'), list) else [])
            log(f'{name}: {count} items')
        except Exception as e:
            log(f'{name} error: {e}')
        time.sleep(0.6)

    # List endpoints (GET or POST)
    list_gets = {
        'catalog_articulos': '/api/catalogos/articulos/listado',
        'catalog_mecanicos': '/api/catalogos/mecanicos/listado',
        'catalog_motociclistas': '/api/catalogos/motociclistas/listado',
    }
    for name, path in list_gets.items():
        items = fetch_list_all(token, path, method='GET')
        save(name, items)
        log(f'{name}: {len(items)} items')

    # Client list
    try:
        client_items = fetch_list_all(token, '/api/catalogos/clientes/listado', method='POST',
                                      payload={'taller': '', 'nombreCompleto': '', 'idClienteEstatus': '-1'})
        save('catalog_clientes', client_items)
        log(f'catalog_clientes: {len(client_items)} items')

        # Fetch full client details for clients referenced in orders
        orders_path = OUT_DIR / 'orders_list.json'
        if orders_path.exists():
            order_ids = {o['idCliente'] for o in json.loads(orders_path.read_text(encoding='utf-8')) if o.get('idCliente')}
            log(f'Fetching details for {len(order_ids)} unique order clients...')
            client_details = {}
            for idx, cid in enumerate(sorted(order_ids), 1):
                detail_path = OUT_DIR / 'client_details' / f'client_{cid}.json'
                if detail_path.exists():
                    client_details[cid] = json.loads(detail_path.read_text(encoding='utf-8'))
                    continue
                try:
                    data = fetch_get(token, f'/api/catalogos/clientes/{cid}')
                    client_details[cid] = data.get('datos')
                    detail_path.parent.mkdir(parents=True, exist_ok=True)
                    detail_path.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding='utf-8')
                except Exception as e:
                    log(f'  client {cid} detail error: {e}')
                if idx % 30 == 0:
                    time.sleep(1)
                else:
                    time.sleep(0.2)
            save('catalog_client_details', client_details)
            log(f'catalog_client_details: {len(client_details)} items')
    except Exception as e:
        log(f'catalog_clientes error: {e}')

    log('Done.')


if __name__ == '__main__':
    main()