Autoparts-DB/scripts/scrape_rached_workshop.js

#!/usr/bin/env node
/**
 * Scrape service orders from Rached's legacy web app.
 * Run with:
 *   RACHED_USER='IVAN' RACHED_PASS='Nexus01' node scripts/scrape_rached_workshop.js
 */
const { chromium } = require('playwright');
const fs = require('fs');
const path = require('path');

const OUT_DIR = process.env.OUT_DIR || path.join(__dirname, '..', 'data', 'rached_import');
const USER = process.env.RACHED_USER || 'IVAN';
const PASS = process.env.RACHED_PASS || 'Nexus01';
const BASE_APP = 'http://app.flechasyventiladores-rached.com';
const BASE_API = 'http://appapi.flechasyventiladores-rached.com';

function sleep(ms) {
    return new Promise(r => setTimeout(r, ms));
}

async function main() {
    fs.mkdirSync(OUT_DIR, { recursive: true });
    const browser = await chromium.launch({ headless: true });
    const context = await browser.newContext({
        userAgent: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
        viewport: { width: 1440, height: 900 }
    });
    const page = await context.newPage();

    // ── LOGIN ──
    console.log('Logging in...');
    await page.goto(`${BASE_APP}/login`);
    await page.waitForSelector('#correo', { timeout: 10000 });
    await page.fill('#correo', USER);
    await page.fill('#password', PASS);
    await page.click('button:has-text("Aceptar")');
    await page.waitForURL('**/administracion/**', { timeout: 20000 });
    console.log('Logged in.');

    // ── ORDERS LIST ──
    console.log('Navigating to orders list...');
    const allOrders = [];
    let pageNum = 1;
    let respPromise = page.waitForResponse(
        r => r.url().includes('/api/servicios/ordenes/listado') && r.status() === 200,
        { timeout: 30000 }
    );
    await page.goto(`${BASE_APP}/administracion/servicios/ordenesservicio`);
    while (true) {
        if (pageNum > 1) {
            const nextBtn = page.locator('button.mat-paginator-navigation-next');
            const isDisabled = await nextBtn.evaluate(el => el.disabled).catch(() => true);
            if (isDisabled) {
                console.log('No more pages.');
                break;
            }
            respPromise = page.waitForResponse(
                r => r.url().includes('/api/servicios/ordenes/listado') && r.status() === 200,
                { timeout: 30000 }
            );
            await nextBtn.click();
        }
        const resp = await respPromise;
        const payload = await resp.json();
        const datos = payload.datos || {};
        const orders = datos.data || [];
        allOrders.push(...orders);
        fs.writeFileSync(
            path.join(OUT_DIR, `orders_page_${pageNum}.json`),
            JSON.stringify(payload, null, 2)
        );
        console.log(`Page ${pageNum}: ${orders.length} orders (total ${allOrders.length}/${datos.total || '?'})`);
        if (datos.current_page >= datos.last_page) break;
        pageNum++;
        await sleep(800);
    }
    fs.writeFileSync(
        path.join(OUT_DIR, 'orders_list.json'),
        JSON.stringify(allOrders, null, 2)
    );
    console.log(`Collected ${allOrders.length} orders summary.`);
    await page.waitForSelector('mat-row', { timeout: 10000 });
    await sleep(3000);

    if (!allOrders.length) {
        await browser.close();
        return;
    }

    // ── ORDER DETAILS ──
    const detailDir = path.join(OUT_DIR, 'details');
    fs.mkdirSync(detailDir, { recursive: true });

    let targetId = null;
    await page.route('**/api/servicios/ordenes/*', async (route, request) => {
        if (targetId && request.url().match(/\/ordenes\/\d+$/)) {
            const url = new URL(request.url());
            const parts = url.pathname.split('/');
            parts[parts.length - 1] = String(targetId);
            url.pathname = parts.join('/');
            await route.continue({ url: url.toString() });
            return;
        }
        await route.continue();
    });

    async function clickFirstEditButton() {
        await page.evaluate(() => {
            const row = document.querySelector('mat-row');
            if (!row) return;
            const btn = row.querySelector('button.mat-icon-button i.fa-edit');
            if (btn) btn.closest('button').click();
        });
    }

    const limit = parseInt(process.env.LIMIT || '0', 10);
    const ordersToFetch = limit > 0 ? allOrders.slice(0, limit) : allOrders;
    for (let i = 0; i < ordersToFetch.length; i++) {
        const o = ordersToFetch[i];
        targetId = o.id;
        const detailPath = path.join(detailDir, `order_${targetId}.json`);
        if (fs.existsSync(detailPath)) {
            process.stdout.write(`\rDetail ${i + 1}/${ordersToFetch.length}: ${targetId} (already saved)  `);
            continue;
        }
        let attempts = 0;
        while (attempts < 3) {
            attempts++;
            try {
                const respPromise = page.waitForResponse(
                    r => r.url().includes(`/api/servicios/ordenes/${targetId}`) && r.status() === 200,
                    { timeout: 20000 }
                );
                await clickFirstEditButton();
                const resp = await respPromise;
                const detail = await resp.json();
                fs.writeFileSync(
                    path.join(detailDir, `order_${targetId}.json`),
                    JSON.stringify(detail, null, 2)
                );
                process.stdout.write(`\rDetail ${i + 1}/${ordersToFetch.length}: ${targetId}  `);
                // close modal if opened
                await page.keyboard.press('Escape').catch(() => {});
                await sleep(300);
                break;
            } catch (e) {
                console.error(`\nError fetching detail for ${targetId} (attempt ${attempts}):`, e.message);
                await page.keyboard.press('Escape').catch(() => {});
                await sleep(1000);
            }
        }
    }
    console.log('\nDetail extraction complete.');

    await browser.close();
    console.log(`Output saved to ${OUT_DIR}`);
}

main().catch(e => {
    console.error(e);
    process.exit(1);
});