consultoria-as/Horux360
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf: optimize bulk XML upload for 100k+ files

Browse Source 
Backend:
- Add batch insert using multi-row INSERT with ON CONFLICT
- Process in batches of 500 records for optimal DB performance
- Return detailed batch results (inserted, duplicates, errors)

Frontend:
- Parse files in chunks of 500 to prevent memory issues
- Upload in batches of 200 CFDIs per request
- Add detailed progress bar with real-time stats
- Show upload statistics (loaded, duplicates, errors)
- Add cancel functionality during upload
- Refresh data after upload completes

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Consultoria AS
2026-01-22 07:01:04 +00:00
parent c3ce7199af
commit db1f2eaecd
4 changed files with 567 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
apps/api/src/controllers/cfdi.controller.ts
View File

@@ -96,15 +96,25 @@ export async function createManyCfdis(req: Request, res: Response, next: NextFun
return next(new AppError(400, 'Se requiere un array de CFDIs'));
}
console.log(`[CFDI Bulk] Recibidos ${req.body.cfdis.length} CFDIs para schema ${req.tenantSchema}`);
const batchInfo = {
batchNumber: req.body.batchNumber || 1,
totalBatches: req.body.totalBatches || 1,
totalFiles: req.body.totalFiles || req.body.cfdis.length
};
// Log first CFDI for debugging
if (req.body.cfdis.length > 0) {
console.log('[CFDI Bulk] Primer CFDI:', JSON.stringify(req.body.cfdis[0], null, 2));
}
console.log(`[CFDI Bulk] Lote ${batchInfo.batchNumber}/${batchInfo.totalBatches} - ${req.body.cfdis.length} CFDIs para schema ${req.tenantSchema}`);
const count = await cfdiService.createManyCfdis(req.tenantSchema, req.body.cfdis);
res.status(201).json({ message: `${count} CFDIs creados exitosamente`, count });
const result = await cfdiService.createManyCfdisBatch(req.tenantSchema, req.body.cfdis);
res.status(201).json({
message: `Lote ${batchInfo.batchNumber} procesado`,
batchNumber: batchInfo.batchNumber,
totalBatches: batchInfo.totalBatches,
inserted: result.inserted,
duplicates: result.duplicates,
errors: result.errors,
errorMessages: result.errorMessages.slice(0, 5) // Limit error messages
});
} catch (error: any) {
console.error('[CFDI Bulk Error]', error.message, error.stack);
next(new AppError(400, error.message || 'Error al procesar CFDIs'));