perf: optimize bulk XML upload for 100k+ files

Backend:
- Add batch insert using multi-row INSERT with ON CONFLICT
- Process in batches of 500 records for optimal DB performance
- Return detailed batch results (inserted, duplicates, errors)

Frontend:
- Parse files in chunks of 500 to prevent memory issues
- Upload in batches of 200 CFDIs per request
- Add detailed progress bar with real-time stats
- Show upload statistics (loaded, duplicates, errors)
- Add cancel functionality during upload
- Refresh data after upload completes

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Consultoria AS
2026-01-22 07:01:04 +00:00
parent c3ce7199af
commit db1f2eaecd
4 changed files with 567 additions and 137 deletions

View File

@@ -1,18 +1,34 @@
'use client';
import { useState, useRef } from 'react';
import { useState, useRef, useCallback } from 'react';
import { Header } from '@/components/layouts/header';
import { Card, CardContent, CardHeader, CardTitle, CardDescription } from '@/components/ui/card';
import { Button } from '@/components/ui/button';
import { Input } from '@/components/ui/input';
import { Label } from '@/components/ui/label';
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select';
import { useCfdis, useCreateCfdi, useCreateManyCfdis, useDeleteCfdi } from '@/lib/hooks/use-cfdi';
import { useCfdis, useCreateCfdi, useDeleteCfdi } from '@/lib/hooks/use-cfdi';
import { createManyCfdis } from '@/lib/api/cfdi';
import type { CfdiFilters, TipoCfdi } from '@horux/shared';
import type { CreateCfdiData } from '@/lib/api/cfdi';
import { FileText, Search, ChevronLeft, ChevronRight, Plus, Upload, Trash2, X, FileUp, CheckCircle, AlertCircle } from 'lucide-react';
import { FileText, Search, ChevronLeft, ChevronRight, Plus, Upload, Trash2, X, FileUp, CheckCircle, AlertCircle, Loader2 } from 'lucide-react';
import { useAuthStore } from '@/stores/auth-store';
import { useTenantViewStore } from '@/stores/tenant-view-store';
import { useQueryClient } from '@tanstack/react-query';
// Upload progress state
interface UploadProgress {
status: 'idle' | 'parsing' | 'uploading' | 'complete' | 'error';
totalFiles: number;
parsedFiles: number;
validFiles: number;
currentBatch: number;
totalBatches: number;
uploaded: number;
duplicates: number;
errors: number;
errorMessages: string[];
}
type CfdiTipo = 'ingreso' | 'egreso' | 'traslado' | 'nomina' | 'pago';
@@ -195,10 +211,15 @@ function parseCfdiXml(xmlString: string, tenantRfc: string): CreateCfdiData | nu
}
}
// Chunk size for batch uploads
const PARSE_CHUNK_SIZE = 500; // Parse 500 files at a time
const UPLOAD_CHUNK_SIZE = 200; // Upload 200 CFDIs per request
export default function CfdiPage() {
const { user } = useAuthStore();
const { viewingTenantRfc } = useTenantViewStore();
const fileInputRef = useRef<HTMLInputElement>(null);
const queryClient = useQueryClient();
// Get the effective tenant RFC (viewing tenant or user's tenant)
const tenantRfc = viewingTenantRfc || user?.tenantRfc || '';
@@ -211,13 +232,27 @@ export default function CfdiPage() {
const [showBulkForm, setShowBulkForm] = useState(false);
const [formData, setFormData] = useState<CreateCfdiData>(initialFormData);
const [bulkData, setBulkData] = useState('');
const [xmlFiles, setXmlFiles] = useState<File[]>([]);
const [parsedXmls, setParsedXmls] = useState<{ file: string; data: CreateCfdiData | null; error?: string }[]>([]);
const [uploadMode, setUploadMode] = useState<'xml' | 'json'>('xml');
const [jsonUploading, setJsonUploading] = useState(false);
// Optimized upload state
const [uploadProgress, setUploadProgress] = useState<UploadProgress>({
status: 'idle',
totalFiles: 0,
parsedFiles: 0,
validFiles: 0,
currentBatch: 0,
totalBatches: 0,
uploaded: 0,
duplicates: 0,
errors: 0,
errorMessages: []
});
const [parsedCfdis, setParsedCfdis] = useState<CreateCfdiData[]>([]);
const uploadAbortRef = useRef(false);
const { data, isLoading } = useCfdis(filters);
const createCfdi = useCreateCfdi();
const createManyCfdis = useCreateManyCfdis();
const deleteCfdi = useDeleteCfdi();
const canEdit = user?.role === 'admin' || user?.role === 'contador';
@@ -243,80 +278,204 @@ export default function CfdiPage() {
const handleBulkSubmit = async (e: React.FormEvent) => {
e.preventDefault();
setJsonUploading(true);
try {
const cfdis = JSON.parse(bulkData);
if (!Array.isArray(cfdis)) {
throw new Error('El formato debe ser un array de CFDIs');
}
const result = await createManyCfdis.mutateAsync(cfdis);
alert(`Se crearon ${result.count} CFDIs exitosamente`);
const result = await createManyCfdis(cfdis);
alert(`Se crearon ${result.inserted} CFDIs exitosamente`);
setBulkData('');
setShowBulkForm(false);
queryClient.invalidateQueries({ queryKey: ['cfdis'] });
} catch (error: any) {
alert(error.message || 'Error al procesar CFDIs');
} finally {
setJsonUploading(false);
}
};
const handleXmlFilesChange = async (e: React.ChangeEvent<HTMLInputElement>) => {
// Optimized: Parse files in chunks to prevent memory issues
const handleXmlFilesChange = useCallback(async (e: React.ChangeEvent<HTMLInputElement>) => {
const files = Array.from(e.target.files || []);
setXmlFiles(files);
if (files.length === 0) return;
// Parse each XML file
const parsed = await Promise.all(
files.map(async (file) => {
try {
const text = await file.text();
const data = parseCfdiXml(text, tenantRfc);
if (!data) {
return { file: file.name, data: null, error: 'No se pudo parsear el XML' };
uploadAbortRef.current = false;
setUploadProgress({
status: 'parsing',
totalFiles: files.length,
parsedFiles: 0,
validFiles: 0,
currentBatch: 0,
totalBatches: 0,
uploaded: 0,
duplicates: 0,
errors: 0,
errorMessages: []
});
setParsedCfdis([]);
const validCfdis: CreateCfdiData[] = [];
let parsedCount = 0;
let errorCount = 0;
// Process in chunks to prevent memory issues
for (let i = 0; i < files.length; i += PARSE_CHUNK_SIZE) {
if (uploadAbortRef.current) break;
const chunk = files.slice(i, i + PARSE_CHUNK_SIZE);
// Parse chunk in parallel
const results = await Promise.all(
chunk.map(async (file) => {
try {
const text = await file.text();
const data = parseCfdiXml(text, tenantRfc);
return data;
} catch {
return null;
}
if (!data.uuidFiscal) {
return { file: file.name, data: null, error: 'UUID no encontrado en el XML' };
}
return { file: file.name, data };
} catch (error) {
return { file: file.name, data: null, error: 'Error al leer el archivo' };
})
);
// Collect valid results
results.forEach((data) => {
if (data && data.uuidFiscal) {
validCfdis.push(data);
} else {
errorCount++;
}
})
);
});
setParsedXmls(parsed);
};
parsedCount += chunk.length;
setUploadProgress(prev => ({
...prev,
parsedFiles: parsedCount,
validFiles: validCfdis.length,
errors: errorCount
}));
// Small delay to allow UI to update
await new Promise(r => setTimeout(r, 10));
}
setParsedCfdis(validCfdis);
setUploadProgress(prev => ({
...prev,
status: uploadAbortRef.current ? 'idle' : 'idle',
totalBatches: Math.ceil(validCfdis.length / UPLOAD_CHUNK_SIZE)
}));
}, [tenantRfc]);
// Optimized: Upload in batches with progress
const handleXmlBulkSubmit = async (e: React.FormEvent) => {
e.preventDefault();
const validCfdis = parsedXmls
.filter((p) => p.data !== null)
.map((p) => p.data as CreateCfdiData);
if (validCfdis.length === 0) {
if (parsedCfdis.length === 0) {
alert('No hay CFDIs validos para cargar');
return;
}
try {
const result = await createManyCfdis.mutateAsync(validCfdis);
alert(`Se crearon ${result.count} CFDIs exitosamente`);
setXmlFiles([]);
setParsedXmls([]);
setShowBulkForm(false);
if (fileInputRef.current) {
fileInputRef.current.value = '';
uploadAbortRef.current = false;
const totalBatches = Math.ceil(parsedCfdis.length / UPLOAD_CHUNK_SIZE);
setUploadProgress(prev => ({
...prev,
status: 'uploading',
currentBatch: 0,
totalBatches,
uploaded: 0,
duplicates: 0,
errors: 0,
errorMessages: []
}));
let totalUploaded = 0;
let totalDuplicates = 0;
let totalErrors = 0;
const allErrors: string[] = [];
// Upload in batches
for (let i = 0; i < parsedCfdis.length; i += UPLOAD_CHUNK_SIZE) {
if (uploadAbortRef.current) break;
const batchNumber = Math.floor(i / UPLOAD_CHUNK_SIZE) + 1;
const chunk = parsedCfdis.slice(i, i + UPLOAD_CHUNK_SIZE);
setUploadProgress(prev => ({
...prev,
currentBatch: batchNumber
}));
try {
const result = await createManyCfdis(chunk, batchNumber, totalBatches, parsedCfdis.length);
totalUploaded += result.inserted;
totalDuplicates += result.duplicates;
totalErrors += result.errors;
if (result.errorMessages) {
allErrors.push(...result.errorMessages);
}
setUploadProgress(prev => ({
...prev,
uploaded: totalUploaded,
duplicates: totalDuplicates,
errors: prev.errors + result.errors,
errorMessages: allErrors.slice(0, 20) // Limit error messages
}));
} catch (error: any) {
console.error(`Error en lote ${batchNumber}:`, error);
totalErrors += chunk.length;
allErrors.push(`Lote ${batchNumber}: ${error.message || 'Error desconocido'}`);
setUploadProgress(prev => ({
...prev,
errors: prev.errors + chunk.length,
errorMessages: allErrors.slice(0, 20)
}));
}
} catch (error: any) {
alert(error.response?.data?.message || 'Error al cargar CFDIs');
// Small delay between batches
await new Promise(r => setTimeout(r, 100));
}
setUploadProgress(prev => ({
...prev,
status: 'complete'
}));
// Invalidate queries to refresh the list
queryClient.invalidateQueries({ queryKey: ['cfdis'] });
};
const clearXmlFiles = () => {
setXmlFiles([]);
setParsedXmls([]);
uploadAbortRef.current = true;
setParsedCfdis([]);
setUploadProgress({
status: 'idle',
totalFiles: 0,
parsedFiles: 0,
validFiles: 0,
currentBatch: 0,
totalBatches: 0,
uploaded: 0,
duplicates: 0,
errors: 0,
errorMessages: []
});
if (fileInputRef.current) {
fileInputRef.current.value = '';
}
};
const cancelUpload = () => {
uploadAbortRef.current = true;
setUploadProgress(prev => ({ ...prev, status: 'idle' }));
};
const handleDelete = async (id: string) => {
if (confirm('¿Eliminar este CFDI?')) {
try {
@@ -662,74 +821,182 @@ export default function CfdiPage() {
{uploadMode === 'xml' ? (
<form onSubmit={handleXmlBulkSubmit} className="space-y-4">
<div className="space-y-2">
<Label>Archivos XML de CFDI</Label>
<div className="border-2 border-dashed rounded-lg p-6 text-center">
<input
ref={fileInputRef}
type="file"
accept=".xml"
multiple
onChange={handleXmlFilesChange}
className="hidden"
id="xml-upload"
/>
<label htmlFor="xml-upload" className="cursor-pointer">
<FileUp className="h-10 w-10 mx-auto text-muted-foreground mb-2" />
<p className="text-sm text-muted-foreground">
Haz clic para seleccionar archivos XML o arrastralos aqui
</p>
<p className="text-xs text-muted-foreground mt-1">
Puedes seleccionar multiples archivos
</p>
</label>
</div>
</div>
{/* Show parsed results */}
{parsedXmls.length > 0 && (
{/* File input - only show when idle */}
{uploadProgress.status === 'idle' && (
<div className="space-y-2">
<div className="flex items-center justify-between">
<Label>Archivos procesados ({parsedXmls.filter(p => p.data).length} validos de {parsedXmls.length})</Label>
<Button type="button" variant="ghost" size="sm" onClick={clearXmlFiles}>
Limpiar
</Button>
</div>
<div className="max-h-48 overflow-y-auto border rounded-lg divide-y">
{parsedXmls.map((parsed, idx) => (
<div key={idx} className="p-2 flex items-center gap-2 text-sm">
{parsed.data ? (
<CheckCircle className="h-4 w-4 text-success flex-shrink-0" />
) : (
<AlertCircle className="h-4 w-4 text-destructive flex-shrink-0" />
)}
<div className="flex-1 min-w-0">
<p className="font-medium truncate">{parsed.file}</p>
{parsed.data ? (
<p className="text-xs text-muted-foreground">
{parsed.data.tipo === 'ingreso' ? 'Ingreso' : 'Egreso'} - {parsed.data.nombreEmisor?.substring(0, 30)}... - ${parsed.data.total?.toLocaleString()}
</p>
) : (
<p className="text-xs text-destructive">{parsed.error}</p>
)}
</div>
</div>
))}
<Label>Archivos XML de CFDI</Label>
<div className="border-2 border-dashed rounded-lg p-6 text-center">
<input
ref={fileInputRef}
type="file"
accept=".xml"
multiple
onChange={handleXmlFilesChange}
className="hidden"
id="xml-upload"
/>
<label htmlFor="xml-upload" className="cursor-pointer">
<FileUp className="h-10 w-10 mx-auto text-muted-foreground mb-2" />
<p className="text-sm text-muted-foreground">
Haz clic para seleccionar archivos XML
</p>
<p className="text-xs text-muted-foreground mt-1">
Optimizado para cargas masivas (+100,000 archivos)
</p>
</label>
</div>
</div>
)}
<div className="flex gap-2 justify-end">
<Button type="button" variant="outline" onClick={() => { setShowBulkForm(false); clearXmlFiles(); }}>
Cancelar
</Button>
<Button
type="submit"
disabled={createManyCfdis.isPending || parsedXmls.filter(p => p.data).length === 0}
>
{createManyCfdis.isPending ? 'Procesando...' : `Cargar ${parsedXmls.filter(p => p.data).length} CFDIs`}
</Button>
</div>
{/* Parsing progress */}
{uploadProgress.status === 'parsing' && (
<div className="space-y-4">
<div className="flex items-center gap-3">
<Loader2 className="h-5 w-5 animate-spin text-primary" />
<div className="flex-1">
<p className="font-medium">Analizando archivos XML...</p>
<p className="text-sm text-muted-foreground">
{uploadProgress.parsedFiles.toLocaleString()} de {uploadProgress.totalFiles.toLocaleString()} archivos
</p>
</div>
<Button type="button" variant="outline" size="sm" onClick={cancelUpload}>
Cancelar
</Button>
</div>
<div className="h-2 bg-muted rounded-full overflow-hidden">
<div
className="h-full bg-primary transition-all duration-300"
style={{ width: `${(uploadProgress.parsedFiles / uploadProgress.totalFiles) * 100}%` }}
/>
</div>
<div className="flex justify-between text-xs text-muted-foreground">
<span>{uploadProgress.validFiles.toLocaleString()} validos</span>
<span>{Math.round((uploadProgress.parsedFiles / uploadProgress.totalFiles) * 100)}%</span>
</div>
</div>
)}
{/* Upload progress */}
{uploadProgress.status === 'uploading' && (
<div className="space-y-4">
<div className="flex items-center gap-3">
<Loader2 className="h-5 w-5 animate-spin text-primary" />
<div className="flex-1">
<p className="font-medium">Subiendo CFDIs al servidor...</p>
<p className="text-sm text-muted-foreground">
Lote {uploadProgress.currentBatch} de {uploadProgress.totalBatches}
</p>
</div>
<Button type="button" variant="outline" size="sm" onClick={cancelUpload}>
Cancelar
</Button>
</div>
<div className="h-2 bg-muted rounded-full overflow-hidden">
<div
className="h-full bg-primary transition-all duration-300"
style={{ width: `${(uploadProgress.currentBatch / uploadProgress.totalBatches) * 100}%` }}
/>
</div>
<div className="grid grid-cols-3 gap-4 text-center">
<div className="p-3 bg-success/10 rounded-lg">
<p className="text-2xl font-bold text-success">{uploadProgress.uploaded.toLocaleString()}</p>
<p className="text-xs text-muted-foreground">Cargados</p>
</div>
<div className="p-3 bg-warning/10 rounded-lg">
<p className="text-2xl font-bold text-warning">{uploadProgress.duplicates.toLocaleString()}</p>
<p className="text-xs text-muted-foreground">Duplicados</p>
</div>
<div className="p-3 bg-destructive/10 rounded-lg">
<p className="text-2xl font-bold text-destructive">{uploadProgress.errors.toLocaleString()}</p>
<p className="text-xs text-muted-foreground">Errores</p>
</div>
</div>
</div>
)}
{/* Upload complete */}
{uploadProgress.status === 'complete' && (
<div className="space-y-4">
<div className="flex items-center gap-3">
<CheckCircle className="h-6 w-6 text-success" />
<div className="flex-1">
<p className="font-medium text-success">Carga completada</p>
<p className="text-sm text-muted-foreground">
Se procesaron {uploadProgress.validFiles.toLocaleString()} archivos
</p>
</div>
</div>
<div className="grid grid-cols-3 gap-4 text-center">
<div className="p-3 bg-success/10 rounded-lg">
<p className="text-2xl font-bold text-success">{uploadProgress.uploaded.toLocaleString()}</p>
<p className="text-xs text-muted-foreground">Cargados</p>
</div>
<div className="p-3 bg-warning/10 rounded-lg">
<p className="text-2xl font-bold text-warning">{uploadProgress.duplicates.toLocaleString()}</p>
<p className="text-xs text-muted-foreground">Duplicados</p>
</div>
<div className="p-3 bg-destructive/10 rounded-lg">
<p className="text-2xl font-bold text-destructive">{uploadProgress.errors.toLocaleString()}</p>
<p className="text-xs text-muted-foreground">Errores</p>
</div>
</div>
{uploadProgress.errorMessages.length > 0 && (
<div className="p-3 bg-destructive/10 rounded-lg">
<p className="text-sm font-medium text-destructive mb-2">Errores:</p>
<ul className="text-xs text-destructive space-y-1 max-h-24 overflow-y-auto">
{uploadProgress.errorMessages.map((err, i) => (
<li key={i}>{err}</li>
))}
</ul>
</div>
)}
<div className="flex gap-2 justify-end">
<Button type="button" variant="outline" onClick={clearXmlFiles}>
Cargar mas archivos
</Button>
<Button type="button" onClick={() => { setShowBulkForm(false); clearXmlFiles(); }}>
Cerrar
</Button>
</div>
</div>
)}
{/* Ready to upload - show summary and upload button */}
{uploadProgress.status === 'idle' && parsedCfdis.length > 0 && (
<div className="space-y-4">
<div className="p-4 bg-muted/50 rounded-lg">
<div className="flex items-center justify-between">
<div>
<p className="font-medium">{parsedCfdis.length.toLocaleString()} CFDIs listos para cargar</p>
<p className="text-sm text-muted-foreground">
Se enviaran en {Math.ceil(parsedCfdis.length / UPLOAD_CHUNK_SIZE)} lotes de {UPLOAD_CHUNK_SIZE} registros
</p>
</div>
<Button type="button" variant="ghost" size="sm" onClick={clearXmlFiles}>
Limpiar
</Button>
</div>
</div>
<div className="flex gap-2 justify-end">
<Button type="button" variant="outline" onClick={() => { setShowBulkForm(false); clearXmlFiles(); }}>
Cancelar
</Button>
<Button type="submit">
<Upload className="h-4 w-4 mr-2" />
Iniciar carga
</Button>
</div>
</div>
)}
{/* Initial state - no files */}
{uploadProgress.status === 'idle' && parsedCfdis.length === 0 && uploadProgress.totalFiles === 0 && (
<div className="flex gap-2 justify-end">
<Button type="button" variant="outline" onClick={() => setShowBulkForm(false)}>
Cancelar
</Button>
</div>
)}
</form>
) : (
<form onSubmit={handleBulkSubmit} className="space-y-4">
@@ -761,8 +1028,8 @@ export default function CfdiPage() {
<Button type="button" variant="outline" onClick={() => setShowBulkForm(false)}>
Cancelar
</Button>
<Button type="submit" disabled={createManyCfdis.isPending}>
{createManyCfdis.isPending ? 'Procesando...' : 'Cargar CFDIs'}
<Button type="submit" disabled={jsonUploading}>
{jsonUploading ? 'Procesando...' : 'Cargar CFDIs'}
</Button>
</div>
</form>