Files
GRH/water-api/src/services/bulk-upload.service.ts
Exteban08 6c7d448b2f Fix: Corregir pantalla blanca y mejorar carga masiva
- Fix error .toFixed() con valores DECIMAL de PostgreSQL (string vs number)
- Fix modal de carga masiva que se cerraba sin mostrar resultados
- Validar fechas antes de insertar en BD (evita error con "Installed")
- Agregar mapeos de columnas comunes (device_status, device_name, etc.)
- Normalizar valores de status (Installed -> ACTIVE, New_LoRa -> ACTIVE)
- Actualizar documentación del proyecto

Archivos modificados:
- src/pages/meters/MetersTable.tsx
- src/pages/consumption/ConsumptionPage.tsx
- src/pages/meters/MeterPage.tsx
- water-api/src/services/bulk-upload.service.ts
- ESTADO_ACTUAL.md
- CAMBIOS_SESION.md

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2026-01-23 23:13:48 +00:00

661 lines
20 KiB
TypeScript

import * as XLSX from 'xlsx';
import { query } from '../config/database';
/**
* Result of a bulk upload operation
*/
export interface BulkUploadResult {
success: boolean;
totalRows: number;
inserted: number;
errors: Array<{
row: number;
error: string;
data?: Record<string, unknown>;
}>;
}
/**
* Expected columns in the Excel file for meters
*/
interface MeterRow {
serial_number: string;
meter_id?: string;
name: string;
concentrator_serial: string; // We'll look up the concentrator by serial
location?: string;
type?: string;
status?: string;
installation_date?: string;
}
/**
* Parse Excel file buffer and return rows
*/
function parseExcelBuffer(buffer: Buffer): Record<string, unknown>[] {
const workbook = XLSX.read(buffer, { type: 'buffer' });
const sheetName = workbook.SheetNames[0];
const worksheet = workbook.Sheets[sheetName];
// Convert to JSON with header row
const rows = XLSX.utils.sheet_to_json<Record<string, unknown>>(worksheet, {
defval: null,
raw: false,
});
return rows;
}
/**
* Normalize column names (handle variations)
*/
function normalizeColumnName(name: string): string {
const normalized = name
.toLowerCase()
.trim()
.replace(/\s+/g, '_')
.replace(/[áàäâ]/g, 'a')
.replace(/[éèëê]/g, 'e')
.replace(/[íìïî]/g, 'i')
.replace(/[óòöô]/g, 'o')
.replace(/[úùüû]/g, 'u')
.replace(/ñ/g, 'n');
// Map common variations
const mappings: Record<string, string> = {
// Serial number
'serial': 'serial_number',
'numero_de_serie': 'serial_number',
'serial_number': 'serial_number',
'device_s/n': 'serial_number',
'device_sn': 'serial_number',
's/n': 'serial_number',
'sn': 'serial_number',
// Meter ID
'meter_id': 'meter_id',
'meterid': 'meter_id',
'id_medidor': 'meter_id',
// Name
'nombre': 'name',
'name': 'name',
'device_name': 'name',
'meter_name': 'name',
'nombre_medidor': 'name',
// Concentrator
'concentrador': 'concentrator_serial',
'concentrator': 'concentrator_serial',
'concentrator_serial': 'concentrator_serial',
'serial_concentrador': 'concentrator_serial',
'gateway': 'concentrator_serial',
'gateway_serial': 'concentrator_serial',
// Location
'ubicacion': 'location',
'location': 'location',
'direccion': 'location',
'address': 'location',
// Type
'tipo': 'type',
'type': 'type',
'device_type': 'type',
'tipo_dispositivo': 'type',
'protocol': 'type',
'protocolo': 'type',
// Status
'estado': 'status',
'status': 'status',
'device_status': 'status',
'estado_dispositivo': 'status',
// Installation date
'fecha_instalacion': 'installation_date',
'installation_date': 'installation_date',
'fecha_de_instalacion': 'installation_date',
'installed_time': 'installation_date',
'installed_date': 'installation_date',
};
return mappings[normalized] || normalized;
}
/**
* Normalize row data with column name mapping
*/
function normalizeRow(row: Record<string, unknown>): Record<string, unknown> {
const normalized: Record<string, unknown> = {};
for (const [key, value] of Object.entries(row)) {
const normalizedKey = normalizeColumnName(key);
normalized[normalizedKey] = value;
}
return normalized;
}
/**
* Validate a meter row
*/
function validateMeterRow(row: Record<string, unknown>, rowIndex: number): { valid: boolean; error?: string } {
if (!row.serial_number || String(row.serial_number).trim() === '') {
return { valid: false, error: `Fila ${rowIndex}: serial_number es requerido` };
}
if (!row.name || String(row.name).trim() === '') {
return { valid: false, error: `Fila ${rowIndex}: name es requerido` };
}
if (!row.concentrator_serial || String(row.concentrator_serial).trim() === '') {
return { valid: false, error: `Fila ${rowIndex}: concentrator_serial es requerido` };
}
return { valid: true };
}
/**
* Bulk upload meters from Excel buffer
*/
export async function bulkUploadMeters(buffer: Buffer): Promise<BulkUploadResult> {
const result: BulkUploadResult = {
success: true,
totalRows: 0,
inserted: 0,
errors: [],
};
try {
// Parse Excel file
const rawRows = parseExcelBuffer(buffer);
result.totalRows = rawRows.length;
if (rawRows.length === 0) {
result.success = false;
result.errors.push({ row: 0, error: 'El archivo está vacío o no tiene datos válidos' });
return result;
}
// Normalize column names
const rows = rawRows.map(row => normalizeRow(row));
// Get all concentrators for lookup
const concentratorsResult = await query<{ id: string; serial_number: string }>(
'SELECT id, serial_number FROM concentrators'
);
const concentratorMap = new Map(
concentratorsResult.rows.map(c => [c.serial_number.toLowerCase(), c.id])
);
// Process each row
for (let i = 0; i < rows.length; i++) {
const row = rows[i];
const rowIndex = i + 2; // Excel row number (1-indexed + header row)
// Validate row
const validation = validateMeterRow(row, rowIndex);
if (!validation.valid) {
result.errors.push({ row: rowIndex, error: validation.error!, data: row });
continue;
}
// Look up concentrator
const concentratorSerial = String(row.concentrator_serial).trim().toLowerCase();
const concentratorId = concentratorMap.get(concentratorSerial);
if (!concentratorId) {
result.errors.push({
row: rowIndex,
error: `Concentrador con serial "${row.concentrator_serial}" no encontrado`,
data: row,
});
continue;
}
// Prepare meter data
// Validate installation_date is actually a valid date
let installationDate: string | undefined = undefined;
if (row.installation_date) {
const dateStr = String(row.installation_date).trim();
// Check if it looks like a date (contains numbers and possibly dashes/slashes)
if (/^\d{4}[-/]\d{1,2}[-/]\d{1,2}/.test(dateStr) || /^\d{1,2}[-/]\d{1,2}[-/]\d{2,4}/.test(dateStr)) {
const parsed = new Date(dateStr);
if (!isNaN(parsed.getTime())) {
installationDate = parsed.toISOString().split('T')[0];
}
}
}
const meterData: MeterRow = {
serial_number: String(row.serial_number).trim(),
meter_id: row.meter_id ? String(row.meter_id).trim() : undefined,
name: String(row.name).trim(),
concentrator_serial: String(row.concentrator_serial).trim(),
location: row.location ? String(row.location).trim() : undefined,
type: row.type ? String(row.type).trim().toUpperCase() : 'LORA',
status: row.status ? String(row.status).trim().toUpperCase() : 'ACTIVE',
installation_date: installationDate,
};
// Validate type
const validTypes = ['LORA', 'LORAWAN', 'GRANDES'];
if (!validTypes.includes(meterData.type!)) {
meterData.type = 'LORA';
}
// Validate and normalize status
const statusMappings: Record<string, string> = {
'ACTIVE': 'ACTIVE',
'INACTIVE': 'INACTIVE',
'MAINTENANCE': 'MAINTENANCE',
'FAULTY': 'FAULTY',
'REPLACED': 'REPLACED',
'INSTALLED': 'ACTIVE',
'NEW_LORA': 'ACTIVE',
'NEW': 'ACTIVE',
'ENABLED': 'ACTIVE',
'DISABLED': 'INACTIVE',
'OFFLINE': 'INACTIVE',
'ONLINE': 'ACTIVE',
};
const normalizedStatus = meterData.status?.toUpperCase().replace(/\s+/g, '_') || 'ACTIVE';
meterData.status = statusMappings[normalizedStatus] || 'ACTIVE';
// Insert meter
try {
await query(
`INSERT INTO meters (serial_number, meter_id, name, concentrator_id, location, type, status, installation_date)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`,
[
meterData.serial_number,
meterData.meter_id || null,
meterData.name,
concentratorId,
meterData.location || null,
meterData.type,
meterData.status,
meterData.installation_date || null,
]
);
result.inserted++;
} catch (err) {
const error = err as Error & { code?: string; detail?: string };
let errorMessage = error.message;
if (error.code === '23505') {
errorMessage = `Serial "${meterData.serial_number}" ya existe en la base de datos`;
}
result.errors.push({
row: rowIndex,
error: errorMessage,
data: row,
});
}
}
result.success = result.errors.length === 0;
} catch (err) {
const error = err as Error;
result.success = false;
result.errors.push({ row: 0, error: `Error procesando archivo: ${error.message}` });
}
return result;
}
/**
* Generate Excel template for meters
*/
export function generateMeterTemplate(): Buffer {
const templateData = [
{
serial_number: 'EJEMPLO-001',
meter_id: 'MID-001',
name: 'Medidor Ejemplo 1',
concentrator_serial: 'CONC-001',
location: 'Ubicación ejemplo',
type: 'LORA',
status: 'ACTIVE',
installation_date: '2024-01-15',
},
{
serial_number: 'EJEMPLO-002',
meter_id: 'MID-002',
name: 'Medidor Ejemplo 2',
concentrator_serial: 'CONC-001',
location: 'Otra ubicación',
type: 'LORAWAN',
status: 'ACTIVE',
installation_date: '2024-01-16',
},
];
const worksheet = XLSX.utils.json_to_sheet(templateData);
// Set column widths
worksheet['!cols'] = [
{ wch: 15 }, // serial_number
{ wch: 12 }, // meter_id
{ wch: 25 }, // name
{ wch: 20 }, // concentrator_serial
{ wch: 25 }, // location
{ wch: 10 }, // type
{ wch: 12 }, // status
{ wch: 15 }, // installation_date
];
const workbook = XLSX.utils.book_new();
XLSX.utils.book_append_sheet(workbook, worksheet, 'Medidores');
// Add instructions sheet
const instructionsData = [
{ Campo: 'serial_number', Descripcion: 'Número de serie del medidor (REQUERIDO, único)', Ejemplo: 'MED-2024-001' },
{ Campo: 'meter_id', Descripcion: 'ID del medidor (opcional)', Ejemplo: 'ID-001' },
{ Campo: 'name', Descripcion: 'Nombre del medidor (REQUERIDO)', Ejemplo: 'Medidor Casa 1' },
{ Campo: 'concentrator_serial', Descripcion: 'Serial del concentrador (REQUERIDO)', Ejemplo: 'CONC-001' },
{ Campo: 'location', Descripcion: 'Ubicación (opcional)', Ejemplo: 'Calle Principal #123' },
{ Campo: 'type', Descripcion: 'Tipo: LORA, LORAWAN, GRANDES (opcional, default: LORA)', Ejemplo: 'LORA' },
{ Campo: 'status', Descripcion: 'Estado: ACTIVE, INACTIVE, MAINTENANCE, FAULTY, REPLACED (opcional, default: ACTIVE)', Ejemplo: 'ACTIVE' },
{ Campo: 'installation_date', Descripcion: 'Fecha de instalación YYYY-MM-DD (opcional)', Ejemplo: '2024-01-15' },
];
const instructionsSheet = XLSX.utils.json_to_sheet(instructionsData);
instructionsSheet['!cols'] = [
{ wch: 20 },
{ wch: 60 },
{ wch: 20 },
];
XLSX.utils.book_append_sheet(workbook, instructionsSheet, 'Instrucciones');
return Buffer.from(XLSX.write(workbook, { type: 'buffer', bookType: 'xlsx' }));
}
/**
* Expected columns in the Excel file for readings
*/
interface ReadingRow {
meter_serial: string;
reading_value: number;
reading_type?: string;
received_at?: string;
battery_level?: number;
signal_strength?: number;
}
/**
* Normalize column name for readings
*/
function normalizeReadingColumnName(name: string): string {
const normalized = name
.toLowerCase()
.trim()
.replace(/\s+/g, '_')
.replace(/[áàäâ]/g, 'a')
.replace(/[éèëê]/g, 'e')
.replace(/[íìïî]/g, 'i')
.replace(/[óòöô]/g, 'o')
.replace(/[úùüû]/g, 'u')
.replace(/ñ/g, 'n');
const mappings: Record<string, string> = {
// Meter serial
'serial': 'meter_serial',
'serial_number': 'meter_serial',
'meter_serial': 'meter_serial',
'numero_de_serie': 'meter_serial',
'serial_medidor': 'meter_serial',
'medidor': 'meter_serial',
// Reading value
'valor': 'reading_value',
'value': 'reading_value',
'reading_value': 'reading_value',
'lectura': 'reading_value',
'consumo': 'reading_value',
// Reading type
'tipo': 'reading_type',
'type': 'reading_type',
'reading_type': 'reading_type',
'tipo_lectura': 'reading_type',
// Received at
'fecha': 'received_at',
'date': 'received_at',
'received_at': 'received_at',
'fecha_lectura': 'received_at',
'fecha_hora': 'received_at',
// Battery
'bateria': 'battery_level',
'battery': 'battery_level',
'battery_level': 'battery_level',
'nivel_bateria': 'battery_level',
// Signal
'senal': 'signal_strength',
'signal': 'signal_strength',
'signal_strength': 'signal_strength',
'intensidad_senal': 'signal_strength',
};
return mappings[normalized] || normalized;
}
/**
* Normalize row for readings
*/
function normalizeReadingRow(row: Record<string, unknown>): Record<string, unknown> {
const normalized: Record<string, unknown> = {};
for (const [key, value] of Object.entries(row)) {
const normalizedKey = normalizeReadingColumnName(key);
normalized[normalizedKey] = value;
}
return normalized;
}
/**
* Validate a reading row
*/
function validateReadingRow(row: Record<string, unknown>, rowIndex: number): { valid: boolean; error?: string } {
if (!row.meter_serial || String(row.meter_serial).trim() === '') {
return { valid: false, error: `Fila ${rowIndex}: meter_serial es requerido` };
}
if (row.reading_value === null || row.reading_value === undefined || row.reading_value === '') {
return { valid: false, error: `Fila ${rowIndex}: reading_value es requerido` };
}
const value = parseFloat(String(row.reading_value));
if (isNaN(value)) {
return { valid: false, error: `Fila ${rowIndex}: reading_value debe ser un número` };
}
return { valid: true };
}
/**
* Bulk upload readings from Excel buffer
*/
export async function bulkUploadReadings(buffer: Buffer): Promise<BulkUploadResult> {
const result: BulkUploadResult = {
success: true,
totalRows: 0,
inserted: 0,
errors: [],
};
try {
// Parse Excel file
const rawRows = parseExcelBuffer(buffer);
result.totalRows = rawRows.length;
if (rawRows.length === 0) {
result.success = false;
result.errors.push({ row: 0, error: 'El archivo está vacío o no tiene datos válidos' });
return result;
}
// Normalize column names
const rows = rawRows.map(row => normalizeReadingRow(row));
// Get all meters for lookup by serial number
const metersResult = await query<{ id: string; serial_number: string }>(
'SELECT id, serial_number FROM meters'
);
const meterMap = new Map(
metersResult.rows.map(m => [m.serial_number.toLowerCase(), m.id])
);
// Process each row
for (let i = 0; i < rows.length; i++) {
const row = rows[i];
const rowIndex = i + 2; // Excel row number (1-indexed + header row)
// Validate row
const validation = validateReadingRow(row, rowIndex);
if (!validation.valid) {
result.errors.push({ row: rowIndex, error: validation.error!, data: row });
continue;
}
// Look up meter
const meterSerial = String(row.meter_serial).trim().toLowerCase();
const meterId = meterMap.get(meterSerial);
if (!meterId) {
result.errors.push({
row: rowIndex,
error: `Medidor con serial "${row.meter_serial}" no encontrado`,
data: row,
});
continue;
}
// Prepare reading data
const readingValue = parseFloat(String(row.reading_value));
const readingType = row.reading_type ? String(row.reading_type).trim().toUpperCase() : 'MANUAL';
const receivedAt = row.received_at ? String(row.received_at).trim() : null;
// Parse battery level - handle NaN and invalid values
let batteryLevel: number | null = null;
if (row.battery_level !== undefined && row.battery_level !== null && row.battery_level !== '') {
const parsed = parseFloat(String(row.battery_level));
if (!isNaN(parsed) && isFinite(parsed)) {
batteryLevel = parsed;
}
}
// Parse signal strength - handle NaN and invalid values
let signalStrength: number | null = null;
if (row.signal_strength !== undefined && row.signal_strength !== null && row.signal_strength !== '') {
const parsed = parseFloat(String(row.signal_strength));
if (!isNaN(parsed) && isFinite(parsed)) {
signalStrength = parsed;
}
}
// Validate reading type
const validTypes = ['AUTOMATIC', 'MANUAL', 'SCHEDULED'];
const finalReadingType = validTypes.includes(readingType) ? readingType : 'MANUAL';
// Insert reading
try {
await query(
`INSERT INTO meter_readings (meter_id, reading_value, reading_type, battery_level, signal_strength, received_at)
VALUES ($1, $2, $3, $4, $5, COALESCE($6::timestamp, NOW()))`,
[
meterId,
readingValue,
finalReadingType,
batteryLevel,
signalStrength,
receivedAt,
]
);
// Update meter's last reading
await query(
`UPDATE meters
SET last_reading_value = $1, last_reading_at = COALESCE($2::timestamp, NOW()), updated_at = NOW()
WHERE id = $3`,
[readingValue, receivedAt, meterId]
);
result.inserted++;
} catch (err) {
const error = err as Error & { code?: string; detail?: string };
result.errors.push({
row: rowIndex,
error: error.message,
data: row,
});
}
}
result.success = result.errors.length === 0;
} catch (err) {
const error = err as Error;
result.success = false;
result.errors.push({ row: 0, error: `Error procesando archivo: ${error.message}` });
}
return result;
}
/**
* Generate Excel template for readings
*/
export function generateReadingTemplate(): Buffer {
const templateData = [
{
meter_serial: '24151158',
reading_value: 123.45,
reading_type: 'MANUAL',
received_at: '2024-01-15 10:30:00',
battery_level: 85,
signal_strength: -70,
},
{
meter_serial: '24151159',
reading_value: 456.78,
reading_type: 'MANUAL',
received_at: '2024-01-15 10:35:00',
battery_level: 90,
signal_strength: -65,
},
];
const worksheet = XLSX.utils.json_to_sheet(templateData);
// Set column widths
worksheet['!cols'] = [
{ wch: 15 }, // meter_serial
{ wch: 15 }, // reading_value
{ wch: 12 }, // reading_type
{ wch: 20 }, // received_at
{ wch: 15 }, // battery_level
{ wch: 15 }, // signal_strength
];
const workbook = XLSX.utils.book_new();
XLSX.utils.book_append_sheet(workbook, worksheet, 'Lecturas');
// Add instructions sheet
const instructionsData = [
{ Campo: 'meter_serial', Descripcion: 'Número de serie del medidor (REQUERIDO)', Ejemplo: '24151158' },
{ Campo: 'reading_value', Descripcion: 'Valor de la lectura en m³ (REQUERIDO)', Ejemplo: '123.45' },
{ Campo: 'reading_type', Descripcion: 'Tipo: AUTOMATIC, MANUAL, SCHEDULED (opcional, default: MANUAL)', Ejemplo: 'MANUAL' },
{ Campo: 'received_at', Descripcion: 'Fecha y hora de la lectura YYYY-MM-DD HH:MM:SS (opcional, default: ahora)', Ejemplo: '2024-01-15 10:30:00' },
{ Campo: 'battery_level', Descripcion: 'Nivel de batería en % (opcional)', Ejemplo: '85' },
{ Campo: 'signal_strength', Descripcion: 'Intensidad de señal en dBm (opcional)', Ejemplo: '-70' },
];
const instructionsSheet = XLSX.utils.json_to_sheet(instructionsData);
instructionsSheet['!cols'] = [
{ wch: 20 },
{ wch: 60 },
{ wch: 25 },
];
XLSX.utils.book_append_sheet(workbook, instructionsSheet, 'Instrucciones');
return Buffer.from(XLSX.write(workbook, { type: 'buffer', bookType: 'xlsx' }));
}