Add CSV upload panel for meters and readings

- Add CSV upload service with upsert logic for meters
- Add CSV upload routes (POST /csv-upload/meters, POST /csv-upload/readings)
- Add template download endpoints for CSV format
- Create standalone upload-panel React app on port 5174
- Support concentrator_serial lookup for meter creation

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Exteban08
2026-02-03 11:08:30 +00:00
parent 7301be7544
commit 71db4219ca
18 changed files with 3708 additions and 0 deletions

View File

@@ -0,0 +1,146 @@
import { Router, Request, Response } from 'express';
import multer from 'multer';
import {
uploadMetersCSV,
uploadReadingsCSV,
generateMeterCSVTemplate,
generateReadingCSVTemplate
} from '../services/csv-upload.service';
const router = Router();
// Configure multer for memory storage (we'll process the file content directly)
const upload = multer({
storage: multer.memoryStorage(),
limits: {
fileSize: 10 * 1024 * 1024, // 10MB limit
},
fileFilter: (_req, file, cb) => {
// Accept only CSV files
if (file.mimetype === 'text/csv' ||
file.originalname.endsWith('.csv') ||
file.mimetype === 'application/vnd.ms-excel') {
cb(null, true);
} else {
cb(new Error('Solo se permiten archivos CSV'));
}
}
});
/**
* POST /api/csv-upload/meters
* Upload CSV file with meters data (upsert logic)
*/
router.post('/meters', upload.single('file'), async (req: Request, res: Response) => {
try {
if (!req.file) {
return res.status(400).json({
success: false,
message: 'No se proporcionó archivo CSV'
});
}
const csvContent = req.file.buffer.toString('utf-8');
if (!csvContent.trim()) {
return res.status(400).json({
success: false,
message: 'El archivo CSV está vacío'
});
}
const result = await uploadMetersCSV(csvContent);
return res.status(200).json({
success: true,
message: `Procesamiento completado: ${result.inserted} insertados, ${result.updated} actualizados, ${result.errors.length} errores`,
data: result
});
} catch (error) {
console.error('Error uploading meters CSV:', error);
return res.status(500).json({
success: false,
message: error instanceof Error ? error.message : 'Error al procesar el archivo CSV'
});
}
});
/**
* POST /api/csv-upload/readings
* Upload CSV file with readings data
*/
router.post('/readings', upload.single('file'), async (req: Request, res: Response) => {
try {
if (!req.file) {
return res.status(400).json({
success: false,
message: 'No se proporcionó archivo CSV'
});
}
const csvContent = req.file.buffer.toString('utf-8');
if (!csvContent.trim()) {
return res.status(400).json({
success: false,
message: 'El archivo CSV está vacío'
});
}
const result = await uploadReadingsCSV(csvContent);
return res.status(200).json({
success: true,
message: `Procesamiento completado: ${result.inserted} lecturas insertadas, ${result.errors.length} errores`,
data: result
});
} catch (error) {
console.error('Error uploading readings CSV:', error);
return res.status(500).json({
success: false,
message: error instanceof Error ? error.message : 'Error al procesar el archivo CSV'
});
}
});
/**
* GET /api/csv-upload/meters/template
* Download CSV template for meters upload
*/
router.get('/meters/template', (_req: Request, res: Response) => {
try {
const template = generateMeterCSVTemplate();
res.setHeader('Content-Type', 'text/csv');
res.setHeader('Content-Disposition', 'attachment; filename="plantilla_medidores.csv"');
return res.send(template);
} catch (error) {
console.error('Error generating meters template:', error);
return res.status(500).json({
success: false,
message: 'Error al generar la plantilla'
});
}
});
/**
* GET /api/csv-upload/readings/template
* Download CSV template for readings upload
*/
router.get('/readings/template', (_req: Request, res: Response) => {
try {
const template = generateReadingCSVTemplate();
res.setHeader('Content-Type', 'text/csv');
res.setHeader('Content-Disposition', 'attachment; filename="plantilla_lecturas.csv"');
return res.send(template);
} catch (error) {
console.error('Error generating readings template:', error);
return res.status(500).json({
success: false,
message: 'Error al generar la plantilla'
});
}
});
export default router;

View File

@@ -13,6 +13,7 @@ import roleRoutes from './role.routes';
import ttsRoutes from './tts.routes';
import readingRoutes from './reading.routes';
import bulkUploadRoutes from './bulk-upload.routes';
import csvUploadRoutes from './csv-upload.routes';
import auditRoutes from './audit.routes';
import notificationRoutes from './notification.routes';
import testRoutes from './test.routes';
@@ -145,6 +146,15 @@ router.use('/readings', readingRoutes);
*/
router.use('/bulk-upload', bulkUploadRoutes);
/**
* CSV upload routes (no authentication required):
* - POST /csv-upload/meters - Upload CSV file with meters data (upsert)
* - POST /csv-upload/readings - Upload CSV file with readings data
* - GET /csv-upload/meters/template - Download CSV template for meters
* - GET /csv-upload/readings/template - Download CSV template for readings
*/
router.use('/csv-upload', csvUploadRoutes);
/**
* Audit routes:
* - GET /audit-logs - List all audit logs (admin only)

View File

@@ -0,0 +1,523 @@
import { query } from '../config/database';
/**
* CSV Upload Service
* Handles parsing and processing of CSV files for meters and readings
*/
// ==================== INTERFACES ====================
export interface CSVMeterRow {
serial_number: string;
name: string;
project_id: string;
concentrator_serial: string;
area_name: string;
location: string;
meter_type: string;
status: string;
installation_date: string;
[key: string]: string;
}
export interface CSVReadingRow {
meter_serial: string;
reading_value: string;
received_at: string;
reading_type: string;
battery_level: string;
signal_strength: string;
[key: string]: string;
}
export interface UploadResult {
total: number;
inserted: number;
updated: number;
errors: UploadError[];
}
export interface UploadError {
row: number;
field?: string;
message: string;
data?: Record<string, unknown>;
}
// ==================== CSV PARSING ====================
/**
* Parse a CSV string into an array of objects
* @param csvContent - Raw CSV string content
* @returns Array of parsed row objects
*/
export function parseCSV<T extends Record<string, string>>(csvContent: string): T[] {
const lines = csvContent.trim().split(/\r?\n/);
if (lines.length < 2) {
return [];
}
// Parse header row
const headers = parseCSVLine(lines[0]).map(h => h.trim().toLowerCase());
// Parse data rows
const rows: T[] = [];
for (let i = 1; i < lines.length; i++) {
const line = lines[i].trim();
if (!line) continue;
const values = parseCSVLine(line);
const row: Record<string, string> = {};
headers.forEach((header, index) => {
row[header] = values[index]?.trim() || '';
});
rows.push(row as T);
}
return rows;
}
/**
* Parse a single CSV line, handling quoted values and commas within quotes
*/
function parseCSVLine(line: string): string[] {
const result: string[] = [];
let current = '';
let inQuotes = false;
for (let i = 0; i < line.length; i++) {
const char = line[i];
if (char === '"') {
if (inQuotes && line[i + 1] === '"') {
// Escaped quote
current += '"';
i++;
} else {
inQuotes = !inQuotes;
}
} else if (char === ',' && !inQuotes) {
result.push(current);
current = '';
} else {
current += char;
}
}
result.push(current);
return result;
}
// ==================== METERS UPLOAD ====================
/**
* Process CSV upload for meters (upsert logic)
* If serial_number exists -> UPDATE, if not -> INSERT
*/
export async function uploadMetersCSV(csvContent: string): Promise<UploadResult> {
const rows = parseCSV<CSVMeterRow>(csvContent);
const result: UploadResult = {
total: rows.length,
inserted: 0,
updated: 0,
errors: []
};
for (let i = 0; i < rows.length; i++) {
const row = rows[i];
const rowNumber = i + 2; // +2 because row 1 is header, and we're 0-indexed
try {
// Validate required fields
if (!row.serial_number) {
result.errors.push({
row: rowNumber,
field: 'serial_number',
message: 'El campo serial_number es requerido',
data: row as unknown as Record<string, unknown>
});
continue;
}
// Check if meter exists
const existingMeter = await query<{ id: string; project_id: string }>(
'SELECT id, project_id FROM meters WHERE serial_number = $1',
[row.serial_number]
);
if (existingMeter.rows.length > 0) {
// UPDATE existing meter
const meterId = existingMeter.rows[0].id;
await updateMeterFromCSV(meterId, row);
result.updated++;
} else {
// INSERT new meter
// Validate required fields for creation
if (!row.name) {
result.errors.push({
row: rowNumber,
field: 'name',
message: 'El campo name es requerido para crear un nuevo medidor',
data: row as unknown as Record<string, unknown>
});
continue;
}
if (!row.concentrator_serial) {
result.errors.push({
row: rowNumber,
field: 'concentrator_serial',
message: 'El campo concentrator_serial es requerido para crear un nuevo medidor',
data: row as unknown as Record<string, unknown>
});
continue;
}
// Find concentrator by serial number
const concentrator = await query<{ id: string; project_id: string }>(
'SELECT id, project_id FROM concentrators WHERE serial_number = $1',
[row.concentrator_serial]
);
if (concentrator.rows.length === 0) {
result.errors.push({
row: rowNumber,
field: 'concentrator_serial',
message: `Concentrador con serial "${row.concentrator_serial}" no encontrado`,
data: row as unknown as Record<string, unknown>
});
continue;
}
const concentratorId = concentrator.rows[0].id;
const projectId = row.project_id || concentrator.rows[0].project_id;
await createMeterFromCSV(row, concentratorId, projectId);
result.inserted++;
}
} catch (error) {
const errorMessage = error instanceof Error ? error.message : 'Error desconocido';
result.errors.push({
row: rowNumber,
message: errorMessage,
data: row as unknown as Record<string, unknown>
});
}
}
return result;
}
/**
* Create a new meter from CSV row data
*/
async function createMeterFromCSV(row: CSVMeterRow, concentratorId: string, projectId: string): Promise<void> {
const meterType = validateMeterType(row.meter_type) || 'WATER';
const status = validateStatus(row.status) || 'ACTIVE';
const installationDate = parseDate(row.installation_date);
await query(
`INSERT INTO meters (serial_number, name, project_id, concentrator_id, area_name, location, type, status, installation_date)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`,
[
row.serial_number,
row.name,
projectId,
concentratorId,
row.area_name || null,
row.location || null,
meterType,
status,
installationDate
]
);
}
/**
* Update an existing meter from CSV row data
*/
async function updateMeterFromCSV(meterId: string, row: CSVMeterRow): Promise<void> {
const updates: string[] = [];
const params: unknown[] = [];
let paramIndex = 1;
if (row.name) {
updates.push(`name = $${paramIndex}`);
params.push(row.name);
paramIndex++;
}
if (row.project_id) {
updates.push(`project_id = $${paramIndex}`);
params.push(row.project_id);
paramIndex++;
}
if (row.area_name !== undefined) {
updates.push(`area_name = $${paramIndex}`);
params.push(row.area_name || null);
paramIndex++;
}
if (row.location !== undefined) {
updates.push(`location = $${paramIndex}`);
params.push(row.location || null);
paramIndex++;
}
if (row.meter_type) {
const meterType = validateMeterType(row.meter_type);
if (meterType) {
updates.push(`type = $${paramIndex}`);
params.push(meterType);
paramIndex++;
}
}
if (row.status) {
const status = validateStatus(row.status);
if (status) {
updates.push(`status = $${paramIndex}`);
params.push(status);
paramIndex++;
}
}
if (row.installation_date) {
const date = parseDate(row.installation_date);
if (date) {
updates.push(`installation_date = $${paramIndex}`);
params.push(date);
paramIndex++;
}
}
if (updates.length === 0) {
return;
}
updates.push(`updated_at = NOW()`);
params.push(meterId);
await query(
`UPDATE meters SET ${updates.join(', ')} WHERE id = $${paramIndex}`,
params
);
}
// ==================== READINGS UPLOAD ====================
/**
* Process CSV upload for readings
*/
export async function uploadReadingsCSV(csvContent: string): Promise<UploadResult> {
const rows = parseCSV<CSVReadingRow>(csvContent);
const result: UploadResult = {
total: rows.length,
inserted: 0,
updated: 0,
errors: []
};
for (let i = 0; i < rows.length; i++) {
const row = rows[i];
const rowNumber = i + 2;
try {
// Validate required fields
if (!row.meter_serial) {
result.errors.push({
row: rowNumber,
field: 'meter_serial',
message: 'El campo meter_serial es requerido',
data: row as unknown as Record<string, unknown>
});
continue;
}
if (!row.reading_value) {
result.errors.push({
row: rowNumber,
field: 'reading_value',
message: 'El campo reading_value es requerido',
data: row as unknown as Record<string, unknown>
});
continue;
}
const readingValue = parseFloat(row.reading_value);
if (isNaN(readingValue)) {
result.errors.push({
row: rowNumber,
field: 'reading_value',
message: `Valor de lectura inválido: "${row.reading_value}"`,
data: row as unknown as Record<string, unknown>
});
continue;
}
// Find meter by serial number
const meter = await query<{ id: string }>(
'SELECT id FROM meters WHERE serial_number = $1',
[row.meter_serial]
);
if (meter.rows.length === 0) {
result.errors.push({
row: rowNumber,
field: 'meter_serial',
message: `Medidor con serial "${row.meter_serial}" no encontrado`,
data: row as unknown as Record<string, unknown>
});
continue;
}
const meterId = meter.rows[0].id;
const readingType = validateReadingType(row.reading_type) || 'MANUAL';
const receivedAt = parseDateTime(row.received_at) || new Date();
const batteryLevel = row.battery_level ? parseInt(row.battery_level, 10) : null;
const signalStrength = row.signal_strength ? parseInt(row.signal_strength, 10) : null;
// Validate battery level range
if (batteryLevel !== null && (batteryLevel < 0 || batteryLevel > 100)) {
result.errors.push({
row: rowNumber,
field: 'battery_level',
message: `Nivel de batería inválido: "${row.battery_level}" (debe ser 0-100)`,
data: row as unknown as Record<string, unknown>
});
continue;
}
// Insert reading
await query(
`INSERT INTO meter_readings (meter_id, reading_value, reading_type, battery_level, signal_strength, received_at)
VALUES ($1, $2, $3, $4, $5, $6)`,
[meterId, readingValue, readingType, batteryLevel, signalStrength, receivedAt]
);
// Update meter's last reading
await query(
`UPDATE meters SET last_reading_value = $1, last_reading_at = $2, updated_at = NOW() WHERE id = $3`,
[readingValue, receivedAt, meterId]
);
result.inserted++;
} catch (error) {
const errorMessage = error instanceof Error ? error.message : 'Error desconocido';
result.errors.push({
row: rowNumber,
message: errorMessage,
data: row as unknown as Record<string, unknown>
});
}
}
return result;
}
// ==================== CSV TEMPLATES ====================
/**
* Generate CSV template for meters upload
*/
export function generateMeterCSVTemplate(): string {
const headers = [
'serial_number',
'name',
'concentrator_serial',
'area_name',
'location',
'meter_type',
'status',
'installation_date'
];
const exampleRow = [
'MED001',
'Medidor Centro',
'CONC001',
'Zona A',
'Calle 1 #100',
'WATER',
'ACTIVE',
'2024-01-15'
];
return headers.join(',') + '\n' + exampleRow.join(',') + '\n';
}
/**
* Generate CSV template for readings upload
*/
export function generateReadingCSVTemplate(): string {
const headers = [
'meter_serial',
'reading_value',
'received_at',
'reading_type',
'battery_level',
'signal_strength'
];
const exampleRow = [
'MED001',
'1234.56',
'2024-01-20 10:30:00',
'MANUAL',
'85',
'-45'
];
return headers.join(',') + '\n' + exampleRow.join(',') + '\n';
}
// ==================== VALIDATION HELPERS ====================
const VALID_METER_TYPES = ['WATER', 'GAS', 'ELECTRIC'];
const VALID_STATUSES = ['ACTIVE', 'INACTIVE', 'OFFLINE', 'MAINTENANCE', 'ERROR'];
const VALID_READING_TYPES = ['AUTOMATIC', 'MANUAL', 'SCHEDULED'];
function validateMeterType(type?: string): string | null {
if (!type) return null;
const upperType = type.toUpperCase();
return VALID_METER_TYPES.includes(upperType) ? upperType : null;
}
function validateStatus(status?: string): string | null {
if (!status) return null;
const upperStatus = status.toUpperCase();
return VALID_STATUSES.includes(upperStatus) ? upperStatus : null;
}
function validateReadingType(type?: string): string | null {
if (!type) return null;
const upperType = type.toUpperCase();
return VALID_READING_TYPES.includes(upperType) ? upperType : null;
}
function parseDate(dateStr?: string): string | null {
if (!dateStr) return null;
// Try to parse YYYY-MM-DD format
const match = dateStr.match(/^(\d{4})-(\d{2})-(\d{2})$/);
if (match) {
return dateStr;
}
// Try DD/MM/YYYY format
const match2 = dateStr.match(/^(\d{2})\/(\d{2})\/(\d{4})$/);
if (match2) {
return `${match2[3]}-${match2[2]}-${match2[1]}`;
}
return null;
}
function parseDateTime(dateTimeStr?: string): Date | null {
if (!dateTimeStr) return null;
const date = new Date(dateTimeStr);
return isNaN(date.getTime()) ? null : date;
}