From 8d3d0296cf3e7db8bc26e32801d48729fc6aeb2f Mon Sep 17 00:00:00 2001 From: Tomek Zebrowski Date: Sat, 31 Jan 2026 15:37:59 +0100 Subject: [PATCH 1/2] feat: nested objects support --- src/dataprocessor.js | 91 +++++++++++++++++++++++++++++--------------- 1 file changed, 60 insertions(+), 31 deletions(-) diff --git a/src/dataprocessor.js b/src/dataprocessor.js index 8dfbcf7..7d6f897 100644 --- a/src/dataprocessor.js +++ b/src/dataprocessor.js @@ -117,8 +117,7 @@ class DataProcessor { let telemetryPoints = data; let fileMetadata = {}; - // OPTIONAL: Check if the first element is a metadata block - // Example format: [{ "metadata": { ... } }, { "t": 1, "s": "sig", "v": 10 }, ...] + // Check if the first element is a metadata block if (data.length > 0 && data[0].metadata) { fileMetadata = data[0].metadata; // The rest of the array is the actual telemetry data @@ -130,28 +129,28 @@ class DataProcessor { console.warn( 'Preprocessing: File contains metadata but no telemetry points.' ); - // Create an empty result structure or handle as needed } // Detect schema based on the first actual data point const schema = this.#detectSchema(telemetryPoints[0]); - const processedPoints = telemetryPoints - .map((item) => this.#applyMappingAndCleaning(item, schema)) - .filter((point) => point !== null); + // CHANGED: Use flatMap to handle 1-to-many expansion (e.g. Object -> Multiple Signals) + const processedPoints = telemetryPoints.flatMap((item) => + this.#applyMappingAndCleaning(item, schema) + ); const result = this.#transformRawData(processedPoints, fileName); // Attach the extracted metadata to the result object result.metadata = fileMetadata; - result.size = telemetryPoints.length; // Update size to reflect actual data count + result.size = telemetryPoints.length; AppState.files.push(result); projectManager.registerFile({ name: fileName, size: result.size, - metadata: result.metadata, // Register metadata with project manager if supported + metadata: result.metadata, }); return result; @@ -174,29 +173,67 @@ class DataProcessor { } /** - * Combines key mapping and data sanitization in one pass. + * Combines key mapping, object flattening, and data sanitization. + * Returns an array of points to support 1-to-many mapping. * @private */ #applyMappingAndCleaning(rawPoint, schema) { try { - const mapped = { - signal: rawPoint[schema.signal], - timestamp: Number(rawPoint[schema.timestamp]), - value: Number(rawPoint[schema.value]), - }; + const baseSignal = rawPoint[schema.signal]; + const timestamp = Number(rawPoint[schema.timestamp]); + const rawValue = rawPoint[schema.value]; + + // Validate Timestamp + if (isNaN(timestamp)) return []; + + // Clean base signal name + let prefix = ''; + if (typeof baseSignal === 'string') { + prefix = baseSignal.replace(/\n/g, ' ').trim(); + } + + // Supports GPS, Accelerometer, or any complex object structure + if (typeof rawValue === 'object' && rawValue !== null) { + const derivedPoints = []; + + for (const [key, val] of Object.entries(rawValue)) { + const numVal = Number(val); - if (typeof mapped.signal === 'string') { - mapped.signal = mapped.signal.replace(/\n/g, ' ').trim(); + // Strict check: we only want to graph numbers + if (isNaN(numVal)) continue; + + // Format Key: "latitude" -> "Latitude" + const formattedKey = key.charAt(0).toUpperCase() + key.slice(1); + + // Construct Composite Signal Name: "GPS" + "Latitude" -> "GPS Latitude" + const finalSignal = prefix + ? `${prefix}-${formattedKey}` + : formattedKey; + + derivedPoints.push({ + signal: finalSignal, + timestamp: timestamp, + value: numVal, + }); + } + return derivedPoints; } - if (isNaN(mapped.timestamp) || isNaN(mapped.value)) { - console.warn('Preprocessing: Dropping malformed point', rawPoint); - return null; + const numValue = Number(rawValue); + if (isNaN(numValue)) { + return []; } - return mapped; - } catch { - return null; + return [ + { + signal: prefix || String(baseSignal), + timestamp: timestamp, + value: numValue, + }, + ]; + } catch (e) { + console.error('Data cleaning error:', e); + return []; } } @@ -211,10 +248,8 @@ class DataProcessor { const headers = lines[0].split(',').map((h) => h.trim()); return lines.slice(1).map((line) => { - // Handle simplistic CSV splitting (warning: doesn't handle commas in quotes) const values = line.split(','); return headers.reduce((obj, header, i) => { - // Guard against row length mismatch obj[header] = values[i] !== undefined ? values[i].trim() : ''; return obj; }, {}); @@ -223,7 +258,6 @@ class DataProcessor { /** * Converts Wide Format (Time, Sig1, Sig2...) to Long Format (SensorName, Time_ms, Reading) - * This enables importing files generated by the "Export" feature. * @private */ #normalizeWideCSV(rows) { @@ -239,26 +273,21 @@ class DataProcessor { return rows; } - // 2. Detect Time Column (common variations: "Time", "Time (s)", "time") + // 2. Detect Time Column const timeKey = keys.find((k) => k.toLowerCase().includes('time')); - - // If no time column found, we can't pivot. Return original and let schema detection fail naturally. if (!timeKey) return rows; const normalized = []; - // All other keys are treated as Signals const signalKeys = keys.filter((k) => k !== timeKey); rows.forEach((row) => { const timeVal = parseFloat(row[timeKey]); if (isNaN(timeVal)) return; - // Exports are usually in Seconds (e.g. 0.1), internals need Milliseconds (e.g. 100) const timestampMs = timeKey.includes('(s)') ? timeVal * 1000 : timeVal; signalKeys.forEach((sigKey) => { const val = row[sigKey]; - // Only add if value exists and is not empty string if (val !== '' && val !== null && val !== undefined) { normalized.push({ SensorName: sigKey, From 43cf7fdb29939b5b592af452e88b40897882046f Mon Sep 17 00:00:00 2001 From: Tomek Zebrowski Date: Sat, 31 Jan 2026 15:43:06 +0100 Subject: [PATCH 2/2] feat: add tests for nested objects --- tests/dataprocessor.test.js | 93 +++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/tests/dataprocessor.test.js b/tests/dataprocessor.test.js index fbe1661..7b9bc1d 100644 --- a/tests/dataprocessor.test.js +++ b/tests/dataprocessor.test.js @@ -599,3 +599,96 @@ invalid,2000 expect(result.availableSignals).toHaveLength(0); }); }); + +describe('DataProcessor: Nested Object Support', () => { + beforeEach(() => { + jest.clearAllMocks(); + AppState.files = []; + DOM.get = jest.fn((id) => document.getElementById(id)); + }); + + test('should flatten nested objects into composite signals', () => { + const rawData = [ + { + t: 1000, + s: 'GPS', + v: { + latitude: 54.1, + longitude: 16.2, + altitude: 85, + }, + }, + ]; + + const result = dataProcessor.process(rawData, 'gps.json'); + + // Expect multiple signals created from one point + expect(result.availableSignals).toEqual( + expect.arrayContaining(['GPS-Latitude', 'GPS-Longitude', 'GPS-Altitude']) + ); + + // Check values + expect(result.signals['GPS-Latitude'][0].y).toBe(54.1); + expect(result.signals['GPS-Longitude'][0].y).toBe(16.2); + expect(result.signals['GPS-Altitude'][0].y).toBe(85); + }); + + test('should capitalize keys in nested objects', () => { + const rawData = [{ t: 1000, s: 'IMU', v: { accelX: 0.5, gyroZ: 0.1 } }]; + + const result = dataProcessor.process(rawData, 'imu.json'); + + // "accelX" -> "IMU AccelX" + expect(result.availableSignals).toContain('IMU-AccelX'); + expect(result.availableSignals).toContain('IMU-GyroZ'); + }); + + test('should handle objects without a prefix signal name', () => { + // Case where 's' is empty or missing, but v is an object + // Though usually schema requires 's', let's simulate empty string + const rawData = [{ t: 1000, s: '', v: { speed: 50, rpm: 2000 } }]; + + const result = dataProcessor.process(rawData, 'noprefix.json'); + + // "speed" -> "Speed" (since prefix is empty) + expect(result.availableSignals).toContain('Speed'); + expect(result.availableSignals).toContain('Rpm'); + expect(result.signals['Speed'][0].y).toBe(50); + }); + + test('should ignore non-numeric values inside nested objects', () => { + const rawData = [ + { + t: 1000, + s: 'Status', + v: { + code: 200, + message: 'OK', // String -> should be ignored + isValid: true, // Boolean -> true=1, so usually included + }, + }, + ]; + + const result = dataProcessor.process(rawData, 'status.json'); + + expect(result.availableSignals).toContain('Status-Code'); + expect(result.availableSignals).not.toContain('Status-Message'); + + // Check boolean handling: true casts to 1 + expect(result.availableSignals).toContain('Status-IsValid'); + expect(result.signals['Status-IsValid'][0].y).toBe(1); + }); + + test('should handle mixed flat and nested data in the same file', () => { + const rawData = [ + { t: 1000, s: 'RPM', v: 2000 }, + { t: 1000, s: 'GPS', v: { lat: 50, lon: 10 } }, + ]; + + const result = dataProcessor.process(rawData, 'mixed.json'); + + expect(result.availableSignals).toContain('RPM'); + expect(result.availableSignals).toContain('GPS-Lat'); + expect(result.availableSignals).toContain('GPS-Lon'); + }); +});