diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 504c45f..df198bf 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -11,7 +11,7 @@ on: permissions: id-token: write # Required for OIDC - contents: read + contents: write # Required for creating releases jobs: publish: @@ -35,4 +35,15 @@ jobs: - name: publish module run: just publish + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + + - name: Create GitHub Release + if: startsWith(github.ref, 'refs/tags/v') + run: | + gh release create ${{ github.ref_name }} \ + --title "Release ${{ github.ref_name }}" \ + --generate-notes + env: + GH_TOKEN: ${{ github.token }} diff --git a/README.md b/README.md index f4353fe..76c6ca8 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,71 @@ npm install @metapages/dataref ## Quick Start +### Primary API (Recommended) + +The simplest way to work with binary data in JSON is to use the primary serialize/deserialize functions: + +```typescript +import { serializeDataRefs, deserializeDataRefs } from "@metapages/dataref"; + +// Automatically converts ALL binary types to datarefs +const data = { + uint8: new Uint8Array([1, 2, 3]), + float32: new Float32Array([1.1, 2.2, 3.3]), + buffer: new Uint8Array([255, 128, 0]).buffer, + blob: new Blob(["hello"], { type: "text/plain" }), + file: new File(["content"], "test.txt"), + string: "regular data", + nested: { + array: new Int16Array([100, -100]) + } +}; + +// Serialize: Convert all binary types to dataref strings +const serialized = await serializeDataRefs(data); +// All binary types are now dataref strings, safe for JSON.stringify() + +// Store or transmit +const json = JSON.stringify(serialized); + +// Deserialize: Convert all datarefs back to original types +const deserialized = await deserializeDataRefs(JSON.parse(json)); +// All binary types are restored to their original types +``` + +**With upload/download support for large objects:** + +```typescript +// Serialize with upload for large data +const serialized = await serializeDataRefs(data, { + uploadFn: async (data, metadata) => { + // Upload to your storage service + const response = await fetch("/upload", { + method: "POST", + body: data + }); + const { url } = await response.json(); + return url; + }, + maxSizeBytes: 10240 // Upload objects > 10KB +}); + +// Deserialize with custom download +const deserialized = await deserializeDataRefs(serialized, { + downloadFn: async (url) => { + const response = await fetch(url); + return response.arrayBuffer(); + } +}); +``` + +**What gets converted:** +- ✅ All TypedArray types (Int8Array, Uint8Array, Float32Array, etc.) +- ✅ ArrayBuffer +- ✅ Blob (with MIME type preservation) +- ✅ File (with name preservation) +- ❌ Regular data (strings, numbers, objects, arrays) — unchanged + ### The Core Workflow: Encode → Serialize → Decode The most common use case is encoding complex data for JSON serialization, then decoding it later: @@ -179,6 +244,8 @@ The library supports all JavaScript data types: | ArrayBuffer | Base64 binary | `bufferToDataUrl(buffer)` | | Uint8Array | Base64 binary | `bufferToDataUrl(uint8Array)` | | TypedArrays | Base64 with type | `typedArrayToDataUrl(array, type)` | +| Blob | Base64 with MIME type | `blobToDataUrl(blob)` | +| File | Base64 with name | `fileToDataUrl(file)` | | URL reference | URL-encoded URI | `urlToDataUrl("https://...")` | **Supported TypedArray types:** @@ -326,6 +393,81 @@ isDataRef("regular string"); // false ## API Reference +### Primary Functions (Recommended) + +#### `serializeDataRefs(json: T, options?: SerializeOptions): Promise` + +Primary serialize function that automatically converts all binary types in a JSON object to dataref strings. + +**Converts:** +- TypedArrays (Int8Array, Uint8Array, Float32Array, etc.) → dataref strings +- ArrayBuffer → dataref strings +- Blob → dataref strings (with MIME type preservation) +- File → dataref strings (with name preservation) +- Regular data (strings, numbers, objects, arrays) → unchanged + +**Options:** +```typescript +interface SerializeOptions { + uploadFn?: (data: Blob | ArrayBuffer, metadata: { + type: string; + size: number; + mimeType?: string; + }) => Promise; + maxSizeBytes?: number; // If provided, objects > size get uploaded +} +``` + +**Example:** +```typescript +const serialized = await serializeDataRefs({ + array: new Uint8Array([1, 2, 3]), + blob: new Blob(["test"], { type: "text/plain" }), + string: "unchanged" +}); +``` + +**With upload:** +```typescript +const serialized = await serializeDataRefs(data, { + uploadFn: async (data, metadata) => { + // Upload and return URL + return "https://storage.example.com/..."; + }, + maxSizeBytes: 10240 // Upload if > 10KB +}); +``` + +#### `deserializeDataRefs(json: T, options?: DeserializeOptions): Promise` + +Primary deserialize function that converts all dataref strings in a JSON object back to their original binary types. + +**Options:** +```typescript +interface DeserializeOptions { + fetchOptions?: RequestInit; // For URL-based datarefs + downloadFn?: (url: string) => Promise; // Custom download +} +``` + +**Example:** +```typescript +const deserialized = await deserializeDataRefs(serialized); +// All datarefs are converted back to original types +``` + +**With custom download:** +```typescript +const deserialized = await deserializeDataRefs(serialized, { + downloadFn: async (url) => { + const response = await fetch(url, { + headers: { "Authorization": "Bearer token" } + }); + return response.arrayBuffer(); + } +}); +``` + ### Encoding Functions (to Data URL) #### `textToDataUrl(text: string): DataUrl` @@ -360,6 +502,15 @@ typedArrayToDataUrl(new Float32Array([1.1, 2.2]), "Float32Array"); // => "data:application/octet-stream;type=Float32Array;base64,..." ``` +#### `blobToDataUrl(blob: Blob): Promise` +Converts a Blob to a data URL with MIME type preservation. + +```typescript +const blob = new Blob(["content"], { type: "text/plain" }); +await blobToDataUrl(blob); +// => "data:text/plain;type=Blob;base64,..." +``` + #### `urlToDataUrl(url: string, fetchOptions?: RequestInit): Promise` Creates a URL reference or fetches and encodes URL content. @@ -389,6 +540,14 @@ Decodes a data URL to an ArrayBuffer. #### `dataUrlToTypedArray(dataUrl: DataUrl, fetchOptions?: RequestInit): Promise` Decodes a data URL to a TypedArray with type preservation. +#### `dataUrlToBlob(dataUrl: DataUrl, fetchOptions?: RequestInit): Promise` +Decodes a data URL to a Blob with MIME type preservation. + +```typescript +const blob = await dataUrlToBlob(dataUrl); +// => Blob { type: "text/plain", size: 7, ... } +``` + #### `dataUrlToFile(dataUrl: DataUrl, name?: string, fetchOptions?: RequestInit): Promise` Converts a data URL to a File object. diff --git a/examples/serialize-example.ts b/examples/serialize-example.ts new file mode 100644 index 0000000..abd2d35 --- /dev/null +++ b/examples/serialize-example.ts @@ -0,0 +1,165 @@ +/** + * Example demonstrating the primary serialize/deserialize API + */ +import { + serializeDataRefs, + deserializeDataRefs, + type SerializeOptions, +} from "../src/index"; + +async function basicExample() { + console.log("=== Basic Serialize/Deserialize Example ===\n"); + + // Create data with various binary types + const originalData = { + metadata: { + name: "Sensor Data", + timestamp: new Date().toISOString(), + }, + readings: { + temperature: new Float32Array([20.5, 21.2, 22.1, 21.8]), + humidity: new Uint8Array([65, 68, 70, 69]), + rawData: new Int16Array([-100, -50, 0, 50, 100]), + }, + buffer: new Uint8Array([0xff, 0x00, 0x80]).buffer, + blob: new Blob(["Sample text content"], { type: "text/plain" }), + file: new File(["Document content"], "report.txt", { type: "text/plain" }), + regularString: "This stays as a regular string", + regularNumber: 42, + }; + + console.log("Original data structure:"); + console.log("- temperature:", originalData.readings.temperature.constructor.name); + console.log("- humidity:", originalData.readings.humidity.constructor.name); + console.log("- rawData:", originalData.readings.rawData.constructor.name); + console.log("- buffer:", originalData.buffer.constructor.name); + console.log("- blob:", originalData.blob.constructor.name); + console.log("- file:", originalData.file.constructor.name); + console.log(); + + // Serialize - convert all binary types to dataref strings + console.log("Serializing..."); + const serialized = await serializeDataRefs(originalData); + + console.log("\nSerialized data (binary types are now strings):"); + console.log("- temperature:", typeof serialized.readings.temperature, + serialized.readings.temperature.substring(0, 50) + "..."); + console.log("- humidity:", typeof serialized.readings.humidity, + serialized.readings.humidity.substring(0, 50) + "..."); + console.log("- buffer:", typeof serialized.buffer, + serialized.buffer.substring(0, 50) + "..."); + console.log("- regularString:", typeof serialized.regularString, "=", serialized.regularString); + console.log(); + + // Can now safely stringify as JSON + const jsonString = JSON.stringify(serialized); + console.log("JSON size:", jsonString.length, "bytes"); + console.log(); + + // Deserialize - convert all datarefs back to original types + console.log("Deserializing..."); + const deserialized = await deserializeDataRefs(JSON.parse(jsonString)); + + console.log("\nDeserialized data (restored to original types):"); + console.log("- temperature:", deserialized.readings.temperature.constructor.name, "=", + Array.from(deserialized.readings.temperature)); + console.log("- humidity:", deserialized.readings.humidity.constructor.name, "=", + Array.from(deserialized.readings.humidity)); + console.log("- rawData:", deserialized.readings.rawData.constructor.name, "=", + Array.from(deserialized.readings.rawData)); + console.log("- buffer:", deserialized.buffer.constructor.name); + console.log("- blob:", deserialized.blob.constructor.name, "type:", deserialized.blob.type); + console.log("- regularString:", deserialized.regularString); + console.log("- regularNumber:", deserialized.regularNumber); + console.log(); +} + +async function uploadDownloadExample() { + console.log("=== Upload/Download Example ===\n"); + + // Simulate cloud storage + const mockStorage = new Map(); + let uploadCounter = 0; + + // Mock upload function + const mockUploadFn = async ( + data: Blob | ArrayBuffer, + metadata: { type: string; size: number; mimeType?: string } + ): Promise => { + const buffer = data instanceof Blob ? await data.arrayBuffer() : data; + const url = `https://storage.example.com/uploads/file-${++uploadCounter}`; + mockStorage.set(url, buffer); + console.log(` ✓ Uploaded ${metadata.size} bytes (${metadata.type}) to ${url}`); + return url; + }; + + // Mock download function + const mockDownloadFn = async (url: string): Promise => { + const data = mockStorage.get(url); + if (!data) { + throw new Error(`URL not found: ${url}`); + } + console.log(` ✓ Downloaded ${data.byteLength} bytes from ${url}`); + return data; + }; + + // Create data with a large array that should be uploaded + const largeArray = new Float32Array(50000); // 200KB + for (let i = 0; i < largeArray.length; i++) { + largeArray[i] = Math.sin(i / 100); + } + + const data = { + smallData: new Uint8Array([1, 2, 3, 4, 5]), + largeData: largeArray, + metadata: { + description: "Signal processing results", + samples: largeArray.length, + }, + }; + + // Serialize with upload for large objects + console.log("Serializing with upload (threshold: 10KB)..."); + const serialized = await serializeDataRefs(data, { + uploadFn: mockUploadFn, + maxSizeBytes: 10240, // 10KB threshold + }); + console.log(); + + console.log("Result:"); + console.log("- smallData: inline dataref (small, not uploaded)"); + console.log("- largeData: URL dataref (uploaded)"); + console.log(); + + // JSON is now much smaller + const jsonString = JSON.stringify(serialized); + console.log("JSON size:", jsonString.length, "bytes (instead of ~200KB)"); + console.log(); + + // Deserialize with download + console.log("Deserializing with download..."); + const deserialized = await deserializeDataRefs(serialized, { + downloadFn: mockDownloadFn, + }); + console.log(); + + console.log("Verification:"); + console.log("- smallData matches:", + JSON.stringify(Array.from(deserialized.smallData)) === JSON.stringify([1, 2, 3, 4, 5])); + console.log("- largeData length:", deserialized.largeData.length); + console.log("- largeData type:", deserialized.largeData.constructor.name); + console.log("- First 5 values:", Array.from(deserialized.largeData.slice(0, 5))); + console.log(); +} + +// Run examples +(async () => { + try { + await basicExample(); + await uploadDownloadExample(); + console.log("✓ All examples completed successfully!"); + } catch (error) { + console.error("Error:", error); + process.exit(1); + } +})(); diff --git a/package.json b/package.json index 37ee764..82e0a34 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@metapages/dataref", - "version": "2.0.1", + "version": "2.1.0", "author": "Dion Whitehead ", "repository": { "type": "git", diff --git a/src/index.ts b/src/index.ts index 73bffe7..c15252c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,11 +2,16 @@ // v1 is kept internal for backwards compatibility only export { + // Primary API (recommended for most use cases) + serializeDataRefs, + deserializeDataRefs, + // Core encoding functions textToDataUrl, jsonToDataUrl, bufferToDataUrl, typedArrayToDataUrl, + blobToDataUrl, urlToDataUrl, fileToDataUrl, @@ -15,6 +20,7 @@ export { dataUrlToJson, dataUrlToBuffer, dataUrlToTypedArray, + dataUrlToBlob, dataUrlToUrl, dataUrlToFile, @@ -33,6 +39,8 @@ export { type DataUrl, type TypedArrayType, type DataRefTypedArray, + type SerializeOptions, + type DeserializeOptions, MIME_TYPES, } from "./v2/types"; diff --git a/src/test/serialize.test.ts b/src/test/serialize.test.ts new file mode 100644 index 0000000..ed60639 --- /dev/null +++ b/src/test/serialize.test.ts @@ -0,0 +1,467 @@ +import { describe, it, expect } from "vitest"; +import { + serializeDataRefs, + deserializeDataRefs, + type SerializeOptions, +} from "../index"; + +describe("Primary Serialize/Deserialize API", () => { + describe("serializeDataRefs", () => { + it("should convert TypedArray to dataref string", async () => { + const input = { + data: new Uint8Array([1, 2, 3, 4, 5]), + name: "test", + }; + + const serialized = await serializeDataRefs(input); + + expect(serialized.name).toBe("test"); + expect(typeof serialized.data).toBe("string"); + expect(serialized.data).toMatch(/^data:/); + expect(serialized.data).toContain("type=Uint8Array"); + }); + + it("should convert ArrayBuffer to dataref string", async () => { + const buffer = new Uint8Array([10, 20, 30]).buffer; + const input = { + buffer, + other: "value", + }; + + const serialized = await serializeDataRefs(input); + + expect(serialized.other).toBe("value"); + expect(typeof serialized.buffer).toBe("string"); + expect(serialized.buffer).toMatch(/^data:/); + }); + + it("should convert Blob to dataref string", async () => { + const blob = new Blob(["hello blob"], { type: "text/plain" }); + const input = { + blob, + count: 42, + }; + + const serialized = await serializeDataRefs(input); + + expect(serialized.count).toBe(42); + expect(typeof serialized.blob).toBe("string"); + expect(serialized.blob).toMatch(/^data:/); + expect(serialized.blob).toContain("text/plain"); + }); + + it("should convert File to dataref string", async () => { + const file = new File(["file content"], "test.txt", { type: "text/plain" }); + const input = { + file, + id: 123, + }; + + const serialized = await serializeDataRefs(input); + + expect(serialized.id).toBe(123); + expect(typeof serialized.file).toBe("string"); + expect(serialized.file).toMatch(/^data:/); + }); + + it("should handle mixed binary types", async () => { + const input = { + array: new Float32Array([1.1, 2.2, 3.3]), + buffer: new Uint8Array([255, 0, 128]).buffer, + blob: new Blob(["test"], { type: "text/plain" }), + file: new File(["content"], "doc.txt"), + string: "regular string", + number: 42, + nested: { + innerArray: new Int16Array([100, -100]), + innerString: "nested value", + }, + }; + + const serialized = await serializeDataRefs(input); + + // Check regular values are unchanged + expect(serialized.string).toBe("regular string"); + expect(serialized.number).toBe(42); + expect(serialized.nested.innerString).toBe("nested value"); + + // Check binary types are converted + expect(typeof serialized.array).toBe("string"); + expect(serialized.array).toContain("type=Float32Array"); + + expect(typeof serialized.buffer).toBe("string"); + expect(serialized.buffer).toMatch(/^data:/); + + expect(typeof serialized.blob).toBe("string"); + expect(serialized.blob).toContain("text/plain"); + + expect(typeof serialized.file).toBe("string"); + expect(serialized.file).toMatch(/^data:/); + + expect(typeof serialized.nested.innerArray).toBe("string"); + expect(serialized.nested.innerArray).toContain("type=Int16Array"); + }); + + it("should handle arrays with binary types", async () => { + const input = { + items: [ + new Uint8Array([1, 2, 3]), + "string", + new Blob(["blob"]), + 42, + new Float32Array([1.5, 2.5]), + ], + }; + + const serialized = await serializeDataRefs(input); + + expect(serialized.items.length).toBe(5); + expect(typeof serialized.items[0]).toBe("string"); + expect(serialized.items[0]).toContain("type=Uint8Array"); + expect(serialized.items[1]).toBe("string"); + expect(typeof serialized.items[2]).toBe("string"); + expect(serialized.items[2]).toMatch(/^data:/); + expect(serialized.items[3]).toBe(42); + expect(typeof serialized.items[4]).toBe("string"); + expect(serialized.items[4]).toContain("type=Float32Array"); + }); + + it("should handle empty/null/undefined values", async () => { + const input = { + empty: new Uint8Array([]), + nullValue: null, + undefinedValue: undefined, + emptyString: "", + }; + + const serialized = await serializeDataRefs(input); + + expect(typeof serialized.empty).toBe("string"); + expect(serialized.nullValue).toBe(null); + expect(serialized.undefinedValue).toBe(undefined); + expect(serialized.emptyString).toBe(""); + }); + + it("should not convert existing dataref strings", async () => { + const existingDataRef = "data:text/plain;charset=utf-8,existing"; + const input = { + existing: existingDataRef, + new: new Uint8Array([1, 2, 3]), + }; + + const serialized = await serializeDataRefs(input); + + expect(serialized.existing).toBe(existingDataRef); + expect(typeof serialized.new).toBe("string"); + expect(serialized.new).not.toBe(existingDataRef); + }); + + it("should support upload for large binary data", async () => { + const largeArray = new Uint8Array(100000); // 100KB + for (let i = 0; i < largeArray.length; i++) { + largeArray[i] = i % 256; + } + + const mockUploadFn = async ( + data: Blob | ArrayBuffer, + metadata: { type: string; size: number } + ): Promise => { + expect(metadata.size).toBeGreaterThan(10000); + expect(metadata.type).toBe("Uint8Array"); + return `https://storage.example.com/uploads/${Date.now()}`; + }; + + const input = { + largeData: largeArray, + smallData: new Uint8Array([1, 2, 3]), + }; + + const options: SerializeOptions = { + uploadFn: mockUploadFn, + maxSizeBytes: 10240, // 10KB threshold + }; + + const serialized = await serializeDataRefs(input, options); + + // Large data should be uploaded (URL dataref) + expect(typeof serialized.largeData).toBe("string"); + expect(serialized.largeData).toContain("text/x-uri"); + // URL is encoded in the dataref + expect(decodeURIComponent(serialized.largeData)).toContain("https://storage.example.com"); + + // Small data should be inline + expect(typeof serialized.smallData).toBe("string"); + expect(serialized.smallData).toContain("type=Uint8Array"); + expect(serialized.smallData).not.toContain("text/x-uri"); + }); + + it("should preserve MIME type in upload metadata for Blobs", async () => { + const blob = new Blob(["content"], { type: "application/json" }); + + const mockUploadFn = async ( + data: Blob | ArrayBuffer, + metadata: { type: string; size: number; mimeType?: string } + ): Promise => { + expect(metadata.mimeType).toBe("application/json"); + expect(metadata.type).toBe("Blob"); + return "https://storage.example.com/blob"; + }; + + const input = { blob }; + + const options: SerializeOptions = { + uploadFn: mockUploadFn, + maxSizeBytes: 1, // Force upload + }; + + const serialized = await serializeDataRefs(input, options); + expect(serialized.blob).toContain("mimeType=application/json"); + }); + }); + + describe("deserializeDataRefs", () => { + it("should convert dataref string back to TypedArray", async () => { + const original = new Uint8Array([1, 2, 3, 4, 5]); + const serialized = await serializeDataRefs({ data: original }); + const deserialized = await deserializeDataRefs(serialized); + + expect(deserialized.data).toEqual(original); + expect(deserialized.data).toBeInstanceOf(Uint8Array); + }); + + it("should convert dataref string back to ArrayBuffer", async () => { + const original = new Uint8Array([10, 20, 30]).buffer; + const serialized = await serializeDataRefs({ buffer: original }); + const deserialized = await deserializeDataRefs(serialized); + + expect(deserialized.buffer).toBeInstanceOf(ArrayBuffer); + expect(new Uint8Array(deserialized.buffer)).toEqual(new Uint8Array([10, 20, 30])); + }); + + it("should convert dataref string back to Blob", async () => { + const original = new Blob(["hello blob"], { type: "text/plain" }); + const serialized = await serializeDataRefs({ blob: original }); + const deserialized = await deserializeDataRefs(serialized); + + expect(deserialized.blob).toBeInstanceOf(Blob); + expect(deserialized.blob.type).toBe("text/plain"); + expect(await deserialized.blob.text()).toBe("hello blob"); + }); + + it("should perform full round-trip for all binary types", async () => { + const original = { + uint8: new Uint8Array([1, 2, 3]), + int16: new Int16Array([-100, 0, 100]), + float32: new Float32Array([1.1, 2.2, 3.3]), + buffer: new Uint8Array([255, 128, 0]).buffer, + blob: new Blob(["blob content"], { type: "text/plain" }), + string: "regular", + number: 42, + nested: { + array: new Uint32Array([1000, 2000]), + value: "nested", + }, + }; + + const serialized = await serializeDataRefs(original); + const deserialized = await deserializeDataRefs(serialized); + + expect(deserialized.uint8).toEqual(original.uint8); + expect(deserialized.int16).toEqual(original.int16); + expect(deserialized.float32).toEqual(original.float32); + expect(new Uint8Array(deserialized.buffer)).toEqual(new Uint8Array([255, 128, 0])); + expect(deserialized.blob).toBeInstanceOf(Blob); + expect(await deserialized.blob.text()).toBe("blob content"); + expect(deserialized.string).toBe("regular"); + expect(deserialized.number).toBe(42); + expect(deserialized.nested.array).toEqual(original.nested.array); + expect(deserialized.nested.value).toBe("nested"); + }); + + it("should handle arrays with mixed types", async () => { + const original = { + items: [ + new Uint8Array([1, 2, 3]), + "string", + 42, + new Float32Array([1.5]), + ], + }; + + const serialized = await serializeDataRefs(original); + const deserialized = await deserializeDataRefs(serialized); + + expect(deserialized.items[0]).toEqual(original.items[0]); + expect(deserialized.items[1]).toBe("string"); + expect(deserialized.items[2]).toBe(42); + expect(deserialized.items[3]).toEqual(original.items[3]); + }); + + it("should handle empty values", async () => { + const original = { + empty: new Uint8Array([]), + nullValue: null, + undefinedValue: undefined, + }; + + const serialized = await serializeDataRefs(original); + const deserialized = await deserializeDataRefs(serialized); + + expect(deserialized.empty).toEqual(new Uint8Array([])); + expect(deserialized.nullValue).toBe(null); + expect(deserialized.undefinedValue).toBe(undefined); + }); + + it("should preserve TypedArray types exactly", async () => { + const types = { + int8: new Int8Array([-128, 0, 127]), + uint8: new Uint8Array([0, 128, 255]), + int16: new Int16Array([-32768, 0, 32767]), + uint16: new Uint16Array([0, 32768, 65535]), + int32: new Int32Array([-2147483648, 0, 2147483647]), + uint32: new Uint32Array([0, 2147483648, 4294967295]), + float32: new Float32Array([1.1, 2.2, 3.3]), + float64: new Float64Array([1.1, 2.2, 3.3]), + uint8clamped: new Uint8ClampedArray([0, 128, 255]), + }; + + const serialized = await serializeDataRefs(types); + const deserialized = await deserializeDataRefs(serialized); + + expect(deserialized.int8.constructor.name).toBe("Int8Array"); + expect(deserialized.uint8.constructor.name).toBe("Uint8Array"); + expect(deserialized.int16.constructor.name).toBe("Int16Array"); + expect(deserialized.uint16.constructor.name).toBe("Uint16Array"); + expect(deserialized.int32.constructor.name).toBe("Int32Array"); + expect(deserialized.uint32.constructor.name).toBe("Uint32Array"); + expect(deserialized.float32.constructor.name).toBe("Float32Array"); + expect(deserialized.float64.constructor.name).toBe("Float64Array"); + expect(deserialized.uint8clamped.constructor.name).toBe("Uint8ClampedArray"); + }); + + it("should support custom download function for URL datarefs", async () => { + // Simulate uploaded data + const originalData = new Uint8Array([1, 2, 3, 4, 5]); + const mockStorage = new Map(); + + const uploadUrl = "https://storage.example.com/data123"; + mockStorage.set(uploadUrl, originalData.buffer); + + // Create a URL dataref manually + const urlDataRef = `data:text/x-uri;type=Uint8Array;charset=utf-8,${encodeURIComponent(uploadUrl)}`; + + const input = { + uploaded: urlDataRef, + inline: "data:application/octet-stream;type=Uint8Array;base64,AQIDBAU=", + }; + + const mockDownloadFn = async (url: string): Promise => { + const data = mockStorage.get(url); + if (!data) { + throw new Error(`URL not found: ${url}`); + } + return data; + }; + + const deserialized = await deserializeDataRefs(input, { + downloadFn: mockDownloadFn, + }); + + expect(deserialized.uploaded).toEqual(originalData); + expect(deserialized.inline).toEqual(new Uint8Array([1, 2, 3, 4, 5])); + }); + + it("should handle upload and download round-trip", async () => { + const mockStorage = new Map(); + let uploadCounter = 0; + + const mockUploadFn = async ( + data: Blob | ArrayBuffer + ): Promise => { + const buffer = data instanceof Blob ? await data.arrayBuffer() : data; + const url = `https://storage.example.com/file${++uploadCounter}`; + mockStorage.set(url, buffer); + return url; + }; + + const mockDownloadFn = async (url: string): Promise => { + const data = mockStorage.get(url); + if (!data) { + throw new Error(`URL not found: ${url}`); + } + return data; + }; + + const original = { + smallArray: new Uint8Array([1, 2, 3]), + largeArray: new Uint8Array(20000).fill(42), + string: "unchanged", + }; + + // Serialize with upload + const serialized = await serializeDataRefs(original, { + uploadFn: mockUploadFn, + maxSizeBytes: 1000, + }); + + // Small array should be inline, large should be URL + expect(serialized.smallArray).toContain("base64"); + expect(serialized.largeArray).toContain("text/x-uri"); + expect(serialized.largeArray).toContain("storage.example.com"); + + // Deserialize with download + const deserialized = await deserializeDataRefs(serialized, { + downloadFn: mockDownloadFn, + }); + + expect(deserialized.smallArray).toEqual(original.smallArray); + expect(deserialized.largeArray).toEqual(original.largeArray); + expect(deserialized.string).toBe("unchanged"); + }); + }); + + describe("Edge cases", () => { + it("should handle deeply nested structures", async () => { + const original = { + level1: { + level2: { + level3: { + level4: { + data: new Uint8Array([99, 100, 101]), + }, + }, + }, + }, + }; + + const serialized = await serializeDataRefs(original); + const deserialized = await deserializeDataRefs(serialized); + + expect(deserialized.level1.level2.level3.level4.data).toEqual( + original.level1.level2.level3.level4.data + ); + }); + + it("should handle objects with no binary types", async () => { + const original = { + name: "test", + count: 42, + nested: { + value: "string", + array: [1, 2, 3], + }, + }; + + const serialized = await serializeDataRefs(original); + const deserialized = await deserializeDataRefs(serialized); + + expect(deserialized).toEqual(original); + }); + + it("should return input unchanged if no binary types found", async () => { + const original = { a: 1, b: "test", c: [1, 2, 3] }; + const serialized = await serializeDataRefs(original); + expect(serialized).toBe(original); // Should be same reference + }); + }); +}); diff --git a/src/test/v2.test.ts b/src/test/v2.test.ts index 806cb25..2394a1d 100644 --- a/src/test/v2.test.ts +++ b/src/test/v2.test.ts @@ -4,10 +4,12 @@ import { jsonToDataUrl, bufferToDataUrl, typedArrayToDataUrl, + blobToDataUrl, dataUrlToText, dataUrlToJson, dataUrlToBuffer, dataUrlToTypedArray, + dataUrlToBlob, isDataUrl, urlToDataUrl, dataUrlToUrl, @@ -485,4 +487,77 @@ describe("v2 DataRef - Basic Type Conversions", () => { expect((result.items[2] as any).key).toEqual({ nested: { value: 123 } }); }); }); + + describe("Blob conversions", () => { + it("should convert Blob to data URL and back", async () => { + const originalBlob = new Blob(["Hello, Blob!"], { type: "text/plain" }); + const dataUrl = await blobToDataUrl(originalBlob); + + expect(isDataUrl(dataUrl)).toBe(true); + expect(dataUrl).toContain("text/plain"); + + const decodedBlob = await dataUrlToBlob(dataUrl); + expect(decodedBlob.type).toBe("text/plain"); + expect(decodedBlob.size).toBe(originalBlob.size); + + const decodedText = await decodedBlob.text(); + expect(decodedText).toBe("Hello, Blob!"); + }); + + it("should handle Blob with no type", async () => { + const originalBlob = new Blob(["No type"]); + const dataUrl = await blobToDataUrl(originalBlob); + + const decodedBlob = await dataUrlToBlob(dataUrl); + const decodedText = await decodedBlob.text(); + expect(decodedText).toBe("No type"); + }); + + it("should handle empty Blob", async () => { + const originalBlob = new Blob([]); + const dataUrl = await blobToDataUrl(originalBlob); + + const decodedBlob = await dataUrlToBlob(dataUrl); + expect(decodedBlob.size).toBe(0); + }); + + it("should preserve MIME type for various Blob types", async () => { + const testCases = [ + { type: "application/json", content: '{"test": true}' }, + { type: "text/html", content: "" }, + { type: "application/octet-stream", content: "binary data" }, + ]; + + for (const { type, content } of testCases) { + const blob = new Blob([content], { type }); + const dataUrl = await blobToDataUrl(blob); + const decoded = await dataUrlToBlob(dataUrl); + + expect(decoded.type).toBe(type); + expect(await decoded.text()).toBe(content); + } + }); + + it("should handle binary Blob data", async () => { + const binaryData = new Uint8Array([0, 1, 2, 255, 254, 253]); + const originalBlob = new Blob([binaryData], { type: "application/octet-stream" }); + const dataUrl = await blobToDataUrl(originalBlob); + + const decodedBlob = await dataUrlToBlob(dataUrl); + const decodedBuffer = await decodedBlob.arrayBuffer(); + const decodedArray = new Uint8Array(decodedBuffer); + + expect(decodedArray).toEqual(binaryData); + }); + + it("should handle large Blob", async () => { + const largeContent = "x".repeat(10000); + const originalBlob = new Blob([largeContent], { type: "text/plain" }); + const dataUrl = await blobToDataUrl(originalBlob); + + const decodedBlob = await dataUrlToBlob(dataUrl); + expect(decodedBlob.size).toBe(originalBlob.size); + expect(await decodedBlob.text()).toBe(largeContent); + }); + }); }); diff --git a/src/v2/dataref.ts b/src/v2/dataref.ts index bc49cef..75d5c40 100644 --- a/src/v2/dataref.ts +++ b/src/v2/dataref.ts @@ -55,6 +55,14 @@ export const typedArrayToDataUrl = ( return `data:${MIME_TYPES.TYPED_ARRAY}${type};base64,${base64}`; }; +export const blobToDataUrl = async (blob: Blob): Promise => { + const buffer = await blob.arrayBuffer(); + const bytes = new Uint8Array(buffer); + const base64 = btoa(String.fromCharCode(...bytes)); + const mimeType = blob.type || MIME_TYPES.OCTET_STREAM; + return `data:${mimeType};base64,${base64}`; +}; + // Update core conversion functions to handle URLs export const dataUrlToBuffer = async ( dataUrl: DataUrl, @@ -136,6 +144,15 @@ export const dataUrlToTypedArray = async ( return new TypedArray(buffer) as T; }; +export const dataUrlToBlob = async ( + dataUrl: DataUrl, + fetchOptions?: RequestInit +): Promise => { + const mimeType = getMimeType(dataUrl); + const buffer = await dataUrlToBuffer(dataUrl, fetchOptions); + return new Blob([buffer], { type: mimeType }); +}; + // Update file handling to use async buffer conversion export const dataUrlToFile = async ( dataUrl: DataUrl, @@ -217,6 +234,194 @@ export const fetchDataUrlContent = async ( // Import mutative for efficient JSON traversal and modification import { create } from "mutative"; +import type { SerializeOptions, DeserializeOptions } from "./types"; + +/** + * Primary serialize function that converts all binary types in a JSON object to dataref strings. + * + * Automatically converts: + * - TypedArrays (Int8Array, Uint8Array, Float32Array, etc.) → dataref strings + * - ArrayBuffer → dataref strings + * - Blob → dataref strings + * - File → dataref strings + * - Regular data (strings, numbers, objects, arrays) → unchanged + * + * If uploadFn and maxSizeBytes are provided, binary objects exceeding the size threshold + * will be uploaded and replaced with URL-based datarefs. + * + * @param json - The JSON object to serialize + * @param options - Optional configuration for upload behavior + * @returns A new JSON object with binary types converted to datarefs + */ +export const serializeDataRefs = async ( + json: T, + options?: SerializeOptions +): Promise => { + const { uploadFn, maxSizeBytes } = options || {}; + + // Track all async conversions + const promises: Array<{ + path: (string | number)[]; + promise: Promise; + }> = []; + + // Helper to check if value is a binary type + const isBinaryType = (value: any): boolean => { + return ( + value instanceof ArrayBuffer || + ArrayBuffer.isView(value) || + value instanceof Blob || + (typeof File !== "undefined" && value instanceof File) + ); + }; + + // Helper to get size of binary data + const getBinarySize = (value: any): number => { + if (value instanceof ArrayBuffer) { + return value.byteLength; + } else if (ArrayBuffer.isView(value)) { + return value.byteLength; + } else if (value instanceof Blob) { + return value.size; + } + return 0; + }; + + // Helper to get type name for binary data + const getBinaryTypeName = (value: any): string => { + if (value instanceof File) return "File"; + if (value instanceof Blob) return "Blob"; + if (value instanceof ArrayBuffer) return "ArrayBuffer"; + if (ArrayBuffer.isView(value)) return value.constructor.name; + return "unknown"; + }; + + // Helper to convert binary to dataref string + const convertBinaryToDataRef = async (value: any): Promise => { + if (value instanceof File) { + return fileToDataUrl(value); + } else if (value instanceof Blob) { + // Add type=Blob parameter to distinguish from plain ArrayBuffer + const dataUrl = await blobToDataUrl(value); + // Insert type parameter after MIME type + const commaIndex = dataUrl.indexOf(","); + const header = dataUrl.substring(0, commaIndex); + const data = dataUrl.substring(commaIndex); + return `${header};type=Blob${data}`; + } else if (value instanceof ArrayBuffer) { + // Add type=ArrayBuffer parameter to distinguish from Blob + const dataUrl = bufferToDataUrl(value); + const commaIndex = dataUrl.indexOf(","); + const header = dataUrl.substring(0, commaIndex); + const data = dataUrl.substring(commaIndex); + return `${header};type=ArrayBuffer${data}`; + } else if (ArrayBuffer.isView(value)) { + // Typed array + const typeName = value.constructor.name as TypedArrayType; + return typedArrayToDataUrl(value as any, typeName); + } + throw new Error(`Unsupported binary type: ${typeof value}`); + }; + + // Helper to upload binary data + const uploadBinary = async (value: any): Promise => { + if (!uploadFn) { + throw new Error("Upload function not provided"); + } + + const size = getBinarySize(value); + const typeName = getBinaryTypeName(value); + const mimeType = value instanceof Blob ? value.type : undefined; + + // Convert to Blob or ArrayBuffer for upload + let uploadData: Blob | ArrayBuffer; + if (value instanceof Blob) { + uploadData = value; + } else if (value instanceof ArrayBuffer) { + uploadData = value; + } else if (ArrayBuffer.isView(value)) { + uploadData = value.buffer; + } else { + throw new Error(`Cannot upload type: ${typeName}`); + } + + const url = await uploadFn(uploadData, { type: typeName, size, mimeType }); + + // Create a URL-based dataref with type information + const encodedUrl = encodeURIComponent(url); + return `data:${MIME_TYPES.URI};type=${typeName}${mimeType ? `;mimeType=${mimeType}` : ""};charset=utf-8,${encodedUrl}`; + }; + + // Traverse and collect conversion promises + const collectConversions = (obj: any, path: (string | number)[] = []) => { + if (obj === null || obj === undefined) { + return; + } + + // Check if this is a binary type + if (isBinaryType(obj)) { + const size = getBinarySize(obj); + let promise: Promise; + + // Should we upload this? + if (uploadFn && maxSizeBytes && size > maxSizeBytes) { + promise = uploadBinary(obj); + } else { + promise = convertBinaryToDataRef(obj); + } + + promises.push({ path: [...path], promise }); + return; // Don't traverse into binary types + } + + // Don't process existing data URLs + if (typeof obj === "string" && isDataUrl(obj)) { + return; + } + + // Check if this is a primitive type + if (typeof obj !== "object") { + return; + } + + // Traverse children + if (Array.isArray(obj)) { + obj.forEach((item, index) => { + collectConversions(item, [...path, index]); + }); + } else { + Object.keys(obj).forEach((key) => { + collectConversions(obj[key], [...path, key]); + }); + } + }; + + // First pass: collect all conversion promises + collectConversions(json); + + // If nothing to convert, return original + if (promises.length === 0) { + return json; + } + + // Wait for all conversions to complete + const results = await Promise.all(promises.map((p) => p.promise)); + + // Second pass: use mutative to update the JSON with datarefs + return create(json, (draft: any) => { + promises.forEach(({ path }, index) => { + const dataRef = results[index]; + + // Navigate to the parent and set the value + let current = draft; + for (let i = 0; i < path.length - 1; i++) { + current = current[path[i]]; + } + const lastKey = path[path.length - 1]; + current[lastKey] = dataRef; + }); + }); +}; /** * Converts large objects in a JSON structure to data URL references. @@ -397,7 +602,7 @@ export const dereferenceDataRefs = async ( /** * Dereferences a single data URL to its actual value. - * Attempts to parse as JSON first, falls back to text, then buffer. + * Determines the appropriate type based on MIME type and parameters. * * @param dataUrl - The data URL to dereference * @param fetchOptions - Optional fetch options for URL-based datarefs @@ -410,16 +615,168 @@ const dereferenceDataUrl = async ( const mimeType = getMimeType(dataUrl); const params = getParameters(dataUrl); - // Handle different MIME types appropriately + // Handle URL-based datarefs + if (mimeType === MIME_TYPES.URI && params.type) { + // This is an uploaded binary, download it first + const url = dataUrlToUrl(dataUrl); + if (!url) { + throw new Error("Invalid URL dataref"); + } + + const response = await fetch(url, { ...fetchOptions, redirect: "follow" }); + if (!response.ok) { + throw new Error(`Failed to fetch ${url}: ${response.statusText}`); + } + + const buffer = await response.arrayBuffer(); + + // Convert based on original type + const typeName = params.type; + const mimeTypeParam = params.mimeType; + + if (typeName === "Blob") { + return new Blob([buffer], { type: mimeTypeParam || MIME_TYPES.OCTET_STREAM }); + } else if (typeName === "File") { + const fileName = params.name || "downloaded-file"; + return new File([buffer], fileName, { type: mimeTypeParam || MIME_TYPES.OCTET_STREAM }); + } else if (typeName === "ArrayBuffer") { + return buffer; + } else { + // TypedArray + const TypedArray = globalThis[typeName as TypedArrayType]; + if (TypedArray) { + return new TypedArray(buffer); + } + return buffer; + } + } + + // Check type parameter first (most specific) + if (params.type === "ArrayBuffer") { + // Explicitly marked as ArrayBuffer + return dataUrlToBuffer(dataUrl, fetchOptions); + } else if (params.type === "Blob") { + // Explicitly marked as Blob + return dataUrlToBlob(dataUrl, fetchOptions); + } else if (params.type && mimeType === MIME_TYPES.OCTET_STREAM) { + // This is a typed array + return dataUrlToTypedArray(dataUrl, fetchOptions); + } + + // Then handle MIME types if (mimeType === MIME_TYPES.JSON) { return dataUrlToJson(dataUrl, fetchOptions); } else if (mimeType === MIME_TYPES.TEXT) { return dataUrlToText(dataUrl, fetchOptions); - } else if (mimeType === MIME_TYPES.OCTET_STREAM && params.type) { - // This is a typed array - return dataUrlToTypedArray(dataUrl, fetchOptions); + } else if (mimeType === MIME_TYPES.OCTET_STREAM) { + // Plain ArrayBuffer (no type parameter) + return dataUrlToBuffer(dataUrl, fetchOptions); } else { - // For octet-stream and other binary types, return as ArrayBuffer + // For other binary types, return as ArrayBuffer by default return dataUrlToBuffer(dataUrl, fetchOptions); } }; + +/** + * Primary deserialize function that converts all dataref strings in a JSON object + * back to their original binary types. + * + * Alias for dereferenceDataRefs with support for custom download function. + * + * @param json - The JSON object to deserialize + * @param options - Optional configuration for fetch/download behavior + * @returns A new JSON object with all datarefs converted back to original types + */ +export const deserializeDataRefs = async ( + json: T, + options?: DeserializeOptions +): Promise => { + const { fetchOptions, downloadFn } = options || {}; + + // If custom download function provided, we need to handle it differently + if (downloadFn) { + // Track all promises for async dereferencing + const promises: Array<{ + path: (string | number)[]; + promise: Promise; + }> = []; + + // Helper function to traverse and collect promises + const collectPromises = (obj: any, path: (string | number)[] = []) => { + if (obj === null || obj === undefined) { + return; + } + + if (typeof obj === "string" && isDataUrl(obj)) { + // Check if this is a URL dataref + if (isUrlDataUrl(obj)) { + const url = dataUrlToUrl(obj); + const params = getParameters(obj); + if (url) { + // Use custom download function + const promise = downloadFn(url).then((buffer) => { + // Convert based on original type + const typeName = params.type; + const mimeTypeParam = params.mimeType; + + if (typeName === "Blob") { + return new Blob([buffer], { type: mimeTypeParam || MIME_TYPES.OCTET_STREAM }); + } else if (typeName === "File") { + const fileName = params.name || "downloaded-file"; + return new File([buffer], fileName, { type: mimeTypeParam || MIME_TYPES.OCTET_STREAM }); + } else if (typeName === "ArrayBuffer") { + return buffer; + } else { + // TypedArray + const TypedArray = globalThis[typeName as TypedArrayType]; + if (TypedArray) { + return new TypedArray(buffer); + } + return buffer; + } + }); + promises.push({ path: [...path], promise }); + return; + } + } + // Regular dataref, use normal dereferencing + const promise = dereferenceDataUrl(obj, fetchOptions); + promises.push({ path: [...path], promise }); + } else if (Array.isArray(obj)) { + obj.forEach((item, index) => { + collectPromises(item, [...path, index]); + }); + } else if (typeof obj === "object") { + Object.keys(obj).forEach((key) => { + collectPromises(obj[key], [...path, key]); + }); + } + }; + + // First pass: collect all promises + collectPromises(json); + + // If no datarefs found, return original + if (promises.length === 0) { + return json; + } + + // Wait for all promises to resolve + const results = await Promise.all(promises.map((p) => p.promise)); + + // Second pass: use mutative to update the JSON with resolved values + return create(json, (draft: any) => { + promises.forEach(({ path }, index) => { + let current = draft; + for (let i = 0; i < path.length - 1; i++) { + current = current[path[i]]; + } + const lastKey = path[path.length - 1]; + current[lastKey] = results[index]; + }); + }); + } + + // No custom download function, use default dereferencing + return dereferenceDataRefs(json, fetchOptions); +}; diff --git a/src/v2/types.ts b/src/v2/types.ts index b971139..8a5ae71 100644 --- a/src/v2/types.ts +++ b/src/v2/types.ts @@ -45,3 +45,19 @@ export type TypedArrayConstructor = { (typeof globalThis)[K] >; }[TypedArrayType]; + +// Options for serialize function +export interface SerializeOptions { + uploadFn?: (data: Blob | ArrayBuffer, metadata: { + type: string; + size: number; + mimeType?: string; + }) => Promise; + maxSizeBytes?: number; // If provided, objects > size get uploaded +} + +// Options for deserialize function +export interface DeserializeOptions { + fetchOptions?: RequestInit; // For URL-based datarefs + downloadFn?: (url: string) => Promise; // Custom download logic +}