better-core-functions

.github/workflows/publish.yml

@@ -11,7 +11,7 @@ on:
 
 permissions:
   id-token: write  # Required for OIDC
-  contents: read
+  contents: write  # Required for creating releases
 
 jobs:
   publish:
@@ -35,4 +35,15 @@ jobs:
 
       - name: publish module
         run: just publish
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+
+      - name: Create GitHub Release
+        if: startsWith(github.ref, 'refs/tags/v')
+        run: |
+          gh release create ${{ github.ref_name }} \
+            --title "Release ${{ github.ref_name }}" \
+            --generate-notes
+        env:
+          GH_TOKEN: ${{ github.token }}
 

README.md

@@ -88,6 +88,71 @@ npm install @metapages/dataref
 
 ## Quick Start
 
+### Primary API (Recommended)
+
+The simplest way to work with binary data in JSON is to use the primary serialize/deserialize functions:
+
+```typescript
+import { serializeDataRefs, deserializeDataRefs } from "@metapages/dataref";
+
+// Automatically converts ALL binary types to datarefs
+const data = {
+  uint8: new Uint8Array([1, 2, 3]),
+  float32: new Float32Array([1.1, 2.2, 3.3]),
+  buffer: new Uint8Array([255, 128, 0]).buffer,
+  blob: new Blob(["hello"], { type: "text/plain" }),
+  file: new File(["content"], "test.txt"),
+  string: "regular data",
+  nested: {
+    array: new Int16Array([100, -100])
+  }
+};
+
+// Serialize: Convert all binary types to dataref strings
+const serialized = await serializeDataRefs(data);
+// All binary types are now dataref strings, safe for JSON.stringify()
+
+// Store or transmit
+const json = JSON.stringify(serialized);
+
+// Deserialize: Convert all datarefs back to original types
+const deserialized = await deserializeDataRefs(JSON.parse(json));
+// All binary types are restored to their original types
+```
+
+**With upload/download support for large objects:**
+
+```typescript
+// Serialize with upload for large data
+const serialized = await serializeDataRefs(data, {
+  uploadFn: async (data, metadata) => {
+    // Upload to your storage service
+    const response = await fetch("/upload", {
+      method: "POST",
+      body: data
+    });
+    const { url } = await response.json();
+    return url;
+  },
+  maxSizeBytes: 10240 // Upload objects > 10KB
+});
+
+// Deserialize with custom download
+const deserialized = await deserializeDataRefs(serialized, {
+  downloadFn: async (url) => {
+    const response = await fetch(url);
+    return response.arrayBuffer();
+  }
+});
+```
+
+**What gets converted:**
+- ✅ All TypedArray types (Int8Array, Uint8Array, Float32Array, etc.)
+- ✅ ArrayBuffer
+- ✅ Blob (with MIME type preservation)
+- ✅ File (with name preservation)
+- ❌ Regular data (strings, numbers, objects, arrays) — unchanged
+
 ### The Core Workflow: Encode → Serialize → Decode
 
 The most common use case is encoding complex data for JSON serialization, then decoding it later:
@@ -179,6 +244,8 @@ The library supports all JavaScript data types:
 | ArrayBuffer | Base64 binary | `bufferToDataUrl(buffer)` |
 | Uint8Array | Base64 binary | `bufferToDataUrl(uint8Array)` |
 | TypedArrays | Base64 with type | `typedArrayToDataUrl(array, type)` |
+| Blob | Base64 with MIME type | `blobToDataUrl(blob)` |
+| File | Base64 with name | `fileToDataUrl(file)` |
 | URL reference | URL-encoded URI | `urlToDataUrl("https://...")` |
 
 **Supported TypedArray types:**
@@ -326,6 +393,81 @@ isDataRef("regular string");                     // false
 
 ## API Reference
 
+### Primary Functions (Recommended)
+
+#### `serializeDataRefs<T>(json: T, options?: SerializeOptions): Promise<T>`
+
+Primary serialize function that automatically converts all binary types in a JSON object to dataref strings.
+
+**Converts:**
+- TypedArrays (Int8Array, Uint8Array, Float32Array, etc.) → dataref strings
+- ArrayBuffer → dataref strings
+- Blob → dataref strings (with MIME type preservation)
+- File → dataref strings (with name preservation)
+- Regular data (strings, numbers, objects, arrays) → unchanged
+
+**Options:**
+```typescript
+interface SerializeOptions {
+  uploadFn?: (data: Blob | ArrayBuffer, metadata: {
+    type: string;
+    size: number;
+    mimeType?: string;
+  }) => Promise<string>;
+  maxSizeBytes?: number; // If provided, objects > size get uploaded
+}
+```
+
+**Example:**
+```typescript
+const serialized = await serializeDataRefs({
+  array: new Uint8Array([1, 2, 3]),
+  blob: new Blob(["test"], { type: "text/plain" }),
+  string: "unchanged"
+});
+```
+
+**With upload:**
+```typescript
+const serialized = await serializeDataRefs(data, {
+  uploadFn: async (data, metadata) => {
+    // Upload and return URL
+    return "https://storage.example.com/...";
+  },
+  maxSizeBytes: 10240 // Upload if > 10KB
+});
+```
+
+#### `deserializeDataRefs<T>(json: T, options?: DeserializeOptions): Promise<T>`
+
+Primary deserialize function that converts all dataref strings in a JSON object back to their original binary types.
+
+**Options:**
+```typescript
+interface DeserializeOptions {
+  fetchOptions?: RequestInit; // For URL-based datarefs
+  downloadFn?: (url: string) => Promise<ArrayBuffer>; // Custom download
+}
+```
+
+**Example:**
+```typescript
+const deserialized = await deserializeDataRefs(serialized);
+// All datarefs are converted back to original types
+```
+
+**With custom download:**
+```typescript
+const deserialized = await deserializeDataRefs(serialized, {
+  downloadFn: async (url) => {
+    const response = await fetch(url, {
+      headers: { "Authorization": "Bearer token" }
+    });
+    return response.arrayBuffer();
+  }
+});
+```
+
 ### Encoding Functions (to Data URL)
 
 #### `textToDataUrl(text: string): DataUrl`
@@ -360,6 +502,15 @@ typedArrayToDataUrl(new Float32Array([1.1, 2.2]), "Float32Array");
 // => "data:application/octet-stream;type=Float32Array;base64,..."
 ```
 
+#### `blobToDataUrl(blob: Blob): Promise<DataUrl>`
+Converts a Blob to a data URL with MIME type preservation.
+
+```typescript
+const blob = new Blob(["content"], { type: "text/plain" });
+await blobToDataUrl(blob);
+// => "data:text/plain;type=Blob;base64,..."
+```
+
 #### `urlToDataUrl(url: string, fetchOptions?: RequestInit): Promise<DataUrl>`
 Creates a URL reference or fetches and encodes URL content.
 
@@ -389,6 +540,14 @@ Decodes a data URL to an ArrayBuffer.
 #### `dataUrlToTypedArray<T>(dataUrl: DataUrl, fetchOptions?: RequestInit): Promise<T>`
 Decodes a data URL to a TypedArray with type preservation.
 
+#### `dataUrlToBlob(dataUrl: DataUrl, fetchOptions?: RequestInit): Promise<Blob>`
+Decodes a data URL to a Blob with MIME type preservation.
+
+```typescript
+const blob = await dataUrlToBlob(dataUrl);
+// => Blob { type: "text/plain", size: 7, ... }
+```
+
 #### `dataUrlToFile(dataUrl: DataUrl, name?: string, fetchOptions?: RequestInit): Promise<File>`
 Converts a data URL to a File object.
 

examples/serialize-example.ts

@@ -0,0 +1,165 @@
+/**
+ * Example demonstrating the primary serialize/deserialize API
+ */
+import {
+  serializeDataRefs,
+  deserializeDataRefs,
+  type SerializeOptions,
+} from "../src/index";
+
+async function basicExample() {
+  console.log("=== Basic Serialize/Deserialize Example ===\n");
+
+  // Create data with various binary types
+  const originalData = {
+    metadata: {
+      name: "Sensor Data",
+      timestamp: new Date().toISOString(),
+    },
+    readings: {
+      temperature: new Float32Array([20.5, 21.2, 22.1, 21.8]),
+      humidity: new Uint8Array([65, 68, 70, 69]),
+      rawData: new Int16Array([-100, -50, 0, 50, 100]),
+    },
+    buffer: new Uint8Array([0xff, 0x00, 0x80]).buffer,
+    blob: new Blob(["Sample text content"], { type: "text/plain" }),
+    file: new File(["Document content"], "report.txt", { type: "text/plain" }),
+    regularString: "This stays as a regular string",
+    regularNumber: 42,
+  };
+
+  console.log("Original data structure:");
+  console.log("- temperature:", originalData.readings.temperature.constructor.name);
+  console.log("- humidity:", originalData.readings.humidity.constructor.name);
+  console.log("- rawData:", originalData.readings.rawData.constructor.name);
+  console.log("- buffer:", originalData.buffer.constructor.name);
+  console.log("- blob:", originalData.blob.constructor.name);
+  console.log("- file:", originalData.file.constructor.name);
+  console.log();
+
+  // Serialize - convert all binary types to dataref strings
+  console.log("Serializing...");
+  const serialized = await serializeDataRefs(originalData);
+
+  console.log("\nSerialized data (binary types are now strings):");
+  console.log("- temperature:", typeof serialized.readings.temperature,
+              serialized.readings.temperature.substring(0, 50) + "...");
+  console.log("- humidity:", typeof serialized.readings.humidity,
+              serialized.readings.humidity.substring(0, 50) + "...");
+  console.log("- buffer:", typeof serialized.buffer,
+              serialized.buffer.substring(0, 50) + "...");
+  console.log("- regularString:", typeof serialized.regularString, "=", serialized.regularString);
+  console.log();
+
+  // Can now safely stringify as JSON
+  const jsonString = JSON.stringify(serialized);
+  console.log("JSON size:", jsonString.length, "bytes");
+  console.log();
+
+  // Deserialize - convert all datarefs back to original types
+  console.log("Deserializing...");
+  const deserialized = await deserializeDataRefs(JSON.parse(jsonString));
+
+  console.log("\nDeserialized data (restored to original types):");
+  console.log("- temperature:", deserialized.readings.temperature.constructor.name, "=",
+              Array.from(deserialized.readings.temperature));
+  console.log("- humidity:", deserialized.readings.humidity.constructor.name, "=",
+              Array.from(deserialized.readings.humidity));
+  console.log("- rawData:", deserialized.readings.rawData.constructor.name, "=",
+              Array.from(deserialized.readings.rawData));
+  console.log("- buffer:", deserialized.buffer.constructor.name);
+  console.log("- blob:", deserialized.blob.constructor.name, "type:", deserialized.blob.type);
+  console.log("- regularString:", deserialized.regularString);
+  console.log("- regularNumber:", deserialized.regularNumber);
+  console.log();
+}
+
+async function uploadDownloadExample() {
+  console.log("=== Upload/Download Example ===\n");
+
+  // Simulate cloud storage
+  const mockStorage = new Map<string, ArrayBuffer>();
+  let uploadCounter = 0;
+
+  // Mock upload function
+  const mockUploadFn = async (
+    data: Blob | ArrayBuffer,
+    metadata: { type: string; size: number; mimeType?: string }
+  ): Promise<string> => {
+    const buffer = data instanceof Blob ? await data.arrayBuffer() : data;
+    const url = `https://storage.example.com/uploads/file-${++uploadCounter}`;
+    mockStorage.set(url, buffer);
+    console.log(`  ✓ Uploaded ${metadata.size} bytes (${metadata.type}) to ${url}`);
+    return url;
+  };
+
+  // Mock download function
+  const mockDownloadFn = async (url: string): Promise<ArrayBuffer> => {
+    const data = mockStorage.get(url);
+    if (!data) {
+      throw new Error(`URL not found: ${url}`);
+    }
+    console.log(`  ✓ Downloaded ${data.byteLength} bytes from ${url}`);
+    return data;
+  };
+
+  // Create data with a large array that should be uploaded
+  const largeArray = new Float32Array(50000); // 200KB
+  for (let i = 0; i < largeArray.length; i++) {
+    largeArray[i] = Math.sin(i / 100);
+  }
+
+  const data = {
+    smallData: new Uint8Array([1, 2, 3, 4, 5]),
+    largeData: largeArray,
+    metadata: {
+      description: "Signal processing results",
+      samples: largeArray.length,
+    },
+  };
+
+  // Serialize with upload for large objects
+  console.log("Serializing with upload (threshold: 10KB)...");
+  const serialized = await serializeDataRefs(data, {
+    uploadFn: mockUploadFn,
+    maxSizeBytes: 10240, // 10KB threshold
+  });
+  console.log();
+
+  console.log("Result:");
+  console.log("- smallData: inline dataref (small, not uploaded)");
+  console.log("- largeData: URL dataref (uploaded)");
+  console.log();
+
+  // JSON is now much smaller
+  const jsonString = JSON.stringify(serialized);
+  console.log("JSON size:", jsonString.length, "bytes (instead of ~200KB)");
+  console.log();
+
+  // Deserialize with download
+  console.log("Deserializing with download...");
+  const deserialized = await deserializeDataRefs(serialized, {
+    downloadFn: mockDownloadFn,
+  });
+  console.log();
+
+  console.log("Verification:");
+  console.log("- smallData matches:",
+    JSON.stringify(Array.from(deserialized.smallData)) === JSON.stringify([1, 2, 3, 4, 5]));
+  console.log("- largeData length:", deserialized.largeData.length);
+  console.log("- largeData type:", deserialized.largeData.constructor.name);
+  console.log("- First 5 values:", Array.from(deserialized.largeData.slice(0, 5)));
+  console.log();
+}
+
+// Run examples
+(async () => {
+  try {
+    await basicExample();
+    await uploadDownloadExample();
+    console.log("✓ All examples completed successfully!");
+  } catch (error) {
+    console.error("Error:", error);
+    process.exit(1);
+  }
+})();

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@metapages/dataref",
-  "version": "2.0.1",
+  "version": "2.1.0",
   "author": "Dion Whitehead <dion@metapage.io>",
   "repository": {
     "type": "git",