From 01218873bffec72b3555270e19f300163f9b8b02 Mon Sep 17 00:00:00 2001 From: Russell Stern Date: Fri, 24 Apr 2026 10:19:05 -0400 Subject: [PATCH 1/5] Added xml parsing and compression building blocks --- building-blocks/README.md | 25 +- building-blocks/compression-utils/README.md | 69 +++++ .../compression-utils-ts/.gitignore | 1 + .../compression-utils-ts/README.md | 251 ++++++++++++++++++ .../compression-utils-ts/project.yaml | 44 +++ .../compression-utils-ts/secrets.yaml | 1 + .../compression-utils-ts/workflow/README.md | 27 ++ .../workflow/config.production.json | 3 + .../workflow/config.staging.json | 3 + .../compression-utils-ts/workflow/main.ts | 238 +++++++++++++++++ .../workflow/package.json | 17 ++ .../workflow/tsconfig.json | 15 ++ .../workflow/workflow.yaml | 34 +++ building-blocks/xml-utils/README.md | 70 +++++ .../xml-utils/xml-utils-ts/.gitignore | 1 + .../xml-utils/xml-utils-ts/README.md | 208 +++++++++++++++ .../xml-utils/xml-utils-ts/project.yaml | 44 +++ .../xml-utils/xml-utils-ts/secrets.yaml | 1 + .../workflow/config.production.json | 3 + .../xml-utils-ts/workflow/config.staging.json | 3 + .../xml-utils/xml-utils-ts/workflow/main.ts | 248 +++++++++++++++++ .../xml-utils-ts/workflow/package.json | 17 ++ .../xml-utils-ts/workflow/tsconfig.json | 15 ++ .../xml-utils-ts/workflow/workflow.yaml | 34 +++ 24 files changed, 1371 insertions(+), 1 deletion(-) create mode 100644 building-blocks/compression-utils/README.md create mode 100644 building-blocks/compression-utils/compression-utils-ts/.gitignore create mode 100644 building-blocks/compression-utils/compression-utils-ts/README.md create mode 100644 building-blocks/compression-utils/compression-utils-ts/project.yaml create mode 100644 building-blocks/compression-utils/compression-utils-ts/secrets.yaml create mode 100644 building-blocks/compression-utils/compression-utils-ts/workflow/README.md create mode 100644 building-blocks/compression-utils/compression-utils-ts/workflow/config.production.json create mode 100644 building-blocks/compression-utils/compression-utils-ts/workflow/config.staging.json create mode 100644 building-blocks/compression-utils/compression-utils-ts/workflow/main.ts create mode 100644 building-blocks/compression-utils/compression-utils-ts/workflow/package.json create mode 100644 building-blocks/compression-utils/compression-utils-ts/workflow/tsconfig.json create mode 100644 building-blocks/compression-utils/compression-utils-ts/workflow/workflow.yaml create mode 100644 building-blocks/xml-utils/README.md create mode 100644 building-blocks/xml-utils/xml-utils-ts/.gitignore create mode 100644 building-blocks/xml-utils/xml-utils-ts/README.md create mode 100644 building-blocks/xml-utils/xml-utils-ts/project.yaml create mode 100644 building-blocks/xml-utils/xml-utils-ts/secrets.yaml create mode 100644 building-blocks/xml-utils/xml-utils-ts/workflow/config.production.json create mode 100644 building-blocks/xml-utils/xml-utils-ts/workflow/config.staging.json create mode 100644 building-blocks/xml-utils/xml-utils-ts/workflow/main.ts create mode 100644 building-blocks/xml-utils/xml-utils-ts/workflow/package.json create mode 100644 building-blocks/xml-utils/xml-utils-ts/workflow/tsconfig.json create mode 100644 building-blocks/xml-utils/xml-utils-ts/workflow/workflow.yaml diff --git a/building-blocks/README.md b/building-blocks/README.md index f7624eaa..280c9eb0 100644 --- a/building-blocks/README.md +++ b/building-blocks/README.md @@ -59,11 +59,34 @@ Path: [`./read-data-feeds`](./read-data-feeds) --- +### 3) **Compression Utils** +Path: [`./compression-utils`](./compression-utils) + +- Demonstrates how to perform compression and decompression operations in CRE workflows using **fflate** as a pure JavaScript alternative to the Node.js `zlib` module. +- Covers gzip, deflate, zlib formats, compression levels, ZIP archives, and string utilities. +- Shows how to fetch a large JSON payload via CRE HTTP and compress it to stay within the 100KB response body limit. + +👉 See the block’s README for setup, config, and sample logs. + +--- + +### 4) **XML Utils** +Path: [`./xml-utils`](./xml-utils) + +- Demonstrates how to use XML in CRE workflows using **fast-xml-parser** as a pure JavaScript alternative to the browser's `DOMParser` and the Node.js `xml2js` module. +- Covers parsing, validating, and building XML. +- Shows how to fetch a XML payload via CRE HTTP, validate it, parse it with attributes and namespaces, and rebuild a filtered XML document. + +👉 See the block’s README for setup, config, and sample logs. + +--- + ## When to Use Which Block * **kv-store**: You want to see an **off-chain write** pattern (AWS S3), secrets usage, SigV4 signing, and a **consensus read → single write** flow. * **read-data-feeds**: You want to **read on-chain data** via contract calls, manage ABIs/bindings, and configure **RPC** access. - +* **compression-utils**: You want to handle **data compression** in CRE Typescript workflows, especially when working with large datasets and the CRE HTTP response limit. +* **xml-utils**: You want to work with **XML data** in CRE workflows, including parsing, validating, and building XML documents. --- ## License diff --git a/building-blocks/compression-utils/README.md b/building-blocks/compression-utils/README.md new file mode 100644 index 00000000..77a35476 --- /dev/null +++ b/building-blocks/compression-utils/README.md @@ -0,0 +1,69 @@ +
+ + Chainlink logo + + +[![License](https://img.shields.io/badge/license-MIT-blue)](https://github.com/smartcontractkit/cre-templates/blob/main/LICENSE) +[![CRE Home](https://img.shields.io/static/v1?label=CRE&message=Home&color=blue)](https://chain.link/chainlink-runtime-environment) +[![CRE Documentation](https://img.shields.io/static/v1?label=CRE&message=Docs&color=blue)](https://docs.chain.link/cre) + +
+ +# Compression Utils - CRE Building Block + +This building block demonstrates how to perform compression and decompression operations in CRE workflows using **fflate** as a pure JavaScript alternative to the Node.js `zlib` module. + +## The Problem + +The CRE TypeScript SDK runs on **QuickJS**, a lightweight JavaScript engine that does not support Node.js native modules. This means the standard `zlib` module from Node.js is **not available** in CRE workflows. Additionally, CRE's HTTP capability has a **100KB response body limit**, making compression essential for working with large datasets. + +## The Solution + +The [fflate library](https://github.com/101arrowz/fflate) provides pure JavaScript implementations of compression algorithms: + +- **Pure JavaScript** - No native dependencies, works in any JS environment including QuickJS +- **Standards-compliant** - Output is interoperable with native gzip, zlib, and deflate tools +- **Small** - ~8KB minified, tree-shakeable +- **Sync-capable** - Provides synchronous APIs (`gzipSync`, `deflateSync`, etc.) that work reliably in QuickJS +- **Practical** - Enable workflows to handle large datasets despite the 100KB HTTP limit + +--- + +## What's Covered + +This template demonstrates: + +| Category | fflate API | Use Cases | +|----------|------------|-----------| +| **Gzip Compression** | `gzipSync` / `gunzipSync` | Compress/decompress with header (standard `.gz` format) | +| **Raw Deflate** | `deflateSync` / `inflateSync` | Lightweight compression without headers | +| **Zlib Format** | `zlibSync` / `unzlibSync` | zlib wrapper compression (common in APIs) | +| **Auto-detect Decompression** | `decompressSync` | Handle gzip, zlib, or deflate automatically | +| **Compression Levels** | `gzipSync(data, { level })` | Tune speed vs. compression (0-9) | +| **ZIP Archives** | `zipSync` / `unzipSync` | Bundle multiple files into a single archive | +| **String Utilities** | `strToU8` / `strFromU8` | Convert between strings and byte arrays | + +## Get Started + +- **TypeScript**: See the [TypeScript README](./compression-utils-ts/README.md) for detailed setup, usage examples, and a complete Node.js zlib-to-fflate mapping table. + +## Quick Example + +```typescript +// Instead of Node.js zlib: +// const compressed = require('zlib').gzipSync(data); + +// Use fflate: +import { gzipSync, gunzipSync, strToU8, strFromU8 } from "fflate"; + +const data = strToU8(JSON.stringify(largeObject)); +const compressed = gzipSync(data); +const restored = JSON.parse(strFromU8(gunzipSync(compressed))); +``` + +## Reference Documentation + +- [fflate GitHub](https://github.com/101arrowz/fflate) +- [fflate npm](https://www.npmjs.com/package/fflate) +- [CRE Documentation](https://docs.chain.link/cre) +- [QuickJS Engine](https://bellard.org/quickjs/) \ No newline at end of file diff --git a/building-blocks/compression-utils/compression-utils-ts/.gitignore b/building-blocks/compression-utils/compression-utils-ts/.gitignore new file mode 100644 index 00000000..03bd4129 --- /dev/null +++ b/building-blocks/compression-utils/compression-utils-ts/.gitignore @@ -0,0 +1 @@ +*.env diff --git a/building-blocks/compression-utils/compression-utils-ts/README.md b/building-blocks/compression-utils/compression-utils-ts/README.md new file mode 100644 index 00000000..b5dc8e3d --- /dev/null +++ b/building-blocks/compression-utils/compression-utils-ts/README.md @@ -0,0 +1,251 @@ +# Compression Utils - CRE Building Block (TypeScript) + +**⚠️ DISCLAIMER** + +This tutorial represents an educational example to use a Chainlink system, product, or service and is provided to demonstrate how to interact with Chainlink's systems, products, and services to integrate them into your own. This template is provided "AS IS" and "AS AVAILABLE" without warranties of any kind, it has not been audited, and it may be missing key checks or error handling to make the usage of the system, product or service more clear. Do not use the code in this example in a production environment without completing your own audits and application of best practices. Neither Chainlink Labs, the Chainlink Foundation, nor Chainlink node operators are responsible for unintended outputs that are generated due to errors in code. + +--- + +This building block demonstrates how to use compression and decompression in CRE TypeScript workflows. Since the CRE TypeScript SDK runs on QuickJS (a lightweight JavaScript engine), the standard Node.js `zlib` module is not available. This template shows how to use **[fflate](https://github.com/101arrowz/fflate)** as a pure JavaScript drop-in alternative. + +## Why fflate? + +[fflate](https://github.com/101arrowz/fflate) is: +- **Pure JavaScript** - No native dependencies, works in any JS environment including QuickJS +- **Standards-compliant** - Output is interoperable with native gzip, zlib, and deflate tools +- **Small** - ~8KB minified, tree-shakeable +- **Sync-capable** - Provides synchronous APIs (`gzipSync`, `deflateSync`, etc.) that work reliably in QuickJS + +## Features Demonstrated + +This workflow fetches a large JSON payload (~94KB, 300 records) via HTTP and uses it as the input for each compression demo: + +| Category | fflate API | Node.js Equivalent | +|----------|------------|--------------------| +| **Gzip** | `gzipSync` / `gunzipSync` | `zlib.gzipSync` / `zlib.gunzipSync` | +| **Raw Deflate** | `deflateSync` / `inflateSync` | `zlib.deflateRawSync` / `zlib.inflateRawSync` | +| **Zlib** | `zlibSync` / `unzlibSync` | `zlib.deflateSync` / `zlib.inflateSync` | +| **Auto-detect** | `decompressSync` | — | +| **Compression levels** | `gzipSync(data, { level })` | `zlib.gzipSync(data, { level })` | +| **ZIP archives** | `zipSync` / `unzipSync` | — (no single-call equivalent) | +| **String utilities** | `strToU8` / `strFromU8` | `Buffer.from()` / `TextEncoder` / `TextDecoder` | + +--- + +## HTTP Fetch in CRE + +The workflow fetches its payload using the CRE HTTP capability (`HTTPClient`), which routes requests through the DON's off-chain network layer. The standard `fetch` / `node:https` APIs are not available in QuickJS. + +> **Note:** The CRE HTTP capability has a response body limit of **100,000 bytes**. This workflow fetches `/comments?_limit=300` (~94KB) to stay within that limit. + +```typescript +import { HTTPClient, consensusIdenticalAggregation, type HTTPSendRequester, text } from "@chainlink/cre-sdk"; + +const fetchComments = (sendRequester: HTTPSendRequester): string => { + const resp = sendRequester + .sendRequest({ url: "https://jsonplaceholder.typicode.com/comments?_limit=300", method: "GET" }) + .result(); + return text(resp); +}; + +// In the handler: +const httpClient = new HTTPClient(); +const rawJson = httpClient + .sendRequest(runtime, fetchComments, consensusIdenticalAggregation())() + .result(); +``` + +`consensusIdenticalAggregation` ensures all DON nodes agree on an identical response before the workflow proceeds. + +--- + +## Node.js zlib to fflate Mapping + +| Node.js zlib | fflate | Import | Code Reference | +|---|---|---|---| +| `zlib.gzipSync(data)` | `gzipSync(data)` | `import { gzipSync } from "fflate"` | [main.ts:62](./workflow/main.ts#L62) | +| `zlib.gunzipSync(data)` | `gunzipSync(data)` | `import { gunzipSync } from "fflate"` | [main.ts:66](./workflow/main.ts#L66) | +| `zlib.deflateRawSync(data)` | `deflateSync(data)` | `import { deflateSync } from "fflate"` | [main.ts:75](./workflow/main.ts#L75) | +| `zlib.inflateRawSync(data)` | `inflateSync(data)` | `import { inflateSync } from "fflate"` | [main.ts:79](./workflow/main.ts#L79) | +| `zlib.deflateSync(data)` | `zlibSync(data)` | `import { zlibSync } from "fflate"` | [main.ts:88](./workflow/main.ts#L88) | +| `zlib.inflateSync(data)` | `unzlibSync(data)` | `import { unzlibSync } from "fflate"` | [main.ts:92](./workflow/main.ts#L92) | +| *(auto-detect format)* | `decompressSync(data)` | `import { decompressSync } from "fflate"` | [main.ts:108](./workflow/main.ts#L108) | +| `zlib.gzipSync(data, { level })` | `gzipSync(data, { level })` | `import { gzipSync } from "fflate"` | [main.ts:125](./workflow/main.ts#L125) | +| *(no equivalent)* | `zipSync(files)` | `import { zipSync } from "fflate"` | [main.ts:147](./workflow/main.ts#L147) | +| *(no equivalent)* | `unzipSync(data)` | `import { unzipSync } from "fflate"` | [main.ts:150](./workflow/main.ts#L150) | +| `Buffer.from(str)` / `TextEncoder` | `strToU8(str)` | `import { strToU8 } from "fflate"` | [main.ts:162](./workflow/main.ts#L162) | +| `buf.toString()` / `TextDecoder` | `strFromU8(bytes)` | `import { strFromU8 } from "fflate"` | [main.ts:165](./workflow/main.ts#L165) | + +--- + +## Dependencies + +```json +{ + "fflate": "^0.8.2" +} +``` + +--- + +## Setup and Prerequisites + +1. **Install CRE CLI** + ```bash + # See https://docs.chain.link/cre for installation instructions + ``` + +2. **Login to CRE** + ```bash + cre login + ``` + +3. **Install Bun** (if not already installed) + ```bash + # See https://bun.sh/docs/installation + ``` + +4. **Install dependencies** + ```bash + cd building-blocks/compression-utils/compression-utils-ts/workflow + bun install + ``` + +--- + +## Running the Workflow + +### Simulate the workflow + +From the project root directory (`compression-utils-ts`): + +```bash +cre workflow simulate workflow +``` + +--- + +## Example Output + +``` +======================================== +FFLATE COMPRESSION LIBRARIES DEMO +Alternative to Node.js zlib module +Compatible with QuickJS / CRE Workflows +======================================== + +Fetching payload from JSONPlaceholder /comments... +Fetched: 94.32 KB — 300 records + +=== GZIP / GUNZIP === +Alternative to: zlib.gzipSync() / zlib.gunzipSync() +Original: 94.32 KB +Compressed: 21.48 KB (77.2% smaller) +Verified: 300 records restored via gunzipSync + +=== DEFLATE / INFLATE (raw) === +Alternative to: zlib.deflateRawSync() / zlib.inflateRawSync() +Original: 94.32 KB +Compressed: 21.46 KB (77.3% smaller) +Verified: 300 records restored via inflateSync + +=== ZLIB / UNZLIB === +Alternative to: zlib.deflateSync() / zlib.inflateSync() +Original: 94.32 KB +Compressed: 21.48 KB (77.2% smaller) +Verified: 300 records restored via unzlibSync + +=== AUTO-DETECT DECOMPRESSION === +decompressSync() detects gzip / zlib / deflate automatically. +From gzip : 300 records (21.48 KB) +From zlib : 300 records (21.48 KB) +From deflate: 300 records (21.46 KB) + +=== COMPRESSION LEVELS === +Level 0 = store only | Level 4 | Level 9 = max +Level 0 (store): 94.35 KB (-0.0% smaller) +Level 4: 21.78 KB (76.9% smaller) +Level 9 (max): 21.04 KB (77.7% smaller) + +=== ZIP ARCHIVES === +Multi-file archiving — no Node.js zlib equivalent. +Archive: 21.92 KB (4 files, 300 total records) +Extracted: batch-1.json, batch-2.json, batch-3.json, manifest.json +Verified: 300 records across 3 batches + +=== STRING UTILITIES === +strToU8 / strFromU8 — alternative to Buffer.from() / TextEncoder / TextDecoder +strToU8(rawJson): 94.32 KB Uint8Array +strFromU8(bytes): 300 records decoded +strToU8("Hello, CRE Workflow!"): [72, 101, 108, 108, 111, 44, 32, 67, 82, 69, 32, 87, 111, 114, 107, 102, 108, 111, 119, 33] +strFromU8(...): "Hello, CRE Workflow!" + +======================================== +DEMO COMPLETE +======================================== +``` + +--- + +## Use Cases + +### 1. Compress large API responses before on-chain storage +Reduce gas costs by compressing data before writing it on-chain or to decentralized storage: +```typescript +import { gzipSync, gunzipSync, strToU8, strFromU8 } from "fflate"; + +const payload = JSON.stringify(largeDataObject); +const compressed = gzipSync(strToU8(payload)); +// Store `compressed` — decompress on read +const restored = JSON.parse(strFromU8(gunzipSync(compressed))); +``` + +### 2. Decompress API responses to get around 100KB limit +The CRE HTTP capability has a 100KB response body limit. Compress large payloads on the server side and decompress in the workflow: +```typescript +import { gunzipSync, strFromU8 } from "fflate"; + +// Server sends gzip-compressed data (~25KB) +// which decompresses to the full ~94KB payload +const compressedData = sendRequester.sendRequest({ + url: "https://api.example.com/data?format=gzip" +}).result(); + +const fullData = JSON.parse(strFromU8(gunzipSync(compressedData))); +// Now you can work with the full dataset despite the 100KB limit +``` + +### 3. Auto-detect and decompress unknown formats +Handle compressed payloads from external sources without knowing the format upfront: +```typescript +import { decompressSync, strFromU8 } from "fflate"; + +// Works regardless of whether the source used gzip, zlib, or raw deflate +const decompressed = strFromU8(decompressSync(incomingBytes)); +``` + +### 4. Tune compression for speed vs. size +Use lower levels for real-time workflows, higher levels when minimizing output size matters: +```typescript +import { gzipSync, strToU8 } from "fflate"; + +// Fast path — minimal runtime +const fast = gzipSync(data, { level: 1 }); + +// Storage path — smallest possible output +const compact = gzipSync(data, { level: 9 }); +``` + +--- + +## Reference Documentation + +- [CRE Documentation](https://docs.chain.link/cre) +- [fflate GitHub](https://github.com/101arrowz/fflate) +- [fflate npm](https://www.npmjs.com/package/fflate) + +--- + +## License + +MIT - see the repository's [LICENSE](https://github.com/smartcontractkit/cre-templates/blob/main/LICENSE). diff --git a/building-blocks/compression-utils/compression-utils-ts/project.yaml b/building-blocks/compression-utils/compression-utils-ts/project.yaml new file mode 100644 index 00000000..3b25aacf --- /dev/null +++ b/building-blocks/compression-utils/compression-utils-ts/project.yaml @@ -0,0 +1,44 @@ +# ========================================================================== +# CRE PROJECT SETTINGS FILE +# ========================================================================== +# Project-specific settings for CRE CLI targets. +# Each target defines cre-cli, account, and rpcs groups. +# +# Example custom target: +# my-target: +# account: +# workflow-owner-address: "0x123..." # Optional: Owner wallet/MSIG address (used for --unsigned transactions) +# rpcs: +# - chain-name: ethereum-testnet-sepolia # Required if your workflow interacts with this chain +# url: "" +# +# RPC URLs support ${VAR_NAME} syntax to reference environment variables. +# This keeps secrets out of project.yaml (which is committed to git). +# Variables are resolved from your .env file or exported shell variables. +# Example: +# - chain-name: ethereum-testnet-sepolia +# url: https://rpc.example.com/${CRE_SECRET_RPC_SEPOLIA} +# +# Experimental chains (automatically used by the simulator when present): +# Use this for chains not yet in official chain-selectors (e.g., hackathons, new chain integrations). +# In your workflow, reference the chain as evm:ChainSelector:@1.0.0 +# +# experimental-chains: +# - chain-selector: 12345 # The chain selector value +# rpc-url: "https://rpc.example.com" # RPC endpoint URL +# forwarder: "0x..." # Forwarder contract address on the chain + +# ========================================================================== +staging-settings: + rpcs: + - chain-name: ethereum-testnet-sepolia + url: https://ethereum-sepolia-rpc.publicnode.com + +# ========================================================================== +production-settings: + rpcs: + - chain-name: ethereum-testnet-sepolia + url: https://ethereum-sepolia-rpc.publicnode.com + diff --git a/building-blocks/compression-utils/compression-utils-ts/secrets.yaml b/building-blocks/compression-utils/compression-utils-ts/secrets.yaml new file mode 100644 index 00000000..7b85d864 --- /dev/null +++ b/building-blocks/compression-utils/compression-utils-ts/secrets.yaml @@ -0,0 +1 @@ +secretsNames: diff --git a/building-blocks/compression-utils/compression-utils-ts/workflow/README.md b/building-blocks/compression-utils/compression-utils-ts/workflow/README.md new file mode 100644 index 00000000..dfe20076 --- /dev/null +++ b/building-blocks/compression-utils/compression-utils-ts/workflow/README.md @@ -0,0 +1,27 @@ +# Hello World (TypeScript) + +This template provides a blank TypeScript workflow example. It aims to give a starting point for writing a workflow from scratch and to get started with local simulation. + +Steps to run the example + +## 1. Update .env file + +You need to add a private key to env file. This is specifically required if you want to simulate chain writes. For that to work the key should be valid and funded. +If your workflow does not do any chain write then you can just put any dummy key as a private key. e.g. +``` +CRE_ETH_PRIVATE_KEY=0000000000000000000000000000000000000000000000000000000000000001 +``` + +## 2. Install dependencies +```bash +bun install +``` + +## 3. Simulate the workflow +Run the command from project root directory + +```bash +cre workflow simulate --target=staging-settings +``` + +It is recommended to look into other existing examples to see how to write a workflow. You can generate them by running the `cre init` command. diff --git a/building-blocks/compression-utils/compression-utils-ts/workflow/config.production.json b/building-blocks/compression-utils/compression-utils-ts/workflow/config.production.json new file mode 100644 index 00000000..1a360cb3 --- /dev/null +++ b/building-blocks/compression-utils/compression-utils-ts/workflow/config.production.json @@ -0,0 +1,3 @@ +{ + "schedule": "*/30 * * * * *" +} diff --git a/building-blocks/compression-utils/compression-utils-ts/workflow/config.staging.json b/building-blocks/compression-utils/compression-utils-ts/workflow/config.staging.json new file mode 100644 index 00000000..1a360cb3 --- /dev/null +++ b/building-blocks/compression-utils/compression-utils-ts/workflow/config.staging.json @@ -0,0 +1,3 @@ +{ + "schedule": "*/30 * * * * *" +} diff --git a/building-blocks/compression-utils/compression-utils-ts/workflow/main.ts b/building-blocks/compression-utils/compression-utils-ts/workflow/main.ts new file mode 100644 index 00000000..4bf88877 --- /dev/null +++ b/building-blocks/compression-utils/compression-utils-ts/workflow/main.ts @@ -0,0 +1,238 @@ +import { + CronCapability, + consensusIdenticalAggregation, + handler, + HTTPClient, + type HTTPSendRequester, + Runner, + text, + type Runtime, +} from "@chainlink/cre-sdk"; + +import { + deflateSync, + decompressSync, + gzipSync, + gunzipSync, + inflateSync, + strFromU8, + strToU8, + unzlibSync, + unzipSync, + zlibSync, + zipSync, +} from "fflate"; + +type Config = { + schedule: string; +}; + +// ============================================================================ +// HTTP Fetch — JSONPlaceholder /comments?_limit=300 (~92KB, 300 records) +// Used as the input payload for all compression demos below. +// ============================================================================ + +const fetchComments = (sendRequester: HTTPSendRequester): string => { + const resp = sendRequester + .sendRequest({ + url: "https://jsonplaceholder.typicode.com/comments?_limit=300", + method: "GET", + }) + .result(); + return text(resp); +}; + +// ============================================================================ +// Helpers +// ============================================================================ + +function formatBytes(bytes: number): string { + if (bytes < 1024) return `${bytes} B`; + return `${(bytes / 1024).toFixed(2)} KB`; +} + +function reductionPct(original: number, compressed: number): string { + return `${((1 - compressed / original) * 100).toFixed(1)}% smaller`; +} + +function demoGzip(runtime: Runtime, jsonBytes: Uint8Array): void { + runtime.log("=== GZIP / GUNZIP ==="); + runtime.log("Alternative to: zlib.gzipSync() / zlib.gunzipSync()"); + + const compressed = gzipSync(jsonBytes); + runtime.log(`Original: ${formatBytes(jsonBytes.length)}`); + runtime.log(`Compressed: ${formatBytes(compressed.length)} (${reductionPct(jsonBytes.length, compressed.length)})`); + + const decompressed = gunzipSync(compressed); + const records = (JSON.parse(strFromU8(decompressed)) as unknown[]).length; + runtime.log(`Verified: ${records} records restored via gunzipSync`); +} + +function demoDeflate(runtime: Runtime, jsonBytes: Uint8Array): void { + runtime.log("=== DEFLATE / INFLATE (raw) ==="); + runtime.log("Alternative to: zlib.deflateRawSync() / zlib.inflateRawSync()"); + + const compressed = deflateSync(jsonBytes); + runtime.log(`Original: ${formatBytes(jsonBytes.length)}`); + runtime.log(`Compressed: ${formatBytes(compressed.length)} (${reductionPct(jsonBytes.length, compressed.length)})`); + + const decompressed = inflateSync(compressed); + const records = (JSON.parse(strFromU8(decompressed)) as unknown[]).length; + runtime.log(`Verified: ${records} records restored via inflateSync`); +} + +function demoZlib(runtime: Runtime, jsonBytes: Uint8Array): void { + runtime.log("=== ZLIB / UNZLIB ==="); + runtime.log("Alternative to: zlib.deflateSync() / zlib.inflateSync()"); + + const compressed = zlibSync(jsonBytes); + runtime.log(`Original: ${formatBytes(jsonBytes.length)}`); + runtime.log(`Compressed: ${formatBytes(compressed.length)} (${reductionPct(jsonBytes.length, compressed.length)})`); + + const decompressed = unzlibSync(compressed); + const records = (JSON.parse(strFromU8(decompressed)) as unknown[]).length; + runtime.log(`Verified: ${records} records restored via unzlibSync`); +} + +function demoAutoDetect(runtime: Runtime, jsonBytes: Uint8Array): void { + runtime.log("=== AUTO-DETECT DECOMPRESSION ==="); + runtime.log("decompressSync() detects gzip / zlib / deflate automatically."); + + const formats: Array<{ label: string; compressed: Uint8Array }> = [ + { label: "gzip", compressed: gzipSync(jsonBytes) }, + { label: "zlib", compressed: zlibSync(jsonBytes) }, + { label: "deflate", compressed: deflateSync(jsonBytes) }, + ]; + + for (const { label, compressed } of formats) { + const decompressed = decompressSync(compressed); + const records = (JSON.parse(strFromU8(decompressed)) as unknown[]).length; + runtime.log(`From ${label.padEnd(7)}: ${records} records (${formatBytes(compressed.length)})`); + } +} + +function demoCompressionLevels(runtime: Runtime, jsonBytes: Uint8Array): void { + runtime.log("=== COMPRESSION LEVELS ==="); + runtime.log("Level 0 = store only | Level 4 | Level 9 = max"); + + const levels = [ + { level: 0 as const, label: "Level 0 (store)" }, + { level: 4 as const, label: "Level 4" }, + { level: 9 as const, label: "Level 9 (max)" }, + ]; + + for (const { level, label } of levels) { + const compressed = gzipSync(jsonBytes, { level }); + runtime.log(`${label}: ${formatBytes(compressed.length)} (${reductionPct(jsonBytes.length, compressed.length)})`); + } +} + +function demoZip(runtime: Runtime, rawJson: string): void { + runtime.log("=== ZIP ARCHIVES ==="); + runtime.log("Multi-file archiving — no Node.js zlib equivalent."); + + const allComments = JSON.parse(rawJson) as unknown[]; + const third = Math.floor(allComments.length / 3); + + const files = { + "batch-1.json": strToU8(JSON.stringify(allComments.slice(0, third))), + "batch-2.json": strToU8(JSON.stringify(allComments.slice(third, 2 * third))), + "batch-3.json": strToU8(JSON.stringify(allComments.slice(2 * third))), + "manifest.json": strToU8(JSON.stringify({ + total: allComments.length, + files: ["batch-1.json", "batch-2.json", "batch-3.json"], + })), + }; + + const zipped = zipSync(files); + runtime.log(`Archive: ${formatBytes(zipped.length)} (4 files, ${allComments.length} total records)`); + + const unzipped = unzipSync(zipped); + const batch1 = JSON.parse(strFromU8(unzipped["batch-1.json"])) as unknown[]; + const batch2 = JSON.parse(strFromU8(unzipped["batch-2.json"])) as unknown[]; + const batch3 = JSON.parse(strFromU8(unzipped["batch-3.json"])) as unknown[]; + runtime.log(`Extracted: ${Object.keys(unzipped).join(", ")}`); + runtime.log(`Verified: ${batch1.length + batch2.length + batch3.length} records across 3 batches`); +} + +function demoStringUtils(runtime: Runtime, rawJson: string): void { + runtime.log("=== STRING UTILITIES ==="); + runtime.log("strToU8 / strFromU8 — alternative to Buffer.from() / TextEncoder / TextDecoder"); + + const bytes = strToU8(rawJson); + runtime.log(`strToU8(rawJson): ${formatBytes(bytes.length)} Uint8Array`); + + const restored = strFromU8(bytes); + runtime.log(`strFromU8(bytes): ${(JSON.parse(restored) as unknown[]).length} records decoded`); + + const hello = strToU8("Hello, CRE Workflow!"); + runtime.log(`strToU8("Hello, CRE Workflow!"): [${hello.join(", ")}]`); + runtime.log(`strFromU8(...): "${strFromU8(hello)}"`); +} + +// ============================================================================ +// Main Workflow Handler +// ============================================================================ + +const onCronTrigger = (runtime: Runtime): string => { + runtime.log("========================================"); + runtime.log("FFLATE COMPRESSION LIBRARIES DEMO"); + runtime.log("Alternative to Node.js zlib module"); + runtime.log("Compatible with QuickJS / CRE Workflows"); + runtime.log("========================================"); + runtime.log(""); + + runtime.log("Fetching payload from JSONPlaceholder /comments..."); + const httpClient = new HTTPClient(); + const rawJson = httpClient + .sendRequest(runtime, fetchComments, consensusIdenticalAggregation())() + .result(); + + const jsonBytes = strToU8(rawJson); + const recordCount = (JSON.parse(rawJson) as unknown[]).length; + runtime.log(`Fetched: ${formatBytes(jsonBytes.length)} — ${recordCount} records`); + runtime.log(""); + + demoGzip(runtime, jsonBytes); + runtime.log(""); + + demoDeflate(runtime, jsonBytes); + runtime.log(""); + + demoZlib(runtime, jsonBytes); + runtime.log(""); + + demoAutoDetect(runtime, jsonBytes); + runtime.log(""); + + demoCompressionLevels(runtime, jsonBytes); + runtime.log(""); + + demoZip(runtime, rawJson); + runtime.log(""); + + demoStringUtils(runtime, rawJson); + runtime.log(""); + + runtime.log("========================================"); + runtime.log("DEMO COMPLETE"); + runtime.log("========================================"); + + return "Compression demo completed successfully"; +}; + +export const initWorkflow = (config: Config) => { + const cron = new CronCapability(); + + return [ + handler( + cron.trigger({ schedule: config.schedule }), + onCronTrigger + ), + ]; +}; + +export async function main() { + const runner = await Runner.newRunner(); + await runner.run(initWorkflow); +} diff --git a/building-blocks/compression-utils/compression-utils-ts/workflow/package.json b/building-blocks/compression-utils/compression-utils-ts/workflow/package.json new file mode 100644 index 00000000..bf290586 --- /dev/null +++ b/building-blocks/compression-utils/compression-utils-ts/workflow/package.json @@ -0,0 +1,17 @@ +{ + "name": "typescript-simple-template", + "version": "1.0.0", + "main": "dist/main.js", + "private": true, + "scripts": { + "typecheck": "tsc --noEmit" + }, + "license": "UNLICENSED", + "dependencies": { + "@chainlink/cre-sdk": "^1.5.0", + "fflate": "^0.8.2" + }, + "devDependencies": { + "typescript": "5.9.3" + } +} diff --git a/building-blocks/compression-utils/compression-utils-ts/workflow/tsconfig.json b/building-blocks/compression-utils/compression-utils-ts/workflow/tsconfig.json new file mode 100644 index 00000000..d142bddd --- /dev/null +++ b/building-blocks/compression-utils/compression-utils-ts/workflow/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "esnext", + "module": "ESNext", + "moduleResolution": "bundler", + "lib": ["ESNext"], + "outDir": "./dist", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "types": [] + }, + "include": ["main.ts"] +} diff --git a/building-blocks/compression-utils/compression-utils-ts/workflow/workflow.yaml b/building-blocks/compression-utils/compression-utils-ts/workflow/workflow.yaml new file mode 100644 index 00000000..45b0e62b --- /dev/null +++ b/building-blocks/compression-utils/compression-utils-ts/workflow/workflow.yaml @@ -0,0 +1,34 @@ +# ========================================================================== +# CRE WORKFLOW SETTINGS FILE +# ========================================================================== +# Workflow-specific settings for CRE CLI targets. +# Each target defines user-workflow and workflow-artifacts groups. +# Settings here override CRE Project Settings File values. +# +# Example custom target: +# my-target: +# user-workflow: +# workflow-name: "MyExampleWorkflow" # Required: Workflow Registry name +# workflow-artifacts: +# workflow-path: "./main.ts" # Path to workflow entry point +# config-path: "./config.yaml" # Path to config file +# secrets-path: "../secrets.yaml" # Path to secrets file (project root by default) + +# ========================================================================== +staging-settings: + user-workflow: + workflow-name: "workflow-staging" + workflow-artifacts: + workflow-path: "./main.ts" + config-path: "./config.staging.json" + secrets-path: "" + + +# ========================================================================== +production-settings: + user-workflow: + workflow-name: "workflow-production" + workflow-artifacts: + workflow-path: "./main.ts" + config-path: "./config.production.json" + secrets-path: "" \ No newline at end of file diff --git a/building-blocks/xml-utils/README.md b/building-blocks/xml-utils/README.md new file mode 100644 index 00000000..c68345bd --- /dev/null +++ b/building-blocks/xml-utils/README.md @@ -0,0 +1,70 @@ +
+ + Chainlink logo + + +[![License](https://img.shields.io/badge/license-MIT-blue)](https://github.com/smartcontractkit/cre-templates/blob/main/LICENSE) +[![CRE Home](https://img.shields.io/static/v1?label=CRE&message=Home&color=blue)](https://chain.link/chainlink-runtime-environment) +[![CRE Documentation](https://img.shields.io/static/v1?label=CRE&message=Docs&color=blue)](https://docs.chain.link/cre) + +
+ +# XML Utils - CRE Building Block + +This building block demonstrates how to parse, validate, and build XML in CRE workflows using **fast-xml-parser** as an alternative to the browser `DOMParser` and Node.js `xml2js`. + +## The Problem + +The CRE TypeScript SDK runs on **QuickJS**, a lightweight JavaScript engine that does not support browser or Node.js XML APIs. This means the standard `DOMParser` and `xml2js` modules are **not available** in CRE workflows. + +## The Solution + +[fast-xml-parser](https://github.com/NaturalIntelligence/fast-xml-parser) is a pure JavaScript XML library that works in QuickJS. It provides: + +- **Pure JavaScript** - No native dependencies, works in any JS environment +- **Bidirectional** - Parse XML to JS objects and build XML from JS objects +- **Attribute-aware** - Exposes XML attributes as typed properties +- **Namespace-aware** - Can strip namespace prefixes from element names automatically +- **Standards-compliant** - Handles real-world XML including SDMX, RSS, Atom, and custom schemas + +--- + +## What's Covered + +This template demonstrates: + +| Operation | API | Use Case | +|-----------|-----|----------| +| **Validation** | `XMLValidator.validate()` | Validate XML before parsing; get structured error info instead of thrown exceptions | +| **Parsing** | `XMLParser.parse()` | Parse XML with attributes, namespace stripping, and automatic type casting | +| **Array Consistency** | `isArray` option | Force single child elements to parse as arrays instead of objects | +| **XML Building** | `XMLBuilder.build()` | Reconstruct or transform XML from a parsed and filtered object | + +## Get Started + +- **TypeScript**: See the [TypeScript README](./xml-utils-ts/README.md) for detailed setup, API mapping table, example output, and use cases. + +## Quick Example + +```typescript +// Instead of DOMParser: +// const doc = new DOMParser().parseFromString(xml, 'text/xml'); +// const rate = doc.querySelector('rate')?.getAttribute('value'); + +// Use fast-xml-parser: +import { XMLParser } from "fast-xml-parser"; + +const parser = new XMLParser({ + ignoreAttributes: false, + attributeNamePrefix: "@_", + parseAttributeValue: true, +}); +const result = parser.parse(xml); +const rate = result.root.rate["@_value"]; +``` + +## Reference Documentation + +- [fast-xml-parser GitHub](https://github.com/NaturalIntelligence/fast-xml-parser) +- [fast-xml-parser Documentation](https://naturalintelligence.github.io/fast-xml-parser/) +- [CRE Documentation](https://docs.chain.link/cre) diff --git a/building-blocks/xml-utils/xml-utils-ts/.gitignore b/building-blocks/xml-utils/xml-utils-ts/.gitignore new file mode 100644 index 00000000..03bd4129 --- /dev/null +++ b/building-blocks/xml-utils/xml-utils-ts/.gitignore @@ -0,0 +1 @@ +*.env diff --git a/building-blocks/xml-utils/xml-utils-ts/README.md b/building-blocks/xml-utils/xml-utils-ts/README.md new file mode 100644 index 00000000..f868cff3 --- /dev/null +++ b/building-blocks/xml-utils/xml-utils-ts/README.md @@ -0,0 +1,208 @@ +# XML Utils - CRE Building Block (TypeScript) + +**⚠️ DISCLAIMER** + +This tutorial represents an educational example to use a Chainlink system, product, or service and is provided to demonstrate how to interact with Chainlink's systems, products, and services to integrate them into your own. This template is provided "AS IS" and "AS AVAILABLE" without warranties of any kind, it has not been audited, and it may be missing key checks or error handling to make the usage of the system, product or service more clear. Do not use the code in this example in a production environment without completing your own audits and application of best practices. Neither Chainlink Labs, the Chainlink Foundation, nor Chainlink node operators are responsible for unintended outputs that are generated due to errors in code. + +--- + +This building block demonstrates how to parse, validate, and build XML in CRE TypeScript workflows. Since the CRE TypeScript SDK runs on QuickJS (a lightweight JavaScript engine), the standard browser `DOMParser` and Node.js `xml2js` are not available. This template shows how to use **fast-xml-parser** as a drop-in alternative. + +## Why fast-xml-parser? + +[fast-xml-parser](https://github.com/NaturalIntelligence/fast-xml-parser) is: +- **Pure JavaScript** - No native dependencies, works in any JS environment including QuickJS +- **Bidirectional** - Parse XML to JS objects and build XML from JS objects +- **Attribute-aware** - Exposes XML attributes as typed properties, not just text nodes +- **Namespace-aware** - Can strip namespace prefixes from element names automatically +- **Standards-compliant** - Handles real-world XML including SDMX, RSS, Atom, and custom schemas + +## Features Demonstrated + +This workflow fetches live ECB EUR FX reference rates from the [ECB Data API](https://data-api.ecb.europa.eu) (SDMX format) and demonstrates: + +| Demo | Description | +|------|-------------| +| **Validation** | Validate XML before parsing — get structured error info instead of a thrown exception | +| **Parsing** | Parse XML with attributes, namespace stripping, and automatic type casting | +| **isArray** | Force consistent array shape for elements that may appear once or many times | +| **XMLBuilder** | Reconstruct a clean, flat XML document from a parsed and filtered object | + +--- + +## Node.js / Browser API to fast-xml-parser Mapping + +Use this table to migrate from `DOMParser` or `xml2js` to fast-xml-parser: + +| DOMParser / xml2js | fast-xml-parser | Code Reference | +|--------------------|-----------------|----------------| +| `try { new DOMParser().parseFromString(xml, ...) }` | `XMLValidator.validate(xml)` | [main.ts:84](./workflow/main.ts#L84) | +| `new DOMParser().parseFromString(xml, 'text/xml')` | `new XMLParser(opts).parse(xml)` | [main.ts:102](./workflow/main.ts#L102) | +| `element.getAttribute('rate')` (always string) | `parseAttributeValue: true` | [main.ts:106](./workflow/main.ts#L106) | +| `element.localName` (strips namespace manually) | `removeNSPrefix: true` | [main.ts:105](./workflow/main.ts#L105) | +| `querySelectorAll('entry')` (always a NodeList) | `isArray: (name) => name === 'entry'` | [main.ts:142](./workflow/main.ts#L142) | +| `xml2js` `explicitArray: true` (default) | `isArray: (name) => ...` | [main.ts:142](./workflow/main.ts#L142) | +| `document.createElement` + `XMLSerializer` | `new XMLBuilder(opts).build(obj)` | [main.ts:190](./workflow/main.ts#L190) | +| `xml2js.Builder.buildObject(obj)` | `new XMLBuilder(opts).build(obj)` | [main.ts:190](./workflow/main.ts#L190) | + +--- + +## Dependencies + +```json +{ + "fast-xml-parser": "^4.4.0" +} +``` + +--- + +## Setup and Prerequisites + +1. **Install CRE CLI** + ```bash + # See https://docs.chain.link/cre for installation instructions + ``` + +2. **Login to CRE** + ```bash + cre login + ``` + +3. **Install Bun** (if not already installed) + ```bash + # See https://bun.sh/docs/installation + ``` + +4. **Install dependencies** + ```bash + cd building-blocks/xml-utils/xml-utils-ts/workflow + bun install + ``` + +--- + +## Running the Workflow + +### Simulate the workflow + +From the project root directory (`xml-utils-ts`): + +```bash +cre workflow simulate workflow +``` + +--- + +## Example Output + +When the workflow runs, it logs the output of each XML operation: + +``` +======================================== +FAST-XML-PARSER DEMO +Alternative to xml2js / DOMParser +Compatible with QuickJS / CRE Workflows +======================================== + + +=== VALIDATION — XMLValidator.validate() === +Alternative to: try/catch around DOMParser.parseFromString() + +Validating ECB XML... +ECB XML valid: true + +Validating malformed XML... +Malformed XML — code: InvalidAttr, line: 1 +Attributes for 'entry' have open quote. + +=== PARSING — attributes + namespace removal + type casting === +Alternative to: xml2js / DOMParser with manual attribute handling and type casting + +Date: 2026-04-17 +Series: 7 + +1 EUR = 1.6438 AUD +1 EUR = 1.6129 CAD +1 EUR = 0.9231 CHF +1 EUR = 8.0483 CNY +1 EUR = 0.87168 GBP +1 EUR = 187.72 JPY +1 EUR = 1.1797 USD + +=== isArray — force consistent array shape === +Alternative to: DOMParser querySelectorAll() / xml2js explicitArray:true — both always return a collection + +1 entry, isArray off -> Array.isArray=false (object — .map() will throw) +1 entry, isArray on -> Array.isArray=true (always safe) +2 entries,isArray off -> Array.isArray=true (naturally an array) + +=== XML BUILDER — parse -> filter -> rebuild === +Alternative to: DOM createElement/setAttribute + XMLSerializer, or xml2js.Builder.buildObject() + +Compact (356 chars): + + + + + + + + +======================================== +DEMO COMPLETE +======================================== +``` + +--- + +## Use Cases + +### 1. Parsing Financial Data Feeds +Consume XML-native price feeds from central banks, financial data providers, or government agencies: +```typescript +const parser = new XMLParser({ + ignoreAttributes: false, + attributeNamePrefix: "@_", + removeNSPrefix: true, + parseAttributeValue: true, +}); +const result = parser.parse(rawXml) as Record; +``` + +### 2. Validating External XML Before Use +Check XML validity before parsing to surface structured errors instead of silent failures: +```typescript +const valid = XMLValidator.validate(rawXml); +if (valid !== true) { + throw new Error(`Invalid XML at line ${valid.err.line}: ${valid.err.msg}`); +} +``` + +### 3. Transforming and Subsetting XML +Parse a verbose upstream XML response, filter it down, and rebuild a clean output: +```typescript +const parser = new XMLParser({ ignoreAttributes: false, attributeNamePrefix: "@_" }); +const builder = new XMLBuilder({ ignoreAttributes: false, attributeNamePrefix: "@_", format: true }); + +const parsed = parser.parse(rawXml); +const filtered = { FXRates: { Rate: parsed.rates.filter(...) } }; +const output = builder.build(filtered); +``` + +--- + +## Reference Documentation + +- [CRE Documentation](https://docs.chain.link/cre) +- [fast-xml-parser GitHub](https://github.com/NaturalIntelligence/fast-xml-parser) +- [fast-xml-parser docs](https://naturalintelligence.github.io/fast-xml-parser/) +- [ECB Data API](https://data-api.ecb.europa.eu) + +--- + +## License + +MIT - see the repository's [LICENSE](https://github.com/smartcontractkit/cre-templates/blob/main/LICENSE). diff --git a/building-blocks/xml-utils/xml-utils-ts/project.yaml b/building-blocks/xml-utils/xml-utils-ts/project.yaml new file mode 100644 index 00000000..3b25aacf --- /dev/null +++ b/building-blocks/xml-utils/xml-utils-ts/project.yaml @@ -0,0 +1,44 @@ +# ========================================================================== +# CRE PROJECT SETTINGS FILE +# ========================================================================== +# Project-specific settings for CRE CLI targets. +# Each target defines cre-cli, account, and rpcs groups. +# +# Example custom target: +# my-target: +# account: +# workflow-owner-address: "0x123..." # Optional: Owner wallet/MSIG address (used for --unsigned transactions) +# rpcs: +# - chain-name: ethereum-testnet-sepolia # Required if your workflow interacts with this chain +# url: "" +# +# RPC URLs support ${VAR_NAME} syntax to reference environment variables. +# This keeps secrets out of project.yaml (which is committed to git). +# Variables are resolved from your .env file or exported shell variables. +# Example: +# - chain-name: ethereum-testnet-sepolia +# url: https://rpc.example.com/${CRE_SECRET_RPC_SEPOLIA} +# +# Experimental chains (automatically used by the simulator when present): +# Use this for chains not yet in official chain-selectors (e.g., hackathons, new chain integrations). +# In your workflow, reference the chain as evm:ChainSelector:@1.0.0 +# +# experimental-chains: +# - chain-selector: 12345 # The chain selector value +# rpc-url: "https://rpc.example.com" # RPC endpoint URL +# forwarder: "0x..." # Forwarder contract address on the chain + +# ========================================================================== +staging-settings: + rpcs: + - chain-name: ethereum-testnet-sepolia + url: https://ethereum-sepolia-rpc.publicnode.com + +# ========================================================================== +production-settings: + rpcs: + - chain-name: ethereum-testnet-sepolia + url: https://ethereum-sepolia-rpc.publicnode.com + diff --git a/building-blocks/xml-utils/xml-utils-ts/secrets.yaml b/building-blocks/xml-utils/xml-utils-ts/secrets.yaml new file mode 100644 index 00000000..7b85d864 --- /dev/null +++ b/building-blocks/xml-utils/xml-utils-ts/secrets.yaml @@ -0,0 +1 @@ +secretsNames: diff --git a/building-blocks/xml-utils/xml-utils-ts/workflow/config.production.json b/building-blocks/xml-utils/xml-utils-ts/workflow/config.production.json new file mode 100644 index 00000000..1a360cb3 --- /dev/null +++ b/building-blocks/xml-utils/xml-utils-ts/workflow/config.production.json @@ -0,0 +1,3 @@ +{ + "schedule": "*/30 * * * * *" +} diff --git a/building-blocks/xml-utils/xml-utils-ts/workflow/config.staging.json b/building-blocks/xml-utils/xml-utils-ts/workflow/config.staging.json new file mode 100644 index 00000000..1a360cb3 --- /dev/null +++ b/building-blocks/xml-utils/xml-utils-ts/workflow/config.staging.json @@ -0,0 +1,3 @@ +{ + "schedule": "*/30 * * * * *" +} diff --git a/building-blocks/xml-utils/xml-utils-ts/workflow/main.ts b/building-blocks/xml-utils/xml-utils-ts/workflow/main.ts new file mode 100644 index 00000000..d4376215 --- /dev/null +++ b/building-blocks/xml-utils/xml-utils-ts/workflow/main.ts @@ -0,0 +1,248 @@ +import { + CronCapability, + consensusIdenticalAggregation, + handler, + HTTPClient, + type HTTPSendRequester, + Runner, + text, + type Runtime, +} from "@chainlink/cre-sdk"; + +import { XMLParser, XMLBuilder, XMLValidator } from "fast-xml-parser"; + +export type Config = { + schedule: string; +}; + +// ============================================================================ +// HTTP Fetch — ECB Daily EUR FX Reference Rates (SDMX format) +// +// XML structure (simplified, after removeNSPrefix strips message:/generic:): +// +// +// +// +// +// ... +// +// +// +// +// +// +// ... +// +// +// ============================================================================ + +const fetchECBRates = (sendRequester: HTTPSendRequester): string => { + const resp = sendRequester + .sendRequest({ + url: "https://data-api.ecb.europa.eu/service/data/EXR/D.USD+GBP+JPY+CHF+CNY+AUD+CAD.EUR.SP00.A?format=genericData&lastNObservations=1", + method: "GET", + }) + .result(); + return text(resp); +}; + +type SDMXKeyValue = { "@_id": string; "@_value": string | number }; + +type SDMXSeries = { + SeriesKey: { Value: SDMXKeyValue | SDMXKeyValue[] }; + Obs: { + ObsDimension: { "@_value": string }; + ObsValue: { "@_value": number }; + }; +}; + +type ECBData = { + Header: { Prepared: string }; + DataSet: { Series: SDMXSeries | SDMXSeries[] }; +}; + +function getDataSet(result: Record): ECBData { + const data = result["GenericData"] as ECBData | undefined; + if (!data) throw new Error(`ECB XML: GenericData not found. Keys: ${Object.keys(result).join(", ")}`); + return data; +} + +// SDMX stores currency as a Value node with id="CURRENCY" rather than as a direct attribute. +function getCurrency(series: SDMXSeries): string { + const vals = Array.isArray(series.SeriesKey.Value) + ? series.SeriesKey.Value + : [series.SeriesKey.Value]; + return String(vals.find((v) => v["@_id"] === "CURRENCY")?.["@_value"] ?? "???"); +} + +function demoValidation(runtime: Runtime, rawXml: string): void { + runtime.log("=== VALIDATION — XMLValidator.validate() ==="); + runtime.log("Alternative to: try/catch around DOMParser.parseFromString()"); + runtime.log(""); + + runtime.log("Validating ECB XML..."); + const valid = XMLValidator.validate(rawXml); + runtime.log(`ECB XML valid: ${valid === true}`); + runtime.log(""); + + runtime.log("Validating malformed XML..."); + const malformed = ` GenericData, generic:Series -> Series + parseAttributeValue: true, // ObsValue value="1.1797" -> 1.1797 (number, not string) + }); + + const result = parser.parse(rawXml) as Record; + const ecb = getDataSet(result); + const rawSeries = ecb.DataSet.Series; + const series = Array.isArray(rawSeries) ? rawSeries : [rawSeries]; + const date = series[0]?.Obs?.ObsDimension["@_value"] ?? "unknown"; + + runtime.log(`Date: ${date}`); + runtime.log(`Series: ${series.length}`); + runtime.log(""); + + for (const s of series) { + const ccy = getCurrency(s); + const rate = s.Obs.ObsValue["@_value"]; + runtime.log(` 1 EUR = ${rate} ${ccy}`); + } +} + +// ============================================================================ +// isArray — force consistent array shape +// Without isArray, a single child element parses as an object, not an array. +// This silently breaks any code that calls .map()/.filter() on the result. +// ============================================================================ + +function demoIsArray(runtime: Runtime): void { + runtime.log("=== isArray — force consistent array shape ==="); + runtime.log("Alternative to: DOMParser querySelectorAll() / xml2js explicitArray:true — both always return a collection"); + runtime.log(""); + + const single = `Only Entry`; + const multi = `AB`; + + const opts = { ignoreAttributes: false, attributeNamePrefix: "@_" }; + const base = new XMLParser(opts); + const forced = new XMLParser({ ...opts, isArray: (name: string) => name === "entry" }); + + const singleBase = (base.parse(single) as { feed: { entry: unknown } }).feed.entry; + const singleForced = (forced.parse(single) as { feed: { entry: unknown[] } }).feed.entry; + const multiBase = (base.parse(multi) as { feed: { entry: unknown[] } }).feed.entry; + + runtime.log(`1 entry, isArray off -> Array.isArray=${Array.isArray(singleBase)} (object — .map() will throw)`); + runtime.log(`1 entry, isArray on -> Array.isArray=${Array.isArray(singleForced)} (always safe)`); + runtime.log(`2 entries,isArray off -> Array.isArray=${Array.isArray(multiBase)} (naturally an array)`); +} + +// ============================================================================ +// XML BUILDER — parse -> filter -> rebuild +// Use when a workflow needs to transform or subset XML before passing it on. +// ============================================================================ + +function demoXMLBuilder(runtime: Runtime, rawXml: string): void { + runtime.log("=== XML BUILDER — parse -> filter -> rebuild ==="); + runtime.log("Alternative to: DOM createElement/setAttribute + XMLSerializer, or xml2js.Builder.buildObject()"); + runtime.log(""); + + const parser = new XMLParser({ + ignoreAttributes: false, + attributeNamePrefix: "@_", + removeNSPrefix: true, + parseAttributeValue: true, + }); + + const result = parser.parse(rawXml) as Record; + const ecb = getDataSet(result); + const rawSeries = ecb.DataSet.Series; + const series = Array.isArray(rawSeries) ? rawSeries : [rawSeries]; + const date = series[0]?.Obs?.ObsDimension["@_value"] ?? "unknown"; + + const outputObj = { + FXRates: { + "@_source": "ECB", + "@_date": date, + "@_base": "EUR", + Rate: series.map((s: SDMXSeries) => ({ + "@_currency": getCurrency(s), + "@_rate": s.Obs.ObsValue["@_value"], + })), + }, + }; + + const builderOpts = { ignoreAttributes: false, attributeNamePrefix: "@_" }; + + const compact = new XMLBuilder(builderOpts).build(outputObj) as string; + runtime.log(`Compact (${compact.length} chars): ${compact.slice(0, 120)}...`); + + const formatted = new XMLBuilder({ ...builderOpts, format: true, indentBy: " " }).build(outputObj) as string; + runtime.log(`Formatted (${formatted.length} chars):`); + formatted.split("\n").slice(0, 8).forEach((l: string) => runtime.log(` ${l}`)); +} + +// ============================================================================ +// Main Workflow Handler +// ============================================================================ + +export const onCronTrigger = (runtime: Runtime): string => { + runtime.log("========================================"); + runtime.log("FAST-XML-PARSER DEMO"); + runtime.log("Alternative to xml2js / DOMParser"); + runtime.log("Compatible with QuickJS / CRE Workflows"); + runtime.log("========================================"); + runtime.log(""); + + const httpClient = new HTTPClient(); + const rawXml = httpClient + .sendRequest(runtime, fetchECBRates, consensusIdenticalAggregation())() + .result(); + + demoValidation(runtime, rawXml); + runtime.log(""); + + demoParsing(runtime, rawXml); + runtime.log(""); + + demoIsArray(runtime); + runtime.log(""); + + demoXMLBuilder(runtime, rawXml); + runtime.log(""); + + runtime.log("========================================"); + runtime.log("DEMO COMPLETE"); + runtime.log("========================================"); + + return "XML parsing demo completed successfully"; +}; + +export const initWorkflow = (config: Config) => { + const cron = new CronCapability(); + + return [ + handler( + cron.trigger({ schedule: config.schedule }), + onCronTrigger + ), + ]; +}; + +export async function main() { + const runner = await Runner.newRunner(); + await runner.run(initWorkflow); +} diff --git a/building-blocks/xml-utils/xml-utils-ts/workflow/package.json b/building-blocks/xml-utils/xml-utils-ts/workflow/package.json new file mode 100644 index 00000000..bb3f2c40 --- /dev/null +++ b/building-blocks/xml-utils/xml-utils-ts/workflow/package.json @@ -0,0 +1,17 @@ +{ + "name": "typescript-simple-template", + "version": "1.0.0", + "main": "dist/main.js", + "private": true, + "scripts": { + "typecheck": "tsc --noEmit" + }, + "license": "UNLICENSED", + "dependencies": { + "@chainlink/cre-sdk": "^1.5.0", + "fast-xml-parser": "^4.4.0" + }, + "devDependencies": { + "typescript": "5.9.3" + } +} diff --git a/building-blocks/xml-utils/xml-utils-ts/workflow/tsconfig.json b/building-blocks/xml-utils/xml-utils-ts/workflow/tsconfig.json new file mode 100644 index 00000000..d142bddd --- /dev/null +++ b/building-blocks/xml-utils/xml-utils-ts/workflow/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "esnext", + "module": "ESNext", + "moduleResolution": "bundler", + "lib": ["ESNext"], + "outDir": "./dist", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "types": [] + }, + "include": ["main.ts"] +} diff --git a/building-blocks/xml-utils/xml-utils-ts/workflow/workflow.yaml b/building-blocks/xml-utils/xml-utils-ts/workflow/workflow.yaml new file mode 100644 index 00000000..45b0e62b --- /dev/null +++ b/building-blocks/xml-utils/xml-utils-ts/workflow/workflow.yaml @@ -0,0 +1,34 @@ +# ========================================================================== +# CRE WORKFLOW SETTINGS FILE +# ========================================================================== +# Workflow-specific settings for CRE CLI targets. +# Each target defines user-workflow and workflow-artifacts groups. +# Settings here override CRE Project Settings File values. +# +# Example custom target: +# my-target: +# user-workflow: +# workflow-name: "MyExampleWorkflow" # Required: Workflow Registry name +# workflow-artifacts: +# workflow-path: "./main.ts" # Path to workflow entry point +# config-path: "./config.yaml" # Path to config file +# secrets-path: "../secrets.yaml" # Path to secrets file (project root by default) + +# ========================================================================== +staging-settings: + user-workflow: + workflow-name: "workflow-staging" + workflow-artifacts: + workflow-path: "./main.ts" + config-path: "./config.staging.json" + secrets-path: "" + + +# ========================================================================== +production-settings: + user-workflow: + workflow-name: "workflow-production" + workflow-artifacts: + workflow-path: "./main.ts" + config-path: "./config.production.json" + secrets-path: "" \ No newline at end of file From 248e42ebfdf7dee1ea1f053f9a7d572166b41267 Mon Sep 17 00:00:00 2001 From: Russell Stern Date: Fri, 24 Apr 2026 10:44:25 -0400 Subject: [PATCH 2/5] Updated readme --- building-blocks/README.md | 1 + .../compression-utils-ts/workflow/README.md | 27 ------------------- 2 files changed, 1 insertion(+), 27 deletions(-) delete mode 100644 building-blocks/compression-utils/compression-utils-ts/workflow/README.md diff --git a/building-blocks/README.md b/building-blocks/README.md index 280c9eb0..675ecc88 100644 --- a/building-blocks/README.md +++ b/building-blocks/README.md @@ -87,6 +87,7 @@ Path: [`./xml-utils`](./xml-utils) * **read-data-feeds**: You want to **read on-chain data** via contract calls, manage ABIs/bindings, and configure **RPC** access. * **compression-utils**: You want to handle **data compression** in CRE Typescript workflows, especially when working with large datasets and the CRE HTTP response limit. * **xml-utils**: You want to work with **XML data** in CRE workflows, including parsing, validating, and building XML documents. + --- ## License diff --git a/building-blocks/compression-utils/compression-utils-ts/workflow/README.md b/building-blocks/compression-utils/compression-utils-ts/workflow/README.md deleted file mode 100644 index dfe20076..00000000 --- a/building-blocks/compression-utils/compression-utils-ts/workflow/README.md +++ /dev/null @@ -1,27 +0,0 @@ -# Hello World (TypeScript) - -This template provides a blank TypeScript workflow example. It aims to give a starting point for writing a workflow from scratch and to get started with local simulation. - -Steps to run the example - -## 1. Update .env file - -You need to add a private key to env file. This is specifically required if you want to simulate chain writes. For that to work the key should be valid and funded. -If your workflow does not do any chain write then you can just put any dummy key as a private key. e.g. -``` -CRE_ETH_PRIVATE_KEY=0000000000000000000000000000000000000000000000000000000000000001 -``` - -## 2. Install dependencies -```bash -bun install -``` - -## 3. Simulate the workflow -Run the command from project root directory - -```bash -cre workflow simulate --target=staging-settings -``` - -It is recommended to look into other existing examples to see how to write a workflow. You can generate them by running the `cre init` command. From 37b9c900795acd511e25fbabe82b4b2255196c12 Mon Sep 17 00:00:00 2001 From: Russell Stern Date: Fri, 24 Apr 2026 12:10:56 -0400 Subject: [PATCH 3/5] Switched to lower http capability limit --- .../compression-utils-ts/README.md | 46 +++++++++---------- .../workflow/config.production.json | 2 +- .../workflow/config.staging.json | 2 +- .../compression-utils-ts/workflow/main.ts | 4 +- 4 files changed, 27 insertions(+), 27 deletions(-) diff --git a/building-blocks/compression-utils/compression-utils-ts/README.md b/building-blocks/compression-utils/compression-utils-ts/README.md index b5dc8e3d..f3ca6aa3 100644 --- a/building-blocks/compression-utils/compression-utils-ts/README.md +++ b/building-blocks/compression-utils/compression-utils-ts/README.md @@ -18,7 +18,7 @@ This building block demonstrates how to use compression and decompression in CRE ## Features Demonstrated -This workflow fetches a large JSON payload (~94KB, 300 records) via HTTP and uses it as the input for each compression demo: +This workflow fetches a large JSON payload (~22KB, 75 records) via HTTP and uses it as the input for each compression demo: | Category | fflate API | Node.js Equivalent | |----------|------------|--------------------| @@ -36,14 +36,14 @@ This workflow fetches a large JSON payload (~94KB, 300 records) via HTTP and use The workflow fetches its payload using the CRE HTTP capability (`HTTPClient`), which routes requests through the DON's off-chain network layer. The standard `fetch` / `node:https` APIs are not available in QuickJS. -> **Note:** The CRE HTTP capability has a response body limit of **100,000 bytes**. This workflow fetches `/comments?_limit=300` (~94KB) to stay within that limit. +> **Note:** The CRE HTTP capability has a response body limit of **25kb**. This workflow fetches `/comments?_limit=75` (~22KB) to stay within that limit. ```typescript import { HTTPClient, consensusIdenticalAggregation, type HTTPSendRequester, text } from "@chainlink/cre-sdk"; const fetchComments = (sendRequester: HTTPSendRequester): string => { const resp = sendRequester - .sendRequest({ url: "https://jsonplaceholder.typicode.com/comments?_limit=300", method: "GET" }) + .sendRequest({ url: "https://jsonplaceholder.typicode.com/comments?_limit=75", method: "GET" }) .result(); return text(resp); }; @@ -135,48 +135,48 @@ Compatible with QuickJS / CRE Workflows ======================================== Fetching payload from JSONPlaceholder /comments... -Fetched: 94.32 KB — 300 records +Fetched: 22.41 KB — 75 records === GZIP / GUNZIP === Alternative to: zlib.gzipSync() / zlib.gunzipSync() -Original: 94.32 KB -Compressed: 21.48 KB (77.2% smaller) -Verified: 300 records restored via gunzipSync +Original: 22.41 KB +Compressed: 6.60 KB (70.5% smaller) +Verified: 75 records restored via gunzipSync === DEFLATE / INFLATE (raw) === Alternative to: zlib.deflateRawSync() / zlib.inflateRawSync() -Original: 94.32 KB -Compressed: 21.46 KB (77.3% smaller) -Verified: 300 records restored via inflateSync +Original: 22.41 KB +Compressed: 6.58 KB (70.6% smaller) +Verified: 75 records restored via inflateSync === ZLIB / UNZLIB === Alternative to: zlib.deflateSync() / zlib.inflateSync() -Original: 94.32 KB -Compressed: 21.48 KB (77.2% smaller) -Verified: 300 records restored via unzlibSync +Original: 22.41 KB +Compressed: 6.59 KB (70.6% smaller) +Verified: 75 records restored via unzlibSync === AUTO-DETECT DECOMPRESSION === decompressSync() detects gzip / zlib / deflate automatically. -From gzip : 300 records (21.48 KB) -From zlib : 300 records (21.48 KB) -From deflate: 300 records (21.46 KB) +From gzip : 75 records (6.60 KB) +From zlib : 75 records (6.59 KB) +From deflate: 75 records (6.58 KB) === COMPRESSION LEVELS === Level 0 = store only | Level 4 | Level 9 = max -Level 0 (store): 94.35 KB (-0.0% smaller) -Level 4: 21.78 KB (76.9% smaller) -Level 9 (max): 21.04 KB (77.7% smaller) +Level 0 (store): 22.43 KB (-0.1% smaller) +Level 4: 6.66 KB (70.3% smaller) +Level 9 (max): 6.60 KB (70.5% smaller) === ZIP ARCHIVES === Multi-file archiving — no Node.js zlib equivalent. -Archive: 21.92 KB (4 files, 300 total records) +Archive: 7.76 KB (4 files, 75 total records) Extracted: batch-1.json, batch-2.json, batch-3.json, manifest.json -Verified: 300 records across 3 batches +Verified: 75 records across 3 batches === STRING UTILITIES === strToU8 / strFromU8 — alternative to Buffer.from() / TextEncoder / TextDecoder -strToU8(rawJson): 94.32 KB Uint8Array -strFromU8(bytes): 300 records decoded +strToU8(rawJson): 22.41 KB Uint8Array +strFromU8(bytes): 75 records decoded strToU8("Hello, CRE Workflow!"): [72, 101, 108, 108, 111, 44, 32, 67, 82, 69, 32, 87, 111, 114, 107, 102, 108, 111, 119, 33] strFromU8(...): "Hello, CRE Workflow!" diff --git a/building-blocks/compression-utils/compression-utils-ts/workflow/config.production.json b/building-blocks/compression-utils/compression-utils-ts/workflow/config.production.json index 1a360cb3..3f6bb772 100644 --- a/building-blocks/compression-utils/compression-utils-ts/workflow/config.production.json +++ b/building-blocks/compression-utils/compression-utils-ts/workflow/config.production.json @@ -1,3 +1,3 @@ { - "schedule": "*/30 * * * * *" + "schedule": "0 */5 * * * *" } diff --git a/building-blocks/compression-utils/compression-utils-ts/workflow/config.staging.json b/building-blocks/compression-utils/compression-utils-ts/workflow/config.staging.json index 1a360cb3..3f6bb772 100644 --- a/building-blocks/compression-utils/compression-utils-ts/workflow/config.staging.json +++ b/building-blocks/compression-utils/compression-utils-ts/workflow/config.staging.json @@ -1,3 +1,3 @@ { - "schedule": "*/30 * * * * *" + "schedule": "0 */5 * * * *" } diff --git a/building-blocks/compression-utils/compression-utils-ts/workflow/main.ts b/building-blocks/compression-utils/compression-utils-ts/workflow/main.ts index 4bf88877..62843dff 100644 --- a/building-blocks/compression-utils/compression-utils-ts/workflow/main.ts +++ b/building-blocks/compression-utils/compression-utils-ts/workflow/main.ts @@ -28,14 +28,14 @@ type Config = { }; // ============================================================================ -// HTTP Fetch — JSONPlaceholder /comments?_limit=300 (~92KB, 300 records) +// HTTP Fetch — JSONPlaceholder /comments?_limit=75 (~24KB, 75 records) // Used as the input payload for all compression demos below. // ============================================================================ const fetchComments = (sendRequester: HTTPSendRequester): string => { const resp = sendRequester .sendRequest({ - url: "https://jsonplaceholder.typicode.com/comments?_limit=300", + url: "https://jsonplaceholder.typicode.com/comments?_limit=75", method: "GET", }) .result(); From f57385aca6e281cb36421055b6ecb6a800970f23 Mon Sep 17 00:00:00 2001 From: Russell Stern Date: Fri, 24 Apr 2026 12:17:44 -0400 Subject: [PATCH 4/5] Updated readmes --- building-blocks/README.md | 2 +- building-blocks/compression-utils/README.md | 4 ++-- .../compression-utils/compression-utils-ts/README.md | 10 +++++----- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/building-blocks/README.md b/building-blocks/README.md index 675ecc88..a4bfb82f 100644 --- a/building-blocks/README.md +++ b/building-blocks/README.md @@ -64,7 +64,7 @@ Path: [`./compression-utils`](./compression-utils) - Demonstrates how to perform compression and decompression operations in CRE workflows using **fflate** as a pure JavaScript alternative to the Node.js `zlib` module. - Covers gzip, deflate, zlib formats, compression levels, ZIP archives, and string utilities. -- Shows how to fetch a large JSON payload via CRE HTTP and compress it to stay within the 100KB response body limit. +- Shows how to fetch a large JSON payload via CRE HTTP and compress it to stay within the 25KB response body limit. 👉 See the block’s README for setup, config, and sample logs. diff --git a/building-blocks/compression-utils/README.md b/building-blocks/compression-utils/README.md index 77a35476..28a79e97 100644 --- a/building-blocks/compression-utils/README.md +++ b/building-blocks/compression-utils/README.md @@ -15,7 +15,7 @@ This building block demonstrates how to perform compression and decompression op ## The Problem -The CRE TypeScript SDK runs on **QuickJS**, a lightweight JavaScript engine that does not support Node.js native modules. This means the standard `zlib` module from Node.js is **not available** in CRE workflows. Additionally, CRE's HTTP capability has a **100KB response body limit**, making compression essential for working with large datasets. +The CRE TypeScript SDK runs on **QuickJS**, a lightweight JavaScript engine that does not support Node.js native modules. This means the standard `zlib` module from Node.js is **not available** in CRE workflows. Additionally, CRE's HTTP capability has a **25KB response body limit**, making compression essential for working with large datasets. ## The Solution @@ -25,7 +25,7 @@ The [fflate library](https://github.com/101arrowz/fflate) provides pure JavaScri - **Standards-compliant** - Output is interoperable with native gzip, zlib, and deflate tools - **Small** - ~8KB minified, tree-shakeable - **Sync-capable** - Provides synchronous APIs (`gzipSync`, `deflateSync`, etc.) that work reliably in QuickJS -- **Practical** - Enable workflows to handle large datasets despite the 100KB HTTP limit +- **Practical** - Enable workflows to handle large datasets despite the 25KB HTTP limit --- diff --git a/building-blocks/compression-utils/compression-utils-ts/README.md b/building-blocks/compression-utils/compression-utils-ts/README.md index f3ca6aa3..282940d2 100644 --- a/building-blocks/compression-utils/compression-utils-ts/README.md +++ b/building-blocks/compression-utils/compression-utils-ts/README.md @@ -36,7 +36,7 @@ This workflow fetches a large JSON payload (~22KB, 75 records) via HTTP and uses The workflow fetches its payload using the CRE HTTP capability (`HTTPClient`), which routes requests through the DON's off-chain network layer. The standard `fetch` / `node:https` APIs are not available in QuickJS. -> **Note:** The CRE HTTP capability has a response body limit of **25kb**. This workflow fetches `/comments?_limit=75` (~22KB) to stay within that limit. +> **Note:** The CRE HTTP capability has a response body limit of **25KB**. This workflow fetches `/comments?_limit=75` (~22KB) to stay within that limit. ```typescript import { HTTPClient, consensusIdenticalAggregation, type HTTPSendRequester, text } from "@chainlink/cre-sdk"; @@ -200,19 +200,19 @@ const compressed = gzipSync(strToU8(payload)); const restored = JSON.parse(strFromU8(gunzipSync(compressed))); ``` -### 2. Decompress API responses to get around 100KB limit -The CRE HTTP capability has a 100KB response body limit. Compress large payloads on the server side and decompress in the workflow: +### 2. Decompress API responses to get around 25KB limit +The CRE HTTP capability has a 25KB response body limit. Compress large payloads on the server side and decompress in the workflow: ```typescript import { gunzipSync, strFromU8 } from "fflate"; // Server sends gzip-compressed data (~25KB) -// which decompresses to the full ~94KB payload +// which decompresses to the full ~80KB payload const compressedData = sendRequester.sendRequest({ url: "https://api.example.com/data?format=gzip" }).result(); const fullData = JSON.parse(strFromU8(gunzipSync(compressedData))); -// Now you can work with the full dataset despite the 100KB limit +// Now you can work with the full dataset despite the 25KB limit ``` ### 3. Auto-detect and decompress unknown formats From 852abc4cb7fc1f7bde2f0c972d3085bdcbfc6121 Mon Sep 17 00:00:00 2001 From: Russell Stern Date: Thu, 30 Apr 2026 13:40:27 -0400 Subject: [PATCH 5/5] Switched compressing files after http response but before consensus --- building-blocks/compression-utils/README.md | 4 +- .../compression-utils-ts/README.md | 62 ++++++++++--------- .../compression-utils-ts/workflow/main.ts | 25 +++++--- .../workflow/package.json | 2 + 4 files changed, 56 insertions(+), 37 deletions(-) diff --git a/building-blocks/compression-utils/README.md b/building-blocks/compression-utils/README.md index 28a79e97..544e1c45 100644 --- a/building-blocks/compression-utils/README.md +++ b/building-blocks/compression-utils/README.md @@ -15,7 +15,7 @@ This building block demonstrates how to perform compression and decompression op ## The Problem -The CRE TypeScript SDK runs on **QuickJS**, a lightweight JavaScript engine that does not support Node.js native modules. This means the standard `zlib` module from Node.js is **not available** in CRE workflows. Additionally, CRE's HTTP capability has a **25KB response body limit**, making compression essential for working with large datasets. +The CRE TypeScript SDK runs on **QuickJS**, a lightweight JavaScript engine that does not support Node.js native modules. This means the standard `zlib` module from Node.js is **not available** in CRE workflows. Additionally, CRE's Consensus capability has a **25KB response body limit**, making compression essential for working with large datasets. ## The Solution @@ -25,7 +25,7 @@ The [fflate library](https://github.com/101arrowz/fflate) provides pure JavaScri - **Standards-compliant** - Output is interoperable with native gzip, zlib, and deflate tools - **Small** - ~8KB minified, tree-shakeable - **Sync-capable** - Provides synchronous APIs (`gzipSync`, `deflateSync`, etc.) that work reliably in QuickJS -- **Practical** - Enable workflows to handle large datasets despite the 25KB HTTP limit +- **Practical** - Enable workflows to handle large datasets despite the 25KB Consensus limit --- diff --git a/building-blocks/compression-utils/compression-utils-ts/README.md b/building-blocks/compression-utils/compression-utils-ts/README.md index 282940d2..3e08a664 100644 --- a/building-blocks/compression-utils/compression-utils-ts/README.md +++ b/building-blocks/compression-utils/compression-utils-ts/README.md @@ -18,7 +18,7 @@ This building block demonstrates how to use compression and decompression in CRE ## Features Demonstrated -This workflow fetches a large JSON payload (~22KB, 75 records) via HTTP and uses it as the input for each compression demo: +This workflow fetches a large JSON payload (~65KB, 215 records) via HTTP, compresses it to pass consensus, decompresses it, and uses it as the input for each compression demo: | Category | fflate API | Node.js Equivalent | |----------|------------|--------------------| @@ -36,14 +36,14 @@ This workflow fetches a large JSON payload (~22KB, 75 records) via HTTP and uses The workflow fetches its payload using the CRE HTTP capability (`HTTPClient`), which routes requests through the DON's off-chain network layer. The standard `fetch` / `node:https` APIs are not available in QuickJS. -> **Note:** The CRE HTTP capability has a response body limit of **25KB**. This workflow fetches `/comments?_limit=75` (~22KB) to stay within that limit. +> **Note:** The CRE HTTP capability has a response body limit of **100KB** but the Consensus capability has a smaller limit **25KB**. This workflow fetches `/comments?_limit=215` (~65KB) and compresses it (~24KB) to stay within those limits. ```typescript import { HTTPClient, consensusIdenticalAggregation, type HTTPSendRequester, text } from "@chainlink/cre-sdk"; const fetchComments = (sendRequester: HTTPSendRequester): string => { const resp = sendRequester - .sendRequest({ url: "https://jsonplaceholder.typicode.com/comments?_limit=75", method: "GET" }) + .sendRequest({ url: "https://jsonplaceholder.typicode.com/comments?_limit=215", method: "GET" }) .result(); return text(resp); }; @@ -134,49 +134,50 @@ Alternative to Node.js zlib module Compatible with QuickJS / CRE Workflows ======================================== -Fetching payload from JSONPlaceholder /comments... -Fetched: 22.41 KB — 75 records +Fetching payload from JSONPlaceholder /comments (compressed over consensus)... +Consensus payload: 23.58 KB (base64-encoded gzip) +Decompressed: 65.04 KB — 215 records === GZIP / GUNZIP === Alternative to: zlib.gzipSync() / zlib.gunzipSync() -Original: 22.41 KB -Compressed: 6.60 KB (70.5% smaller) -Verified: 75 records restored via gunzipSync +Original: 65.04 KB +Compressed: 17.71 KB (72.8% smaller) +Verified: 215 records restored via gunzipSync === DEFLATE / INFLATE (raw) === Alternative to: zlib.deflateRawSync() / zlib.inflateRawSync() -Original: 22.41 KB -Compressed: 6.58 KB (70.6% smaller) -Verified: 75 records restored via inflateSync +Original: 65.04 KB +Compressed: 17.69 KB (72.8% smaller) +Verified: 215 records restored via inflateSync === ZLIB / UNZLIB === Alternative to: zlib.deflateSync() / zlib.inflateSync() -Original: 22.41 KB -Compressed: 6.59 KB (70.6% smaller) -Verified: 75 records restored via unzlibSync +Original: 65.04 KB +Compressed: 17.70 KB (72.8% smaller) +Verified: 215 records restored via unzlibSync === AUTO-DETECT DECOMPRESSION === decompressSync() detects gzip / zlib / deflate automatically. -From gzip : 75 records (6.60 KB) -From zlib : 75 records (6.59 KB) -From deflate: 75 records (6.58 KB) +From gzip : 215 records (17.71 KB) +From zlib : 215 records (17.70 KB) +From deflate: 215 records (17.69 KB) === COMPRESSION LEVELS === Level 0 = store only | Level 4 | Level 9 = max -Level 0 (store): 22.43 KB (-0.1% smaller) -Level 4: 6.66 KB (70.3% smaller) -Level 9 (max): 6.60 KB (70.5% smaller) +Level 0 (store): 65.07 KB (-0.0% smaller) +Level 4: 17.99 KB (72.3% smaller) +Level 9 (max): 17.68 KB (72.8% smaller) === ZIP ARCHIVES === Multi-file archiving — no Node.js zlib equivalent. -Archive: 7.76 KB (4 files, 75 total records) +Archive: 19.09 KB (4 files, 215 total records) Extracted: batch-1.json, batch-2.json, batch-3.json, manifest.json -Verified: 75 records across 3 batches +Verified: 215 records across 3 batches === STRING UTILITIES === strToU8 / strFromU8 — alternative to Buffer.from() / TextEncoder / TextDecoder -strToU8(rawJson): 22.41 KB Uint8Array -strFromU8(bytes): 75 records decoded +strToU8(rawJson): 65.04 KB Uint8Array +strFromU8(bytes): 215 records decoded strToU8("Hello, CRE Workflow!"): [72, 101, 108, 108, 111, 44, 32, 67, 82, 69, 32, 87, 111, 114, 107, 102, 108, 111, 119, 33] strFromU8(...): "Hello, CRE Workflow!" @@ -201,17 +202,22 @@ const restored = JSON.parse(strFromU8(gunzipSync(compressed))); ``` ### 2. Decompress API responses to get around 25KB limit -The CRE HTTP capability has a 25KB response body limit. Compress large payloads on the server side and decompress in the workflow: +The CRE HTTP capability has a 100KB response body limit but the Consensus capability has a 25KB limit. Compress large payloads before passing it to consensus and decompress in the workflow: ```typescript import { gunzipSync, strFromU8 } from "fflate"; -// Server sends gzip-compressed data (~25KB) -// which decompresses to the full ~80KB payload +// Server sends data (~65KB) +// which compresses to ~24KB consensus payload const compressedData = sendRequester.sendRequest({ url: "https://api.example.com/data?format=gzip" }).result(); -const fullData = JSON.parse(strFromU8(gunzipSync(compressedData))); +const raw = text(resp); +const compressed = gzipSync(strToU8(raw), { level: 9 }); +const compressedBase64 = fromByteArray(compressed); + +const compressedBytes = toByteArray(compressedBase64); +const fullData = JSON.parse(strFromU8(gunzipSync(compressedBytes))); // Now you can work with the full dataset despite the 25KB limit ``` diff --git a/building-blocks/compression-utils/compression-utils-ts/workflow/main.ts b/building-blocks/compression-utils/compression-utils-ts/workflow/main.ts index 62843dff..bdbb9fb6 100644 --- a/building-blocks/compression-utils/compression-utils-ts/workflow/main.ts +++ b/building-blocks/compression-utils/compression-utils-ts/workflow/main.ts @@ -23,23 +23,29 @@ import { zipSync, } from "fflate"; +import { fromByteArray, toByteArray } from "base64-js"; + type Config = { schedule: string; }; // ============================================================================ -// HTTP Fetch — JSONPlaceholder /comments?_limit=75 (~24KB, 75 records) -// Used as the input payload for all compression demos below. +// HTTP Fetch — JSONPlaceholder /comments?_limit=220 (~65KB, 220 records) +// Response is gzip-compressed and base64-encoded so it fits under the +// consensus capability size limit. Decompressed after consensus. // ============================================================================ const fetchComments = (sendRequester: HTTPSendRequester): string => { const resp = sendRequester .sendRequest({ - url: "https://jsonplaceholder.typicode.com/comments?_limit=75", + url: "https://jsonplaceholder.typicode.com/comments?_limit=215", method: "GET", }) .result(); - return text(resp); + const raw = text(resp); + // mtime: 0 makes output deterministic across nodes (required for consensus) + const compressed = gzipSync(strToU8(raw), { level: 9, mtime: 0 }); + return fromByteArray(compressed); }; // ============================================================================ @@ -182,15 +188,20 @@ const onCronTrigger = (runtime: Runtime): string => { runtime.log("========================================"); runtime.log(""); - runtime.log("Fetching payload from JSONPlaceholder /comments..."); + runtime.log("Fetching payload from JSONPlaceholder /comments (compressed over consensus)..."); const httpClient = new HTTPClient(); - const rawJson = httpClient + const compressedB64 = httpClient .sendRequest(runtime, fetchComments, consensusIdenticalAggregation())() .result(); + // Decode base64 and decompress — the large payload only crosses consensus compressed + const compressedBytes = toByteArray(compressedB64); + runtime.log(`Consensus payload: ${formatBytes(compressedB64.length)} (base64-encoded gzip)`); + const rawJson = strFromU8(gunzipSync(compressedBytes)); + const jsonBytes = strToU8(rawJson); const recordCount = (JSON.parse(rawJson) as unknown[]).length; - runtime.log(`Fetched: ${formatBytes(jsonBytes.length)} — ${recordCount} records`); + runtime.log(`Decompressed: ${formatBytes(jsonBytes.length)} — ${recordCount} records`); runtime.log(""); demoGzip(runtime, jsonBytes); diff --git a/building-blocks/compression-utils/compression-utils-ts/workflow/package.json b/building-blocks/compression-utils/compression-utils-ts/workflow/package.json index bf290586..9de2d9ad 100644 --- a/building-blocks/compression-utils/compression-utils-ts/workflow/package.json +++ b/building-blocks/compression-utils/compression-utils-ts/workflow/package.json @@ -9,9 +9,11 @@ "license": "UNLICENSED", "dependencies": { "@chainlink/cre-sdk": "^1.5.0", + "base64-js": "^1.5.1", "fflate": "^0.8.2" }, "devDependencies": { + "@types/base64-js": "^1.3.2", "typescript": "5.9.3" } }