From 7e1b1d658469a7699cac9f4bc01a879ec58793bf Mon Sep 17 00:00:00 2001 From: Pavel Tcholakov Date: Tue, 3 Feb 2026 21:14:14 +0200 Subject: [PATCH 1/4] Add TypeScript e2e OTEL tracing example - Use TextMapGetter in extractTraceContext so all propagator formats (W3C, B3, Jaeger) work without hardcoding header names - Explain in README why Node.js auto-instrumentation can't substitute for manual extraction: Restate wraps the HTTP transport layer and handlers replay, so one span per logical invocation requires extracting from ctx.request().attemptHeaders - Remove committed screenshot (trace.png); add *.png to .gitignore --- .tools/run_node_tests.sh | 1 + typescript/tracing/otel/.gitignore | 19 +++ typescript/tracing/otel/README.md | 135 ++++++++++++++++++ typescript/tracing/otel/package.json | 28 ++++ typescript/tracing/otel/src/client.ts | 91 ++++++++++++ typescript/tracing/otel/src/downstream.ts | 93 ++++++++++++ .../tracing/otel/src/restate-service.ts | 130 +++++++++++++++++ typescript/tracing/otel/tsconfig.json | 18 +++ typescript/tracing/package.json | 28 ++++ typescript/tracing/tsconfig.json | 18 +++ 10 files changed, 561 insertions(+) create mode 100644 typescript/tracing/otel/.gitignore create mode 100644 typescript/tracing/otel/README.md create mode 100644 typescript/tracing/otel/package.json create mode 100644 typescript/tracing/otel/src/client.ts create mode 100644 typescript/tracing/otel/src/downstream.ts create mode 100644 typescript/tracing/otel/src/restate-service.ts create mode 100644 typescript/tracing/otel/tsconfig.json create mode 100644 typescript/tracing/package.json create mode 100644 typescript/tracing/tsconfig.json diff --git a/.tools/run_node_tests.sh b/.tools/run_node_tests.sh index 66be72dd..c485eaf5 100755 --- a/.tools/run_node_tests.sh +++ b/.tools/run_node_tests.sh @@ -10,6 +10,7 @@ function npm_install_check() { } npm_install_check $PROJECT_ROOT/typescript/basics +npm_install_check $PROJECT_ROOT/typescript/tracing npm_install_check $PROJECT_ROOT/typescript/templates/node npm_install_check $PROJECT_ROOT/typescript/templates/lambda diff --git a/typescript/tracing/otel/.gitignore b/typescript/tracing/otel/.gitignore new file mode 100644 index 00000000..39895742 --- /dev/null +++ b/typescript/tracing/otel/.gitignore @@ -0,0 +1,19 @@ +# Node +node_modules +dist + +# screenshots +*.png + +# debug +npm-debug.log* + +# env files +.env* + +# typescript +*.tsbuildinfo + +# Restate +.restate +restate-data diff --git a/typescript/tracing/otel/README.md b/typescript/tracing/otel/README.md new file mode 100644 index 00000000..e5eeecf2 --- /dev/null +++ b/typescript/tracing/otel/README.md @@ -0,0 +1,135 @@ +# End-to-End OpenTelemetry Tracing with Restate + +This example demonstrates distributed tracing across a fictional multi-tier system: + +``` +┌──────────┐ ┌─────────────┐ ┌─────────────────┐ ┌────────────┐ +│ Client │────▶│ Restate │────▶│ Greeter Service │────▶│ Downstream │ +│ App │ │ Server │ │ (SDK/Node) │ │ Service │ +└──────────┘ └─────────────┘ └─────────────────┘ └────────────┘ + │ │ │ │ + │ │ │ │ + ▼ ▼ ▼ ▼ +┌────────────────────────────────────────────────────────────────────────┐ +│ Jaeger │ +└────────────────────────────────────────────────────────────────────────┘ +``` + +**What gets traced:** + +1. **Client App** - Creates the root span and injects W3C trace context into the Restate request +2. **Restate Server** - Receives trace context, emits spans for ingress requests and handler invocations +3. **Greeter Service** - SDK handler that creates custom spans and propagates context to downstream calls +4. **Downstream Service** - Receives and logs the propagated trace headers + +## Prerequisites + +- Node.js 18+ +- Docker (for Jaeger) + +## Setup + +### 1. Start Jaeger + +```bash +docker run -d --name jaeger \ + -p 4317:4317 \ + -p 16686:16686 \ + jaegertracing/all-in-one:latest +``` + +Jaeger UI will be available at `http://localhost:16686` + +### 2. Install dependencies + +```bash +npm install +``` + +### 3. Start Restate Server with tracing enabled + +```bash +npx @restatedev/restate-server --tracing-endpoint http://localhost:4317 +``` + +### 4. Start the downstream service (terminal 1) + +```bash +npm run downstream +``` + +### 5. Start the Greeter service (terminal 2) + +```bash +npm run service +``` + +### 6. Register the service with Restate + +```bash +npx @restatedev/restate deployments register http://localhost:9080 +``` + +### 7. Run the client + +```bash +npm run client Alice +``` + +## Viewing Traces + +After running the client, you'll see output like: + +``` +Root Trace ID: abc123... +View in Jaeger: `http://localhost:16686/trace/abc123...` +``` + +Open the Jaeger link to see the complete distributed trace spanning all four components. + +## What You'll See in Jaeger + +The trace will show spans from all four services: + +- **client-app**: The root `client-request` span +- **Greeter**: Restate server spans for ingress, invoke, and journal operations +- **restate-greeter-service**: Custom `Greeter.greet` span with events +- **downstream-service**: `handle-request` span (may show errors due to 50% failure rate) + +## Key Pattern: Extracting Trace Context in TypeScript SDK + +The Restate server propagates W3C trace context to handlers via HTTP headers. In the TypeScript SDK, you need to manually extract this from `ctx.request().attemptHeaders`. + +**Why not use Node.js auto-instrumentation?** Unlike Java and Go, Node.js does have OTEL auto-instrumentation packages (e.g. `@opentelemetry/auto-instrumentations-node`). However, they operate at the raw HTTP transport layer, which Restate wraps internally. More importantly, Restate provides durable execution — a handler may be invoked multiple times due to retries. Extracting trace context from `ctx.request().attemptHeaders` ensures exactly one span per logical invocation, correctly positioned in the trace hierarchy regardless of retries. + +```typescript +import { context, propagation, type Context } from "@opentelemetry/api"; + +function extractTraceContext(ctx: restate.Context): Context { + const headers = ctx.request().attemptHeaders; + // TextMapGetter lets any propagator format (W3C, B3, Jaeger…) work automatically + return propagation.extract(context.active(), headers, { + get: (carrier, key) => { + const val = carrier.get(key); + return Array.isArray(val) ? val[0] : (val ?? undefined); + }, + keys: (carrier) => [...carrier.keys()], + }); +} +``` + +Then run your handler logic within that context: + +```typescript +const traceContext = extractTraceContext(ctx); +return context.with(traceContext, () => { + const span = tracer.startSpan("MyHandler"); + // ... your logic here, span is now a child of Restate's span +}); +``` + +## Files + +- `src/client.ts` - Client app that initiates traced requests +- `src/restate-service.ts` - Restate Greeter service with OpenTelemetry instrumentation +- `src/downstream.ts` - HTTP server with tracing and random failure rate diff --git a/typescript/tracing/otel/package.json b/typescript/tracing/otel/package.json new file mode 100644 index 00000000..ebbe23b4 --- /dev/null +++ b/typescript/tracing/otel/package.json @@ -0,0 +1,28 @@ +{ + "name": "@restatedev/examples-tracing", + "version": "0.0.1", + "description": "End-to-end OpenTelemetry tracing with Restate", + "license": "MIT", + "author": "Restate developers", + "email": "code@restate.dev", + "type": "commonjs", + "scripts": { + "build": "tsc --noEmitOnError", + "service": "tsx ./src/restate-service.ts", + "client": "tsx ./src/client.ts", + "downstream": "tsx ./src/downstream.ts" + }, + "dependencies": { + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/exporter-trace-otlp-grpc": "^0.57.0", + "@opentelemetry/resources": "^1.30.0", + "@opentelemetry/sdk-node": "^0.57.0", + "@opentelemetry/semantic-conventions": "^1.28.0", + "@restatedev/restate-sdk": "^1.10.2" + }, + "devDependencies": { + "@types/node": "^20.14.2", + "tsx": "^4.19.2", + "typescript": "^5.4.5" + } +} diff --git a/typescript/tracing/otel/src/client.ts b/typescript/tracing/otel/src/client.ts new file mode 100644 index 00000000..73e3447b --- /dev/null +++ b/typescript/tracing/otel/src/client.ts @@ -0,0 +1,91 @@ +// OpenTelemetry must be initialized before other imports +import { NodeSDK } from "@opentelemetry/sdk-node"; +import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-grpc"; +import { Resource } from "@opentelemetry/resources"; +import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions"; +import { + trace, + context, + propagation, + SpanKind, + SpanStatusCode, +} from "@opentelemetry/api"; + +const sdk = new NodeSDK({ + resource: new Resource({ + [ATTR_SERVICE_NAME]: "client-app", + }), + traceExporter: new OTLPTraceExporter({ + url: "http://localhost:4317", + }), +}); + +sdk.start(); + +const RESTATE_INGRESS = "http://localhost:8080"; +const tracer = trace.getTracer("client-app"); + +async function main() { + const name = process.argv[2] || "World"; + + console.log("=== Client App ==="); + console.log(`Calling Restate Greeter service with name: ${name}`); + + // Create the root span for this request + const rootSpan = tracer.startSpan("client-request", { + kind: SpanKind.CLIENT, + attributes: { + "request.name": name, + }, + }); + + try { + const result = await context.with( + trace.setSpan(context.active(), rootSpan), + async () => { + const headers: Record = { + "Content-Type": "application/json", + }; + + propagation.inject(context.active(), headers); + console.log(`Injected W3C trace context headers:`, headers); + + const traceId = rootSpan.spanContext().traceId; + console.log(`Root Trace ID: ${traceId}`); + console.log(`View in Jaeger: http://localhost:16686/trace/${traceId}`); + console.log(""); + + const response = await fetch(`${RESTATE_INGRESS}/Greeter/greet`, { + method: "POST", + headers, + body: JSON.stringify(name), + }); + + if (!response.ok) { + throw new Error(`HTTP ${response.status}: ${await response.text()}`); + } + + return response.json(); + }, + ); + + rootSpan.addEvent("response_received", { + "response.value": JSON.stringify(result), + }); + rootSpan.setStatus({ code: SpanStatusCode.OK }); + + console.log(`Response: ${JSON.stringify(result)}`); + } catch (err) { + rootSpan.setStatus({ + code: SpanStatusCode.ERROR, + message: err instanceof Error ? err.message : "Unknown error", + }); + console.error("Error:", err); + process.exitCode = 1; + } finally { + rootSpan.end(); + await sdk.shutdown(); + } +} + +main(); diff --git a/typescript/tracing/otel/src/downstream.ts b/typescript/tracing/otel/src/downstream.ts new file mode 100644 index 00000000..4074a631 --- /dev/null +++ b/typescript/tracing/otel/src/downstream.ts @@ -0,0 +1,93 @@ +// OpenTelemetry must be initialized before other imports +import { NodeSDK } from "@opentelemetry/sdk-node"; +import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-grpc"; +import { Resource } from "@opentelemetry/resources"; +import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions"; +import { + trace, + context, + propagation, + SpanKind, + SpanStatusCode, +} from "@opentelemetry/api"; +import { createServer } from "node:http"; + +const sdk = new NodeSDK({ + resource: new Resource({ + [ATTR_SERVICE_NAME]: "downstream-service", + }), + traceExporter: new OTLPTraceExporter({ + url: "http://localhost:4317", + }), +}); + +sdk.start(); + +const PORT = 3000; +const FAILURE_RATE = 0.5; // 50% chance + +const tracer = trace.getTracer("downstream-service"); + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +const server = createServer((req, res) => { + // Extract trace context from incoming HTTP headers + const traceContext = propagation.extract(context.active(), req.headers, { + get: (carrier, key) => { + const val = carrier[key]; + return Array.isArray(val) ? val[0] : (val ?? undefined); + }, + keys: (carrier) => Object.keys(carrier), + }); + + // Run request handling within the extracted trace context + context.with(traceContext, async () => { + const span = tracer.startSpan("handle-request", { + kind: SpanKind.SERVER, + attributes: { + "http.method": req.method, + "http.url": req.url, + }, + }); + + try { + // Simulate some work + await sleep(50 + Math.random() * 100); + + // Random failure + if (Math.random() < FAILURE_RATE) { + span.setStatus({ + code: SpanStatusCode.ERROR, + message: "Random failure", + }); + span.addEvent("failure_triggered", { rate: FAILURE_RATE }); + + res.writeHead(500, { "Content-Type": "application/json" }); + res.end( + JSON.stringify({ + error: "Random failure", + receivedTrace: !!req.headers["traceparent"], + }), + ); + return; + } + + span.addEvent("processing_complete"); + span.setStatus({ code: SpanStatusCode.OK }); + + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ status: "ok", receivedTrace: !!req.headers["traceparent"] })); + } finally { + span.end(); + } + }); +}); + +server.listen(PORT, () => { + console.log(`Downstream service listening on http://localhost:${PORT}`); + console.log(`Failure rate: ${FAILURE_RATE * 100}%`); +}); + +process.on("SIGTERM", () => { + sdk.shutdown().then(() => process.exit(0)); +}); diff --git a/typescript/tracing/otel/src/restate-service.ts b/typescript/tracing/otel/src/restate-service.ts new file mode 100644 index 00000000..0f61feb6 --- /dev/null +++ b/typescript/tracing/otel/src/restate-service.ts @@ -0,0 +1,130 @@ +// OpenTelemetry must be initialized before other imports +import { NodeSDK } from "@opentelemetry/sdk-node"; +import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-grpc"; +import { Resource } from "@opentelemetry/resources"; +import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions"; +import { + trace, + context, + propagation, + SpanKind, + SpanStatusCode, + type Context, +} from "@opentelemetry/api"; + +const sdk = new NodeSDK({ + resource: new Resource({ + [ATTR_SERVICE_NAME]: "restate-greeter-service", + }), + traceExporter: new OTLPTraceExporter({ + url: "http://localhost:4317", + }), +}); + +sdk.start(); + +import * as restate from "@restatedev/restate-sdk"; + +const DOWNSTREAM_URL = "http://localhost:3000/api/process"; + +const tracer = trace.getTracer("greeter-service"); + +// Extract trace context propagated by Restate via attempt headers. +// +// Unlike Java/Go, Node.js does have OTEL auto-instrumentation packages, but they +// operate at the raw HTTP transport level. Restate wraps the HTTP layer and provides +// durable execution semantics — a handler may be replayed multiple times. Extracting +// from ctx.request().attemptHeaders ensures one span per logical invocation, +// correctly positioned in the trace hierarchy regardless of retries. +function extractTraceContext(ctx: restate.Context): Context { + const headers = ctx.request().attemptHeaders; + // Use a TextMapGetter so any propagator format (W3C, B3, Jaeger…) is supported + return propagation.extract(context.active(), headers, { + get: (carrier, key) => { + const val = carrier.get(key); + return Array.isArray(val) ? val[0] : (val ?? undefined); + }, + keys: (carrier) => [...carrier.keys()], + }); +} + +const greeter = restate.service({ + name: "Greeter", + handlers: { + greet: async (ctx: restate.Context, name: string): Promise => { + const traceContext = extractTraceContext(ctx); + + // Create span under the extracted trace context + const span = tracer.startSpan( + "Greeter.greet", + { kind: SpanKind.INTERNAL, attributes: { "greeter.name": name } }, + traceContext, + ); + + // Create context with our span as parent for downstream calls + const spanContext = trace.setSpan(traceContext, span); + + return context.with(spanContext, () => { + return (async () => { + try { + span.addEvent("processing_started", { name }); + + const greeting = `Hello, ${name}!`; + + // Call downstream - our span becomes the parent + const downstreamResult = await ctx.run("call-downstream", () => + callDownstreamWithTrace(name, spanContext), + ); + + span.addEvent("downstream_completed", { + "downstream.result": JSON.stringify(downstreamResult), + }); + + span.setStatus({ code: SpanStatusCode.OK }); + return greeting; + } catch (err) { + span.setStatus({ + code: SpanStatusCode.ERROR, + message: err instanceof Error ? err.message : "Unknown error", + }); + throw err; + } finally { + span.end(); + } + })(); + }); + }, + }, +}); + +async function callDownstreamWithTrace( + name: string, + traceContext: Context, +): Promise<{ status: string; receivedTrace: boolean }> { + const headers: Record = { + "Content-Type": "application/json", + }; + propagation.inject(traceContext, headers); + + const response = await fetch(DOWNSTREAM_URL, { + method: "POST", + headers, + body: JSON.stringify({ name }), + }); + + if (!response.ok) { + const body = (await response.json()) as { error?: string }; + throw new Error(`Downstream failed: ${body.error ?? response.statusText}`); + } + + return response.json() as Promise<{ status: string; receivedTrace: boolean }>; +} + +restate.serve({ + services: [greeter], + port: 9080, +}); + +process.on("SIGTERM", () => { + sdk.shutdown().then(() => process.exit(0)); +}); diff --git a/typescript/tracing/otel/tsconfig.json b/typescript/tracing/otel/tsconfig.json new file mode 100644 index 00000000..c2946b24 --- /dev/null +++ b/typescript/tracing/otel/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "target": "esnext", + "lib": ["esnext"], + "module": "nodenext", + "allowJs": true, + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "outDir": "./dist", + "allowSyntheticDefaultImports": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "strict": true, + "skipDefaultLibCheck": true, + "skipLibCheck": true + } +} diff --git a/typescript/tracing/package.json b/typescript/tracing/package.json new file mode 100644 index 00000000..ebbe23b4 --- /dev/null +++ b/typescript/tracing/package.json @@ -0,0 +1,28 @@ +{ + "name": "@restatedev/examples-tracing", + "version": "0.0.1", + "description": "End-to-end OpenTelemetry tracing with Restate", + "license": "MIT", + "author": "Restate developers", + "email": "code@restate.dev", + "type": "commonjs", + "scripts": { + "build": "tsc --noEmitOnError", + "service": "tsx ./src/restate-service.ts", + "client": "tsx ./src/client.ts", + "downstream": "tsx ./src/downstream.ts" + }, + "dependencies": { + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/exporter-trace-otlp-grpc": "^0.57.0", + "@opentelemetry/resources": "^1.30.0", + "@opentelemetry/sdk-node": "^0.57.0", + "@opentelemetry/semantic-conventions": "^1.28.0", + "@restatedev/restate-sdk": "^1.10.2" + }, + "devDependencies": { + "@types/node": "^20.14.2", + "tsx": "^4.19.2", + "typescript": "^5.4.5" + } +} diff --git a/typescript/tracing/tsconfig.json b/typescript/tracing/tsconfig.json new file mode 100644 index 00000000..c2946b24 --- /dev/null +++ b/typescript/tracing/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "target": "esnext", + "lib": ["esnext"], + "module": "nodenext", + "allowJs": true, + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "outDir": "./dist", + "allowSyntheticDefaultImports": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "strict": true, + "skipDefaultLibCheck": true, + "skipLibCheck": true + } +} From e6de977a003db549e2814f0ac7d41fa14138f19d Mon Sep 17 00:00:00 2001 From: slinkydeveloper Date: Thu, 9 Apr 2026 17:59:25 +0200 Subject: [PATCH 2/4] Use the new hook api --- typescript/tracing/otel/package.json | 3 +- .../tracing/otel/src/restate-service.ts | 79 +++++-------------- 2 files changed, 21 insertions(+), 61 deletions(-) diff --git a/typescript/tracing/otel/package.json b/typescript/tracing/otel/package.json index ebbe23b4..c0a49ceb 100644 --- a/typescript/tracing/otel/package.json +++ b/typescript/tracing/otel/package.json @@ -18,7 +18,8 @@ "@opentelemetry/resources": "^1.30.0", "@opentelemetry/sdk-node": "^0.57.0", "@opentelemetry/semantic-conventions": "^1.28.0", - "@restatedev/restate-sdk": "^1.10.2" + "@restatedev/restate-sdk": "dev", + "@restatedev/restate-sdk-opentelemetry": "dev" }, "devDependencies": { "@types/node": "^20.14.2", diff --git a/typescript/tracing/otel/src/restate-service.ts b/typescript/tracing/otel/src/restate-service.ts index 0f61feb6..ee11c03f 100644 --- a/typescript/tracing/otel/src/restate-service.ts +++ b/typescript/tracing/otel/src/restate-service.ts @@ -7,9 +7,7 @@ import { trace, context, propagation, - SpanKind, SpanStatusCode, - type Context, } from "@opentelemetry/api"; const sdk = new NodeSDK({ @@ -24,87 +22,48 @@ const sdk = new NodeSDK({ sdk.start(); import * as restate from "@restatedev/restate-sdk"; +import { openTelemetryHook } from "@restatedev/restate-sdk-opentelemetry"; const DOWNSTREAM_URL = "http://localhost:3000/api/process"; const tracer = trace.getTracer("greeter-service"); -// Extract trace context propagated by Restate via attempt headers. -// -// Unlike Java/Go, Node.js does have OTEL auto-instrumentation packages, but they -// operate at the raw HTTP transport level. Restate wraps the HTTP layer and provides -// durable execution semantics — a handler may be replayed multiple times. Extracting -// from ctx.request().attemptHeaders ensures one span per logical invocation, -// correctly positioned in the trace hierarchy regardless of retries. -function extractTraceContext(ctx: restate.Context): Context { - const headers = ctx.request().attemptHeaders; - // Use a TextMapGetter so any propagator format (W3C, B3, Jaeger…) is supported - return propagation.extract(context.active(), headers, { - get: (carrier, key) => { - const val = carrier.get(key); - return Array.isArray(val) ? val[0] : (val ?? undefined); - }, - keys: (carrier) => [...carrier.keys()], - }); -} - const greeter = restate.service({ name: "Greeter", handlers: { greet: async (ctx: restate.Context, name: string): Promise => { - const traceContext = extractTraceContext(ctx); + // This span is created automatically by the hook we install below + const span = trace.getActiveSpan()!; + span.addEvent("processing_started", { name }); - // Create span under the extracted trace context - const span = tracer.startSpan( - "Greeter.greet", - { kind: SpanKind.INTERNAL, attributes: { "greeter.name": name } }, - traceContext, + const greeting = `Hello, ${name}!`; + + // Execute ctx.run -> this will create a child span, parent of the attempt span. + const downstreamResult = await ctx.run("call-downstream", () => + // OTEL context is propagated downstream here as well + callDownstreamWithTrace(name), ); - // Create context with our span as parent for downstream calls - const spanContext = trace.setSpan(traceContext, span); - - return context.with(spanContext, () => { - return (async () => { - try { - span.addEvent("processing_started", { name }); - - const greeting = `Hello, ${name}!`; - - // Call downstream - our span becomes the parent - const downstreamResult = await ctx.run("call-downstream", () => - callDownstreamWithTrace(name, spanContext), - ); - - span.addEvent("downstream_completed", { - "downstream.result": JSON.stringify(downstreamResult), - }); - - span.setStatus({ code: SpanStatusCode.OK }); - return greeting; - } catch (err) { - span.setStatus({ - code: SpanStatusCode.ERROR, - message: err instanceof Error ? err.message : "Unknown error", - }); - throw err; - } finally { - span.end(); - } - })(); + span.addEvent("downstream_completed", { + "downstream.result": JSON.stringify(downstreamResult), }); + + return greeting; }, }, + options: { + // Set up the OTEL hook + hooks: [openTelemetryHook({ tracer })] + } }); async function callDownstreamWithTrace( name: string, - traceContext: Context, ): Promise<{ status: string; receivedTrace: boolean }> { const headers: Record = { "Content-Type": "application/json", }; - propagation.inject(traceContext, headers); + propagation.inject(context.active(), headers); const response = await fetch(DOWNSTREAM_URL, { method: "POST", From 582a4186efc60885ce2d492d24f52a892431d229 Mon Sep 17 00:00:00 2001 From: slinkydeveloper Date: Thu, 9 Apr 2026 18:03:44 +0200 Subject: [PATCH 3/4] Move around + update readme --- .tools/run_node_tests.sh | 3 +- .tools/update_node_examples.sh | 1 + typescript/README.md | 1 + .../opentelemetry}/.gitignore | 0 .../opentelemetry}/README.md | 34 +------------------ .../opentelemetry}/package.json | 4 +-- .../opentelemetry}/src/client.ts | 0 .../opentelemetry}/src/downstream.ts | 0 .../opentelemetry}/src/restate-service.ts | 1 - .../opentelemetry}/tsconfig.json | 0 typescript/tracing/package.json | 28 --------------- typescript/tracing/tsconfig.json | 18 ---------- 12 files changed, 6 insertions(+), 84 deletions(-) rename typescript/{tracing/otel => integrations/opentelemetry}/.gitignore (100%) rename typescript/{tracing/otel => integrations/opentelemetry}/README.md (67%) rename typescript/{tracing/otel => integrations/opentelemetry}/package.json (89%) rename typescript/{tracing/otel => integrations/opentelemetry}/src/client.ts (100%) rename typescript/{tracing/otel => integrations/opentelemetry}/src/downstream.ts (100%) rename typescript/{tracing/otel => integrations/opentelemetry}/src/restate-service.ts (99%) rename typescript/{tracing/otel => integrations/opentelemetry}/tsconfig.json (100%) delete mode 100644 typescript/tracing/package.json delete mode 100644 typescript/tracing/tsconfig.json diff --git a/.tools/run_node_tests.sh b/.tools/run_node_tests.sh index c485eaf5..f9794dcf 100755 --- a/.tools/run_node_tests.sh +++ b/.tools/run_node_tests.sh @@ -10,11 +10,9 @@ function npm_install_check() { } npm_install_check $PROJECT_ROOT/typescript/basics -npm_install_check $PROJECT_ROOT/typescript/tracing npm_install_check $PROJECT_ROOT/typescript/templates/node npm_install_check $PROJECT_ROOT/typescript/templates/lambda -RESTATE_ENV_ID=env_test RESTATE_API_KEY=key_test npm_install_check $PROJECT_ROOT/typescript/integrations/deployment-lambda-cdk npm_install_check $PROJECT_ROOT/typescript/templates/cloudflare-worker npm_install_check $PROJECT_ROOT/typescript/templates/vercel npm_install_check $PROJECT_ROOT/typescript/templates/nextjs @@ -31,4 +29,5 @@ npm_install_check $PROJECT_ROOT/typescript/end-to-end-applications/ai-image-work npm_install_check $PROJECT_ROOT/typescript/end-to-end-applications/food-ordering/app npm_install_check $PROJECT_ROOT/typescript/end-to-end-applications/chat-bot +npm_install_check $PROJECT_ROOT/python/integrations/opentelemetry RESTATE_ENV_ID=env_test RESTATE_API_KEY=key_test npm_install_check $PROJECT_ROOT/python/integrations/deployment-lambda-cdk diff --git a/.tools/update_node_examples.sh b/.tools/update_node_examples.sh index 25e922ba..b7c6fd6d 100755 --- a/.tools/update_node_examples.sh +++ b/.tools/update_node_examples.sh @@ -54,6 +54,7 @@ bump_restate_sdk_deps $PROJECT_ROOT/typescript/templates/nextjs bump_restate_sdk_deps $PROJECT_ROOT/typescript/templates/vercel bump_restate_sdk_deps $PROJECT_ROOT/typescript/templates/cloudflare-worker bump_restate_sdk_deps $PROJECT_ROOT/typescript/integrations/deployment-lambda-cdk +bump_restate_sdk_deps $PROJECT_ROOT/typescript/integrations/opentelemetry bump_restate_sdk_deps $PROJECT_ROOT/typescript/tutorials/tour-of-orchestration-typescript bump_restate_sdk_deps $PROJECT_ROOT/typescript/tutorials/tour-of-workflows-typescript bump_restate_sdk_deps $PROJECT_ROOT/typescript/patterns-use-cases diff --git a/typescript/README.md b/typescript/README.md index 6582c0e5..704d317c 100644 --- a/typescript/README.md +++ b/typescript/README.md @@ -49,6 +49,7 @@ Common tasks and patterns implemented with Restate: Examples integrating Restate with other tools and frameworks: +- **[OpenTelemetry](integrations/opentelemetry)**: Integration with OpenTelemetry for distributed tracing and monitoring. - **[AWS Lambda + CDK](integrations/deployment-lambda-cdk)**: Sample project deploying a TypeScript-based Restate service to AWS Lambda using the AWS Cloud Development Kit (CDK). - **[XState](integrations/xstate)**: Resilient, distributed durable state machines with Restate and XState. diff --git a/typescript/tracing/otel/.gitignore b/typescript/integrations/opentelemetry/.gitignore similarity index 100% rename from typescript/tracing/otel/.gitignore rename to typescript/integrations/opentelemetry/.gitignore diff --git a/typescript/tracing/otel/README.md b/typescript/integrations/opentelemetry/README.md similarity index 67% rename from typescript/tracing/otel/README.md rename to typescript/integrations/opentelemetry/README.md index e5eeecf2..009f0c1f 100644 --- a/typescript/tracing/otel/README.md +++ b/typescript/integrations/opentelemetry/README.md @@ -19,7 +19,7 @@ This example demonstrates distributed tracing across a fictional multi-tier syst 1. **Client App** - Creates the root span and injects W3C trace context into the Restate request 2. **Restate Server** - Receives trace context, emits spans for ingress requests and handler invocations -3. **Greeter Service** - SDK handler that creates custom spans and propagates context to downstream calls +3. **Greeter Service** - SDK handler using `@restatedev/restate-sdk-opentelemetry` that creates spans per attempt, per `ctx.run` and propagates context to downstream calls 4. **Downstream Service** - Receives and logs the propagated trace headers ## Prerequisites @@ -96,38 +96,6 @@ The trace will show spans from all four services: - **restate-greeter-service**: Custom `Greeter.greet` span with events - **downstream-service**: `handle-request` span (may show errors due to 50% failure rate) -## Key Pattern: Extracting Trace Context in TypeScript SDK - -The Restate server propagates W3C trace context to handlers via HTTP headers. In the TypeScript SDK, you need to manually extract this from `ctx.request().attemptHeaders`. - -**Why not use Node.js auto-instrumentation?** Unlike Java and Go, Node.js does have OTEL auto-instrumentation packages (e.g. `@opentelemetry/auto-instrumentations-node`). However, they operate at the raw HTTP transport layer, which Restate wraps internally. More importantly, Restate provides durable execution — a handler may be invoked multiple times due to retries. Extracting trace context from `ctx.request().attemptHeaders` ensures exactly one span per logical invocation, correctly positioned in the trace hierarchy regardless of retries. - -```typescript -import { context, propagation, type Context } from "@opentelemetry/api"; - -function extractTraceContext(ctx: restate.Context): Context { - const headers = ctx.request().attemptHeaders; - // TextMapGetter lets any propagator format (W3C, B3, Jaeger…) work automatically - return propagation.extract(context.active(), headers, { - get: (carrier, key) => { - const val = carrier.get(key); - return Array.isArray(val) ? val[0] : (val ?? undefined); - }, - keys: (carrier) => [...carrier.keys()], - }); -} -``` - -Then run your handler logic within that context: - -```typescript -const traceContext = extractTraceContext(ctx); -return context.with(traceContext, () => { - const span = tracer.startSpan("MyHandler"); - // ... your logic here, span is now a child of Restate's span -}); -``` - ## Files - `src/client.ts` - Client app that initiates traced requests diff --git a/typescript/tracing/otel/package.json b/typescript/integrations/opentelemetry/package.json similarity index 89% rename from typescript/tracing/otel/package.json rename to typescript/integrations/opentelemetry/package.json index c0a49ceb..e50d1e9b 100644 --- a/typescript/tracing/otel/package.json +++ b/typescript/integrations/opentelemetry/package.json @@ -18,8 +18,8 @@ "@opentelemetry/resources": "^1.30.0", "@opentelemetry/sdk-node": "^0.57.0", "@opentelemetry/semantic-conventions": "^1.28.0", - "@restatedev/restate-sdk": "dev", - "@restatedev/restate-sdk-opentelemetry": "dev" + "@restatedev/restate-sdk": "^1.12.0", + "@restatedev/restate-sdk-opentelemetry": "^1.12.0" }, "devDependencies": { "@types/node": "^20.14.2", diff --git a/typescript/tracing/otel/src/client.ts b/typescript/integrations/opentelemetry/src/client.ts similarity index 100% rename from typescript/tracing/otel/src/client.ts rename to typescript/integrations/opentelemetry/src/client.ts diff --git a/typescript/tracing/otel/src/downstream.ts b/typescript/integrations/opentelemetry/src/downstream.ts similarity index 100% rename from typescript/tracing/otel/src/downstream.ts rename to typescript/integrations/opentelemetry/src/downstream.ts diff --git a/typescript/tracing/otel/src/restate-service.ts b/typescript/integrations/opentelemetry/src/restate-service.ts similarity index 99% rename from typescript/tracing/otel/src/restate-service.ts rename to typescript/integrations/opentelemetry/src/restate-service.ts index ee11c03f..0bcc49e1 100644 --- a/typescript/tracing/otel/src/restate-service.ts +++ b/typescript/integrations/opentelemetry/src/restate-service.ts @@ -7,7 +7,6 @@ import { trace, context, propagation, - SpanStatusCode, } from "@opentelemetry/api"; const sdk = new NodeSDK({ diff --git a/typescript/tracing/otel/tsconfig.json b/typescript/integrations/opentelemetry/tsconfig.json similarity index 100% rename from typescript/tracing/otel/tsconfig.json rename to typescript/integrations/opentelemetry/tsconfig.json diff --git a/typescript/tracing/package.json b/typescript/tracing/package.json deleted file mode 100644 index ebbe23b4..00000000 --- a/typescript/tracing/package.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "name": "@restatedev/examples-tracing", - "version": "0.0.1", - "description": "End-to-end OpenTelemetry tracing with Restate", - "license": "MIT", - "author": "Restate developers", - "email": "code@restate.dev", - "type": "commonjs", - "scripts": { - "build": "tsc --noEmitOnError", - "service": "tsx ./src/restate-service.ts", - "client": "tsx ./src/client.ts", - "downstream": "tsx ./src/downstream.ts" - }, - "dependencies": { - "@opentelemetry/api": "^1.9.0", - "@opentelemetry/exporter-trace-otlp-grpc": "^0.57.0", - "@opentelemetry/resources": "^1.30.0", - "@opentelemetry/sdk-node": "^0.57.0", - "@opentelemetry/semantic-conventions": "^1.28.0", - "@restatedev/restate-sdk": "^1.10.2" - }, - "devDependencies": { - "@types/node": "^20.14.2", - "tsx": "^4.19.2", - "typescript": "^5.4.5" - } -} diff --git a/typescript/tracing/tsconfig.json b/typescript/tracing/tsconfig.json deleted file mode 100644 index c2946b24..00000000 --- a/typescript/tracing/tsconfig.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "compilerOptions": { - "target": "esnext", - "lib": ["esnext"], - "module": "nodenext", - "allowJs": true, - "declaration": true, - "declarationMap": true, - "sourceMap": true, - "outDir": "./dist", - "allowSyntheticDefaultImports": true, - "esModuleInterop": true, - "forceConsistentCasingInFileNames": true, - "strict": true, - "skipDefaultLibCheck": true, - "skipLibCheck": true - } -} From 8f5bc31777907e632c16f2db5d6dd308a0216773 Mon Sep 17 00:00:00 2001 From: slinkydeveloper Date: Fri, 10 Apr 2026 15:23:23 +0200 Subject: [PATCH 4/4] Fix --- .tools/run_node_tests.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.tools/run_node_tests.sh b/.tools/run_node_tests.sh index f9794dcf..6a403463 100755 --- a/.tools/run_node_tests.sh +++ b/.tools/run_node_tests.sh @@ -29,5 +29,7 @@ npm_install_check $PROJECT_ROOT/typescript/end-to-end-applications/ai-image-work npm_install_check $PROJECT_ROOT/typescript/end-to-end-applications/food-ordering/app npm_install_check $PROJECT_ROOT/typescript/end-to-end-applications/chat-bot -npm_install_check $PROJECT_ROOT/python/integrations/opentelemetry +npm_install_check $PROJECT_ROOT/typescript/integrations/opentelemetry +RESTATE_ENV_ID=env_test RESTATE_API_KEY=key_test npm_install_check $PROJECT_ROOT/typescript/integrations/deployment-lambda-cdk + RESTATE_ENV_ID=env_test RESTATE_API_KEY=key_test npm_install_check $PROJECT_ROOT/python/integrations/deployment-lambda-cdk