diff --git a/.tools/run_node_tests.sh b/.tools/run_node_tests.sh index 66be72dd..6a403463 100755 --- a/.tools/run_node_tests.sh +++ b/.tools/run_node_tests.sh @@ -13,7 +13,6 @@ npm_install_check $PROJECT_ROOT/typescript/basics npm_install_check $PROJECT_ROOT/typescript/templates/node npm_install_check $PROJECT_ROOT/typescript/templates/lambda -RESTATE_ENV_ID=env_test RESTATE_API_KEY=key_test npm_install_check $PROJECT_ROOT/typescript/integrations/deployment-lambda-cdk npm_install_check $PROJECT_ROOT/typescript/templates/cloudflare-worker npm_install_check $PROJECT_ROOT/typescript/templates/vercel npm_install_check $PROJECT_ROOT/typescript/templates/nextjs @@ -30,4 +29,7 @@ npm_install_check $PROJECT_ROOT/typescript/end-to-end-applications/ai-image-work npm_install_check $PROJECT_ROOT/typescript/end-to-end-applications/food-ordering/app npm_install_check $PROJECT_ROOT/typescript/end-to-end-applications/chat-bot +npm_install_check $PROJECT_ROOT/typescript/integrations/opentelemetry +RESTATE_ENV_ID=env_test RESTATE_API_KEY=key_test npm_install_check $PROJECT_ROOT/typescript/integrations/deployment-lambda-cdk + RESTATE_ENV_ID=env_test RESTATE_API_KEY=key_test npm_install_check $PROJECT_ROOT/python/integrations/deployment-lambda-cdk diff --git a/.tools/update_node_examples.sh b/.tools/update_node_examples.sh index 25e922ba..b7c6fd6d 100755 --- a/.tools/update_node_examples.sh +++ b/.tools/update_node_examples.sh @@ -54,6 +54,7 @@ bump_restate_sdk_deps $PROJECT_ROOT/typescript/templates/nextjs bump_restate_sdk_deps $PROJECT_ROOT/typescript/templates/vercel bump_restate_sdk_deps $PROJECT_ROOT/typescript/templates/cloudflare-worker bump_restate_sdk_deps $PROJECT_ROOT/typescript/integrations/deployment-lambda-cdk +bump_restate_sdk_deps $PROJECT_ROOT/typescript/integrations/opentelemetry bump_restate_sdk_deps $PROJECT_ROOT/typescript/tutorials/tour-of-orchestration-typescript bump_restate_sdk_deps $PROJECT_ROOT/typescript/tutorials/tour-of-workflows-typescript bump_restate_sdk_deps $PROJECT_ROOT/typescript/patterns-use-cases diff --git a/typescript/README.md b/typescript/README.md index 6582c0e5..704d317c 100644 --- a/typescript/README.md +++ b/typescript/README.md @@ -49,6 +49,7 @@ Common tasks and patterns implemented with Restate: Examples integrating Restate with other tools and frameworks: +- **[OpenTelemetry](integrations/opentelemetry)**: Integration with OpenTelemetry for distributed tracing and monitoring. - **[AWS Lambda + CDK](integrations/deployment-lambda-cdk)**: Sample project deploying a TypeScript-based Restate service to AWS Lambda using the AWS Cloud Development Kit (CDK). - **[XState](integrations/xstate)**: Resilient, distributed durable state machines with Restate and XState. diff --git a/typescript/integrations/opentelemetry/.gitignore b/typescript/integrations/opentelemetry/.gitignore new file mode 100644 index 00000000..39895742 --- /dev/null +++ b/typescript/integrations/opentelemetry/.gitignore @@ -0,0 +1,19 @@ +# Node +node_modules +dist + +# screenshots +*.png + +# debug +npm-debug.log* + +# env files +.env* + +# typescript +*.tsbuildinfo + +# Restate +.restate +restate-data diff --git a/typescript/integrations/opentelemetry/README.md b/typescript/integrations/opentelemetry/README.md new file mode 100644 index 00000000..009f0c1f --- /dev/null +++ b/typescript/integrations/opentelemetry/README.md @@ -0,0 +1,103 @@ +# End-to-End OpenTelemetry Tracing with Restate + +This example demonstrates distributed tracing across a fictional multi-tier system: + +``` +┌──────────┐ ┌─────────────┐ ┌─────────────────┐ ┌────────────┐ +│ Client │────▶│ Restate │────▶│ Greeter Service │────▶│ Downstream │ +│ App │ │ Server │ │ (SDK/Node) │ │ Service │ +└──────────┘ └─────────────┘ └─────────────────┘ └────────────┘ + │ │ │ │ + │ │ │ │ + ▼ ▼ ▼ ▼ +┌────────────────────────────────────────────────────────────────────────┐ +│ Jaeger │ +└────────────────────────────────────────────────────────────────────────┘ +``` + +**What gets traced:** + +1. **Client App** - Creates the root span and injects W3C trace context into the Restate request +2. **Restate Server** - Receives trace context, emits spans for ingress requests and handler invocations +3. **Greeter Service** - SDK handler using `@restatedev/restate-sdk-opentelemetry` that creates spans per attempt, per `ctx.run` and propagates context to downstream calls +4. **Downstream Service** - Receives and logs the propagated trace headers + +## Prerequisites + +- Node.js 18+ +- Docker (for Jaeger) + +## Setup + +### 1. Start Jaeger + +```bash +docker run -d --name jaeger \ + -p 4317:4317 \ + -p 16686:16686 \ + jaegertracing/all-in-one:latest +``` + +Jaeger UI will be available at `http://localhost:16686` + +### 2. Install dependencies + +```bash +npm install +``` + +### 3. Start Restate Server with tracing enabled + +```bash +npx @restatedev/restate-server --tracing-endpoint http://localhost:4317 +``` + +### 4. Start the downstream service (terminal 1) + +```bash +npm run downstream +``` + +### 5. Start the Greeter service (terminal 2) + +```bash +npm run service +``` + +### 6. Register the service with Restate + +```bash +npx @restatedev/restate deployments register http://localhost:9080 +``` + +### 7. Run the client + +```bash +npm run client Alice +``` + +## Viewing Traces + +After running the client, you'll see output like: + +``` +Root Trace ID: abc123... +View in Jaeger: `http://localhost:16686/trace/abc123...` +``` + +Open the Jaeger link to see the complete distributed trace spanning all four components. + +## What You'll See in Jaeger + +The trace will show spans from all four services: + +- **client-app**: The root `client-request` span +- **Greeter**: Restate server spans for ingress, invoke, and journal operations +- **restate-greeter-service**: Custom `Greeter.greet` span with events +- **downstream-service**: `handle-request` span (may show errors due to 50% failure rate) + +## Files + +- `src/client.ts` - Client app that initiates traced requests +- `src/restate-service.ts` - Restate Greeter service with OpenTelemetry instrumentation +- `src/downstream.ts` - HTTP server with tracing and random failure rate diff --git a/typescript/integrations/opentelemetry/package.json b/typescript/integrations/opentelemetry/package.json new file mode 100644 index 00000000..e50d1e9b --- /dev/null +++ b/typescript/integrations/opentelemetry/package.json @@ -0,0 +1,29 @@ +{ + "name": "@restatedev/examples-tracing", + "version": "0.0.1", + "description": "End-to-end OpenTelemetry tracing with Restate", + "license": "MIT", + "author": "Restate developers", + "email": "code@restate.dev", + "type": "commonjs", + "scripts": { + "build": "tsc --noEmitOnError", + "service": "tsx ./src/restate-service.ts", + "client": "tsx ./src/client.ts", + "downstream": "tsx ./src/downstream.ts" + }, + "dependencies": { + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/exporter-trace-otlp-grpc": "^0.57.0", + "@opentelemetry/resources": "^1.30.0", + "@opentelemetry/sdk-node": "^0.57.0", + "@opentelemetry/semantic-conventions": "^1.28.0", + "@restatedev/restate-sdk": "^1.12.0", + "@restatedev/restate-sdk-opentelemetry": "^1.12.0" + }, + "devDependencies": { + "@types/node": "^20.14.2", + "tsx": "^4.19.2", + "typescript": "^5.4.5" + } +} diff --git a/typescript/integrations/opentelemetry/src/client.ts b/typescript/integrations/opentelemetry/src/client.ts new file mode 100644 index 00000000..73e3447b --- /dev/null +++ b/typescript/integrations/opentelemetry/src/client.ts @@ -0,0 +1,91 @@ +// OpenTelemetry must be initialized before other imports +import { NodeSDK } from "@opentelemetry/sdk-node"; +import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-grpc"; +import { Resource } from "@opentelemetry/resources"; +import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions"; +import { + trace, + context, + propagation, + SpanKind, + SpanStatusCode, +} from "@opentelemetry/api"; + +const sdk = new NodeSDK({ + resource: new Resource({ + [ATTR_SERVICE_NAME]: "client-app", + }), + traceExporter: new OTLPTraceExporter({ + url: "http://localhost:4317", + }), +}); + +sdk.start(); + +const RESTATE_INGRESS = "http://localhost:8080"; +const tracer = trace.getTracer("client-app"); + +async function main() { + const name = process.argv[2] || "World"; + + console.log("=== Client App ==="); + console.log(`Calling Restate Greeter service with name: ${name}`); + + // Create the root span for this request + const rootSpan = tracer.startSpan("client-request", { + kind: SpanKind.CLIENT, + attributes: { + "request.name": name, + }, + }); + + try { + const result = await context.with( + trace.setSpan(context.active(), rootSpan), + async () => { + const headers: Record = { + "Content-Type": "application/json", + }; + + propagation.inject(context.active(), headers); + console.log(`Injected W3C trace context headers:`, headers); + + const traceId = rootSpan.spanContext().traceId; + console.log(`Root Trace ID: ${traceId}`); + console.log(`View in Jaeger: http://localhost:16686/trace/${traceId}`); + console.log(""); + + const response = await fetch(`${RESTATE_INGRESS}/Greeter/greet`, { + method: "POST", + headers, + body: JSON.stringify(name), + }); + + if (!response.ok) { + throw new Error(`HTTP ${response.status}: ${await response.text()}`); + } + + return response.json(); + }, + ); + + rootSpan.addEvent("response_received", { + "response.value": JSON.stringify(result), + }); + rootSpan.setStatus({ code: SpanStatusCode.OK }); + + console.log(`Response: ${JSON.stringify(result)}`); + } catch (err) { + rootSpan.setStatus({ + code: SpanStatusCode.ERROR, + message: err instanceof Error ? err.message : "Unknown error", + }); + console.error("Error:", err); + process.exitCode = 1; + } finally { + rootSpan.end(); + await sdk.shutdown(); + } +} + +main(); diff --git a/typescript/integrations/opentelemetry/src/downstream.ts b/typescript/integrations/opentelemetry/src/downstream.ts new file mode 100644 index 00000000..4074a631 --- /dev/null +++ b/typescript/integrations/opentelemetry/src/downstream.ts @@ -0,0 +1,93 @@ +// OpenTelemetry must be initialized before other imports +import { NodeSDK } from "@opentelemetry/sdk-node"; +import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-grpc"; +import { Resource } from "@opentelemetry/resources"; +import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions"; +import { + trace, + context, + propagation, + SpanKind, + SpanStatusCode, +} from "@opentelemetry/api"; +import { createServer } from "node:http"; + +const sdk = new NodeSDK({ + resource: new Resource({ + [ATTR_SERVICE_NAME]: "downstream-service", + }), + traceExporter: new OTLPTraceExporter({ + url: "http://localhost:4317", + }), +}); + +sdk.start(); + +const PORT = 3000; +const FAILURE_RATE = 0.5; // 50% chance + +const tracer = trace.getTracer("downstream-service"); + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +const server = createServer((req, res) => { + // Extract trace context from incoming HTTP headers + const traceContext = propagation.extract(context.active(), req.headers, { + get: (carrier, key) => { + const val = carrier[key]; + return Array.isArray(val) ? val[0] : (val ?? undefined); + }, + keys: (carrier) => Object.keys(carrier), + }); + + // Run request handling within the extracted trace context + context.with(traceContext, async () => { + const span = tracer.startSpan("handle-request", { + kind: SpanKind.SERVER, + attributes: { + "http.method": req.method, + "http.url": req.url, + }, + }); + + try { + // Simulate some work + await sleep(50 + Math.random() * 100); + + // Random failure + if (Math.random() < FAILURE_RATE) { + span.setStatus({ + code: SpanStatusCode.ERROR, + message: "Random failure", + }); + span.addEvent("failure_triggered", { rate: FAILURE_RATE }); + + res.writeHead(500, { "Content-Type": "application/json" }); + res.end( + JSON.stringify({ + error: "Random failure", + receivedTrace: !!req.headers["traceparent"], + }), + ); + return; + } + + span.addEvent("processing_complete"); + span.setStatus({ code: SpanStatusCode.OK }); + + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ status: "ok", receivedTrace: !!req.headers["traceparent"] })); + } finally { + span.end(); + } + }); +}); + +server.listen(PORT, () => { + console.log(`Downstream service listening on http://localhost:${PORT}`); + console.log(`Failure rate: ${FAILURE_RATE * 100}%`); +}); + +process.on("SIGTERM", () => { + sdk.shutdown().then(() => process.exit(0)); +}); diff --git a/typescript/integrations/opentelemetry/src/restate-service.ts b/typescript/integrations/opentelemetry/src/restate-service.ts new file mode 100644 index 00000000..0bcc49e1 --- /dev/null +++ b/typescript/integrations/opentelemetry/src/restate-service.ts @@ -0,0 +1,88 @@ +// OpenTelemetry must be initialized before other imports +import { NodeSDK } from "@opentelemetry/sdk-node"; +import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-grpc"; +import { Resource } from "@opentelemetry/resources"; +import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions"; +import { + trace, + context, + propagation, +} from "@opentelemetry/api"; + +const sdk = new NodeSDK({ + resource: new Resource({ + [ATTR_SERVICE_NAME]: "restate-greeter-service", + }), + traceExporter: new OTLPTraceExporter({ + url: "http://localhost:4317", + }), +}); + +sdk.start(); + +import * as restate from "@restatedev/restate-sdk"; +import { openTelemetryHook } from "@restatedev/restate-sdk-opentelemetry"; + +const DOWNSTREAM_URL = "http://localhost:3000/api/process"; + +const tracer = trace.getTracer("greeter-service"); + +const greeter = restate.service({ + name: "Greeter", + handlers: { + greet: async (ctx: restate.Context, name: string): Promise => { + // This span is created automatically by the hook we install below + const span = trace.getActiveSpan()!; + span.addEvent("processing_started", { name }); + + const greeting = `Hello, ${name}!`; + + // Execute ctx.run -> this will create a child span, parent of the attempt span. + const downstreamResult = await ctx.run("call-downstream", () => + // OTEL context is propagated downstream here as well + callDownstreamWithTrace(name), + ); + + span.addEvent("downstream_completed", { + "downstream.result": JSON.stringify(downstreamResult), + }); + + return greeting; + }, + }, + options: { + // Set up the OTEL hook + hooks: [openTelemetryHook({ tracer })] + } +}); + +async function callDownstreamWithTrace( + name: string, +): Promise<{ status: string; receivedTrace: boolean }> { + const headers: Record = { + "Content-Type": "application/json", + }; + propagation.inject(context.active(), headers); + + const response = await fetch(DOWNSTREAM_URL, { + method: "POST", + headers, + body: JSON.stringify({ name }), + }); + + if (!response.ok) { + const body = (await response.json()) as { error?: string }; + throw new Error(`Downstream failed: ${body.error ?? response.statusText}`); + } + + return response.json() as Promise<{ status: string; receivedTrace: boolean }>; +} + +restate.serve({ + services: [greeter], + port: 9080, +}); + +process.on("SIGTERM", () => { + sdk.shutdown().then(() => process.exit(0)); +}); diff --git a/typescript/integrations/opentelemetry/tsconfig.json b/typescript/integrations/opentelemetry/tsconfig.json new file mode 100644 index 00000000..c2946b24 --- /dev/null +++ b/typescript/integrations/opentelemetry/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "target": "esnext", + "lib": ["esnext"], + "module": "nodenext", + "allowJs": true, + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "outDir": "./dist", + "allowSyntheticDefaultImports": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "strict": true, + "skipDefaultLibCheck": true, + "skipLibCheck": true + } +}