Cmaf Native Support

js/hang/src/catalog/audio.ts

@@ -15,7 +15,7 @@ export const AudioConfigSchema = z.object({
 	codec: z.string(),
 
 	// Container format for timestamp encoding
-	// Defaults to "legacy" when not specified in catalog (backward compatibility)
+	// Defaults to "native" when not specified in catalog (backward compatibility)
 	container: ContainerSchema.default(DEFAULT_CONTAINER),
 
 	// The description is used for some codecs.
@@ -32,6 +32,12 @@ export const AudioConfigSchema = z.object({
 	// The bitrate of the audio in bits per second
 	// TODO: Support up to Number.MAX_SAFE_INTEGER
 	bitrate: u53Schema.optional(),
+
+	// Init segment (ftyp+moov) for CMAF/fMP4 containers.
+	// This is the initialization segment needed for MSE playback.
+	// Stored as base64-encoded bytes. If not provided, init segments
+	// will be sent over the data track (legacy behavior).
+	initSegment: z.string().optional(), // base64-encoded
 });
 
 export const AudioSchema = z

js/hang/src/catalog/container.ts

@@ -3,16 +3,16 @@ import { z } from "zod";
 /**
  * Container format for frame timestamp encoding.
  *
- * - "legacy": Uses QUIC VarInt encoding (1-8 bytes, variable length)
+ * - "native": Uses QUIC VarInt encoding (1-8 bytes, variable length)
  * - "raw": Uses fixed u64 encoding (8 bytes, big-endian)
- * - "fmp4": Fragmented MP4 container (future)
+ * - "cmaf": Fragmented MP4 container (future)
  */
-export const ContainerSchema = z.enum(["legacy", "raw", "fmp4"]);
+export const ContainerSchema = z.enum(["native", "raw", "cmaf"]);
 
 export type Container = z.infer<typeof ContainerSchema>;
 
 /**
  * Default container format when not specified.
- * Set to legacy for backward compatibility.
+ * Set to native for backward compatibility.
  */
-export const DEFAULT_CONTAINER: Container = "legacy";
+export const DEFAULT_CONTAINER: Container = "native";

js/hang/src/catalog/video.ts

@@ -14,7 +14,7 @@ export const VideoConfigSchema = z.object({
 	codec: z.string(),
 
 	// Container format for timestamp encoding
-	// Defaults to "legacy" when not specified in catalog (backward compatibility)
+	// Defaults to "native" when not specified in catalog (backward compatibility)
 	container: ContainerSchema.default(DEFAULT_CONTAINER),
 
 	// The description is used for some codecs.
@@ -43,6 +43,12 @@ export const VideoConfigSchema = z.object({
 	// If true, the decoder will optimize for latency.
 	// Default: true
 	optimizeForLatency: z.boolean().optional(),
+
+	// Init segment (ftyp+moov) for CMAF/fMP4 containers.
+	// This is the initialization segment needed for MSE playback.
+	// Stored as base64-encoded bytes. If not provided, init segments
+	// will be sent over the data track (legacy behavior).
+	initSegment: z.string().optional(), // base64-encoded
 });
 
 // Mirrors VideoDecoderConfig

js/hang/src/container/codec.ts

@@ -11,12 +11,14 @@ import type * as Time from "../time";
  */
 export function encodeTimestamp(timestamp: Time.Micro, container: Catalog.Container = DEFAULT_CONTAINER): Uint8Array {
 	switch (container) {
-		case "legacy":
+		case "native":
 			return encodeVarInt(timestamp);
 		case "raw":
 			return encodeU64(timestamp);
-		case "fmp4":
-			throw new Error("fmp4 container not yet implemented");
+		case "cmaf": {
+			// CMAF fragments contain timestamps in moof atoms, no header needed
+			return new Uint8Array(0);
+		}
 	}
 }
 
@@ -32,16 +34,17 @@ export function decodeTimestamp(
 	container: Catalog.Container = DEFAULT_CONTAINER,
 ): [Time.Micro, Uint8Array] {
 	switch (container) {
-		case "legacy": {
+		case "native": {
 			const [value, remaining] = decodeVarInt(buffer);
 			return [value as Time.Micro, remaining];
 		}
 		case "raw": {
 			const [value, remaining] = decodeU64(buffer);
 			return [value as Time.Micro, remaining];
 		}
-		case "fmp4":
-			throw new Error("fmp4 container not yet implemented");
+		case "cmaf": {
+			return [0 as Time.Micro, buffer];
+		}
 	}
 }
 
@@ -54,12 +57,12 @@ export function decodeTimestamp(
  */
 export function getTimestampSize(container: Catalog.Container = DEFAULT_CONTAINER): number {
 	switch (container) {
-		case "legacy":
+		case "native":
 			return 8; // VarInt maximum size
 		case "raw":
 			return 8; // u64 fixed size
-		case "fmp4":
-			throw new Error("fmp4 container not yet implemented");
+		case "cmaf":
+			return 8; // VarInt maximum size (same as native)
 	}
 }
 

js/hang/src/frame.ts

@@ -20,6 +20,16 @@ export function encode(source: Uint8Array | Source, timestamp: Time.Micro, conta
 	// Encode timestamp using the specified container format
 	const timestampBytes = Container.encodeTimestamp(timestamp, container);
 
+	// For CMAF, timestampBytes will be empty, so we just return the source
+	if (container === "cmaf") {
+		if (source instanceof Uint8Array) {
+			return source;
+		}
+		const data = new Uint8Array(source.byteLength);
+		source.copyTo(data);
+		return data;
+	}
+
 	// Allocate buffer for timestamp + payload
 	const payloadSize = source instanceof Uint8Array ? source.byteLength : source.byteLength;
 	const data = new Uint8Array(timestampBytes.byteLength + payloadSize);
@@ -112,19 +122,18 @@ export class Consumer {
 
 	async #run() {
 		// Start fetching groups in the background
+
 		for (;;) {
 			const consumer = await this.#track.nextGroup();
-			if (!consumer) break;
+			if (!consumer) {
+				break;
+			}
 
-			// To improve TTV, we always start with the first group.
-			// For higher latencies we might need to figure something else out, as its racey.
 			if (this.#active === undefined) {
 				this.#active = consumer.sequence;
 			}
 
 			if (consumer.sequence < this.#active) {
-				console.warn(`skipping old group: ${consumer.sequence} < ${this.#active}`);
-				// Skip old groups.
 				consumer.close();
 				continue;
 			}
@@ -150,7 +159,9 @@ export class Consumer {
 
 			for (;;) {
 				const next = await group.consumer.readFrame();
-				if (!next) break;
+				if (!next) {
+					break;
+				}
 
 				const { data, timestamp } = decode(next, this.#container);
 				const frame = {
@@ -223,8 +234,6 @@ export class Consumer {
 		if (this.#active !== undefined && first.consumer.sequence <= this.#active) {
 			this.#groups.shift();
 
-			console.warn(`skipping slow group: ${first.consumer.sequence} < ${this.#groups[0]?.consumer.sequence}`);
-
 			first.consumer.close();
 			first.frames.length = 0;
 		}
@@ -246,7 +255,9 @@ export class Consumer {
 				this.#groups[0].consumer.sequence <= this.#active
 			) {
 				const frame = this.#groups[0].frames.shift();
-				if (frame) return frame;
+				if (frame) {
+					return frame;
+				}
 
 				// Check if the group is done and then remove it.
 				if (this.#active > this.#groups[0].consumer.sequence) {
@@ -261,7 +272,9 @@ export class Consumer {
 
 			const wait = new Promise<void>((resolve) => {
 				this.#notify = resolve;
-			}).then(() => true);
+			}).then(() => {
+				return true;
+			});
 
 			if (!(await Promise.race([wait, this.#signals.closed]))) {
 				this.#notify = undefined;

js/hang/src/util/mime.ts

@@ -0,0 +1,52 @@
+import type * as Catalog from "../catalog";
+
+/**
+ * Builds a MIME type string for MediaSource from a codec string.
+ *
+ * @param codec - The codec string from the catalog (e.g., "avc1.42E01E", "mp4a.40.2")
+ * @param type - "video" or "audio"
+ * @returns MIME type string (e.g., "video/mp4; codecs=\"avc1.42E01E\"")
+ */
+export function buildMimeType(codec: string, type: "video" | "audio"): string {
+	// For MP4 containers, we use the standard MIME type format
+	// Most codecs are already in the correct format for MSE
+	return `${type}/mp4; codecs="${codec}"`;
+}
+
+/**
+ * Checks if a MIME type is supported by MediaSource.
+ *
+ * @param mimeType - The MIME type to check
+ * @returns true if supported, false otherwise
+ */
+export function isMimeTypeSupported(mimeType: string): boolean {
+	return MediaSource.isTypeSupported(mimeType);
+}
+
+/**
+ * Builds and validates a MIME type for video from catalog config.
+ *
+ * @param config - Video configuration from catalog
+ * @returns MIME type string or undefined if not supported
+ */
+export function buildVideoMimeType(config: Catalog.VideoConfig): string | undefined {
+	const mimeType = buildMimeType(config.codec, "video");
+	if (isMimeTypeSupported(mimeType)) {
+		return mimeType;
+	}
+	return undefined;
+}
+
+/**
+ * Builds and validates a MIME type for audio from catalog config.
+ *
+ * @param config - Audio configuration from catalog
+ * @returns MIME type string or undefined if not supported
+ */
+export function buildAudioMimeType(config: Catalog.AudioConfig): string | undefined {
+	const mimeType = buildMimeType(config.codec, "audio");
+	if (isMimeTypeSupported(mimeType)) {
+		return mimeType;
+	}
+	return undefined;
+}

js/hang/src/watch/audio/emitter.ts

@@ -46,7 +46,8 @@ export class Emitter {
 		});
 
 		this.#signals.effect((effect) => {
-			const enabled = !effect.get(this.paused) && !effect.get(this.muted);
+			const paused = effect.get(this.paused);
+			const enabled = !paused;
 			this.source.enabled.set(enabled);
 		});
 
@@ -56,7 +57,44 @@ export class Emitter {
 			this.muted.set(volume === 0);
 		});
 
+		// Handle MSE path (HTMLAudioElement) vs WebCodecs path (AudioWorklet)
 		this.#signals.effect((effect) => {
+			const mseAudio = effect.get(this.source.mseAudioElement);
+			if (mseAudio) {
+				// MSE path: control HTMLAudioElement directly
+				effect.effect(() => {
+					const volume = effect.get(this.volume);
+					const muted = effect.get(this.muted);
+					const paused = effect.get(this.paused);
+					mseAudio.volume = volume;
+					mseAudio.muted = muted;
+
+					// Control play/pause state
+					if (paused && !mseAudio.paused) {
+						mseAudio.pause();
+					} else if (!paused && mseAudio.paused) {
+						// Resume if paused - try to play even if readyState is low
+						const tryPlay = () => {
+							if (!paused && mseAudio.paused) {
+								mseAudio
+									.play()
+									.catch((err) => console.error("[Audio Emitter] Failed to resume audio:", err));
+							}
+						};
+
+						// Try to play if we have metadata (HAVE_METADATA = 1), browser will start when ready
+						if (mseAudio.readyState >= HTMLMediaElement.HAVE_METADATA) {
+							tryPlay();
+						} else {
+							// Wait for loadedmetadata event if not ready yet
+							mseAudio.addEventListener("loadedmetadata", tryPlay, { once: true });
+						}
+					}
+				});
+				return;
+			}
+
+			// WebCodecs path: use AudioWorklet with GainNode
 			const root = effect.get(this.source.root);
 			if (!root) return;
 
@@ -76,9 +114,10 @@ export class Emitter {
 			});
 		});
 
+		// Only apply gain transitions for WebCodecs path (when gain node exists)
 		this.#signals.effect((effect) => {
 			const gain = effect.get(this.#gain);
-			if (!gain) return;
+			if (!gain) return; // MSE path doesn't use gain node
 
 			// Cancel any scheduled transitions on change.
 			effect.cleanup(() => gain.gain.cancelScheduledValues(gain.context.currentTime));