Merge pull request #124 from shotstack/derk/eng-653-studio-shared-resolver

fix: shared rich-caption font resolution + preview/export render-parity fixes

Author: dazzatronus

package.json

@@ -100,7 +100,7 @@
 	},
 	"dependencies": {
 		"@shotstack/schemas": "1.11.0",
-		"@shotstack/shotstack-canvas": "^2.9.0",
+		"@shotstack/shotstack-canvas": "^2.10.0",
 		"howler": "^2.2.4",
 		"mediabunny": "^1.11.2",
 		"opentype.js": "^1.3.4",

src/components/canvas/players/html5-player.ts

@@ -3,6 +3,7 @@ import type { Edit } from "@core/edit-session";
 import { EditEvent } from "@core/events/edit-events";
 import { type Size } from "@layouts/geometry";
 import type { ResolvedClip } from "@schemas";
+import { nextFrame } from "@shared/utils";
 import { Html5AssetSchema, composeHtml5IframeSrcdoc, computeHtml5FrameCount, type Html5Asset } from "@shotstack/shotstack-canvas";
 import * as pixi from "pixi.js";
 
@@ -19,13 +20,6 @@ type HarnessWindow = Window & {
 const SEEK_KEY = "__shotstackSeek" as const;
 const CAPTURE_CONCURRENCY = 4;
 
-function yieldFrame(): Promise<void> {
-	return new Promise<void>(resolve => {
-		if (typeof requestAnimationFrame === "function") requestAnimationFrame(() => resolve());
-		else setTimeout(() => resolve(), 0);
-	});
-}
-
 function forceLayout(el: HTMLElement): void {
 	el.getBoundingClientRect();
 }
@@ -313,7 +307,7 @@ export class Html5Player extends Player {
 		} catch {
 			/* older browsers — proceed */
 		}
-		await yieldFrame();
+		await nextFrame();
 		if (stale()) return null;
 
 		const { frameCount } = computeHtml5FrameCount({
@@ -330,7 +324,7 @@ export class Html5Player extends Player {
 
 		const blobs: Blob[] = [];
 		for (let i = 0; i < frameCount; i += CAPTURE_CONCURRENCY) {
-			await yieldFrame();
+			await nextFrame();
 			if (stale()) return null;
 			const batch: Promise<Uint8Array>[] = [];
 			for (let j = i; j < Math.min(i + CAPTURE_CONCURRENCY, frameCount); j += 1) {

src/components/canvas/players/player.ts

@@ -2,6 +2,7 @@ import { ComposedKeyframeBuilder } from "@animations/composed-keyframe-builder";
 import { EffectPresetBuilder } from "@animations/effect-preset-builder";
 import { KeyframeBuilder } from "@animations/keyframe-builder";
 import { TransitionPresetBuilder } from "@animations/transition-preset-builder";
+import { WipeFilter } from "@animations/wipe-filter";
 import { type Edit } from "@core/edit-session";
 import { InternalEvent } from "@core/events/edit-events";
 import { calculateContainerScale, calculateFitScale, calculateSpriteTransform, type FitMode } from "@core/layout/fit-system";
@@ -87,6 +88,11 @@ export abstract class Player extends Entity {
 	private skewXKeyframeBuilder?: ComposedKeyframeBuilder;
 	private skewYKeyframeBuilder?: ComposedKeyframeBuilder;
 	private maskXKeyframeBuilder?: KeyframeBuilder;
+	private wipeBrightnessBuilder?: KeyframeBuilder;
+	private wipeInFromRight: boolean = false;
+	private wipeOutFromRight: boolean = false;
+	private wipeOutStart: number = Number.POSITIVE_INFINITY;
+	private wipeFilter: WipeFilter | null = null;
 
 	private wipeMask: pixi.Graphics | null;
 	protected lumaWrapper: pixi.Container;
@@ -208,11 +214,20 @@ export abstract class Player extends Entity {
 		this.opacityKeyframeBuilder.addLayer(transitionSet.out.opacityKeyframes);
 		this.rotationKeyframeBuilder.addLayer(transitionSet.out.rotationKeyframes);
 
-		// Mask keyframes (wipe/reveal effects)
+		// Mask keyframes (reveal effect)
 		const maskXKeyframes: Keyframe[] = [...transitionSet.in.maskXKeyframes, ...transitionSet.out.maskXKeyframes];
 		if (maskXKeyframes.length) {
 			this.maskXKeyframeBuilder = new KeyframeBuilder(maskXKeyframes, length);
 		}
+
+		// Wipe brightness sweep (wipeLeft/wipeRight) — driven through WipeFilter, not a mask.
+		const brightnessKeyframes: Keyframe[] = [...transitionSet.in.brightnessKeyframes, ...transitionSet.out.brightnessKeyframes];
+		if (brightnessKeyframes.length) {
+			this.wipeBrightnessBuilder = new KeyframeBuilder(brightnessKeyframes, length);
+			this.wipeInFromRight = transitionSet.in.wipeFromRight;
+			this.wipeOutFromRight = transitionSet.out.wipeFromRight;
+			this.wipeOutStart = transitionSet.out.brightnessKeyframes[0]?.start ?? Number.POSITIVE_INFINITY;
+		}
 	}
 
 	public override async load(): Promise<void> {
@@ -234,6 +249,14 @@ export abstract class Player extends Entity {
 	}
 
 	public override update(_: number, __: number): void {
+		this.applyPlayheadState();
+	}
+
+	/**
+	 * Apply all playhead-time-dependent container state (visibility, transform, opacity, wipe mask)
+	 * for the current playback time. Called by the per-frame tick and directly by static capture.
+	 */
+	public applyPlayheadState(): void {
 		this.getContainer().visible = this.isActive();
 		this.getContainer().zIndex = 100000 - this.layer * 100;
 		if (!this.isActive()) {
@@ -262,6 +285,7 @@ export abstract class Player extends Entity {
 
 		// Update wipe/reveal mask animation
 		this.updateWipeMask();
+		this.updateWipeFilter();
 
 		if (this.shouldDiscardFrame()) {
 			this.contentContainer.alpha = 0;
@@ -294,9 +318,32 @@ export abstract class Player extends Entity {
 		this.wipeMask.fill(0xffffff);
 	}
 
+	private updateWipeFilter(): void {
+		if (!this.wipeBrightnessBuilder) {
+			this.removeWipeFilter();
+			return;
+		}
+		if (!this.wipeFilter) {
+			this.wipeFilter = new WipeFilter();
+			this.contentContainer.filters = [this.wipeFilter];
+		}
+		const time = this.getPlaybackTime();
+		this.wipeFilter.brightness = this.wipeBrightnessBuilder.getValue(time);
+		this.wipeFilter.revealFromRight = time >= this.wipeOutStart ? this.wipeOutFromRight : this.wipeInFromRight;
+	}
+
+	private removeWipeFilter(): void {
+		if (!this.wipeFilter) return;
+		this.contentContainer.filters = [];
+		this.wipeFilter.destroy();
+		this.wipeFilter = null;
+	}
+
 	public override dispose(): void {
 		this.wipeMask?.destroy();
 		this.wipeMask = null;
+		this.wipeFilter?.destroy();
+		this.wipeFilter = null;
 
 		this.contentContainer?.destroy();
 		this.lumaWrapper?.destroy();

src/components/canvas/players/rich-caption-player.ts

@@ -12,6 +12,7 @@ import {
 	createDefaultGeneratorConfig,
 	createWebPainter,
 	buildCaptionLayoutConfig,
+	resolveCaptionFonts,
 	parseSubtitleToWords,
 	CanvasRichCaptionAssetSchema,
 	type CanvasRichCaptionAsset,
@@ -34,6 +35,7 @@ export class RichCaptionPlayer extends Player {
 
 	private canvas: HTMLCanvasElement | null = null;
 	private painter: ReturnType<typeof createWebPainter> | null = null;
+	private currentRender: Promise<void> | null = null; // serialises the async paint so captures await a finished frame
 	private texture: pixi.Texture | null = null;
 	private sprite: pixi.Sprite | null = null;
 
@@ -139,6 +141,16 @@ export class RichCaptionPlayer extends Player {
 		this.renderFrameSync(currentTimeMs);
 	}
 
+	/**
+	 * Render the exact playhead frame to completion for an off-playback capture. The web painter is
+	 * async, so this awaits {@link renderFrame} (which awaits the paint) before captureFrame extracts —
+	 * otherwise the snapshot is a half-drawn caption. Asset loading is awaited by the caller.
+	 * @internal
+	 */
+	public override async prepareStaticRender(): Promise<void> {
+		if (this.loadComplete) await this.renderFrame(this.getPlaybackTime() * 1000);
+	}
+
 	public override async reloadAsset(): Promise<void> {
 		const asset = this.clipConfiguration.asset as RichCaptionAsset;
 
@@ -218,11 +230,6 @@ export class RichCaptionPlayer extends Player {
 
 			const { width, height } = this.getSize();
 			const layoutConfig = buildCaptionLayoutConfig(this.validatedAsset, width, height);
-			const letterSpacing = this.validatedAsset?.style?.letterSpacing;
-			const canvasTextMeasurer = this.createCanvasTextMeasurer(letterSpacing);
-			if (canvasTextMeasurer) {
-				layoutConfig.measureTextWidth = canvasTextMeasurer;
-			}
 			this.captionLayout = await this.layoutEngine.layoutCaption(this.words, layoutConfig);
 
 			this.generatorConfig = createDefaultGeneratorConfig(width, height, 1);
@@ -253,12 +260,6 @@ export class RichCaptionPlayer extends Player {
 		const { width, height } = this.getSize();
 		const layoutConfig = buildCaptionLayoutConfig(this.validatedAsset, width, height);
 
-		const letterSpacing = this.validatedAsset?.style?.letterSpacing;
-		const canvasTextMeasurer = this.createCanvasTextMeasurer(letterSpacing);
-		if (canvasTextMeasurer) {
-			layoutConfig.measureTextWidth = canvasTextMeasurer;
-		}
-
 		this.captionLayout = await this.layoutEngine.layoutCaption(words, layoutConfig);
 
 		this.generatorConfig = createDefaultGeneratorConfig(width, height, 1);
@@ -273,7 +274,31 @@ export class RichCaptionPlayer extends Player {
 		this.loadComplete = true;
 	}
 
+	/**
+	 * Render the caption at `timeMs`, serialised so the shared canvas isn't cleared mid-paint.
+	 * Awaited by {@link prepareStaticRender} for off-playback captures.
+	 */
+	private async renderFrame(timeMs: number): Promise<void> {
+		while (this.currentRender) {
+			await this.currentRender;
+		}
+		this.currentRender = this.paintFrame(timeMs).finally(() => {
+			this.currentRender = null;
+		});
+		await this.currentRender;
+	}
+
+	/** Fire-and-forget render for the live tick and lifecycle hooks; the texture updates when the paint settles. */
 	private renderFrameSync(timeMs: number): void {
+		this.renderFrame(timeMs).catch(() => {});
+	}
+
+	/**
+	 * Paint the caption frame to completion. The web painter is async (glyph fills resolve
+	 * asynchronously), so the Pixi texture is only updated once the paint has finished — otherwise a
+	 * snapshot captures a half-drawn canvas (a single glyph).
+	 */
+	private async paintFrame(timeMs: number): Promise<void> {
 		if (!this.layoutEngine || !this.captionLayout || !this.canvas || !this.painter || !this.validatedAsset || !this.generatorConfig) {
 			return;
 		}
@@ -289,7 +314,7 @@ export class RichCaptionPlayer extends Player {
 			const ctx = this.canvas.getContext("2d");
 			if (ctx) ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);
 
-			this.painter.render(ops);
+			await this.painter.render(ops);
 
 			if (!this.texture) {
 				this.texture = pixi.Texture.from(this.canvas);
@@ -323,20 +348,46 @@ export class RichCaptionPlayer extends Player {
 		}
 	}
 
+	// Shared font resolution used by both registration and the render payload, so the registered
+	// face and the rendered family are always the same. Delegates to the canvas resolver.
+	private resolveFontsForAsset(asset: RichCaptionAsset) {
+		const family = asset.font?.family ?? "Roboto";
+		const weight = asset.font?.weight ? parseInt(String(asset.font.weight), 10) || 400 : 400;
+		const fontNameMap = new Map<string, string>();
+		for (const [src, meta] of this.edit.getFontMetadata()) fontNameMap.set(src, meta.baseFamilyName);
+		const activeFamily = asset.active?.font?.family;
+		const activeWeight = asset.active?.font?.weight;
+		return resolveCaptionFonts({
+			family,
+			weight,
+			timelineFonts: this.edit.getTimelineFonts(),
+			fontNameMap,
+			...(activeFamily ? { activeFamily } : {}),
+			...(activeWeight != null ? { activeWeight: activeWeight as string | number } : {})
+		});
+	}
+
 	private async registerFonts(asset: RichCaptionAsset): Promise<void> {
 		if (!this.fontRegistry) return;
 
 		const family = asset.font?.family ?? "Roboto";
 		const assetWeight = asset.font?.weight ? parseInt(String(asset.font.weight), 10) || 400 : 400;
+
+		// Registration and the render payload (buildCanvasPayload) share this one resolution, so the
+		// render looks up exactly the face that was registered.
+		const resolution = this.resolveFontsForAsset(asset);
+
+		if (resolution.matched) {
+			for (const font of resolution.fonts) {
+				if (font.src) await this.registerFontFromUrl(font.src, font.family, parseInt(font.weight, 10) || 400);
+			}
+			return;
+		}
+
 		const resolved = this.resolveFontWithWeight(family, assetWeight);
 		if (resolved) {
 			await this.registerFontFromUrl(resolved.url, resolved.baseFontFamily, resolved.fontWeight);
 		}
-
-		const customFonts = this.buildCustomFontsFromTimeline(asset);
-		for (const customFont of customFonts) {
-			await this.registerFontFromUrl(customFont.src, customFont.family, parseInt(customFont.weight, 10) || 400);
-		}
 	}
 
 	private async registerFontFromUrl(url: string, family: string, weight: number): Promise<boolean> {
@@ -465,8 +516,12 @@ export class RichCaptionPlayer extends Player {
 
 	private buildCanvasPayload(asset: RichCaptionAsset, words: WordTiming[]): Record<string, unknown> {
 		const { width, height } = this.getSize();
-		const customFonts = this.buildCustomFontsFromTimeline(asset);
-		const resolvedFamily = getFontDisplayName(asset.font?.family ?? "Roboto");
+		// Use the same resolution as registration, so the rendered family matches the registered face.
+		const resolution = this.resolveFontsForAsset(asset);
+		const resolvedFamily = resolution.matched ? resolution.resolvedFamily : getFontDisplayName(asset.font?.family ?? "Roboto");
+		const customFonts = resolution.matched
+			? resolution.fonts.filter(f => f.src).map(f => ({ src: f.src as string, family: f.family, weight: f.weight }))
+			: this.buildCustomFontsFromTimeline(asset);
 
 		const payload: Record<string, unknown> = {
 			type: asset.type,
@@ -484,7 +539,7 @@ export class RichCaptionPlayer extends Player {
 			border: asset.border,
 			padding: asset.padding,
 			style: asset.style,
-			wordAnimation: asset.animation,
+			animation: asset.animation,
 			align: asset.align,
 			pauseThreshold: this.resolvedPauseThreshold
 		};
@@ -502,25 +557,6 @@ export class RichCaptionPlayer extends Player {
 		return payload;
 	}
 
-	private createCanvasTextMeasurer(letterSpacing?: number): ((text: string, font: string) => number) | undefined {
-		try {
-			const measureCanvas = document.createElement("canvas");
-			const ctx = measureCanvas.getContext("2d");
-			if (!ctx) return undefined;
-
-			if (letterSpacing) {
-				(ctx as unknown as Record<string, unknown>)["letterSpacing"] = `${letterSpacing}px`;
-			}
-
-			return (text: string, font: string): number => {
-				ctx.font = font;
-				return ctx.measureText(text).width;
-			};
-		} catch {
-			return undefined;
-		}
-	}
-
 	private createFallbackGraphic(message: string): void {
 		const { width, height } = this.getSize();
 
@@ -652,11 +688,6 @@ export class RichCaptionPlayer extends Player {
 		if (!this.layoutEngine) return;
 
 		const layoutConfig = buildCaptionLayoutConfig(this.validatedAsset, width, height);
-		const letterSpacing = this.validatedAsset?.style?.letterSpacing;
-		const canvasTextMeasurer = this.createCanvasTextMeasurer(letterSpacing);
-		if (canvasTextMeasurer) {
-			layoutConfig.measureTextWidth = canvasTextMeasurer;
-		}
 
 		this.pendingLayoutId += 1;
 		const layoutId = this.pendingLayoutId;