diff --git a/packages/cli/src/background-removal/pipeline.test.ts b/packages/cli/src/background-removal/pipeline.test.ts index afb8ad7fc5..4504e396bb 100644 --- a/packages/cli/src/background-removal/pipeline.test.ts +++ b/packages/cli/src/background-removal/pipeline.test.ts @@ -79,8 +79,15 @@ describe("background-removal/pipeline — buildEncoderArgs", () => { }, ); - it.each(["mov", "png"] as const)("keeps the %s output color path", (format) => { - const args = buildEncoderArgs(format, 64, 64, 30, `/tmp/cutout.${format}`); + it("converts raw RGBA to the tagged BT.709 matrix for the ProRes mov", () => { + const args = buildEncoderArgs("mov", 64, 64, 30, "/tmp/cutout.mov"); + const filter = args[args.indexOf("-vf") + 1]; + expect(filter).toContain("out_color_matrix=bt709"); + expect(filter).toContain("setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709"); + }); + + it("keeps the png output color path", () => { + const args = buildEncoderArgs("png", 64, 64, 30, "/tmp/cutout.png"); expect(args).not.toContain("-vf"); }); diff --git a/packages/cli/src/background-removal/pipeline.ts b/packages/cli/src/background-removal/pipeline.ts index 9a49c19c0e..9ead622129 100644 --- a/packages/cli/src/background-removal/pipeline.ts +++ b/packages/cli/src/background-removal/pipeline.ts @@ -23,6 +23,7 @@ import { type Device, type ModelId } from "./manager.js"; import { DEFAULT_VP9_CPU_USED, SDR_RGB_TO_BT709_FILTER, + SDR_RGB_TO_TAGGED_BT709_FILTER, renderProvenanceArgs, } from "@hyperframes/engine"; @@ -192,6 +193,8 @@ export function buildEncoderArgs( if (format === "mov") { return [ ...base, + "-vf", + SDR_RGB_TO_TAGGED_BT709_FILTER, "-c:v", "prores_ks", "-profile:v", diff --git a/packages/cli/src/background-removal/pipeline.windowsHide.test.ts b/packages/cli/src/background-removal/pipeline.windowsHide.test.ts index 69eb558c47..05a3a5e22b 100644 --- a/packages/cli/src/background-removal/pipeline.windowsHide.test.ts +++ b/packages/cli/src/background-removal/pipeline.windowsHide.test.ts @@ -36,6 +36,8 @@ vi.mock("./inference.js", () => ({ vi.mock("@hyperframes/engine", () => ({ DEFAULT_VP9_CPU_USED: 4, SDR_RGB_TO_BT709_FILTER: "scale=out_color_matrix=bt709:out_range=tv:flags=neighbor", + SDR_RGB_TO_TAGGED_BT709_FILTER: + "scale=out_color_matrix=bt709:out_range=tv:flags=neighbor,setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709:range=tv", renderProvenanceArgs: () => [], extractMediaMetadata: async () => ({ width: 1, height: 1, fps: 1, durationSeconds: 1 }), })); diff --git a/packages/engine/src/index.ts b/packages/engine/src/index.ts index 4fd3df11fa..f6215595b2 100644 --- a/packages/engine/src/index.ts +++ b/packages/engine/src/index.ts @@ -61,7 +61,10 @@ export { type EngineConfig, type ExtractCacheDirResolution, } from "./config.js"; -export { SDR_RGB_TO_BT709_FILTER } from "./utils/sdrCaptureColor.js"; +export { + SDR_RGB_TO_BT709_FILTER, + SDR_RGB_TO_TAGGED_BT709_FILTER, +} from "./utils/sdrCaptureColor.js"; export { DEFAULT_VP9_CPU_USED, MAX_VP9_CPU_USED, diff --git a/packages/engine/src/services/chunkEncoder.test.ts b/packages/engine/src/services/chunkEncoder.test.ts index d664ff2758..e437c0d30f 100644 --- a/packages/engine/src/services/chunkEncoder.test.ts +++ b/packages/engine/src/services/chunkEncoder.test.ts @@ -14,8 +14,11 @@ import { resolveLockedGopSize, } from "./chunkEncoder.js"; import { renderProvenanceArgs } from "../utils/renderProvenance.js"; -import { SDR_CAPTURE_TO_BT709_FILTER } from "../utils/sdrCaptureColor.js"; -import { getFfmpegBinary } from "../utils/ffmpegBinaries.js"; +import { + SDR_CAPTURE_TO_BT709_FILTER, + SDR_RGB_TO_TAGGED_BT709_FILTER, +} from "../utils/sdrCaptureColor.js"; +import { getFfmpegBinary, getFfprobeBinary } from "../utils/ffmpegBinaries.js"; const HAS_FFMPEG = spawnSync(getFfmpegBinary(), ["-version"]).status === 0; @@ -1152,7 +1155,7 @@ describe("buildEncoderArgs color space", () => { expect(args[args.indexOf("-vf") + 1]).toBe("pad=ceil(iw/2)*2:ceil(ih/2)*2"); }); - it("leaves alpha ProRes untouched (no even-dim pad)", () => { + it("converts alpha ProRes to BT.709 without an even-dim pad", () => { const args = buildEncoderArgs( { ...baseOptions, @@ -1164,7 +1167,7 @@ describe("buildEncoderArgs color space", () => { inputArgs, "out.mov", ); - expect(args.indexOf("-vf")).toBe(-1); + expect(args[args.indexOf("-vf") + 1]).toBe(SDR_RGB_TO_TAGGED_BT709_FILTER); expect(args.join(" ")).not.toContain("pad="); }); @@ -1746,26 +1749,27 @@ describe("buildConcatArgs", () => { }); describe.skipIf(!HAS_FFMPEG)("buildEncoderArgs SDR colour", () => { + const ffmpeg = getFfmpegBinary(); + const pixelAt = (file: string, decode: string, x: number, format = "rgb24"): number[] => [ + ...spawnSync(ffmpeg, [ + "-v", + "error", + "-i", + file, + "-vf", + `${decode}format=${format},crop=1:1:${x}:8`, + "-frames:v", + "1", + "-f", + "rawvideo", + "-", + ]).stdout, + ]; + // Chrome captures are BT.601 JPEGs. A direct YUV-to-YUV scale keeps that matrix under the BT.709 // tag (ffmpeg 7 and older) or tints greys (8 and newer), so the encode goes through RGB. it("delivers Chrome's JPEG colours in the BT.709 the mp4 is tagged with", () => { - const ffmpeg = getFfmpegBinary(); const dir = mkdtempSync(join(tmpdir(), "hf-sdr-colour-")); - const rgbAt = (file: string, decode: string, x: number): number[] => [ - ...spawnSync(ffmpeg, [ - "-v", - "error", - "-i", - file, - "-vf", - `${decode}format=rgb24,crop=1:1:${x}:8`, - "-frames:v", - "1", - "-f", - "rawvideo", - "-", - ]).stdout, - ]; try { for (const color of ["0xC83C28", "0xFE0000", "0x101010", "0x2050E0"]) { const jpg = join(dir, "frame.jpg"); @@ -1805,8 +1809,8 @@ describe.skipIf(!HAS_FFMPEG)("buildEncoderArgs SDR colour", () => { [8, 2], [32, 2], ] as const) { - const captured = rgbAt(jpg, "", x); - const delivered = rgbAt(out, "scale=in_color_matrix=bt709:in_range=tv,", x); + const captured = pixelAt(jpg, "", x); + const delivered = pixelAt(out, "scale=in_color_matrix=bt709:in_range=tv,", x); const worst = Math.max(...delivered.map((v, i) => Math.abs(v - captured[i]!))); expect( worst, @@ -1818,4 +1822,69 @@ describe.skipIf(!HAS_FFMPEG)("buildEncoderArgs SDR colour", () => { rmSync(dir, { recursive: true, force: true }); } }, 30_000); + + // Untagged HD reads as BT.709 in QuickTime, so ProRes has to be BT.709 too. MOV captures are + // RGBA PNGs, and the conversion must leave their alpha plane alone. The tags come from + // setparams because prores_ks ignores -color_* (ffmpeg 5.1 writes none, 8.1 only the matrix). + it("delivers PNG colours and alpha in the BT.709 the ProRes mov is tagged with", () => { + const dir = mkdtempSync(join(tmpdir(), "hf-prores-colour-")); + const png = join(dir, "frame.png"); + const out = join(dir, "out.mov"); + try { + for (const color of ["0xC83C28", "0x00FF00", "0x101010"]) { + const synth = spawnSync(ffmpeg, [ + "-v", + "error", + "-y", + "-f", + "lavfi", + "-i", + `color=c=${color}@0.5:s=64x16,format=rgba`, + "-frames:v", + "1", + png, + ]); + expect(synth.status).toBe(0); + const args = buildEncoderArgs( + { + fps: { num: 30, den: 1 }, + width: 64, + height: 16, + codec: "prores", + preset: "4444", + quality: 23, + pixelFormat: "yuva444p10le", + }, + ["-i", png], + out, + ); + expect(spawnSync(ffmpeg, args).status).toBe(0); + + const captured = pixelAt(png, "", 8, "rgba"); + const delivered = pixelAt(out, "scale=in_color_matrix=bt709:in_range=tv,", 8, "rgba"); + const worst = Math.max(...delivered.map((v, i) => Math.abs(v - captured[i]!))); + expect(worst, `${color}: capture ${captured} delivered ${delivered}`).toBeLessThanOrEqual( + 2, + ); + } + // Read the first frame, not the stream: a prores_ks mov's stream-level range depends on + // the ffprobe version (7.1 and 8.0 report unknown), while every version tags the frames. + const tags = spawnSync(getFfprobeBinary(), [ + "-v", + "error", + "-select_streams", + "v:0", + "-read_intervals", + "%+#1", + "-show_entries", + "frame=color_range,color_space,color_primaries,color_transfer", + "-of", + "csv=p=0", + out, + ]); + expect(tags.stdout.toString().trim()).toBe("tv,bt709,bt709,bt709"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }, 30_000); }); diff --git a/packages/engine/src/services/chunkEncoder.ts b/packages/engine/src/services/chunkEncoder.ts index 28243068a3..f90bd2ec6c 100644 --- a/packages/engine/src/services/chunkEncoder.ts +++ b/packages/engine/src/services/chunkEncoder.ts @@ -31,7 +31,10 @@ import { } from "../utils/gpuEncoder.js"; import { type HdrTransfer, getHdrEncoderColorParams } from "../utils/hdr.js"; import { withEvenDimensionPad } from "../utils/evenDimensions.js"; -import { SDR_CAPTURE_TO_BT709_FILTER } from "../utils/sdrCaptureColor.js"; +import { + SDR_CAPTURE_TO_BT709_FILTER, + SDR_RGB_TO_TAGGED_BT709_FILTER, +} from "../utils/sdrCaptureColor.js"; import { ffmpegStatsReader, formatFfmpegError, @@ -407,6 +410,7 @@ export function buildEncoderArgs( } } else if (codec === "prores") { args.push("-c:v", "prores_ks", "-profile:v", preset, "-vendor", "apl0"); + args.push("-vf", SDR_RGB_TO_TAGGED_BT709_FILTER); args.push("-pix_fmt", pixelFormat); appendRenderProvenanceArgs(args, outputPath); return [...args, "-y", outputPath]; diff --git a/packages/engine/src/services/streamingEncoder.test.ts b/packages/engine/src/services/streamingEncoder.test.ts index fbe6267252..74b9111873 100644 --- a/packages/engine/src/services/streamingEncoder.test.ts +++ b/packages/engine/src/services/streamingEncoder.test.ts @@ -26,7 +26,10 @@ import { } from "./streamingEncoder.js"; import { DEFAULT_HDR10_MASTERING } from "../utils/hdr.js"; import { type GpuEncoder } from "../utils/gpuEncoder.js"; -import { SDR_CAPTURE_TO_BT709_FILTER } from "../utils/sdrCaptureColor.js"; +import { + SDR_CAPTURE_TO_BT709_FILTER, + SDR_RGB_TO_TAGGED_BT709_FILTER, +} from "../utils/sdrCaptureColor.js"; const baseHdrPq: StreamingEncoderOptions = { fps: { num: 30, den: 1 }, @@ -241,6 +244,15 @@ describe("buildStreamingArgs", () => { `${SDR_CAPTURE_TO_BT709_FILTER},pad=ceil(iw/2)*2:ceil(ih/2)*2`, ); }); + + it("converts ProRes to the BT.709 it is tagged with and keeps the alpha format", () => { + const args = buildStreamingArgs( + { ...baseVp9, codec: "prores", preset: "4444", pixelFormat: "yuva444p10le" }, + "/tmp/out.mov", + ); + expect(args[args.indexOf("-vf") + 1]).toBe(SDR_RGB_TO_TAGGED_BT709_FILTER); + expect(args[args.indexOf("-pix_fmt") + 1]).toBe("yuva444p10le"); + }); }); describe("output path", () => { diff --git a/packages/engine/src/services/streamingEncoder.ts b/packages/engine/src/services/streamingEncoder.ts index 6c379c1351..605f450380 100644 --- a/packages/engine/src/services/streamingEncoder.ts +++ b/packages/engine/src/services/streamingEncoder.ts @@ -39,7 +39,11 @@ import { import { getFfmpegBinary } from "../utils/ffmpegBinaries.js"; import { getHdrEncoderColorParams } from "../utils/hdr.js"; import { withEvenDimensionPad } from "../utils/evenDimensions.js"; -import { SDR_CAPTURE_TO_BT709_FILTER, SDR_RGB_TO_BT709_FILTER } from "../utils/sdrCaptureColor.js"; +import { + SDR_CAPTURE_TO_BT709_FILTER, + SDR_RGB_TO_BT709_FILTER, + SDR_RGB_TO_TAGGED_BT709_FILTER, +} from "../utils/sdrCaptureColor.js"; import { DEFAULT_CONFIG, type EngineConfig } from "../config.js"; import { fpsToFfmpegArg, fpsToNumber, type Fps } from "@hyperframes/core"; import { appendVp9CpuUsedArg } from "./vp9Options.js"; @@ -385,6 +389,7 @@ export function buildStreamingArgs( } } else if (codec === "prores") { args.push("-c:v", "prores_ks", "-profile:v", preset, "-vendor", "apl0"); + args.push("-vf", SDR_RGB_TO_TAGGED_BT709_FILTER); args.push("-pix_fmt", pixelFormat); appendRenderProvenanceArgs(args, outputPath); return [...args, "-y", outputPath]; diff --git a/packages/engine/src/utils/sdrCaptureColor.ts b/packages/engine/src/utils/sdrCaptureColor.ts index 3a8e240144..7cc94e0fb6 100644 --- a/packages/engine/src/utils/sdrCaptureColor.ts +++ b/packages/engine/src/utils/sdrCaptureColor.ts @@ -1,3 +1,5 @@ export const SDR_RGB_TO_BT709_FILTER = "scale=out_color_matrix=bt709:out_range=tv:flags=neighbor"; export const SDR_CAPTURE_TO_BT709_FILTER = `scale=in_color_matrix=bt601:in_range=pc:flags=neighbor,format=gbrp,${SDR_RGB_TO_BT709_FILTER}`; + +export const SDR_RGB_TO_TAGGED_BT709_FILTER = `${SDR_RGB_TO_BT709_FILTER},setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709:range=tv`;