diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..a27ca95 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,112 @@ +# Changelog + +## 0.9.0 + +MMCP protocol 1.2: motion and video references as prompts. + +### Added + +- `video_reference` segment (`VideoReferenceSegment`): a prompt given as a + video instead of text. `video` is exactly one of `{"url": "https://..."}` + (the server fetches it) or `{"data": , "media_type": "video/mp4" | + "video/quicktime" | "video/webm"}`; optional `start_s` / `end_s` trim + (`0 <= start_s < end_s`) and `fps` (a hint); the server follows the most + prominent person in the video. Otherwise it **is a text segment**, + validated with the text rules exactly as `motion_reference` is. The SDK + checks the payload (one source; an https URL with a host and no user + info, whitespace or bad port; standard padded base64, its size worked + out from its length; finite numbers; the trim; 422) and never decodes + the video: a backbone gets it parsed (`VideoSource.decoded()`, + `.num_bytes`). +- `limits.max_video_bytes` (decoded inline data over it is + `413 payload_too_large`) and `limits.max_video_seconds` (a trimmed video + over it is `400 invalid_options`), both optional, `details.segment` + naming the segment, left out of `/capabilities` when unset. A URL's size + and an untrimmed video's length are the server's to enforce. +- Models opt in by listing `"video_reference"` in `supported_segments`; + the existing gate answers `unsupported_segment` for models that don't. + No server implements it yet. +- `GenerateRequest.video_reference` / `.video_references`, `VideoSource`. +- `motionmcp.client.video_source(path | url=, max_bytes=)`: a video file + (base64, media type from the extension, size checked before reading) or + an https URL (the server's URL rule) as a `video` object. Standard + library only. +- `motionmcp.fetch_video_url(url, max_bytes=, timeout=)`: fetch a + `video.url` without SSRF -- https only, every resolved address and every + redirect hop must be public (IPv6 forms that embed an IPv4 address -- + mapped, compatible, NAT64, 6to4, Teredo -- judged by it), the connection + pinned to the checked address, a streamed byte cap (default 100 MiB) and + one hard deadline over connect, TLS, headers and body. Synchronous, + standard library only. +- `motion_reference` segment (`MotionReferenceSegment`): a prompt given as a + motion instead of text. Send a clip (`joint_names`, per-frame local + `rotations` T×J×4, `root_positions` T×3, `fps`) where a text segment + carries its prompt; the model reads it as it reads a caption and makes new + motion of that kind. Otherwise the segment **is a text segment** and the + SDK validates it with the text rules: `duration_frames` (required), + per-segment `seed`, any number of them mixed in any order with `text` / + `unconditioned` / `pose` segments, `options.loop` (on a `supports_loop` + model, one segment), constraints over the whole take, `num_samples`, + guidance, batch bodies. The response is a text response's (no extra + metadata). +- Its payload's own checks: T ≥ 2 and consistent lengths (422); joints on + the request skeleton, or on the segment's own `skeleton` when it carries + one (`unknown_joint`); a non-canonical segment `skeleton` needs + `supports_retargeting` (`retargeting_unsupported`). +- `MotionReferenceSegment.skeleton` (optional, the request `Skeleton` + schema): the clip's own rig, so a reference can come from any rig. +- `limits.max_reference_frames` (optional): when set, a clip longer than it + is `400 invalid_options` (`details.segment` names it). It counts frames + as sent: a server MUST accept a clip within it at any `fps` (resampling + or trimming to its own window). It caps the clip, not the output (that is + `max_duration_seconds`, as for text). Left out of `/capabilities` when + unset. +- A motion reference must include its skeleton's root joint + (`400 invalid_request`), and its numbers must be finite (422). +- Models opt in by listing `"motion_reference"` in `supported_segments`; the + existing gate still answers `unsupported_segment` for models that don't. + `supports_motion_reference_mixed` is advertised `true` exactly when a + model lists `"motion_reference"` (a reference always mixes like text). +- `GenerateRequest.motion_reference` / `.motion_references`, + `MotionReferenceSegment.reference_frames`. +- Error code `invalid_request` (400), for backbones. +- Client: `poll_job(server_url, location, ...)` resumes polling an + accepted job, e.g. after refreshing a token on a 401 mid-poll, instead of + starting a second job. (#8) + +### Changed + +- `PROTOCOL_VERSION` is `"1.2"` (was `"1.0"`; 1.1 was never stamped here). + The SDK still sets no `options.guidance` default: a backbone applies its + text default, to a reference as to text. +- Trajectory: the root moves along its shape (still / line / arc / Bezier, + chosen by fit) with the body's centre of mass, not at a constant speed, so + a stride's speeding up and slowing down no longer shows as the body + floating forward and back when played in place. Models are just + still/line/arc/bezier; params gain `speed_range` and drop the distance + keys. (#7) +- Client: every `MmcpError` from `generate()` carries `details["phase"]` + (`"generate"` or `"poll"`); poll errors also carry `details["location"]`. + Codes and messages are unchanged. (#8) + +- *Clarified:* clients SHOULD send identity `rest_rotation` on every + joint (1.0 meanings unchanged: `rest_translation` / `rest_rotation` are + parent-local, `fill_mode: "rest"` pins to `rest_rotation`); a server that + can't honour a non-identity one MUST refuse it with `400 invalid_skeleton`. +- The SDK server refuses a body over `limits.max_request_bytes` before + parsing it (`413 payload_too_large`: from `Content-Length`, or while + streaming one sent without), and each model's own cap after. +- A segment type the SDK knows but the model doesn't list is + `unsupported_segment`, checked (after the protocol major) before the + segment's payload is parsed; a type the SDK doesn't know stays 422. +- Batch items are held to their own model's `max_request_bytes`. + +### Fixed + +- A request failing one of the schema's own validators (e.g. `root_path` + lengths) is a `422 schema_validation` envelope with the message, not a + 500 from an unserialisable error context; the envelope no longer echoes + the offending input (which could be megabytes of `video.data`). A body + nested too deeply to parse is a 422 envelope too, not a 500. +- A skeleton with a bad topology (the request's or a segment's) is + `400 invalid_skeleton`, as documented, not `422 schema_validation`. diff --git a/docs/docs/concepts/constraints.mdx b/docs/docs/concepts/constraints.mdx index aea9ca6..ec65dc9 100644 --- a/docs/docs/concepts/constraints.mdx +++ b/docs/docs/concepts/constraints.mdx @@ -102,7 +102,8 @@ A pose at a single frame. Fill mode semantics: - **`rest`** — unspecified joints are pinned to their `rest_rotation` at - this frame. + this frame (identity, i.e. the rest pose, when identity rest rotations + are sent, as clients SHOULD; see [Rest rotations](./skeleton#rest-rotations)). - **`generate`** — unspecified joints are unconstrained; the backbone generates them as part of the motion. The constraint pins only the listed joints. diff --git a/docs/docs/concepts/segments.mdx b/docs/docs/concepts/segments.mdx index ae8babc..6cc0b8f 100644 --- a/docs/docs/concepts/segments.mdx +++ b/docs/docs/concepts/segments.mdx @@ -1,7 +1,7 @@ --- id: segments title: Segments -description: How to compose a motion request from text, unconditioned, and pose spans. +description: How to compose a motion request from text, unconditioned, pose, motion-reference and video-reference spans. sidebar_position: 2 --- @@ -11,9 +11,12 @@ A `Segment` describes one stretch of motion in time. Sequential segments compose into a clip — the request's total length is `sum(segments[].duration_frames)`. A single [`pose`](#posesegment--type-pose) segment is the one exception: it produces exactly one frame and can't be -composed with others. +composed with others. A [`motion_reference`](#motionreferencesegment--type-motion_reference) +segment (1.2) is a prompt given as a clip instead of words, and a +[`video_reference`](#videoreferencesegment--type-video_reference) segment +(1.2) one given as a video; both compose exactly like a `text` segment. -Three segment types are defined in v1: +Five segment types are defined in v1: ```json // Multi-segment clip @@ -27,13 +30,34 @@ Three segment types are defined in v1: [ { "type": "pose", "prompt": "person waves hello with their right hand" } ] + +// Motion like a reference clip (1.2) +[ + { "type": "motion_reference", "duration_frames": 90, "joint_names": ["Hips", "Spine1"], + "rotations": [[[0, 0, 0, 1], [0, 0, 0, 1]], [[0, 0, 0, 1], [0, 0.01, 0, 1]]], + "root_positions": [[0, 0.95, 0], [0, 0.95, 0.03]], "fps": 30 } +] + +// A walk like one clip, then a sit like another (1.2) +[ + { "type": "motion_reference", "duration_frames": 90, "joint_names": [/* … */], + "rotations": [/* walk clip */], "root_positions": [/* … */], "fps": 30 }, + { "type": "motion_reference", "duration_frames": 60, "joint_names": [/* … */], + "rotations": [/* sit clip */], "root_positions": [/* … */], "fps": 30 } +] + +// A dance like the one in a video, seconds 2–6 of it (1.2) +[ + { "type": "video_reference", "duration_frames": 120, + "video": { "url": "https://cdn.example.com/dance.mp4" }, "start_s": 2, "end_s": 6 } +] ``` `Segment` is a **discriminated union** with `type` as the discriminator. Servers advertise which types they accept via `ModelSpec.supported_segments` in `/capabilities` — clients SHOULD check before sending a non-default type. Future minor versions MAY add -types (e.g. `"audio"`, `"motion_reference"`) under the same key. +types (e.g. `"audio"`) under the same key. ## TextSegment — `type: "text"` @@ -96,6 +120,165 @@ rejects pose segments sent to a model that doesn't claim support, with the `unsupported_segment` error code. See **[Capabilities reference →](../reference/capabilities)**. +## MotionReferenceSegment — `type: "motion_reference"` + +*Since 1.2.* A **prompt given as a motion instead of words**. The client +sends a clip sampled from its own rig (a take, hand keys, mocap) where a +text segment would carry its prompt; the model reads the clip the way it +reads a caption — a backbone turns it into the same kind of conditioning +vector its text encoder makes for a prompt — and generates new motion *of +that kind*, not a copy of the clip. Apart from that, a reference segment +**is a text segment**: it covers `duration_frames` of the take, takes a +`seed`, and mixes with `text`, `unconditioned` and other reference segments +in any order, stitched with the same transitions; constraints, +`options.loop`, `num_samples`, guidance and batching apply exactly as they +do to text, and the SDK validates it with the text segment's rules. The +clip can come from **any rig**: send that rig as the segment's own +`skeleton` and the server retargets the clip from it; the output is always +on the request `skeleton`. + +```json +{ + "type": "motion_reference", + "duration_frames": 150, + "skeleton": { "joints": [ /* the clip's rig, optional */ ] }, + "joint_names": ["Hips", "Spine1", "..."], + "rotations": [[[0, 0, 0, 1], "... one per joint"], "... one per frame"], + "root_positions": [[0, 0.95, 0], "... one per frame"], + "fps": 30 +} +``` + +| Field | Type | Required | Notes | +|---|---|---|---| +| `type` | `"motion_reference"` | yes | Discriminator | +| `duration_frames` | int (≥ 1) | yes | Output length, in frames at the request's effective fps, as for `text`. Independent of the clip's length | +| `skeleton` | [`Skeleton`](./skeleton), same schema as the request `skeleton` | no | The reference clip's own rig. When present, `joint_names`, `rotations` and `root_positions` refer to it and the server retargets the clip from it; when absent they refer to the request `skeleton` | +| `joint_names` | string[] (unique, ≥ 1) | yes | Joints of the segment's `skeleton` if given, else of the request `skeleton`; named as in `pose_keyframe.joint_rotations` | +| `rotations` | `[x, y, z, w][][]`, T × J | yes | Local-to-parent quaternions per frame, one per `joint_names` entry, in the same convention and order as `pose_keyframe.joint_rotations`. T ≥ 2, at most `limits.max_reference_frames` | +| `root_positions` | `[x, y, z][]`, T | yes | The root joint's world position per frame, Y-up metres, as `pose_keyframe.root_position` | +| `fps` | number (> 0) | yes | Frame rate of the clip's samples (may differ from the request's) | +| `seed` | int | no | Per-segment seed, as [`TextSegment.seed`](#textsegment--type-text) | + +To build one from a rig (the conventions, sampling a clip in numpy or +Blender, BVH and FBX, the size budget), see +**[Building reference segments →](../guides/reference-segments)**. + +The payload's own checks: + +- `len(rotations) == len(root_positions) == T`, every frame has one + quaternion per `joint_names` entry, and T ≥ 2 → otherwise + `422 schema_validation`. Every number must be finite (no `NaN` / + `Infinity`). Quaternions are taken as sent (as for `pose_keyframe`): not + required to be exactly unit length. +- Every `joint_names` entry must be a joint of the segment's `skeleton` + when it has one, else of the request `skeleton` → `unknown_joint`. The + clip must include that skeleton's root joint → `400 invalid_request`. +- A segment `skeleton` with a bad topology (no root, several roots, a + parent listed after its child) → `400 invalid_skeleton`, as for the + request skeleton. +- A segment `skeleton` other than the model's canonical skeleton needs a + model with `supports_retargeting` → otherwise `retargeting_unsupported`. +- A clip longer than `limits.max_reference_frames` (when set) → + `400 invalid_options`. `limits.max_reference_frames` counts the clip's frames **as sent** (`len(rotations)`), whatever its `fps`: a server MUST accept a clip within it, and MUST NOT refuse one within it because of how it resamples the clip internally (it resamples or trims to fit its own window instead). + +Everything else is the `text` segment's rule, unchanged. The response is +the usual glTF, exactly as for a text request. + +`MotionReferenceSegment` is an **optional** capability: a model lists +`"motion_reference"` in `supported_segments` and SHOULD set +`limits.max_reference_frames`. The SDK rejects the segment for a model that +doesn't (`unsupported_segment`), before reading the segment's payload. +`supports_motion_reference_mixed` is `true` exactly when a model lists +`"motion_reference"` (a reference mixes like text). + +## VideoReferenceSegment — `type: "video_reference"` + +*Since 1.2.* A **prompt given as a video instead of words**: footage of a +person moving — a phone clip, a reference from the web, a performance +capture pass. The model reads the person's motion in it as it reads a +caption and generates new motion *of that kind* on the request `skeleton`, +not a copy of the footage. It is to video what +[`motion_reference`](#motionreferencesegment--type-motion_reference) is to +a clip, and in every other respect it **is a text segment**: it covers +`duration_frames` of the take, takes a `seed`, and mixes with `text`, +`unconditioned`, `motion_reference` and other video segments in any order; +constraints, `options.loop`, `num_samples`, guidance and batching apply +exactly as they do to text, and the SDK validates it with the text +segment's rules. + +```json +// The server fetches the video +{ + "type": "video_reference", + "duration_frames": 120, + "video": { "url": "https://cdn.example.com/dance.mp4" }, + "start_s": 2.0, + "end_s": 6.0 +} + +// The video inline +{ + "type": "video_reference", + "duration_frames": 90, + "video": { "data": "AAAAGGZ0eXBtcDQy…", "media_type": "video/mp4" }, + "fps": 30 +} +``` + +| Field | Type | Required | Notes | +|---|---|---|---| +| `type` | `"video_reference"` | yes | Discriminator | +| `duration_frames` | int (≥ 1) | yes | Output length, in frames at the request's effective fps, as for `text`. Independent of the video's length | +| `video` | object, exactly one of the two forms below | yes | The video | +| `video.url` | string, an `https` URL with a host | one of | The server fetches it. No other fields go with it | +| `video.data` | string, standard base64 (RFC 4648, padded) | one of | The video file inline, non-empty | +| `video.media_type` | `"video/mp4"` \| `"video/quicktime"` \| `"video/webm"` | with `data` | The container of `data` | +| `start_s` | number (≥ 0) | no | Where to start reading, in seconds into the video. Default: the start | +| `end_s` | number (> `start_s`) | no | Where to stop reading, in seconds into the video. Default: the end | +| `fps` | number (> 0) | no | A hint for the video's frame rate, for containers that report it badly | +| `seed` | int | no | Per-segment seed, as [`TextSegment.seed`](#textsegment--type-text) | + +Unknown fields are rejected (`422`), in the segment and in `video`. + +**Which person.** The server follows the **most prominent person** in the +video — the largest, most visible one over the stretch it reads. There is +no field to pick another in 1.2; trim or crop the video so the person you +want is the clear subject. (Choosing among several people may come in a +later version; see the [roadmap](../protocol/roadmap).) + +To build one (inline or by URL, the size math, trimming), +see **[Building reference segments →](../guides/reference-segments)**. + +The payload's own checks: + +- `video` has exactly one of `url` / `data`; a `url` is `https` with a host + and has no `media_type`; `data` is valid, non-empty base64 with one of + the three `media_type`s; `0 ≤ start_s < end_s`; `fps > 0` + → otherwise `422 schema_validation`. +- Inline `data` whose decoded size is over `limits.max_video_bytes` (when + set) → `413 payload_too_large`, `details.segment` naming it. The whole + body must still fit `limits.max_request_bytes`; base64 is a third bigger + than the file. +- `end_s − (start_s or 0)` over `limits.max_video_seconds` (when set) → + `400 invalid_options`. It caps the video read, not the output (that is + `max_duration_seconds`, as for text). +- What the SDK cannot see, the server checks once it has the video: a + `url`'s size against `max_video_bytes`, an untrimmed video's length + against `max_video_seconds`, and whether the video can be fetched and + decoded and has a person in it (`400 invalid_request`). + +Everything else is the `text` segment's rule, unchanged. The response is +the usual glTF, exactly as for a text request. The SDK never decodes video: +a backbone receives the segment parsed (`VideoReferenceSegment`, with +`video.decoded()` for inline data) and reads the video itself. + +`VideoReferenceSegment` is an **optional** capability: a model lists +`"video_reference"` in `supported_segments` and SHOULD set +`limits.max_video_bytes` and `limits.max_video_seconds`. The SDK rejects the +segment for a model that doesn't list it (`unsupported_segment`). No server +implements it yet. + ## Composing ```json diff --git a/docs/docs/concepts/skeleton.mdx b/docs/docs/concepts/skeleton.mdx index 1fe01dd..f4b8eec 100644 --- a/docs/docs/concepts/skeleton.mdx +++ b/docs/docs/concepts/skeleton.mdx @@ -42,10 +42,37 @@ skeleton. | `joints[].display_name` | string \| null | Optional human-readable name. Echoed in errors; ignored elsewhere | | `joints[].parent` | string \| null | Parent joint name. Exactly one joint MUST have `parent: null` (the root) | | `joints[].rest_translation` | `[x, y, z]` | Local-space, relative to parent. Meters | -| `joints[].rest_rotation` | `[x, y, z, w]` | Local-space quaternion, relative to parent. glTF order | +| `joints[].rest_rotation` | `[x, y, z, w]` | Local-space quaternion, relative to parent. glTF order. Clients SHOULD send identity; see [Rest rotations](#rest-rotations) | | `coordinate_system` | `"right_handed_y_up"` | Required value in v1 | | `units` | `"meters"` | Required value in v1 | +## Rest rotations + +`rest_translation` and `rest_rotation` mean what they mean in glTF: the +joint's rest transform in its parent's frame. Every rotation in a request +(`pose_keyframe.joint_rotations`, `effector_target.rotations`, +`motion_reference.rotations`) is a joint's local rotation in that same +parent frame, so at rest it equals the joint's `rest_rotation`. + +*Clarified in 1.2:* + +- Clients SHOULD send `rest_rotation: [0, 0, 0, 1]` on every joint. Then + every parent frame at rest is aligned with the world axes: a + `rest_translation` is simply the joint's rest position minus its + parent's, a local rotation is the rotation the joint has turned through + since the rest pose (in world-aligned axes, relative to its parent's), + and `[0, 0, 0, 1]` means "as in the rest pose". A rig whose bones have + their own rest orientations is sent this way by folding those + orientations into the rotations (as the Blender example in the guide + does). +- A server that cannot honour a non-identity `rest_rotation` MUST reject + the request with `400 invalid_skeleton` rather than ignore it. (Kimodo + takes identity rest rotations only, and says so with that error.) + +See +**[Building reference segments →](../guides/reference-segments#the-data-a-motion_reference-carries)** +for sampling rotations this way from a rig. + ## Canonical vs custom Servers publish a **canonical skeleton** in `/capabilities`. It's what the diff --git a/docs/docs/guides/reference-segments.mdx b/docs/docs/guides/reference-segments.mdx new file mode 100644 index 0000000..cb9cda1 --- /dev/null +++ b/docs/docs/guides/reference-segments.mdx @@ -0,0 +1,500 @@ +--- +id: reference-segments +title: Building reference segments +description: How to build a motion_reference from a clip (numpy, Blender, BVH, FBX) and a video_reference from a video, step by step. +sidebar_position: 1 +--- + +# Building reference segments + +A [`motion_reference`](../concepts/segments#motionreferencesegment--type-motion_reference) +or [`video_reference`](../concepts/segments#videoreferencesegment--type-video_reference) +segment is a prompt given as a clip or a video instead of words. The +[Segments](../concepts/segments) page lists their fields; this page is +for the client author who has to fill them in: what each number means, +how to sample it from a rig, and what the server says when it is wrong. + +The conventions below are what the reference server (Kimodo, the first +server to implement `motion_reference`) reads, and what the Animatica +Blender add-on sends. + +## The data a `motion_reference` carries + +A clip is T frames of J joints. `joint_names` fixes the joint order once; +every frame then lists one rotation per joint in that order, and one root +position: + +```text +joint_names = [ "Hips", "Spine", "Head" ] J = 3 + j = 0 j = 1 j = 2 +rotations[0] = [ [x, y, z, w], [x, y, z, w], [x, y, z, w] ] frame 0 +rotations[1] = [ [x, y, z, w], [x, y, z, w], [x, y, z, w] ] frame 1 + ... ... +rotations[T-1] = [ ... ] frame T-1 + +root_positions = [ [x, y, z], [x, y, z], ..., [x, y, z] ] T entries +fps = 30 the clip's sample rate +``` + +| Quantity | Convention | +|---|---| +| Axes, units | Right-handed, **Y-up, metres**, as everywhere in MMCP (see [Coordinate systems](../concepts/coordinates)) | +| `rotations[t][j]` | Quaternion **`[x, y, z, w]`** (glTF order, `w` last) | +| What a rotation is | Joint `j`'s **local rotation relative to its parent**, the same convention as `pose_keyframe.joint_rotations`. With identity rest rotations (below) this is the rotation **from the rest pose**: `[0, 0, 0, 1]` means "as in the rest pose" | +| `root_positions[t]` | The **root joint's world position** at frame t (the joint whose `parent` is `null`), not an offset from frame 0 | +| Quaternion length | Taken as sent; the server normalises. Round to 4 decimals | + +**Rest rotations are identity.** Clients SHOULD send +`rest_rotation: [0, 0, 0, 1]` on every joint of the skeleton (see +[below](#the-skeleton) and [Rest rotations](../concepts/skeleton#rest-rotations)) +and fold each bone's own rest orientation into its rotations. Then every +parent frame at rest is aligned with the world axes, and a joint's local +rotation is simply *the rotation it has turned through since the rest +pose, expressed in world (MMCP) axes*, relative to its parent's. That is +what the Blender example below computes. A server that can't honour a +non-identity `rest_rotation` refuses it with `400 invalid_skeleton` +(Kimodo does), so identity is the form every server takes. + +**Absolute root positions.** Send where the root really is, frame by +frame, in the rig's world. You do not need to move the clip to the +origin or turn it to face +Z: the reference server canonicalises it +(the clip's first frame becomes heading 0 at x = z = 0), so the same motion +recorded anywhere in the scene gives the same prompt. The height is kept +as sent. + +## The skeleton + +The clip's joints are joints of a skeleton, and the server needs that +skeleton to read the clip. It is either the **request `skeleton`** (the +character you are generating for) or the segment's **own `skeleton`** +(the rig the clip was recorded on): + +| Case | Send | +|---|---| +| The clip was sampled from the character you are generating for | nothing extra: `joint_names` refer to the request `skeleton` | +| The clip comes from another rig (a mocap library, another character, a Mixamo download) | that rig as the segment's `skeleton`; `joint_names` refer to it, and the server retargets the clip from it | + +Either way the output is on the request skeleton. A segment `skeleton` +that isn't the model's canonical skeleton needs a model with +`supports_retargeting` (Kimodo has it). + +Each joint of a skeleton is: + +```json +{ "name": "Spine", "parent": "Hips", + "rest_translation": [0.0, 0.25, 0.0], "rest_rotation": [0.0, 0.0, 0.0, 1.0] } +``` + +- **`name`** — any unique string. Names are not a fixed vocabulary: the + reference server finds hips, spine, legs, arms and head from the + skeleton's structure and rest positions (with the names as hints, after + dropping a common prefix such as `mixamorig:`), so a Mixamo, Rigify or + BVH rig can be sent as it is. +- **`parent`** — the parent joint's name; exactly one joint has `null` + (the root). List every parent before its children. +- **`rest_translation`** — the joint's rest offset in its parent's frame, + in metres. With identity rest rotations (as you send them) the parent's + frame is world-aligned, so this is the joint's rest position minus its + parent's, along the world axes; for the root, its rest position in the + world. +- **`rest_rotation`** — `[0, 0, 0, 1]`. + +**Joint coverage.** + +- `joint_names` must include the skeleton's **root joint** + (`400 invalid_request` otherwise), and every name must be a joint of + the skeleton (`unknown_joint`). +- Joints of the skeleton that the clip leaves out stay at rest. +- Joints the model has no use for (fingers, twist bones, face) are + accepted and follow their parent, so you can send the whole rig. +- If the rig has a scene root above the pelvis (an `Armature` or + `Reference` joint), that is the root: `root_positions` is its position, + and it must be in `joint_names`. + +## Sampling a clip + +1. **Evaluate every frame** of the stretch you want, in order, at the + clip's own rate. Evaluate the rig (constraints, drivers, IK) rather + than reading raw keys, so the reference is what the viewer sees. +2. For each frame take **each joint's local rotation from rest** and the + **root's world position**, converted to Y-up metres. +3. Keep **quaternion sign continuity**: `q` and `−q` are the same + rotation, but a sign flip between frames looks like a jump to anything + that interpolates. For each joint, if `dot(q_prev, q) < 0`, send `−q`. +4. Set **`fps`** to the rate you sampled at. It need not be the request's + rate: the server resamples the clip to the model's own rate. +5. Set **`duration_frames`** to the length you want *out*, at the + request's fps. It is independent of the clip's length: a 1-second clip + can prompt a 5-second segment, and a 10-second clip a 2-second one. + +**Length.** `limits.max_reference_frames` (300 on Kimodo) caps the +number of frames you send, counted as sent, whatever the `fps`. A clip +within it is always accepted: the server resamples or trims it to its own +window. To fit a longer one, either trim the clip to the part that +matters or thin it: keep every k-th frame and divide `fps` by k. Two to +ten seconds is usually plenty to say "this kind of motion". + +**Size.** A frame is `4·J + 3` numbers. Rounded to 4 decimals, a number +is about 8 bytes of JSON, so a frame is about `8 × (4·J + 3)` bytes: + +| Rig | Per frame | 2 s at 30 fps (60 frames) | 10 s at 30 fps (300 frames) | +|---|---|---|---| +| 30 joints (a body rig) | ≈ 1.0 KB | ≈ 58 KB | ≈ 290 KB | +| 77 joints (with fingers) | ≈ 2.4 KB | ≈ 146 KB | ≈ 730 KB | + +The whole request must fit `limits.max_request_bytes` (1 MiB by +default), and that includes the request skeleton, any segment skeleton, +and every other segment. A long clip on a finger-rig is the case to watch: +drop to 3 decimals, thin the clip, or leave the fingers out of +`joint_names` (they stay at rest and the model doesn't use them). Send +compact JSON (no indentation) as well. + +## Example: numpy + +A 3-joint chain, built by hand. It runs as it is; to send it, POST +`request` with `motionmcp.client.generate(server_url, request)` to a +model that lists `"motion_reference"`. + +```python +import json + +import numpy as np + +# A 3-joint chain: Hips -> Spine -> Head, Y-up metres. +skeleton = {"joints": [ + {"name": "Hips", "parent": None, "rest_translation": [0.0, 0.95, 0.0], + "rest_rotation": [0.0, 0.0, 0.0, 1.0]}, + {"name": "Spine", "parent": "Hips", "rest_translation": [0.0, 0.25, 0.0], + "rest_rotation": [0.0, 0.0, 0.0, 1.0]}, + {"name": "Head", "parent": "Spine", "rest_translation": [0.0, 0.35, 0.0], + "rest_rotation": [0.0, 0.0, 0.0, 1.0]}, +]} +joint_names = [j["name"] for j in skeleton["joints"]] + + +def quat_about_y(angle): + """[x, y, z, w] for a rotation of `angle` radians about +Y.""" + return np.array([0.0, np.sin(angle / 2), 0.0, np.cos(angle / 2)]) + + +def quat_about_x(angle): + return np.array([np.sin(angle / 2), 0.0, 0.0, np.cos(angle / 2)]) + + +fps = 30.0 +T = 4 # 4 frames here; a real clip has ~2 s +t = np.arange(T) / fps + +rotations = np.empty((T, len(joint_names), 4)) +rotations[:, 0] = [quat_about_y(0.3 * np.sin(2 * np.pi * s)) for s in t] # hips sway +rotations[:, 1] = [quat_about_x(0.1) for _ in t] # spine bent forward +rotations[:, 2] = [quat_about_y(-0.2) for _ in t] # head turned +root_positions = np.stack([np.zeros(T), np.full(T, 0.95), 1.2 * t], axis=1) # walking +Z + +# Keep each joint's quaternions on one hemisphere: q and -q are the same +# rotation, but a sign flip between frames reads as a jump. +for i in range(1, T): + flip = np.sum(rotations[i] * rotations[i - 1], axis=-1) < 0 + rotations[i][flip] *= -1 + +segment = { + "type": "motion_reference", + "duration_frames": 90, # the output: 3 s at the request's fps + "joint_names": joint_names, + "rotations": np.round(rotations, 4).tolist(), + "root_positions": np.round(root_positions, 4).tolist(), + "fps": fps, +} +request = { + "protocol_version": "1.2", + "model": "kimodo-soma-rp", + "skeleton": skeleton, + "segments": [segment], +} +print(json.dumps(segment, indent=2)) +``` + +The segment it prints (4 frames, 3 joints; wrapped here): + +```json +{ + "type": "motion_reference", + "duration_frames": 90, + "joint_names": ["Hips", "Spine", "Head"], + "rotations": [ + [[0.0, 0.0, 0.0, 1.0], [0.05, 0.0, 0.0, 0.9988], [0.0, -0.0998, 0.0, 0.995]], + [[0.0, 0.0312, 0.0, 0.9995], [0.05, 0.0, 0.0, 0.9988], [0.0, -0.0998, 0.0, 0.995]], + [[0.0, 0.061, 0.0, 0.9981], [0.05, 0.0, 0.0, 0.9988], [0.0, -0.0998, 0.0, 0.995]], + [[0.0, 0.0881, 0.0, 0.9961], [0.05, 0.0, 0.0, 0.9988], [0.0, -0.0998, 0.0, 0.995]] + ], + "root_positions": [[0.0, 0.95, 0.0], [0.0, 0.95, 0.04], [0.0, 0.95, 0.08], [0.0, 0.95, 0.12]], + "fps": 30.0 +} +``` + +Here the clip is on the request skeleton, so the segment carries no +`skeleton` of its own. Real clips need at least 2 frames and usually +have 60 or more. + +## Example: Blender + +This samples the active armature's action into a `motion_reference` +with the clip's own `skeleton`, converting Blender's Z-up to MMCP's Y-up +the way the Animatica add-on does: + +- positions `(x, y, z)` → `(x, z, −y)`, after the object's world matrix + (so an armature scaled 0.01, as Mixamo imports are, comes out in + metres); +- rotations as a change of basis: the bone's rotation from rest is taken + from the evaluated `pose_bone.matrix`, moved into armature axes by the + bone's rest matrix, into world axes by the object's rotation, and into + MMCP axes by the Z-up → Y-up swap `S`. + +Run it in Blender's Python console or with +`blender file.blend --background --python sample_reference.py`; it writes +`reference.json` next to the `.blend`. + +```python +import json + +import bpy +from mathutils import Matrix + +# MMCP (Y-up) axes in Blender (Z-up) coordinates: MMCP x, y, z = Blender x, z, -y. +S = Matrix(((1.0, 0.0, 0.0), + (0.0, 0.0, -1.0), + (0.0, 1.0, 0.0))) + + +def to_mmcp_pos(v): + return [round(v.x, 4) + 0.0, round(v.z, 4) + 0.0, round(-v.y, 4) + 0.0] + + +def bones_parents_first(arm): + """The armature's bones, every parent before its children.""" + order, todo = [], [b for b in arm.data.bones if b.parent is None] + while todo: + b = todo.pop(0) + order.append(b) + todo.extend(b.children) + return order + + +def build_skeleton(arm): + """The armature's rest pose as an MMCP skeleton: each joint at its rest head, + offset from its parent's in world axes, with an identity rest rotation.""" + mw = arm.matrix_world + head = {b.name: mw @ b.head_local for b in arm.data.bones} + joints = [] + for b in bones_parents_first(arm): + offset = head[b.name] - head[b.parent.name] if b.parent else head[b.name] + joints.append({"name": b.name, + "parent": b.parent.name if b.parent else None, + "rest_translation": to_mmcp_pos(offset), + "rest_rotation": [0.0, 0.0, 0.0, 1.0]}) + return {"joints": joints} + + +def local_basis(pb): + """The bone's evaluated rotation relative to its rest, in its own rest frame + (pose_bone.matrix, so constraints and drivers count).""" + if pb.parent is None: + return pb.bone.matrix_local.inverted() @ pb.matrix + offset = pb.bone.matrix_local.inverted() @ pb.parent.bone.matrix_local + return offset @ pb.parent.matrix.inverted() @ pb.matrix + + +def mmcp_rotation(pb, mw_rot): + """[x, y, z, w]: the bone's rotation from rest, in MMCP world axes.""" + ml = pb.bone.matrix_local.to_3x3() + r = ml @ local_basis(pb).to_3x3() @ ml.transposed() # armature axes + r = mw_rot @ r @ mw_rot.transposed() # world axes + w, x, y, z = (S.transposed() @ r @ S).to_quaternion() # MMCP axes + return [x, y, z, w] + + +def sample_reference(arm, frame_start, frame_end, duration_frames): + scene = bpy.context.scene + names = [b.name for b in bones_parents_first(arm)] + root = arm.pose.bones[names[0]] # the one bone with no parent + mw_rot = arm.matrix_world.to_quaternion().to_matrix() + rotations, roots = [], [] + for f in range(frame_start, frame_end + 1): # every frame + scene.frame_set(f) + pose = [mmcp_rotation(arm.pose.bones[n], mw_rot) for n in names] + if rotations: # sign continuity + for q, prev in zip(pose, rotations[-1]): + if sum(a * b for a, b in zip(q, prev)) < 0.0: + q[:] = [-c for c in q] + rotations.append(pose) + roots.append(to_mmcp_pos((arm.matrix_world @ root.matrix).translation)) + return { + "type": "motion_reference", + "duration_frames": duration_frames, + "joint_names": names, + "rotations": [[[round(c, 4) + 0.0 for c in q] for q in pose] for pose in rotations], + "root_positions": roots, + "fps": scene.render.fps / scene.render.fps_base, + } + + +arm = bpy.context.object # the armature, with its action on +start, end = map(int, arm.animation_data.action.frame_range) +reference = sample_reference(arm, start, end, duration_frames=90) +reference["skeleton"] = build_skeleton(arm) # the clip's own rig (see "The skeleton") +with open(bpy.path.abspath("//reference.json"), "w") as f: + json.dump(reference, f) +``` + +Notes: + +- It sends every bone. That suits a deform rig with one root bone (a + Mixamo or BVH import). A control rig (Rigify and the like) has + unparented control bones and several roots: send only its deform bones + (`bone.use_deform`), each parented to its nearest deform ancestor, as + the add-on does. +- When you are generating *for* this same armature, you can drop + `reference["skeleton"]` and send `build_skeleton(arm)` as the request + `skeleton` instead. +- `scene.frame_set` evaluates the whole scene; for long clips on heavy + scenes, sample only the stretch you need. + +## Other sources + +- **BVH.** A BVH hierarchy is already a skeleton in this sense: + `OFFSET` → `rest_translation` (BVH rest rotations are identity; convert + centimetres to metres and, if the file is Z-up, swap axes as above); + the `ROOT` joint's `Xposition Yposition Zposition` channels → + `root_positions`. Turn each joint's Euler channels into a quaternion in + the order the channels are listed (`Zrotation Xrotation Yrotation` + means `R = Rz · Rx · Ry`, angles in degrees), e.g. with + `scipy.spatial.transform.Rotation.from_euler("ZXY", angles, degrees=True).as_quat()`, + which returns `[x, y, z, w]`. The frame rate is `1 / Frame Time`. + End Sites carry no rotation: leave them out. +- **FBX, glTF, USD.** Import into Blender (File → Import, or + `bpy.ops.import_scene.fbx(filepath=...)`), then sample it as in the + Blender example. Other DCCs work the same way: sample local rotations + from rest and the root's world position, and convert with the tables in + [Coordinate systems](../concepts/coordinates). + +## Building a `video_reference` + +A `video_reference` has no skeleton and no per-frame data: it is a video +file, and the server does the reading. No server implements it yet, so +this section covers what the protocol and the SDK check; what makes a +reference work *well* will depend on the model that implements it. + +### The `video` object + +[`motionmcp.client.video_source`](../sdk/client#a-video-as-the-prompt-12) +builds it, with the standard library alone: + +```python +from motionmcp.client import video_source + +video_source(url="https://cdn.example.com/dance.mp4") +# {"url": "https://cdn.example.com/dance.mp4"} + +video_source("dance.mov") +# {"data": "AAAAFGZ0eXBxdCAg...", "media_type": "video/quicktime"} +``` + +| | `{"url": ...}` | `{"data": ..., "media_type": ...}` | +|---|---|---| +| What goes on the wire | the URL | the whole file, base64 | +| Who fetches it | the server, over `https` | nobody: it is in the body | +| Size checked by | the server, when it fetches | the SDK server, before the backbone sees it | +| Good for | files already hosted, long videos | local files, private footage | + +A URL must be `https`, with a host and no `user:password@`, whitespace +or odd port, and reachable **from the public internet**: servers fetch +it with SSRF protections, so a URL on `localhost`, a private network or a +link-local address is refused (`400 invalid_request`) even if the server +could reach it. A signed URL with an expiry works if it outlives the job. Inline data needs a `media_type`: +`video/mp4` (`.mp4`, `.m4v`), `video/quicktime` (`.mov`) or `video/webm` +(`.webm`); `video_source` takes it from the extension. + +### Size + +**Use the URL form for anything over a few seconds.** Inline video is +for short clips: with the default `max_request_bytes` of 1 MiB, the whole +file has to be under about 750 KB, which is a few seconds of +phone-quality footage at most. A longer video, or one at a higher bitrate, +goes by `url`. + +Inline video is limited twice: + +- **`limits.max_video_bytes`** caps the **file** (the decoded bytes). +- **`limits.max_request_bytes`** caps the **whole body**, and base64 is + 4/3 the size of the file: a 3 MB file is 4 MB of `data`. + +So the largest file you can send inline is about +`min(max_video_bytes, 0.75 × max_request_bytes)` less the rest of the +request. Check both in `/capabilities` before encoding: + +```python +import os + +limits = model["limits"] +size = os.path.getsize("dance.mp4") +fits = (size <= limits.get("max_video_bytes", size) + and size * 4 / 3 + 50_000 < limits["max_request_bytes"]) +video = video_source("dance.mp4") if fits else video_source(url=upload("dance.mp4")) +``` + +(`upload` is yours: anything that gives back an `https` URL the server can +fetch.) Trimming with `start_s` / `end_s` does not make an inline file any +smaller; cut the file itself if it is too big. + +### Trimming, fps, and which person + +```json +{ "type": "video_reference", "duration_frames": 120, + "video": { "url": "https://cdn.example.com/dance.mp4" }, + "start_s": 2.0, "end_s": 6.0, "fps": 30 } +``` + +- **`start_s` / `end_s`** — the stretch of the video to read, in seconds + from its start (`0 ≤ start_s < end_s`). Either can be left out: from the + start, or to the end. When `end_s` is given, `end_s − (start_s or 0)` + must fit `limits.max_video_seconds`; an untrimmed video's length is + checked by the server once it has it. +- **Which person** — the server follows the **most prominent person**: + the largest, most visible one over the stretch it reads. There is no + field to choose another in 1.2. If the video has several people, trim + it (or crop the file) to a stretch where the one you want is the clear + subject. Choosing among several people may come in a later version. +- **`fps`** — a hint for the video's frame rate, for footage whose + container reports it wrongly (some screen recordings and re-encodes). + Normally leave it out. +- **`duration_frames`** — as for every segment, the length of the + *output*, at the request's fps. A 4-second stretch of video can prompt + a 2-second or a 10-second segment. + +### A good reference video + +Until a model says otherwise, the safe choice is footage a person-tracker +reads easily: + +- one person, or one who is clearly the most prominent, +- the whole body in frame for the whole stretch (feet included), +- a steady camera, and enough light, +- the stretch trimmed to the motion you want, without lead-in. + +## Troubleshooting + +| What you see | Why | Fix | +|---|---|---| +| `400 unsupported_segment` | The model doesn't list the segment type in `supported_segments` | Check `/capabilities` first (`model_supported_segments(model)`); fall back to a `text` segment | +| `422 schema_validation`, "root_positions must have one entry per frame" | `len(root_positions) != len(rotations)` | One root position per sampled frame | +| `422 schema_validation`, "rotations[i] must have one quaternion per joint_names entry" | A frame has more or fewer rotations than `joint_names` | Sample every joint on every frame, in `joint_names` order | +| `422 schema_validation` on `rotations` | A rotation isn't 4 numbers, T < 2, or `joint_names` has a duplicate | `[x, y, z, w]` per joint; at least 2 frames; unique names | +| `400 unknown_joint` | A `joint_names` entry isn't a joint of the skeleton it refers to (the segment's `skeleton` if it has one, else the request's) | Send the clip's rig as the segment `skeleton`, or rename to the request skeleton's joints | +| `400 invalid_request`, "must include the skeleton's root joint" | The root joint isn't in `joint_names` | Include it, with its world positions in `root_positions` | +| `400 retargeting_unsupported` | A segment `skeleton` other than the canonical one, on a model without retargeting | Sample the clip onto the canonical skeleton, or use a model with `supports_retargeting` | +| `400 invalid_options`, "max_reference_frames" | The clip has more frames than the model takes | Trim it, or thin it (every k-th frame, `fps / k`) | +| `413 payload_too_large` | The body is over the server's `max_request_bytes`, or an inline video is over `max_video_bytes` | Round to fewer decimals, thin the clip or drop fingers; for video, send a `url` or a shorter file | +| `400 invalid_options`, "max_video_seconds" | `end_s − (start_s or 0)` is longer than the model reads | Trim to a shorter stretch | +| `422 schema_validation` on `video` | Both or neither of `url` / `data`, a non-`https` URL, bad base64, a missing or unknown `media_type`, or `start_s ≥ end_s` | Build the object with `video_source`; check the trim | +| The motion looks mirrored, lying down, or twisted | Axes not converted, or rotations sent in the rig's own bone axes rather than from rest | Convert Z-up to Y-up for positions *and* rotations; send identity `rest_rotation` and rotations measured from rest | +| The output drifts or pops where the clip is smooth | Quaternion sign flips between frames | Apply the `dot(q_prev, q) < 0 → −q` rule per joint | diff --git a/docs/docs/protocol/roadmap.mdx b/docs/docs/protocol/roadmap.mdx index 7b7b05b..b7d13bd 100644 --- a/docs/docs/protocol/roadmap.mdx +++ b/docs/docs/protocol/roadmap.mdx @@ -36,8 +36,17 @@ Each lands additively — clients on `v1.0` keep working. - **Sparse-keyframe responses** — bezier handles instead of dense linear channels, for clip sizes that scale with motion complexity rather than duration. +- **Choosing a person in a video reference** — in 1.2 a + [`video_reference`](../concepts/segments#videoreferencesegment--type-video_reference) + always follows the most prominent person. A later version may let the + client choose among several, e.g. the server reports the people it found + and the client picks one by a box or a point. - **Round-trip motion editing** — sending an existing motion as input - alongside constraints for refinement. + alongside constraints for refinement. *1.2 takes the first step: + [`motion_reference`](../concepts/segments#motionreferencesegment--type-motion_reference) + sends a clip as the prompt and gets new motion of its kind back, and + [`video_reference`](../concepts/segments#videoreferencesegment--type-video_reference) + does the same from a video of a person.* - **Vendor extensions** — formal `extensions._*` blocks in the response, for backbones that want to surface metadata not yet in core. diff --git a/docs/docs/protocol/versioning.mdx b/docs/docs/protocol/versioning.mdx index 248edab..d48c93d 100644 --- a/docs/docs/protocol/versioning.mdx +++ b/docs/docs/protocol/versioning.mdx @@ -14,6 +14,28 @@ sidebar_position: 1 fields with defaults, new response extension fields — bump the minor (`"1.1"`). +## Changes by version + +- **1.2** — references as prompts, each composing exactly like a text + segment: the [`motion_reference`](../concepts/segments#motionreferencesegment--type-motion_reference) + segment (a clip as the prompt) with `limits.max_reference_frames`, and + the [`video_reference`](../concepts/segments#videoreferencesegment--type-video_reference) + segment (a video as the prompt, by `https` URL or inline base64) with + `limits.max_video_bytes` and `limits.max_video_seconds`; and the + `invalid_request` error code. Opt-in per type: servers advertise + `"motion_reference"` / `"video_reference"` in `supported_segments` + (`supports_motion_reference_mixed` is then always true for a motion + reference). A 1.0 client never sends either and sees no change; a + server that doesn't list a type keeps refusing it with + `unsupported_segment`. *Clarified* (1.0 meanings unchanged: + `rest_translation` / `rest_rotation` are parent-local, and + `fill_mode: "rest"` pins to `rest_rotation`): clients SHOULD send + identity `rest_rotation` on every joint, so parent frames are + world-aligned at rest and rotations read as "turned from rest"; a server + that cannot honour a non-identity `rest_rotation` MUST reject the + request with `400 invalid_skeleton` rather than ignore it. See + [Rest rotations](../concepts/skeleton#rest-rotations). + ## Comparison Servers compare versions **numerically**, not lexicographically. So: diff --git a/docs/docs/reference/capabilities.mdx b/docs/docs/reference/capabilities.mdx index 6c76645..596b328 100644 --- a/docs/docs/reference/capabilities.mdx +++ b/docs/docs/reference/capabilities.mdx @@ -14,7 +14,7 @@ request body. Response is a `Capabilities` object. ```json { - "protocol_version": "1.0", + "protocol_version": "1.2", "rotation_format": "quaternion_xyzw", "coordinate_system": "right_handed_y_up", "units": "meters", @@ -55,10 +55,11 @@ request body. Response is a `Capabilities` object. | `supports_async` | bool | If `true`, server may return `202 Accepted` for long generations (see [Async](./async)) | | `supports_segment_seed` | bool | If `true`, the model honours a per-segment `seed` | | `supports_loop` | bool | If `true`, the model samples `Options.loop` as a seamless cycle (one segment) | +| `supports_motion_reference_mixed` | bool | *1.2.* `true` exactly when the model lists `"motion_reference"` in `supported_segments` (a reference composes like a text segment), `false` otherwise | | `supports_trajectory` | bool | If `true`, every response sample carries its travel trajectory (`MMCP_motion.samples[].trajectory`, see [Response](./response#the-travel-trajectory)) | | `supports_batch` | bool | If `true`, `POST /generate` takes a batch body of up to `limits.max_batch_size` requests (see [Endpoints](./endpoints#batches)) | | `supported_constraints` | string[] | Constraint type names the model accepts | -| `supported_segments` | string[] | Segment types the model accepts. Defaults to `["text", "unconditioned"]`; backbones with a specialized text-to-pose model add `"pose"` | +| `supported_segments` | string[] | Segment types the model accepts. Defaults to `["text", "unconditioned"]`; backbones with a specialized text-to-pose model add `"pose"`, ones that take a motion as a prompt add `"motion_reference"` (1.2, see [Segments](../concepts/segments#motionreferencesegment--type-motion_reference)), and ones that take a video as a prompt add `"video_reference"` (1.2, see [Segments](../concepts/segments#videoreferencesegment--type-video_reference)). A client SHOULD check this before sending a reference; a type not listed is refused with `unsupported_segment` | | `supported_guidance_types` | string[] | Subset of `["nocfg", "regular", "separated"]` | | `predicted_contact_joints` | string[] | Joint names (from `canonical_skeleton`) for which the model emits `foot_contacts`. Empty means no contact prediction | | `native_clip_seconds` | number | Duration the model was trained on. Requests beyond this are stitched per `chunking` | @@ -75,8 +76,11 @@ request body. Response is a `Capabilities` object. | `max_num_samples` | int | Largest accepted `Options.num_samples` | | `max_constraints_per_request` | int | Sum across all constraints in the request | | `max_prompt_length` | int | Codepoints, per `TextSegment.prompt` | -| `max_request_bytes` | int | Body size limit. Exceeding returns `413 payload_too_large` | +| `max_request_bytes` | int | Body size limit. Exceeding returns `413 payload_too_large`. The SDK server enforces it before parsing: from `Content-Length`, and while reading a body sent without one | | `max_batch_size` | int | Most requests in one batch body | +| `max_reference_frames` | int, optional | *1.2.* Most frames a `motion_reference` segment's clip may carry, counted **as sent**: a server MUST accept a clip of at most this many frames at any `fps`, resampling or trimming it to its own window as needed (its `duration_frames` is capped like any segment's, by `max_duration_seconds`). Set by models that list `"motion_reference"`; absent otherwise | +| `max_video_bytes` | int, optional | *1.2.* Most bytes a `video_reference`'s video may be. The SDK enforces it on inline `video.data` (decoded size, `413 payload_too_large`); a server enforces it on a `video.url` when it fetches it. Set by models that list `"video_reference"`; absent otherwise | +| `max_video_seconds` | number, optional | *1.2.* Most seconds of video a `video_reference` may ask to read. The SDK enforces it on a trimmed video (`end_s − (start_s or 0)`, `400 invalid_options`); a server enforces it on an untrimmed one once it has it. It caps the video, not the output (that is `max_duration_seconds`). Set by models that list `"video_reference"`; absent otherwise | ## Using the canonical skeleton diff --git a/docs/docs/reference/errors.mdx b/docs/docs/reference/errors.mdx index 6c79045..daa4996 100644 --- a/docs/docs/reference/errors.mdx +++ b/docs/docs/reference/errors.mdx @@ -29,19 +29,20 @@ Error responses use a uniform shape: | Code | HTTP | Meaning | |---|---|---| -| `schema_validation` | 422 | Request fails JSON schema validation | +| `schema_validation` | 422 | Request fails JSON schema validation (e.g. a `video_reference` with both or neither of `video.url` / `video.data`, a non-`https` URL, bad base64, or `start_s ≥ end_s`). `details.errors` lists each failure (location, type, message; the offending input is not echoed) | | `unknown_model` | 400 | `model` is not in `/capabilities` | | `retargeting_unsupported` | 400 | Custom skeleton sent to a backbone with `supports_retargeting=false` | | `unsupported_constraint` | 400 | A constraint type is not in `model.supported_constraints` | -| `unsupported_segment` | 400 | A segment type (e.g. `"pose"`) is not in `model.supported_segments` | -| `unknown_joint` | 400 | A constraint references a joint name not in `skeleton.joints` | -| `invalid_skeleton` | 400 | Skeleton has no root, multiple roots, cycles, or out-of-order joints | +| `unsupported_segment` | 400 | A segment type (e.g. `"pose"`) is not in `model.supported_segments`. Checked before the segment's payload is read. A type the server doesn't know at all is `422 schema_validation` | +| `unknown_joint` | 400 | A constraint references a joint name not in `skeleton.joints` (for a `motion_reference`: not in its own `skeleton` when it carries one) | +| `invalid_skeleton` | 400 | Skeleton (the request's, or a `motion_reference`'s own) has no root, multiple roots, cycles, or out-of-order joints; or (1.2) has a non-identity `rest_rotation` the server can't honour | | `frame_out_of_range` | 400 | A constraint references a frame index outside the request timeline | -| `invalid_options` | 400 | An option value is out of range (e.g. `diffusion_steps: 9999`) or inconsistent (e.g. `guidance.weight` length mismatch) | +| `invalid_request` | 400 | *1.2.* Parts of the request that are valid alone but can't go together (e.g. a `motion_reference` clip that lacks its skeleton's root joint; a `video_reference` whose video can't be fetched or decoded, or has no person in it) | +| `invalid_options` | 400 | An option value is out of range (e.g. `diffusion_steps: 9999`) or inconsistent (e.g. `guidance.weight` length mismatch), or a reference is over its limit: a `motion_reference` clip over `max_reference_frames`, a `video_reference` reading more than `max_video_seconds` (`details.segment` names it) | | `version_unsupported` | 400 | Server does not implement the requested `protocol_version` | | `unauthorized` | 401 | Missing or invalid credentials | | `forbidden` | 403 | Authenticated but not permitted to call this model | -| `payload_too_large` | 413 | Request body exceeds `model.limits.max_request_bytes` | +| `payload_too_large` | 413 | Request body exceeds `model.limits.max_request_bytes` (the SDK server checks it before parsing), or (1.2) a `video_reference`'s video exceeds `max_video_bytes` (`details`: `max_video_bytes`, `video_bytes`, `segment`) | | `rate_limited` | 429 | Caller is rate-limited; check `Retry-After` | | `internal_error` | 500 | Backbone failure (non-retryable; clients SHOULD NOT retry) | | `resource_exhausted` | 503 | Transient capacity / GPU memory pressure. Retryable with backoff per `Retry-After` | @@ -52,6 +53,6 @@ Error responses use a uniform shape: | Code | Retry? | |---|---| -| `schema_validation`, `unknown_model`, `retargeting_unsupported`, `unsupported_constraint`, `unknown_joint`, `invalid_skeleton`, `frame_out_of_range`, `invalid_options`, `version_unsupported`, `unauthorized`, `forbidden`, `payload_too_large` | No — fix the request | +| `schema_validation`, `unknown_model`, `retargeting_unsupported`, `unsupported_constraint`, `unknown_joint`, `invalid_skeleton`, `frame_out_of_range`, `invalid_request`, `invalid_options`, `version_unsupported`, `unauthorized`, `forbidden`, `payload_too_large` | No — fix the request | | `rate_limited`, `resource_exhausted`, `model_unavailable`, `timeout` | Yes — honor `Retry-After`, exponential backoff | | `internal_error` | **No** — backbone failure, not transient | diff --git a/docs/docs/reference/generate-request.mdx b/docs/docs/reference/generate-request.mdx index 4bd74e6..808f984 100644 --- a/docs/docs/reference/generate-request.mdx +++ b/docs/docs/reference/generate-request.mdx @@ -16,7 +16,7 @@ The body of `POST /generate`. | `protocol_version` | string matching `^\d+\.\d+$` | yes | Wire version. A v1.x server MUST accept any value matching `^1\.\d+$` and MAY refuse other majors with `version_unsupported` | | `model` | string | yes | One of `Capabilities.models[].id` | | `skeleton` | [`Skeleton`](../concepts/skeleton) | yes | The character's skeleton; must be sent every time | -| `segments` | [`Segment[]`](../concepts/segments) | no, default `[]` | Discriminated union of `TextSegment` and `UnconditionedSegment`. At least one of `segments` or `constraints` must be non-empty | +| `segments` | [`Segment[]`](../concepts/segments) | no, default `[]` | Discriminated union of `TextSegment`, `UnconditionedSegment`, `PoseSegment` and (1.2) `MotionReferenceSegment` and `VideoReferenceSegment`. A pose stands alone; a motion or video reference composes exactly like a text segment. At least one of `segments` or `constraints` must be non-empty | | `constraints` | [`Constraint[]`](../concepts/constraints) | no, default `[]` | See constraint primitives | | `duration_frames` | int (> 0) | conditional | **Required** when `segments` is empty (constraints-only); **forbidden** when `segments` is non-empty | | `timing` | `{ "fps": number }` | no | If set, frames are interpreted in this fps and the response is resampled. See [Timing →](../concepts/timing) | @@ -125,3 +125,124 @@ request with a derived seed. ] } ``` + +### A clip as the prompt (1.2) + +Four new performances of the kind of motion in a 2-second reference, +3 seconds long, to a model that lists `"motion_reference"` in +`supported_segments` — the request a text prompt would make, with the clip +where the prompt was: + +```json +{ + "protocol_version": "1.2", + "model": "kimodo-soma-rp", + "skeleton": { "joints": [ /* … */ ] }, + "segments": [{ + "type": "motion_reference", + "duration_frames": 90, + "joint_names": ["Hips", "Spine1", "..."], + "rotations": [ /* 60 frames × one [x, y, z, w] per joint */ ], + "root_positions": [ /* 60 × [x, y, z] */ ], + "fps": 30 + }], + "options": { "num_samples": 4 } +} +``` + +A clip from another rig carries that rig as the segment's `skeleton` (same +schema as the request `skeleton`); its `joint_names` are then that rig's, +and the server retargets the clip from it: + +```json +{ "type": "motion_reference", "duration_frames": 120, + "skeleton": { "joints": [ { "name": "mixamorig:Hips", "parent": null, /* … */ }, /* … */ ] }, + "joint_names": ["mixamorig:Hips", "mixamorig:Spine", "..."], + "rotations": [ /* … */ ], "root_positions": [ /* … */ ], "fps": 30 } +``` + +### Two clips in one take (1.2) + +References are prompts like text segments, so they mix with them: here a +walk like one clip, a text prompt, then a sit like another clip (sampled on +a Mixamo rig), stitched with the usual transitions. + +```json +{ + "protocol_version": "1.2", + "model": "kimodo-soma-rp", + "skeleton": { "joints": [ /* the character */ ] }, + "segments": [ + { "type": "motion_reference", "duration_frames": 90, + "joint_names": ["Hips", "Spine1", "..."], + "rotations": [ /* walk: T × J × [x, y, z, w] */ ], "root_positions": [ /* T × 3 */ ], "fps": 30 }, + { "type": "text", "prompt": "a person stops and looks around", "duration_frames": 45 }, + { "type": "motion_reference", "duration_frames": 75, + "skeleton": { "joints": [ /* the sit clip's rig */ ] }, + "joint_names": ["mixamorig:Hips", "..."], + "rotations": [ /* sit */ ], "root_positions": [ /* … */ ], "fps": 30 } + ], + "options": { "num_samples": 2, "transition_frames": 5 } +} +``` + +See **[Segments →](../concepts/segments#motionreferencesegment--type-motion_reference)** +for the rules, and **[Building reference segments →](../guides/reference-segments)** for how to sample +`rotations` and `root_positions` from a rig. + +### A video as the prompt (1.2) + +The dance in seconds 2–6 of a video, as a 4-second take on the character, +to a model that lists `"video_reference"` in `supported_segments`. The +server fetches the URL and follows the most prominent dancer in it: + +```json +{ + "protocol_version": "1.2", + "model": "my-video-model", + "skeleton": { "joints": [ /* … */ ] }, + "segments": [{ + "type": "video_reference", + "duration_frames": 120, + "video": { "url": "https://cdn.example.com/dance.mp4" }, + "start_s": 2.0, + "end_s": 6.0 + }] +} +``` + +A video from disk goes inline as base64, with its container's media type +(`motionmcp.client.video_source("clip.mov")` builds this object): + +```json +{ "type": "video_reference", "duration_frames": 90, + "video": { "data": "AAAAFGZ0eXBxdCAg…", "media_type": "video/quicktime" } } +``` + +### Text, a clip and a video in one take (1.2) + +All three kinds of prompt mix in any order: a walk from words, a stop like a +reference clip, then a turn like the one in a video, stitched with the usual +transitions. Each segment can take its own `seed`. + +```json +{ + "protocol_version": "1.2", + "model": "my-video-model", + "skeleton": { "joints": [ /* the character */ ] }, + "segments": [ + { "type": "text", "prompt": "a person walks forward", "duration_frames": 90, "seed": 7 }, + { "type": "motion_reference", "duration_frames": 45, + "joint_names": ["Hips", "Spine1", "..."], + "rotations": [ /* stop: T × J × [x, y, z, w] */ ], "root_positions": [ /* T × 3 */ ], + "fps": 30 }, + { "type": "video_reference", "duration_frames": 60, + "video": { "url": "https://cdn.example.com/turn.mp4" }, "end_s": 2.5 } + ], + "options": { "num_samples": 2, "transition_frames": 5 } +} +``` + +See **[Segments →](../concepts/segments#videoreferencesegment--type-video_reference)** +for the rules, and **[Building reference segments →](../guides/reference-segments)** for how to build +each one. diff --git a/docs/docs/reference/response.mdx b/docs/docs/reference/response.mdx index 88c08b0..d4d3bf3 100644 --- a/docs/docs/reference/response.mdx +++ b/docs/docs/reference/response.mdx @@ -70,7 +70,8 @@ response only animates joints that were sent in the request's | `samples[].name` | string | Matches the corresponding `animations[].name` | | `samples[].num_frames` | int | Frame count | | `samples[].chunk_boundaries` | int[] | Frame indices at which the server stitched internal chunks (relevant when the model declares `chunking: "stitched"`). Empty array when the request fit in one chunk | -| `samples[].foot_contacts` | `{ joint_name: bool[] }` \| `{ "$accessor": int }` \| null | Per-frame contact state, keyed by joint names from the request's `skeleton.joints`. For long clips, MAY reference an `accessor` (uint8 booleans, length `num_frames × len(joints)`) via `{ "$accessor": }`; the joint-name → channel-index mapping then lives in `samples[].foot_contact_joints: string[]` || `samples[].trajectory` | object \| null | The travel trajectory: the path the motion moves along, without the body's own sway, surge, height or twist (see [below](#the-travel-trajectory)). Sent by servers with `supports_trajectory: true`; null when the body could not be read | +| `samples[].foot_contacts` | `{ joint_name: bool[] }` \| `{ "$accessor": int }` \| null | Per-frame contact state, keyed by joint names from the request's `skeleton.joints`. For long clips, MAY reference an `accessor` (uint8 booleans, length `num_frames × len(joints)`) via `{ "$accessor": }`; the joint-name → channel-index mapping then lives in `samples[].foot_contact_joints: string[]` | +| `samples[].trajectory` | object \| null | The travel trajectory: the path the motion moves along, without the body's own sway, surge, height or twist (see [below](#the-travel-trajectory)). Sent by servers with `supports_trajectory: true`; null when the body could not be read | Clients that don't need the metadata can ignore the extension entirely and still get a valid glTF. diff --git a/docs/docs/sdk/backbone.mdx b/docs/docs/sdk/backbone.mdx index a25666f..8f5d424 100644 --- a/docs/docs/sdk/backbone.mdx +++ b/docs/docs/sdk/backbone.mdx @@ -122,6 +122,124 @@ Anything that isn't a `ProtocolError` becomes `500 internal_error`. See **[Errors →](./errors)** for the full list. +### Motion references (1.2) + +A [`motion_reference`](../concepts/segments#motionreferencesegment--type-motion_reference) +segment is a prompt given as a motion. To take one, list it and cap the +clip's length: + +```python +spec = ModelSpec(..., supported_segments=["text", "unconditioned", "motion_reference"]) +spec.limits.max_reference_frames = 300 +``` + +The SDK validates a reference with the text segment's rules (it has a +`duration_frames` and a `seed`, mixes with other segments, goes with loop, +constraints and guidance as text does) plus its payload: joints on its own +`skeleton` (or the request skeleton when it has none), the root joint +among them, finite numbers, the clip no longer than +`max_reference_frames` frames as sent. It reaches you as parsed, not +resampled or retargeted. A clip within `max_reference_frames` MUST be +accepted at any `fps`: if resampling it to your model's rate gives more +frames than your window, resample it to fit (or trim it), don't refuse it. Treat it as a text segment whose caption embedding comes from +the clip instead of the text encoder, and run everything else — lengths, +seeds, guidance defaults, transitions, loop, batching — through your text +path unchanged: + +```python +for seg in req.segments: + if seg.type == "motion_reference": + # seg.rotations (T_ref × J × 4), seg.root_positions (T_ref × 3), + # seg.joint_names, seg.fps; seg.skeleton (Skeleton | None): the + # clip's own rig, to retarget from when set + caption = embed_clip(seg) # clip -> the text encoder's kind of vector + elif seg.type == "text": + caption = encode_text(seg.prompt) + ... # the same path for both +``` + +`/capabilities` advertises `supports_motion_reference_mixed: true` for such a +model and `false` for any other (a reference always mixes like text). If +your model can't take a non-identity `rest_rotation` (on the request +skeleton or a segment's), raise `400 invalid_skeleton` for it rather than +ignoring it. What clients send — rotations +from rest with identity `rest_rotation`, the root's world position, any +rig's own skeleton — is in **[Building reference segments →](../guides/reference-segments)**. + +### Video references (1.2) + +A [`video_reference`](../concepts/segments#videoreferencesegment--type-video_reference) +segment is a prompt given as a video of a person. To take one, list it and +cap the video: + +```python +spec = ModelSpec(..., supported_segments=["text", "unconditioned", "video_reference"]) +spec.limits.max_video_bytes = 50_000_000 # decoded size of the video file +spec.limits.max_video_seconds = 20.0 # seconds of video to read +``` + +The SDK validates it with the text segment's rules plus its payload: one +source (an `https` `video.url`, or base64 `video.data` with a +`media_type`), `0 ≤ start_s < end_s`, the inline data within +`max_video_bytes` (`413 payload_too_large`) and a trimmed video within +`max_video_seconds` (`400 invalid_options`). It never decodes the video; +the rest is yours, once you have it: fetch the URL, check an untrimmed +video's length, find the most prominent person (the largest, most visible +one) to follow, and raise `invalid_request` when you can't. + +**Fetching a URL safely.** A `video.url` comes from the client, so +fetching it makes your server an SSRF target (its loopback, its cloud +metadata endpoint, its private network). A backbone that fetches one MUST: + +- fetch `https` only, with the URL rule the schema applies (a host, no + `user:pw@`, no whitespace or control characters, a valid port); +- resolve the host and refuse it if **any** address is loopback, private, + link-local (which covers `169.254.169.254`), carrier-grade NAT, + multicast, reserved or unspecified, then connect to the address it + checked (so a second DNS answer can't swap it); +- follow redirects by hand, a few at most, checking every hop the same way; +- stream the body and stop past `limits.max_video_bytes` + (`413 payload_too_large`), and bound the whole fetch with a timeout. + +`motionmcp.fetch_video_url(url, max_bytes=None, timeout=30.0)` does all of +this with the standard library, raising `ProtocolError`s you can let +propagate (`invalid_request`, `payload_too_large`, `timeout`). IPv6 +addresses that reach an IPv4 one (mapped, NAT64, 6to4, Teredo) are judged +by that address, and `timeout` is one hard deadline over connect, TLS, +headers and body. `max_bytes=None` (your model advertises no +`max_video_bytes`) means `motionmcp.fetch.DEFAULT_MAX_VIDEO_BYTES`, +100 MiB. It is synchronous: from an `async def generate`, run it with +`asyncio.to_thread` so it doesn't block the event loop. + +```python +import asyncio + +from motionmcp import ProtocolError, fetch_video_url + +async def generate(self, req): + cap = self.capabilities().limits.max_video_bytes # None -> the 100 MiB default + for seg in req.segments: + if seg.type == "video_reference": + raw = (seg.video.decoded() if seg.video.data else + await asyncio.to_thread(fetch_video_url, seg.video.url, max_bytes=cap)) + # seg.video.media_type (inline), seg.start_s / seg.end_s (seconds + # into the video, None = its start / end), seg.fps (a hint, or None) + track = track_most_prominent_person(raw, seg.start_s, seg.end_s) + if track is None: + raise ProtocolError("invalid_request", "no person to follow in the video", + details={"segment": req.segments.index(seg)}) + caption = embed_track(track) # video -> the text encoder's kind of vector + ... # then the text path, as for a motion reference +``` + +`req.video_references` lists them in order (`req.video_reference` is the +first). `seg.video.num_bytes` is the inline file's size, worked out from +the base64's length (the SDK checks the alphabet but doesn't decode it); +`seg.video.decoded()` decodes it. Keep `limits.max_request_bytes` above +`max_video_bytes` × 4/3 if you take inline video: the body carries it as +base64, and the SDK server refuses a body over `max_request_bytes` before +parsing it. + ## `generate_batch(requests)` Optional. The SDK serves batch bodies for every backbone (and advertises diff --git a/docs/docs/sdk/client.mdx b/docs/docs/sdk/client.mdx index cfc8c38..1eb0000 100644 --- a/docs/docs/sdk/client.mdx +++ b/docs/docs/sdk/client.mdx @@ -168,6 +168,47 @@ doc = glb_to_gltf(open("motion.glb", "rb").read()) See **[Response →](../reference/response)** for the document itself. +### A video as the prompt (1.2) + +A [`video_reference`](../concepts/segments#videoreferencesegment--type-video_reference) +segment carries its video by `https` URL or inline as base64. +**`video_source(path=None, *, url=None, media_type=None)`** builds its +`video` object from either, with the standard library alone (it reads and +encodes the file; it never decodes the video): + +```python +from motionmcp.client import video_source, model_supported_segments + +model = client.pick_model(caps) +if "video_reference" in model_supported_segments(model): + video = video_source("dance.mov") # {"data": "", "media_type": "video/quicktime"} + # or: video_source(url="https://cdn.example.com/dance.mp4") -> {"url": ...} + request_body["segments"] = [ + {"type": "text", "prompt": "a person walks in", "duration_frames": 60}, + {"type": "video_reference", "duration_frames": 120, "video": video, + "start_s": 2.0, "end_s": 6.0}, + ] +``` + +The media type comes from the extension (`.mp4` / `.m4v`, `.mov`, `.webm`) +unless `media_type` is passed; anything else raises `ValueError`, as does +an empty file or a URL the server would refuse (not `https`, no host, +`user:pw@`, whitespace, a bad port). Pass `max_bytes=` (the model's +`limits.max_video_bytes`) to refuse a file over it before reading it. + +`data` must be **standard base64, padded, on one line**: the alphabet +`A–Z a–z 0–9 + /` with `=` padding, and no line breaks. `video_source` +and Python's `base64.b64encode` produce exactly that. The GNU `base64` +command wraps its output at 76 columns by default, which the server +refuses; use `base64 -w0 clip.mp4` (on macOS, `base64 -i clip.mp4` does +not wrap). URL-safe base64 (`-` and `_`) is refused too. Check `limits.max_video_bytes` before +sending a file inline: a server refuses one over it with +`413 payload_too_large`, and the whole body must still fit +`limits.max_request_bytes` (base64 is a third bigger than the file). + +Building a `motion_reference` from a rig, and more on video references, +is in **[Building reference segments →](../guides/reference-segments)**. + ## Errors Every transport or server-side failure raises `MmcpError` (a diff --git a/docs/docs/sdk/errors.mdx b/docs/docs/sdk/errors.mdx index b9b33a4..8a264d7 100644 --- a/docs/docs/sdk/errors.mdx +++ b/docs/docs/sdk/errors.mdx @@ -51,9 +51,12 @@ The server already raises these — you don't need to: | `version_unsupported` | `protocol_version` major doesn't match this server | | `retargeting_unsupported` | Custom skeleton sent to a `supports_retargeting=False` model | | `unsupported_constraint` | Constraint type isn't in `model.supported_constraints` | +| `unsupported_segment` | Segment type isn't in `model.supported_segments` | +| `invalid_request` | Parts of the request that are valid alone but can't go together (raised by backbones, e.g. a `motion_reference` clip without its skeleton's root joint, a `video_reference` video that can't be fetched or decoded or has no person in it) | | `unknown_joint` | Constraint references a joint name not in the skeleton | | `frame_out_of_range` | Constraint frame outside `[0, total_frames - 1]` | -| `invalid_options` | num_samples / prompt length / duration / constraint count exceeds limits | +| `invalid_options` | num_samples / prompt length / duration / constraint count / a `motion_reference` clip longer than `max_reference_frames` / a `video_reference` trimmed longer than `max_video_seconds` | +| `payload_too_large` | a `video_reference`'s inline video over `max_video_bytes` (raise it yourself for a fetched `video.url` over it) | ## What you raise diff --git a/docs/sidebars.ts b/docs/sidebars.ts index 7979317..4aeac49 100644 --- a/docs/sidebars.ts +++ b/docs/sidebars.ts @@ -49,6 +49,13 @@ const sidebars: SidebarsConfig = { "concepts/coordinates", ], }, + { + type: "category", + label: "Guides", + collapsed: false, + collapsible: false, + items: ["guides/reference-segments"], + }, { type: "category", label: "Reference", diff --git a/src/motionmcp/__init__.py b/src/motionmcp/__init__.py index 56c9e29..c7888ce 100644 --- a/src/motionmcp/__init__.py +++ b/src/motionmcp/__init__.py @@ -47,6 +47,7 @@ async def generate(self, req: GenerateRequest) -> MotionResult: "GenerateRequest": ".schemas", "Guidance": ".schemas", "Joint": ".schemas", + "MotionReferenceSegment": ".schemas", "Options": ".schemas", "PoseKeyframeConstraint": ".schemas", "PoseSegment": ".schemas", @@ -56,7 +57,10 @@ async def generate(self, req: GenerateRequest) -> MotionResult: "TextSegment": ".schemas", "Timing": ".schemas", "UnconditionedSegment": ".schemas", + "VideoReferenceSegment": ".schemas", + "VideoSource": ".schemas", "build_app": ".server", + "fetch_video_url": ".fetch", "serve": ".server", } @@ -69,6 +73,7 @@ async def generate(self, req: GenerateRequest) -> MotionResult: "Guidance", "Joint", "ModelSpec", + "MotionReferenceSegment", "MotionResult", "Options", "PROTOCOL_VERSION", @@ -84,7 +89,10 @@ async def generate(self, req: GenerateRequest) -> MotionResult: "TextSegment", "Timing", "UnconditionedSegment", + "VideoReferenceSegment", + "VideoSource", "build_app", + "fetch_video_url", "serve", ] diff --git a/src/motionmcp/_urls.py b/src/motionmcp/_urls.py new file mode 100644 index 0000000..e28a156 --- /dev/null +++ b/src/motionmcp/_urls.py @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The one rule for a ``video_reference`` URL, shared by the request schema, +the client's ``video_source`` and ``fetch_video_url``. Standard library only +(the client imports it without the [server] extras).""" + +from __future__ import annotations + +from urllib.parse import urlsplit + +MAX_URL_LENGTH = 4096 + + +def check_video_url(url: str) -> str: + """Return ``url`` if it is an acceptable video URL, else raise ValueError. + + ``https`` (any case) with a host; no user info (``user:pw@``), no + whitespace or control characters, a valid port if one is given, at most + :data:`MAX_URL_LENGTH` characters. Where the host points is not checked + here: that needs a DNS lookup, and is ``fetch_video_url``'s job. + """ + if not isinstance(url, str) or not url: + raise ValueError("video.url must be a non-empty string") + if len(url) > MAX_URL_LENGTH: + raise ValueError(f"video.url is longer than {MAX_URL_LENGTH} characters") + if any(c.isspace() or ord(c) < 0x20 or ord(c) == 0x7F for c in url): + raise ValueError("video.url must not contain whitespace or control characters") + parts = urlsplit(url) + if parts.scheme.lower() != "https": + raise ValueError("video.url must be an https URL") + if "@" in parts.netloc: + raise ValueError("video.url must not carry user info (user:password@)") + if not parts.hostname: + raise ValueError("video.url must have a host") + try: + port = parts.port + except ValueError: + raise ValueError("video.url has an invalid port") from None + if port is not None and not 0 < port < 65536: + raise ValueError("video.url has an invalid port") + return url diff --git a/src/motionmcp/backbone.py b/src/motionmcp/backbone.py index 9998c1c..e776485 100644 --- a/src/motionmcp/backbone.py +++ b/src/motionmcp/backbone.py @@ -17,7 +17,7 @@ import abc import inspect from dataclasses import dataclass, field -from typing import Awaitable, Sequence +from typing import Awaitable, Optional, Sequence import numpy as np from pydantic import BaseModel, ConfigDict, Field @@ -40,6 +40,18 @@ class Limits(BaseModel): max_prompt_length: int = int(DEFAULT_LIMITS["max_prompt_length"]) max_request_bytes: int = int(DEFAULT_LIMITS["max_request_bytes"]) max_batch_size: int = int(DEFAULT_LIMITS["max_batch_size"]) + # Most frames a ``motion_reference`` segment's clip may carry (the clip + # itself; its ``duration_frames`` is capped like any segment's, by + # ``max_duration_seconds``). Set it when ``supported_segments`` lists + # ``"motion_reference"``; None = no cap. + max_reference_frames: Optional[int] = Field(None, gt=0) + # A ``video_reference`` segment's video (1.2): most bytes of inline + # ``video.data`` (decoded), and most seconds of video to read when the + # request trims it (``end_s``). Set them when ``supported_segments`` lists + # ``"video_reference"``; the SDK enforces what it can see and the backbone + # the rest (a URL's size, an untrimmed video's length). None = no cap. + max_video_bytes: Optional[int] = Field(None, gt=0) + max_video_seconds: Optional[float] = Field(None, gt=0) class ModelSpec(BaseModel): @@ -65,6 +77,12 @@ class ModelSpec(BaseModel): # should only set ``loop`` when this is advertised; the SDK rejects it # otherwise. supports_loop: bool = False + # ``motion_reference`` segments are prompts like text segments (MMCP 1.2): + # several in one request, mixed with ``text`` / ``unconditioned`` ones. + # Implied by ``"motion_reference"`` in ``supported_segments`` -- the SDK + # server advertises it as true for such a model, whatever is set here -- + # and kept for clients that read it. + supports_motion_reference_mixed: bool = False # True when POST /generate accepts a batch body, ``{"requests": [...]}``, # of up to ``limits.max_batch_size`` generate requests. Advertised by the # SDK server for every model it serves (it can always run a batch item by @@ -79,8 +97,12 @@ class ModelSpec(BaseModel): supported_constraints: list[str] = Field(default_factory=list) # Wire-format segment types this model accepts. Defaults to "text" and # "unconditioned" (every conforming server supports those). Backbones - # with a specialized text-to-pose model add "pose"; the SDK rejects - # any segment whose type isn't listed here. + # with a specialized text-to-pose model add "pose", and ones that can + # take a motion as a prompt add "motion_reference" (with + # ``limits.max_reference_frames``), a video as a prompt + # "video_reference" (with ``limits.max_video_bytes`` / + # ``max_video_seconds``); the SDK rejects any segment whose type isn't + # listed here. supported_segments: list[str] = Field( default_factory=lambda: list(SUPPORTED_SEGMENTS) ) diff --git a/src/motionmcp/client/__init__.py b/src/motionmcp/client/__init__.py index b52b4db..75a1d07 100644 --- a/src/motionmcp/client/__init__.py +++ b/src/motionmcp/client/__init__.py @@ -8,6 +8,8 @@ same plain glTF JSON document the JSON path returns. :mod:`motionmcp.client.gltf_parser` turns that glTF 2.0 document into plain arrays; numpy is its only dependency. +:mod:`motionmcp.client.video` builds a ``video_reference`` segment's +``video`` from a file or a URL. """ from .glb import glb_to_gltf @@ -25,6 +27,7 @@ probe_server, retarget_state, ) +from .video import video_source __all__ = [ "MmcpError", @@ -41,5 +44,6 @@ "poll_job", "probe_server", "retarget_state", + "video_source", "xyzw_to_rotmat", ] diff --git a/src/motionmcp/client/video.py b/src/motionmcp/client/video.py new file mode 100644 index 0000000..f986189 --- /dev/null +++ b/src/motionmcp/client/video.py @@ -0,0 +1,65 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Build the ``video`` of a ``video_reference`` segment (MMCP 1.2). + +Standard library only, like the rest of the client: this reads a file and +base64-encodes it; it never decodes the video. +""" + +from __future__ import annotations + +import base64 +import os + +from .._urls import check_video_url + +__all__ = ["video_source"] + +# The containers a video_reference may carry inline, by file extension. +_MEDIA_TYPES = { + ".mp4": "video/mp4", + ".m4v": "video/mp4", + ".mov": "video/quicktime", + ".webm": "video/webm", +} + + +def video_source(path=None, *, url=None, media_type=None, max_bytes=None): + """Return a ``video_reference`` segment's ``video`` object. + + ``video_source(url="https://...")`` gives ``{"url": ...}`` for the server + to fetch. ``video_source("clip.mp4")`` reads the file and gives + ``{"data": , "media_type": ...}``, the media type taken from the + extension (``.mp4`` / ``.m4v``, ``.mov``, ``.webm``) unless + ``media_type`` is passed. The URL must pass the server's rule (https, + a host, no ``user:pw@``, no whitespace, a valid port). + + Pass the model's ``limits.max_video_bytes`` as ``max_bytes`` to refuse a + file over it before reading it (``ValueError``): the SDK server refuses + inline video over it (413 ``payload_too_large``), and the body must still + fit ``limits.max_request_bytes`` (base64 is a third bigger than the file). + The encoding is standard, padded base64 on one line, as the protocol + requires. + """ + if (path is None) == (url is None): + raise ValueError("pass exactly one of a file path or url=") + if url is not None: + if media_type is not None: + raise ValueError("media_type goes with a file, not a url") + return {"url": check_video_url(str(url))} + if media_type is None: + ext = os.path.splitext(str(path))[1].lower() + media_type = _MEDIA_TYPES.get(ext) + if media_type is None: + raise ValueError( + "cannot tell the media type of %r; pass media_type= (one of %s)" + % (str(path), sorted(set(_MEDIA_TYPES.values())))) + elif media_type not in _MEDIA_TYPES.values(): + raise ValueError("media_type must be one of %s" % sorted(set(_MEDIA_TYPES.values()))) + if max_bytes is not None and os.path.getsize(path) > max_bytes: + raise ValueError("%r is %d bytes; max_bytes is %d" + % (str(path), os.path.getsize(path), max_bytes)) + with open(path, "rb") as f: + data = f.read() + if not data: + raise ValueError("%r is empty" % str(path)) + return {"data": base64.b64encode(data).decode("ascii"), "media_type": media_type} diff --git a/src/motionmcp/errors.py b/src/motionmcp/errors.py index 4f847f8..d6c9cc4 100644 --- a/src/motionmcp/errors.py +++ b/src/motionmcp/errors.py @@ -26,6 +26,9 @@ "invalid_skeleton": 400, "frame_out_of_range": 400, "invalid_options": 400, + # 1.2: a request whose parts are individually valid but can't go together + # (e.g. a motion_reference clip without the root joint of its skeleton). + "invalid_request": 400, "version_unsupported": 400, "unauthorized": 401, "forbidden": 403, @@ -101,10 +104,11 @@ def unsupported_segment( ) -def unknown_joint(joint: str, skeleton_joints: list[str]) -> ProtocolError: +def unknown_joint(joint: str, skeleton_joints: list[str], + skeleton: str = "request skeleton") -> ProtocolError: return ProtocolError( "unknown_joint", - f"joint {joint!r} is not in the request skeleton", + f"joint {joint!r} is not in the {skeleton}", details={"skeleton_joints": skeleton_joints}, ) diff --git a/src/motionmcp/fetch.py b/src/motionmcp/fetch.py new file mode 100644 index 0000000..7c0d1f9 --- /dev/null +++ b/src/motionmcp/fetch.py @@ -0,0 +1,225 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Fetch a ``video_reference``'s ``video.url`` the safe way (MMCP 1.2). + +A server that fetches a URL a client sent is an SSRF target: the URL can name +the server's own loopback, its cloud metadata endpoint, or a host on its +private network, directly or through a redirect or a DNS name that resolves +there. :func:`fetch_video_url` is the SDK's answer, standard library only: + +- ``https`` only, with the URL rule the request schema applies; +- every host is resolved and **every** address it resolves to must be + public (no loopback, private, link-local -- which covers the + 169.254.169.254 metadata address --, carrier-grade NAT, multicast, + reserved or unspecified addresses); the connection then goes to the + address that was checked, so a second DNS answer cannot swap it; +- redirects are followed by hand, at most ``max_redirects``, and each hop + is checked the same way; +- the body is streamed and cut off past ``max_bytes`` (413 + ``payload_too_large``), and the whole fetch -- connect, TLS, headers, + body, every redirect -- has one hard deadline: a timer shuts the socket + when it passes, however slowly the other end drips. + +It is synchronous; an ``async`` backbone calls it with +``await asyncio.to_thread(fetch_video_url, url, max_bytes=...)``. + +Failures are :class:`~motionmcp.errors.ProtocolError`\\ s a backbone can let +propagate: ``invalid_request`` for a URL that can't or mustn't be fetched, +``payload_too_large`` for one over the cap, ``timeout`` for the deadline. +""" + +from __future__ import annotations + +import http.client +import ipaddress +import socket +import ssl +import threading +import time +from typing import Callable, Optional +from urllib.parse import urljoin, urlsplit + +from ._urls import check_video_url +from .errors import ProtocolError + +__all__ = ["DEFAULT_MAX_VIDEO_BYTES", "fetch_video_url", "is_public_address"] + +# The byte cap when the model advertises no ``limits.max_video_bytes``. +DEFAULT_MAX_VIDEO_BYTES = 100 * 1024 * 1024 + +_REDIRECTS = (301, 302, 303, 307, 308) +_CHUNK = 64 * 1024 + + +_NAT64 = ipaddress.ip_network("64:ff9b::/96") # RFC 6052 well-known prefix +_NAT64_LOCAL = ipaddress.ip_network("64:ff9b:1::/48") # RFC 8215 local-use +_IPV4_COMPATIBLE = ipaddress.ip_network("::/96") # deprecated ::a.b.c.d + + +def _embedded_ipv4(ip: ipaddress.IPv6Address): + """The IPv4 address an IPv6 address carries and would reach, if any: + IPv4-mapped, IPv4-compatible, NAT64 (64:ff9b::/96), 6to4, Teredo.""" + if ip.ipv4_mapped is not None: + return ip.ipv4_mapped + if ip in _NAT64 or ip in _IPV4_COMPATIBLE: + return ipaddress.IPv4Address(int(ip) & 0xFFFFFFFF) + if ip.sixtofour is not None: + return ip.sixtofour + if ip.teredo is not None: + return ip.teredo[1] # the client end + return None + + +def is_public_address(address: str) -> bool: + """True if ``address`` (IPv4 or IPv6) is a globally routable unicast + address a server may fetch from. An IPv6 address that embeds an IPv4 one + (mapped, compatible, NAT64, 6to4, Teredo) is judged by that IPv4 + address; NAT64 local-use (64:ff9b:1::/48) is never public.""" + ip = ipaddress.ip_address(address.split("%", 1)[0]) + if isinstance(ip, ipaddress.IPv6Address): + if ip in _NAT64_LOCAL: + return False + v4 = _embedded_ipv4(ip) + if v4 is not None: + return v4.is_global and not v4.is_multicast + return ip.is_global and not ip.is_multicast + + +def _resolve(host: str, port: int) -> list[str]: + try: + infos = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM) + except (socket.gaierror, UnicodeError) as exc: + raise ProtocolError("invalid_request", f"video.url host {host!r} does not resolve: {exc}", + details={"host": host}) from None + return list(dict.fromkeys(info[4][0] for info in infos)) + + +class _PinnedHTTPSConnection(http.client.HTTPSConnection): + """An HTTPS connection to one checked address, verifying the certificate + (and sending SNI) for the URL's host name.""" + + def __init__(self, host: str, address: str, port: int, timeout: float, + context: ssl.SSLContext): + super().__init__(host, port, timeout=timeout, context=context) + self._address = address + + def connect(self) -> None: + sock = socket.create_connection((self._address, self.port), self.timeout) + self._raw = sock # for the deadline to shut, mid-handshake too + self.sock = self._context.wrap_socket(sock, server_hostname=self.host) + + +def _default_connect(host: str, address: str, port: int, timeout: float) -> http.client.HTTPConnection: + return _PinnedHTTPSConnection(host, address, port, timeout, ssl.create_default_context()) + + +# Replaceable for tests: (host, checked address, port, timeout) -> connection. +_connect: Callable[[str, str, int, float], http.client.HTTPConnection] = _default_connect + + +def fetch_video_url(url: str, *, max_bytes: Optional[int] = None, timeout: float = 30.0, + max_redirects: int = 3, user_agent: str = "motionmcp-fetch", + resolve: Optional[Callable[[str, int], list[str]]] = None) -> bytes: + """Download ``url`` (a ``video_reference``'s ``video.url``) and return its + bytes, refusing anything a server must not fetch. See the module doc. + + ``max_bytes``: pass the model's ``limits.max_video_bytes``; ``None`` + means :data:`DEFAULT_MAX_VIDEO_BYTES` (100 MiB). ``timeout`` (seconds) + bounds the whole fetch, redirects included, as a hard deadline. + """ + if max_bytes is None: + max_bytes = DEFAULT_MAX_VIDEO_BYTES + if isinstance(max_bytes, bool) or not isinstance(max_bytes, int) or max_bytes <= 0: + raise ValueError(f"max_bytes must be a positive int, got {max_bytes!r}") + resolve = resolve or _resolve + deadline = time.monotonic() + timeout + for _hop in range(max_redirects + 1): + try: + check_video_url(url) + except ValueError as exc: + raise ProtocolError("invalid_request", str(exc), details={"url": url}) from None + parts = urlsplit(url) + host, port = parts.hostname, parts.port or 443 + addresses = resolve(host, port) + blocked = [a for a in addresses if not is_public_address(a)] + if not addresses or blocked: + raise ProtocolError( + "invalid_request", + f"video.url host {host!r} resolves to an address the server won't fetch", + details={"host": host}) + path = (parts.path or "/") + (f"?{parts.query}" if parts.query else "") + conn = _connect(host, addresses[0], port, _left(deadline)) + # The hard deadline: shut the socket when it passes, whatever phase + # the fetch is in (a per-read timeout alone lets a drip run forever). + timer = threading.Timer(max(0.0, deadline - time.monotonic()), _abort, args=(conn,)) + timer.daemon = True + timer.start() + try: + conn.request("GET", path, headers={"Host": parts.netloc, "User-Agent": user_agent, + "Accept": "video/*"}) + resp = conn.getresponse() + if resp.status in _REDIRECTS: + location = resp.getheader("Location") + if not location: + raise ProtocolError("invalid_request", "video.url redirect without Location") + url = urljoin(url, location) + continue + if resp.status != 200: + raise ProtocolError("invalid_request", + f"video.url answered HTTP {resp.status}", + details={"status": resp.status}) + declared = resp.getheader("Content-Length") + if declared and declared.isdigit() and int(declared) > max_bytes: + raise _too_large(int(declared), max_bytes) + return _read_capped(resp, conn, max_bytes, deadline) + except (OSError, ValueError, http.client.HTTPException) as exc: + if (isinstance(exc, socket.timeout) or getattr(conn, "mmcp_timed_out", False) + or time.monotonic() >= deadline): + raise ProtocolError("timeout", "fetching video.url timed out") from None + raise ProtocolError("invalid_request", f"could not fetch video.url: {exc}") from None + finally: + timer.cancel() + conn.close() + raise ProtocolError("invalid_request", f"video.url redirected more than {max_redirects} times") + + +def _abort(conn) -> None: + """Shut the connection's sockets (called by the deadline timer).""" + conn.mmcp_timed_out = True + for sock in (getattr(conn, "_raw", None), getattr(conn, "sock", None)): + if sock is not None: + try: + sock.shutdown(socket.SHUT_RDWR) + except OSError: + pass + + +def _left(deadline: float) -> float: + left = deadline - time.monotonic() + if left <= 0: + raise ProtocolError("timeout", "fetching video.url timed out") + return left + + +def _read_capped(resp, conn, max_bytes: int, deadline: float) -> bytes: + chunks, total = [], 0 + while True: + if conn.sock is not None: + conn.sock.settimeout(_left(deadline)) + chunk = resp.read1(_CHUNK) # what has arrived, not a full chunk + if getattr(conn, "mmcp_timed_out", False): + raise ProtocolError("timeout", "fetching video.url timed out") + if not chunk: + return b"".join(chunks) + total += len(chunk) + if total > max_bytes: + raise _too_large(None, max_bytes) + chunks.append(chunk) + + +def _too_large(size: Optional[int], cap: int) -> ProtocolError: + return ProtocolError( + "payload_too_large", + f"video.url is {'more than ' + str(cap) if size is None else size} bytes; " + f"max_video_bytes is {cap}", + details={"max_video_bytes": cap, **({} if size is None else {"video_bytes": size})}, + ) diff --git a/src/motionmcp/protocol.py b/src/motionmcp/protocol.py index 415b196..b9e226a 100644 --- a/src/motionmcp/protocol.py +++ b/src/motionmcp/protocol.py @@ -4,7 +4,7 @@ from __future__ import annotations -PROTOCOL_VERSION = "1.0" +PROTOCOL_VERSION = "1.2" PROTOCOL_MAJOR = 1 ROTATION_FORMAT = "quaternion_xyzw" @@ -19,7 +19,10 @@ # optional extension — backbones that have a specialized text-to-pose model # include it in their ``ModelSpec.supported_segments``; backbones without # it leave this default and the SDK rejects ``pose`` segments before they -# reach the backbone. +# reach the backbone. ``"motion_reference"`` (1.2) is optional the same way: +# a backbone that can take a motion as a prompt lists it and sets +# ``limits.max_reference_frames``; so is ``"video_reference"`` (1.2), with +# ``limits.max_video_bytes`` / ``limits.max_video_seconds``. SUPPORTED_SEGMENTS = ["text", "unconditioned"] SUPPORTED_GUIDANCE_TYPES = ["nocfg", "regular", "separated"] diff --git a/src/motionmcp/schemas.py b/src/motionmcp/schemas.py index 5d7a7b3..fbfcae4 100644 --- a/src/motionmcp/schemas.py +++ b/src/motionmcp/schemas.py @@ -7,9 +7,14 @@ from __future__ import annotations +import base64 +import re from typing import Annotated, Literal, Optional, Union -from pydantic import BaseModel, ConfigDict, Field, model_validator +from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, model_validator +from pydantic_core import PydanticCustomError + +from ._urls import check_video_url # ---- Type aliases --------------------------------------------------------- @@ -18,6 +23,16 @@ Vec3 = tuple[float, float, float] Quaternion = tuple[float, float, float, float] # (x, y, z, w) +# Finite floats only (no NaN / Infinity, which Python's JSON parser accepts). +Finite = Annotated[float, Field(allow_inf_nan=False)] +FiniteVec3 = tuple[Finite, Finite, Finite] +FiniteQuaternion = tuple[Finite, Finite, Finite, Finite] + + +def _invalid_skeleton(message: str) -> PydanticCustomError: + """A topology error, reported as ``400 invalid_skeleton`` (not 422).""" + return PydanticCustomError("invalid_skeleton", message) + # ---- Skeleton ------------------------------------------------------------- @@ -40,22 +55,27 @@ class Skeleton(BaseModel): def _validate_topology(self) -> "Skeleton": names = [j.name for j in self.joints] if len(set(names)) != len(names): - raise ValueError("joint names must be unique within the skeleton") + raise _invalid_skeleton("joint names must be unique within the skeleton") roots = [j for j in self.joints if j.parent is None] if len(roots) != 1: - raise ValueError( + raise _invalid_skeleton( f"exactly one joint must have parent=null; got {len(roots)}" ) seen: set[str] = set() for j in self.joints: if j.parent is not None and j.parent not in seen: - raise ValueError( + raise _invalid_skeleton( f"joint {j.name!r} references parent {j.parent!r} that is " "not defined or appears later in the joints list" ) seen.add(j.name) return self + @property + def root(self) -> Joint: + """The joint with ``parent=None``.""" + return next(j for j in self.joints if j.parent is None) + # ---- Segments ------------------------------------------------------------- @@ -106,8 +126,204 @@ def duration_frames(self) -> int: return 1 +class MotionReferenceSegment(BaseModel): + """A prompt given as a motion instead of text (MMCP 1.2). + + The client sends a clip -- sampled from its own rig -- in place of a text + prompt. The model reads it as it would read a caption (a backbone turns the + clip into the same kind of conditioning vector its text encoder makes), so + the segment is a ``text`` segment in every other respect: it covers + ``duration_frames`` of the take, takes a per-segment ``seed``, mixes with + ``text`` / ``unconditioned`` / other reference segments in any order, + goes with constraints, ``options.loop``, ``num_samples`` and guidance + exactly as a text segment does, and the SDK validates it with the same + rules. What "a walk like this clip" produces is what the prompt "walk" + would: new performances of that kind of motion, not a copy of the clip. + + ``rotations`` are local-to-parent quaternions ``(x, y, z, w)`` per frame, + one per ``joint_names`` entry, in the same convention as + ``pose_keyframe.joint_rotations``; ``root_positions`` are the root joint's + world position per frame (Y-up metres), as ``pose_keyframe.root_position``. + Quaternions are taken as sent, like ``pose_keyframe``: not required to be + exactly unit length; a backbone normalises them as it needs. Every number + must be finite. ``joint_names`` must include the skeleton's root joint + (checked by the SDK server: ``invalid_request``). ``fps`` is + the rate of the clip's samples, which may differ from the request's; the + clip's length is independent of ``duration_frames``. + + ``skeleton`` (optional, same shape as ``GenerateRequest.skeleton``) is the + reference clip's own rig: when present, ``joint_names`` / ``rotations`` / + ``root_positions`` refer to it and the server retargets the clip from it, + so the clip can come from any rig; when absent they refer to the request + skeleton. Either way the output is on the request skeleton. + + Servers advertise support by listing ``"motion_reference"`` in + ``supported_segments``; ``limits.max_reference_frames`` caps the clip's + own length, in frames as sent (the output length is capped like any + segment's, by ``max_duration_seconds``). + """ + model_config = ConfigDict(extra="forbid") + type: Literal["motion_reference"] + duration_frames: int = Field(..., gt=0) + skeleton: Optional[Skeleton] = None + joint_names: list[str] = Field(..., min_length=1) + rotations: list[list[FiniteQuaternion]] = Field(..., min_length=2) + root_positions: list[FiniteVec3] = Field(..., min_length=2) + fps: float = Field(..., gt=0, allow_inf_nan=False) + # See ``TextSegment.seed`` -- same per-segment override semantics. + seed: Optional[int] = None + + @model_validator(mode="after") + def _check_shapes(self) -> "MotionReferenceSegment": + if len(set(self.joint_names)) != len(self.joint_names): + raise ValueError("joint_names must be unique") + t = len(self.rotations) + if len(self.root_positions) != t: + raise ValueError( + f"root_positions must have one entry per frame ({t}), " + f"got {len(self.root_positions)}" + ) + j = len(self.joint_names) + for i, frame in enumerate(self.rotations): + if len(frame) != j: + raise ValueError( + f"rotations[{i}] must have one quaternion per joint_names " + f"entry ({j}), got {len(frame)}" + ) + return self + + @property + def reference_frames(self) -> int: + """Frames in the reference clip (T).""" + return len(self.rotations) + + +# Container formats a ``video_reference`` may carry inline (``video.data``). +VIDEO_MEDIA_TYPES = ("video/mp4", "video/quicktime", "video/webm") + +# Standard base64 (RFC 4648 section 4), padded, no line breaks. +_BASE64 = re.compile(r"[A-Za-z0-9+/]*={0,2}") + + +def base64_decoded_size(data: str) -> int: + """Bytes that standard, padded base64 ``data`` decodes to, worked out from + its length and padding (no decoded copy). ValueError if it isn't such + base64 (another alphabet, line breaks, a length not a multiple of 4).""" + if len(data) % 4 or not _BASE64.fullmatch(data): + raise ValueError("video.data must be standard base64 (A-Z a-z 0-9 + /, " + "padded with =, no line breaks)") + padding = (data[-2:] == "==") + (data[-1:] == "=") # no copy of the payload + return len(data) // 4 * 3 - padding + + +class VideoSource(BaseModel): + """Where a ``video_reference`` segment's video comes from (MMCP 1.2). + + Exactly one of: + + - ``{"url": "https://..."}`` -- the server fetches it. An ``https`` URL + with a host; the SDK cannot see how big it is, so a server that + advertises ``limits.max_video_bytes`` enforces that itself on the fetch. + - ``{"data": "", "media_type": "video/mp4"}`` -- the file inline, + standard base64 (RFC 4648, padded); ``media_type`` is one of + :data:`VIDEO_MEDIA_TYPES`. The SDK checks the base64 alphabet and + works out the decoded size (``num_bytes``) from its length, without + decoding; ``decoded()`` decodes it when a backbone wants the bytes. + """ + model_config = ConfigDict(extra="forbid") + url: Optional[str] = Field(None, min_length=1) + data: Optional[str] = Field(None, min_length=1) + media_type: Optional[Literal["video/mp4", "video/quicktime", "video/webm"]] = None + _num_bytes: Optional[int] = PrivateAttr(None) + + @model_validator(mode="after") + def _one_source(self) -> "VideoSource": + if (self.url is None) == (self.data is None): + raise ValueError("video needs exactly one of `url` or `data`") + if self.url is not None: + if self.media_type is not None: + raise ValueError("`media_type` goes with `data`, not `url`") + check_video_url(self.url) + return self + if self.media_type is None: + raise ValueError( + f"video.data needs a `media_type`, one of {list(VIDEO_MEDIA_TYPES)}") + size = base64_decoded_size(self.data) + if not size: + raise ValueError("video.data is empty") + self._num_bytes = size + return self + + @property + def num_bytes(self) -> Optional[int]: + """Decoded size of inline ``data`` in bytes; None for a ``url``.""" + return self._num_bytes + + def decoded(self) -> bytes: + """The inline video file's bytes (raises for a ``url`` source).""" + if self.data is None: + raise ValueError("a url video has no inline data; fetch video.url") + return base64.b64decode(self.data, validate=True) + + +class VideoReferenceSegment(BaseModel): + """A prompt given as a video instead of text (MMCP 1.2). + + The client sends a video of a person moving -- by ``https`` URL or inline + as base64 (see :class:`VideoSource`) -- in place of a text prompt. The + model reads the person's motion in it as it would read a caption and makes + new motion of that kind, on the request skeleton. Like + :class:`MotionReferenceSegment` the segment **is a text segment** in + every other respect: it covers ``duration_frames`` of the take, takes a + per-segment ``seed``, mixes with any other segment type in any order, and + goes with constraints, ``options.loop``, ``num_samples`` and guidance + exactly as a text segment does; the SDK validates it with the same rules. + + ``start_s`` / ``end_s`` (optional, seconds into the video, ``0 <= start_s + < end_s``) trim it to the stretch to read. The backbone follows the + most prominent person in the video (the largest, most visible one); + choosing among several people is not part of 1.2. ``fps`` (optional) is a hint for the video's frame rate, for containers + that report it badly; the video's length is independent of + ``duration_frames``. + + A backbone receives the segment parsed and does the fetching / decoding + itself; the SDK never decodes video. Servers advertise support by + listing ``"video_reference"`` in ``supported_segments``; + ``limits.max_video_bytes`` caps the inline data's decoded size and + ``limits.max_video_seconds`` the trimmed length when the request states + it (``end_s``). Anything the SDK cannot see -- a URL's size, an untrimmed + video's length -- the server checks when it has the video. + """ + model_config = ConfigDict(extra="forbid") + type: Literal["video_reference"] + duration_frames: int = Field(..., gt=0) + video: VideoSource + start_s: Optional[float] = Field(None, ge=0, allow_inf_nan=False) + end_s: Optional[float] = Field(None, gt=0, allow_inf_nan=False) + fps: Optional[float] = Field(None, gt=0, allow_inf_nan=False) + # See ``TextSegment.seed`` -- same per-segment override semantics. + seed: Optional[int] = None + + @model_validator(mode="after") + def _check_trim(self) -> "VideoReferenceSegment": + if self.start_s is not None and self.end_s is not None and not self.start_s < self.end_s: + raise ValueError( + f"start_s ({self.start_s}) must be before end_s ({self.end_s})") + return self + + @property + def trimmed_seconds(self) -> Optional[float]: + """Seconds of video to read when the request states it (``end_s`` + given): ``end_s - (start_s or 0)``; else None (the whole video, or + from ``start_s`` to its end -- a length only the server can know).""" + if self.end_s is None: + return None + return self.end_s - (self.start_s or 0.0) + + Segment = Annotated[ - Union[TextSegment, UnconditionedSegment, PoseSegment], + Union[TextSegment, UnconditionedSegment, PoseSegment, MotionReferenceSegment, + VideoReferenceSegment], Field(discriminator="type"), ] @@ -251,6 +467,32 @@ def total_frames(self) -> int: assert self.duration_frames is not None return self.duration_frames + @property + def motion_reference(self) -> Optional[MotionReferenceSegment]: + """The request's first ``motion_reference`` segment, or None.""" + for s in self.segments: + if isinstance(s, MotionReferenceSegment): + return s + return None + + @property + def motion_references(self) -> list[MotionReferenceSegment]: + """Every ``motion_reference`` segment of the request, in order.""" + return [s for s in self.segments if isinstance(s, MotionReferenceSegment)] + + @property + def video_reference(self) -> Optional[VideoReferenceSegment]: + """The request's first ``video_reference`` segment, or None.""" + for s in self.segments: + if isinstance(s, VideoReferenceSegment): + return s + return None + + @property + def video_references(self) -> list[VideoReferenceSegment]: + """Every ``video_reference`` segment of the request, in order.""" + return [s for s in self.segments if isinstance(s, VideoReferenceSegment)] + def fps(self, model_native_fps: float) -> float: """Effective fps for this request: ``timing.fps`` if set, else the model native.""" return self.timing.fps if self.timing is not None else model_native_fps diff --git a/src/motionmcp/server.py b/src/motionmcp/server.py index 76e220e..1b139dd 100644 --- a/src/motionmcp/server.py +++ b/src/motionmcp/server.py @@ -13,6 +13,9 @@ from __future__ import annotations import inspect +import json +import re +import typing from contextlib import asynccontextmanager from typing import Iterable, Mapping @@ -41,7 +44,7 @@ ROTATION_FORMAT, UNITS, ) -from .schemas import GenerateRequest +from .schemas import GenerateRequest, Segment # ---- Backbone registry ---------------------------------------------------- @@ -138,24 +141,25 @@ async def get_capabilities() -> dict: "response_formats": RESPONSE_FORMATS, # Every model served here takes a batch body at POST /generate: # the SDK runs one item by item when the backbone cannot batch. - "models": [b.capabilities().model_copy(update={"supports_batch": True, - "supports_trajectory": True}) - .model_dump(mode="json") - for b in registry.values()], + "models": [_spec_json(b.capabilities()) for b in registry.values()], } # ----- /generate --------------------------------------------------------- def _parse(payload) -> GenerateRequest: - """One generate request, schema- and version-checked.""" + """One generate request, schema- and version-checked. + + A protocol major this server doesn't speak is refused first; then + segment types the SDK knows but the named model doesn't take + (``unsupported_segment``), before their payload is parsed at all.""" + version = payload.get("protocol_version") if isinstance(payload, dict) else None + if isinstance(version, str) and _VERSION.fullmatch(version): + _check_version(version) + _gate_segment_types(payload, registry) try: req = GenerateRequest.model_validate(payload) except ValidationError as exc: - raise ProtocolError( - "schema_validation", - "request fails schema validation", - details={"errors": exc.errors(include_url=False)}, - ) from exc + raise _schema_error(exc) from exc _check_version(req.protocol_version) return req @@ -200,15 +204,26 @@ def _encode(req: GenerateRequest, spec: ModelSpec, result) -> dict: trajectories=trajectories, ) + # The largest body any model here takes: the cap before the body is read + # (which model a request names is only known after parsing it). + def _body_cap() -> int: + return max((b.capabilities().limits.max_request_bytes for b in registry.values()), + default=int(DEFAULT_LIMITS["max_request_bytes"])) + @app.post("/generate") async def post_generate(request: Request) -> Response: + cap = _body_cap() + body = await _read_body(request, cap) + size = len(body) try: - payload = await request.json() - except Exception as exc: + payload = json.loads(body) + except (ValueError, RecursionError) as exc: + # RecursionError: nesting deeper than the parser goes. raise ProtocolError( "schema_validation", - f"request body is not valid JSON: {exc}", - ) + f"request body is not valid JSON: {type(exc).__name__}: {exc}"[:500], + ) from None + del body # A batch: several generate requests in one body (see _generate_batch). if isinstance(payload, dict) and "requests" in payload: @@ -217,6 +232,8 @@ async def post_generate(request: Request) -> Response: req = _parse(payload) backbone, spec = _backbone_for(req) + if size > spec.limits.max_request_bytes: # this model's own cap + raise payload_too_large(size, spec.limits.max_request_bytes) result = await _call_generate(backbone, req) return JSONResponse( content=_encode(req, spec, result), @@ -262,6 +279,11 @@ async def _generate_batch(payload: dict) -> dict: raise ProtocolError("schema_validation", "each request must be an object") req = _parse({"protocol_version": version, **raw}) _, spec = _backbone_for(req) + # Each item against its own model's cap (the body as a whole + # was checked against the largest one). + item_bytes = len(json.dumps(raw, separators=(",", ":"))) + if item_bytes > spec.limits.max_request_bytes: + raise payload_too_large(item_bytes, spec.limits.max_request_bytes) except ProtocolError as exc: results[i] = exc.to_envelope() continue @@ -298,8 +320,102 @@ async def _http_handler(_: Request, exc: HTTPException) -> JSONResponse: return app +# ---- Reading and parsing --------------------------------------------------- + +async def _read_body(request: Request, cap: int) -> bytes: + """The request body, refused with 413 as soon as it is known to exceed + ``cap`` bytes: from ``Content-Length`` before reading, else while + streaming (a body without one, or one that lies).""" + declared = request.headers.get("content-length") + if declared is not None: + try: + n = int(declared) + except ValueError: + raise ProtocolError("schema_validation", "invalid Content-Length") from None + if n > cap: + raise payload_too_large(n, cap) + chunks: list[bytes] = [] + total = 0 + async for chunk in request.stream(): + total += len(chunk) + if total > cap: + raise payload_too_large(total, cap, streamed=True) + chunks.append(chunk) + return b"".join(chunks) + + +def payload_too_large(size: int, cap: int, streamed: bool = False) -> ProtocolError: + what = f"more than {cap}" if streamed else str(size) + return ProtocolError( + "payload_too_large", + f"request body is {what} bytes; max_request_bytes is {cap}", + details={"max_request_bytes": cap, **({} if streamed else {"request_bytes": size})}, + ) + + +def _schema_error(exc: ValidationError) -> ProtocolError: + """A pydantic failure as the MMCP envelope: ``400 invalid_skeleton`` when + a skeleton's topology is wrong (the request's, or a segment's own), else + ``422 schema_validation``. Serialised via JSON (a validator's ValueError + in ``ctx`` would not serialise) and without the offending input, which + can be megabytes (a bad ``video.data``).""" + errors = json.loads(exc.json(include_url=False, include_input=False)) + topology = [e for e in errors if e.get("type") == "invalid_skeleton"] + if topology: + return ProtocolError("invalid_skeleton", topology[0]["msg"], + details={"errors": topology}) + return ProtocolError("schema_validation", "request fails schema validation", + details={"errors": errors}) + + +_VERSION = re.compile(r"\d+\.\d+") +# Segment types the SDK's schema knows; any other type is a schema error. +_KNOWN_SEGMENTS = frozenset( + typing.get_args(model.model_fields["type"].annotation)[0] + for model in typing.get_args(typing.get_args(Segment)[0])) + + +def _gate_segment_types(payload, registry) -> None: + """Refuse a segment type the SDK knows but the named model doesn't list, + from the raw body, so an unsupported payload (a video, a clip) is never + parsed. A request this can't read (no such model, no segment list) and a + type the SDK doesn't know are left to the schema (422).""" + if not isinstance(payload, dict): + return + backbone = registry.get(payload.get("model")) if isinstance(payload.get("model"), str) else None + segments = payload.get("segments") + if backbone is None or not isinstance(segments, list): + return + supported = list(backbone.capabilities().supported_segments) + for s in segments: + t = s.get("type") if isinstance(s, dict) else None + if t in _KNOWN_SEGMENTS and t not in supported: + raise unsupported_segment(t, supported) + + # ---- Generic per-spec validation ------------------------------------------ +def _spec_json(spec: ModelSpec) -> dict: + """One model's /capabilities entry. Every model served here takes a batch + body and gets trajectories (the SDK provides both); a model that takes a + ``motion_reference`` takes it as a prompt, mixed like text + (``supports_motion_reference_mixed``); an optional limit left unset + (``max_reference_frames``, ``max_video_bytes``, ``max_video_seconds``) is + left out rather than sent as null.""" + update = {"supports_batch": True, "supports_trajectory": True, + "supports_motion_reference_mixed": "motion_reference" in spec.supported_segments} + out = spec.model_copy(update=update).model_dump(mode="json") + limits = out.get("limits", {}) + for key in _OPTIONAL_LIMITS: + if limits.get(key) is None: + limits.pop(key, None) + return out + + +# Limits a model advertises only when it sets them (1.2 reference segments). +_OPTIONAL_LIMITS = ("max_reference_frames", "max_video_bytes", "max_video_seconds") + + def _check_version(version: str) -> None: try: major = int(version.split(".", 1)[0]) @@ -392,6 +508,12 @@ def _validate_against_spec(req: GenerateRequest, spec) -> None: if s.type not in spec.supported_segments: raise unsupported_segment(s.type, list(spec.supported_segments)) + # A motion or video reference (1.2) is a prompt given as a motion or a + # video: the rules below are the text segment's; these check only the + # payload (a clip's joints and length, a video's size and trimmed length). + _validate_motion_reference(req, spec) + _validate_video_reference(req, spec) + # Looping: only where the backbone says it can, and over one segment. if req.options is not None and req.options.loop: if not getattr(spec, "supports_loop", False): @@ -467,6 +589,75 @@ def _validate_against_spec(req: GenerateRequest, spec) -> None: ) +def _validate_motion_reference(req: GenerateRequest, spec) -> None: + """A reference's payload: everything else about it is a text segment's + (length, seed, mixing, loop, constraints), checked with the text rules.""" + for i, ref in enumerate(req.segments): + if ref.type == "motion_reference": + _validate_one_reference(req, spec, ref, i) + + +def _validate_one_reference(req: GenerateRequest, spec, ref, index: int) -> None: + # The clip's joints are its own skeleton's when it carries one (a clip from + # another rig, retargeted by the server), else the request skeleton's. + ref_skeleton = ref.skeleton if ref.skeleton is not None else req.skeleton + if ref.skeleton is not None and not spec.supports_retargeting: + canonical_names = [j.name for j in spec.canonical_skeleton.joints] + if [j.name for j in ref.skeleton.joints] != canonical_names: + raise retargeting_unsupported() + skeleton_joint_names = {j.name for j in ref_skeleton.joints} + where = ("motion_reference skeleton" if ref.skeleton is not None + else "request skeleton") + for j in ref.joint_names: + if j not in skeleton_joint_names: + raise unknown_joint(j, sorted(skeleton_joint_names), where) + # The clip places the rig by its root: without it there is nothing to place. + root = ref_skeleton.root.name + if root not in ref.joint_names: + raise ProtocolError( + "invalid_request", + f"motion_reference must include the {where}'s root joint {root!r}", + details={"root_joint": root, "segment": index}, + ) + cap = spec.limits.max_reference_frames + if cap is not None and ref.reference_frames > cap: + raise ProtocolError( + "invalid_options", + f"motion_reference clip has {ref.reference_frames} frames; " + f"max_reference_frames is {cap}", + details={"max_reference_frames": cap, "reference_frames": ref.reference_frames, + "segment": index}, + ) + + +def _validate_video_reference(req: GenerateRequest, spec) -> None: + """A video reference's payload against the model's limits, as far as the + SDK can see it: inline data's decoded size, and the trimmed length when + the request states one. A URL's size, an untrimmed video's length, finding a + person to follow are the backbone's to check once it has the video.""" + max_bytes = spec.limits.max_video_bytes + max_seconds = spec.limits.max_video_seconds + for i, s in enumerate(req.segments): + if s.type != "video_reference": + continue + size = s.video.num_bytes + if max_bytes is not None and size is not None and size > max_bytes: + raise ProtocolError( + "payload_too_large", + f"video_reference video is {size} bytes; max_video_bytes is {max_bytes}", + details={"max_video_bytes": max_bytes, "video_bytes": size, "segment": i}, + ) + seconds = s.trimmed_seconds + if max_seconds is not None and seconds is not None and seconds > max_seconds: + raise ProtocolError( + "invalid_options", + f"video_reference reads {seconds:g}s of video; " + f"max_video_seconds is {max_seconds:g}", + details={"max_video_seconds": max_seconds, "video_seconds": seconds, + "segment": i}, + ) + + # ---- Convenience runner --------------------------------------------------- def serve( diff --git a/tests/capabilities_0_9_null.json b/tests/capabilities_0_9_null.json new file mode 100644 index 0000000..c1a8299 --- /dev/null +++ b/tests/capabilities_0_9_null.json @@ -0,0 +1,133 @@ +{ + "coordinate_system": "right_handed_y_up", + "models": [ + { + "canonical_skeleton": { + "coordinate_system": "right_handed_y_up", + "joints": [ + { + "display_name": null, + "name": "Hips", + "parent": null, + "rest_rotation": [ + 0.0, + 0.0, + 0.0, + 1.0 + ], + "rest_translation": [ + 0.0, + 0.95, + 0.0 + ] + }, + { + "display_name": null, + "name": "Spine", + "parent": "Hips", + "rest_rotation": [ + 0.0, + 0.0, + 0.0, + 1.0 + ], + "rest_translation": [ + 0.0, + 0.5, + 0.0 + ] + }, + { + "display_name": null, + "name": "Head", + "parent": "Spine", + "rest_rotation": [ + 0.0, + 0.0, + 0.0, + 1.0 + ], + "rest_translation": [ + 0.0, + 0.3, + 0.0 + ] + }, + { + "display_name": null, + "name": "LeftFoot", + "parent": "Hips", + "rest_rotation": [ + 0.0, + 0.0, + 0.0, + 1.0 + ], + "rest_translation": [ + 0.1, + -0.95, + 0.0 + ] + }, + { + "display_name": null, + "name": "RightFoot", + "parent": "Hips", + "rest_rotation": [ + 0.0, + 0.0, + 0.0, + 1.0 + ], + "rest_translation": [ + -0.1, + -0.95, + 0.0 + ] + } + ], + "units": "meters" + }, + "chunking": "none", + "fps": 30.0, + "id": "null", + "limits": { + "max_batch_size": 16, + "max_constraints_per_request": 64, + "max_duration_seconds": 30.0, + "max_num_samples": 16, + "max_prompt_length": 1000, + "max_request_bytes": 1048576 + }, + "native_clip_seconds": 10.0, + "predicted_contact_joints": [], + "recommended_max_duration_seconds": 12.0, + "supported_constraints": [ + "root_path", + "effector_target", + "pose_keyframe" + ], + "supported_guidance_types": [ + "nocfg", + "regular", + "separated" + ], + "supported_segments": [ + "text", + "unconditioned" + ], + "supports_async": false, + "supports_batch": true, + "supports_loop": false, + "supports_retargeting": false, + "supports_segment_seed": false, + "supports_trajectory": true + } + ], + "protocol_version": "1.0", + "response_formats": [ + "gltf_2.0_json" + ], + "rotation_format": "quaternion_xyzw", + "units": "meters" +} \ No newline at end of file diff --git a/tests/test_client_video.py b/tests/test_client_video.py new file mode 100644 index 0000000..806ca1a --- /dev/null +++ b/tests/test_client_video.py @@ -0,0 +1,94 @@ +# SPDX-License-Identifier: Apache-2.0 +"""motionmcp.client.video_source: a video_reference's ``video`` from a file or +a URL. Standard library only (runs in the client-only job).""" + +from __future__ import annotations + +import base64 + +import pytest + +from motionmcp.client import video_source + + +@pytest.mark.parametrize("name,media_type", [ + ("clip.mp4", "video/mp4"), ("clip.M4V", "video/mp4"), + ("clip.mov", "video/quicktime"), ("clip.webm", "video/webm"), +]) +def test_file_is_base64_with_its_media_type(tmp_path, name, media_type) -> None: + raw = b"\x00\x00\x00\x18ftypmp42" + bytes(range(256)) + path = tmp_path / name + path.write_bytes(raw) + got = video_source(path) + assert set(got) == {"data", "media_type"} + assert got["media_type"] == media_type + assert base64.b64decode(got["data"], validate=True) == raw + + +def test_media_type_can_be_given(tmp_path) -> None: + path = tmp_path / "capture.bin" + path.write_bytes(b"x" * 10) + assert video_source(path, media_type="video/webm")["media_type"] == "video/webm" + with pytest.raises(ValueError): + video_source(path) # unknown extension + with pytest.raises(ValueError): + video_source(path, media_type="video/avi") + + +def test_url() -> None: + assert video_source(url="https://cdn.example.com/a.mp4") == {"url": "https://cdn.example.com/a.mp4"} + assert video_source(url="HTTPS://cdn.example.com/a.mp4")["url"] == "HTTPS://cdn.example.com/a.mp4" + with pytest.raises(ValueError): + video_source(url="https://cdn.example.com/a.mp4", media_type="video/mp4") + + +@pytest.mark.parametrize("url", [ + "http://cdn.example.com/a.mp4", "https:///a.mp4", "https://user:pw@cdn.example.com/a.mp4", + "https://cdn.example.com/a b.mp4", "https://cdn.example.com/a.mp4\n", "https://cdn.example.com:0/a", + "https://cdn.example.com:70000/a", "cdn.example.com/a.mp4", +]) +def test_url_uses_the_server_rule(url) -> None: + with pytest.raises(ValueError): + video_source(url=url) + + +def test_max_bytes_is_checked_before_reading(tmp_path, monkeypatch) -> None: + path = tmp_path / "big.mp4" + path.write_bytes(b"x" * 101) + assert video_source(path, max_bytes=101)["media_type"] == "video/mp4" + import builtins + real_open = builtins.open + monkeypatch.setattr(builtins, "open", lambda *a, **k: (_ for _ in ()).throw(AssertionError("read"))) + with pytest.raises(ValueError): + video_source(path, max_bytes=100) + monkeypatch.setattr(builtins, "open", real_open) + + +def test_encoding_is_one_padded_line(tmp_path) -> None: + path = tmp_path / "long.mp4" + path.write_bytes(bytes(range(256)) * 40 + b"x") # 10241 bytes: padded, > 76 chars + data = video_source(path)["data"] + assert "\n" not in data and data.endswith("=") and len(data) % 4 == 0 + + +def test_exactly_one_source(tmp_path) -> None: + with pytest.raises(ValueError): + video_source() + with pytest.raises(ValueError): + video_source(tmp_path / "a.mp4", url="https://cdn.example.com/a.mp4") + + +def test_empty_file(tmp_path) -> None: + path = tmp_path / "empty.mp4" + path.write_bytes(b"") + with pytest.raises(ValueError): + video_source(path) + + +def test_matches_the_server_schema(tmp_path) -> None: + pytest.importorskip("pydantic", reason="needs the [server] extras") + from motionmcp import schemas + path = tmp_path / "clip.mov" + path.write_bytes(b"moov" * 100) + src = schemas.VideoSource.model_validate(video_source(path)) + assert src.num_bytes == 400 and src.decoded() == b"moov" * 100 diff --git a/tests/test_fetch.py b/tests/test_fetch.py new file mode 100644 index 0000000..cdb4b4c --- /dev/null +++ b/tests/test_fetch.py @@ -0,0 +1,253 @@ +# SPDX-License-Identifier: Apache-2.0 +"""motionmcp.fetch_video_url: fetching a video_reference's URL without SSRF. +Standard library only (runs in the client-only job). The network is faked: +``resolve`` and the connection factory are replaced.""" + +from __future__ import annotations + +import http.client +import json +import socket +import threading +import time + +import pytest + +from motionmcp import fetch as fetch_mod +from motionmcp.errors import ProtocolError +from motionmcp.fetch import fetch_video_url, is_public_address + +PUBLIC = "93.184.216.34" + + +class FakeResponse: + def __init__(self, status=200, body=b"", headers=None): + self.status, self._body, self._headers = status, body, headers or {} + + def getheader(self, name, default=None): + return self._headers.get(name, default) + + def read1(self, n): + chunk, self._body = self._body[:n], self._body[n:] + return chunk + + +class FakeConnection: + sock = None + + def __init__(self, routes, log, host, address, port, timeout): + self.routes, self.log = routes, log + self.host, self.address, self.port = host, address, port + + def request(self, method, path, headers): + self.log.append((self.host, self.address, path, headers["Host"])) + self._resp = self.routes[(self.host, path)] + + def getresponse(self): + return self._resp + + def close(self): + pass + + +@pytest.fixture +def net(monkeypatch): + routes, log = {}, [] + monkeypatch.setattr(fetch_mod, "_connect", + lambda *a: FakeConnection(routes, log, *a)) + return routes, log + + +def _dns(table): + return lambda host, port: table[host] + + +def test_fetches_a_public_url(net) -> None: + routes, log = net + routes[("cdn.example.com", "/a.mp4?sig=1")] = FakeResponse(body=b"x" * 1000) + got = fetch_video_url("https://cdn.example.com/a.mp4?sig=1", max_bytes=1000, + resolve=_dns({"cdn.example.com": [PUBLIC]})) + assert got == b"x" * 1000 + assert log == [("cdn.example.com", PUBLIC, "/a.mp4?sig=1", "cdn.example.com")], \ + "connected to the checked address, with the URL's host for TLS and Host" + + +@pytest.mark.parametrize("address", [ + "127.0.0.1", "10.0.0.5", "172.16.3.4", "192.168.1.1", "169.254.169.254", + "100.64.0.1", "0.0.0.0", "224.0.0.1", "::1", "fe80::1", "fc00::1", "::ffff:127.0.0.1", + # IPv6 forms that reach an IPv4 address: judged by that address + "64:ff9b::a9fe:a9fe", "64:ff9b::7f00:1", "64:ff9b::a00:1", # NAT64 -> metadata, loopback, 10/8 + "64:ff9b:1::1", "64:ff9b:1::5db8:d822", # NAT64 local-use: never + "::127.0.0.1", "::a9fe:a9fe", "::10.0.0.1", # IPv4-compatible + "2002:7f00:1::1", "2002:a9fe:a9fe::", # 6to4 + "2001:0:4136:e378:8000:63bf:80ff:fffe", # Teredo, client 127.0.0.1 +]) +def test_non_public_addresses_are_refused(net, address) -> None: + assert not is_public_address(address) + with pytest.raises(ProtocolError) as e: + fetch_video_url("https://evil.example.com/v.mp4", max_bytes=10, + resolve=_dns({"evil.example.com": [address]})) + assert e.value.code == "invalid_request" + assert net[1] == [], "never connected" + + +@pytest.mark.parametrize("address", [PUBLIC, "2606:2800:220:1:248:1893:25c8:1946", + "64:ff9b::5db8:d822", "::ffff:93.184.216.34", + "2002:5db8:d822::1"]) +def test_public_addresses_pass(address) -> None: + assert is_public_address(address) + + +def test_the_refusal_does_not_name_the_addresses(net) -> None: + with pytest.raises(ProtocolError) as e: + fetch_video_url("https://evil.example.com/v.mp4", max_bytes=10, + resolve=_dns({"evil.example.com": ["10.0.0.7"]})) + assert "10.0.0.7" not in json.dumps(e.value.to_envelope()) + + +def test_max_bytes_defaults_and_must_be_positive(net) -> None: + routes, _ = net + dns = _dns({"cdn.example.com": [PUBLIC]}) + routes[("cdn.example.com", "/v.mp4")] = FakeResponse(body=b"abc") + assert fetch_video_url("https://cdn.example.com/v.mp4", resolve=dns) == b"abc" + routes[("cdn.example.com", "/v.mp4")] = FakeResponse(body=b"abc") + assert fetch_video_url("https://cdn.example.com/v.mp4", max_bytes=None, resolve=dns) == b"abc" + assert fetch_mod.DEFAULT_MAX_VIDEO_BYTES == 100 * 1024 * 1024 + for bad in (0, -1, 1.5, "10", True): + with pytest.raises(ValueError): + fetch_video_url("https://cdn.example.com/v.mp4", max_bytes=bad, resolve=dns) + + +def test_one_private_answer_among_public_ones_is_refused(net) -> None: + with pytest.raises(ProtocolError): + fetch_video_url("https://mixed.example.com/v.mp4", max_bytes=10, + resolve=_dns({"mixed.example.com": [PUBLIC, "10.1.2.3"]})) + + +def test_every_redirect_is_checked(net) -> None: + routes, log = net + routes[("cdn.example.com", "/v.mp4")] = FakeResponse( + 302, headers={"Location": "https://internal.example.com/secret"}) + with pytest.raises(ProtocolError) as e: + fetch_video_url("https://cdn.example.com/v.mp4", max_bytes=10, + resolve=_dns({"cdn.example.com": [PUBLIC], + "internal.example.com": ["10.0.0.9"]})) + assert e.value.code == "invalid_request" and len(log) == 1 + + +def test_redirect_to_http_is_refused(net) -> None: + routes, _ = net + routes[("cdn.example.com", "/v.mp4")] = FakeResponse( + 301, headers={"Location": "http://cdn.example.com/v.mp4"}) + with pytest.raises(ProtocolError): + fetch_video_url("https://cdn.example.com/v.mp4", max_bytes=10, + resolve=_dns({"cdn.example.com": [PUBLIC]})) + + +def test_relative_redirects_and_the_hop_limit(net) -> None: + routes, log = net + routes[("cdn.example.com", "/a")] = FakeResponse(302, headers={"Location": "/b"}) + routes[("cdn.example.com", "/b")] = FakeResponse(302, headers={"Location": "/a"}) + with pytest.raises(ProtocolError) as e: + fetch_video_url("https://cdn.example.com/a", max_bytes=10, max_redirects=3, + resolve=_dns({"cdn.example.com": [PUBLIC]})) + assert "redirected more than 3" in e.value.message and len(log) == 4 + + +def test_byte_cap_while_streaming_and_by_content_length(net) -> None: + routes, _ = net + routes[("cdn.example.com", "/big")] = FakeResponse(body=b"x" * 200_000) + routes[("cdn.example.com", "/said")] = FakeResponse(body=b"", headers={"Content-Length": "999"}) + dns = _dns({"cdn.example.com": [PUBLIC]}) + for path in ("/big", "/said"): + with pytest.raises(ProtocolError) as e: + fetch_video_url("https://cdn.example.com" + path, max_bytes=100, resolve=dns) + assert e.value.code == "payload_too_large" + + +def test_http_errors(net) -> None: + routes, _ = net + routes[("cdn.example.com", "/gone")] = FakeResponse(404) + with pytest.raises(ProtocolError) as e: + fetch_video_url("https://cdn.example.com/gone", max_bytes=10, + resolve=_dns({"cdn.example.com": [PUBLIC]})) + assert e.value.code == "invalid_request" and e.value.details["status"] == 404 + + +@pytest.mark.parametrize("url", [ + "http://cdn.example.com/v.mp4", "https://user:pw@cdn.example.com/v.mp4", + "https://cdn.example.com:99999/v.mp4", "https://cdn.example.com/a b.mp4", + "file:///etc/passwd", "https:///v.mp4", +]) +def test_bad_urls_are_refused_before_any_lookup(url) -> None: + def resolve(host, port): + raise AssertionError("looked up") + with pytest.raises(ProtocolError) as e: + fetch_video_url(url, max_bytes=10, resolve=resolve) + assert e.value.code == "invalid_request" + + +def test_deadline() -> None: + with pytest.raises(ProtocolError) as e: + fetch_video_url("https://cdn.example.com/v.mp4", max_bytes=10, timeout=0, + resolve=_dns({"cdn.example.com": [PUBLIC]})) + assert e.value.code == "timeout" + + +# ---- the hard deadline, against a real (plain-TCP) server that drips ---------- + +class DripServer: + """Answers one connection by sending ``head`` then ``body`` one byte + every ``gap`` seconds.""" + + def __init__(self, head: bytes, body: bytes = b"", gap: float = 0.4, drip_head=True): + self.sock = socket.socket() + self.sock.bind(("127.0.0.1", 0)) + self.sock.listen(1) + self.port = self.sock.getsockname()[1] + self.head, self.body, self.gap, self.drip_head = head, body, gap, drip_head + self.thread = threading.Thread(target=self._serve, daemon=True) + self.thread.start() + + def _serve(self): + conn, _ = self.sock.accept() + try: + conn.recv(65536) + if not self.drip_head: + conn.sendall(self.head) + for b in (self.head if self.drip_head else b"") + self.body: + conn.sendall(bytes([b])) + time.sleep(self.gap) + except OSError: + pass + finally: + conn.close() + self.sock.close() + + +def _via(server: DripServer, monkeypatch): + monkeypatch.setattr(fetch_mod, "_connect", + lambda host, addr, port, timeout: http.client.HTTPConnection( + "127.0.0.1", server.port, timeout=timeout)) + + +@pytest.mark.parametrize("phase", ["headers", "body"]) +def test_a_dripping_server_is_cut_off_at_the_deadline(monkeypatch, phase) -> None: + head = b"HTTP/1.1 200 OK\r\nContent-Length: 1000\r\n\r\n" + server = (DripServer(head, gap=0.2) if phase == "headers" + else DripServer(head, b"x" * 1000, gap=0.2, drip_head=False)) + _via(server, monkeypatch) + start = time.monotonic() + with pytest.raises(ProtocolError) as e: + fetch_video_url("https://cdn.example.com/v.mp4", max_bytes=10_000, timeout=1.0, + resolve=_dns({"cdn.example.com": [PUBLIC]})) + elapsed = time.monotonic() - start + assert e.value.code == "timeout" + assert elapsed < 1.0 + 0.5, f"held {elapsed:.1f}s for a 1s timeout" + + +def test_a_prompt_server_is_not_cut_off(monkeypatch) -> None: + server = DripServer(b"HTTP/1.1 200 OK\r\nContent-Length: 3\r\n\r\nabc", gap=0, drip_head=False) + _via(server, monkeypatch) + assert fetch_video_url("https://cdn.example.com/v.mp4", max_bytes=10, timeout=2.0, + resolve=_dns({"cdn.example.com": [PUBLIC]})) == b"abc" diff --git a/tests/test_hardening.py b/tests/test_hardening.py new file mode 100644 index 0000000..2c34926 --- /dev/null +++ b/tests/test_hardening.py @@ -0,0 +1,318 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The 1.2 review fixes: body size before parsing, small error envelopes, +finite numbers, skeleton errors as invalid_skeleton, the root joint in a +motion_reference, the segment gate before payloads, and /capabilities +unchanged for a backbone that takes no references.""" + +from __future__ import annotations + +import base64 +import json +from pathlib import Path + +import pytest + +pytest.importorskip("fastapi", reason="needs the [server] extras") + +from fastapi.testclient import TestClient # noqa: E402 + +from motionmcp import ModelSpec, PROTOCOL_VERSION, VideoSource, build_app # noqa: E402 +from motionmcp.null_backbone import NullBackbone # noqa: E402 + + +class RefNull(NullBackbone): + def __init__(self, segments=("motion_reference", "video_reference"), + max_request_bytes=None, retargets=True, **kw): + super().__init__(**kw) + self.segments, self.max_request_bytes, self.retargets = segments, max_request_bytes, retargets + + def capabilities(self) -> ModelSpec: + spec = super().capabilities() + spec.supported_segments = ["text", "unconditioned", *self.segments] + spec.supports_retargeting = self.retargets + if self.max_request_bytes is not None: + spec.limits.max_request_bytes = self.max_request_bytes + return spec + + +def _client(**kw) -> TestClient: + return TestClient(build_app(RefNull(**kw))) + + +def _skel(c: TestClient) -> dict: + return c.get("/capabilities").json()["models"][0]["canonical_skeleton"] + + +def _ref(names, t=4, **kw) -> dict: + return {"type": "motion_reference", "duration_frames": 10, "joint_names": names, + "rotations": [[[0, 0, 0, 1]] * len(names)] * t, + "root_positions": [[0, 0.9, 0]] * t, "fps": 30, **kw} + + +def _body(skel, *segments) -> dict: + return {"protocol_version": "1.2", "model": "null", "skeleton": skel, "segments": list(segments)} + + +def _post_raw(c: TestClient, text: str): + return c.post("/generate", content=text.encode(), headers={"Content-Type": "application/json"}) + + +# ---- NaN / Infinity ------------------------------------------------------- + +@pytest.mark.parametrize("token", ["NaN", "Infinity", "-Infinity"]) +@pytest.mark.parametrize("where", ["fps", "rotation", "root"]) +def test_motion_reference_numbers_must_be_finite(token, where) -> None: + c = _client() + skel = _skel(c) + names = [j["name"] for j in skel["joints"]] + seg = _ref(names) + marker = 12345.678 + if where == "fps": + seg["fps"] = marker + elif where == "rotation": + seg["rotations"] = [[[0, 0, 0, marker]] + [[0, 0, 0, 1]] * (len(names) - 1)] * 4 + else: + seg["root_positions"] = [[0, marker, 0]] * 4 + text = json.dumps(_body(skel, seg)).replace(str(marker), token) + assert token in text + r = _post_raw(c, text) + assert r.status_code == 422, r.text + assert r.json()["error"]["code"] == "schema_validation" + + +@pytest.mark.parametrize("field", ["start_s", "end_s", "fps"]) +def test_video_reference_numbers_must_be_finite(field) -> None: + c = _client() + seg = {"type": "video_reference", "duration_frames": 10, + "video": {"url": "https://cdn.example.com/a.mp4"}, field: 12345.678} + text = json.dumps(_body(_skel(c), seg)).replace("12345.678", "NaN") + assert _post_raw(c, text).status_code == 422 + + +# ---- the error envelope stays small ------------------------------------------- + +def test_a_huge_bad_video_data_gives_a_small_error() -> None: + c = _client(max_request_bytes=8_000_000) + seg = {"type": "video_reference", "duration_frames": 10, + "video": {"data": "!" * 4_000_000, "media_type": "video/mp4"}} + r = c.post("/generate", json=_body(_skel(c), seg)) + assert r.status_code == 422 + assert len(r.content) < 4_000, "the offending input is not echoed back" + assert "!!!!" not in r.text + + +# ---- max_request_bytes before parsing ---------------------------------------- + +def test_body_over_max_request_bytes_is_413_from_content_length() -> None: + c = _client(max_request_bytes=10_000) + body = json.dumps(_body(_skel(c), {"type": "text", "prompt": "x" * 20_000, + "duration_frames": 10})) + r = _post_raw(c, body) + assert r.status_code == 413 + err = r.json()["error"] + assert err["code"] == "payload_too_large" + assert err["details"] == {"max_request_bytes": 10_000, "request_bytes": len(body)} + + +def test_a_streamed_body_is_cut_off_at_the_cap() -> None: + """No Content-Length (chunked): the server stops reading past the cap + instead of buffering the whole body. Driven over raw ASGI, since the test + client buffers a streamed body before the app sees it.""" + import asyncio + + app = build_app(RefNull(max_request_bytes=1_000_000)) + pulled, sent = [], [] + + async def receive(): + pulled.append(1) + if len(pulled) > 200: # 200 MB if it were all read + return {"type": "http.request", "body": b"", "more_body": False} + return {"type": "http.request", "body": b"x" * 1_000_000, "more_body": True} + + async def send(message): + sent.append(message) + + scope = {"type": "http", "asgi": {"version": "3.0"}, "http_version": "1.1", + "method": "POST", "scheme": "http", "path": "/generate", "raw_path": b"/generate", + "query_string": b"", "root_path": "", "server": ("test", 80), "client": ("c", 1), + "headers": [(b"content-type", b"application/json"), + (b"transfer-encoding", b"chunked")]} + asyncio.run(app(scope, receive, send)) + start = next(m for m in sent if m["type"] == "http.response.start") + body = b"".join(m.get("body", b"") for m in sent if m["type"] == "http.response.body") + assert start["status"] == 413 + assert json.loads(body)["error"]["code"] == "payload_too_large" + assert len(pulled) <= 3, f"read {len(pulled)} MB for a 1 MB cap" + + +def test_each_model_s_own_cap_applies() -> None: + small = RefNull(model_id="small", max_request_bytes=1_000) + big = RefNull(model_id="big", max_request_bytes=100_000) + c = TestClient(build_app([small, big])) + skel = _skel(c) + seg = {"type": "text", "prompt": "x" * 900, "duration_frames": 10} + body = {**_body(skel, seg)} + size = len(json.dumps({**body, "model": "big"})) + assert 1_000 < size < 100_000 + assert c.post("/generate", json={**body, "model": "big"}).status_code == 200 + r = c.post("/generate", json={**body, "model": "small"}) + assert r.status_code == 413 and r.json()["error"]["details"]["max_request_bytes"] == 1_000 + + +# ---- base64 size without decoding ------------------------------------------------ + +def test_num_bytes_is_arithmetic_not_a_decode(monkeypatch) -> None: + raw = bytes(range(256)) * 10 + b"ab" + data = base64.b64encode(raw).decode() + import motionmcp.schemas as schemas + monkeypatch.setattr(schemas.base64, "b64decode", + lambda *a, **k: (_ for _ in ()).throw(AssertionError("decoded"))) + src = VideoSource.model_validate({"data": data, "media_type": "video/mp4"}) + assert src.num_bytes == len(raw) + monkeypatch.undo() + assert src.decoded() == raw + + +@pytest.mark.parametrize("data", ["QUJD\nREVG", "QUJD REVG", "QUJ", "QUJD=REVG", "QU==QUJD", + "QUJD-_==", "QQ===", "QUJDRA"]) +def test_base64_must_be_standard_padded_one_line(data) -> None: + with pytest.raises(ValueError): + VideoSource.model_validate({"data": data, "media_type": "video/mp4"}) + + +# ---- skeletons: 400 invalid_skeleton --------------------------------------------- + +_TWO_ROOTS = {"joints": [ + {"name": "a", "parent": None, "rest_translation": [0, 0, 0], "rest_rotation": [0, 0, 0, 1]}, + {"name": "b", "parent": None, "rest_translation": [0, 0, 0], "rest_rotation": [0, 0, 0, 1]}]} +_LATE_PARENT = {"joints": [ + {"name": "b", "parent": "a", "rest_translation": [0, 0, 0], "rest_rotation": [0, 0, 0, 1]}, + {"name": "a", "parent": None, "rest_translation": [0, 0, 0], "rest_rotation": [0, 0, 0, 1]}]} + + +@pytest.mark.parametrize("bad", [_TWO_ROOTS, _LATE_PARENT]) +def test_request_skeleton_topology_is_invalid_skeleton(bad) -> None: + c = _client() + r = c.post("/generate", json=_body(bad, {"type": "text", "prompt": "walk", "duration_frames": 5})) + assert r.status_code == 400, r.text + assert r.json()["error"]["code"] == "invalid_skeleton" + + +def test_segment_skeleton_topology_is_invalid_skeleton() -> None: + c = _client() + r = c.post("/generate", json=_body(_skel(c), _ref(["a"], skeleton=_TWO_ROOTS))) + assert r.status_code == 400 + err = r.json()["error"] + assert err["code"] == "invalid_skeleton" + assert err["details"]["errors"][0]["loc"][:3] == ["segments", 0, "motion_reference"] + + +def test_a_malformed_joint_is_still_schema_validation() -> None: + c = _client() + skel = _skel(c) + skel["joints"][0]["rest_rotation"] = [0, 0, 1] + r = c.post("/generate", json=_body(skel, {"type": "text", "prompt": "walk", "duration_frames": 5})) + assert r.status_code == 422 + + +# ---- the root joint --------------------------------------------------------------- + +def test_a_motion_reference_needs_its_skeleton_s_root() -> None: + c = _client() + skel = _skel(c) + root = skel["joints"][0]["name"] + names = [j["name"] for j in skel["joints"] if j["parent"] is not None] + r = c.post("/generate", json=_body(skel, _ref(names))) + assert r.status_code == 400 + err = r.json()["error"] + assert err["code"] == "invalid_request" + assert err["details"] == {"root_joint": root, "segment": 0} + + +def test_the_root_is_the_segment_skeleton_s_when_it_has_one() -> None: + c = _client() + own = {"joints": [ + {"name": "pelvis", "parent": None, "rest_translation": [0, 1, 0], "rest_rotation": [0, 0, 0, 1]}, + {"name": "chest", "parent": "pelvis", "rest_translation": [0, .3, 0], "rest_rotation": [0, 0, 0, 1]}]} + skel = _skel(c) + assert c.post("/generate", json=_body(skel, _ref(["chest"], skeleton=own))).json()[ + "error"]["details"]["root_joint"] == "pelvis" + assert c.post("/generate", json=_body(skel, _ref(["pelvis", "chest"], skeleton=own))).status_code == 200 + + +# ---- the segment gate comes first ------------------------------------------------------ + +def test_unsupported_segment_before_its_payload_is_parsed() -> None: + c = _client(segments=()) + seg = {"type": "video_reference", "duration_frames": 10, + "video": {"data": "not base64 at all", "media_type": "video/mp4"}} + r = c.post("/generate", json=_body(_skel(c), seg)) + assert r.status_code == 400 + assert r.json()["error"]["code"] == "unsupported_segment" + + +def test_an_unknown_segment_type_is_still_a_schema_error() -> None: + c = _client() + r = c.post("/generate", json=_body(_skel(c), {"type": "audio", "duration_frames": 10})) + assert r.status_code == 422 and r.json()["error"]["code"] == "schema_validation" + + +def test_the_protocol_major_is_checked_before_the_segment_gate() -> None: + c = _client(segments=()) + body = {**_body(_skel(c), {"type": "video_reference", "duration_frames": 10, + "video": {"url": "https://cdn.example.com/a.mp4"}}), + "protocol_version": "2.0"} + r = c.post("/generate", json=body) + assert r.status_code == 400 and r.json()["error"]["code"] == "version_unsupported" + + +# ---- deeply nested JSON --------------------------------------------------------------- + +@pytest.mark.parametrize("wrap", [lambda deep: deep, + lambda deep: '{"protocol_version": "1.2", "requests": [' + deep + ']}']) +def test_deeply_nested_json_is_a_422_envelope(wrap) -> None: + c = _client() + r = _post_raw(c, wrap("[" * 50_000 + "]" * 50_000)) + assert r.status_code == 422, r.text[:200] + assert r.json()["error"]["code"] == "schema_validation" + + +# ---- batch items against their own model's cap ------------------------------------------ + +def test_batch_items_are_capped_by_their_own_model() -> None: + small = RefNull(model_id="small", max_request_bytes=1_000) + big = RefNull(model_id="big", max_request_bytes=100_000) + c = TestClient(build_app([small, big])) + item = {k: v for k, v in _body(_skel(c), {"type": "text", "prompt": "x" * 900, + "duration_frames": 10}).items() + if k != "protocol_version"} + r = c.post("/generate", json={"protocol_version": "1.2", "requests": [ + {**item, "model": "big"}, {**item, "model": "small"}]}) + res = r.json()["results"] + assert "gltf" in res[0] + assert res[1]["error"]["code"] == "payload_too_large" + assert res[1]["error"]["details"]["max_request_bytes"] == 1_000 + + +# ---- /capabilities for a backbone that takes no references --------------------------------- + +def test_capabilities_unchanged_for_an_old_backbone() -> None: + before = json.loads((Path(__file__).parent / "capabilities_0_9_null.json").read_text()) + now = TestClient(build_app(NullBackbone())).get("/capabilities").json() + assert now["protocol_version"] == PROTOCOL_VERSION == "1.2" + assert now["models"][0].pop("supports_motion_reference_mixed") is False + before.pop("protocol_version") + now.pop("protocol_version") + assert now == before + + +def test_mixed_flag_follows_supported_segments() -> None: + class Claims(NullBackbone): + def capabilities(self): + spec = super().capabilities() + spec.supports_motion_reference_mixed = True # without the segment type + return spec + m = TestClient(build_app(Claims())).get("/capabilities").json()["models"][0] + assert m["supports_motion_reference_mixed"] is False + m = _client().get("/capabilities").json()["models"][0] + assert m["supports_motion_reference_mixed"] is True diff --git a/tests/test_motion_reference.py b/tests/test_motion_reference.py new file mode 100644 index 0000000..9802c48 --- /dev/null +++ b/tests/test_motion_reference.py @@ -0,0 +1,365 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The motion_reference segment (MMCP 1.2): a prompt given as a motion. + +Its payload has its own checks (shapes, joints on its skeleton, +max_reference_frames); everything else is a text segment's, so every request +rule is tested here against the same request with a text segment in its place.""" + +from __future__ import annotations + +import json + +import pytest + +pytest.importorskip("fastapi", reason="needs the [server] extras") + +from fastapi.testclient import TestClient # noqa: E402 +from pydantic import ValidationError # noqa: E402 + +from motionmcp import GenerateRequest, ModelSpec, MotionReferenceSegment, build_app # noqa: E402 +from motionmcp.schemas import TextSegment # noqa: E402 +from motionmcp.null_backbone import NullBackbone # noqa: E402 + + +class ReferenceNull(NullBackbone): + """A NullBackbone that advertises motion_reference and keeps what it got.""" + + def __init__(self, max_reference_frames: int | None = 300, retargets: bool = False, + loops: bool = False, poses: bool = False, **kw): + super().__init__(**kw) + self.max_reference_frames = max_reference_frames + self.retargets = retargets + self.loops = loops + self.poses = poses + self.received: list[GenerateRequest] = [] + + def capabilities(self) -> ModelSpec: + spec = super().capabilities() + spec.supported_segments = ["text", "unconditioned", "motion_reference", + *(["pose"] if self.poses else [])] + spec.limits.max_reference_frames = self.max_reference_frames + spec.supports_retargeting = self.retargets + spec.supports_loop = self.loops + return spec + + async def generate(self, request: GenerateRequest): + self.received.append(request) + return await super().generate(request) + + +def _joints(client: TestClient) -> tuple[dict, list[str]]: + skel = client.get("/capabilities").json()["models"][0]["canonical_skeleton"] + return skel, [j["name"] for j in skel["joints"]] + + +def _segment(names: list[str], t: int = 10, **kw) -> dict: + return { + "type": "motion_reference", + "duration_frames": t, + "joint_names": names, + "rotations": [[[0.0, 0.0, 0.0, 1.0]] * len(names)] * t, + "root_positions": [[0.0, 0.9, 0.01 * i] for i in range(t)], + "fps": 30, + **kw, + } + + +def _request(skel: dict, *segments: dict, **kw) -> dict: + return {"protocol_version": "1.2", "model": "null", "skeleton": skel, + "segments": list(segments), **kw} + + +@pytest.fixture +def backbone() -> ReferenceNull: + return ReferenceNull() + + +@pytest.fixture +def client(backbone: ReferenceNull) -> TestClient: + return TestClient(build_app(backbone)) + + +# ---- schema ------------------------------------------------------------- + +def test_segment_parses() -> None: + seg = MotionReferenceSegment.model_validate(_segment(["a", "b"], t=5, duration_frames=12)) + assert seg.duration_frames == 12 and seg.reference_frames == 5 and seg.seed is None + assert MotionReferenceSegment.model_validate(_segment(["a"], t=3, seed=5)).seed == 5 + + +def test_duration_frames_is_required_as_for_text() -> None: + raw = _segment(["a"], t=4) + del raw["duration_frames"] + with pytest.raises(ValidationError): + MotionReferenceSegment.model_validate(raw) + with pytest.raises(ValidationError): + TextSegment.model_validate({"type": "text", "prompt": "walk"}) + + +def test_fidelity_is_gone() -> None: + with pytest.raises(ValidationError): + MotionReferenceSegment.model_validate(_segment(["a"], t=3, fidelity=0.0)) + + +def test_non_unit_quaternions_are_taken_as_sent() -> None: + raw = _segment(["a"], t=2) + raw["rotations"] = [[[0.0, 0.0, 0.0, 1.02]], [[0.0, 0.1, 0.0, 0.99]]] + seg = MotionReferenceSegment.model_validate(raw) + assert seg.rotations[0][0] == (0.0, 0.0, 0.0, 1.02) + + +@pytest.mark.parametrize("change", [ + {"rotations": [[[0, 0, 0, 1]]]}, # T = 1 + {"root_positions": [[0, 0, 0]] * 3}, # T mismatch + {"rotations": [[[0, 0, 0, 1]]] * 4}, # frame joint count + {"rotations": [[[0, 0, 0, 1], [0, 0, 1]]] * 4}, # not a quaternion + {"joint_names": []}, + {"joint_names": ["a", "a"]}, + {"fidelity": 0.0}, # removed in 1.2 + {"fps": float("nan")}, + {"fps": float("inf")}, + {"rotations": [[[0, 0, 0, float("nan")], [0, 0, 0, 1]]] * 4}, + {"root_positions": [[0, float("inf"), 0]] * 4}, + {"duration_frames": 0}, + {"fps": 0}, + {"prompt": "walk"}, # extra field +]) +def test_segment_rejects(change: dict) -> None: + raw = {**_segment(["a", "b"], t=4), **change} + with pytest.raises(ValidationError): + MotionReferenceSegment.model_validate(raw) + + +def test_json_round_trip() -> None: + raw = _request({"joints": [{"name": "a", "parent": None, "rest_translation": [0, 0, 0], + "rest_rotation": [0, 0, 0, 1]}]}, + _segment(["a"], t=3, duration_frames=7)) + req = GenerateRequest.model_validate(raw) + again = GenerateRequest.model_validate(json.loads(req.model_dump_json())) + assert again == req + seg = again.segments[0] + assert isinstance(seg, MotionReferenceSegment) and again.motion_reference is seg + assert again.total_frames == 7, "the segment's duration_frames, not the clip's length" + + +# ---- it reaches the backbone as sent ------------------------------------ + +def test_happy_path_reaches_backbone_untouched(client, backbone) -> None: + skel, names = _joints(client) + seg = _segment(names, t=15, duration_frames=40, seed=3) + r = client.post("/generate", json=_request(skel, seg, options={"num_samples": 3})) + assert r.status_code == 200, r.text + got = backbone.received[-1].motion_reference + assert got.model_dump(mode="json") == {**seg, "fps": 30.0, "skeleton": None} + ext = r.json()["extensions"]["MMCP_motion"] + assert [s["num_frames"] for s in ext["samples"]] == [40] * 3 + assert all("reference" not in s for s in ext["samples"]), "a response like a text one" + + +def test_capabilities_advertise_reference(client) -> None: + m = client.get("/capabilities").json()["models"][0] + assert "motion_reference" in m["supported_segments"] + assert m["limits"]["max_reference_frames"] == 300 + assert m["supports_motion_reference_mixed"] is True, "implied: a reference mixes like text" + + +def test_gate_rejects_without_the_capability() -> None: + c = TestClient(build_app(NullBackbone())) + skel, names = _joints(c) + caps = c.get("/capabilities").json()["models"][0] + assert "motion_reference" not in caps["supported_segments"] + assert "max_reference_frames" not in caps["limits"] + assert caps["supports_motion_reference_mixed"] is False + r = c.post("/generate", json=_request(skel, _segment(names))) + assert r.status_code == 400 + assert r.json()["error"]["code"] == "unsupported_segment" + + +# ---- the text segment's rules ------------------------------------------- + +def _as_text(seg: dict) -> dict: + """The text segment in a reference's place: same length, same seed.""" + return {"type": "text", "prompt": "a person walks", "duration_frames": seg["duration_frames"], + **({"seed": seg["seed"]} if "seed" in seg else {})} + + +def _outcome(c: TestClient, body: dict): + r = c.post("/generate", json=body) + if r.status_code == 200: + return 200, [s["num_frames"] for s in r.json()["extensions"]["MMCP_motion"]["samples"]] + return r.status_code, r.json()["error"]["code"] + + +_REF = "ref" + + +def _layouts(names): + """(segments with _REF placeholders, extra request fields) -- requests a + text segment makes or breaks in every way the SDK checks.""" + pk = lambda f: {"type": "pose_keyframe", "frame": f, "joint_rotations": {names[0]: [0, 0, 0, 1]}} # noqa: E731 + return [ + ([_REF], {}), + ([_REF, _REF], {}), + ([_REF, "text"], {}), + (["text", _REF, "text"], {}), + ([_REF, "unconditioned", _REF], {}), + ([_REF], {"options": {"num_samples": 4, "seed": 1}}), + ([_REF], {"options": {"loop": True}}), + ([_REF, _REF], {"options": {"loop": True}}), + ([_REF, "pose"], {}), + ([_REF], {"options": {"guidance": {"type": "separated", "weight": [2.0, 2.0]}}}), + ([_REF], {"options": {"transition_frames": 8}}), + ([_REF, _REF], {"constraints": [pk(39)]}), + ([_REF, _REF], {"constraints": [pk(40)]}), # frame_out_of_range + ([_REF], {"options": {"num_samples": 99}}), # 422 schema + ([_REF], {"options": {"num_samples": 16}}), # the most samples + ] + + +@pytest.mark.parametrize("loops", [False, True]) +@pytest.mark.parametrize("i", range(15)) +def test_a_reference_is_validated_like_a_text_segment(i, loops) -> None: + b = ReferenceNull(loops=loops, poses=True) + c = TestClient(build_app(b)) + skel, names = _joints(c) + segments, extra = _layouts(names)[i] + others = {"text": {"type": "text", "prompt": "a person sits", "duration_frames": 17}, + "unconditioned": {"type": "unconditioned", "duration_frames": 9}, + "pose": {"type": "pose", "prompt": "a person sits"}} + refs = [_segment(names, t=12, duration_frames=20, seed=10 + k) for k in range(len(segments))] + with_refs = [refs[k] if s == _REF else others[s] for k, s in enumerate(segments)] + with_text = [_as_text(refs[k]) if s == _REF else others[s] for k, s in enumerate(segments)] + got = _outcome(c, _request(skel, *with_refs, **extra)) + want = _outcome(c, _request(skel, *with_text, **extra)) + assert got == want + + +def test_mixed_segments_reach_the_backbone_in_order(backbone, client) -> None: + skel, names = _joints(client) + segs = [_segment(names, t=10, duration_frames=15, seed=100), + {"type": "text", "prompt": "a person sits", "duration_frames": 20, "seed": 7}, + _segment(names, t=30, duration_frames=25, seed=101)] + r = client.post("/generate", json=_request(skel, *segs)) + assert r.status_code == 200, r.text + got = backbone.received[-1] + assert [s.type for s in got.segments] == ["motion_reference", "text", "motion_reference"] + assert got.motion_references == [got.segments[0], got.segments[2]] + assert [s.seed for s in got.segments] == [100, 7, 101] + assert got.total_frames == 60 + + +def test_the_output_length_is_duration_frames_not_the_clips() -> None: + c = TestClient(build_app(ReferenceNull(max_reference_frames=50))) + skel, names = _joints(c) + # a clip within the cap, a take far longer than it: fine (text rules) + r = c.post("/generate", json=_request(skel, _segment(names[:2], t=40, duration_frames=200))) + assert r.status_code == 200, r.text + assert r.json()["extensions"]["MMCP_motion"]["samples"][0]["num_frames"] == 200 + + +# ---- the payload's own checks ------------------------------------------- + +def test_reference_joints_must_be_on_the_skeleton(client) -> None: + skel, names = _joints(client) + good = _segment(names) + r = client.post("/generate", json=_request(skel, good, _segment(names[:-1] + ["NotAJoint"]))) + assert r.status_code == 400 + assert r.json()["error"]["code"] == "unknown_joint" + + +@pytest.mark.parametrize("t,ok", [(300, True), (301, False)]) +def test_max_reference_frames_caps_the_clip(t, ok) -> None: + c = TestClient(build_app(ReferenceNull(max_reference_frames=300))) + skel, names = _joints(c) + r = c.post("/generate", json=_request(skel, _segment(names[:2], t=10), + _segment(names[:2], t=t, duration_frames=10))) + if ok: + assert r.status_code == 200, r.text + else: + assert r.status_code == 400 + err = r.json()["error"] + assert err["code"] == "invalid_options" + assert err["details"] == {"max_reference_frames": 300, "reference_frames": 301, "segment": 1} + + +def test_no_frame_cap_when_not_advertised() -> None: + c = TestClient(build_app(ReferenceNull(max_reference_frames=None))) + skel, names = _joints(c) + assert "max_reference_frames" not in c.get("/capabilities").json()["models"][0]["limits"] + r = c.post("/generate", json=_request(skel, _segment(names[:2], t=400, duration_frames=30))) + assert r.status_code == 200, r.text + + +# ---- a reference on its own skeleton ------------------------------------- + +_MIXAMO = {"joints": [ + {"name": "mixamorig:Hips", "parent": None, "rest_translation": [0, 1, 0], "rest_rotation": [0, 0, 0, 1]}, + {"name": "mixamorig:Spine", "parent": "mixamorig:Hips", "rest_translation": [0, 0.1, 0], + "rest_rotation": [0, 0, 0, 1]}, + {"name": "mixamorig:Head", "parent": "mixamorig:Spine", "rest_translation": [0, 0.5, 0], + "rest_rotation": [0, 0, 0, 1]}, +]} + + +def test_segment_skeleton_parses_and_round_trips() -> None: + seg = MotionReferenceSegment.model_validate( + _segment(["mixamorig:Hips", "mixamorig:Head"], t=4, skeleton=_MIXAMO)) + assert [j.name for j in seg.skeleton.joints][0] == "mixamorig:Hips" + again = MotionReferenceSegment.model_validate(json.loads(seg.model_dump_json())) + assert again == seg + + +def test_segment_skeleton_uses_the_skeleton_schema() -> None: + bad = {"joints": [{"name": "a", "parent": None, "rest_translation": [0, 0, 0], + "rest_rotation": [0, 0, 0, 1]}, + {"name": "b", "parent": None, "rest_translation": [0, 0, 0], + "rest_rotation": [0, 0, 0, 1]}]} # two roots + with pytest.raises(ValidationError): + MotionReferenceSegment.model_validate(_segment(["a"], t=3, skeleton=bad)) + + +def test_reference_from_another_rig_reaches_backbone() -> None: + b = ReferenceNull(retargets=True) + c = TestClient(build_app(b)) + skel, _ = _joints(c) + names = [j["name"] for j in _MIXAMO["joints"]] + seg = _segment(names, t=12, duration_frames=40, skeleton=_MIXAMO) + r = c.post("/generate", json=_request(skel, seg)) + assert r.status_code == 200, r.text + got = b.received[-1].motion_reference + assert [j.name for j in got.skeleton.joints] == names + assert got.reference_frames == 12 and got.duration_frames == 40 + + +def test_unknown_joint_checks_the_reference_skeleton() -> None: + c = TestClient(build_app(ReferenceNull(retargets=True))) + skel, request_names = _joints(c) + # Request-skeleton names are unknown on the clip's own rig. + r = c.post("/generate", json=_request(skel, _segment(request_names[:2], skeleton=_MIXAMO))) + assert r.status_code == 400 + err = r.json()["error"] + assert err["code"] == "unknown_joint" + assert "mixamorig:Hips" in json.dumps(err) + + +def test_reference_skeleton_needs_retargeting_unless_canonical(client) -> None: + skel, names = _joints(client) + r = client.post("/generate", json=_request( + skel, _segment(["mixamorig:Hips"], skeleton=_MIXAMO))) + assert r.status_code == 400 + assert r.json()["error"]["code"] == "retargeting_unsupported" + # The model's own skeleton, sent explicitly, is fine without retargeting. + r = client.post("/generate", json=_request(skel, _segment(names, skeleton=skel))) + assert r.status_code == 200, r.text + + +# ---- batches -------------------------------------------------------------- + +def test_batch_items_with_references(client) -> None: + skel, names = _joints(client) + good = _request(skel, _segment(names, t=8), {"type": "text", "prompt": "sit", "duration_frames": 8}) + bad = _request(skel, _segment(names[:-1] + ["NotAJoint"])) + r = client.post("/generate", json={"protocol_version": "1.2", "requests": [good, good, bad]}) + res = r.json()["results"] + assert "gltf" in res[0] and "gltf" in res[1] + assert res[2]["error"]["code"] == "unknown_joint" diff --git a/tests/test_smoke.py b/tests/test_smoke.py index d11aaec..644fa39 100644 --- a/tests/test_smoke.py +++ b/tests/test_smoke.py @@ -27,7 +27,7 @@ def test_capabilities_shape(client: TestClient) -> None: r = client.get("/capabilities") assert r.status_code == 200 body = r.json() - assert body["protocol_version"] == "1.0" + assert body["protocol_version"] == "1.2" assert body["coordinate_system"] == "right_handed_y_up" assert body["units"] == "meters" assert body["rotation_format"] == "quaternion_xyzw" @@ -192,6 +192,18 @@ def test_schema_validation_bad_quat_length(client: TestClient) -> None: assert r.json()["error"]["code"] == "schema_validation" +def test_schema_validation_from_a_model_validator_is_422(client: TestClient) -> None: + """A validator's own ValueError (here root_path's lengths) is a 422 + envelope with its message, not a 500 from an unserialisable ctx.""" + req = _request_for(client) + req["constraints"] = [{"type": "root_path", "frames": [0, 1], "positions_xz": [[0, 0]]}] + r = client.post("/generate", json=req) + assert r.status_code == 422 + err = r.json()["error"] + assert err["code"] == "schema_validation" + assert "positions_xz must have length 2" in json.dumps(err["details"]) + + def test_version_unsupported(client: TestClient) -> None: req = _request_for(client) req["protocol_version"] = "2.0" diff --git a/tests/test_video_reference.py b/tests/test_video_reference.py new file mode 100644 index 0000000..d58a096 --- /dev/null +++ b/tests/test_video_reference.py @@ -0,0 +1,422 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The video_reference segment (MMCP 1.2): a prompt given as a video. + +Its payload has its own checks (one source, https URL or base64 data with a +media type, the trim, max_video_bytes / max_video_seconds); everything else is +a text segment's, so every request rule is tested here against the same +request with a text segment in its place (as in test_motion_reference).""" + +from __future__ import annotations + +import base64 +import json + +import pytest + +pytest.importorskip("fastapi", reason="needs the [server] extras") + +from fastapi.testclient import TestClient # noqa: E402 +from pydantic import ValidationError # noqa: E402 + +from motionmcp import ( # noqa: E402 + GenerateRequest, + ModelSpec, + VideoReferenceSegment, + VideoSource, + build_app, +) +from motionmcp.null_backbone import NullBackbone # noqa: E402 + +_MP4 = b"\x00\x00\x00\x18ftypmp42" + bytes(range(256)) * 4 # 1036 bytes; never decoded +_DATA = base64.b64encode(_MP4).decode("ascii") +_URL = "https://cdn.example.com/clips/walk.mp4" + + +class VideoNull(NullBackbone): + """A NullBackbone that advertises video_reference (and motion_reference) + and keeps what it got.""" + + def __init__(self, max_video_bytes: int | None = 50_000_000, + max_video_seconds: float | None = 30.0, + loops: bool = False, poses: bool = False, **kw): + super().__init__(**kw) + self.max_video_bytes = max_video_bytes + self.max_video_seconds = max_video_seconds + self.loops = loops + self.poses = poses + self.received: list[GenerateRequest] = [] + + def capabilities(self) -> ModelSpec: + spec = super().capabilities() + spec.supported_segments = ["text", "unconditioned", "motion_reference", + "video_reference", *(["pose"] if self.poses else [])] + spec.limits.max_video_bytes = self.max_video_bytes + spec.limits.max_video_seconds = self.max_video_seconds + spec.supports_loop = self.loops + return spec + + async def generate(self, request: GenerateRequest): + self.received.append(request) + return await super().generate(request) + + +def _skeleton(client: TestClient) -> dict: + return client.get("/capabilities").json()["models"][0]["canonical_skeleton"] + + +def _url_segment(t: int = 30, **kw) -> dict: + return {"type": "video_reference", "duration_frames": t, "video": {"url": _URL}, **kw} + + +def _data_segment(t: int = 30, data: str = _DATA, **kw) -> dict: + return {"type": "video_reference", "duration_frames": t, + "video": {"data": data, "media_type": "video/mp4"}, **kw} + + +def _request(skel: dict, *segments: dict, **kw) -> dict: + return {"protocol_version": "1.2", "model": "null", "skeleton": skel, + "segments": list(segments), **kw} + + +@pytest.fixture +def backbone() -> VideoNull: + return VideoNull() + + +@pytest.fixture +def client(backbone: VideoNull) -> TestClient: + return TestClient(build_app(backbone)) + + +# ---- schema ------------------------------------------------------------- + +def test_url_segment_parses() -> None: + seg = VideoReferenceSegment.model_validate(_url_segment(t=48)) + assert seg.duration_frames == 48 and seg.video.url == _URL + assert seg.video.data is None and seg.video.num_bytes is None + assert (seg.start_s, seg.end_s, seg.fps, seg.seed) == (None,) * 4 + assert seg.trimmed_seconds is None + + +def test_data_segment_parses_and_is_not_decoded_as_video() -> None: + seg = VideoReferenceSegment.model_validate( + _data_segment(start_s=1.5, end_s=4.0, fps=29.97, seed=9)) + assert seg.video.media_type == "video/mp4" + assert seg.video.num_bytes == len(_MP4) + assert seg.video.decoded() == _MP4 + assert seg.trimmed_seconds == pytest.approx(2.5) + assert (seg.fps, seg.seed) == (29.97, 9) + + +@pytest.mark.parametrize("media_type", ["video/mp4", "video/quicktime", "video/webm"]) +def test_every_media_type(media_type) -> None: + raw = {"data": _DATA, "media_type": media_type} + assert VideoSource.model_validate(raw).media_type == media_type + + +def test_url_scheme_is_case_insensitive() -> None: + assert VideoSource.model_validate({"url": "HTTPS://cdn.example.com:8443/a.mp4"}).url + + +def test_trimmed_seconds_from_end_alone() -> None: + assert VideoReferenceSegment.model_validate(_url_segment(end_s=3)).trimmed_seconds == 3 + assert VideoReferenceSegment.model_validate(_url_segment(start_s=3)).trimmed_seconds is None + + +def test_url_video_cannot_be_decoded_by_the_sdk() -> None: + with pytest.raises(ValueError): + VideoSource.model_validate({"url": _URL}).decoded() + + +def test_duration_frames_is_required_as_for_text() -> None: + raw = _url_segment() + del raw["duration_frames"] + with pytest.raises(ValidationError): + VideoReferenceSegment.model_validate(raw) + + +@pytest.mark.parametrize("video", [ + {}, # no source + {"url": _URL, "data": _DATA, "media_type": "video/mp4"}, # both + {"url": "http://cdn.example.com/walk.mp4"}, # not https + {"url": "ftp://cdn.example.com/walk.mp4"}, + {"url": "https:///walk.mp4"}, # no host + {"url": "walk.mp4"}, + {"url": ""}, + {"url": "https://user:pw@cdn.example.com/walk.mp4"}, # user info + {"url": "https://cdn.example.com/walk .mp4"}, # whitespace + {"url": "https://cdn.example.com/walk.mp4\r\nX: y"}, # control chars + {"url": "https://cdn.example.com:99999/walk.mp4"}, # invalid port + {"url": "https://cdn.example.com:0/walk.mp4"}, + {"url": "https://cdn.example.com:x/walk.mp4"}, + {"url": "https://cdn.example.com/" + "a" * 5000}, # too long + {"url": _URL, "media_type": "video/mp4"}, # media_type with url + {"data": _DATA}, # no media_type + {"data": _DATA, "media_type": "video/avi"}, # not a listed type + {"data": _DATA, "media_type": "image/gif"}, + {"data": "not base64!", "media_type": "video/mp4"}, + {"data": _DATA[:-2], "media_type": "video/mp4"}, # bad padding + {"data": "", "media_type": "video/mp4"}, + {"data": "====", "media_type": "video/mp4"}, # decodes to nothing + {"url": _URL, "headers": {"Authorization": "x"}}, # extra field +]) +def test_video_source_rejects(video: dict) -> None: + with pytest.raises(ValidationError): + VideoReferenceSegment.model_validate({**_url_segment(), "video": video}) + + +@pytest.mark.parametrize("change", [ + {"duration_frames": 0}, + {"start_s": -0.1}, + {"end_s": 0}, + {"start_s": 2.0, "end_s": 2.0}, # start == end + {"start_s": 3.0, "end_s": 2.0}, # start > end + {"start_s": float("inf")}, + {"end_s": float("nan")}, + {"person": 0}, # not in 1.2: the most prominent + {"fps": 0}, + {"fps": -24}, + {"prompt": "walk"}, # extra field + {"video": None}, +]) +def test_segment_rejects(change: dict) -> None: + with pytest.raises(ValidationError): + VideoReferenceSegment.model_validate({**_url_segment(), **change}) + + +def test_zero_start_is_a_trim_from_the_beginning() -> None: + seg = VideoReferenceSegment.model_validate(_url_segment(start_s=0, end_s=5)) + assert seg.trimmed_seconds == 5 + + +@pytest.mark.parametrize("seg", [_url_segment(t=7, end_s=2.5), + _data_segment(t=7, start_s=0.5, fps=24, seed=3)]) +def test_json_round_trip(seg) -> None: + raw = _request({"joints": [{"name": "a", "parent": None, "rest_translation": [0, 0, 0], + "rest_rotation": [0, 0, 0, 1]}]}, + seg, {"type": "text", "prompt": "a person sits", "duration_frames": 5}) + req = GenerateRequest.model_validate(raw) + again = GenerateRequest.model_validate(json.loads(req.model_dump_json())) + assert again == req + got = again.segments[0] + assert isinstance(got, VideoReferenceSegment) and again.video_reference is got + assert again.video_references == [got] + assert got.video.num_bytes == req.segments[0].video.num_bytes + assert again.total_frames == 12, "the segments' duration_frames, not the video's length" + + +# ---- it reaches the backbone as sent ------------------------------------ + +@pytest.mark.parametrize("seg", [_url_segment(t=40, seed=3, start_s=1, end_s=3), + _data_segment(t=40, fps=30)]) +def test_happy_path_reaches_backbone_parsed(client, backbone, seg) -> None: + skel = _skeleton(client) + r = client.post("/generate", json=_request(skel, seg, options={"num_samples": 3})) + assert r.status_code == 200, r.text + got = backbone.received[-1].video_reference + sent = {"start_s": None, "end_s": None, "fps": None, "seed": None, **seg} + sent["video"] = {"url": None, "data": None, "media_type": None, **seg["video"]} + assert got.model_dump(mode="json") == json.loads(json.dumps(sent)) + ext = r.json()["extensions"]["MMCP_motion"] + assert [s["num_frames"] for s in ext["samples"]] == [40] * 3 + + +def test_capabilities_advertise_video_reference(client) -> None: + m = client.get("/capabilities").json()["models"][0] + assert "video_reference" in m["supported_segments"] + assert m["limits"]["max_video_bytes"] == 50_000_000 + assert m["limits"]["max_video_seconds"] == 30.0 + + +def test_gate_rejects_without_the_capability() -> None: + c = TestClient(build_app(NullBackbone())) + caps = c.get("/capabilities").json()["models"][0] + assert "video_reference" not in caps["supported_segments"] + assert "max_video_bytes" not in caps["limits"] + assert "max_video_seconds" not in caps["limits"] + for seg in (_url_segment(), _data_segment()): + r = c.post("/generate", json=_request(_skeleton(c), seg)) + assert r.status_code == 400 + err = r.json()["error"] + assert err["code"] == "unsupported_segment" + assert "video_reference" not in err["details"]["supported_segments"] + + +def test_gate_is_per_type() -> None: + """A model that takes motion references but not video references.""" + class MotionOnly(VideoNull): + def capabilities(self) -> ModelSpec: + spec = super().capabilities() + spec.supported_segments = ["text", "unconditioned", "motion_reference"] + return spec + + c = TestClient(build_app(MotionOnly())) + r = c.post("/generate", json=_request(_skeleton(c), _url_segment())) + assert r.status_code == 400 and r.json()["error"]["code"] == "unsupported_segment" + + +def test_schema_errors_are_422(client) -> None: + skel = _skeleton(client) + bad = {**_url_segment(), "video": {"url": "http://insecure.example.com/a.mp4"}} + r = client.post("/generate", json=_request(skel, bad)) + assert r.status_code == 422 + assert r.json()["error"]["code"] == "schema_validation" + + +# ---- the text segment's rules ------------------------------------------- + +def _as_text(seg: dict) -> dict: + """The text segment in a reference's place: same length, same seed.""" + return {"type": "text", "prompt": "a person walks", "duration_frames": seg["duration_frames"], + **({"seed": seg["seed"]} if "seed" in seg else {})} + + +def _outcome(c: TestClient, body: dict): + r = c.post("/generate", json=body) + if r.status_code == 200: + return 200, [s["num_frames"] for s in r.json()["extensions"]["MMCP_motion"]["samples"]] + return r.status_code, r.json()["error"]["code"] + + +_REF = "ref" + + +def _layouts(names): + """(segments with _REF placeholders, extra request fields) -- requests a + text segment makes or breaks in every way the SDK checks.""" + pk = lambda f: {"type": "pose_keyframe", "frame": f, "joint_rotations": {names[0]: [0, 0, 0, 1]}} # noqa: E731 + return [ + ([_REF], {}), + ([_REF, _REF], {}), + ([_REF, "text"], {}), + (["text", _REF, "text"], {}), + ([_REF, "unconditioned", _REF], {}), + ([_REF, "motion_reference"], {}), + (["motion_reference", _REF, "text"], {}), + ([_REF], {"options": {"num_samples": 4, "seed": 1}}), + ([_REF], {"options": {"loop": True}}), + ([_REF, _REF], {"options": {"loop": True}}), + ([_REF, "pose"], {}), + ([_REF], {"options": {"guidance": {"type": "separated", "weight": [2.0, 2.0]}}}), + ([_REF], {"options": {"transition_frames": 8}}), + ([_REF, _REF], {"constraints": [pk(39)]}), + ([_REF, _REF], {"constraints": [pk(40)]}), # frame_out_of_range + ([_REF], {"options": {"num_samples": 99}}), # 422 schema + ([_REF], {"options": {"num_samples": 16}}), # the most samples + ([_REF] * 50, {}), # max_duration_seconds + ] + + +@pytest.mark.parametrize("loops", [False, True]) +@pytest.mark.parametrize("i", range(18)) +def test_a_video_reference_is_validated_like_a_text_segment(i, loops) -> None: + b = VideoNull(loops=loops, poses=True) + c = TestClient(build_app(b)) + skel = _skeleton(c) + names = [j["name"] for j in skel["joints"]] + segments, extra = _layouts(names)[i] + motion = {"type": "motion_reference", "duration_frames": 11, "joint_names": names[:1], + "rotations": [[[0, 0, 0, 1]]] * 4, "root_positions": [[0, 0.9, 0]] * 4, "fps": 30} + others = {"text": {"type": "text", "prompt": "a person sits", "duration_frames": 17}, + "unconditioned": {"type": "unconditioned", "duration_frames": 9}, + "pose": {"type": "pose", "prompt": "a person sits"}, + "motion_reference": motion} + refs = [(_url_segment if k % 2 else _data_segment)(t=20, seed=10 + k) + for k in range(len(segments))] + with_refs = [refs[k] if s == _REF else others[s] for k, s in enumerate(segments)] + with_text = [_as_text(refs[k]) if s == _REF else others[s] for k, s in enumerate(segments)] + got = _outcome(c, _request(skel, *with_refs, **extra)) + want = _outcome(c, _request(skel, *with_text, **extra)) + assert got == want + + +def test_mixed_segments_reach_the_backbone_in_order(backbone, client) -> None: + skel = _skeleton(client) + names = [j["name"] for j in skel["joints"]] + segs = [{"type": "text", "prompt": "a person walks forward", "duration_frames": 30, "seed": 1}, + {"type": "motion_reference", "duration_frames": 20, "joint_names": names[:1], + "rotations": [[[0, 0, 0, 1]]] * 5, "root_positions": [[0, 0.9, 0]] * 5, + "fps": 30, "seed": 2}, + _url_segment(t=25, seed=3, start_s=0.5, end_s=2.5), + _data_segment(t=15, seed=4)] + r = client.post("/generate", json=_request(skel, *segs)) + assert r.status_code == 200, r.text + got = backbone.received[-1] + assert [s.type for s in got.segments] == ["text", "motion_reference", "video_reference", + "video_reference"] + assert got.video_references == [got.segments[2], got.segments[3]] + assert got.video_reference is got.segments[2] + assert [s.seed for s in got.segments] == [1, 2, 3, 4] + assert got.total_frames == 90 + + +# ---- the payload's own limits ------------------------------------------- + +@pytest.mark.parametrize("cap,ok", [(len(_MP4), True), (len(_MP4) - 1, False)]) +def test_max_video_bytes_caps_inline_data(cap, ok) -> None: + c = TestClient(build_app(VideoNull(max_video_bytes=cap))) + r = c.post("/generate", json=_request(_skeleton(c), _url_segment(), _data_segment())) + if ok: + assert r.status_code == 200, r.text + else: + assert r.status_code == 413 + err = r.json()["error"] + assert err["code"] == "payload_too_large" + assert err["details"] == {"max_video_bytes": cap, "video_bytes": len(_MP4), "segment": 1} + + +def test_max_video_bytes_cannot_see_a_url() -> None: + c = TestClient(build_app(VideoNull(max_video_bytes=1))) + r = c.post("/generate", json=_request(_skeleton(c), _url_segment())) + assert r.status_code == 200, "the server checks a URL's size when it fetches it" + + +@pytest.mark.parametrize("trim,ok", [ + ({"end_s": 10.0}, True), + ({"start_s": 5.0, "end_s": 15.0}, True), + ({"end_s": 10.5}, False), + ({"start_s": 1.0, "end_s": 11.5}, False), + ({"start_s": 100.0}, True), # length unknown to the SDK: the server's to check + ({}, True), +]) +def test_max_video_seconds_caps_the_trimmed_length(trim, ok) -> None: + c = TestClient(build_app(VideoNull(max_video_seconds=10.0))) + r = c.post("/generate", json=_request(_skeleton(c), _url_segment(**trim))) + if ok: + assert r.status_code == 200, r.text + else: + assert r.status_code == 400 + err = r.json()["error"] + assert err["code"] == "invalid_options" + assert err["details"]["max_video_seconds"] == 10.0 + assert err["details"]["segment"] == 0 + assert err["details"]["video_seconds"] == pytest.approx( + trim["end_s"] - trim.get("start_s", 0.0)) + + +def test_no_video_caps_when_not_advertised() -> None: + c = TestClient(build_app(VideoNull(max_video_bytes=None, max_video_seconds=None))) + limits = c.get("/capabilities").json()["models"][0]["limits"] + assert "max_video_bytes" not in limits and "max_video_seconds" not in limits + r = c.post("/generate", json=_request(_skeleton(c), _data_segment(end_s=3600))) + assert r.status_code == 200, r.text + + +def test_the_output_length_is_duration_frames_not_the_videos() -> None: + c = TestClient(build_app(VideoNull(max_video_seconds=2.0))) + r = c.post("/generate", json=_request(_skeleton(c), _url_segment(t=200, end_s=2.0))) + assert r.status_code == 200, r.text + assert r.json()["extensions"]["MMCP_motion"]["samples"][0]["num_frames"] == 200 + + +# ---- batches -------------------------------------------------------------- + +def test_batch_items_with_video_references() -> None: + c = TestClient(build_app(VideoNull(max_video_bytes=len(_MP4) - 1))) + skel = _skeleton(c) + good = _request(skel, _url_segment(t=8), {"type": "text", "prompt": "sit", "duration_frames": 8}) + bad = _request(skel, _data_segment(t=8)) + r = c.post("/generate", json={"protocol_version": "1.2", "requests": [good, bad, good]}) + res = r.json()["results"] + assert "gltf" in res[0] and "gltf" in res[2] + assert res[1]["error"]["code"] == "payload_too_large"