From 0efaa5c18ff2b157c96c6e23df0530da4abda1e1 Mon Sep 17 00:00:00 2001 From: a-baran-orhan Date: Sun, 4 Oct 2026 00:39:07 +0300 Subject: [PATCH] feat(render): export UniMate constraint manifests --- .changeset/clean-lions-inbetween.md | 5 + docs/README.md | 1 + docs/integrations/unimate.md | 64 ++++ packages/posecode-render/README.md | 18 + packages/posecode-render/src/index.ts | 11 + packages/posecode-render/src/unimate.ts | 327 ++++++++++++++++++ packages/posecode-render/test/unimate.test.ts | 119 +++++++ 7 files changed, 545 insertions(+) create mode 100644 .changeset/clean-lions-inbetween.md create mode 100644 docs/integrations/unimate.md create mode 100644 packages/posecode-render/src/unimate.ts create mode 100644 packages/posecode-render/test/unimate.test.ts diff --git a/.changeset/clean-lions-inbetween.md b/.changeset/clean-lions-inbetween.md new file mode 100644 index 0000000..50ea354 --- /dev/null +++ b/.changeset/clean-lions-inbetween.md @@ -0,0 +1,5 @@ +--- +"posecode-render": minor +--- + +Add a deterministic UniMate constraint manifest exporter for sparse key-pose in-betweening workflows. diff --git a/docs/README.md b/docs/README.md index 0d8f4d8..17fe6ee 100644 --- a/docs/README.md +++ b/docs/README.md @@ -27,5 +27,6 @@ The repository root retains the canonical [`LICENSE`](../LICENSE) and [`NOTICE`] ## Development references - [Product usage analytics](product-analytics.md) +- [Posecode × UniMate sparse key-pose bridge](integrations/unimate.md) - [Vercel agent notes](development/VERCEL_AGENTS.md) - [Pose diagnostics summary](diagnostics/pose-summary.json) diff --git a/docs/integrations/unimate.md b/docs/integrations/unimate.md new file mode 100644 index 0000000..cf9947f --- /dev/null +++ b/docs/integrations/unimate.md @@ -0,0 +1,64 @@ +# Posecode × UniMate: sparse key-pose bridge + +**Status:** implementable interchange boundary, with the rig-aware UniMate adapter proposed as the next joint step. + +Posecode should remain the editable source of truth for named phases, sparse key poses, root intent, and contact constraints. UniMate should generate only the motion between those authored anchors. This avoids treating a generated 60 FPS clip as the primary artifact and directly matches UniMate's existing `x1_known` / `keep_mask` replacement path. + +```mermaid +flowchart LR + A[Posecode document] --> B[constraint manifest v1] + B --> C[rig mapping and FK] + C --> D[UniMate pose encoder] + D --> E[x1_known + keep_mask] + E --> F[UniMate in-betweening] + F --> G[contact and ROM diagnostics] + G --> H[editable BVH / GLB / Blender Action] +``` + +## Implemented boundary + +`buildUniMateConstraintManifest(ir, { fps: 30 })` now emits a deterministic JSON-safe schedule with: + +- one-based contiguous prompt clips compatible with Blender timelines; +- the start pose and every phase endpoint as a named key-pose reference; +- carried local Euler channels, root travel and facing intent; +- phase-range `ground-lock`, `reach`, `pin`, and `grip` constraints; +- display-only cues kept separate from model prompts; +- Posecode-to-Mixamo bone bindings; and +- an explicit mask contract: position and rotation are known at reference frames, velocity remains generative. + +The manifest intentionally does **not** claim to be UniMate's normalized `(T, J, 12)` tensor. That final encoding requires the chosen rig's rest hierarchy, global FK positions, and `dataset_stats.npy` from the selected checkpoint. The adapter must compute those values next to UniMate, where that context exists. + +## Smallest useful joint prototype + +1. Add a manifest importer to [UniMate-B3D](https://github.com/nopeburger/UniMate-B3D). +2. Map each Posecode semantic bone to the selected Blender deform rig, apply every reference pose, then reuse B3D's `encode_pose` path. +3. Build `x1_known` and `keep_mask` with channels `0:9` fixed at reference frames and channels `9:12` left free, matching B3D's current pose-reference behavior. +4. Generate the gaps, then retain Posecode keyframes as hard anchors during retiming. +5. Return an editable Blender Action plus machine-readable diagnostics. + +## Evaluation + +Use the released 22-joint Mixamo checkpoint first, with 10 to 20 short movements containing two to five key poses. Compare text-only UniMate, Posecode interpolation, and the combined system on: + +| Measure | Target | +| --- | --- | +| Authored key-pose rotation error | effectively zero at pinned frames | +| Root endpoint error | under 2 cm | +| Planted-foot drift | under 2 cm during declared contacts | +| ROM violations | none after validation/post-process | +| Animator correction effort | fewer key edits than either baseline | + +The most informative failure set is also small: quadruped rigs that rotate upright, rare topologies, opposing contact constraints, and phase references placed too close for a 60-frame generation window. + +## Contribution split and licensing + +- **Posecode:** manifest producer, Mixamo binding profile, fixtures, browser demo, contact/ROM diagnostics, and evaluation report. +- **UniMate / UniMate-B3D:** rig-specific FK and pose encoding, checkpoint normalization, generation, and Blender Action creation. +- **Shared:** benchmark clips, failure taxonomy, and a short technical report if results warrant it. + +The new Posecode adapter is part of `posecode-render` and therefore AGPL-3.0-only. UniMate's repository code is MIT; UniMate-B3D is GPL-3.0-or-later. The released UniMate weights are CC BY-NC 4.0, so model-backed commercial use is a separate question from code reuse and must not be represented as MIT-covered. + +## Acceptance test + +A bridge is complete when four Posecode-authored Mixamo key poses can be imported, used as fixed UniMate references, generated between, and exported as a Blender Action while preserving all four anchors and reporting contact/ROM residuals. diff --git a/packages/posecode-render/README.md b/packages/posecode-render/README.md index b1a31ac..9d46591 100644 --- a/packages/posecode-render/README.md +++ b/packages/posecode-render/README.md @@ -73,6 +73,24 @@ bounded self-collision pairs. They measure the post-solver procedural driver, before optional skinned-character or mocap surface reconciliation, and report outcomes without changing the authored motion. +## UniMate constraint interchange + +Use `buildUniMateConstraintManifest()` to turn a validated IR into a JSON-safe +sparse key-pose schedule for a rig-aware UniMate adapter: + +```ts +import { buildUniMateConstraintManifest } from "posecode-render"; + +const manifest = buildUniMateConstraintManifest(ir, { fps: 30 }); +``` + +The manifest preserves named phase endpoints, root intent, Mixamo bone +bindings, and contact constraints. It deliberately stops before UniMate's +normalized motion tensor because that conversion requires the destination +rig's rest geometry and the selected checkpoint's normalization statistics. +See the [integration note](https://github.com/posecode-dev/posecode/blob/main/docs/integrations/unimate.md) +for the bridge boundary and acceptance test. + No GPU, no diffusion model: generation is a fraction of a cent of text, and rendering is plain forward kinematics. diff --git a/packages/posecode-render/src/index.ts b/packages/posecode-render/src/index.ts index f81e6c7..7dd3441 100644 --- a/packages/posecode-render/src/index.ts +++ b/packages/posecode-render/src/index.ts @@ -1684,3 +1684,14 @@ export { export type { PhaseSegment } from "./timeline.js"; export { exportBVH, type BvhExportOptions } from "./bvh.js"; export { exportGLTF, buildAnimatedRig, type GltfExportOptions } from "./gltf.js"; +export { + UNIMATE_CONSTRAINT_SCHEMA, + POSECODE_MIXAMO_BINDINGS, + buildUniMateConstraintManifest, + type UniMateBoneBinding, + type UniMateConstraintKeyframe, + type UniMateConstraintManifest, + type UniMateConstraintOptions, + type UniMateConstraintSet, + type UniMatePromptClip, +} from "./unimate.js"; diff --git a/packages/posecode-render/src/unimate.ts b/packages/posecode-render/src/unimate.ts new file mode 100644 index 0000000..b9ad375 --- /dev/null +++ b/packages/posecode-render/src/unimate.ts @@ -0,0 +1,327 @@ +/** + * Sparse, rig-neutral key-pose interchange for a UniMate in-betweening bridge. + * + * This module deliberately stops before UniMate's `(T, J, 12)` tensor. Building + * that tensor requires the destination rig's rest geometry and the selected + * checkpoint's normalization statistics. The manifest keeps Posecode's exact + * authored intent intact so a rig-aware consumer (for example a Blender + * adapter) can perform that final conversion without guessing. + */ + +import type { + EulerDeg, + GripTarget, + PinTarget, + PosecodeIR, + ReachTarget, +} from "posecode-parser"; +import { poseFor } from "./poses.js"; + +export const UNIMATE_CONSTRAINT_SCHEMA = "posecode.unimate.constraints.v1" as const; + +export interface UniMateConstraintOptions { + /** Output timeline rate. UniMate's released checkpoints use 30 FPS. */ + fps?: number; + /** Optional context prepended to every phase prompt. */ + promptPrefix?: string; +} + +export interface UniMateBoneBinding { + posecode: string; + /** Mixamo convention without a `mixamorig` namespace. */ + mixamo: string; +} + +export interface UniMateConstraintSet { + groundLock: string[]; + reaches: ReachTarget[]; + pins: PinTarget[]; + grips: GripTarget[]; +} + +export interface UniMateConstraintKeyframe { + id: string; + /** One-based, matching Blender and UniMate-B3D timeline ranges. */ + frame: number; + timeSeconds: number; + phase: string; + /** Local XYZ Euler channels in Posecode's anatomical convention. */ + localEulerDeg: Record; + root: { + /** Authored root translation before grounding/contact solving. */ + positionMeters: [number, number, number]; + /** Base-pose XYZ rotation; locomotion facing remains a separate yaw. */ + rotationDeg: [number, number, number]; + yawDeg: number; + }; + constraints: UniMateConstraintSet; +} + +export interface UniMatePromptClip { + prompt: string; + /** Display-only coaching metadata; it never changes generation prompts. */ + cue?: string; + /** Inclusive, one-based frame range. */ + start: number; + end: number; + /** Contact intent active across this complete phase range. */ + constraints: UniMateConstraintSet; + references: Array<{ frame: number; keyframeId: string }>; +} + +export interface UniMateConstraintManifest { + schema: typeof UNIMATE_CONSTRAINT_SCHEMA; + source: { + name: string; + kind: string; + posecodeVersion: string; + rig: string; + startPose: string; + repeat: number; + }; + timing: { + fps: number; + frames: number; + durationSeconds: number; + frameIndexing: "one-based-inclusive"; + }; + rig: { + profile: "posecode-humanoid"; + boneBindings: UniMateBoneBinding[]; + }; + clips: UniMatePromptClip[]; + keyframes: UniMateConstraintKeyframe[]; + generation: { + knownChannels: ["position", "rotation6d"]; + generatedChannels: ["velocity"]; + sourceOfTruth: "keyframes"; + }; + caveats: string[]; +} + +/** Driver bone id to common Mixamo bone name. */ +export const POSECODE_MIXAMO_BINDINGS: readonly UniMateBoneBinding[] = [ + { posecode: "pelvis", mixamo: "Hips" }, + { posecode: "spine", mixamo: "Spine" }, + { posecode: "chest", mixamo: "Spine2" }, + { posecode: "neck", mixamo: "Neck" }, + { posecode: "head", mixamo: "Head" }, + { posecode: "shoulder_left", mixamo: "LeftArm" }, + { posecode: "elbow_left", mixamo: "LeftForeArm" }, + { posecode: "wrist_left", mixamo: "LeftHand" }, + { posecode: "shoulder_right", mixamo: "RightArm" }, + { posecode: "elbow_right", mixamo: "RightForeArm" }, + { posecode: "wrist_right", mixamo: "RightHand" }, + { posecode: "hip_left", mixamo: "LeftUpLeg" }, + { posecode: "knee_left", mixamo: "LeftLeg" }, + { posecode: "ankle_left", mixamo: "LeftFoot" }, + { posecode: "hip_right", mixamo: "RightUpLeg" }, + { posecode: "knee_right", mixamo: "RightLeg" }, + { posecode: "ankle_right", mixamo: "RightFoot" }, + { posecode: "thumb_left", mixamo: "LeftHandThumb1" }, + { posecode: "index_left", mixamo: "LeftHandIndex1" }, + { posecode: "middle_left", mixamo: "LeftHandMiddle1" }, + { posecode: "ring_left", mixamo: "LeftHandRing1" }, + { posecode: "pinky_left", mixamo: "LeftHandPinky1" }, + { posecode: "thumb_right", mixamo: "RightHandThumb1" }, + { posecode: "index_right", mixamo: "RightHandIndex1" }, + { posecode: "middle_right", mixamo: "RightHandMiddle1" }, + { posecode: "ring_right", mixamo: "RightHandRing1" }, + { posecode: "pinky_right", mixamo: "RightHandPinky1" }, +] as const; + +type EulerTuple = [number, number, number]; + +/** + * Convert validated Posecode IR into a sparse constraint schedule. + * + * The result is JSON-safe and deterministic. It is not a normalized UniMate + * model tensor: a consumer must first apply these local rotations to its own + * rest skeleton, run FK, then call the checkpoint-specific pose encoder. + */ +export function buildUniMateConstraintManifest( + ir: PosecodeIR, + options: UniMateConstraintOptions = {}, +): UniMateConstraintManifest { + const fps = options.fps ?? 30; + if (!Number.isInteger(fps) || fps <= 0 || fps > 240) { + throw new RangeError("buildUniMateConstraintManifest: fps must be an integer from 1 to 240"); + } + + const base = poseFor(ir.startPose); + const authored = new Map( + Object.entries(base.joints ?? {}).map(([bone, euler]) => [bone, [...euler]]), + ); + mergeTargets(authored, ir.startPoseOverrides ?? []); + + const basePosition = [...(base.root?.position ?? [0, 0, 0])] as EulerTuple; + const baseRotation = [...(base.root?.rotationDeg ?? [0, 0, 0])] as EulerTuple; + const keyframes: UniMateConstraintKeyframe[] = [ + makeKeyframe( + "start", + 1, + 0, + ir.startPose ?? "neutral", + authored, + basePosition, + baseRotation, + 0, + emptyConstraints(), + ), + ]; + const clips: UniMatePromptClip[] = []; + let cursor = 1; + let timeSeconds = 0; + let yawDeg = 0; + let travel = { x: 0, z: 0 }; + + ir.phases.forEach((phase, phaseIndex) => { + const frames = Math.round(phase.durationSec * fps); + if (frames < 2) { + throw new RangeError( + `buildUniMateConstraintManifest: phase ${JSON.stringify(phase.name)} needs at least 2 frames at ${fps} FPS`, + ); + } + mergeTargets(authored, phase.targets); + if (phase.turnDeg !== undefined) yawDeg = phase.turnDeg; + if (phase.travel) travel = { ...phase.travel }; + timeSeconds += phase.durationSec; + + const start = cursor; + const end = start + frames - 1; + const id = `phase-${phaseIndex + 1}`; + keyframes.push( + makeKeyframe( + id, + end, + timeSeconds, + phase.name, + authored, + [basePosition[0] + travel.x, basePosition[1], basePosition[2] + travel.z], + baseRotation, + yawDeg, + { + groundLock: [...phase.groundLock], + reaches: phase.reaches.map((target) => ({ ...target })), + pins: phase.pins.map((target) => ({ ...target })), + grips: phase.grips.map((target) => ({ ...target })), + }, + ), + ); + + const constraints: UniMateConstraintSet = { + groundLock: [...phase.groundLock], + reaches: phase.reaches.map((target) => ({ ...target })), + pins: phase.pins.map((target) => ({ ...target })), + grips: phase.grips.map((target) => ({ ...target })), + }; + const prompt = [options.promptPrefix?.trim(), ir.name, phase.name] + .filter(Boolean) + .join(". "); + clips.push({ + prompt, + ...(phase.cue ? { cue: phase.cue } : {}), + start, + end, + constraints, + references: [ + ...(phaseIndex === 0 ? [{ frame: 1, keyframeId: "start" }] : []), + { frame: end, keyframeId: id }, + ], + }); + cursor = end + 1; + }); + + return { + schema: UNIMATE_CONSTRAINT_SCHEMA, + source: { + name: ir.name, + kind: ir.kind, + posecodeVersion: ir.version, + rig: ir.rig, + startPose: ir.startPose ?? "neutral", + repeat: ir.repeat, + }, + timing: { + fps, + frames: Math.max(1, cursor - 1), + durationSeconds: timeSeconds, + frameIndexing: "one-based-inclusive", + }, + rig: { + profile: "posecode-humanoid", + boneBindings: POSECODE_MIXAMO_BINDINGS.map((binding) => ({ ...binding })), + }, + clips, + keyframes, + generation: { + knownChannels: ["position", "rotation6d"], + generatedChannels: ["velocity"], + sourceOfTruth: "keyframes", + }, + caveats: [ + "Keyframes contain authored local rotations and root travel before IK, grounding, and contact solving.", + "A rig-aware adapter must apply FK and checkpoint normalization before constructing x1_known and keep_mask.", + "Model weights and generated motion remain subject to the UniMate checkpoint and training-data terms.", + ], + }; +} + +function mergeTargets( + pose: Map, + targets: readonly { boneId: string; euler: EulerDeg; axes?: readonly ("x" | "y" | "z")[] }[], +): void { + const axisIndex = { x: 0, y: 1, z: 2 } as const; + for (const target of targets) { + const next = [...(pose.get(target.boneId) ?? [0, 0, 0])] as EulerTuple; + for (const axis of target.axes ?? ["x", "y", "z"]) { + next[axisIndex[axis]] = target.euler[axis]; + } + pose.set(target.boneId, next); + } +} + +function makeKeyframe( + id: string, + frame: number, + timeSeconds: number, + phase: string, + authored: Map, + positionMeters: EulerTuple, + rotationDeg: EulerTuple, + yawDeg: number, + constraints: UniMateConstraintSet, +): UniMateConstraintKeyframe { + const pose = new Map(); + for (const [bone, euler] of authored) pose.set(bone, [...euler]); + + // Match the renderer's hip-hinge coupling: pelvis flexion tips the torso, + // while equal counter-rotation keeps the legs in their authored frame. + const pelvisX = authored.get("pelvis")?.[0] ?? 0; + if (pelvisX !== 0) { + for (const hip of ["hip_left", "hip_right"]) { + const [x, y, z] = authored.get(hip) ?? [0, 0, 0]; + pose.set(hip, [Math.max(-135, Math.min(20, x - pelvisX)), y, z]); + } + } + + return { + id, + frame, + timeSeconds, + phase, + localEulerDeg: Object.fromEntries( + [...pose.entries()].sort(([a], [b]) => a.localeCompare(b)), + ), + root: { + positionMeters: [...positionMeters], + rotationDeg: [...rotationDeg], + yawDeg, + }, + constraints, + }; +} + +function emptyConstraints(): UniMateConstraintSet { + return { groundLock: [], reaches: [], pins: [], grips: [] }; +} diff --git a/packages/posecode-render/test/unimate.test.ts b/packages/posecode-render/test/unimate.test.ts new file mode 100644 index 0000000..43ec4ea --- /dev/null +++ b/packages/posecode-render/test/unimate.test.ts @@ -0,0 +1,119 @@ +import { describe, expect, it } from "vitest"; +import { parse } from "posecode-parser"; +import { + UNIMATE_CONSTRAINT_SCHEMA, + buildUniMateConstraintManifest, +} from "../src/unimate.js"; + +const SOURCE = `posecode exercise "Controlled squat reach" + rig humanoid + pose start = standing + + step "Lower" 1s settle: + pelvis: hinge 20 + hips: flex 70 + knees: flex 90 + ground-lock: feet + cue "lower while keeping both feet planted" + + step "Reach" 0.5s flow: + shoulder_left: flex 80 + reach: hand_left ankle_left + turn: 30 + travel: 0.2 0.1 + repeat 2 +`; + +describe("buildUniMateConstraintManifest", () => { + it("exports contiguous prompt clips and exact sparse key-pose references", () => { + const parsed = parse(SOURCE); + expect(parsed.errors).toEqual([]); + const manifest = buildUniMateConstraintManifest(parsed.ir!, { fps: 30 }); + + expect(manifest.schema).toBe(UNIMATE_CONSTRAINT_SCHEMA); + expect(manifest.source).toMatchObject({ + name: "Controlled squat reach", + startPose: "standing", + repeat: 2, + }); + expect(manifest.timing).toEqual({ + fps: 30, + frames: 45, + durationSeconds: 1.5, + frameIndexing: "one-based-inclusive", + }); + expect(manifest.clips).toEqual([ + { + prompt: "Controlled squat reach. Lower", + cue: "lower while keeping both feet planted", + start: 1, + end: 30, + constraints: { + groundLock: ["feet"], + reaches: [], + pins: [], + grips: [], + }, + references: [ + { frame: 1, keyframeId: "start" }, + { frame: 30, keyframeId: "phase-1" }, + ], + }, + { + prompt: "Controlled squat reach. Reach", + start: 31, + end: 45, + constraints: { + groundLock: [], + reaches: [{ effector: "hand_left", target: "ankle_left" }], + pins: [], + grips: [], + }, + references: [{ frame: 45, keyframeId: "phase-2" }], + }, + ]); + }); + + it("carries joint state, root intent, contacts, and renderer hip coupling", () => { + const manifest = buildUniMateConstraintManifest(parse(SOURCE).ir!); + const lower = manifest.keyframes[1]!; + const reach = manifest.keyframes[2]!; + + expect(lower.localEulerDeg.pelvis).toEqual([20, 0, 0]); + expect(lower.localEulerDeg.hip_left).toEqual([-90, 0, 0]); + expect(lower.localEulerDeg.hip_right).toEqual([-90, 0, 0]); + expect(lower.constraints.groundLock).toEqual(["feet"]); + expect(reach.localEulerDeg.knee_left).toEqual([90, 0, 0]); + expect(reach.localEulerDeg.shoulder_left).toEqual([-80, 0, 0]); + expect(reach.constraints.reaches).toEqual([ + { effector: "hand_left", target: "ankle_left" }, + ]); + expect(reach.root).toMatchObject({ + positionMeters: [0.2, 0, 0.1], + yawDeg: 30, + }); + }); + + it("keeps checkpoint-specific encoding outside the interchange", () => { + const manifest = buildUniMateConstraintManifest(parse(SOURCE).ir!); + expect(manifest.generation).toEqual({ + knownChannels: ["position", "rotation6d"], + generatedChannels: ["velocity"], + sourceOfTruth: "keyframes", + }); + expect(JSON.stringify(manifest)).not.toContain("dataset_stats"); + expect(manifest.caveats.join(" ")).toContain("rig-aware adapter"); + }); + + it("rejects phases that collapse below UniMate-B3D's two-frame minimum", () => { + const parsed = parse(`posecode posture "Flash" + rig humanoid + step "Too short" 0.01s snap: + head: flex 5 +`); + expect(parsed.errors).toEqual([]); + expect(() => buildUniMateConstraintManifest(parsed.ir!, { fps: 30 })).toThrow( + /needs at least 2 frames/, + ); + }); +});