Examples: add `webgpu_animation_retargeting` (#29338)

* add webgpu_animation_retargeting

* cleanup

* Revert "cleanup"

This reverts commit 7d51a197cddd5618a949e7a142c0b3f0bbd35937.

* Update webgpu_animation_retargeting.html

* adding floor

* update

* Update webgpu_animation_retargeting.html

* update

* update

* Update webgpu_animation_retargeting.html

* updates

* update

examples/files.json

@@ -301,6 +301,7 @@
 		"webgl_performance"
 	],
 	"webgpu (wip)": [
+		"webgpu_animation_retargeting",
 		"webgpu_backdrop",
 		"webgpu_backdrop_area",
 		"webgpu_backdrop_water",

examples/jsm/utils/SkeletonUtils.js

@@ -12,25 +12,23 @@ import {
 
 function retarget( target, source, options = {} ) {
 
-	const pos = new Vector3(),
-		quat = new Quaternion(),
+	const quat = new Quaternion(),
 		scale = new Vector3(),
-		bindBoneMatrix = new Matrix4(),
 		relativeMatrix = new Matrix4(),
 		globalMatrix = new Matrix4();
 
-	options.preserveMatrix = options.preserveMatrix !== undefined ? options.preserveMatrix : true;
-	options.preservePosition = options.preservePosition !== undefined ? options.preservePosition : true;
+	options.preserveBoneMatrix = options.preserveBoneMatrix !== undefined ? options.preserveBoneMatrix : true;
+	options.preserveBonePositions = options.preserveBonePositions !== undefined ? options.preserveBonePositions : true;
 	options.preserveHipPosition = options.preserveHipPosition !== undefined ? options.preserveHipPosition : false;
 	options.useTargetMatrix = options.useTargetMatrix !== undefined ? options.useTargetMatrix : false;
 	options.hip = options.hip !== undefined ? options.hip : 'hip';
+	options.scale = options.scale !== undefined ? options.scale : 1;
 	options.names = options.names || {};
 
 	const sourceBones = source.isObject3D ? source.skeleton.bones : getBones( source ),
 		bones = target.isObject3D ? target.skeleton.bones : getBones( target );
 
-	let bindBones,
-		bone, name, boneTo,
+	let bone, name, boneTo,
 		bonesPosition;
 
 	// reset bones
@@ -42,11 +40,11 @@ function retarget( target, source, options = {} ) {
 	} else {
 
 		options.useTargetMatrix = true;
-		options.preserveMatrix = false;
+		options.preserveBoneMatrix = false;
 
 	}
 
-	if ( options.preservePosition ) {
+	if ( options.preserveBonePositions ) {
 
 		bonesPosition = [];
 
@@ -58,7 +56,7 @@ function retarget( target, source, options = {} ) {
 
 	}
 
-	if ( options.preserveMatrix ) {
+	if ( options.preserveBoneMatrix ) {
 
 		// reset matrix
 
@@ -76,35 +74,10 @@ function retarget( target, source, options = {} ) {
 
 	}
 
-	if ( options.offsets ) {
-
-		bindBones = [];
-
-		for ( let i = 0; i < bones.length; ++ i ) {
-
-			bone = bones[ i ];
-			name = options.names[ bone.name ] || bone.name;
-
-			if ( options.offsets[ name ] ) {
-
-				bone.matrix.multiply( options.offsets[ name ] );
-
-				bone.matrix.decompose( bone.position, bone.quaternion, bone.scale );
-
-				bone.updateMatrixWorld();
-
-			}
-
-			bindBones.push( bone.matrixWorld.clone() );
-
-		}
-
-	}
-
 	for ( let i = 0; i < bones.length; ++ i ) {
 
 		bone = bones[ i ];
-		name = options.names[ bone.name ] || bone.name;
+		name = options.names[ bone.name ];
 
 		boneTo = getBoneByName( name, sourceBones );
 
@@ -136,10 +109,15 @@ function retarget( target, source, options = {} ) {
 
 			if ( target.isObject3D ) {
 
-				const boneIndex = bones.indexOf( bone ),
-					wBindMatrix = bindBones ? bindBones[ boneIndex ] : bindBoneMatrix.copy( target.skeleton.boneInverses[ boneIndex ] ).invert();
+				if ( options.localOffsets ) {
 
-				globalMatrix.multiply( wBindMatrix );
+					if ( options.localOffsets[ bone.name ] ) {
+
+						globalMatrix.multiply( options.localOffsets[ bone.name ] );
+
+					}
+
+				}
 
 			}
 
@@ -147,20 +125,28 @@ function retarget( target, source, options = {} ) {
 
 		}
 
-		if ( bone.parent && bone.parent.isBone ) {
+		if ( name === options.hip ) {
 
-			bone.matrix.copy( bone.parent.matrixWorld ).invert();
-			bone.matrix.multiply( globalMatrix );
+			globalMatrix.elements[ 12 ] *= options.scale;
+			globalMatrix.elements[ 13 ] *= options.scale;
+			globalMatrix.elements[ 14 ] *= options.scale;
 
-		} else {
+			if ( options.preserveHipPosition ) {
 
-			bone.matrix.copy( globalMatrix );
+				globalMatrix.elements[ 12 ] = globalMatrix.elements[ 14 ] = 0;
+
+			}
 
 		}
 
-		if ( options.preserveHipPosition && name === options.hip ) {
+		if ( bone.parent ) {
 
-			bone.matrix.setPosition( pos.set( 0, bone.position.y, 0 ) );
+			bone.matrix.copy( bone.parent.matrixWorld ).invert();
+			bone.matrix.multiply( globalMatrix );
+
+		} else {
+
+			bone.matrix.copy( globalMatrix );
 
 		}
 
@@ -170,7 +156,7 @@ function retarget( target, source, options = {} ) {
 
 	}
 
-	if ( options.preservePosition ) {
+	if ( options.preserveBonePositions ) {
 
 		for ( let i = 0; i < bones.length; ++ i ) {
 
@@ -187,7 +173,7 @@ function retarget( target, source, options = {} ) {
 
 	}
 
-	if ( options.preserveMatrix ) {
+	if ( options.preserveBoneMatrix ) {
 
 		// restore matrix
 
@@ -200,6 +186,7 @@ function retarget( target, source, options = {} ) {
 function retargetClip( target, source, clip, options = {} ) {
 
 	options.useFirstFramePosition = options.useFirstFramePosition !== undefined ? options.useFirstFramePosition : false;
+
 	// Calculate the fps from the source clip based on the track with the most frames, unless fps is already provided.
 	options.fps = options.fps !== undefined ? options.fps : ( Math.max( ...clip.tracks.map( track => track.times.length ) ) / clip.duration );
 	options.names = options.names || [];

examples/webgpu_animation_retargeting.html

@@ -0,0 +1,259 @@
+<html lang="en">
+	<head>
+		<title>three.js webgpu - animation retargeting</title>
+		<meta charset="utf-8">
+		<meta name="viewport" content="width=device-width, user-scalable=no, minimum-scale=1.0, maximum-scale=1.0">
+		<link type="text/css" rel="stylesheet" href="main.css">
+	</head>
+	<body>
+
+		<div id="info">
+			<a href="https://threejs.org" target="_blank" rel="noopener">three.js</a> webgpu - animation retargeting
+		</div>
+
+		<script type="importmap">
+			{
+				"imports": {
+					"three": "../build/three.webgpu.js",
+					"three/tsl": "../build/three.webgpu.js",
+					"three/addons/": "./jsm/"
+				}
+			}
+		</script>
+
+		<script type="module">
+
+			import * as THREE from 'three';
+			import { color, viewportUV, reflector } from 'three/tsl';
+
+			import Stats from 'three/addons/libs/stats.module.js';
+
+			import { OrbitControls } from 'three/addons/controls/OrbitControls.js';
+			import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js';
+			import { GUI } from 'three/addons/libs/lil-gui.module.min.js';
+
+			import * as SkeletonUtils from 'three/addons/utils/SkeletonUtils.js';
+
+			const [ sourceModel, targetModel ] = await Promise.all( [
+
+				new Promise( ( resolve, reject ) => {
+
+					new GLTFLoader().load( './models/gltf/Michelle.glb', resolve, undefined, reject );
+
+				} ),
+
+				new Promise( ( resolve, reject ) => {
+
+					new GLTFLoader().load( './models/gltf/Soldier.glb', resolve, undefined, reject );
+
+				} )
+
+			] );
+
+			//
+
+			const clock = new THREE.Clock();
+
+			const stats = new Stats();
+			document.body.appendChild( stats.dom );
+
+			// scene
+			const scene = new THREE.Scene();
+			scene.backgroundNode = viewportUV.distance( .5 ).mix( color( 0x0175ad ), color( 0x02274f ) );
+
+			const helpers = new THREE.Group();
+			helpers.visible = false;
+			scene.add( helpers );
+
+			const light = new THREE.HemisphereLight( 0xe9c0a5, 0x0175ad, 5 );
+			scene.add( light );
+
+			const dirLight = new THREE.DirectionalLight( 0xfff9ea, 4 );
+			dirLight.position.set( 2, 5, 2 );
+			scene.add( dirLight );
+
+			const camera = new THREE.PerspectiveCamera( 40, window.innerWidth / window.innerHeight, .25, 50 );
+			camera.position.set( - 2, 2, 4 );
+
+			// add models to scene
+			scene.add( sourceModel.scene );
+			scene.add( targetModel.scene );
+
+			// reposition models
+			sourceModel.scene.position.x -= 1;
+			targetModel.scene.position.x += 1;
+
+			// reajust model
+			targetModel.scene.scale.setScalar( .01 );
+
+			// retarget
+			const source = getSource( sourceModel );
+			const mixer = retargetModel( source, targetModel );
+
+			// floor
+			const reflection = reflector();
+			reflection.target.rotateX( - Math.PI / 2 );
+			scene.add( reflection.target );
+
+			const floorMaterial = new THREE.NodeMaterial();
+			floorMaterial.colorNode = reflection;
+			floorMaterial.opacity = .2;
+			floorMaterial.transparent = true;
+
+			const floor = new THREE.Mesh( new THREE.BoxGeometry( 50, .001, 50 ), floorMaterial );
+			floor.receiveShadow = true;
+
+			floor.position.set( 0, 0, 0 );
+			scene.add( floor );
+
+			// renderer
+			const renderer = new THREE.WebGPURenderer( { antialias: true } );
+			renderer.toneMapping = THREE.NeutralToneMapping;
+			renderer.setAnimationLoop( animate );
+			renderer.setPixelRatio( window.devicePixelRatio );
+			renderer.setSize( window.innerWidth, window.innerHeight );
+			document.body.appendChild( renderer.domElement );
+
+			const controls = new OrbitControls( camera, renderer.domElement );
+			controls.minDistance = 3;
+			controls.maxDistance = 12;
+			controls.target.set( 0, 1, 0 );
+			controls.maxPolarAngle = Math.PI / 2;
+
+			const gui = new GUI();
+			gui.add( helpers, 'visible' ).name( 'helpers' );
+
+			//
+
+			function getSource( sourceModel ) {
+
+				const clip = sourceModel.animations[ 0 ]
+
+				const helper = new THREE.SkeletonHelper( sourceModel.scene );
+				helpers.add( helper );
+
+				const skeleton = new THREE.Skeleton( helper.bones );
+
+				const mixer = new THREE.AnimationMixer( sourceModel.scene );
+				mixer.clipAction( sourceModel.animations[ 0 ] ).play();
+
+				return { clip, skeleton, mixer };
+
+			}
+
+			function retargetModel( sourceModel, targetModel ) {
+
+				const targetSkin = targetModel.scene.children[ 0 ].children[ 0 ];
+
+				const targetSkelHelper = new THREE.SkeletonHelper( targetModel.scene );
+				helpers.add( targetSkelHelper );
+
+				const rotateCW45 = new THREE.Matrix4().makeRotationY( THREE.MathUtils.degToRad( 45 ) );
+				const rotateCCW180 = new THREE.Matrix4().makeRotationY( THREE.MathUtils.degToRad( - 180 ) );
+				const rotateCW180 = new THREE.Matrix4().makeRotationY( THREE.MathUtils.degToRad( 180 ) );
+				const rotateFoot = new THREE.Matrix4().makeRotationFromEuler( new THREE.Euler( THREE.MathUtils.degToRad( 45 ), THREE.MathUtils.degToRad( 180 ), THREE.MathUtils.degToRad( 0 ) ) );
+
+				const retargetOptions = {
+
+					preserveHipPosition: false, // lock XZ position of the hip bone to the origin
+
+					// specify the name of the source's hip bone.
+					hip: 'mixamorigHips',
+
+					// preserve the scale of the target model
+					scale: 1 / targetModel.scene.scale.y,
+
+					// offset target bones -> { targetBoneName: offsetMatrix }
+					localOffsets: {
+
+						mixamorigLeftShoulder: rotateCW45,
+						mixamorigRightShoulder: rotateCCW180,
+						mixamorigLeftArm: rotateCW45,
+						mixamorigRightArm: rotateCCW180,
+						mixamorigLeftForeArm: rotateCW45,
+						mixamorigRightForeArm: rotateCCW180,
+						mixamorigLeftHand: rotateCW45,
+						mixamorigRightHand: rotateCCW180,
+
+						mixamorigLeftUpLeg: rotateCW180,
+						mixamorigRightUpLeg: rotateCW180,
+						mixamorigLeftLeg: rotateCW180,
+						mixamorigRightLeg: rotateCW180,
+						mixamorigLeftFoot: rotateFoot,
+						mixamorigRightFoot: rotateFoot,
+						mixamorigLeftToeBase: rotateCW180,
+						mixamorigRightToeBase: rotateCW180
+
+					},
+
+					// Map of target's bone names to source's bone names -> { targetBoneName: sourceBoneName }
+					names: {
+
+						mixamorigHips: 'mixamorigHips',
+
+						mixamorigSpine: 'mixamorigSpine',
+						mixamorigSpine2: 'mixamorigSpine2',
+						mixamorigHead: 'mixamorigHead',
+
+						mixamorigLeftShoulder: 'mixamorigLeftShoulder',
+						mixamorigRightShoulder: 'mixamorigRightShoulder',
+						mixamorigLeftArm: 'mixamorigLeftArm',
+						mixamorigRightArm: 'mixamorigRightArm',
+						mixamorigLeftForeArm: 'mixamorigLeftForeArm',
+						mixamorigRightForeArm: 'mixamorigRightForeArm',
+						mixamorigLeftHand: 'mixamorigLeftHand',
+						mixamorigRightHand: 'mixamorigRightHand',
+
+						mixamorigLeftUpLeg: 'mixamorigLeftUpLeg',
+						mixamorigRightUpLeg: 'mixamorigRightUpLeg',
+						mixamorigLeftLeg: 'mixamorigLeftLeg',
+						mixamorigRightLeg: 'mixamorigRightLeg',
+						mixamorigLeftFoot: 'mixamorigLeftFoot',
+						mixamorigRightFoot: 'mixamorigRightFoot',
+						mixamorigLeftToeBase: 'mixamorigLeftToeBase',
+						mixamorigRightToeBase: 'mixamorigRightToeBase'
+
+					}
+
+				};
+
+				const retargetedClip = SkeletonUtils.retargetClip( targetSkin, sourceModel.skeleton, sourceModel.clip, retargetOptions );
+
+				// Apply the mixer directly to the SkinnedMesh, not any
+				// ancestor node, because that's what
+				// SkeletonUtils.retargetClip outputs the clip to be
+				// compatible with.
+				const mixer = new THREE.AnimationMixer( targetSkin );
+				mixer.clipAction( retargetedClip ).play();
+
+				return mixer;
+
+			}
+
+			window.onresize = function () {
+
+				camera.aspect = window.innerWidth / window.innerHeight;
+				camera.updateProjectionMatrix();
+
+				renderer.setSize( window.innerWidth, window.innerHeight );
+
+			};
+
+			function animate() {
+
+				const delta = clock.getDelta();
+
+				source.mixer.update( delta );
+				mixer.update( delta );
+
+				controls.update();
+
+				stats.update();
+
+				renderer.render( scene, camera );
+
+			}
+
+		</script>
+	</body>
+</html>