Răsfoiți Sursa

WebXRManager: Add Raw Camera Access module. (#31487)

* WebXRManager: Add raw camera access module support

* Remove unused test variables

* Raw Texture class

* Improve example

* Flip bool

* Address deleteTexture changes

* Update RawTexture.js

Improve JSDoc.

* Update RawTexture.js

Remove opaque property.

* Update RawTexture.js

* Update WebXRDepthSensing.js

* Update WebXRManager.js

* Update WebXRManager.js

* Update WebXRDepthSensing.js

Fix typo.

---------

Co-authored-by: Michael Herzog <michael.herzog@human-interactive.org>
Mike Nisbet 5 luni în urmă
părinte
comite
95febf473c

+ 1 - 0
examples/files.json

@@ -475,6 +475,7 @@
 		"webaudio_visualizer"
 	],
 	"webxr": [
+		"webxr_ar_camera_access",
 		"webxr_ar_cones",
 		"webxr_ar_hittest",
 		"webxr_ar_lighting",

BIN
examples/screenshots/webxr_ar_camera_access.jpg


+ 138 - 0
examples/webxr_ar_camera_access.html

@@ -0,0 +1,138 @@
+<!DOCTYPE html>
+<html lang="en">
+	<head>
+		<title>three.js ar - camera access</title>
+		<meta charset="utf-8">
+		<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=no">
+		<link type="text/css" rel="stylesheet" href="main.css">
+	</head>
+	<body>
+
+		<div id="info">
+			<a href="https://threejs.org" target="_blank" rel="noopener">three.js</a> ar - camera access<br/>
+		</div>
+
+		<script type="importmap">
+			{
+				"imports": {
+					"three": "../build/three.module.js",
+					"three/addons/": "./jsm/"
+				}
+			}
+		</script>
+
+		<script type="module">
+
+			import * as THREE from 'three';
+			import { ARButton } from 'three/addons/webxr/ARButton.js';
+
+			const clock = new THREE.Clock();
+
+			let camera, scene, renderer;
+			let cube;
+
+			init();
+
+			async function init() {
+
+				const container = document.createElement( 'div' );
+				document.body.appendChild( container );
+
+				scene = new THREE.Scene();
+
+				camera = new THREE.PerspectiveCamera( 70, window.innerWidth / window.innerHeight, 0.01, 20 );
+
+				const defaultLight = new THREE.HemisphereLight( 0xffffff, 0xbbbbff, 3 );
+				defaultLight.position.set( 0.5, 1, 0.25 );
+				scene.add( defaultLight );
+
+				//
+
+				renderer = new THREE.WebGLRenderer( { antialias: true, alpha: true } );
+				renderer.setPixelRatio( window.devicePixelRatio );
+				renderer.setSize( window.innerWidth, window.innerHeight );
+				renderer.setAnimationLoop( animate );
+				renderer.xr.enabled = true;
+				container.appendChild( renderer.domElement );
+
+				//
+
+				document.body.appendChild( ARButton.createButton( renderer, { requiredFeatures: [ 'camera-access' ] } ) );
+
+				//
+
+				const boxGeometry = new THREE.BoxGeometry( 1, 1, 1 );
+				const boxMaterial = new THREE.MeshStandardMaterial();
+				cube = new THREE.Mesh( boxGeometry, boxMaterial );
+				cube.position.z = -3;
+
+				scene.add( cube );
+
+				//
+
+				window.addEventListener( 'resize', onWindowResize );
+
+			}
+
+			function onWindowResize() {
+
+				camera.aspect = window.innerWidth / window.innerHeight;
+				camera.updateProjectionMatrix();
+
+				renderer.setSize( window.innerWidth, window.innerHeight );
+
+			}
+
+			//
+
+			function setCameraTexture() {
+
+				if ( ! renderer.xr.getSession() ) {
+
+					if ( cube.material.map ) {
+
+						cube.material.map = null;
+						cube.material.needsUpdate = true;
+
+					}
+
+					return;
+
+				}
+
+				const frame = renderer.xr.getFrame();
+				const referenceSpace = renderer.xr.getReferenceSpace();
+
+				if ( !frame || !referenceSpace ) return;
+
+				const viewerPose = frame.getViewerPose( referenceSpace );
+
+				if ( ! viewerPose ) return;
+
+				const view = viewerPose.views.find( view => view.camera );
+				
+				const cameraTexture = renderer.xr.getCameraTexture( view.camera );
+
+				if ( cube.material.map === cameraTexture ) return;
+
+				cube.material.map = cameraTexture;
+				cube.material.needsUpdate = true;
+				
+			}
+
+			function animate() {
+
+				const delta = clock.getDelta();
+
+				setCameraTexture();
+
+				cube.rotation.x += delta;
+				cube.rotation.y += delta;
+
+				renderer.render( scene, camera );
+
+			}
+
+		</script>
+	</body>
+</html>

+ 5 - 1
src/renderers/webgl/WebGLTextures.js

@@ -515,7 +515,7 @@ function WebGLTextures( _gl, extensions, state, properties, capabilities, utils,
 
 		if ( texture.isVideoTexture ) updateVideoTexture( texture );
 
-		if ( texture.isRenderTargetTexture === false && texture.version > 0 && textureProperties.__version !== texture.version ) {
+		if ( texture.isRenderTargetTexture === false && texture.isRawTexture !== true && texture.version > 0 && textureProperties.__version !== texture.version ) {
 
 			const image = texture.image;
 
@@ -534,6 +534,10 @@ function WebGLTextures( _gl, extensions, state, properties, capabilities, utils,
 
 			}
 
+		} else if ( texture.isRawTexture ) {
+
+			textureProperties.__webglTexture = texture.sourceTexture ? texture.sourceTexture : null;
+
 		}
 
 		state.bindTexture( _gl.TEXTURE_2D, textureProperties.__webglTexture, _gl.TEXTURE0 + slot );

+ 6 - 10
src/renderers/webxr/WebXRDepthSensing.js

@@ -1,7 +1,7 @@
 import { PlaneGeometry } from '../../geometries/PlaneGeometry.js';
 import { ShaderMaterial } from '../../materials/ShaderMaterial.js';
 import { Mesh } from '../../objects/Mesh.js';
-import { Texture } from '../../textures/Texture.js';
+import { RawTexture } from '../../textures/RawTexture.js';
 
 const _occlusion_vertex = `
 void main() {
@@ -42,9 +42,9 @@ class WebXRDepthSensing {
 	constructor() {
 
 		/**
-		 * A texture representing the depth of the user's environment.
+		 * An opaque texture representing the depth of the user's environment.
 		 *
-		 * @type {?Texture}
+		 * @type {?RawTexture}
 		 */
 		this.texture = null;
 
@@ -74,18 +74,14 @@ class WebXRDepthSensing {
 	/**
 	 * Inits the depth sensing module
 	 *
-	 * @param {WebGLRenderer} renderer - The renderer.
 	 * @param {XRWebGLDepthInformation} depthData - The XR depth data.
 	 * @param {XRRenderState} renderState - The XR render state.
 	 */
-	init( renderer, depthData, renderState ) {
+	init( depthData, renderState ) {
 
 		if ( this.texture === null ) {
 
-			const texture = new Texture();
-
-			const texProps = renderer.properties.get( texture );
-			texProps.__webglTexture = depthData.texture;
+			const texture = new RawTexture( depthData.texture );
 
 			if ( ( depthData.depthNear !== renderState.depthNear ) || ( depthData.depthFar !== renderState.depthFar ) ) {
 
@@ -146,7 +142,7 @@ class WebXRDepthSensing {
 	/**
 	 * Returns a texture representing the depth of the user's environment.
 	 *
-	 * @return {?Texture} The depth texture.
+	 * @return {?RawTexture} The depth texture.
 	 */
 	getDepthTexture() {
 

+ 56 - 1
src/renderers/webxr/WebXRManager.js

@@ -9,6 +9,7 @@ import { WebGLAnimation } from '../webgl/WebGLAnimation.js';
 import { WebGLRenderTarget } from '../WebGLRenderTarget.js';
 import { WebXRController } from './WebXRController.js';
 import { DepthTexture } from '../../textures/DepthTexture.js';
+import { RawTexture } from '../../textures/RawTexture.js';
 import { DepthFormat, DepthStencilFormat, RGBAFormat, UnsignedByteType, UnsignedIntType, UnsignedInt248Type } from '../../constants.js';
 import { WebXRDepthSensing } from './WebXRDepthSensing.js';
 
@@ -52,6 +53,7 @@ class WebXRManager extends EventDispatcher {
 		let xrFrame = null;
 
 		const depthSensing = new WebXRDepthSensing();
+		const cameraAccessTextures = {};
 		const attributes = gl.getContextAttributes();
 
 		let initialRenderTarget = null;
@@ -232,6 +234,11 @@ class WebXRManager extends EventDispatcher {
 			_currentDepthFar = null;
 
 			depthSensing.reset();
+			for ( const key in cameraAccessTextures ) {
+
+				delete cameraAccessTextures[ key ];
+
+			}
 
 			// restore framebuffer/rendering state
 
@@ -873,6 +880,19 @@ class WebXRManager extends EventDispatcher {
 
 		};
 
+		/**
+		 * Retrieves an opaque texture from the view-aligned {@link XRCamera}.
+		 * Only available during the current animation loop.
+		 *
+		 * @param {XRCamera} xrCamera - The camera to query.
+		 * @return {?Texture} An opaque texture representing the current raw camera frame.
+		 */
+		this.getCameraTexture = function ( xrCamera ) {
+
+			return cameraAccessTextures[ xrCamera ];
+
+		};
+
 		// Animation Loop
 
 		let onAnimationFrameCallback = null;
@@ -978,7 +998,42 @@ class WebXRManager extends EventDispatcher {
 
 					if ( depthData && depthData.isValid && depthData.texture ) {
 
-						depthSensing.init( renderer, depthData, session.renderState );
+						depthSensing.init( depthData, session.renderState );
+
+					}
+
+				}
+
+				const cameraAccessEnabled = enabledFeatures &&
+				    enabledFeatures.includes( 'camera-access' );
+
+				if ( cameraAccessEnabled ) {
+
+					renderer.state.unbindTexture();
+
+					if ( glBinding ) {
+
+						for ( let i = 0; i < views.length; i ++ ) {
+
+							const camera = views[ i ].camera;
+
+							if ( camera ) {
+
+								let cameraTex = cameraAccessTextures[ camera ];
+
+								if ( ! cameraTex ) {
+
+									cameraTex = new RawTexture();
+									cameraAccessTextures[ camera ] = cameraTex;
+
+								}
+
+								const glTexture = glBinding.getCameraImage( camera );
+								cameraTex.sourceTexture = glTexture;
+
+							}
+
+						}
 
 					}
 

+ 45 - 0
src/textures/RawTexture.js

@@ -0,0 +1,45 @@
+import { Texture } from './Texture.js';
+
+/**
+ * Represents a texture created externally from the renderer context.
+ *
+ * This may be a texture from a protected media stream, device camera feed,
+ * or other data feeds like a depth sensor.
+ *
+ * Note that this class is only supported in {@link WebGLRenderer} right now.
+ *
+ * @augments Texture
+ */
+class RawTexture extends Texture {
+
+	/**
+	 * Creates a new raw texture.
+	 *
+	 * @param {?WebGLTexture} [sourceTexture=null] - The external texture.
+	 */
+	constructor( sourceTexture = null ) {
+
+		super();
+
+		/**
+		 * The external source texture.
+		 *
+		 * @type {?WebGLTexture}
+		 * @default null
+		 */
+		this.sourceTexture = sourceTexture;
+
+		/**
+		 * This flag can be used for type testing.
+		 *
+		 * @type {boolean}
+		 * @readonly
+		 * @default true
+		 */
+		this.isRawTexture = true;
+
+	}
+
+}
+
+export { RawTexture };

粤ICP备19079148号