TSL Transpiler: Introduce `Linker` and improvements (#31314)

* updates

* cleanup

* fix hasAssignment

* fix float import

* fix negate

* improve numeric expression

examples/jsm/transpiler/AST.js

@@ -1,19 +1,119 @@
-export class Program {
+export class ASTNode {
 
 	constructor() {
 
-		this.body = [];
+		this.isASTNode = true;
+
+		this.linker = {
+			accesses: [],
+			assignments: []
+		};
+
+		this.parent = null;
+
+	}
+
+	hasAssignment() {
+
+		if ( this.isAssignment === true ) {
+
+			return true;
+
+		}
+
+		if ( this.parent === null ) {
+
+			return false;
+
+		}
+
+		return this.parent.hasAssignment();
+
+	}
+
+	getParent( parents = [] ) {
+
+		if ( this.parent === null ) {
+
+			return parents;
+
+		}
+
+		parents.push( this.parent );
+
+		return this.parent.getParent( parents );
+
+	}
+
+	initialize() {
+
+		for ( const key in this ) {
+
+			if ( this[ key ] && this[ key ].isASTNode ) {
+
+				this[ key ].parent = this;
+
+			} else if ( Array.isArray( this[ key ] ) ) {
+
+				const array = this[ key ];
+
+				for ( const item of array ) {
+
+					if ( item && item.isASTNode ) {
+
+						item.parent = this;
+
+					}
+
+				}
+
+			}
+
+		}
+
+	}
+
+}
+
+export class Comment extends ASTNode {
+
+	constructor( comment ) {
+
+		super();
+
+		this.comment = comment;
+
+		this.isComment = true;
+
+		this.initialize();
+
+	}
+
+}
+
+
+export class Program extends ASTNode {
+
+	constructor( body = [] ) {
+
+		super();
+
+		this.body = body;
 
 		this.isProgram = true;
 
+		this.initialize();
+
 	}
 
 }
 
-export class VariableDeclaration {
+export class VariableDeclaration extends ASTNode {
 
 	constructor( type, name, value = null, next = null, immutable = false ) {
 
+		super();
+
 		this.type = type;
 		this.name = name;
 		this.value = value;
@@ -23,40 +123,58 @@ export class VariableDeclaration {
 
 		this.isVariableDeclaration = true;
 
+		this.initialize();
+
+	}
+
+	get isAssignment() {
+
+		return this.value !== null;
+
 	}
 
 }
 
-export class Uniform {
+export class Uniform extends ASTNode {
 
 	constructor( type, name ) {
 
+		super();
+
 		this.type = type;
 		this.name = name;
 
 		this.isUniform = true;
 
+		this.initialize();
+
 	}
 
 }
 
-export class Varying {
+export class Varying extends ASTNode {
 
 	constructor( type, name ) {
 
+		super();
+
 		this.type = type;
 		this.name = name;
 
 		this.isVarying = true;
 
+		this.initialize();
+
 	}
 
 }
 
-export class FunctionParameter {
+export class FunctionParameter extends ASTNode {
 
 	constructor( type, name, qualifier = null, immutable = true ) {
 
+		super();
+
 		this.type = type;
 		this.name = name;
 		this.qualifier = qualifier;
@@ -64,271 +182,374 @@ export class FunctionParameter {
 
 		this.isFunctionParameter = true;
 
+		this.initialize();
+
 	}
 
 }
 
-export class FunctionDeclaration {
+export class FunctionDeclaration extends ASTNode {
 
-	constructor( type, name, params = [] ) {
+	constructor( type, name, params = [], body = [] ) {
+
+		super();
 
 		this.type = type;
 		this.name = name;
 		this.params = params;
-		this.body = [];
+		this.body = body;
 
 		this.isFunctionDeclaration = true;
 
+		this.initialize();
+
 	}
 
 }
 
-export class Expression {
+export class Expression extends ASTNode {
 
 	constructor( expression ) {
 
+		super();
+
 		this.expression = expression;
 
 		this.isExpression = true;
 
+		this.initialize();
+
 	}
 
 }
 
-export class Ternary {
+export class Ternary extends ASTNode {
 
 	constructor( cond, left, right ) {
 
+		super();
+
 		this.cond = cond;
 		this.left = left;
 		this.right = right;
 
 		this.isTernary = true;
 
+		this.initialize();
+
 	}
 
 }
 
-export class Operator {
+export class Operator extends ASTNode {
 
 	constructor( type, left, right ) {
 
+		super();
+
 		this.type = type;
 		this.left = left;
 		this.right = right;
 
 		this.isOperator = true;
 
+		this.initialize();
+
+	}
+
+	get isAssignment() {
+
+		return /^(=|\+=|-=|\*=|\/=|%=|<<=|>>=|>>>=|&=|\^=|\|=)$/.test( this.type );
+
 	}
 
 }
 
 
-export class Unary {
+export class Unary extends ASTNode {
 
 	constructor( type, expression, after = false ) {
 
+		super();
+
 		this.type = type;
 		this.expression = expression;
 		this.after = after;
 
 		this.isUnary = true;
 
+		this.initialize();
+
+	}
+
+	get isAssignment() {
+
+		return /^(\+\+|--)$/.test( this.type );
+
 	}
 
 }
 
-export class Number {
+export class Number extends ASTNode {
 
 	constructor( value, type = 'float' ) {
 
+		super();
+
 		this.type = type;
 		this.value = value;
 
 		this.isNumber = true;
 
+		this.initialize();
+
 	}
 
 }
 
-export class String {
+export class String extends ASTNode {
 
 	constructor( value ) {
 
+		super();
+
 		this.value = value;
 
 		this.isString = true;
 
+		this.initialize();
+
 	}
 
 }
 
 
-export class Conditional {
+export class Conditional extends ASTNode {
 
-	constructor( cond = null ) {
+	constructor( cond = null, body = [] ) {
 
-		this.cond = cond;
+		super();
 
-		this.body = [];
+		this.cond = cond;
+		this.body = body;
 		this.elseConditional = null;
 
 		this.isConditional = true;
 
+		this.initialize();
+
 	}
 
 }
 
-export class FunctionCall {
+export class FunctionCall extends ASTNode {
 
 	constructor( name, params = [] ) {
 
+		super();
+
 		this.name = name;
 		this.params = params;
 
 		this.isFunctionCall = true;
 
+		this.initialize();
+
 	}
 
 }
 
-export class Return {
+export class Return extends ASTNode {
 
 	constructor( value ) {
 
+		super();
+
 		this.value = value;
 
 		this.isReturn = true;
 
+		this.initialize();
+
 	}
 
 }
 
-export class Discard {
+export class Discard extends ASTNode {
 
 	constructor() {
 
+		super();
+
 		this.isDiscard = true;
 
+		this.initialize();
+
 	}
 
 }
 
-export class Continue {
+export class Continue extends ASTNode {
 
 	constructor() {
 
+		super();
+
 		this.isContinue = true;
 
+		this.initialize();
+
 	}
 
 }
 
-export class Break {
+export class Break extends ASTNode {
 
 	constructor() {
 
+		super();
+
 		this.isBreak = true;
 
+		this.initialize();
+
 	}
 
 }
 
-export class Accessor {
+export class Accessor extends ASTNode {
 
 	constructor( property ) {
 
+		super();
+
 		this.property = property;
 
 		this.isAccessor = true;
 
+		this.initialize();
+
 	}
 
 }
 
-export class StaticElement {
+export class StaticElement extends ASTNode {
 
 	constructor( value ) {
 
+		super();
+
 		this.value = value;
 
 		this.isStaticElement = true;
 
+		this.initialize();
+
 	}
 
 }
 
-export class DynamicElement {
+export class DynamicElement extends ASTNode {
 
 	constructor( value ) {
 
+		super();
+
 		this.value = value;
 
 		this.isDynamicElement = true;
 
+		this.initialize();
+
 	}
 
 }
 
-export class AccessorElements {
+export class AccessorElements extends ASTNode {
 
 	constructor( object, elements = [] ) {
 
+		super();
+
 		this.object = object;
 		this.elements = elements;
 
 		this.isAccessorElements = true;
 
+		this.initialize();
+
 	}
 
 }
 
-export class For {
+export class For extends ASTNode {
+
+	constructor( initialization, condition, afterthought, body = [] ) {
 
-	constructor( initialization, condition, afterthought ) {
+		super();
 
 		this.initialization = initialization;
 		this.condition = condition;
 		this.afterthought = afterthought;
-
-		this.body = [];
+		this.body = body;
 
 		this.isFor = true;
 
+		this.initialize();
+
 	}
 
 }
 
-export class Switch {
+export class While extends ASTNode {
 
-	constructor( discriminant ) {
+	constructor( condition, body = [] ) {
 
-		this.body = [];
+		super();
+
+		this.condition = condition;
+		this.body = body;
+
+		this.isWhile = true;
+
+		this.initialize();
+
+	}
+
+}
+
+
+export class Switch extends ASTNode {
+
+	constructor( discriminant, cases ) {
+
+		super();
 
 		this.discriminant = discriminant;
-		this.case = null;
+		this.cases = cases;
+
 		this.isSwitch = true;
 
+		this.initialize();
+
 	}
 
 }
 
-export class SwitchCase {
+export class SwitchCase extends ASTNode {
 
-	constructor( caseCondition ) {
+	constructor( body, conditions = null ) {
 
-		// Condition for the case body to execute
-		this.caseCondition = caseCondition;
+		super();
 
-		// Body of the case statement
-		this.body = [];
+		this.body = body;
+		this.conditions = conditions;
 
-		// Next case to fall to if current case fails
-		this.nextCase = null;
-
-		this.isDefault = caseCondition === null ? true : false;
+		this.isDefault = conditions === null ? true : false;
 		this.isSwitchCase = true;
 
+		this.initialize();
+
 	}
 
 }

examples/jsm/transpiler/GLSLDecoder.js

@@ -1,4 +1,4 @@
-import { Program, FunctionDeclaration, Switch, For, AccessorElements, Ternary, Varying, DynamicElement, StaticElement, FunctionParameter, Unary, Conditional, VariableDeclaration, Operator, Number, String, FunctionCall, Return, Accessor, Uniform, Discard, SwitchCase, Continue, Break } from './AST.js';
+import { Program, FunctionDeclaration, Switch, For, AccessorElements, Ternary, Varying, DynamicElement, StaticElement, FunctionParameter, Unary, Conditional, VariableDeclaration, Operator, Number, String, FunctionCall, Return, Accessor, Uniform, Discard, SwitchCase, Continue, Break, While, Comment } from './AST.js';
 
 const unaryOperators = [
 	'+', '-', '~', '!', '++', '--'
@@ -45,7 +45,7 @@ const samplers3D = [ 'sampler3D', 'isampler3D', 'usampler3D' ];
 const spaceRegExp = /^((\t| )\n*)+/;
 const lineRegExp = /^\n+/;
 const commentRegExp = /^\/\*[\s\S]*?\*\//;
-const inlineCommentRegExp = /^\/\/.*?(\n|$)/;
+const inlineCommentRegExp = /^\/\/.*?(?=\n|$)/;
 
 const numberRegExp = /^((0x\w+)|(\.?\d+\.?\d*((e-?\d+)|\w)?))/;
 const stringDoubleRegExp = /^(\"((?:[^"\\]|\\.)*)\")/;
@@ -67,7 +67,7 @@ function getFunctionName( str ) {
 function getGroupDelta( str ) {
 
 	if ( str === '(' || str === '[' || str === '{' ) return 1;
-	if ( str === ')' || str === ']' || str === '}' || str === 'case' || str === 'default' ) return - 1;
+	if ( str === ')' || str === ']' || str === '}' ) return - 1;
 
 	return 0;
 
@@ -84,7 +84,9 @@ class Token {
 		this.str = str;
 		this.pos = pos;
 
-		this.tag = null;
+		this.isTag = false;
+
+		this.tags = null;
 
 	}
 
@@ -193,7 +195,7 @@ class Tokenizer {
 
 	}
 
-	readToken() {
+	nextToken() {
 
 		const remainingCode = this.skip( spaceRegExp );
 
@@ -205,29 +207,42 @@ class Tokenizer {
 			if ( result ) {
 
 				const token = new Token( this, parser.type, result[ parser.group || 0 ], this.position );
+				token.isTag = parser.isTag;
 
 				this.position += result[ 0 ].length;
 
-				if ( parser.isTag ) {
+				return token;
 
-					const nextToken = this.readToken();
+			}
 
-					if ( nextToken ) {
+		}
 
-						nextToken.tag = token;
+	}
 
-					}
+	readToken() {
 
-					return nextToken;
+		let token = this.nextToken();
 
-				}
+		if ( token && token.isTag ) {
 
-				return token;
+			const tags = [];
+
+			while ( token.isTag ) {
+
+				tags.push( token );
+
+				token = this.nextToken();
+
+				if ( ! token ) return;
 
 			}
 
+			token.tags = tags;
+
 		}
 
+		return token;
+
 	}
 
 }
@@ -242,8 +257,6 @@ class GLSLDecoder {
 		this.tokenizer = null;
 		this.keywords = [];
 
-		this._currentFunction = null;
-
 		this.addPolyfill( 'gl_FragCoord', 'vec3 gl_FragCoord = vec3( screenCoordinate.x, screenCoordinate.y.oneMinus(), screenCoordinate.z );' );
 
 	}
@@ -332,7 +345,7 @@ class GLSLDecoder {
 				if ( ! token.isOperator || i === 0 || i === tokens.length - 1 ) return;
 
 				// important for negate operator after arithmetic operator: a * -1, a * -( b )
-				if ( ( inverse && arithmeticOperators.includes( tokens[ i - 1 ].str ) ) || ( ! inverse && arithmeticOperators.includes( tokens[ i + 1 ].str ) ) ) {
+				if ( inverse && arithmeticOperators.includes( tokens[ i - 1 ].str ) ) {
 
 					return;
 
@@ -357,7 +370,7 @@ class GLSLDecoder {
 						const left = this.parseExpressionFromTokens( tokens.slice( 0, i ) );
 						const right = this.parseExpressionFromTokens( tokens.slice( i + 1, tokens.length ) );
 
-						return this._evalOperator( new Operator( operator, left, right ) );
+						return new Operator( operator, left, right );
 
 					}
 
@@ -458,7 +471,7 @@ class GLSLDecoder {
 				const rightTokens = tokens.slice( leftTokens.length + 1 );
 				const right = this.parseExpressionFromTokens( rightTokens );
 
-				return this._evalOperator( new Operator( operator.str, left, right ) );
+				return new Operator( operator.str, left, right );
 
 			}
 
@@ -676,14 +689,9 @@ class GLSLDecoder {
 		const paramsTokens = this.readTokensUntil( ')' );
 
 		const params = this.parseFunctionParams( paramsTokens.slice( 1, paramsTokens.length - 1 ) );
+		const body = this.parseBlock();
 
-		const func = new FunctionDeclaration( type, name, params );
-
-		this._currentFunction = func;
-
-		this.parseBlock( func );
-
-		this._currentFunction = null;
+		const func = new FunctionDeclaration( type, name, params, body );
 
 		return func;
 
@@ -779,6 +787,31 @@ class GLSLDecoder {
 
 	}
 
+	parseWhile() {
+
+		this.readToken(); // skip 'while'
+
+		const conditionTokens = this.readTokensUntil( ')' ).slice( 1, - 1 );
+		const condition = this.parseExpressionFromTokens( conditionTokens );
+
+		let body;
+
+		if ( this.getToken().str === '{' ) {
+
+			body = this.parseBlock();
+
+		} else {
+
+			body = [ this.parseExpression() ];
+
+		}
+
+		const statement = new While( condition, body );
+
+		return statement;
+
+	}
+
 	parseFor() {
 
 		this.readToken(); // skip 'for'
@@ -804,123 +837,97 @@ class GLSLDecoder {
 		const condition = this.parseExpressionFromTokens( conditionTokens );
 		const afterthought = this.parseExpressionFromTokens( afterthoughtTokens );
 
-		const statement = new For( initialization, condition, afterthought );
+		let body;
 
 		if ( this.getToken().str === '{' ) {
 
-			this.parseBlock( statement );
+			body = this.parseBlock();
 
 		} else {
 
-			statement.body.push( this.parseExpression() );
+			body = [ this.parseExpression() ];
 
 		}
 
+		const statement = new For( initialization, condition, afterthought, body );
+
 		return statement;
 
 	}
 
 	parseSwitch() {
 
-		const parseSwitchExpression = () => {
+		this.readToken(); // Skip 'switch'
 
-			this.readToken(); // Skip 'switch'
+		const switchDeterminantTokens = this.readTokensUntil( ')' );
 
-			const switchDeterminantTokens = this.readTokensUntil( ')' );
+		// Parse expresison between parentheses. Index 1: char after '('. Index -1: char before ')'
+		const discriminant = this.parseExpressionFromTokens( switchDeterminantTokens.slice( 1, - 1 ) );
 
-			// Parse expresison between parentheses. Index 1: char after '('. Index -1: char before ')'
-			return this.parseExpressionFromTokens( switchDeterminantTokens.slice( 1, - 1 ) );
+		// Validate curly braces
+		if ( this.getToken().str !== '{' ) {
 
+			throw new Error( 'Expected \'{\' after switch(...) ' );
 
-		};
-
-		const parseSwitchBlock = ( switchStatement ) => {
-
-			// Validate curly braces
-			if ( this.getToken().str === '{' ) {
-
-				this.readToken(); // Skip '{'
-
-			} else {
-
-				throw new Error( 'Expected \'{\' after switch(...) ' );
-
-			}
-
-			if ( this.getToken() && ( this.getToken().str === 'case' || this.getToken().str === 'default' ) ) {
-
-				switchStatement.case = this.parseSwitchCase();
-
-			} else {
-
-				this.parseBlock( switchStatement );
-
-			}
-
+		}
 
-		};
+		this.readToken(); // Skip '{'
 
-		const switchStatement = new Switch( parseSwitchExpression() );
+		const cases = this.parseSwitchCases();
 
-		parseSwitchBlock( switchStatement );
+		const switchStatement = new Switch( discriminant, cases );
 
 		return switchStatement;
 
-
 	}
 
-	parseSwitchCase() {
-
-		const parseCaseExpression = ( token ) => {
+	parseSwitchCases() {
 
-			const caseTypeToken = token ? token : this.readToken(); // Skip 'case' or 'default
+		const cases = [];
 
-			const caseTokens = this.readTokensUntil( ':' );
+		let token = this.getToken();
+		let conditions = null;
 
-			// No case condition on default
-			if ( caseTypeToken.str === 'default' ) {
+		const isCase = ( token ) => token.str === 'case' || token.str === 'default';
 
-				return null;
+		while ( isCase( token ) ) {
 
-			}
+			this.readToken(); // Skip 'case' or 'default'
 
-			return this.parseExpressionFromTokens( caseTokens.slice( 0, - 1 ) );
+			if ( token.str === 'case' ) {
 
-		};
+				const caseTokens = this.readTokensUntil( ':' );
+				const caseStatement = this.parseExpressionFromTokens( caseTokens.slice( 0, - 1 ) );
 
-		let lastReadToken = null;
+				conditions = conditions || [];
+				conditions.push( caseStatement );
 
-		// No '{' so use different approach
-		const parseCaseBlock = ( caseStatement ) => {
+			} else {
 
-			lastReadToken = this.parseBlock( caseStatement );
+				this.readTokensUntil( ':' ); // Skip 'default:'
 
-		};
+				conditions = null;
 
-		// Parse case condition
-		const caseCondition = parseCaseExpression();
-		const switchCase = new SwitchCase( caseCondition );
+			}
 
-		// Get case body
-		parseCaseBlock( switchCase );
+			token = this.getToken();
 
-		let currentCase = switchCase;
+			if ( isCase( token ) ) {
 
-		// If block ended with case, then continue chaining cases, otherwise, ended with '}' and no more case blocks to parse
-		while ( lastReadToken.str === 'case' || lastReadToken.str === 'default' ) {
+				// If the next token is another case/default, continue parsing
+				continue;
 
-			const previousCase = currentCase;
+			}
 
-			// case and default already skipped at block end, so need to pass it in as last read token
-			currentCase = new SwitchCase( parseCaseExpression( lastReadToken ) );
+			cases.push( new SwitchCase( this.parseBlock(), conditions ) );
 
-			previousCase.nextCase = currentCase;
+			token = this.getToken();
 
-			parseCaseBlock( currentCase );
+			conditions = null;
 
 		}
 
-		return switchCase;
+		return cases;
 
 	}
 
@@ -936,25 +943,28 @@ class GLSLDecoder {
 
 		};
 
-		const parseIfBlock = ( cond ) => {
+		const parseIfBlock = () => {
+
+			let body;
 
 			if ( this.getToken().str === '{' ) {
 
-				this.parseBlock( cond );
+				body = this.parseBlock();
 
 			} else {
 
-				cond.body.push( this.parseExpression() );
+				body = [ this.parseExpression() ];
 
 			}
 
+			return body;
+
 		};
 
 		//
 
-		const conditional = new Conditional( parseIfExpression() );
-
-		parseIfBlock( conditional );
+		// Parse the first if statement
+		const conditional = new Conditional( parseIfExpression(), parseIfBlock() );
 
 		//
 
@@ -967,31 +977,31 @@ class GLSLDecoder {
 			// Assign the current if/else statement as the previous within the chain of conditionals
 			const previous = current;
 
+			let expression = null;
+
 			// If an 'else if' statement, parse the conditional within the if
 			if ( this.getToken().str === 'if' ) {
 
 				// Current conditional now equal to next conditional in the chain
-				current = new Conditional( parseIfExpression() );
-
-			} else {
-
-				current = new Conditional();
+				expression = parseIfExpression();
 
 			}
 
+			current = new Conditional( expression, parseIfBlock() );
+			current.parent = previous;
+
 			// n - 1 conditional's else statement assigned to new if/else statement
 			previous.elseConditional = current;
 
-			// Parse conditional of latest if statement
-			parseIfBlock( current );
-
 		}
 
 		return conditional;
 
 	}
 
-	parseBlock( scope ) {
+	parseBlock() {
+
+		const body = [];
 
 		const firstToken = this.getToken();
 
@@ -1011,17 +1021,47 @@ class GLSLDecoder {
 
 			groupIndex += getGroupDelta( token.str );
 
-			if ( groupIndex < 0 ) {
+			if ( groupIndex === 0 && ( token.str === 'case' || token.str === 'default' ) ) {
 
-				this.readToken(); // skip '}', ']', 'case', or other block ending tokens'
+				return body; // switch case or default statement, return body
 
-				// Return skipped token
-				return token;
+			} else if ( groupIndex < 0 ) {
+
+				this.readToken(); // skip '}'
+
+				return body;
 
 			}
 
 			//
 
+			if ( token.tags ) {
+
+				let lastStatement = null;
+
+				for ( const tag of token.tags ) {
+
+					if ( tag.type === Token.COMMENT ) {
+
+						const str = tag.str.replace( /\t/g, '' );
+
+						if ( ! lastStatement || lastStatement.isComment !== true ) {
+
+							lastStatement = new Comment( str );
+							body.push( lastStatement );
+
+						} else {
+
+							lastStatement.comment += '\n' + str;
+
+						}
+
+					}
+
+				}
+
+			}
+
 			if ( token.isLiteral || token.isOperator ) {
 
 				if ( token.str === 'const' ) {
@@ -1060,6 +1100,10 @@ class GLSLDecoder {
 
 					statement = this.parseFor();
 
+				} else if ( token.str === 'while' ) {
+
+					statement = this.parseWhile();
+
 				} else if ( token.str === 'switch' ) {
 
 					statement = this.parseSwitch();
@@ -1074,7 +1118,7 @@ class GLSLDecoder {
 
 			if ( statement ) {
 
-				scope.body.push( statement );
+				body.push( statement );
 
 			} else {
 
@@ -1084,43 +1128,7 @@ class GLSLDecoder {
 
 		}
 
-	}
-
-	_evalOperator( operator ) {
-
-		if ( operator.type.includes( '=' ) ) {
-
-			const parameter = this._getFunctionParameter( operator.left.property );
-
-			if ( parameter !== undefined ) {
-
-				// Parameters are immutable in WGSL
-
-				parameter.immutable = false;
-
-			}
-
-		}
-
-		return operator;
-
-	}
-
-	_getFunctionParameter( name ) {
-
-		if ( this._currentFunction ) {
-
-			for ( const param of this._currentFunction.params ) {
-
-				if ( param.name === name ) {
-
-					return param;
-
-				}
-
-			}
-
-		}
+		return body;
 
 	}
 
@@ -1147,9 +1155,8 @@ class GLSLDecoder {
 		this.index = 0;
 		this.tokenizer = new Tokenizer( polyfill + source ).tokenize();
 
-		const program = new Program();
-
-		this.parseBlock( program );
+		const body = this.parseBlock();
+		const program = new Program( body );
 
 		return program;
 

examples/jsm/transpiler/Linker.js

@@ -0,0 +1,325 @@
+class Block {
+
+	constructor( node, parent = null ) {
+
+		this.node = node;
+		this.parent = parent;
+
+		this.properties = {};
+
+	}
+
+	setProperty( name, value ) {
+
+		this.properties[ name ] = value;
+
+	}
+
+	getProperty( name ) {
+
+		let value = this.properties[ name ];
+
+		if ( value === undefined && this.parent !== null ) {
+
+			value = this.parent.getProperty( name );
+
+		}
+
+		return value;
+
+	}
+
+}
+
+class Linker {
+
+	constructor() {
+
+		this.block = null;
+
+	}
+
+	addBlock( node ) {
+
+		this.block = new Block( node, this.block );
+
+	}
+
+	removeBlock( node ) {
+
+		if ( this.block === null || this.block.node !== node ) {
+
+			throw new Error( 'No block to remove or block mismatch.' );
+
+		}
+
+		this.block = this.block.parent;
+
+	}
+
+	processVariables( node ) {
+
+		this.block.setProperty( node.name, node );
+
+		if ( node.value ) {
+
+			this.processExpression( node.value );
+
+		}
+
+	}
+
+	processUniform( node ) {
+
+		this.block.setProperty( node.name, node );
+
+	}
+
+	processVarying( node ) {
+
+		this.block.setProperty( node.name, node );
+
+	}
+
+	evalProperty( node ) {
+
+		let property = '';
+
+		if ( node.isAccessor ) {
+
+			property += node.property;
+
+		}
+
+		return property;
+
+	}
+
+	processExpression( node ) {
+
+		if ( node.isAccessor ) {
+
+			const property = this.block.getProperty( this.evalProperty( node ) );
+
+			if ( property ) {
+
+				node.linker.accesses.push( property );
+
+			}
+
+		} else if ( node.isNumber || node.isString ) {
+
+			// Process primitive values
+
+		} else if ( node.isOperator ) {
+
+			this.processExpression( node.left );
+			this.processExpression( node.right );
+
+			if ( node.isAssignment ) {
+
+				const property = this.block.getProperty( this.evalProperty( node.left ) );
+
+				if ( property ) {
+
+					property.linker.assignments.push( node );
+
+				}
+
+			}
+
+		} else if ( node.isFunctionCall ) {
+
+			for ( const param of node.params ) {
+
+				this.processExpression( param );
+
+			}
+
+		} else if ( node.isReturn ) {
+
+			if ( node.value ) this.processExpression( node.value );
+
+		} else if ( node.isDiscard || node.isBreak || node.isContinue ) {
+
+			// Process control flow
+
+		} else if ( node.isAccessorElements ) {
+
+			this.processExpression( node.object );
+
+			for ( const element of node.elements ) {
+
+				this.processExpression( element.value );
+
+			}
+
+		} else if ( node.isDynamicElement || node.isStaticElement ) {
+
+			this.processExpression( node.value );
+
+		} else if ( node.isFor || node.isWhile ) {
+
+			this.processForWhile( node );
+
+		} else if ( node.isSwitch ) {
+
+			this.processSwitch( node );
+
+		} else if ( node.isVariableDeclaration ) {
+
+			this.processVariables( node );
+
+		} else if ( node.isUniform ) {
+
+			this.processUniform( node );
+
+		} else if ( node.isVarying ) {
+
+			this.processVarying( node );
+
+		} else if ( node.isTernary ) {
+
+			this.processExpression( node.cond );
+			this.processExpression( node.left );
+			this.processExpression( node.right );
+
+		} else if ( node.isConditional ) {
+
+			this.processConditional( node );
+
+		} else if ( node.isUnary ) {
+
+			this.processExpression( node.expression );
+
+			if ( node.isAssignment ) {
+
+				if ( node.parent.hasAssignment() !== true ) {
+
+					// optimize increment/decrement operator
+					// to avoid creating a new variable
+
+					node.after = false;
+
+				}
+
+				const property = this.block.getProperty( this.evalProperty( node.expression ) );
+
+				if ( property ) {
+
+					property.linker.assignments.push( node );
+
+				}
+
+			}
+
+		}
+
+	}
+
+	processBody( body ) {
+
+		for ( const statement of body ) {
+
+			this.processExpression( statement );
+
+		}
+
+	}
+
+	processConditional( node ) {
+
+		this.processExpression( node.cond );
+		this.processBody( node.body );
+
+		let current = node;
+
+		while ( current.elseConditional ) {
+
+			if ( current.elseConditional.cond ) {
+
+				this.processExpression( current.elseConditional.cond );
+
+			}
+
+			this.processBody( current.elseConditional.body );
+
+			current = current.elseConditional;
+
+		}
+
+	}
+
+	processForWhile( node ) {
+
+		if ( node.initialization ) this.processExpression( node.initialization );
+		if ( node.condition ) this.processExpression( node.condition );
+		if ( node.afterthought ) this.processExpression( node.afterthought );
+
+		this.processBody( node.body );
+
+	}
+
+	processSwitch( switchNode ) {
+
+		this.processExpression( switchNode.discriminant );
+
+		for ( const switchCase of switchNode.cases ) {
+
+			if ( switchCase.isDefault !== true ) {
+
+				for ( const condition of switchCase.conditions ) {
+
+					this.processExpression( condition );
+
+				}
+
+			}
+
+			this.processBody( switchCase.body );
+
+		}
+
+	}
+
+	processFunction( node ) {
+
+		this.addBlock( node );
+
+		for ( const param of node.params ) {
+
+			this.block.setProperty( param.name, param );
+
+		}
+
+		this.processBody( node.body );
+
+		this.removeBlock( node );
+
+	}
+
+	process( ast ) {
+
+		this.addBlock( ast );
+
+		for ( const statement of ast.body ) {
+
+			if ( statement.isFunctionDeclaration ) {
+
+				this.processFunction( statement );
+
+			} else {
+
+				this.processExpression( statement );
+
+			}
+
+		}
+
+		this.removeBlock( ast );
+
+	}
+
+}
+
+export default Linker;

examples/jsm/transpiler/TSLEncoder.js

@@ -47,8 +47,35 @@ const unaryLib = {
 
 const textureLookupFunctions = [ 'texture', 'texture2D', 'texture3D', 'textureCube', 'textureLod', 'texelFetch', 'textureGrad' ];
 
+const isExpression = ( st ) => st.isFunctionDeclaration !== true && st.isFor !== true && st.isWhile !== true && st.isConditional !== true && st.isSwitch !== true;
 const isPrimitive = ( value ) => /^(true|false|-?(\d|\.\d))/.test( value );
 
+const isNumericExpression = ( node ) => {
+
+	if ( node.isNumber ) {
+
+		return true;
+
+	} else if ( node.isUnary ) {
+
+		if ( node.expression.isNumber ) {
+
+			return true;
+
+		}
+
+	} else if ( node.isOperator ) {
+
+		return isNumericExpression( node.left ) && isNumericExpression( node.right );
+
+	}
+
+	return false;
+
+
+}
+
+
 class TSLEncoder {
 
 	constructor() {
@@ -60,11 +87,6 @@ class TSLEncoder {
 		this.iife = false;
 		this.reference = false;
 
-		this._currentVariable = null;
-
-		this._currentProperties = {};
-		this._lastStatement = null;
-
 		this.block = null;
 
 	}
@@ -75,7 +97,7 @@ class TSLEncoder {
 
 		name = name.split( '.' )[ 0 ];
 
-		if ( TSL[ name ] !== undefined && this.global.has( name ) === false && this._currentProperties[ name ] === undefined ) {
+		if ( TSL[ name ] !== undefined && this.global.has( name ) === false ) {
 
 			this.imports.add( name );
 
@@ -133,29 +155,23 @@ class TSLEncoder {
 
 	}
 
-	emitExpression( node ) {
+	emitExpression( node, output = null ) {
 
 		let code;
 
 		if ( node.isAccessor ) {
 
-			this.addImport( node.property );
+			if ( node.linker.accesses.length === 0 ) {
 
-			code = node.property;
-
-		} else if ( node.isNumber ) {
+				this.addImport( node.property );
 
-			if ( node.type === 'int' || node.type === 'uint' ) {
-
-				code = node.type + '( ' + node.value + ' )';
-
-				this.addImport( node.type );
+			}
 
-			} else {
+			code = node.property;
 
-				code = node.value;
+		} else if ( node.isNumber ) {
 
-			}
+			code = node.value;
 
 		} else if ( node.isString ) {
 
@@ -165,10 +181,10 @@ class TSLEncoder {
 
 			const opFn = opLib[ node.type ] || node.type;
 
-			const left = this.emitExpression( node.left );
-			const right = this.emitExpression( node.right );
+			const left = this.emitExpression( node.left, output );
+			const right = this.emitExpression( node.right, output );
 
-			if ( isPrimitive( left ) && isPrimitive( right ) ) {
+			if ( isNumericExpression( node ) ) {
 
 				return left + ' ' + node.type + ' ' + right;
 
@@ -263,6 +279,7 @@ class TSLEncoder {
 		} else if ( node.isContinue ) {
 
 			this.addImport( 'Continue' );
+
 			code = 'Continue()';
 
 		} else if ( node.isAccessorElements ) {
@@ -305,6 +322,10 @@ class TSLEncoder {
 
 			code = this.emitFor( node );
 
+		} else if ( node.isWhile ) {
+
+			code = this.emitWhile( node );
+
 		} else if ( node.isSwitch ) {
 
 			code = this.emitSwitch( node );
@@ -329,26 +350,23 @@ class TSLEncoder {
 
 			code = this.emitConditional( node );
 
-		} else if ( node.isUnary && node.expression.isNumber ) {
+		} else if ( node.isUnary && node.expression.isNumber && node.type === '-' ) {
 
-			code = node.expression.type + '( ' + node.type + ' ' + node.expression.value + ' )';
+			code = '- ' + node.expression.value;
 
-			this.addImport( node.expression.type );
+			if ( node.expression.type !== 'float' ) {
 
-		} else if ( node.isUnary ) {
+				code = node.expression.type + '( ' + code + ' )';
 
-			let type = unaryLib[ node.type ];
+				this.addImport( node.expression.type );
 
-			if ( node.type === '++' || node.type === '--' ) {
+			}
 
-				if ( this._currentVariable === null ) {
+		} else if ( node.isUnary ) {
 
-					// optimize increment/decrement operator
-					// to avoid creating a new variable
+			let type = unaryLib[ node.type ];
 
-					node.after = false;
-
-				}
+			if ( node.hasAssignment() ) {
 
 				if ( node.after === false ) {
 
@@ -386,29 +404,34 @@ class TSLEncoder {
 
 	emitBody( body ) {
 
-		this.setLastStatement( null );
-
 		let code = '';
 
 		this.tab += '\t';
 
 		for ( const statement of body ) {
 
+			code += this.emitExtraLine( statement, body );
+
+			if ( statement.isComment ) {
+
+				code += this.emitComment( statement, body );
+
+				continue;
+
+			}
+
 			if ( this.block && this.block.isSwitchCase ) {
 
 				if ( statement.isBreak ) continue; // skip break statements in switch cases
 
 			}
 
-			code += this.emitExtraLine( statement );
 			code += this.tab + this.emitExpression( statement );
 
 			if ( code.slice( - 1 ) !== '}' ) code += ';';
 
 			code += '\n';
 
-			this.setLastStatement( statement );
-
 		}
 
 		code = code.slice( 0, - 1 ); // remove the last extra line
@@ -538,35 +561,31 @@ ${ this.tab }} )`;
 
 		let switchString = `Switch( ${ discriminantString } )\n${ this.tab }`;
 
-		let caseNode = switchNode.case;
-
 		const previousBlock = this.block;
 
-		while ( caseNode !== null ) {
+		for ( const switchCase of switchNode.cases ) {
 
-			this.block = caseNode;
+			this.block = switchCase;
 
 			let caseBodyString;
 
-			if ( ! caseNode.isDefault ) {
-
-				const caseConditions = [ this.emitExpression( caseNode.caseCondition ) ];
+			if ( ! switchCase.isDefault ) {
 
-				while ( caseNode.body.length === 0 && caseNode.nextCase !== null && caseNode.nextCase.isDefault !== true ) {
+				const caseConditions = [ ];
 
-					caseNode = caseNode.nextCase;
+				for ( const condition of switchCase.conditions ) {
 
-					caseConditions.push( this.emitExpression( caseNode.caseCondition ) );
+					caseConditions.push( this.emitExpression( condition ) );
 
 				}
 
-				caseBodyString = this.emitBody( caseNode.body );
+				caseBodyString = this.emitBody( switchCase.body );
 
 				switchString += `.Case( ${ caseConditions.join( ', ' ) }, `;
 
 			} else {
 
-				caseBodyString = this.emitBody( caseNode.body );
+				caseBodyString = this.emitBody( switchCase.body );
 
 				switchString += '.Default( ';
 
@@ -578,8 +597,6 @@ ${ caseBodyString }
 
 ${ this.tab }} )`;
 
-			caseNode = caseNode.nextCase;
-
 		}
 
 		this.block = previousBlock;
@@ -639,38 +656,58 @@ ${ this.tab }} )`;
 
 	}
 
-	emitVariables( node, isRoot = true ) {
+	emitWhile( node ) {
 
-		const { name, type, value, next } = node;
+		const condition = this.emitExpression( node.condition );
+
+		let whileStr = `Loop( ${ condition }, () => {\n\n`;
 
-		this._currentVariable = node;
+		whileStr += this.emitBody( node.body ) + '\n\n';
+
+		whileStr += this.tab + '} )';
+
+		this.imports.add( 'Loop' );
 
-		const valueStr = value ? this.emitExpression( value ) : '';
+		return whileStr;
+
+	}
+
+	emitVariables( node, isRoot = true ) {
+
+		const { name, type, value, next } = node;
 
 		let varStr = isRoot ? 'const ' : '';
 		varStr += name;
 
 		if ( value ) {
 
-			if ( value.isFunctionCall && value.name === type ) {
+			let valueStr = this.emitExpression( value );
 
-				varStr += ' = ' + valueStr;
+			if ( isNumericExpression( value ) ) {
 
-			} else {
+				// convert JS primitive to node
+
+				valueStr = `${ type }( ${ valueStr } )`;
 
-				varStr += ` = ${ type }( ${ valueStr } )`;
+				this.addImport( type );
 
 			}
 
-		} else {
+			if ( node.linker.assignments.length > 0 ) {
 
-			varStr += ` = ${ type }()`;
+				varStr += ' = ' + valueStr + '.toVar()';
 
-		}
+			} else {
+
+				varStr += ' = ' + valueStr;
+
+			}
+
+		} else {
 
-		if ( node.immutable === false ) {
+			varStr += ` = property( '${ type }' )`;
 
-			varStr += '.toVar()';
+			this.addImport( 'property' );
 
 		}
 
@@ -680,10 +717,6 @@ ${ this.tab }} )`;
 
 		}
 
-		this.addImport( type );
-
-		this._currentVariable = null;
-
 		return varStr;
 
 	}
@@ -715,8 +748,6 @@ ${ this.tab }} )`;
 
 		const { name, type } = node;
 
-		this._currentProperties = { name: node };
-
 		const params = [];
 		const inputs = [];
 		const mutableParams = [];
@@ -727,7 +758,7 @@ ${ this.tab }} )`;
 
 			let name = param.name;
 
-			if ( param.immutable === false && ( param.qualifier !== 'inout' && param.qualifier !== 'out' ) ) {
+			if ( param.linker.assignments.length > 0 ) {
 
 				name = name + '_immutable';
 
@@ -748,13 +779,15 @@ ${ this.tab }} )`;
 			inputs.push( param.name + ': \'' + param.type + '\'' );
 			params.push( name );
 
-			this._currentProperties[ name ] = param;
-
 		}
 
 		for ( const param of mutableParams ) {
 
-			node.body.unshift( new VariableDeclaration( param.type, param.name, new Accessor( param.name + '_immutable' ) ) );
+			const mutableParam = new VariableDeclaration( param.type, param.name, new Accessor( param.name + '_immutable' ), null, true );
+			mutableParam.parent = param.parent; // link to the original node
+			mutableParam.linker.assignments.push( mutableParam );
+
+			node.body.unshift( mutableParam );
 
 		}
 
@@ -814,21 +847,42 @@ ${ this.tab }}`;
 
 	}
 
-	setLastStatement( statement ) {
+	emitComment( statement, body ) {
+
+		const index = body.indexOf( statement );
+		const previous = body[ index - 1 ];
+		const next = body[ index + 1 ];
+
+		let output = '';
+
+		if ( previous && isExpression( previous ) ) {
+
+			output += '\n';
+
+		}
 
-		this._lastStatement = statement;
+		output += this.tab + statement.comment.replace( /\n/g, '\n' + this.tab ) + '\n';
+
+		if ( next && isExpression( next ) ) {
+
+			output += '\n';
+
+		}
+
+		return output;
 
 	}
 
-	emitExtraLine( statement ) {
+	emitExtraLine( statement, body ) {
+
+		const index = body.indexOf( statement );
+		const previous = body[ index - 1 ];
 
-		const last = this._lastStatement;
-		if ( last === null ) return '';
+		if ( previous === undefined ) return '';
 
 		if ( statement.isReturn ) return '\n';
 
-		const isExpression = ( st ) => st.isFunctionDeclaration !== true && st.isFor !== true && st.isConditional !== true && st.isSwitch !== true;
-		const lastExp = isExpression( last );
+		const lastExp = isExpression( previous );
 		const currExp = isExpression( statement );
 
 		if ( lastExp !== currExp || ( ! lastExp && ! currExp ) ) return '\n';
@@ -863,7 +917,15 @@ ${ this.tab }}`;
 
 		for ( const statement of ast.body ) {
 
-			code += this.emitExtraLine( statement );
+			code += this.emitExtraLine( statement, ast.body );
+
+			if ( statement.isComment ) {
+
+				code += this.emitComment( statement, ast.body );
+
+				continue;
+
+			}
 
 			if ( statement.isFunctionDeclaration ) {
 
@@ -875,8 +937,6 @@ ${ this.tab }}`;
 
 			}
 
-			this.setLastStatement( statement );
-
 		}
 
 		const imports = [ ...this.imports ];

examples/jsm/transpiler/Transpiler.js

@@ -1,3 +1,5 @@
+import Linker from './Linker.js';
+
 /**
  * A class that transpiles shader code from one language into another.
  *
@@ -32,6 +34,15 @@ class Transpiler {
 		 */
 		this.encoder = encoder;
 
+		/**
+		 * The linker. It processes the AST and resolves
+		 * variable and function references, ensuring that all
+		 * dependencies are properly linked.
+		 *
+		 * @type {Linker}
+		 */
+		this.linker = new Linker();
+
 	}
 
 	/**
@@ -42,7 +53,12 @@ class Transpiler {
 	 */
 	parse( source ) {
 
-		return this.encoder.emit( this.decoder.parse( source ) );
+		const ast = this.decoder.parse( source );
+
+		// Process the AST to resolve variable and function references and optimizations.
+		this.linker.process( ast );
+
+		return this.encoder.emit( ast );
 
 	}