From 68158dd5b328650aa30dedfe987320a34c5a5ec3 Mon Sep 17 00:00:00 2001 From: adroitwhiz Date: Sat, 16 Jul 2022 23:41:45 -0400 Subject: [PATCH 1/8] Replace SkinCache with WeakMap --- src/Renderer.js | 37 +++++++++++++++++++----- src/renderer/Skin.js | 1 - src/renderer/SkinCache.js | 59 --------------------------------------- 3 files changed, 30 insertions(+), 67 deletions(-) delete mode 100644 src/renderer/SkinCache.js diff --git a/src/Renderer.js b/src/Renderer.js index 29c682e..652412b 100644 --- a/src/Renderer.js +++ b/src/Renderer.js @@ -1,10 +1,13 @@ import Matrix from "./renderer/Matrix.js"; +import BitmapSkin from "./renderer/BitmapSkin.js"; import PenSkin from "./renderer/PenSkin.js"; +import SpeechBubbleSkin from "./renderer/SpeechBubbleSkin.js"; +import VectorSkin from "./renderer/VectorSkin.js"; import Rectangle from "./renderer/Rectangle.js"; import ShaderManager from "./renderer/ShaderManager.js"; -import SkinCache from "./renderer/SkinCache.js"; import { effectBitmasks } from "./renderer/effectInfo.js"; +import Costume from "./Costume.js"; import { Sprite, Stage } from "./Sprite.js"; export default class Renderer { @@ -22,7 +25,7 @@ export default class Renderer { } this._shaderManager = new ShaderManager(this); - this._skinCache = new SkinCache(this); + this._skins = new WeakMap(); this._currentShader = null; this._currentFramebuffer = null; @@ -63,6 +66,29 @@ export default class Renderer { ); } + // Retrieve a given object (e.g. costume or speech bubble)'s skin. If it doesn't exist, make one. + _getSkin(obj) { + if (this._skins.has(obj)) { + const skin = this._skins.get(obj); + return skin; + } else { + let skin; + + if (obj instanceof Costume) { + if (obj.isBitmap) { + skin = new BitmapSkin(this, obj.img); + } else { + skin = new VectorSkin(this, obj.img); + } + } else { + // If it's not a costume, assume it's a speech bubble. + skin = new SpeechBubbleSkin(this, obj); + } + this._skins.set(obj, skin); + return skin; + } + } + // Create a framebuffer info object, which contains the following: // * The framebuffer itself. // * The texture backing the framebuffer. @@ -291,10 +317,7 @@ export default class Renderer { gl.clearColor(1, 1, 1, 1); gl.clear(gl.COLOR_BUFFER_BIT); - // TODO: find a way to not destroy the skins of hidden sprites - this._skinCache.beginTrace(); this._renderLayers(); - this._skinCache.endTrace(); } createStage(w, h) { @@ -418,7 +441,7 @@ export default class Renderer { : 1; this._setSkinUniforms( - this._skinCache.getSkin(sprite.costume), + this._getSkin(sprite.costume), options.drawMode, this._calculateSpriteMatrix(sprite), spriteScale, @@ -437,7 +460,7 @@ export default class Renderer { sprite._speechBubble && sprite._speechBubble.text !== "" ) { - const speechBubbleSkin = this._skinCache.getSkin(sprite._speechBubble); + const speechBubbleSkin = this._getSkin(sprite._speechBubble); this._setSkinUniforms( speechBubbleSkin, diff --git a/src/renderer/Skin.js b/src/renderer/Skin.js index 8730280..064899f 100644 --- a/src/renderer/Skin.js +++ b/src/renderer/Skin.js @@ -2,7 +2,6 @@ export default class Skin { constructor(renderer) { this.renderer = renderer; this.gl = renderer.gl; - this.used = true; } // Get the skin's texture for a given (screen-space) scale. diff --git a/src/renderer/SkinCache.js b/src/renderer/SkinCache.js deleted file mode 100644 index 74e90e3..0000000 --- a/src/renderer/SkinCache.js +++ /dev/null @@ -1,59 +0,0 @@ -import BitmapSkin from "./BitmapSkin.js"; -import SpeechBubbleSkin from "./SpeechBubbleSkin.js"; -import VectorSkin from "./VectorSkin.js"; - -import Costume from "../Costume.js"; - -// This is a class which manages the creation and destruction of Skin objects. -// A Skin is the renderer's version of a "costume". It is backed by an image, but you render it by getting its texture. -// Different types of Skins can give you textures in different ways. -export default class SkinCache { - constructor(renderer) { - this._renderer = renderer; - this.gl = renderer.gl; - - this._skins = new Map(); - } - - // Begin GC tracing. Any skin retrieved and rendered during tracing will *not* be garbage-collected. - beginTrace() { - // Initialize by assuming no texture is used. - this._skins.forEach(skin => { - skin.used = false; - }); - } - - // End GC tracing. Any skin not retrieved since the tracing begun will be deleted. - endTrace() { - this._skins.forEach((skin, key) => { - if (!skin.used) { - skin.destroy(); - this._skins.delete(key); - } - }); - } - - // Retrieve a given object (e.g. costume or speech bubble)'s skin. If it doesn't exist, make one. - getSkin(obj) { - if (this._skins.has(obj)) { - const skin = this._skins.get(obj); - skin.used = true; - return skin; - } else { - let skin; - - if (obj instanceof Costume) { - if (obj.isBitmap) { - skin = new BitmapSkin(this._renderer, obj.img); - } else { - skin = new VectorSkin(this._renderer, obj.img); - } - } else { - // If it's not a costume, assume it's a speech bubble. - skin = new SpeechBubbleSkin(this._renderer, obj); - } - this._skins.set(obj, skin); - return skin; - } - } -} From 5ae4105b6c5be5fd28139e3b5033b826a678e837 Mon Sep 17 00:00:00 2001 From: adroitwhiz Date: Sat, 16 Jul 2022 23:58:06 -0400 Subject: [PATCH 2/8] Replace _setSkinUniforms with _renderSkin This fixes a WebGL warning when we tried to render skins that had no texture. Now, we return before trying to render them. --- src/Renderer.js | 92 ++++++++++++++++++++++++------------------------- 1 file changed, 45 insertions(+), 47 deletions(-) diff --git a/src/Renderer.js b/src/Renderer.js index 652412b..24dff1b 100644 --- a/src/Renderer.js +++ b/src/Renderer.js @@ -5,7 +5,7 @@ import SpeechBubbleSkin from "./renderer/SpeechBubbleSkin.js"; import VectorSkin from "./renderer/VectorSkin.js"; import Rectangle from "./renderer/Rectangle.js"; import ShaderManager from "./renderer/ShaderManager.js"; -import { effectBitmasks } from "./renderer/effectInfo.js"; +import { effectNames, effectBitmasks } from "./renderer/effectInfo.js"; import Costume from "./Costume.js"; import { Sprite, Stage } from "./Sprite.js"; @@ -245,14 +245,12 @@ export default class Renderer { ); Matrix.translate(penMatrix, penMatrix, -0.5, -0.5); - this._setSkinUniforms( + this._renderSkin( this._penSkin, options.drawMode, penMatrix, - 1, - null + 1 /* spriteScale */ ); - this.gl.drawArrays(this.gl.TRIANGLES, 0, 6); } // Sprites + clones @@ -339,36 +337,6 @@ export default class Renderer { return stage; } - _setSkinUniforms(skin, drawMode, matrix, scale, effects, effectMask) { - const gl = this.gl; - - const skinTexture = skin.getTexture(scale * this._screenSpaceScale); - if (!skinTexture) return; - - let effectBitmask = 0; - if (effects) effectBitmask = effects._bitmask; - if (typeof effectMask === "number") effectBitmask &= effectMask; - const shader = this._shaderManager.getShader(drawMode, effectBitmask); - this._setShader(shader); - gl.uniformMatrix3fv(shader.uniforms.u_transform, false, matrix); - - if (effectBitmask !== 0) { - for (const effect of Object.keys(effects._effectValues)) { - const effectVal = effects._effectValues[effect]; - if (effectVal !== 0) - gl.uniform1f(shader.uniforms[`u_${effect}`], effectVal); - } - - // Pixelate effect needs the skin size - if (effects._effectValues.pixelate !== 0) - gl.uniform2f(shader.uniforms.u_skinSize, skin.width, skin.height); - } - - gl.bindTexture(gl.TEXTURE_2D, skinTexture); - // All textures are bound to texture unit 0, so that's where the texture sampler should point - gl.uniform1i(shader.uniforms.u_texture, 0); - } - // Calculate the transform matrix for a sprite. // TODO: store the transform matrix in the sprite itself. That adds some complexity though, // so it's better off in another PR. @@ -435,25 +403,57 @@ export default class Renderer { return m; } + _renderSkin(skin, drawMode, matrix, scale, effects, effectMask, colorMask) { + const gl = this.gl; + + const skinTexture = skin.getTexture(scale * this._screenSpaceScale); + // Skip rendering the skin if it has no texture. + if (!skinTexture) return; + + let effectBitmask = effects ? effects._bitmask : 0; + if (typeof effectMask === "number") effectBitmask &= effectMask; + const shader = this._shaderManager.getShader(drawMode, effectBitmask); + this._setShader(shader); + gl.uniformMatrix3fv(shader.uniforms.u_transform, false, matrix); + + if (effectBitmask !== 0) { + for (const effect of effectNames) { + const effectVal = effects._effectValues[effect]; + if (effectVal !== 0) + gl.uniform1f(shader.uniforms[`u_${effect}`], effectVal); + } + + // Pixelate effect needs the skin size + if (effects._effectValues.pixelate !== 0) + gl.uniform2f(shader.uniforms.u_skinSize, skin.width, skin.height); + } + + gl.bindTexture(gl.TEXTURE_2D, skinTexture); + // All textures are bound to texture unit 0, so that's where the texture sampler should point + gl.uniform1i(shader.uniforms.u_texture, 0); + + // Enable color masking mode if set + if (Array.isArray(colorMask)) + this.gl.uniform4fv(this._currentShader.uniforms.u_colorMask, colorMask); + + // Actually draw the skin + this.gl.drawArrays(this.gl.TRIANGLES, 0, 6); + } + renderSprite(sprite, options) { const spriteScale = Object.prototype.hasOwnProperty.call(sprite, "size") ? sprite.size / 100 : 1; - this._setSkinUniforms( + this._renderSkin( this._getSkin(sprite.costume), options.drawMode, this._calculateSpriteMatrix(sprite), spriteScale, sprite.effects, - options.effectMask + options.effectMask, + options.colorMask ); - if (Array.isArray(options.colorMask)) - this.gl.uniform4fv( - this._currentShader.uniforms.u_colorMask, - options.colorMask - ); - this.gl.drawArrays(this.gl.TRIANGLES, 0, 6); if ( options.renderSpeechBubbles && @@ -462,14 +462,12 @@ export default class Renderer { ) { const speechBubbleSkin = this._getSkin(sprite._speechBubble); - this._setSkinUniforms( + this.renderSkin( speechBubbleSkin, options.drawMode, this._calculateSpeechBubbleMatrix(sprite, speechBubbleSkin), - 1, - null + 1 /* spriteScale */ ); - this.gl.drawArrays(this.gl.TRIANGLES, 0, 6); } } From 3e008087a731bc42deb497584d6fc8b7d0790d16 Mon Sep 17 00:00:00 2001 From: adroitwhiz Date: Sun, 17 Jul 2022 00:32:34 -0400 Subject: [PATCH 3/8] Cache sprite-specific data in Drawable class --- src/Renderer.js | 88 +++++++++++++--------------------------- src/renderer/Drawable.js | 81 ++++++++++++++++++++++++++++++++++++ src/renderer/Matrix.js | 18 ++++++-- 3 files changed, 124 insertions(+), 63 deletions(-) create mode 100644 src/renderer/Drawable.js diff --git a/src/Renderer.js b/src/Renderer.js index 24dff1b..9db70dc 100644 --- a/src/Renderer.js +++ b/src/Renderer.js @@ -1,4 +1,5 @@ import Matrix from "./renderer/Matrix.js"; +import Drawable from "./renderer/Drawable.js"; import BitmapSkin from "./renderer/BitmapSkin.js"; import PenSkin from "./renderer/PenSkin.js"; import SpeechBubbleSkin from "./renderer/SpeechBubbleSkin.js"; @@ -8,7 +9,6 @@ import ShaderManager from "./renderer/ShaderManager.js"; import { effectNames, effectBitmasks } from "./renderer/effectInfo.js"; import Costume from "./Costume.js"; -import { Sprite, Stage } from "./Sprite.js"; export default class Renderer { constructor(project, renderTarget) { @@ -25,6 +25,7 @@ export default class Renderer { } this._shaderManager = new ShaderManager(this); + this._drawables = new WeakMap(); this._skins = new WeakMap(); this._currentShader = null; @@ -69,24 +70,33 @@ export default class Renderer { // Retrieve a given object (e.g. costume or speech bubble)'s skin. If it doesn't exist, make one. _getSkin(obj) { if (this._skins.has(obj)) { - const skin = this._skins.get(obj); - return skin; - } else { - let skin; - - if (obj instanceof Costume) { - if (obj.isBitmap) { - skin = new BitmapSkin(this, obj.img); - } else { - skin = new VectorSkin(this, obj.img); - } + return this._skins.get(obj); + } + + let skin; + + if (obj instanceof Costume) { + if (obj.isBitmap) { + skin = new BitmapSkin(this, obj.img); } else { - // If it's not a costume, assume it's a speech bubble. - skin = new SpeechBubbleSkin(this, obj); + skin = new VectorSkin(this, obj.img); } - this._skins.set(obj, skin); - return skin; + } else { + // If it's not a costume, assume it's a speech bubble. + skin = new SpeechBubbleSkin(this, obj); } + this._skins.set(obj, skin); + return skin; + } + + // Retrieve the renderer-specific data object for a given sprite or clone. If it doesn't exist, make one. + _getDrawable(sprite) { + if (this._drawables.has(sprite)) { + return this._drawables.get(sprite); + } + const drawable = new Drawable(this, sprite); + this._drawables.set(sprite, drawable); + return drawable; } // Create a framebuffer info object, which contains the following: @@ -337,48 +347,6 @@ export default class Renderer { return stage; } - // Calculate the transform matrix for a sprite. - // TODO: store the transform matrix in the sprite itself. That adds some complexity though, - // so it's better off in another PR. - _calculateSpriteMatrix(spr) { - // These transforms are actually in reverse order because lol matrices - const m = Matrix.create(); - if (!(spr instanceof Stage)) { - Matrix.translate(m, m, spr.x, spr.y); - switch (spr.rotationStyle) { - case Sprite.RotationStyle.ALL_AROUND: { - Matrix.rotate(m, m, spr.scratchToRad(spr.direction)); - break; - } - case Sprite.RotationStyle.LEFT_RIGHT: { - if (spr.direction < 0) Matrix.scale(m, m, -1, 1); - break; - } - } - - const spriteScale = spr.size / 100; - Matrix.scale(m, m, spriteScale, spriteScale); - } - - const scalingFactor = 1 / spr.costume.resolution; - // Rotation centers are in non-Scratch space (positive y-values = down), - // but these transforms are in Scratch space (negative y-values = down). - Matrix.translate( - m, - m, - -spr.costume.center.x * scalingFactor, - (spr.costume.center.y - spr.costume.height) * scalingFactor - ); - Matrix.scale( - m, - m, - spr.costume.width * scalingFactor, - spr.costume.height * scalingFactor - ); - - return m; - } - // Calculate the transform matrix for a speech bubble attached to a sprite. _calculateSpeechBubbleMatrix(spr, speechBubbleSkin) { const sprBounds = this.getBoundingBox(spr); @@ -448,7 +416,7 @@ export default class Renderer { this._renderSkin( this._getSkin(sprite.costume), options.drawMode, - this._calculateSpriteMatrix(sprite), + this._getDrawable(sprite).getMatrix(), spriteScale, sprite.effects, options.effectMask, @@ -472,7 +440,7 @@ export default class Renderer { } getBoundingBox(sprite) { - return Rectangle.fromMatrix(this._calculateSpriteMatrix(sprite)); + return Rectangle.fromMatrix(this._getDrawable(sprite).getMatrix()); } // Mask drawing in to only areas where this sprite is opaque. diff --git a/src/renderer/Drawable.js b/src/renderer/Drawable.js new file mode 100644 index 0000000..e404fb7 --- /dev/null +++ b/src/renderer/Drawable.js @@ -0,0 +1,81 @@ +import Matrix from "./Matrix.js"; + +import { Sprite, Stage } from "../Sprite.js"; + +// Renderer-specific data for an instance (the original or a clone) of a Sprite +export default class Drawable { + constructor(renderer, sprite) { + this._renderer = renderer; + this._sprite = sprite; + + // Transformation matrix for the sprite. + this._matrix = Matrix.create(); + this._calculateSpriteMatrix(); + } + + _calculateSpriteMatrix() { + const m = this._matrix; + Matrix.identity(m); + const spr = this._sprite; + if (!(spr instanceof Stage)) { + Matrix.translate(m, m, spr.x, spr.y); + switch (spr.rotationStyle) { + case Sprite.RotationStyle.ALL_AROUND: { + Matrix.rotate(m, m, spr.scratchToRad(spr.direction)); + break; + } + case Sprite.RotationStyle.LEFT_RIGHT: { + if (spr.direction < 0) Matrix.scale(m, m, -1, 1); + break; + } + } + + const spriteScale = spr.size / 100; + Matrix.scale(m, m, spriteScale, spriteScale); + } + + const scalingFactor = 1 / spr.costume.resolution; + // Rotation centers are in non-Scratch space (positive y-values = down), + // but these transforms are in Scratch space (negative y-values = down). + Matrix.translate( + m, + m, + -spr.costume.center.x * scalingFactor, + (spr.costume.center.y - spr.costume.height) * scalingFactor + ); + Matrix.scale( + m, + m, + spr.costume.width * scalingFactor, + spr.costume.height * scalingFactor + ); + + // Store the values we used to compute the matrix so we only recalculate + // the matrix when we really need to. + this._matrixX = this._sprite.x; + this._matrixY = this._sprite.y; + this._matrixRotation = this._sprite.direction; + this._matrixRotationStyle = this._sprite.rotationStyle; + this._matrixScale = this._sprite.scale; + this._matrixCostume = this._sprite.costume; + this._matrixCostumeLoaded = this._sprite.costume.img.complete; + } + + getMatrix() { + // If all the values we used to calculate the matrix haven't changed since + // we last calculated the matrix, we can just return the matrix as-is. + if ( + this._matrixX !== this._sprite.x || + this._matrixY !== this._sprite.y || + this._matrixRotation !== this._sprite.direction || + this._matrixRotationStyle !== this._sprite.rotationStyle || + this._matrixScale !== this._sprite.scale || + this._matrixCostume !== this._sprite.costume || + this._matrixCostumeLoaded !== this._sprite.costume.img.complete + ) { + this._calculateSpriteMatrix(); + } + + return this._matrix; + } +} diff --git a/src/renderer/Matrix.js b/src/renderer/Matrix.js index 6c8752b..2d12310 100644 --- a/src/renderer/Matrix.js +++ b/src/renderer/Matrix.js @@ -7,12 +7,24 @@ export default class Matrix { // Create a new 3x3 transform matrix, initialized to the identity matrix. static create() { const matrix = new Float32Array(9); - matrix[0] = 1; - matrix[4] = 1; - matrix[8] = 1; + Matrix.identity(matrix); return matrix; } + // Reset a matrix to the identity matrix + static identity(dst) { + dst[0] = 1; + dst[1] = 0; + dst[2] = 0; + dst[3] = 0; + dst[4] = 1; + dst[5] = 0; + dst[6] = 0; + dst[7] = 0; + dst[8] = 1; + return dst; + } + // Translate a matrix by the given X and Y values static translate(dst, src, x, y) { const a00 = src[0], From 581bf2503d56238f17163b14ab1353e4a6cef18a Mon Sep 17 00:00:00 2001 From: adroitwhiz Date: Sun, 17 Jul 2022 04:15:20 -0400 Subject: [PATCH 4/8] Implement tight bounding box and "touching edge" Unfortunately, handling this properly involves a lot of edge cases. We have to implement the bounding box correctly even for sprites with distortion effects, and the pixels' area must be accounted for. --- src/Renderer.js | 4 + src/Sprite.js | 11 + src/renderer/BitmapSkin.js | 18 ++ src/renderer/Drawable.js | 288 +++++++++++++++++++++++++-- src/renderer/Matrix.js | 9 + src/renderer/Skin.js | 6 + src/renderer/VectorSkin.js | 61 ++++-- src/renderer/effectTransformPoint.js | 77 +++++++ 8 files changed, 444 insertions(+), 30 deletions(-) create mode 100644 src/renderer/effectTransformPoint.js diff --git a/src/Renderer.js b/src/Renderer.js index 9db70dc..efe51d4 100644 --- a/src/Renderer.js +++ b/src/Renderer.js @@ -439,6 +439,10 @@ export default class Renderer { } } + getTightBoundingBox(sprite) { + return this._getDrawable(sprite).getTightBoundingBox(); + } + getBoundingBox(sprite) { return Rectangle.fromMatrix(this._getDrawable(sprite).getMatrix()); } diff --git a/src/Sprite.js b/src/Sprite.js index 5c001ca..f98272f 100644 --- a/src/Sprite.js +++ b/src/Sprite.js @@ -527,6 +527,17 @@ export class Sprite extends SpriteBase { }, fast ); + case "edge": { + const bounds = this._project.renderer.getTightBoundingBox(this); + const stageWidth = this.stage.width; + const stageHeight = this.stage.height; + return ( + bounds.left < -stageWidth / 2 || + bounds.right > stageWidth / 2 || + bounds.top > stageHeight / 2 || + bounds.bottom < -stageHeight / 2 + ); + } default: console.error( `Cannot find target "${target}" in "touching". Did you mean to pass a sprite class instead?` diff --git a/src/renderer/BitmapSkin.js b/src/renderer/BitmapSkin.js index f58d7a4..399508e 100644 --- a/src/renderer/BitmapSkin.js +++ b/src/renderer/BitmapSkin.js @@ -5,11 +5,29 @@ export default class BitmapSkin extends Skin { super(renderer); this._image = image; + this._imageData = null; this._texture = null; this._setSizeFromImage(image); } + getImageData() { + // Make sure to handle potentially non-loaded textures + if (!this._image.complete) return null; + + if (!this._imageData) { + const canvas = document.createElement("canvas"); + canvas.width = this._image.naturalWidth || this._image.width; + canvas.height = this._image.naturalHeight || this._image.height; + const ctx = canvas.getContext("2d"); + ctx.drawImage(this._image, 0, 0); + // Cache image data so we can reuse it + this._imageData = ctx.getImageData(0, 0, canvas.width, canvas.height); + } + + return this._imageData; + } + getTexture() { // Make sure to handle potentially non-loaded textures const image = this._image; diff --git a/src/renderer/Drawable.js b/src/renderer/Drawable.js index e404fb7..107b780 100644 --- a/src/renderer/Drawable.js +++ b/src/renderer/Drawable.js @@ -1,7 +1,54 @@ import Matrix from "./Matrix.js"; +import Rectangle from "./Rectangle.js"; +import effectTransformPoint from "./effectTransformPoint.js"; +import { effectBitmasks } from "./effectInfo.js"; + import { Sprite, Stage } from "../Sprite.js"; +// Returns the determinant of two vectors, the vector from A to B and the vector +// from A to C. If positive, it means AC is counterclockwise from AB. +// If negative, AC is clockwise from AB. +const determinant = (a, b, c) => { + return (b[0] - a[0]) * (c[1] - a[1]) - (b[1] - a[1]) * (c[0] - a[0]); +}; + +// Used to track whether a sprite's transform has changed since we last looked +// at it. +// TODO: store renderer-specific data on the sprite and have *it* set a +// "transform changed" flag. +class SpriteTransformDiff { + constructor(sprite) { + this._sprite = sprite; + this._unset = true; + this.update(); + } + + update() { + this._lastX = this._sprite.x; + this._lastY = this._sprite.y; + this._lastRotation = this._sprite.direction; + this._lastRotationStyle = this._sprite.rotationStyle; + this._lastScale = this._sprite.scale; + this._lastCostume = this._sprite.costume; + this._lastCostumeLoaded = this._sprite.costume.img.complete; + this._unset = false; + } + + get changed() { + return ( + this._lastX !== this._sprite.x || + this._lastY !== this._sprite.y || + this._lastRotation !== this._sprite.direction || + this._lastRotationStyle !== this._sprite.rotationStyle || + this._lastScale !== this._sprite.scale || + this._lastCostume !== this._sprite.costume || + this._lastCostumeLoaded !== this._sprite.costume.img.complete || + this._unset + ); + } +} + // Renderer-specific data for an instance (the original or a clone) of a Sprite export default class Drawable { constructor(renderer, sprite) { @@ -10,7 +57,230 @@ export default class Drawable { // Transformation matrix for the sprite. this._matrix = Matrix.create(); + // Track when the sprite's transform changes so we can recalculate the + // transform matrix. + this._matrixDiff = new SpriteTransformDiff(sprite); this._calculateSpriteMatrix(); + + // Track when the image data used to calculate the convex hull, + // or distortion effects that affect how it's drawn, change. + // We also need the image data to know how big the pixels are. + this._convexHullImageData = null; + this._convexHullMosaic = 0; + this._convexHullPixelate = 0; + this._convexHullWhirl = 0; + this._convexHullFisheye = 0; + this._convexHullPoints = null; + + this._aabb = new Rectangle(); + this._tightBoundingBox = new Rectangle(); + // Track when the sprite's transform changes so we can recalculate the + // tight bounding box. + this._convexHullMatrixDiff = new SpriteTransformDiff(sprite); + } + + getCurrentSkin() { + return this._renderer._getSkin(this._sprite.costume); + } + + // Get the rough axis-aligned bounding box for this sprite. Not as tight as + // getTightBoundingBox, especially when rotated. + getAABB() { + return Rectangle.fromMatrix(this.getMatrix(), this._aabb); + } + + // Get the Scratch-space tight bounding box for this sprite. + getTightBoundingBox() { + if (!this._convexHullMatrixDiff.changed) return this._tightBoundingBox; + + const matrix = this.getMatrix(); + const convexHullPoints = this._calculateConvexHull(); + // Maybe the costume isn't loaded yet. Return a 0x0 bounding box around the + // center of the sprite. + if (convexHullPoints === null) { + return Rectangle.fromBounds( + this._sprite.x, + this._sprite.y, + this._sprite.x, + this._sprite.y, + this._tightBoundingBox + ); + } + + let left = Infinity; + let right = -Infinity; + let top = -Infinity; + let bottom = Infinity; + const transformedPoint = [0, 0]; + + // Each convex hull point is the center of a pixel. However, said pixels + // each have area. We must take into account the size of the pixels when + // calculating the bounds. The pixel dimensions depend on the scale and + // rotation (as we're treating pixels as squares, which change dimensions + // when rotated). + const xa = matrix[0] / 2; + const xb = matrix[3] / 2; + const halfPixelX = + (Math.abs(xa) + Math.abs(xb)) / this._convexHullImageData.width; + const ya = matrix[1] / 2; + const yb = matrix[4] / 2; + const halfPixelY = + (Math.abs(ya) + Math.abs(yb)) / this._convexHullImageData.height; + + // Transform every point in the convex hull using our transform matrix, + // and expand the bounds to include that point. + for (let i = 0; i < convexHullPoints.length; i++) { + const point = convexHullPoints[i]; + transformedPoint[0] = point[0]; + transformedPoint[1] = 1 - point[1]; + Matrix.transformPoint(matrix, transformedPoint, transformedPoint); + + left = Math.min(left, transformedPoint[0] - halfPixelX); + right = Math.max(right, transformedPoint[0] + halfPixelX); + top = Math.max(top, transformedPoint[1] + halfPixelY); + bottom = Math.min(bottom, transformedPoint[1] - halfPixelY); + } + + Rectangle.fromBounds(left, right, bottom, top, this._tightBoundingBox); + this._convexHullMatrixDiff.update(); + return this._tightBoundingBox; + } + + _calculateConvexHull() { + const sprite = this._sprite; + const skin = this.getCurrentSkin(); + const imageData = skin.getImageData( + "size" in sprite ? sprite.size / 100 : 1 + ); + if (!imageData) return null; + + // We only need to recalculate the convex hull points if the image data's + // changed since we last calculated the convex hull, or if the sprite's + // effects which distort its shape have changed. + const { mosaic, pixelate, whirl, fisheye } = sprite.effects; + if ( + this._convexHullImageData === imageData && + this._convexHullMosaic === mosaic && + this._convexHullPixelate === pixelate && + this._convexHullWhirl === whirl && + this._convexHullFisheye === fisheye + ) { + return this._convexHullPoints; + } + + const effectBitmask = + sprite.effects._bitmask & + (effectBitmasks.mosaic | + effectBitmasks.pixelate | + effectBitmasks.whirl | + effectBitmasks.fisheye); + + const leftHull = []; + const rightHull = []; + + const { width, height, data } = imageData; + + const pixelPos = [0, 0]; + const effectPos = [0, 0]; + let currentPoint; + // Not Scratch-space: y increases as we go downwards + // Loop over all rows of pixels in the costume, starting at the top + for (let y = 0; y < height; y++) { + pixelPos[1] = (y + 0.5) / height; + + // We start at the leftmost point, then go rightwards until we hit an + // opaque pixel + let x = 0; + for (; x < width; x++) { + pixelPos[0] = (x + 0.5) / width; + let pixelX = x; + let pixelY = y; + if (effectBitmask !== 0) { + effectTransformPoint(this, pixelPos, effectPos); + pixelX = Math.floor(effectPos[0] * width); + pixelY = Math.floor(effectPos[1] * height); + } + // We hit an opaque pixel + if (data[(pixelY * width + pixelX) * 4 + 3] > 0) { + currentPoint = [pixelPos[0], pixelPos[1]]; + break; + } + } + + // There are no opaque pixels on this row. Go to the next one. + if (x >= width) continue; + + // If appending the current point to the left hull makes a + // counterclockwise turn, we want to append the current point to it. + // Otherwise, we remove hull points until the current point makes a + // counterclockwise turn with the last two points. + while (leftHull.length >= 2) { + if ( + determinant( + leftHull[leftHull.length - 1], + leftHull[leftHull.length - 2], + currentPoint + ) > 0 + ) { + break; + } + + leftHull.pop(); + } + + leftHull.push(currentPoint); + + // Now we repeat the process for the right side, looking leftwards for an + // opaque pixel. + for (x = width - 1; x >= 0; x--) { + pixelPos[0] = (x + 0.5) / width; + effectTransformPoint(this, pixelPos, effectPos); + let pixelX = x; + let pixelY = y; + if (effectBitmask !== 0) { + effectTransformPoint(this, pixelPos, effectPos); + pixelX = Math.floor(effectPos[0] * width); + pixelY = Math.floor(effectPos[1] * height); + } + // We hit an opaque pixel + if (data[(pixelY * width + pixelX) * 4 + 3] > 0) { + currentPoint = [pixelPos[0], pixelPos[1]]; + break; + } + } + + // Because we're coming at this from the right, it goes clockwise. + while (rightHull.length >= 2) { + if ( + determinant( + rightHull[rightHull.length - 1], + rightHull[rightHull.length - 2], + currentPoint + ) < 0 + ) { + break; + } + + rightHull.pop(); + } + + rightHull.push(currentPoint); + } + + // Add points from the right side in reverse order so all the points are + // clockwise. + for (let i = rightHull.length - 1; i >= 0; i--) { + leftHull.push(rightHull[i]); + } + + this._convexHullPoints = leftHull; + this._convexHullMosaic = mosaic; + this._convexHullPixelate = pixelate; + this._convexHullWhirl = whirl; + this._convexHullFisheye = fisheye; + this._convexHullImageData = imageData; + + return this._convexHullPoints; } _calculateSpriteMatrix() { @@ -52,27 +322,13 @@ export default class Drawable { // Store the values we used to compute the matrix so we only recalculate // the matrix when we really need to. - this._matrixX = this._sprite.x; - this._matrixY = this._sprite.y; - this._matrixRotation = this._sprite.direction; - this._matrixRotationStyle = this._sprite.rotationStyle; - this._matrixScale = this._sprite.scale; - this._matrixCostume = this._sprite.costume; - this._matrixCostumeLoaded = this._sprite.costume.img.complete; + this._matrixDiff.update(); } getMatrix() { // If all the values we used to calculate the matrix haven't changed since // we last calculated the matrix, we can just return the matrix as-is. - if ( - this._matrixX !== this._sprite.x || - this._matrixY !== this._sprite.y || - this._matrixRotation !== this._sprite.direction || - this._matrixRotationStyle !== this._sprite.rotationStyle || - this._matrixScale !== this._sprite.scale || - this._matrixCostume !== this._sprite.costume || - this._matrixCostumeLoaded !== this._sprite.costume.img.complete - ) { + if (this._matrixDiff.changed) { this._calculateSpriteMatrix(); } diff --git a/src/renderer/Matrix.js b/src/renderer/Matrix.js index 2d12310..44fae75 100644 --- a/src/renderer/Matrix.js +++ b/src/renderer/Matrix.js @@ -94,4 +94,13 @@ export default class Matrix { dst[8] = src[8]; return dst; } + + // Transform a 2D point by the given matrix + static transformPoint(m, dst, src) { + const x = src[0]; + const y = src[1]; + dst[0] = m[0] * x + m[3] * y + m[6]; + dst[1] = m[1] * x + m[4] * y + m[7]; + return dst; + } } diff --git a/src/renderer/Skin.js b/src/renderer/Skin.js index 064899f..dc709f6 100644 --- a/src/renderer/Skin.js +++ b/src/renderer/Skin.js @@ -10,6 +10,12 @@ export default class Skin { return null; } + // Get the skin image's ImageData at a given (screen-space) scale. + // eslint-disable-next-line no-unused-vars + getImageData(scale) { + throw new Error("getImageData not implemented for this skin type"); + } + // Helper function to create a texture from an image and handle all the boilerplate. _makeTexture(image, filtering) { const gl = this.gl; diff --git a/src/renderer/VectorSkin.js b/src/renderer/VectorSkin.js index 28ee5f6..21210ef 100644 --- a/src/renderer/VectorSkin.js +++ b/src/renderer/VectorSkin.js @@ -10,6 +10,9 @@ export default class VectorSkin extends Skin { this._image = image; this._canvas = document.createElement("canvas"); + this._imageDataMipLevel = 0; + this._imageData = null; + this._maxTextureSize = renderer.gl.getParameter( renderer.gl.MAX_TEXTURE_SIZE ); @@ -19,14 +22,31 @@ export default class VectorSkin extends Skin { this._mipmaps = new Map(); } - // TODO: handle proper subpixel positioning when SVG viewbox has non-integer coordinates - // This will require rethinking costume + project loading probably - _createMipmap(mipLevel) { - const scale = 2 ** (mipLevel - MIPMAP_OFFSET); + static mipLevelForScale(scale) { + return Math.max(Math.ceil(Math.log2(scale)) + MIPMAP_OFFSET, 0); + } - // Instead of uploading the image to WebGL as a texture, render the image to a canvas and upload the canvas. - const canvas = this._canvas; - const ctx = canvas.getContext("2d"); + getImageData(scale) { + if (!this._image.complete) return null; + + // Round off the scale of the image data drawn to a given power-of-two mip level. + const mipLevel = VectorSkin.mipLevelForScale(scale); + if (!this._imageData || this._imageDataMipLevel !== mipLevel) { + const canvas = this._drawSvgToCanvas(mipLevel); + if (canvas === null) return null; + + // Cache image data so we can reuse it + this._imageData = canvas + .getContext("2d") + .getImageData(0, 0, canvas.width, canvas.height); + this._imageDataMipLevel = mipLevel; + } + + return this._imageData; + } + + _drawSvgToCanvas(mipLevel) { + const scale = 2 ** (mipLevel - MIPMAP_OFFSET); const image = this._image; let width = image.naturalWidth * scale; @@ -37,22 +57,35 @@ export default class VectorSkin extends Skin { // Prevent IndexSizeErrors if the image is too small to render if (width === 0 || height === 0) { - this._mipmaps.set(mipLevel, null); - return; + return null; } + // Instead of uploading the image to WebGL as a texture, render the image to a canvas and upload the canvas. + const canvas = this._canvas; + const ctx = canvas.getContext("2d"); + canvas.width = width; canvas.height = height; ctx.drawImage(image, 0, 0, width, height); + return this._canvas; + } - // Use linear (i.e. smooth) texture filtering for vectors - this._mipmaps.set(mipLevel, this._makeTexture(canvas, this.gl.LINEAR)); + // TODO: handle proper subpixel positioning when SVG viewbox has non-integer coordinates + // This will require rethinking costume + project loading probably + _createMipmap(mipLevel) { + // Instead of uploading the image to WebGL as a texture, render the image to a canvas and upload the canvas. + const canvas = this._drawSvgToCanvas(mipLevel); + this._mipmaps.set( + mipLevel, + // Use linear (i.e. smooth) texture filtering for vectors + // If the image is 0x0, we return null. Check for that. + canvas === null ? null : this._makeTexture(canvas, this.gl.LINEAR) + ); } getTexture(scale) { - const image = this._image; - if (!image.complete) return null; + if (!this._image.complete) return null; // Because WebGL doesn't support vector graphics, substitute a bunch of bitmaps. // This skin contains several renderings of its image at different scales. @@ -61,7 +94,7 @@ export default class VectorSkin extends Skin { // Math.ceil(Math.log2(scale)) means we use the "2x" texture at 1x-2x scale, the "4x" texture at 2x-4x scale, etc. // This means that one texture pixel will always be between 0.5x and 1x the size of one rendered pixel, // but never bigger than one rendered pixel--this prevents blurriness from blowing up the texture too much. - const mipLevel = Math.max(Math.ceil(Math.log2(scale)) + MIPMAP_OFFSET, 0); + const mipLevel = VectorSkin.mipLevelForScale(scale); if (!this._mipmaps.has(mipLevel)) this._createMipmap(mipLevel); return this._mipmaps.get(mipLevel); diff --git a/src/renderer/effectTransformPoint.js b/src/renderer/effectTransformPoint.js new file mode 100644 index 0000000..bb1590d --- /dev/null +++ b/src/renderer/effectTransformPoint.js @@ -0,0 +1,77 @@ +import { effectBitmasks } from "./effectInfo.js"; + +const CENTER = 0.5; +const EPSILON = 1e-3; + +// Transform a texture-space point using the effects defined on the given drawable. +const effectTransformPoint = (drawable, src, dst) => { + const { effects } = drawable._sprite; + const effectBitmask = effects._bitmask; + + dst[0] = src[0]; + dst[1] = src[1]; + + if ((effectBitmask & effectBitmasks.mosaic) !== 0) { + // float mosaicFactor = clamp(floor(abs(u_mosaic + 10.0) / 10.0 + 0.5), 1.0, 512.0); + const mosaicFactor = Math.max( + 1, + Math.min(Math.floor(Math.abs(effects.mosaic + 10) / 10 + 0.5), 512) + ); + // coord = fract(coord * mosaicFactor); + dst[0] = (mosaicFactor * dst[0]) % 1; + dst[1] = (mosaicFactor * dst[1]) % 1; + } + + if ((effectBitmask & effectBitmasks.pixelate) !== 0) { + // vec2 pixSize = u_skinSize / (abs(u_pixelate) * 0.1); + const skin = drawable.getCurrentSkin(); + const pixSizeX = skin.width / (Math.abs(effects.pixelate) * 0.1); + const pixSizeY = skin.height / (Math.abs(effects.pixelate) * 0.1); + // coord = (floor(coord * pixSize) + CENTER) / pixSize; + dst[0] = (Math.floor(dst[0] * pixSizeX) + CENTER) / pixSizeX; + dst[1] = (Math.floor(dst[1] * pixSizeY) + CENTER) / pixSizeY; + } + + if ((effectBitmask & effectBitmasks.whirl) !== 0) { + // const float PI_OVER_180 = 0.017453292519943295; + const PI_OVER_180 = 0.017453292519943295; + // vec2 offset = coord - CENTER; + const offsetX = dst[0] - CENTER; + const offsetY = dst[1] - CENTER; + // float whirlFactor = max(1.0 - (length(offset) * 2.0), 0.0); + const offsetLength = Math.sqrt(offsetX * offsetX + offsetY * offsetY); + const whirlFactor = Math.max(1 - offsetLength * 2, 0); + // float whirl = (-u_whirl * PI_OVER_180) * whirlFactor * whirlFactor; + const whirl = -effects.whirl * PI_OVER_180 * whirlFactor * whirlFactor; + // float s = sin(whirl); + // float c = cos(whirl); + const s = Math.sin(whirl); + const c = Math.cos(whirl); + // mat2 rotationMatrix = mat2(c, -s, s, c); + // coord = rotationMatrix * offset + CENTER; + dst[0] = c * offsetX + s * offsetY + CENTER; + dst[1] = -s * offsetX + c * offsetY + CENTER; + } + + if ((effectBitmask & effectBitmasks.fisheye) !== 0) { + // vec2 vec = (coord - CENTER) / CENTER; + const vecX = (dst[0] - CENTER) / CENTER; + const vecY = (dst[1] - CENTER) / CENTER; + // float len = length(vec) + epsilon; + const len = Math.sqrt(vecX * vecX + vecY * vecY) + EPSILON; + // float factor = max(0.0, (u_fisheye + 100.0) / 100.0); + const factor = Math.max(0, (effects.fisheye + 100) / 100); + // float r = pow(min(len, 1.0), factor) * max(1.0, len); + const r = Math.pow(Math.min(len, 1), factor) * Math.max(1, len); + // vec2 unit = vec / len; + const unitX = vecX / len; + const unitY = vecY / len; + // coord = CENTER + (r * unit * CENTER); + dst[0] = CENTER + r * unitX * CENTER; + dst[1] = CENTER + r * unitY * CENTER; + } + + return dst; +}; + +export default effectTransformPoint; From cd0f0d8106e7042dce8e6a800c6675efc052b200 Mon Sep 17 00:00:00 2001 From: adroitwhiz Date: Sun, 17 Jul 2022 17:38:20 -0400 Subject: [PATCH 5/8] Fix spriteScale comment --- src/Renderer.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Renderer.js b/src/Renderer.js index efe51d4..5afa3f6 100644 --- a/src/Renderer.js +++ b/src/Renderer.js @@ -259,7 +259,7 @@ export default class Renderer { this._penSkin, options.drawMode, penMatrix, - 1 /* spriteScale */ + 1 /* scale */ ); } From b3e8ec2cd11f478a7774ea3f9fe6cca8f896a487 Mon Sep 17 00:00:00 2001 From: adroitwhiz Date: Sun, 17 Jul 2022 19:09:42 -0400 Subject: [PATCH 6/8] Don't modify rectangles in place Now that we're caching the same Rectangle object, we need to copy it before snapping it to int bounds. --- src/Renderer.js | 21 ++++++++++++++------- src/renderer/Rectangle.js | 9 +++++++++ 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/src/Renderer.js b/src/Renderer.js index 5afa3f6..8e24ff2 100644 --- a/src/Renderer.js +++ b/src/Renderer.js @@ -10,6 +10,10 @@ import { effectNames, effectBitmasks } from "./renderer/effectInfo.js"; import Costume from "./Costume.js"; +// Rectangle used for checking collision bounds. +// Rather than create a new one each time, we can just reuse this one. +const __collisionBox = new Rectangle(); + export default class Renderer { constructor(project, renderTarget) { const w = project.stage.width; @@ -503,7 +507,10 @@ export default class Renderer { } } - const sprBox = this.getBoundingBox(spr).snapToInt(); + const sprBox = Rectangle.copy( + this.getBoundingBox(spr), + __collisionBox + ).snapToInt(); // This is an "impossible rectangle"-- its left bound is infinitely far to the right, // its right bound is infinitely to the left, and so on. Its size is effectively -Infinity. @@ -515,12 +522,9 @@ export default class Renderer { -Infinity ); for (const target of targets) { - Rectangle.union( - targetBox, - this.getBoundingBox(target).snapToInt(), - targetBox - ); + Rectangle.union(targetBox, this.getBoundingBox(target), targetBox); } + targetBox.snapToInt(); if (!sprBox.intersects(targetBox)) return false; if (fast) return true; @@ -573,7 +577,10 @@ export default class Renderer { } checkColorCollision(spr, targetsColor, sprColor) { - const sprBox = this.getBoundingBox(spr).snapToInt(); + const sprBox = Rectangle.copy( + this.getBoundingBox(spr), + __collisionBox + ).snapToInt(); const cx = this._collisionBuffer.width / 2; const cy = this._collisionBuffer.height / 2; diff --git a/src/renderer/Rectangle.js b/src/renderer/Rectangle.js index 6528e6a..149bfa4 100644 --- a/src/renderer/Rectangle.js +++ b/src/renderer/Rectangle.js @@ -41,6 +41,15 @@ export default class Rectangle { return result; } + // Initialize from another rectangle. + static copy(src, dst) { + dst.left = src.left; + dst.right = src.right; + dst.bottom = src.bottom; + dst.top = src.top; + return dst; + } + // Push this rectangle out to integer bounds. // This takes a conservative approach and will always expand the rectangle outwards. snapToInt() { From 67dd3afa45e196097e850e31c3bf530ffcbdea3c Mon Sep 17 00:00:00 2001 From: adroitwhiz Date: Sun, 17 Jul 2022 20:06:13 -0400 Subject: [PATCH 7/8] Fix diffing sprite size --- src/renderer/Drawable.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/renderer/Drawable.js b/src/renderer/Drawable.js index 107b780..d2a7819 100644 --- a/src/renderer/Drawable.js +++ b/src/renderer/Drawable.js @@ -29,7 +29,7 @@ class SpriteTransformDiff { this._lastY = this._sprite.y; this._lastRotation = this._sprite.direction; this._lastRotationStyle = this._sprite.rotationStyle; - this._lastScale = this._sprite.scale; + this._lastSize = this._sprite.size; this._lastCostume = this._sprite.costume; this._lastCostumeLoaded = this._sprite.costume.img.complete; this._unset = false; @@ -41,7 +41,7 @@ class SpriteTransformDiff { this._lastY !== this._sprite.y || this._lastRotation !== this._sprite.direction || this._lastRotationStyle !== this._sprite.rotationStyle || - this._lastScale !== this._sprite.scale || + this._lastSize !== this._sprite.size || this._lastCostume !== this._sprite.costume || this._lastCostumeLoaded !== this._sprite.costume.img.complete || this._unset From 3e3f281a23b079ba41d2c0d65884462638ab73f2 Mon Sep 17 00:00:00 2001 From: adroitwhiz Date: Sun, 17 Jul 2022 20:06:31 -0400 Subject: [PATCH 8/8] Fix error rendering speech bubbles --- src/Renderer.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Renderer.js b/src/Renderer.js index 8e24ff2..ab1c448 100644 --- a/src/Renderer.js +++ b/src/Renderer.js @@ -434,7 +434,7 @@ export default class Renderer { ) { const speechBubbleSkin = this._getSkin(sprite._speechBubble); - this.renderSkin( + this._renderSkin( speechBubbleSkin, options.drawMode, this._calculateSpeechBubbleMatrix(sprite, speechBubbleSkin),