From 742dabc6b1f2f9edb183027c0e38cb45c82111cd Mon Sep 17 00:00:00 2001 From: Andrew Champion Date: Sun, 5 Jan 2020 21:06:07 +0000 Subject: [PATCH] Image block layer: add support for non-uint8 data types This adds support for rendering non-uint8 blocks by adding custom `SpriteRenderers` for each data type. These renderers create shaders based on Pixi's but with appropriate datatypes and scaling to output the data type into a normal floating point render buffer. Note that 64-bit integer types render to 16-bit RGBA, and that float64 is not supported. All other types were tested, though not all are necessary useful as is. This builds toward support for label rendering and 3D texture rendering. See catmaid/CATMAID#1955. --- .../js/layers/pixi-image-block-layer.js | 120 ++++++- .../js/layers/pixi-sprite-extensions.js | 312 ++++++++++++++++++ .../static/js/layers/pixi-tile-layer.js | 3 +- 3 files changed, 425 insertions(+), 10 deletions(-) create mode 100644 django/applications/catmaid/static/js/layers/pixi-sprite-extensions.js diff --git a/django/applications/catmaid/static/js/layers/pixi-image-block-layer.js b/django/applications/catmaid/static/js/layers/pixi-image-block-layer.js index 387136426f..f888575d88 100644 --- a/django/applications/catmaid/static/js/layers/pixi-image-block-layer.js +++ b/django/applications/catmaid/static/js/layers/pixi-image-block-layer.js @@ -36,6 +36,11 @@ // If tiles have been initialized, reinitialize. this.resize(this.stackViewer.viewWidth, this.stackViewer.viewHeight); } + + if (this.tileSource.dataType().endsWith('64')) { + CATMAID.warn('64 bit data is not yet directly renderable, rendering as 16-bpc RGBA'); + } + let numStackLevels = this.stack.downsample_factors.length; let numSourceLevels = this.tileSource.numScaleLevels(); if (numStackLevels > numSourceLevels) { @@ -47,6 +52,9 @@ } _initTiles(rows, cols) { + let dataType = this.tileSource.dataType(); + this._setPermissibleInterpolationMode(dataType); + this.tileConstructor = this._dtypeTileConstructor(dataType); super._initTiles(rows, cols); for (var i = 0; i < rows; ++i) { @@ -264,28 +272,111 @@ return empty; } + _dtypeTileConstructor(dtype) { + switch (dtype) { + // These types use WebGL1-style implicit conversion: + case 'uint8': + // TODO: For now render (u)int32 via implicit conversion to RGBA 8-bit. + // This should be changed once there are configurable rendering modes + // per-datatype, so that an uint32 could either be an `gl.RGBA`, + // `gl.R32UI`, `gl.RG16UI`, etc. + case 'int32': + case 'uint32': + // Floats are fine with the default Pixi shaders: + case 'float32': + case 'float64': + return PIXI.Sprite; + default: + return CATMAID.Pixi.TypedSprite.bind({}, dtype); + } + } + + _setPermissibleInterpolationMode(dtype) { + if (typeof dtype === 'undefined') return; + + // Non-8-bit integer texture formats are not interpolatable. + let current = this.getEffectiveInterpolationMode(); + if (current === CATMAID.StackLayer.INTERPOLATION_MODES.LINEAR && + (dtype.startsWith('uint') || dtype.startsWith('int')) && + dtype !== 'int8') { + this.setInterpolationMode(CATMAID.StackLayer.INTERPOLATION_MODES.NEAREST); + } + } + _dtypeWebGLParams(dtype) { + // See table 2: https://www.khronos.org/registry/OpenGL-Refpages/es3.0/html/glTexImage2D.xhtml + // WebGL2 documentation is often misleading because it only lists the + // subset of this table that is supported for conversion from JS canvases, + // etc., but WebGL2 itself supports all OpenGL ES 3.0 combinations. + const gl = this._context.renderer.gl; var format, type, internalFormat, jsArrayType; + // TODO: float64 is not supported. This may be the one current datatype + // that should be cast (to float32) before uploading. + switch (dtype) { + case 'int8': + format = gl.RED_INTEGER; + type = gl.BYTE; + internalFormat = gl.R8I; + jsArrayType = Int8Array; + break; + case 'int16': + format = gl.RED_INTEGER; + type = gl.SHORT; + internalFormat = gl.R16I; + jsArrayType = Int16Array; + break; + case 'int64': + // TODO: Once render modes per-datatype are available, this could also + // be a `RG32I`. + format = gl.RGBA_INTEGER; + type = gl.SHORT; + internalFormat = gl.RGBA16I; + jsArrayType = Int16Array; + break; case 'uint8': format = gl.LUMINANCE; type = gl.UNSIGNED_BYTE; internalFormat = gl.LUMINANCE; jsArrayType = Uint8Array; break; + case 'uint16': + format = gl.RED_INTEGER; + type = gl.UNSIGNED_SHORT; + internalFormat = gl.R16UI; + jsArrayType = Uint16Array; + break; + case 'uint64': + // TODO: Once render modes per-datatype are available, this could also + // be a `RG32UI`. + format = gl.RGBA_INTEGER; + type = gl.UNSIGNED_SHORT; + internalFormat = gl.RGBA16UI; + jsArrayType = Uint16Array; + break; // The default case can be hit when the layer is drawn before the // image block source has fully loaded. default: - CATMAID.warn(`Unknown data type for stack layer: ${dtype}, using uint32`); + // This default should only catch float64 at time of writing, but + // is a default since sources may generalize beyond N5 to backends + // with other data types. + CATMAID.warn(`Unsupported data type for stack layer: ${dtype}, using uint32`); /* falls through */ + // TODO: See note about 32-bit types in `_dtypeTileConstructor`. case 'uint32': format = gl.RGBA; type = gl.UNSIGNED_BYTE; internalFormat = gl.RGBA; jsArrayType = Uint8Array; break; + case 'float32': + format = gl.RED; + type = gl.FLOAT; + internalFormat = gl.R32F; + jsArrayType = Float32Array; + break; } return {format, type, internalFormat, jsArrayType}; @@ -295,7 +386,8 @@ let renderer = this._context.renderer; let gl = renderer.gl; - let {format, type, internalFormat, jsArrayType} = this._dtypeWebGLParams(slice.dtype); + let dtype = sliceDtypeToBlockDtype(slice.dtype); + let {format, type, internalFormat, jsArrayType} = this._dtypeWebGLParams(dtype); const glScaleMode = this._pixiInterpolationMode === PIXI.SCALE_MODES.LINEAR ? gl.LINEAR : gl.NEAREST; @@ -313,11 +405,11 @@ let height = slice.shape[0]; if (!texture || texture.width !== width || texture.height !== height || - texture.format !== format || texture.type !== type) { + texture.format !== internalFormat || texture.type !== type) { // Make sure Pixi does not have the old texture bound. renderer.unbindTexture(baseTex); if (texture) gl.deleteTexture(texture.texture); - texture = new PIXI.glCore.GLTexture(gl, width, height, format, type); + texture = new PIXI.glCore.GLTexture(gl, width, height, internalFormat, type); baseTex._glTextures[renderer.CONTEXT_UID] = texture; pixiTex._frame.width = baseTex.width = baseTex.realWidth = width; pixiTex._frame.height = baseTex.height = baseTex.realHeight = height; @@ -335,7 +427,7 @@ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); let typedArr = _flattenNdarraySliceToView(slice); - let arrayBuff = new jsArrayType(typedArr, + let arrayBuff = new jsArrayType(typedArr.buffer, typedArr.byteOffset, typedArr.byteLength/jsArrayType.BYTES_PER_ELEMENT); pixiTex._transpose = transpose; @@ -347,9 +439,10 @@ texture.width, texture.height, 0, // Border - texture.format, + format, texture.type, - arrayBuff); + arrayBuff, + 0); } else { gl.texSubImage2D( gl.TEXTURE_2D, @@ -357,9 +450,10 @@ 0, 0, texture.width, texture.height, - texture.format, + format, texture.type, - arrayBuff); + arrayBuff, + 0); } gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, glScaleMode); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, glScaleMode); @@ -427,6 +521,14 @@ CATMAID.PixiImageBlockLayer = PixiImageBlockLayer; + function sliceDtypeToBlockDtype(sliceDtype) { + if (sliceDtype.startsWith('big')) { + return sliceDtype.slice(3); + } else { + return sliceDtype; + } + } + /** Convert a 2-d c-order ndarray into a flattened TypedArray. */ function _flattenNdarraySliceToView(slice) { let sourceArray = slice.selection.data; diff --git a/django/applications/catmaid/static/js/layers/pixi-sprite-extensions.js b/django/applications/catmaid/static/js/layers/pixi-sprite-extensions.js new file mode 100644 index 0000000000..4dec3f9a39 --- /dev/null +++ b/django/applications/catmaid/static/js/layers/pixi-sprite-extensions.js @@ -0,0 +1,312 @@ +(function(CATMAID) { + +'use strict'; + +CATMAID.Pixi = CATMAID.Pixi || {}; + +CATMAID.Pixi.TypedSpriteRenderer = class TypedSpriteRenderer extends PIXI.SpriteRenderer { + constructor(dataType, renderer) { + super(renderer); + + this.dataType = dataType; + } + + // This is mostly a direct copy of PIXI.SpriteRenderer + onContextChange() + { + const gl = this.renderer.gl; + + if (this.renderer.legacy) + { + this.MAX_TEXTURES = 1; + } + else + { + // step 1: first check max textures the GPU can handle. + this.MAX_TEXTURES = Math.min(gl.getParameter(gl.MAX_TEXTURE_IMAGE_UNITS), PIXI.settings.SPRITE_MAX_TEXTURES); + + // step 2: check the maximum number of if statements the shader can have too.. + this.MAX_TEXTURES = checkMaxIfStatmentsInShader(this.MAX_TEXTURES, gl); + } + + // CHANGED LINE ///////////////////////////////////////////////////////// + this.shader = generateMultiTextureShader(gl, this.MAX_TEXTURES, this.dataType); + ///////////////////////////////////////////////////////////////////////// + + // create a couple of buffers + this.indexBuffer = PIXI.glCore.GLBuffer.createIndexBuffer(gl, this.indices, gl.STATIC_DRAW); + + // we use the second shader as the first one depending on your browser may omit aTextureId + // as it is not used by the shader so is optimized out. + + this.renderer.bindVao(null); + + const attrs = this.shader.attributes; + + for (let i = 0; i < this.vaoMax; i++) + { + /* eslint-disable max-len */ + const vertexBuffer = this.vertexBuffers[i] = PIXI.glCore.GLBuffer.createVertexBuffer(gl, null, gl.STREAM_DRAW); + /* eslint-enable max-len */ + + // build the vao object that will render.. + const vao = this.renderer.createVao() + .addIndex(this.indexBuffer) + .addAttribute(vertexBuffer, attrs.aVertexPosition, gl.FLOAT, false, this.vertByteSize, 0) + .addAttribute(vertexBuffer, attrs.aTextureCoord, gl.UNSIGNED_SHORT, true, this.vertByteSize, 2 * 4) + .addAttribute(vertexBuffer, attrs.aColor, gl.UNSIGNED_BYTE, true, this.vertByteSize, 3 * 4); + + if (attrs.aTextureId) + { + vao.addAttribute(vertexBuffer, attrs.aTextureId, gl.FLOAT, false, this.vertByteSize, 4 * 4); + } + + this.vaos[i] = vao; + } + + this.vao = this.vaos[0]; + this.currentBlendMode = 99999; + + this.boundTextures = new Array(this.MAX_TEXTURES); + } +}; + + +/** + * Version of `PIXI.Shader` that does not insert `precision` statements that + * break the requirement of `#version` being first in WebGL2/GLSL 3.0 shaders. + */ +class NonMutatingShader extends PIXI.glCore.GLShader +{ + constructor(gl, vertexSrc, fragmentSrc, attributeLocations, precision) + { + super(gl, vertexSrc, fragmentSrc, undefined, attributeLocations); + } +} + +// From `pixi.js/src/core/sprites/webgl/` +// Modified to change shaders to GLSL 3 and vary types according to input data +// type. + +const vertTemplate = `#version 300 es +precision highp float; +precision highp int; +in vec2 aVertexPosition; +in vec2 aTextureCoord; +in vec4 aColor; +in float aTextureId; + +uniform mat3 projectionMatrix; + +out vec2 vTextureCoord; +out vec4 vColor; +out float vTextureId; + +void main(void){ + gl_Position = vec4((projectionMatrix * vec3(aVertexPosition, 1.0)).xy, 0.0, 1.0); + + vTextureCoord = aTextureCoord; + vTextureId = aTextureId; + vColor = aColor; +}`; + +const fragTemplate = `#version 300 es +precision highp float; +precision highp int; +precision highp %sampler_type%; + +in vec2 vTextureCoord; +in vec4 vColor; +in float vTextureId; +uniform %sampler_type% uSamplers[%count%]; +out vec4 myOutputColor; + +void main(void){ +%color_type% color; +%forloop% +myOutputColor = %color_conversion%; +}`; + +function generateMultiTextureShader(gl, maxTextures, dataType) +{ + const vertexSrc = vertTemplate; + let fragmentSrc = fragTemplate; + + fragmentSrc = fragmentSrc.replace(/%count%/gi, maxTextures); + fragmentSrc = fragmentSrc.replace(/%forloop%/gi, generateSampleSrc(maxTextures)); + + const params = dataTypeParameters(dataType); + fragmentSrc = fragmentSrc.replace(/%sampler_type%/gi, params.samplerType); + fragmentSrc = fragmentSrc.replace(/%color_type%/gi, params.colorType); + fragmentSrc = fragmentSrc.replace(/%color_conversion%/gi, params.colorConversion); + + const shader = new NonMutatingShader(gl, vertexSrc, fragmentSrc); + + const sampleValues = []; + + for (let i = 0; i < maxTextures; i++) + { + sampleValues[i] = i; + } + + shader.bind(); + shader.uniforms.uSamplers = sampleValues; + + return shader; +} + +function dataTypeParameters(dataType) { + let prefix; + if (dataType.startsWith('int')) { + prefix = 'i'; + } else if (dataType.startsWith('uint')) { + prefix = 'u'; + } + + let depth = dataType.substr(-2); + if (depth.endsWith("8")) depth = "8"; + depth = parseInt(depth, 10); + + // TODO: Since this is not channel/mode aware yet, manually set that 64-bit + // data is RGBA16 so should be normalized by 16 bits instead. + if (depth === 64) { + depth = 16; + } + + let typeMax = Math.pow(2, depth); + + return { + colorType: prefix + 'vec4', + colorConversion: 'float(color) * vColor / ' + typeMax.toFixed(1), + samplerType: prefix + 'sampler2D', + }; +} + +function generateSampleSrc(maxTextures) +{ + let src = ''; + + src += '\n'; + src += '\n'; + + for (let i = 0; i < maxTextures; i++) + { + if (i > 0) + { + src += '\nelse '; + } + + if (i < maxTextures - 1) + { + src += `if(vTextureId < ${i}.5)`; + } + + src += '\n{'; + src += `\n\tcolor = texture(uSamplers[${i}], vTextureCoord);`; + src += '\n}'; + } + + src += '\n'; + src += '\n'; + + return src; +} + + +// From pixi.js/src/core/renderers/webgl/utils/checkMaxIfStatmentsInShader.js + +const maxFragTemplate = [ + 'precision mediump float;', + 'void main(void){', + 'float test = 0.1;', + '%forloop%', + 'gl_FragColor = vec4(0.0);', + '}', +].join('\n'); + +function checkMaxIfStatmentsInShader(maxIfs, gl) +{ + const createTempContext = !gl; + + if (createTempContext) + { + const tinyCanvas = document.createElement('canvas'); + + tinyCanvas.width = 1; + tinyCanvas.height = 1; + + gl = PIXI.glCore.createContext(tinyCanvas); + } + + const shader = gl.createShader(gl.FRAGMENT_SHADER); + + while (true) // eslint-disable-line no-constant-condition + { + const fragmentSrc = maxFragTemplate.replace(/%forloop%/gi, generateIfTestSrc(maxIfs)); + + gl.shaderSource(shader, fragmentSrc); + gl.compileShader(shader); + + if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) + { + maxIfs = (maxIfs / 2) | 0; + } + else + { + // valid! + break; + } + } + + if (createTempContext) + { + // get rid of context + if (gl.getExtension('WEBGL_lose_context')) + { + gl.getExtension('WEBGL_lose_context').loseContext(); + } + } + + return maxIfs; +} + +function generateIfTestSrc(maxIfs) +{ + let src = ''; + + for (let i = 0; i < maxIfs; ++i) + { + if (i > 0) + { + src += '\nelse '; + } + + if (i < maxIfs - 1) + { + src += `if(test == ${i}.0){}`; + } + } + + return src; +} + + +CATMAID.Pixi.TypedSprite = class TypedSprite extends PIXI.Sprite { + constructor(dataType, ...args) { + super(...args); + + this.pluginName = 'typedSprite_' + dataType; + } +}; + +let supportedDataTypes = [ + 'int8', 'int16', 'int32', 'int64', + 'uint16', 'uint32', 'uint64', +]; +for (let dataType of supportedDataTypes) { + let renderer = CATMAID.Pixi.TypedSpriteRenderer.bind({}, dataType); + + PIXI.WebGLRenderer.registerPlugin('typedSprite_' + dataType, renderer); +} + +})(CATMAID); diff --git a/django/applications/catmaid/static/js/layers/pixi-tile-layer.js b/django/applications/catmaid/static/js/layers/pixi-tile-layer.js index b25e60a755..1d4e9de42c 100644 --- a/django/applications/catmaid/static/js/layers/pixi-tile-layer.js +++ b/django/applications/catmaid/static/js/layers/pixi-tile-layer.js @@ -29,6 +29,7 @@ this.stackViewer.getLayersView().removeChild(this.tilesContainer); this.tilesContainer = this.renderer.view; this.tilesContainer.className = 'sliceTiles'; + this.tileConstructor = PIXI.Sprite; this.stackViewer.getLayersView().appendChild(this.tilesContainer); this._oldZoom = 0; @@ -111,7 +112,7 @@ this._tiles[i] = []; this._tilesBuffer[i] = []; for (var j = 0; j < cols; ++j) { - this._tiles[i][j] = new PIXI.Sprite(emptyTex); + this._tiles[i][j] = new this.tileConstructor(emptyTex); this.batchContainer.addChild(this._tiles[i][j]); this._tiles[i][j].position.x = j * this.tileWidth * this._anisotropy.x; this._tiles[i][j].position.y = i * this.tileHeight * this._anisotropy.y;