diff --git a/django/applications/catmaid/static/js/layers/pixi-image-block-layer.js b/django/applications/catmaid/static/js/layers/pixi-image-block-layer.js index 387136426f..38e7275b2d 100644 --- a/django/applications/catmaid/static/js/layers/pixi-image-block-layer.js +++ b/django/applications/catmaid/static/js/layers/pixi-image-block-layer.js @@ -36,6 +36,11 @@ // If tiles have been initialized, reinitialize. this.resize(this.stackViewer.viewWidth, this.stackViewer.viewHeight); } + + if (this.tileSource.dataType().endsWith('64')) { + CATMAID.warn('64 bit data is not yet directly renderable, rendering as 16-bpc RGBA'); + } + let numStackLevels = this.stack.downsample_factors.length; let numSourceLevels = this.tileSource.numScaleLevels(); if (numStackLevels > numSourceLevels) { @@ -47,6 +52,9 @@ } _initTiles(rows, cols) { + let dataType = this.tileSource.dataType(); + this._setPermissibleInterpolationMode(dataType); + this.tileConstructor = this._dtypeTileConstructor(dataType); super._initTiles(rows, cols); for (var i = 0; i < rows; ++i) { @@ -264,28 +272,116 @@ return empty; } + _dtypeTileConstructor(dtype) { + switch (dtype) { + // These types use WebGL1-style implicit conversion: + case 'uint8': + // TODO: For now render uint32 via implicit conversion to RGBA 8-bit. + // This should be changed once there are configurable rendering modes + // per-datatype, so that an uint32 could either be an `gl.RGBA`, + // `gl.R32UI`, `gl.RG16UI`, etc. + case 'uint32': + // Floats are fine with the default Pixi shaders: + case 'float32': + case 'float64': + return PIXI.Sprite; + default: + return CATMAID.Pixi.TypedSprite.bind({}, dtype); + } + } + + _setPermissibleInterpolationMode(dtype) { + if (typeof dtype === 'undefined') return; + + // Integer texture formats are not interpolatable. + let current = this.getEffectiveInterpolationMode(); + if (current === CATMAID.StackLayer.INTERPOLATION_MODES.LINEAR && + (dtype.startsWith('uint') || dtype.startsWith('int')) && + dtype !== 'int8') { + this.setInterpolationMode(CATMAID.StackLayer.INTERPOLATION_MODES.NEAREST); + } + } + _dtypeWebGLParams(dtype) { + // See table 2: https://www.khronos.org/registry/OpenGL-Refpages/es3.0/html/glTexImage2D.xhtml + // WebGL2 documentation is often misleading because it only lists the + // subset of this table that is supported for conversion from JS canvases, + // etc., but WebGL2 itself supports all OpenGL ES 3.0 combinations. + const gl = this._context.renderer.gl; var format, type, internalFormat, jsArrayType; + // TODO: float64 is not supported. This may be the one current datatype + // that should be cast (to float32) before uploading. + switch (dtype) { + case 'int8': + format = gl.RED_INTEGER; + type = gl.BYTE; + internalFormat = gl.R8I; + jsArrayType = Int8Array; + break; + case 'int16': + format = gl.RED_INTEGER; + type = gl.SHORT; + internalFormat = gl.R16I; + jsArrayType = Int16Array; + break; + case 'int32': + format = gl.RED_INTEGER; + type = gl.INT; + internalFormat = gl.R32I; + jsArrayType = Int32Array; + break; + case 'int64': + // TODO: Once render modes per-datatype are available, this could also + // be a `RG32I`. + format = gl.RGBA_INTEGER; + type = gl.SHORT; + internalFormat = gl.RGBA16I; + jsArrayType = Int16Array; + break; case 'uint8': format = gl.LUMINANCE; type = gl.UNSIGNED_BYTE; internalFormat = gl.LUMINANCE; jsArrayType = Uint8Array; break; + case 'uint16': + format = gl.RED_INTEGER; + type = gl.UNSIGNED_SHORT; + internalFormat = gl.R16UI; + jsArrayType = Uint16Array; + break; + case 'uint64': + // TODO: Once render modes per-datatype are available, this could also + // be a `RG32UI`. + format = gl.RGBA_INTEGER; + type = gl.UNSIGNED_SHORT; + internalFormat = gl.RGBA16UI; + jsArrayType = Uint16Array; + break; // The default case can be hit when the layer is drawn before the // image block source has fully loaded. default: - CATMAID.warn(`Unknown data type for stack layer: ${dtype}, using uint32`); + // This default should only catch float64 at time of writing, but + // is a default since sources may generalize beyond N5 to backends + // with other data types. + CATMAID.warn(`Unsupported data type for stack layer: ${dtype}, using uint32`); /* falls through */ + // TODO: See note about 32-bit types in `_dtypeTileConstructor`. case 'uint32': format = gl.RGBA; type = gl.UNSIGNED_BYTE; internalFormat = gl.RGBA; jsArrayType = Uint8Array; break; + case 'float32': + format = gl.RED; + type = gl.FLOAT; + internalFormat = gl.R32F; + jsArrayType = Float32Array; + break; } return {format, type, internalFormat, jsArrayType}; @@ -295,7 +391,8 @@ let renderer = this._context.renderer; let gl = renderer.gl; - let {format, type, internalFormat, jsArrayType} = this._dtypeWebGLParams(slice.dtype); + let dtype = sliceDtypeToBlockDtype(slice.dtype); + let {format, type, internalFormat, jsArrayType} = this._dtypeWebGLParams(dtype); const glScaleMode = this._pixiInterpolationMode === PIXI.SCALE_MODES.LINEAR ? gl.LINEAR : gl.NEAREST; @@ -313,11 +410,11 @@ let height = slice.shape[0]; if (!texture || texture.width !== width || texture.height !== height || - texture.format !== format || texture.type !== type) { + texture.format !== internalFormat || texture.type !== type) { // Make sure Pixi does not have the old texture bound. renderer.unbindTexture(baseTex); if (texture) gl.deleteTexture(texture.texture); - texture = new PIXI.glCore.GLTexture(gl, width, height, format, type); + texture = new PIXI.glCore.GLTexture(gl, width, height, internalFormat, type); baseTex._glTextures[renderer.CONTEXT_UID] = texture; pixiTex._frame.width = baseTex.width = baseTex.realWidth = width; pixiTex._frame.height = baseTex.height = baseTex.realHeight = height; @@ -335,7 +432,7 @@ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); let typedArr = _flattenNdarraySliceToView(slice); - let arrayBuff = new jsArrayType(typedArr, + let arrayBuff = new jsArrayType(typedArr.buffer, typedArr.byteOffset, typedArr.byteLength/jsArrayType.BYTES_PER_ELEMENT); pixiTex._transpose = transpose; @@ -347,9 +444,10 @@ texture.width, texture.height, 0, // Border - texture.format, + format, texture.type, - arrayBuff); + arrayBuff, + 0); } else { gl.texSubImage2D( gl.TEXTURE_2D, @@ -357,9 +455,10 @@ 0, 0, texture.width, texture.height, - texture.format, + format, texture.type, - arrayBuff); + arrayBuff, + 0); } gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, glScaleMode); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, glScaleMode); @@ -427,6 +526,14 @@ CATMAID.PixiImageBlockLayer = PixiImageBlockLayer; + function sliceDtypeToBlockDtype(sliceDtype) { + if (sliceDtype.startsWith('big')) { + return sliceDtype.slice(3); + } else { + return sliceDtype; + } + } + /** Convert a 2-d c-order ndarray into a flattened TypedArray. */ function _flattenNdarraySliceToView(slice) { let sourceArray = slice.selection.data; diff --git a/django/applications/catmaid/static/js/layers/pixi-sprite-extensions.js b/django/applications/catmaid/static/js/layers/pixi-sprite-extensions.js new file mode 100644 index 0000000000..4dec3f9a39 --- /dev/null +++ b/django/applications/catmaid/static/js/layers/pixi-sprite-extensions.js @@ -0,0 +1,312 @@ +(function(CATMAID) { + +'use strict'; + +CATMAID.Pixi = CATMAID.Pixi || {}; + +CATMAID.Pixi.TypedSpriteRenderer = class TypedSpriteRenderer extends PIXI.SpriteRenderer { + constructor(dataType, renderer) { + super(renderer); + + this.dataType = dataType; + } + + // This is mostly a direct copy of PIXI.SpriteRenderer + onContextChange() + { + const gl = this.renderer.gl; + + if (this.renderer.legacy) + { + this.MAX_TEXTURES = 1; + } + else + { + // step 1: first check max textures the GPU can handle. + this.MAX_TEXTURES = Math.min(gl.getParameter(gl.MAX_TEXTURE_IMAGE_UNITS), PIXI.settings.SPRITE_MAX_TEXTURES); + + // step 2: check the maximum number of if statements the shader can have too.. + this.MAX_TEXTURES = checkMaxIfStatmentsInShader(this.MAX_TEXTURES, gl); + } + + // CHANGED LINE ///////////////////////////////////////////////////////// + this.shader = generateMultiTextureShader(gl, this.MAX_TEXTURES, this.dataType); + ///////////////////////////////////////////////////////////////////////// + + // create a couple of buffers + this.indexBuffer = PIXI.glCore.GLBuffer.createIndexBuffer(gl, this.indices, gl.STATIC_DRAW); + + // we use the second shader as the first one depending on your browser may omit aTextureId + // as it is not used by the shader so is optimized out. + + this.renderer.bindVao(null); + + const attrs = this.shader.attributes; + + for (let i = 0; i < this.vaoMax; i++) + { + /* eslint-disable max-len */ + const vertexBuffer = this.vertexBuffers[i] = PIXI.glCore.GLBuffer.createVertexBuffer(gl, null, gl.STREAM_DRAW); + /* eslint-enable max-len */ + + // build the vao object that will render.. + const vao = this.renderer.createVao() + .addIndex(this.indexBuffer) + .addAttribute(vertexBuffer, attrs.aVertexPosition, gl.FLOAT, false, this.vertByteSize, 0) + .addAttribute(vertexBuffer, attrs.aTextureCoord, gl.UNSIGNED_SHORT, true, this.vertByteSize, 2 * 4) + .addAttribute(vertexBuffer, attrs.aColor, gl.UNSIGNED_BYTE, true, this.vertByteSize, 3 * 4); + + if (attrs.aTextureId) + { + vao.addAttribute(vertexBuffer, attrs.aTextureId, gl.FLOAT, false, this.vertByteSize, 4 * 4); + } + + this.vaos[i] = vao; + } + + this.vao = this.vaos[0]; + this.currentBlendMode = 99999; + + this.boundTextures = new Array(this.MAX_TEXTURES); + } +}; + + +/** + * Version of `PIXI.Shader` that does not insert `precision` statements that + * break the requirement of `#version` being first in WebGL2/GLSL 3.0 shaders. + */ +class NonMutatingShader extends PIXI.glCore.GLShader +{ + constructor(gl, vertexSrc, fragmentSrc, attributeLocations, precision) + { + super(gl, vertexSrc, fragmentSrc, undefined, attributeLocations); + } +} + +// From `pixi.js/src/core/sprites/webgl/` +// Modified to change shaders to GLSL 3 and vary types according to input data +// type. + +const vertTemplate = `#version 300 es +precision highp float; +precision highp int; +in vec2 aVertexPosition; +in vec2 aTextureCoord; +in vec4 aColor; +in float aTextureId; + +uniform mat3 projectionMatrix; + +out vec2 vTextureCoord; +out vec4 vColor; +out float vTextureId; + +void main(void){ + gl_Position = vec4((projectionMatrix * vec3(aVertexPosition, 1.0)).xy, 0.0, 1.0); + + vTextureCoord = aTextureCoord; + vTextureId = aTextureId; + vColor = aColor; +}`; + +const fragTemplate = `#version 300 es +precision highp float; +precision highp int; +precision highp %sampler_type%; + +in vec2 vTextureCoord; +in vec4 vColor; +in float vTextureId; +uniform %sampler_type% uSamplers[%count%]; +out vec4 myOutputColor; + +void main(void){ +%color_type% color; +%forloop% +myOutputColor = %color_conversion%; +}`; + +function generateMultiTextureShader(gl, maxTextures, dataType) +{ + const vertexSrc = vertTemplate; + let fragmentSrc = fragTemplate; + + fragmentSrc = fragmentSrc.replace(/%count%/gi, maxTextures); + fragmentSrc = fragmentSrc.replace(/%forloop%/gi, generateSampleSrc(maxTextures)); + + const params = dataTypeParameters(dataType); + fragmentSrc = fragmentSrc.replace(/%sampler_type%/gi, params.samplerType); + fragmentSrc = fragmentSrc.replace(/%color_type%/gi, params.colorType); + fragmentSrc = fragmentSrc.replace(/%color_conversion%/gi, params.colorConversion); + + const shader = new NonMutatingShader(gl, vertexSrc, fragmentSrc); + + const sampleValues = []; + + for (let i = 0; i < maxTextures; i++) + { + sampleValues[i] = i; + } + + shader.bind(); + shader.uniforms.uSamplers = sampleValues; + + return shader; +} + +function dataTypeParameters(dataType) { + let prefix; + if (dataType.startsWith('int')) { + prefix = 'i'; + } else if (dataType.startsWith('uint')) { + prefix = 'u'; + } + + let depth = dataType.substr(-2); + if (depth.endsWith("8")) depth = "8"; + depth = parseInt(depth, 10); + + // TODO: Since this is not channel/mode aware yet, manually set that 64-bit + // data is RGBA16 so should be normalized by 16 bits instead. + if (depth === 64) { + depth = 16; + } + + let typeMax = Math.pow(2, depth); + + return { + colorType: prefix + 'vec4', + colorConversion: 'float(color) * vColor / ' + typeMax.toFixed(1), + samplerType: prefix + 'sampler2D', + }; +} + +function generateSampleSrc(maxTextures) +{ + let src = ''; + + src += '\n'; + src += '\n'; + + for (let i = 0; i < maxTextures; i++) + { + if (i > 0) + { + src += '\nelse '; + } + + if (i < maxTextures - 1) + { + src += `if(vTextureId < ${i}.5)`; + } + + src += '\n{'; + src += `\n\tcolor = texture(uSamplers[${i}], vTextureCoord);`; + src += '\n}'; + } + + src += '\n'; + src += '\n'; + + return src; +} + + +// From pixi.js/src/core/renderers/webgl/utils/checkMaxIfStatmentsInShader.js + +const maxFragTemplate = [ + 'precision mediump float;', + 'void main(void){', + 'float test = 0.1;', + '%forloop%', + 'gl_FragColor = vec4(0.0);', + '}', +].join('\n'); + +function checkMaxIfStatmentsInShader(maxIfs, gl) +{ + const createTempContext = !gl; + + if (createTempContext) + { + const tinyCanvas = document.createElement('canvas'); + + tinyCanvas.width = 1; + tinyCanvas.height = 1; + + gl = PIXI.glCore.createContext(tinyCanvas); + } + + const shader = gl.createShader(gl.FRAGMENT_SHADER); + + while (true) // eslint-disable-line no-constant-condition + { + const fragmentSrc = maxFragTemplate.replace(/%forloop%/gi, generateIfTestSrc(maxIfs)); + + gl.shaderSource(shader, fragmentSrc); + gl.compileShader(shader); + + if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) + { + maxIfs = (maxIfs / 2) | 0; + } + else + { + // valid! + break; + } + } + + if (createTempContext) + { + // get rid of context + if (gl.getExtension('WEBGL_lose_context')) + { + gl.getExtension('WEBGL_lose_context').loseContext(); + } + } + + return maxIfs; +} + +function generateIfTestSrc(maxIfs) +{ + let src = ''; + + for (let i = 0; i < maxIfs; ++i) + { + if (i > 0) + { + src += '\nelse '; + } + + if (i < maxIfs - 1) + { + src += `if(test == ${i}.0){}`; + } + } + + return src; +} + + +CATMAID.Pixi.TypedSprite = class TypedSprite extends PIXI.Sprite { + constructor(dataType, ...args) { + super(...args); + + this.pluginName = 'typedSprite_' + dataType; + } +}; + +let supportedDataTypes = [ + 'int8', 'int16', 'int32', 'int64', + 'uint16', 'uint32', 'uint64', +]; +for (let dataType of supportedDataTypes) { + let renderer = CATMAID.Pixi.TypedSpriteRenderer.bind({}, dataType); + + PIXI.WebGLRenderer.registerPlugin('typedSprite_' + dataType, renderer); +} + +})(CATMAID); diff --git a/django/applications/catmaid/static/js/layers/pixi-tile-layer.js b/django/applications/catmaid/static/js/layers/pixi-tile-layer.js index b25e60a755..1d4e9de42c 100644 --- a/django/applications/catmaid/static/js/layers/pixi-tile-layer.js +++ b/django/applications/catmaid/static/js/layers/pixi-tile-layer.js @@ -29,6 +29,7 @@ this.stackViewer.getLayersView().removeChild(this.tilesContainer); this.tilesContainer = this.renderer.view; this.tilesContainer.className = 'sliceTiles'; + this.tileConstructor = PIXI.Sprite; this.stackViewer.getLayersView().appendChild(this.tilesContainer); this._oldZoom = 0; @@ -111,7 +112,7 @@ this._tiles[i] = []; this._tilesBuffer[i] = []; for (var j = 0; j < cols; ++j) { - this._tiles[i][j] = new PIXI.Sprite(emptyTex); + this._tiles[i][j] = new this.tileConstructor(emptyTex); this.batchContainer.addChild(this._tiles[i][j]); this._tiles[i][j].position.x = j * this.tileWidth * this._anisotropy.x; this._tiles[i][j].position.y = i * this.tileHeight * this._anisotropy.y;