From b1f2984b3ea4a607a4d663a7f14e87f5ee6d3d11 Mon Sep 17 00:00:00 2001 From: Edwin Jakobs Date: Fri, 28 Jun 2024 11:12:38 +0200 Subject: [PATCH] [orx-integral-image] Pad source image for npot inputs. Add demo --- orx-integral-image/README.md | 59 ----------- .../src/demo/kotlin/DemoFII01.kt | 42 ++++++++ .../src/main/kotlin/FastIntegralImage.kt | 98 ++++++++++++++++--- .../resources/shaders/gl3/integral-image.frag | 12 +-- 4 files changed, 130 insertions(+), 81 deletions(-) create mode 100644 orx-integral-image/src/demo/kotlin/DemoFII01.kt diff --git a/orx-integral-image/README.md b/orx-integral-image/README.md index 33a84b2fd..283b74688 100644 --- a/orx-integral-image/README.md +++ b/orx-integral-image/README.md @@ -2,62 +2,3 @@ CPU and GPU-based implementation for integral images (summed area tables) -#### Usage - -```kotlin -val image = colorBuffer( ... ) -image.shadow.download() -val integralImage = IntegralImage.fromColorBufferShadow(image.shadow) - -// -- the sum for a given area can be queried using -val sum = integralImage.sum(IntRectangle(20, 20, 100, 100)) -``` - -## Fast Integral Image - -Since v0.0.20 orx-integral-image comes with `FastIntegralImage` which calculates integral images fully on the GPU. - -```kotlin -import org.openrndr.application -import org.openrndr.color.ColorRGBa -import org.openrndr.draw.* -import org.openrndr.extra.integralimage.* - -fun main(args: Array) = application { - configure { - width = 1024 - height = 1024 - } - program { - val fii = FastIntegralImage() - val target = colorBuffer(1024, 512, 1.0, ColorFormat.RGBa, ColorType.FLOAT32) - val rt = renderTarget(1024, 512) { - colorBuffer() - } - extend { - drawer.background(ColorRGBa.PINK) - drawer.isolatedWithTarget(rt) { - drawer.ortho(rt) - drawer.background(ColorRGBa.BLACK) - drawer.fill = ColorRGBa.PINK.shade(1.0) - drawer.circle(mouse.position, 128.0) - } - fii.apply(rt.colorBuffer(0),target) - - // -- here we sample from the integral image - drawer.shadeStyle = shadeStyle { - fragmentTransform = """ - float w = 128.0; - vec2 step = 1.0 / textureSize(image, 0); - vec4 t11 = texture(image, va_texCoord0 + step * vec2(w,w)); - vec4 t01 = texture(image, va_texCoord0 + step * vec2(-w,w)); - vec4 t00 = texture(image, va_texCoord0 + step * vec2(-w,-w)); - vec4 t10 = texture(image, va_texCoord0 + step * vec2(w,-w)); - x_fill = (t11 - t01 - t10 + t00) / (w*w); - """.trimIndent() - } - drawer.image(target) - } - } -} -``` diff --git a/orx-integral-image/src/demo/kotlin/DemoFII01.kt b/orx-integral-image/src/demo/kotlin/DemoFII01.kt new file mode 100644 index 000000000..306919760 --- /dev/null +++ b/orx-integral-image/src/demo/kotlin/DemoFII01.kt @@ -0,0 +1,42 @@ +import org.openrndr.application +import org.openrndr.color.ColorRGBa +import org.openrndr.draw.* +import org.openrndr.extra.integralimage.* + +fun main() = application { + configure { + width = 720 + height = 720 + } + program { + val fii = FastIntegralImage() + val target = colorBuffer(width, height, 1.0, ColorFormat.RGBa, ColorType.FLOAT32) + val rt = renderTarget(width, height) { + colorBuffer() + } + extend { + drawer.clear(ColorRGBa.PINK) + drawer.isolatedWithTarget(rt) { + drawer.ortho(rt) + drawer.clear(ColorRGBa.BLACK) + drawer.fill = ColorRGBa.PINK.shade(1.0) + drawer.circle(mouse.position, 128.0) + } + fii.apply(rt.colorBuffer(0), target) + + // -- here we sample from the integral image + drawer.shadeStyle = shadeStyle { + fragmentTransform = """ + float w = 64.0; + vec2 step = 1.0 / vec2(textureSize(image, 0)); + vec4 t11 = texture(image, va_texCoord0 + step * vec2(w+1.0,w+1.0)); + vec4 t01 = texture(image, va_texCoord0 + step * vec2(-w,w+1.0)); + vec4 t00 = texture(image, va_texCoord0 + step * vec2(-w,-w)); + vec4 t10 = texture(image, va_texCoord0 + step * vec2(w+1.0,-w)); + x_fill = (t11 - t01 - t10 + t00) / ((2.0 * w +1.0) * (2.0 * w + 1.0)); + """.trimIndent() + } + drawer.image(target) + } + } +} \ No newline at end of file diff --git a/orx-integral-image/src/main/kotlin/FastIntegralImage.kt b/orx-integral-image/src/main/kotlin/FastIntegralImage.kt index 558f74e14..32c986d64 100644 --- a/orx-integral-image/src/main/kotlin/FastIntegralImage.kt +++ b/orx-integral-image/src/main/kotlin/FastIntegralImage.kt @@ -5,27 +5,43 @@ import org.openrndr.extra.fx.blend.Passthrough import org.openrndr.math.Vector2 import org.openrndr.resourceUrl +import org.openrndr.shape.IntRectangle import org.openrndr.shape.Rectangle +import kotlin.math.ceil +import kotlin.math.log -class FastIntegralImageFilter : Filter(filterShaderFromUrl(resourceUrl( - "/shaders/gl3/integral-image.frag" -))) { +internal class FastIntegralImageFilter : Filter( + filterShaderFromUrl( + resourceUrl( + "/shaders/gl3/integral-image.frag" + ) + ) +) { var passIndex: Int by parameters var passDirection: Vector2 by parameters var sampleCount: Int by parameters var sampleCountBase: Int by parameters } -class FastIntegralImage : Filter(filterShaderFromUrl(resourceUrl( - "/shaders/gl3/integral-image.frag" -))) { +/** + * Compute an integral image for the source image + */ +class FastIntegralImage : Filter( + filterShaderFromUrl( + resourceUrl( + "/shaders/gl3/integral-image.frag" + ) + ) +) { private val passthrough = Passthrough() var intermediate: ColorBuffer? = null - val filter = FastIntegralImageFilter() + var sourceCropped: ColorBuffer? = null + var targetPadded: ColorBuffer? = null + private val filter = FastIntegralImageFilter() - private fun sampleCounts(size:Int, sampleCountBase:Int) : List { + private fun sampleCounts(size: Int, sampleCountBase: Int): List { var remainder = size val sampleCounts = mutableListOf() while (remainder > 0) { @@ -41,40 +57,90 @@ class FastIntegralImage : Filter(filterShaderFromUrl(resourceUrl( override fun apply(source: Array, target: Array, clip: Rectangle?) { require(clip == null) + require(source[0].isEquivalentTo(target[0], ignoreFormat = true, ignoreType = true)) + + val npotx = ceil(log(source[0].effectiveWidth.toDouble(), 2.0)).toInt() + val npoty = ceil(log(source[0].effectiveHeight.toDouble(), 2.0)).toInt() + + val recWidth = 1 shl npotx + val recHeight = 1 shl npoty + + if (recWidth != source[0].effectiveWidth || recHeight != source[0].effectiveHeight) { + if (sourceCropped?.effectiveWidth != recWidth || sourceCropped?.effectiveHeight != recHeight) { + sourceCropped?.destroy() + targetPadded?.destroy() + } + + if (sourceCropped == null) { + sourceCropped = source[0].createEquivalent(width = recWidth, height = recHeight, contentScale = 1.0) + targetPadded = target[0].createEquivalent( + width = (recWidth / target[0].contentScale).toInt(), + height = (recHeight / target[0].contentScale).toInt(), + contentScale = 1.0 + ) + } + source[0].copyTo(sourceCropped!!, + sourceRectangle = IntRectangle(0, 0, source[0].effectiveWidth, source[0].effectiveHeight), + targetRectangle = IntRectangle(0, recHeight-source[0].effectiveHeight, source[0].effectiveWidth, source[0].effectiveHeight) + ) + } + val sampleCountBase = 16 - val xSampleCounts = sampleCounts(source[0].width, sampleCountBase) - val ySampleCounts = sampleCounts(source[0].height, sampleCountBase) + val xSampleCounts = sampleCounts(recWidth, sampleCountBase) + val ySampleCounts = sampleCounts(recHeight, sampleCountBase) val li = intermediate - if (li == null || (li.width != source[0].width || li.height != source[0].height)) { + if (li == null || (li.effectiveWidth != recWidth || li.effectiveHeight != recHeight)) { intermediate?.destroy() - intermediate = colorBuffer(source[0].width, source[0].height, 1.0, ColorFormat.RGBa, ColorType.FLOAT32) + intermediate = colorBuffer(recWidth, recHeight, 1.0, ColorFormat.RGBa, ColorType.FLOAT32) } - val targets = arrayOf(target, arrayOf(intermediate!!)) + val targets = arrayOf(if (targetPadded == null) target else arrayOf(targetPadded!!), arrayOf(intermediate!!)) var targetIndex = 0 filter.sampleCountBase = sampleCountBase + /* + Perform horizontal steps + */ filter.passDirection = Vector2.UNIT_X for (pass in xSampleCounts.indices) { filter.sampleCount = xSampleCounts[pass] filter.passIndex = pass - filter.apply( if (pass == 0) source else targets[targetIndex%2], targets[(targetIndex+1)%2]) + filter.apply( + if (pass == 0) { + if (sourceCropped == null) source else arrayOf(sourceCropped!!) + } else targets[targetIndex % 2], targets[(targetIndex + 1) % 2] + ) targetIndex++ } + + /* + Perform vertical steps + */ filter.passDirection = Vector2.UNIT_Y for (pass in ySampleCounts.indices) { filter.sampleCount = ySampleCounts[pass] filter.passIndex = pass - filter.apply( targets[targetIndex%2], targets[(targetIndex+1)%2]) + filter.apply(targets[targetIndex % 2], targets[(targetIndex + 1) % 2]) targetIndex++ } - if (targetIndex%2 == 1) { + // this is a bit wasteful + if (targetIndex % 2 == 1) { passthrough.apply(targets[1], targets[0]) } + + /* + When the source is not a power of two we copy from the padded target to the target + */ + if (targetPadded != null) { + targetPadded!!.copyTo(target[0], + sourceRectangle = IntRectangle(0, recHeight-source[0].effectiveHeight, source[0].effectiveWidth, source[0].effectiveHeight), + targetRectangle = IntRectangle(0, 0, source[0].effectiveWidth, source[0].effectiveHeight) + ) + } } } \ No newline at end of file diff --git a/orx-integral-image/src/main/resources/shaders/gl3/integral-image.frag b/orx-integral-image/src/main/resources/shaders/gl3/integral-image.frag index a6dd5bdb6..ade32a47e 100644 --- a/orx-integral-image/src/main/resources/shaders/gl3/integral-image.frag +++ b/orx-integral-image/src/main/resources/shaders/gl3/integral-image.frag @@ -1,5 +1,3 @@ -#version 330 core - uniform sampler2D tex0; in vec2 v_texCoord0; out vec4 o_color; @@ -11,16 +9,18 @@ uniform vec2 passDirection; void main() { - vec2 passOffset = vec2(pow(sampleCountBase, passIndex)) * (1.0/textureSize(tex0, 0)) * passDirection; + vec2 passOffset = vec2( + pow(float(sampleCountBase), + float(passIndex))) * (1.0 / vec2(textureSize(tex0, 0)) + ) * passDirection; vec2 uv0 = v_texCoord0; +// uv0.y = 1.0 - uv0.y; vec4 result = vec4(0.0); for (int i = 0; i < sampleCount; ++i) { - vec2 readUV = v_texCoord0 - vec2(i *passOffset); + vec2 readUV = v_texCoord0 - vec2(float(i) * passOffset); float factor = step(0.0, readUV.x) * step(0.0, readUV.y); result += factor * texture(tex0, readUV); } - o_color = result; - } \ No newline at end of file