From b6c4f0b69e05f3d2952beec0b17cf96234556a2d Mon Sep 17 00:00:00 2001 From: Calixte Denizet Date: Wed, 16 Oct 2024 16:36:46 +0200 Subject: [PATCH] Use ImageDecoder in order to decode jpeg images (bug 1901223) --- src/core/base_stream.js | 4 ++ src/core/image.js | 22 +++++++ src/core/jpeg_stream.js | 101 ++++++++++++++++++++++++------ src/core/jpg.js | 133 ++++++++++++++++++++++++++++------------ src/display/canvas.js | 6 +- 5 files changed, 207 insertions(+), 59 deletions(-) diff --git a/src/core/base_stream.js b/src/core/base_stream.js index 7f79a147c1f13..febb4ed109c15 100644 --- a/src/core/base_stream.js +++ b/src/core/base_stream.js @@ -68,6 +68,10 @@ class BaseStream { return false; } + async getTransferableImage() { + return null; + } + peekByte() { const peekedByte = this.getByte(); if (peekedByte !== -1) { diff --git a/src/core/image.js b/src/core/image.js index 876f4e5a46fc8..d384935d75361 100644 --- a/src/core/image.js +++ b/src/core/image.js @@ -752,6 +752,10 @@ class PDFImage { drawWidth === originalWidth && drawHeight === originalHeight ) { + const image = await this.#getImage(originalWidth, originalHeight); + if (image) { + return image; + } const data = await this.getImageBytes(originalHeight * rowBytes, {}); if (isOffscreenCanvasSupported) { if (mustBeResized) { @@ -810,6 +814,10 @@ class PDFImage { } if (isHandled) { + const image = await this.#getImage(drawWidth, drawHeight); + if (image) { + return image; + } const rgba = await this.getImageBytes(imageLength, { drawWidth, drawHeight, @@ -1013,6 +1021,20 @@ class PDFImage { }; } + async #getImage(width, height) { + const bitmap = await this.image.getTransferableImage(); + if (!bitmap) { + return null; + } + return { + data: null, + width, + height, + bitmap, + interpolate: this.interpolate, + }; + } + async getImageBytes( length, { diff --git a/src/core/jpeg_stream.js b/src/core/jpeg_stream.js index 32cacae59b889..6cf2de2cba764 100644 --- a/src/core/jpeg_stream.js +++ b/src/core/jpeg_stream.js @@ -13,10 +13,10 @@ * limitations under the License. */ +import { shadow, warn } from "../shared/util.js"; import { DecodeStream } from "./decode_stream.js"; import { Dict } from "./primitives.js"; import { JpegImage } from "./jpg.js"; -import { shadow } from "../shared/util.js"; /** * For JPEG's we use a library to decode these images and the stream behaves @@ -32,6 +32,18 @@ class JpegStream extends DecodeStream { this.params = params; } + static get canUseImageDecoder() { + return shadow( + this, + "canUseImageDecoder", + // eslint-disable-next-line no-undef + typeof ImageDecoder === "undefined" + ? Promise.resolve(false) + : // eslint-disable-next-line no-undef + ImageDecoder.isTypeSupported("image/jpeg") + ); + } + get bytes() { // If `this.maybeLength` is null, we'll get the entire stream. return shadow(this, "bytes", this.stream.getBytes(this.maybeLength)); @@ -46,22 +58,7 @@ class JpegStream extends DecodeStream { this.decodeImage(); } - decodeImage(bytes) { - if (this.eof) { - return this.buffer; - } - bytes ||= this.bytes; - - // Some images may contain 'junk' before the SOI (start-of-image) marker. - // Note: this seems to mainly affect inline images. - for (let i = 0, ii = bytes.length - 1; i < ii; i++) { - if (bytes[i] === 0xff && bytes[i + 1] === 0xd8) { - if (i > 0) { - bytes = bytes.subarray(i); - } - break; - } - } + get jpegOptions() { const jpegOptions = { decodeTransform: undefined, colorTransform: undefined, @@ -93,8 +90,34 @@ class JpegStream extends DecodeStream { jpegOptions.colorTransform = colorTransform; } } - const jpegImage = new JpegImage(jpegOptions); + return shadow(this, "jpegOptions", jpegOptions); + } + + #skipUselessBytes(data) { + // Some images may contain 'junk' before the SOI (start-of-image) marker. + // Note: this seems to mainly affect inline images. + for (let i = 0, ii = data.length - 1; i < ii; i++) { + if (data[i] === 0xff && data[i + 1] === 0xd8) { + if (i > 0) { + data = data.subarray(i); + } + break; + } + } + return data; + } + + decodeImage(bytes) { + if (this.eof) { + return this.buffer; + } + bytes = this.#skipUselessBytes(bytes || this.bytes); + // TODO: if an image has a mask we need to combine the data. + // So ideally get a VideoFrame from getTransferableImage and then use + // copyTo. + + const jpegImage = new JpegImage(this.jpegOptions); jpegImage.parse(bytes); const data = jpegImage.getData({ width: this.drawWidth, @@ -113,6 +136,48 @@ class JpegStream extends DecodeStream { get canAsyncDecodeImageFromBuffer() { return this.stream.isAsync; } + + async getTransferableImage() { + if (!(await JpegStream.canUseImageDecoder)) { + return null; + } + const jpegOptions = this.jpegOptions; + if (jpegOptions.decodeTransform) { + // TODO: We could decode the image thanks to ImageDecoder and then + // get the pixels with copyTo and apply the decodeTransform. + return null; + } + let decoder; + try { + // TODO: If the stream is Flate & DCT we could try to just pipe the + // the DecompressionStream into the ImageDecoder: it'll avoid the + // intermediate ArrayBuffer. + const bytes = + (this.canAsyncDecodeImageFromBuffer && + (await this.stream.asyncGetBytes())) || + this.bytes; + if (!bytes) { + return null; + } + const data = this.#skipUselessBytes(bytes); + if (!JpegImage.canUseImageDecoder(data, jpegOptions.colorTransform)) { + return null; + } + // eslint-disable-next-line no-undef + decoder = new ImageDecoder({ + data, + type: "image/jpeg", + preferAnimation: false, + }); + + return (await decoder.decode()).image; + } catch (reason) { + warn(`getTransferableImage - failed: "${reason}".`); + return null; + } finally { + decoder?.close(); + } + } } export { JpegStream }; diff --git a/src/core/jpg.js b/src/core/jpg.js index 82f4fee297380..a7640a6d46dc1 100644 --- a/src/core/jpg.js +++ b/src/core/jpg.js @@ -744,55 +744,109 @@ function findNextFileMarker(data, currentPos, startPos = currentPos) { }; } +function prepareComponents(frame) { + const mcusPerLine = Math.ceil(frame.samplesPerLine / 8 / frame.maxH); + const mcusPerColumn = Math.ceil(frame.scanLines / 8 / frame.maxV); + for (const component of frame.components) { + const blocksPerLine = Math.ceil( + (Math.ceil(frame.samplesPerLine / 8) * component.h) / frame.maxH + ); + const blocksPerColumn = Math.ceil( + (Math.ceil(frame.scanLines / 8) * component.v) / frame.maxV + ); + const blocksPerLineForMcu = mcusPerLine * component.h; + const blocksPerColumnForMcu = mcusPerColumn * component.v; + + const blocksBufferSize = + 64 * blocksPerColumnForMcu * (blocksPerLineForMcu + 1); + component.blockData = new Int16Array(blocksBufferSize); + component.blocksPerLine = blocksPerLine; + component.blocksPerColumn = blocksPerColumn; + } + frame.mcusPerLine = mcusPerLine; + frame.mcusPerColumn = mcusPerColumn; +} + +function readDataBlock(data, offset) { + const length = readUint16(data, offset); + offset += 2; + let endOffset = offset + length - 2; + + const fileMarker = findNextFileMarker(data, endOffset, offset); + if (fileMarker?.invalid) { + warn( + "readDataBlock - incorrect length, current marker is: " + + fileMarker.invalid + ); + endOffset = fileMarker.offset; + } + + const array = data.subarray(offset, endOffset); + offset += array.length; + return { appData: array, newOffset: offset }; +} + +function skipData(data, offset) { + const length = readUint16(data, offset); + offset += 2; + const endOffset = offset + length - 2; + + const fileMarker = findNextFileMarker(data, endOffset, offset); + if (fileMarker?.invalid) { + return fileMarker.offset; + } + return endOffset; +} + class JpegImage { constructor({ decodeTransform = null, colorTransform = -1 } = {}) { this._decodeTransform = decodeTransform; this._colorTransform = colorTransform; } - parse(data, { dnlScanLines = null } = {}) { - function readDataBlock() { - const length = readUint16(data, offset); - offset += 2; - let endOffset = offset + length - 2; - - const fileMarker = findNextFileMarker(data, endOffset, offset); - if (fileMarker?.invalid) { - warn( - "readDataBlock - incorrect length, current marker is: " + - fileMarker.invalid - ); - endOffset = fileMarker.offset; - } - - const array = data.subarray(offset, endOffset); - offset += array.length; - return array; + static canUseImageDecoder(data, colorTransform = -1) { + let offset = 0; + let numComponents = null; + let fileMarker = readUint16(data, offset); + offset += 2; + if (fileMarker !== /* SOI (Start of Image) = */ 0xffd8) { + throw new JpegError("SOI not found"); } + fileMarker = readUint16(data, offset); + offset += 2; - function prepareComponents(frame) { - const mcusPerLine = Math.ceil(frame.samplesPerLine / 8 / frame.maxH); - const mcusPerColumn = Math.ceil(frame.scanLines / 8 / frame.maxV); - for (const component of frame.components) { - const blocksPerLine = Math.ceil( - (Math.ceil(frame.samplesPerLine / 8) * component.h) / frame.maxH - ); - const blocksPerColumn = Math.ceil( - (Math.ceil(frame.scanLines / 8) * component.v) / frame.maxV - ); - const blocksPerLineForMcu = mcusPerLine * component.h; - const blocksPerColumnForMcu = mcusPerColumn * component.v; - - const blocksBufferSize = - 64 * blocksPerColumnForMcu * (blocksPerLineForMcu + 1); - component.blockData = new Int16Array(blocksBufferSize); - component.blocksPerLine = blocksPerLine; - component.blocksPerColumn = blocksPerColumn; + markerLoop: while (fileMarker !== /* EOI (End of Image) = */ 0xffd9) { + switch (fileMarker) { + case 0xffc0: // SOF0 (Start of Frame, Baseline DCT) + case 0xffc1: // SOF1 (Start of Frame, Extended DCT) + case 0xffc2: // SOF2 (Start of Frame, Progressive DCT) + // Skip marker length. + // Skip precision. + // Skip scanLines. + // Skip samplesPerLine. + numComponents = data[offset + (2 + 1 + 2 + 2)]; + break markerLoop; + case 0xffff: // Fill bytes + if (data[offset] !== 0xff) { + // Avoid skipping a valid marker. + offset--; + } + break; } - frame.mcusPerLine = mcusPerLine; - frame.mcusPerColumn = mcusPerColumn; + offset = skipData(data, offset); + fileMarker = readUint16(data, offset); + offset += 2; + } + if (numComponents === 4) { + return false; + } + if (numComponents === 3 && colorTransform === 0) { + return false; } + return true; + } + parse(data, { dnlScanLines = null } = {}) { let offset = 0; let jfif = null; let adobe = null; @@ -830,7 +884,8 @@ class JpegImage { case 0xffee: // APP14 case 0xffef: // APP15 case 0xfffe: // COM (Comment) - const appData = readDataBlock(); + const { appData, newOffset } = readDataBlock(data, offset); + offset = newOffset; if (fileMarker === 0xffe0) { // 'JFIF\x00' diff --git a/src/display/canvas.js b/src/display/canvas.js index 13f0790a9123b..814dc3d811f27 100644 --- a/src/display/canvas.js +++ b/src/display/canvas.js @@ -1059,8 +1059,10 @@ class CanvasGraphics { // Vertical or horizontal scaling shall not be more than 2 to not lose the // pixels during drawImage operation, painting on the temporary canvas(es) // that are twice smaller in size. - const width = img.width; - const height = img.height; + + // displayWidth and displayHeight are used for VideoFrame. + const width = img.width ?? img.displayWidth; + const height = img.height ?? img.displayHeight; let widthScale = Math.max( Math.hypot(inverseTransform[0], inverseTransform[1]), 1