diff --git a/.changeset/quick-roses-fail.md b/.changeset/quick-roses-fail.md new file mode 100644 index 00000000..60b8b5b0 --- /dev/null +++ b/.changeset/quick-roses-fail.md @@ -0,0 +1,5 @@ +--- +'vite-imagetools': patch +--- + +feat: caching of generated images diff --git a/docs/README.md b/docs/README.md index 39bd156a..4c7223ac 100644 --- a/docs/README.md +++ b/docs/README.md @@ -17,3 +17,4 @@ unclear! - [Extend Imagetools](guide/extending.md) - [Sharp's documentation](https://sharp.pixelplumbing.com) +- [Caching](guide/caching.md) \ No newline at end of file diff --git a/docs/guide/caching.md b/docs/guide/caching.md new file mode 100644 index 00000000..621a460b --- /dev/null +++ b/docs/guide/caching.md @@ -0,0 +1,39 @@ +# Caching + +To speed up a build pipeline with many images, the generated images can be cached on disk. +If the source image changes, the cached images will be regenerated. + +## How to enable caching + +To enable caching, the cache directory has to be configured. + +``` +// vite.config.js, etc +... + plugins: [ + react(), + imagetools({ + cacheDir: './node_modules/.cache/imagetools' + }) + ] +... +``` + +## Cache retention to remove unused images + +When an image is no longer there or the transformation parameters change, the previously +cached images will be removed after a configurable retention period. +The default retention is 86400 seconds. A value of 0 will disable this mechanism. + +``` +// vite.config.js, etc +... + plugins: [ + react(), + imagetools({ + cacheDir: './node_modules/.cache/imagetools', + cacheRetention: 172800 + }) + ] +... +``` diff --git a/docs/interfaces/vite_src_types.VitePluginOptions.md b/docs/interfaces/vite_src_types.VitePluginOptions.md index 68a0e0c1..5a6fd145 100644 --- a/docs/interfaces/vite_src_types.VitePluginOptions.md +++ b/docs/interfaces/vite_src_types.VitePluginOptions.md @@ -15,6 +15,8 @@ - [include](vite_src_types.VitePluginOptions.md#include) - [removeMetadata](vite_src_types.VitePluginOptions.md#removemetadata) - [resolveConfigs](vite_src_types.VitePluginOptions.md#resolveconfigs) +- [cacheRetention](vite_src_types.VitePluginOptions.md#cacheretention) +- [cacheDir](vite_src_types.VitePluginOptions.md#cachedir) ## Properties @@ -177,3 +179,35 @@ undefined #### Defined in [packages/vite/src/types.ts:79](https://github.com/JonasKruckenberg/imagetools/blob/4ebc88f/packages/vite/src/types.ts#L79) + +### cacheRetention + +• **cacheRetention**: `number` + +After what time an unused image will be removed from the cache. + +**`Default`** + +```ts +86400 +``` + +#### Defined in + +[packages/vite/src/types.ts:85](https://github.com/JonasKruckenberg/imagetools/blob/4ebc88f/packages/vite/src/types.ts#L97) + +### cacheDir + +• **cacheDir**: `string` + +Where to store generated images on disk as cache. + +**`Default`** + +```ts +undefined +``` + +#### Defined in + +[packages/vite/src/types.ts:85](https://github.com/JonasKruckenberg/imagetools/blob/4ebc88f/packages/vite/src/types.ts#L102) diff --git a/packages/vite/src/__tests__/__image_snapshots__/main-test-ts-src-tests-main-test-ts-vite-imagetools-import-with-space-in-identifier-and-cache-1-snap.png b/packages/vite/src/__tests__/__image_snapshots__/main-test-ts-src-tests-main-test-ts-vite-imagetools-import-with-space-in-identifier-and-cache-1-snap.png new file mode 100644 index 00000000..bd3de489 --- /dev/null +++ b/packages/vite/src/__tests__/__image_snapshots__/main-test-ts-src-tests-main-test-ts-vite-imagetools-import-with-space-in-identifier-and-cache-1-snap.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:895f2fa8cdb7759f75df4b2aff51518cbf1406e1455fdb36268cd129f60736a1 +size 120401 diff --git a/packages/vite/src/__tests__/main.test.ts b/packages/vite/src/__tests__/main.test.ts index 02f7404e..aa20773c 100644 --- a/packages/vite/src/__tests__/main.test.ts +++ b/packages/vite/src/__tests__/main.test.ts @@ -1,4 +1,4 @@ -import { build, createLogger } from 'vite' +import { InlineConfig, build, createLogger } from 'vite' import { imagetools } from '../index' import { join } from 'path' import { getFiles, testEntry } from './util' @@ -7,7 +7,15 @@ import { OutputAsset, OutputChunk, RollupOutput } from 'rollup' import { JSDOM } from 'jsdom' import sharp from 'sharp' import { afterEach, describe, test, expect, it, vi } from 'vitest' -import { createBasePath } from '../utils' +import { createBasePath, generateCacheID } from '../utils' +import { readFile, rm, utimes } from 'fs/promises' + +const processPath = process.cwd() + +const extractCreated = (path: string) => + readFile(path, { encoding: 'utf8' }) + .then((d) => JSON.parse(d).created as number) + .catch(() => undefined) expect.extend({ toMatchImageSnapshot }) @@ -458,6 +466,65 @@ describe('vite-imagetools', () => { expect(window.__IMAGE__).toHaveProperty('hasAlpha') }) }) + describe('cacheRetention', () => { + test('is used to clear cache with default 86400', async () => { + const cacheDir = './node_modules/.cache/imagetools_test_cacheRetention' + await rm(cacheDir, { recursive: true, force: true }) + const root = join(__dirname, '__fixtures__') + const config: (width: number) => InlineConfig = (width) => ({ + root, + logLevel: 'warn', + build: { write: false }, + plugins: [ + testEntry(` + import Image from "./pexels-allec-gomes-5195763.png?w=${width}" + export default Image + `), + imagetools({ cacheDir }) + ] + }) + await build(config(300)) + + const relativeRoot = root.startsWith(processPath) ? root.slice(processPath.length + 1) : root + const cacheID = generateCacheID(`${relativeRoot}/pexels-allec-gomes-5195763.png?w=300`) + const indexPath = `${cacheDir}/${cacheID}/index.json` + const created = await extractCreated(indexPath) + expect(created).toBeTypeOf('number') + + await build(config(200)) + expect(await extractCreated(indexPath)).toBe(created) + + const date = new Date(Date.now() - 86400000) + await utimes(indexPath, date, date) + await build(config(200)) + expect(await extractCreated(indexPath)).not.toBe(created) + }) + }) + describe('cacheDir', () => { + test('is used', async () => { + const cacheDir = './node_modules/.cache/imagetools_test_cacheRetention' + await rm(cacheDir, { recursive: true, force: true }) + const root = join(__dirname, '__fixtures__') + await build({ + root, + logLevel: 'warn', + build: { write: false }, + plugins: [ + testEntry(` + import Image from "./pexels-allec-gomes-5195763.png?w=300" + export default Image + `), + imagetools({ cacheDir }) + ] + }) + + const relativeRoot = root.startsWith(processPath) ? root.slice(processPath.length + 1) : root + const cacheID = generateCacheID(`${relativeRoot}/pexels-allec-gomes-5195763.png?w=300`) + const indexPath = `${cacheDir}/${cacheID}/index.json` + const created = await extractCreated(indexPath) + expect(created).toBeTypeOf('number') + }) + }) }) test('relative import', async () => { @@ -516,6 +583,28 @@ describe('vite-imagetools', () => { expect(files[0].source).toMatchImageSnapshot() }) + test('import with space in identifier and cache', async () => { + const cacheDir = './node_modules/.cache/imagetools_test_import_with_space' + await rm(cacheDir, { recursive: true, force: true }) + const config: InlineConfig = { + root: join(__dirname, '__fixtures__'), + logLevel: 'warn', + build: { write: false }, + plugins: [ + testEntry(` + import Image from "./with space.png?w=300" + export default Image + `), + imagetools({ cacheDir }) + ] + } + await build(config) + const bundle = (await build(config)) as RollupOutput | RollupOutput[] + + const files = getFiles(bundle, '**.png') as OutputAsset[] + expect(files[0].source).toMatchImageSnapshot() + }) + test('non existent file', async () => { const p = build({ root: join(__dirname, '__fixtures__'), diff --git a/packages/vite/src/index.ts b/packages/vite/src/index.ts index bc4f5b76..b4622eaa 100644 --- a/packages/vite/src/index.ts +++ b/packages/vite/src/index.ts @@ -1,4 +1,6 @@ import { basename, extname } from 'node:path' +import { createReadStream, existsSync } from 'node:fs' +import { mkdir, opendir, readFile, rm, stat, utimes, writeFile } from 'node:fs/promises' import type { Plugin, ResolvedConfig } from 'vite' import { applyTransforms, @@ -16,7 +18,7 @@ import { } from 'imagetools-core' import { createFilter, dataToEsm } from '@rollup/pluginutils' import sharp, { type Metadata, type Sharp } from 'sharp' -import { createBasePath, generateImageID } from './utils.js' +import { checksumFile, createBasePath, generateCacheID, generateImageID } from './utils.js' import type { VitePluginOptions } from './types.js' export type { @@ -32,7 +34,12 @@ export type { const defaultOptions: VitePluginOptions = { include: /^[^?]+\.(avif|gif|heif|jpeg|jpg|png|tiff|webp)(\?.*)?$/, exclude: 'public/**/*', - removeMetadata: true + removeMetadata: true, + cacheRetention: 86400 +} + +interface ProcessedCachableImageMetadata extends ProcessedImageMetadata { + imagePath?: string } export * from 'imagetools-core' @@ -51,7 +58,11 @@ export function imagetools(userOptions: Partial = {}): Plugin let viteConfig: ResolvedConfig let basePath: string - const generatedImages = new Map() + const processPath = process.cwd() + + const generatedImages = new Map() + + const isSharp = (image: Sharp | ProcessedCachableImageMetadata): image is Sharp => typeof image.clone === 'function' return { name: 'imagetools', @@ -91,29 +102,6 @@ export function imagetools(userOptions: Partial = {}): Plugin if (!directives.toString()) return null - const img = lazyLoadImage() - const widthParam = directives.get('w') - const heightParam = directives.get('h') - if (directives.get('allowUpscale') !== 'true' && (widthParam || heightParam)) { - const metadata = await lazyLoadMetadata() - const clamp = (s: string, intrinsic: number) => - [...new Set(s.split(';').map((d): string => (parseInt(d) <= intrinsic ? d : intrinsic.toString())))].join(';') - - if (widthParam) { - const intrinsicWidth = metadata.width || 0 - directives.set('w', clamp(widthParam, intrinsicWidth)) - } - - if (heightParam) { - const intrinsicHeight = metadata.height || 0 - directives.set('h', clamp(heightParam, intrinsicHeight)) - } - } - - const parameters = extractEntries(directives) - const imageConfigs = - pluginOptions.resolveConfigs?.(parameters, outputFormats) ?? resolveConfigs(parameters, outputFormats) - const outputMetadatas: Array = [] const logger: Logger = { @@ -122,27 +110,126 @@ export function imagetools(userOptions: Partial = {}): Plugin error: (msg) => this.error(msg) } - for (const config of imageConfigs) { - const { transforms } = generateTransforms(config, transformFactories, srcURL.searchParams, logger) - const { image, metadata } = await applyTransforms(transforms, img.clone(), pluginOptions.removeMetadata) - - if (viteConfig.command === 'serve') { - const id = await generateImageID(srcURL, config, img) - generatedImages.set(id, image) - metadata.src = basePath + id - } else { - const fileHandle = this.emitFile({ - name: basename(pathname, extname(pathname)) + `.${metadata.format}`, - source: await image.toBuffer(), - type: 'asset' - }) - - metadata.src = `__VITE_ASSET__${fileHandle}__` + const relativeID = id.startsWith(processPath) ? id.slice(processPath.length + 1) : id + const cacheID = pluginOptions.cacheDir ? generateCacheID(relativeID) : undefined + if (cacheID && pluginOptions.cacheDir && existsSync(`${pluginOptions.cacheDir}/${cacheID}/index.json`)) { + try { + const srcChecksum = await checksumFile('sha1', pathname) + const { checksum, metadatas } = JSON.parse( + await readFile(`${pluginOptions.cacheDir}/${cacheID}/index.json`, { encoding: 'utf8' }) + ) + + if (srcChecksum === checksum) { + const date = new Date() + utimes(`${pluginOptions.cacheDir}/${cacheID}/index.json`, date, date) + + for (const metadata of metadatas) { + if (viteConfig.command === 'serve') { + const imageID = metadata.imageID + generatedImages.set(imageID, metadata) + metadata.src = basePath + imageID + } else { + const fileHandle = this.emitFile({ + name: basename(pathname, extname(pathname)) + `.${metadata.format}`, + source: await readFile(metadata.imagePath), + type: 'asset' + }) + + metadata.src = `__VITE_ASSET__${fileHandle}__` + } + + outputMetadatas.push(metadata) + } + } + } catch (e) { + console.error('cache error:', e) + outputMetadatas.length = 0 } + } + + if (!outputMetadatas.length) { + const img = lazyLoadImage() + const widthParam = directives.get('w') + const heightParam = directives.get('h') + if (directives.get('allowUpscale') !== 'true' && (widthParam || heightParam)) { + const metadata = await lazyLoadMetadata() + const clamp = (s: string, intrinsic: number) => + [...new Set(s.split(';').map((d): string => (parseInt(d) <= intrinsic ? d : intrinsic.toString())))].join( + ';' + ) + + if (widthParam) { + const intrinsicWidth = metadata.width || 0 + directives.set('w', clamp(widthParam, intrinsicWidth)) + } - metadata.image = image + if (heightParam) { + const intrinsicHeight = metadata.height || 0 + directives.set('h', clamp(heightParam, intrinsicHeight)) + } + } - outputMetadatas.push(metadata as ProcessedImageMetadata) + const parameters = extractEntries(directives) + const imageConfigs = + pluginOptions.resolveConfigs?.(parameters, outputFormats) ?? resolveConfigs(parameters, outputFormats) + + for (const config of imageConfigs) { + const { transforms } = generateTransforms(config, transformFactories, srcURL.searchParams, logger) + const { image, metadata } = await applyTransforms(transforms, img.clone(), pluginOptions.removeMetadata) + const imageBuffer = await image.toBuffer() + const imageID = await generateImageID(srcURL, config, imageBuffer) + + if (viteConfig.command === 'serve') { + generatedImages.set(imageID, image) + metadata.src = basePath + imageID + } else { + const fileHandle = this.emitFile({ + name: basename(pathname, extname(pathname)) + `.${metadata.format}`, + source: imageBuffer, + type: 'asset' + }) + + metadata.src = `__VITE_ASSET__${fileHandle}__` + } + + metadata.imageID = imageID + metadata.image = image + + outputMetadatas.push(metadata as ProcessedImageMetadata) + } + + if (pluginOptions.cacheDir) { + const relativeID = id.startsWith(processPath) ? id.slice(processPath.length + 1) : id + const cacheID = generateCacheID(relativeID) + try { + const checksum = await checksumFile('sha1', pathname) + await mkdir(`${pluginOptions.cacheDir}/${cacheID}`, { recursive: true }) + await Promise.all( + outputMetadatas.map(async (metadata) => { + const { format, image, imageID } = metadata + const imagePath = `${pluginOptions.cacheDir}/${cacheID}/${imageID}.${format}` + if (image) await writeFile(imagePath, await image.toBuffer()) + metadata.imagePath = imagePath + if (viteConfig.command === 'serve') { + generatedImages.set(id, metadata) + } + }) + ) + await writeFile( + `${pluginOptions.cacheDir}/${cacheID}/index.json`, + JSON.stringify({ + checksum, + created: Date.now(), + // eslint-disable-next-line @typescript-eslint/no-unused-vars + metadatas: outputMetadatas.map(({ src, image, ...metadata }) => metadata) + }), + { encoding: 'utf8' } + ) + } catch (e) { + console.debug(`failed to create cache for ${cacheID}`) + await rm(`${pluginOptions.cacheDir}/${cacheID}`, { recursive: true }) + } + } } let outputFormat = urlFormat() @@ -172,17 +259,46 @@ export function imagetools(userOptions: Partial = {}): Plugin if (!image) throw new Error(`vite-imagetools cannot find image with id "${id}" this is likely an internal error`) - if (pluginOptions.removeMetadata === false) { - image.withMetadata() - } - - res.setHeader('Content-Type', `image/${getMetadata(image, 'format')}`) res.setHeader('Cache-Control', 'max-age=360000') - return image.clone().pipe(res) + + if (isSharp(image)) { + if (pluginOptions.removeMetadata === false) { + image.withMetadata() + } + + res.setHeader('Content-Type', `image/${getMetadata(image, 'format')}`) + return image.clone().pipe(res) + } else if (image.imagePath) { + res.setHeader('Content-Type', `image/${image.format}`) + return createReadStream(image.imagePath).pipe(res) + } else { + throw new Error(`vite-imagetools cannot find image with id "${id}" this is likely an internal error`) + } } next() }) + }, + + async buildEnd(error) { + if (!error && pluginOptions.cacheDir && pluginOptions.cacheRetention && viteConfig.command !== 'serve') { + const dir = await opendir(pluginOptions.cacheDir) + for await (const dirent of dir) { + if (dirent.isDirectory()) { + const cacheDir = `${pluginOptions.cacheDir}/${dirent.name}` + try { + const stats = await stat(`${cacheDir}/index.json`) + if (Date.now() - stats.mtimeMs > pluginOptions.cacheRetention * 1000) { + console.debug(`deleting stale cache dir ${dirent.name}`) + await rm(cacheDir, { recursive: true }) + } + } catch (e) { + console.debug(`deleting invalid cache dir ${dirent.name}`) + await rm(cacheDir, { recursive: true }) + } + } + } + } } } } diff --git a/packages/vite/src/types.ts b/packages/vite/src/types.ts index e2d600fa..797bed3c 100644 --- a/packages/vite/src/types.ts +++ b/packages/vite/src/types.ts @@ -90,4 +90,14 @@ export interface VitePluginOptions { * @default undefined */ namedExports?: boolean + + /** + * For how many seconds to keep transformed images cached. Default is 86400. To disable cache specify 0. + */ + cacheRetention?: number + + /** + * Where should the cached images be stored. Default is undefined + */ + cacheDir?: string } diff --git a/packages/vite/src/utils.ts b/packages/vite/src/utils.ts index 9eabd787..98d3128a 100644 --- a/packages/vite/src/utils.ts +++ b/packages/vite/src/utils.ts @@ -2,17 +2,16 @@ import { createHash } from 'node:crypto' import path from 'node:path' import { statSync } from 'node:fs' import type { ImageConfig } from 'imagetools-core' -import type { Sharp } from 'sharp' +import { createReadStream } from 'node:fs' export const createBasePath = (base?: string) => { return (base?.replace(/\/$/, '') || '') + '/@imagetools/' } -export async function generateImageID(url: URL, config: ImageConfig, originalImage: Sharp) { +export async function generateImageID(url: URL, config: ImageConfig, imageBuffer: Buffer) { if (url.host) { const baseURL = new URL(url.origin + url.pathname) - const buffer = await originalImage.toBuffer() - return hash([baseURL.href, JSON.stringify(config), buffer]) + return hash([baseURL.href, JSON.stringify(config), imageBuffer]) } // baseURL isn't a valid URL, but just a string used for an identifier @@ -29,3 +28,18 @@ function hash(keyParts: Array) { } return hash.digest('hex') } + +export const generateCacheID = (path: string) => createHash('sha1').update(path).digest('hex') + +export const checksumFile = (algorithm: string, path: string) => { + return new Promise(function (resolve, reject) { + const hash = createHash(algorithm).setEncoding('hex') + createReadStream(path) + .pipe(hash) + .on('error', reject) + .on('finish', () => { + hash.end() + resolve(hash.digest('hex')) + }) + }) +}