From 54637522a13b5e7032131d99de12f3565e415a0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbigniew=20Zag=C3=B3rski?= <1507542+zbigg@users.noreply.github.com> Date: Tue, 10 Dec 2024 14:48:31 +0100 Subject: [PATCH 1/4] Improve Tilejson typings to include properties used in Carto platform (#37) --- src/sources/index.ts | 5 ++ src/sources/types.ts | 149 +++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 149 insertions(+), 5 deletions(-) diff --git a/src/sources/index.ts b/src/sources/index.ts index 4be769f..99028fa 100644 --- a/src/sources/index.ts +++ b/src/sources/index.ts @@ -4,6 +4,11 @@ export {SOURCE_DEFAULTS} from './base-source'; export type { + VectorLayer, + RasterMetadataBandStats, + RasterBandColorinterp, + RasterMetadataBand, + RasterMetadata, TilejsonResult, GeojsonResult, JsonResult, diff --git a/src/sources/types.ts b/src/sources/types.ts index b039d18..50ec91c 100644 --- a/src/sources/types.ts +++ b/src/sources/types.ts @@ -191,16 +191,51 @@ export interface Tilejson { attribution: string; scheme: string; tiles: string[]; - properties_tiles: string[]; - minresolution: number; - maxresolution: number; minzoom: number; maxzoom: number; - bounds: [number, number, number, number]; - center: [number, number, number]; + bounds: [left: number, bottom: number, right: number, top: number]; + center: [longitute: number, latitude: number, zoom: number]; vector_layers: VectorLayer[]; + + // + // Carto additions over standard Tilejson properties + // + + minresolution: number; + maxresolution: number; + properties_tiles: string[]; tilestats: Tilestats; tileResolution?: TileResolution; + + /** + * Resolution of data in spatial-index dataset (e.g. H3, Quadbin). + * + * @internal + */ + dataresolution?: number; + + /** + * Array of ratios of dropped features per zoom level. + * + * Example: `[0,0,0.5]` - means that 50% of features are dropped at zoom 2 and bigger. + * + * @internal + */ + fraction_dropped_per_zoom?: number[]; + + /** + * Names of bands - rasters only. + * + * @internal + */ + raster_bands?: string[]; + + /** + * Raster metadata - rasters only. + * + * @internal + */ + raster_metadata?: RasterMetadata; } export interface Tilestats { @@ -221,12 +256,116 @@ export interface Attribute { } export interface VectorLayer { + // tilejson standard id: string; minzoom: number; maxzoom: number; fields: Record; + + // Carto additions over standard Tilejson properties + geometry_type?: string; } +export type RasterMetadataBandStats = { + approximated_stats?: boolean; + min: number; + max: number; + mean: number; + stddev: number; + sum: number; + sum_squares: number; + count: number; + + /** + * Quantiles by number of buckets. + * + * Example: + * ```ts + * { + * // for 3 buckets, first 1/3 of items lies in range [min, 20], second 1/3 is in [20, 40], and last 1/3 is in [40, max] + * 3: [20, 40], + * 4: [20, 30, 50], for 4 buckets ... + * } + * ``` + */ + quantiles?: Record; + + /** + * Top values by number of values. + * + * Key of dictionary is value, value is count. + * Key order is random. + * + * Example: + * ``` + * { + * 3: 5, // means there are 5 pixels with value 3 + * 11: 222, + * 12: 333, // means that 12 is most common value with count 333 + * ... // (assuming 333 was largest value in dict) + * } + * ``` + */ + top_values?: Record; + + /** + * Raster loader version. + */ + version?: string; +}; + +export enum RasterBandColorinterp { + Gray = 'gray', + Red = 'red', + Green = 'green', + Blue = 'blue', + Alpha = 'alpha', + Palette = 'palette', +} + +export type RasterMetadataBand = { + type: string; + name: string; + stats: RasterMetadataBandStats; + /** + * Known values: + * * `palette`: use unique value and `colortable` ad default mapping + * * `red`, `green`, `blue`: use the band as color channel + * * `gray`: use the band as grayscale + */ + colorinterp?: string | RasterBandColorinterp; // use RasterBandColorinterp, but it's external value, so let's use string + + /** + * Default color mapping for unique values (or if coloprinterp is `palette`) + */ + colortable?: Record; + + /** + * No value representation. + * Observed values: + * * `'nan'` for `NaN` + * * `number`: both as string as number, so parsing is needed + */ + nodata: string | number; // 255, '0', 'nan' +}; + +export type RasterMetadata = { + block_resolution: number; + minresolution: number; + maxresolution: number; + nodata: number | string; + bands: RasterMetadataBand[]; + bounds: [left: number, bottom: number, right: number, top: number]; + center: [longitute: number, latitude: number, zoom: number]; + width: number; + height: number; + block_width: number; + block_height: number; + num_blocks: number; + num_pixels: number; + pixel_resolution: number; +}; + export type TilejsonResult = Tilejson & {accessToken: string}; export type GeojsonResult = {type: 'FeatureCollection'; features: Feature[]}; export type JsonResult = any[]; From 5681c7c96a676b1047aaa04f0363a7d8ce853a2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbigniew=20Zag=C3=B3rski?= <1507542+zbigg@users.noreply.github.com> Date: Wed, 11 Dec 2024 16:29:43 +0100 Subject: [PATCH 2/4] enable control of cache behavior for source and query APIs (#33) --- CHANGELOG.md | 4 ++++ src/api/query.ts | 2 ++ src/api/request-with-parameters.ts | 38 ++++++++++++++++++++++++++---- src/sources/base-source.ts | 5 +++- src/sources/types.ts | 21 +++++++++++++++++ 5 files changed, 65 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e75851e..fa9dd2c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # CHANGELOG +## Not released + +- add cache control mechanism for sources and query APIs + ## 0.4 ### 0.4.0 diff --git a/src/api/query.ts b/src/api/query.ts index 3678038..b21b1b8 100644 --- a/src/api/query.ts +++ b/src/api/query.ts @@ -23,6 +23,7 @@ export const query = async function ( apiBaseUrl = SOURCE_DEFAULTS.apiBaseUrl, clientId = SOURCE_DEFAULTS.clientId, maxLengthURL = SOURCE_DEFAULTS.maxLengthURL, + localCache, connectionName, sqlQuery, queryParameters, @@ -52,5 +53,6 @@ export const query = async function ( headers, errorContext, maxLengthURL, + localCache, }); }; diff --git a/src/api/request-with-parameters.ts b/src/api/request-with-parameters.ts index 0cffe1a..4261d17 100644 --- a/src/api/request-with-parameters.ts +++ b/src/api/request-with-parameters.ts @@ -7,13 +7,14 @@ import {CartoAPIError, APIErrorContext} from './carto-api-error'; import {V3_MINOR_VERSION} from '../constants-internal'; import {DEFAULT_MAX_LENGTH_URL} from '../constants-internal'; import {getClient} from '../client'; +import {LocalCacheOptions} from '../sources/types'; const DEFAULT_HEADERS = { Accept: 'application/json', 'Content-Type': 'application/json', }; -const REQUEST_CACHE = new Map>(); +const DEFAULT_REQUEST_CACHE = new Map>(); export async function requestWithParameters({ baseUrl, @@ -21,12 +22,14 @@ export async function requestWithParameters({ headers: customHeaders = {}, errorContext, maxLengthURL = DEFAULT_MAX_LENGTH_URL, + localCache, }: { baseUrl: string; parameters?: Record; headers?: Record; errorContext: APIErrorContext; maxLengthURL?: number; + localCache?: LocalCacheOptions; }): Promise { // Parameters added to all requests issued with `requestWithParameters()`. // These parameters override parameters already in the base URL, but not @@ -41,7 +44,14 @@ export async function requestWithParameters({ baseUrl = excludeURLParameters(baseUrl, Object.keys(parameters)); const key = createCacheKey(baseUrl, parameters, customHeaders); - if (REQUEST_CACHE.has(key)) { + + const { + cache: REQUEST_CACHE, + canReadCache, + canStoreInCache, + } = getCacheSettings(localCache); + + if (canReadCache && REQUEST_CACHE.has(key)) { return REQUEST_CACHE.get(key) as Promise; } @@ -73,14 +83,34 @@ export async function requestWithParameters({ return json; }) .catch((error: Error) => { - REQUEST_CACHE.delete(key); + if (canStoreInCache) { + REQUEST_CACHE.delete(key); + } throw new CartoAPIError(error, errorContext, response, responseJson); }); - REQUEST_CACHE.set(key, jsonPromise); + if (canStoreInCache) { + REQUEST_CACHE.set(key, jsonPromise); + } return jsonPromise; } +function getCacheSettings(localCache: LocalCacheOptions | undefined) { + const canReadCache = localCache?.cacheControl?.includes('no-cache') + ? false + : true; + const canStoreInCache = localCache?.cacheControl?.includes('no-store') + ? false + : true; + const cache = localCache?.cache || DEFAULT_REQUEST_CACHE; + + return { + cache, + canReadCache, + canStoreInCache, + }; +} + function createCacheKey( baseUrl: string, parameters: Record, diff --git a/src/sources/base-source.ts b/src/sources/base-source.ts index 3b2f530..ac5878e 100644 --- a/src/sources/base-source.ts +++ b/src/sources/base-source.ts @@ -45,7 +45,7 @@ export async function baseSource>( } } const baseUrl = buildSourceUrl(mergedOptions); - const {clientId, maxLengthURL, format} = mergedOptions; + const {clientId, maxLengthURL, format, localCache} = mergedOptions; const headers = { Authorization: `Bearer ${options.accessToken}`, ...options.headers, @@ -65,6 +65,7 @@ export async function baseSource>( headers, errorContext, maxLengthURL, + localCache, }); const dataUrl = mapInstantiation[format].url[0]; @@ -82,6 +83,7 @@ export async function baseSource>( headers, errorContext, maxLengthURL, + localCache, }); if (accessToken) { json.accessToken = accessToken; @@ -94,5 +96,6 @@ export async function baseSource>( headers, errorContext, maxLengthURL, + localCache, }); } diff --git a/src/sources/types.ts b/src/sources/types.ts index 50ec91c..f61ec7d 100644 --- a/src/sources/types.ts +++ b/src/sources/types.ts @@ -47,6 +47,27 @@ export type SourceOptionalOptions = { * @default {@link DEFAULT_MAX_LENGTH_URL} */ maxLengthURL?: number; + + /** + * By default, local in-memory caching is enabled. + */ + localCache?: LocalCacheOptions; +}; + +export type LocalCacheOptions = { + /** + * Map that stores requests and their responses. + */ + cache?: Map>; + + /** + * Cache control + * * `no-cache`: If present, the source will always fetch from original source. + * * `no-store`: If present, source will not store result in cache (for later reuse). + * + * See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control#directives + */ + cacheControl?: ('no-cache' | 'no-store')[]; }; export type SourceOptions = SourceRequiredOptions & From abc23cea4cb6c3eb8160ee9cb969f7f44253be39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Arag=C3=B3n?= Date: Thu, 12 Dec 2024 16:35:09 +0100 Subject: [PATCH 3/4] chore(release): v0.4.1 (#43) --- CHANGELOG.md | 6 ++++-- package.json | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fa9dd2c..51dfea0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,10 +2,12 @@ ## Not released -- add cache control mechanism for sources and query APIs - ## 0.4 +### 0.4.1 + +- add cache control mechanism for sources and query APIs + ### 0.4.0 - feat: Add Picking Model API diff --git a/package.json b/package.json index 344203d..0b52505 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "repository": "github:CartoDB/carto-api-client", "author": "Don McCurdy ", "packageManager": "yarn@4.3.1", - "version": "0.4.0", + "version": "0.4.1", "license": "MIT", "publishConfig": { "access": "public", From 7b73fe43f6330ebbc128d393b992660543d91d87 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 12 Dec 2024 16:39:55 +0100 Subject: [PATCH 4/4] chore(deps): bump nanoid from 3.3.7 to 3.3.8 (#44) --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index 652234c..603a299 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7202,11 +7202,11 @@ __metadata: linkType: hard "nanoid@npm:^3.3.7": - version: 3.3.7 - resolution: "nanoid@npm:3.3.7" + version: 3.3.8 + resolution: "nanoid@npm:3.3.8" bin: nanoid: bin/nanoid.cjs - checksum: 10c0/e3fb661aa083454f40500473bb69eedb85dc160e763150b9a2c567c7e9ff560ce028a9f833123b618a6ea742e311138b591910e795614a629029e86e180660f3 + checksum: 10c0/4b1bb29f6cfebf3be3bc4ad1f1296fb0a10a3043a79f34fbffe75d1621b4318319211cd420549459018ea3592f0d2f159247a6f874911d6d26eaaadda2478120 languageName: node linkType: hard