Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow replacing fetch tile and decode image for local Dem manager #359

Merged
merged 11 commits into from
Dec 20, 2024
28 changes: 14 additions & 14 deletions benchmark.js
Original file line number Diff line number Diff line change
Expand Up @@ -26,21 +26,21 @@
return buffer;
}

const awsManager = new LocalDemManager(
"https://elevation-tiles-prod.s3.amazonaws.com/terrarium/{z}/{x}/{y}.png",
100,
"terrarium",
12,
10_000,
);
const awsManager = new LocalDemManager({
demUrlPattern: "https://elevation-tiles-prod.s3.amazonaws.com/terrarium/{z}/{x}/{y}.png",
cacheSize: 100,
encoding: "terrarium",
maxzoom: 12,
timeoutMs: 10_000,
});

const demoManager = new LocalDemManager(
"https://demotiles.maplibre.org/terrain-tiles/{z}/{x}/{y}.png",
100,
"mapbox",
11,
10_000,
);
const demoManager = new LocalDemManager({
demUrlPattern: "https://demotiles.maplibre.org/terrain-tiles/{z}/{x}/{y}.png",
cacheSize: 100,
encoding: "mapbox",
maxzoom: 11,
timeoutMs: 10_000,
});
let noMoreFetch = false;
if (typeof document === "undefined") {
demoManager.decodeImage = awsManager.decodeImage = (blob, encoding) =>
Expand Down
16 changes: 10 additions & 6 deletions src/dem-source.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,12 @@
import type { DemManager } from "./dem-manager";
import { LocalDemManager } from "./dem-manager";
import { LocalDemManager } from "./local-dem-manager";
import { decodeOptions, encodeOptions, getOptionsForZoom } from "./utils";
import RemoteDemManager from "./remote-dem-manager";
import type { DemTile, GlobalContourTileOptions, Timing } from "./types";
import type {
DemManager,
DemTile,
GlobalContourTileOptions,
Timing,
} from "./types";
import type WorkerDispatch from "./worker-dispatch";
import Actor from "./actor";
import { Timer } from "./performance";
Expand Down Expand Up @@ -129,14 +133,14 @@ export class DemSource {
this.sharedDemProtocolUrl = `${this.sharedDemProtocolId}://{z}/{x}/{y}`;
this.contourProtocolUrlBase = `${this.contourProtocolId}://{z}/{x}/{y}`;
const ManagerClass = worker ? RemoteDemManager : LocalDemManager;
this.manager = new ManagerClass(
url,
this.manager = new ManagerClass({
demUrlPattern: url,
cacheSize,
encoding,
maxzoom,
timeoutMs,
actor,
);
});
}

/** Registers a callback to be invoked with a performance report after each tile is requested. */
Expand Down
26 changes: 26 additions & 0 deletions src/e2e.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import { MainThreadDispatch } from "./remote-dem-manager";
import type { DemTile, Timing } from "./types";
import { VectorTile } from "@mapbox/vector-tile";
import Pbf from "pbf";
import { LocalDemManager } from "./local-dem-manager";

beforeEach(() => {
jest.useFakeTimers({ now: 0, doNotFake: ["performance"] });
Expand Down Expand Up @@ -282,3 +283,28 @@ test("decode image from worker", async () => {
data: expectedElevations,
});
});

test("fake decode image and fetch tile", async () => {
const getTileSpy = jest.fn().mockReturnValue(Promise.resolve({}));
const demManager = new LocalDemManager({
demUrlPattern: "https://example/{z}/{x}/{y}.png",
cacheSize: 100,
encoding: "terrarium",
maxzoom: 11,
timeoutMs: 10000,
decodeImage: async () => ({
width: 4,
height: 4,
data: expectedElevations,
}),
getTile: getTileSpy,
});
const demTile = await demManager.fetchAndParseTile(
1,
2,
3,
new AbortController(),
);
expect(demTile.data).toEqual(expectedElevations);
expect(getTileSpy.mock.calls[0][0]).toBe("https://example/1/2/3.png");
});
2 changes: 1 addition & 1 deletion src/index.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import generateIsolines from "./isolines";
import { DemSource } from "./dem-source";
import { decodeParsedImage } from "./decode-image";
import { LocalDemManager } from "./dem-manager";
import { LocalDemManager } from "./local-dem-manager";
import CONFIG from "./config";
import { HeightTile } from "./height-tile";

Expand Down
98 changes: 35 additions & 63 deletions src/dem-manager.ts → src/local-dem-manager.ts
Original file line number Diff line number Diff line change
@@ -1,47 +1,39 @@
import AsyncCache from "./cache";
import decodeImage from "./decode-image";
import defaultDecodeImage from "./decode-image";
import { HeightTile } from "./height-tile";
import generateIsolines from "./isolines";
import { encodeIndividualOptions, isAborted, withTimeout } from "./utils";
import type {
ContourTile,
DecodeImageFunction,
DemManager,
DemManagerInitizlizationParameters,
DemTile,
Encoding,
FetchResponse,
GetTileFunction,
IndividualContourTileOptions,
} from "./types";
import encodeVectorTile, { GeomType } from "./vtpbf";
import { Timer } from "./performance";

/**
* Holds cached tile state, and exposes `fetchContourTile` which fetches the necessary
* tiles and returns an encoded contour vector tiles.
*/
export interface DemManager {
loaded: Promise<any>;
fetchTile(
z: number,
x: number,
y: number,
abortController: AbortController,
timer?: Timer,
): Promise<FetchResponse>;
fetchAndParseTile(
z: number,
x: number,
y: number,
abortController: AbortController,
timer?: Timer,
): Promise<DemTile>;
fetchContourTile(
z: number,
x: number,
y: number,
options: IndividualContourTileOptions,
abortController: AbortController,
timer?: Timer,
): Promise<ContourTile>;
}
const defaultGetTile: GetTileFunction = async (
url: string,
abortController: AbortController,
) => {
const options: RequestInit = {
signal: abortController.signal,
};
const response = await fetch(url, options);
if (!response.ok) {
throw new Error(`Bad response: ${response.status} for ${url}`);
}
return {
data: await response.blob(),
expires: response.headers.get("expires") || undefined,
cacheControl: response.headers.get("cache-control") || undefined,
};
};

/**
* Caches, decodes, and processes raster tiles in the current thread.
Expand All @@ -55,26 +47,19 @@ export class LocalDemManager implements DemManager {
maxzoom: number;
timeoutMs: number;
loaded = Promise.resolve();
decodeImage: (
blob: Blob,
encoding: Encoding,
abortController: AbortController,
) => Promise<DemTile> = decodeImage;
decodeImage: DecodeImageFunction;
getTile: GetTileFunction;

constructor(
demUrlPattern: string,
cacheSize: number,
encoding: Encoding,
maxzoom: number,
timeoutMs: number,
) {
this.tileCache = new AsyncCache(cacheSize);
this.parsedCache = new AsyncCache(cacheSize);
this.contourCache = new AsyncCache(cacheSize);
this.timeoutMs = timeoutMs;
this.demUrlPattern = demUrlPattern;
this.encoding = encoding;
this.maxzoom = maxzoom;
constructor(options: DemManagerInitizlizationParameters) {
HarelM marked this conversation as resolved.
Show resolved Hide resolved
this.tileCache = new AsyncCache(options.cacheSize);
this.parsedCache = new AsyncCache(options.cacheSize);
this.contourCache = new AsyncCache(options.cacheSize);
this.timeoutMs = options.timeoutMs;
this.demUrlPattern = options.demUrlPattern;
this.encoding = options.encoding;
this.maxzoom = options.maxzoom;
this.decodeImage = options.decodeImage || defaultDecodeImage;
this.getTile = options.getTile || defaultGetTile;
}

fetchTile(
Expand All @@ -92,24 +77,11 @@ export class LocalDemManager implements DemManager {
return this.tileCache.get(
url,
(_, childAbortController) => {
const options: RequestInit = {
signal: childAbortController.signal,
};
timer?.fetchTile(url);
const mark = timer?.marker("fetch");
msbarry marked this conversation as resolved.
Show resolved Hide resolved
return withTimeout(
this.timeoutMs,
fetch(url, options).then(async (response) => {
mark?.();
if (!response.ok) {
throw new Error(`Bad response: ${response.status} for ${url}`);
}
return {
data: await response.blob(),
expires: response.headers.get("expires") || undefined,
cacheControl: response.headers.get("cache-control") || undefined,
};
}),
this.getTile(url, childAbortController).finally(() => mark?.()),
childAbortController,
);
},
Expand Down
20 changes: 5 additions & 15 deletions src/remote-dem-manager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,11 @@ import Actor from "./actor";
import CONFIG from "./config";
import type WorkerDispatch from "./worker-dispatch";
import decodeImage from "./decode-image";
import type { DemManager } from "./dem-manager";
import { Timer } from "./performance";
import type {
ContourTile,
DemManager,
DemManagerInitizlizationParameters,
DemTile,
Encoding,
FetchResponse,
Expand Down Expand Up @@ -41,28 +42,17 @@ export default class RemoteDemManager implements DemManager {
actor: Actor<WorkerDispatch>;
loaded: Promise<any>;

constructor(
demUrlPattern: string,
cacheSize: number,
encoding: Encoding,
maxzoom: number,
timeoutMs: number,
actor?: Actor<WorkerDispatch>,
) {
constructor(options: DemManagerInitizlizationParameters) {
const managerId = (this.managerId = ++id);
this.actor = actor || defaultActor();
this.actor = options.actor || defaultActor();
this.loaded = this.actor.send(
"init",
[],
new AbortController(),
undefined,
{
cacheSize,
demUrlPattern,
encoding,
maxzoom,
...options,
managerId,
timeoutMs,
},
);
}
Expand Down
73 changes: 64 additions & 9 deletions src/types.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
import type Actor from "./actor";
import type { Timer } from "./performance";
import type WorkerDispatch from "./worker-dispatch";

/** Scheme used to map pixel rgb values elevations. */
export type Encoding = "terrarium" | "mapbox";
export interface IsTransferrable {
Expand Down Expand Up @@ -77,15 +81,6 @@ export interface Image {
data: Uint8Array;
}

export interface InitMessage {
managerId: number;
demUrlPattern: string;
cacheSize: number;
encoding: Encoding;
maxzoom: number;
timeoutMs: number;
}

export type TimingCategory = "main" | "worker" | "fetch" | "decode" | "isoline";

/** Performance profile for a tile request */
Expand Down Expand Up @@ -114,3 +109,63 @@ export interface Timing {
/** If the tile failed with an error */
error?: boolean;
}

/**
* Holds cached tile state, and exposes `fetchContourTile` which fetches the necessary
* tiles and returns an encoded contour vector tiles.
*/
export interface DemManager {
loaded: Promise<any>;
fetchTile(
z: number,
x: number,
y: number,
abortController: AbortController,
timer?: Timer,
): Promise<FetchResponse>;
fetchAndParseTile(
z: number,
x: number,
y: number,
abortController: AbortController,
timer?: Timer,
): Promise<DemTile>;
fetchContourTile(
z: number,
x: number,
y: number,
options: IndividualContourTileOptions,
abortController: AbortController,
timer?: Timer,
): Promise<ContourTile>;
}

export type GetTileFunction = (
url: string,
abortController: AbortController,
) => Promise<FetchResponse>;

export type DecodeImageFunction = (
blob: Blob,
encoding: Encoding,
abortController: AbortController,
) => Promise<DemTile>;

export type DemManagerRequiredInitializationParameters = {
demUrlPattern: string;
cacheSize: number;
encoding: Encoding;
maxzoom: number;
timeoutMs: number;
};

export type DemManagerInitizlizationParameters =
DemManagerRequiredInitializationParameters & {
decodeImage?: DecodeImageFunction;
getTile?: GetTileFunction;
actor?: Actor<WorkerDispatch>;
};

export type InitMessage = DemManagerRequiredInitializationParameters & {
managerId: number;
};
Loading