feat: ability to provide a checksum for global cache (#42)

auth_tokens.ts

@@ -46,6 +46,8 @@ export class AuthTokens {
  tokens.push({ type: "bearer", host, token });
  }
  } else {
+ // todo(dsherret): feel like this should error?
+ // deno-lint-ignore no-console
  console.error("Badly formed auth token discarded.");
  }
  }

cache.ts

@@ -18,8 +18,9 @@ export class FetchCacher {
  specifier: string,
  _isDynamic?: boolean,
  cacheSetting?: CacheSetting,
+ checksum?: string,
  ): Promise<LoadResponse | undefined> => {
  const url = new URL(specifier);
- return this.#fileFetcher.fetch(url, { cacheSetting });
+ return this.#fileFetcher.fetch(url, { cacheSetting, checksum });
  };
 }

deno.jsonc

@@ -6,6 +6,13 @@
  "build": "deno task wasmbuild",
  "wasmbuild": "deno run -A https://deno.land/x/[email protected]/main.ts --sync --features wasm"
  },
+ "lint": {
+ "rules": {
+ "include": [
+ "no-console"
+ ]
+ }
+ },
  "exclude": [
  "target"
  ]

deno_dir_test.ts

@@ -1,6 +1,6 @@
 // Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
 
-import { assertEquals } from "./deps_test.ts";
+import { assertEquals, assertRejects } from "./deps_test.ts";
 import { DenoDir } from "./deno_dir.ts";
 import { assert } from "./util.ts";
 
@@ -12,7 +12,7 @@ Deno.test({
  const deps = denoDir.createHttpCache();
  const headers = (await deps.getHeaders(url))!;
  assert(Object.keys(headers).length > 10);
- const text = new TextDecoder().decode(await deps.get(url));
+ const text = new TextDecoder().decode(await deps.get(url, undefined));
  assertEquals(
  text,
  `// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
@@ -56,5 +56,13 @@ export * from "./_interface.ts";
 export * from "./glob.ts";
 `,
  );
+
+ // ok
+ await deps.get(
+ url,
+ "d3e68d0abb393fb0bf94a6d07c46ec31dc755b544b13144dee931d8d5f06a52d",
+ );
+ // not ok
+ await assertRejects(async () => await deps.get(url, "invalid"));
  },
 });

file_fetcher.ts

@@ -125,7 +125,7 @@ export class FileFetcher {
  specifier: URL,
  cacheSetting: CacheSetting,
  ): Promise<LoadResponse> {
- const cached = await this.#fetchCached(specifier, 0);
+ const cached = await this.#fetchCached(specifier, undefined, 0);
  if (cached) {
  return cached;
  }
@@ -153,6 +153,7 @@ export class FileFetcher {
 
  async #fetchCached(
  specifier: URL,
+ maybeChecksum: string | undefined,
  redirectLimit: number,
  ): Promise<LoadResponse | undefined> {
  if (redirectLimit < 0) {
@@ -168,9 +169,9 @@ export class FileFetcher {
  const location = headers["location"];
  if (location != null && location.length > 0) {
  const redirect = new URL(location, specifier);
- return this.#fetchCached(redirect, redirectLimit - 1);
+ return this.#fetchCached(redirect, maybeChecksum, redirectLimit - 1);
  }
- const content = await this.#httpCache.get(specifier);
+ const content = await this.#httpCache.get(specifier, maybeChecksum);
  if (content == null) {
  return undefined;
  }
@@ -182,19 +183,27 @@ export class FileFetcher {
  };
  }
 
- async #fetchRemote(
- specifier: URL,
- redirectLimit: number,
- cacheSetting: CacheSetting,
- ): Promise<LoadResponse | undefined> {
+ async #fetchRemote(specifier: URL, {
+ redirectLimit,
+ cacheSetting,
+ checksum,
+ }: {
+ redirectLimit: number;
+ cacheSetting: CacheSetting;
+ checksum: string | undefined;
+ }): Promise<LoadResponse | undefined> {
  if (redirectLimit < 0) {
  throw new Deno.errors.Http(
  `Too many redirects.\n Specifier: "${specifier.toString()}"`,
  );
  }
 
  if (shouldUseCache(cacheSetting, specifier)) {
- const response = await this.#fetchCached(specifier, redirectLimit);
+ const response = await this.#fetchCached(
+ specifier,
+ checksum,
+ redirectLimit,
+ );
  if (response) {
  return response;
  }
@@ -218,6 +227,7 @@ export class FileFetcher {
  if (authToken) {
  requestHeaders.append("authorization", authToken);
  }
+ // deno-lint-ignore no-console
  console.error(`${colors.green("Download")} ${specifier.toString()}`);
  const response = await fetchWithRetries(specifier.toString(), {
  headers: requestHeaders,
@@ -242,6 +252,17 @@ export class FileFetcher {
  headers[key.toLowerCase()] = value;
  }
  await this.#httpCache.set(url, headers, content);
+ if (checksum != null) {
+ const digest = await crypto.subtle.digest("SHA-256", content);
+ const actualChecksum = Array.from(new Uint8Array(digest))
+ .map((b) => b.toString(16).padStart(2, "0"))
+ .join("");
+ if (actualChecksum != checksum) {
+ throw new Error(
+ `Integrity check failed for: ${url}\n\nActual: ${actualChecksum}\nExpected: ${checksum}`,
+ );
+ }
+ }
  return {
  kind: "module",
  specifier: response.url,
@@ -252,7 +273,7 @@ export class FileFetcher {
 
  async fetch(
  specifier: URL,
- options?: { cacheSetting?: CacheSetting },
+ options?: { cacheSetting?: CacheSetting; checksum?: string },
  ): Promise<LoadResponse | undefined> {
  const cacheSetting = options?.cacheSetting ?? this.#cacheSetting;
  const scheme = getValidatedScheme(specifier);
@@ -271,7 +292,11 @@ export class FileFetcher {
  `A remote specifier was requested: "${specifier.toString()}", but --no-remote is specified.`,
  );
  } else {
- const response = await this.#fetchRemote(specifier, 10, cacheSetting);
+ const response = await this.#fetchRemote(specifier, {
+ redirectLimit: 10,
+ cacheSetting,
+ checksum: options?.checksum,
+ });
  if (response) {
  await this.#cache.set(specifier.toString(), response);
  }
@@ -299,6 +324,7 @@ export async function fetchWithRetries(
  throw err;
  }
  }
+ // deno-lint-ignore no-console
  console.warn(
  `${
  colors.yellow("WARN")

file_fetcher_test.ts

@@ -1,7 +1,7 @@
 // Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
 
 import { DenoDir } from "./deno_dir.ts";
-import { createGraph } from "./deps_test.ts";
+import { assertRejects, createGraph } from "./deps_test.ts";
 import { FileFetcher } from "./file_fetcher.ts";
 
 Deno.test({
@@ -14,6 +14,64 @@ Deno.test({
  return fileFetcher.fetch(new URL(specifier));
  },
  });
- console.log(graph.toString());
+ // deno-lint-ignore no-console
+ console.log(graph);
+ },
+});
+
+Deno.test({
+ name: "FileFetcher - bad checksum no cache",
+ async fn() {
+ const denoDir = new DenoDir();
+ const fileFetcher = new FileFetcher(denoDir.createHttpCache());
+ {
+ // should error
+ await assertRejects(async () => {
+ await fileFetcher.fetch(
+ new URL("https://deno.land/x/[email protected]/mod.ts"),
+ {
+ checksum: "bad",
+ },
+ );
+ });
+ // ok for good checksum
+ await fileFetcher.fetch(
+ new URL("https://deno.land/x/[email protected]/mod.ts"),
+ {
+ checksum:
+ "7a1b5169ef702e96dd994168879dbcbd8af4f639578b6300cbe1c6995d7f3f32",
+ },
+ );
+ }
+ },
+});
+
+Deno.test({
+ name: "FileFetcher - bad checksum reload",
+ async fn() {
+ const denoDir = new DenoDir();
+ const fileFetcher = new FileFetcher(denoDir.createHttpCache());
+ await assertRejects(async () => {
+ await fileFetcher.fetch(
+ new URL("https://deno.land/x/[email protected]/mod.ts"),
+ {
+ cacheSetting: "reload",
+ checksum: "bad",
+ },
+ );
+ });
+ },
+});
+
+Deno.test({
+ name: "FileFetcher - good checksum reload",
+ async fn() {
+ const denoDir = new DenoDir();
+ const fileFetcher = new FileFetcher(denoDir.createHttpCache());
+ await fileFetcher.fetch(new URL("https://deno.land/x/[email protected]/mod.ts"), {
+ cacheSetting: "reload",
+ checksum:
+ "7a1b5169ef702e96dd994168879dbcbd8af4f639578b6300cbe1c6995d7f3f32",
+ });
  },
 });

http_cache.ts

@@ -60,8 +60,12 @@ export class HttpCache {
 
  async get(
  url: URL,
+ maybeChecksum: string | undefined,
  ): Promise<Uint8Array | undefined> {
- const data = (await this.#ensureCache()).getFileBytes(url.toString());
+ const data = (await this.#ensureCache()).getFileBytes(
+ url.toString(),
+ maybeChecksum,
+ );
  return data == null ? undefined : data;
  }