test: add tests

storacha · May 13, 2024 · fdaa475 · fdaa475
1 parent 58e5821
commit fdaa475
Show file tree

Hide file tree

Showing 6 changed files with 179 additions and 29 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -41,7 +41,7 @@
     "@ipld/dag-json": "^10.1.7",
     "@ipld/dag-pb": "^4.0.8",
     "@web3-storage/content-claims": "^4.0.5",
-    "@web3-storage/gateway-lib": "github:web3-storage/gateway-lib#feat/support-byte-range-reqs-for-raw-block",
+    "@web3-storage/gateway-lib": "github:web3-storage/gateway-lib#5a027e05be62f985407b46bce70748f543d302b7",
     "cardex": "^3.0.0",
     "dagula": "^7.3.0",
     "http-range-parse": "^1.0.0",
@@ -54,6 +54,7 @@
     "@cloudflare/workers-types": "^4.20231218.0",
     "@ucanto/principal": "^8.1.0",
     "ava": "^5.3.1",
+    "byteranges": "^1.1.0",
     "carbites": "^1.0.6",
     "carstream": "^2.1.0",
     "dotenv": "^16.3.1",

diff --git a/src/index.js b/src/index.js
@@ -44,9 +44,9 @@ export default {
       createWithHttpMethod('GET', 'HEAD'),
       withCarBlockHandler,
       withContentClaimsDagula,
-      withRawBlockHandler,
+      withFormatRawHandler,
       withHttpRangeUnsupported,
-      withCarHandler,
+      withFormatCarHandler,
       withContentDispositionHeader,
       withFixedLengthStream
     )
@@ -57,7 +57,7 @@ export default {
 /**
  * @type {import('@web3-storage/gateway-lib').Middleware<BlockContext & UnixfsContext & IpfsUrlContext, BlockContext & UnixfsContext & IpfsUrlContext, Environment>}
  */
-export function withRawBlockHandler (handler) {
+export function withFormatRawHandler (handler) {
   return async (request, env, ctx) => {
     const { headers } = request
     const { searchParams } = ctx
@@ -72,7 +72,7 @@ export function withRawBlockHandler (handler) {
 /**
  * @type {import('@web3-storage/gateway-lib').Middleware<DagContext & IpfsUrlContext, DagContext & IpfsUrlContext, Environment>}
  */
-export function withCarHandler (handler) {
+export function withFormatCarHandler (handler) {
   return async (request, env, ctx) => {
     const { headers } = request
     const { searchParams } = ctx

diff --git a/src/middleware.js b/src/middleware.js
@@ -45,9 +45,10 @@ export function withHttpRangeUnsupported (handler) {
  */
 export function withCarBlockHandler (handler) {
   return async (request, env, ctx) => {
-    const { dataCid } = ctx
+    const { dataCid, searchParams } = ctx
     if (!dataCid) throw new Error('missing data CID')
-    if (dataCid.code !== CAR_CODE) {
+    // if not CAR codec, or if a different format has been requested...
+    if (dataCid.code !== CAR_CODE || searchParams.get('format') || request.headers.get('Accept')) {
       return handler(request, env, ctx) // pass to other handlers
     }
     return handleCarBlock(request, env, ctx)

diff --git a/test/helpers/content-claims.js b/test/helpers/content-claims.js
@@ -122,38 +122,48 @@ export const generateClaims = async (signer, dataCid, carCid, carStream, indexCi
  * @param {import('cardex/api').CARLink} carCid
  * @param {ReadableStream<Uint8Array>} carStream CAR file data
  */
-export const generateLocationClaims = async (signer, carCid, carStream) => {
+export const generateBlockLocationClaims = async (signer, carCid, carStream) => {
   /** @type {Claims} */
   const claims = new LinkMap()
-  const rawCid = Link.create(raw.code, carCid.multihash)
 
   await carStream
     .pipeThrough(new CARReaderStream())
     .pipeTo(new WritableStream({
       async write ({ cid, blockOffset, blockLength }) {
-        const invocation = Assert.location.invoke({
-          issuer: signer,
-          audience: signer,
-          with: signer.did(),
-          nb: {
-            content: cid,
-            location: [
-              /** @type {import('@ucanto/interface').URI<'https:'>} */
-              (`https://w3s.link/ipfs/${rawCid}?format=raw`)
-            ],
-            range: { offset: blockOffset, length: blockLength }
-          }
-        })
-
         const blocks = claims.get(cid) ?? []
-        blocks.push(await encode(invocation))
+        const location = new URL(`https://w3s.link/ipfs/${carCid}?format=raw`)
+        blocks.push(await generateLocationClaim(signer, carCid, location, blockOffset, blockLength))
         claims.set(cid, blocks)
       }
     }))
 
   return claims
 }
 
+/**
+ * @param {import('@ucanto/interface').Signer} signer
+ * @param {import('multiformats').UnknownLink} content
+ * @param {URL} location
+ * @param {number} offset
+ * @param {number} length
+ */
+export const generateLocationClaim = async (signer, content, location, offset, length) => {
+  const invocation = Assert.location.invoke({
+    issuer: signer,
+    audience: signer,
+    with: signer.did(),
+    nb: {
+      content,
+      location: [
+        // @ts-expect-error string is not ${string}:$string
+        location.toString()
+      ],
+      range: { offset, length }
+    }
+  })
+  return await encode(invocation)
+}
+
 /**
  * Encode a claim to a block.
  * @param {import('@ucanto/interface').IPLDViewBuilder<import('@ucanto/interface').Delegation>} invocation
@@ -179,7 +189,7 @@ export const mockClaimsService = async () => {
   const server = http.createServer(async (req, res) => {
     callCount++
     const content = Link.parse(String(req.url?.split('/')[2]))
-    const blocks = claims.get(content) ?? []
+    const blocks = [...claims.get(content) ?? []]
     const readable = new ReadableStream({
       pull (controller) {
         const block = blocks.shift()

diff --git a/test/index.spec.js b/test/index.spec.js
@@ -1,12 +1,19 @@
 import { describe, before, beforeEach, after, it } from 'node:test'
 import assert from 'node:assert'
+import { Buffer } from 'node:buffer'
 import { randomBytes } from 'node:crypto'
 import { Miniflare } from 'miniflare'
 import { equals } from 'uint8arrays'
 import { CarIndexer, CarReader } from '@ipld/car'
+import * as Link from 'multiformats/link'
+import { sha256 } from 'multiformats/hashes/sha2'
+import * as raw from 'multiformats/codecs/raw'
+import { Map as LinkMap } from 'lnmap'
+import { CARReaderStream } from 'carstream'
+import * as ByteRanges from 'byteranges'
 import { Builder, toBlobKey } from './helpers/builder.js'
 import { MAX_CAR_BYTES_IN_MEMORY } from '../src/constants.js'
-import { generateClaims, generateLocationClaims, mockClaimsService } from './helpers/content-claims.js'
+import { generateClaims, generateBlockLocationClaims, mockClaimsService, generateLocationClaim } from './helpers/content-claims.js'
 
 describe('freeway', () => {
   /** @type {Miniflare} */
@@ -228,7 +235,7 @@ describe('freeway', () => {
     assert(res)
 
     // @ts-expect-error nodejs ReadableStream does not implement ReadableStream interface correctly
-    const claims = await generateLocationClaims(claimsService.signer, carCids[0], res.body)
+    const claims = await generateBlockLocationClaims(claimsService.signer, carCids[0], res.body)
     claimsService.setClaims(claims)
 
     const res1 = await miniflare.dispatchFetch(`http://localhost:8787/ipfs/${dataCid}/${input[0].path}`)
@@ -339,4 +346,133 @@ describe('freeway', () => {
     assert.equal(res.headers.get('Content-Type'), 'application/vnd.ipld.car; version=1;')
     assert.equal(res.headers.get('Etag'), `"${carCids[0]}"`)
   })
+
+  it('should GET a raw block', async () => {
+    const input = randomBytes(138)
+    const cid = Link.create(raw.code, await sha256.digest(input))
+
+    const carpark = await miniflare.getR2Bucket('CARPARK')
+    const blobKey = toBlobKey(cid.multihash)
+    await carpark.put(blobKey, input)
+
+    const url = new URL(`https://w3s.link/ipfs/${cid}?format=raw`)
+    const claim = await generateLocationClaim(claimsService.signer, cid, url, 0, input.length)
+    claimsService.setClaims(new LinkMap([[cid, [claim]]]))
+
+    const res = await miniflare.dispatchFetch(`http://localhost:8787/ipfs/${cid}?format=raw`)
+    assert(res.ok)
+
+    const output = new Uint8Array(await res.arrayBuffer())
+    assert.equal(output.length, input.length)
+
+    const contentLength = parseInt(res.headers.get('Content-Length') ?? '0')
+    assert(contentLength)
+    assert.equal(contentLength, input.length)
+    assert.equal(res.headers.get('Content-Type'), 'application/vnd.ipld.raw')
+    assert.equal(res.headers.get('Etag'), `"${cid}.raw"`)
+  })
+
+  it('should HEAD a raw block', async () => {
+    const input = randomBytes(138)
+    const cid = Link.create(raw.code, await sha256.digest(input))
+
+    const carpark = await miniflare.getR2Bucket('CARPARK')
+    const blobKey = toBlobKey(cid.multihash)
+    await carpark.put(blobKey, input)
+
+    const url = new URL(`https://w3s.link/ipfs/${cid}?format=raw`)
+    const claim = await generateLocationClaim(claimsService.signer, cid, url, 0, input.length)
+    claimsService.setClaims(new LinkMap([[cid, [claim]]]))
+
+    const res = await miniflare.dispatchFetch(`http://localhost:8787/ipfs/${cid}?format=raw`, {
+      method: 'HEAD'
+    })
+    assert(res.ok)
+
+    const contentLength = parseInt(res.headers.get('Content-Length') ?? '0')
+    assert(contentLength)
+
+    assert.equal(contentLength, input.length)
+    assert.equal(res.headers.get('Accept-Ranges'), 'bytes')
+    assert.equal(res.headers.get('Etag'), `"${cid}.raw"`)
+  })
+
+  it('should GET a byte range of raw block', async () => {
+    const input = [{ path: 'sargo.tar.xz', content: randomBytes(MAX_CAR_BYTES_IN_MEMORY + 1) }]
+    // no dudewhere or satnav so only content claims can satisfy the request
+    const { carCids } = await builder.add(input, { asBlob: true })
+
+    const carpark = await miniflare.getR2Bucket('CARPARK')
+    const res0 = await carpark.get(toBlobKey(carCids[0].multihash))
+    assert(res0)
+
+    const url = new URL(`https://w3s.link/ipfs/${carCids[0]}?format=raw`)
+    const claim = await generateLocationClaim(claimsService.signer, carCids[0], url, 0, input[0].content.length)
+    claimsService.setClaims(new LinkMap([[carCids[0], [claim]]]))
+
+    const res1 = await carpark.get(toBlobKey(carCids[0].multihash))
+    assert(res1)
+
+    await /** @type {ReadableStream} */ (res1.body)
+      .pipeThrough(new CARReaderStream())
+      .pipeTo(new WritableStream({
+        async write ({ bytes, blockOffset, blockLength }) {
+          const res = await miniflare.dispatchFetch(`http://localhost:8787/ipfs/${carCids[0]}?format=raw`, {
+            headers: {
+              Range: `bytes=${blockOffset}-${blockOffset + blockLength - 1}`
+            }
+          })
+          assert(res.ok)
+          assert(equals(new Uint8Array(await res.arrayBuffer()), bytes))
+
+          const contentLength = parseInt(res.headers.get('Content-Length') ?? '0')
+          assert(contentLength)
+          assert.equal(contentLength, bytes.length)
+          assert.equal(res.headers.get('Content-Range'), `bytes ${blockOffset}-${blockOffset + blockLength - 1}/${input[0].content.length}`)
+          assert.equal(res.headers.get('Content-Type'), 'application/vnd.ipld.raw')
+          assert.equal(res.headers.get('Etag'), `"${carCids[0]}.raw"`)
+        }
+      }))
+  })
+
+  it('should GET a multipart byte range of raw block', async () => {
+    const input = [{ path: 'sargo.tar.xz', content: randomBytes(MAX_CAR_BYTES_IN_MEMORY + 1) }]
+    // no dudewhere or satnav so only content claims can satisfy the request
+    const { carCids } = await builder.add(input, { asBlob: true })
+
+    const url = new URL(`https://w3s.link/ipfs/${carCids[0]}?format=raw`)
+    const claim = await generateLocationClaim(claimsService.signer, carCids[0], url, 0, input[0].content.length)
+    claimsService.setClaims(new LinkMap([[carCids[0], [claim]]]))
+
+    const carpark = await miniflare.getR2Bucket('CARPARK')
+    const res0 = await carpark.get(toBlobKey(carCids[0].multihash))
+    assert(res0)
+
+    /** @type {Array<import('carstream/api').Block & import('carstream/api').Position>} */
+    const blocks = []
+    await /** @type {ReadableStream} */ (res0.body)
+      .pipeThrough(new CARReaderStream())
+      .pipeTo(new WritableStream({ write (block) { blocks.push(block) } }))
+
+    const res1 = await miniflare.dispatchFetch(`http://localhost:8787/ipfs/${carCids[0]}?format=raw`, {
+      headers: {
+        Range: `bytes=${blocks.map(b => `${b.blockOffset}-${b.blockOffset + b.blockLength - 1}`).join(',')}`
+      }
+    })
+    assert(res1.ok)
+
+    const contentType = res1.headers.get('Content-Type')
+    assert(contentType)
+
+    const boundary = contentType.replace('multipart/byteranges; boundary=', '')
+    const body = Buffer.from(await res1.arrayBuffer())
+
+    const parts = ByteRanges.parse(body, boundary)
+    assert.equal(parts.length, blocks.length)
+
+    for (let i = 0; i < parts.length; i++) {
+      assert.equal(parts[i].type, 'application/vnd.ipld.raw')
+      assert(equals(parts[i].octets, blocks[i]?.bytes))
+    }
+  })
 })