Merge pull request #72 from foragepm/metadata-on-ipfs

store proxy http responses in ipfs

docs/add-a-package-manager.md

@@ -17,5 +17,6 @@ Note: This document isn't quite finished yet!
   - watchKnown
   - setConfig
   - unsetConfig
-  - getLatestVersion
   - setupWatcher
+  - matchesUseragent
+  - versionAsJson

docs/metadata.md

@@ -13,17 +13,18 @@ This will require further trusting of other instances as metadata includes integ
 Existing data:
 
 - for a package (manager, name):
-  - list of version numbers
-  - http response body for list of versions for proxy
+  - list of versions
     - key: `pkg:${manager}:${name}`
+  - http response body for list of versions for proxy
+    - key: `response:${manager}:versions:${name}`
   - for each version (manager, name, version):
     - cid for archive
       - `cid:${manager}:${name}:${version}`
     - http response bodies for proxy
       extras for go:
-        - `mod:go:${name}:${version}`
-        - `latest:go:${name}`
-        - `info:go:${name}:${version}`
+        - `response:go:mod:${name}:${version}`
+        - `response:go:sum:${name}`
+        - `response:go:info:${name}:${version}`
 
 Example json:
 
@@ -70,8 +71,8 @@ Example json:
 
 ## Steps to implement
 
-- save/read responses to ipfs and save cids to leveldb
-- construct json object representing a version
+- save/read responses to ipfs and save cids to leveldb ✅
+- construct json object representing a version ✅
 - construct json object representing whole package
 - introduce signing to data
 

forage.lock

@@ -399,7 +399,7 @@
   "npm:node-fetch:2.6.1": "bafkrgqcxq2miht3dckcvdriw6z6joqedyfd3zdw4cmtb2vaipxqhd5cqqk6kbhcupzeokdidpvc3gmpzwert7p5aqdeikru22k3c2n6y5cu4o",
   "npm:node-forge:0.10.0": "bafkrgqb47gxpdyi6dpnrueklzd33pzxa4yyv2ud2nrn7enj4ujioaytscburi3ya2s4pbxnwhln642b2gdcda5dho5ddwhipwhxoz4j3gc5ci",
   "npm:node-gyp-build:4.1.1": "bafkrgqdvfk24mzypbsr5twcbapms5m6d636tjkzkzv2ej2ovnpytkue4rxcsjzhnpfbs5kc52cvm5c3gb6tcyzqjlksdgmnsemzne2so745z2",
-  "npm:node-stream-zip:1.13.3": "bafkrgqcw7543ud54ex52byamlsunk62n4gcjgzvhwtmj3d7gae5sh7cjf4fuy4tgr3ato56ypix435lytnhpfhxdplbhocb3mcjqtcz37qoew",
+  "npm:node-stream-zip:1.13.4": "bafkrgqbtnhh345ewcri747qczbczb2u2dmqzv3gxok7zy66qnqoql767jjlupzyez72remmjwh2zabldf43623mqua5a3tqrnvwzqx6numspi",
   "npm:normalize-package-data:3.0.2": "bafkrgqhie5m2dsm7dgrkoidt2ulbeslhogxeprfmfsv4gagrnhr3ezwltvncrcx4njemawjuwg3jeprqdaarlvemi4vdrbgkiokt5hkn67qgm",
   "npm:normalize-path:3.0.0": "bafkrgqhj4zwojozxllikfmdvvh2s3bstf4o2ussgroafksz5yzvkrbhj5txg6ttv3bclhnlvgbib5axiqknuerrwhz3p7gb6czrirqsha4ycy",
   "npm:normalize-url:4.5.0": "bafkrgqg2zy54wnjroxwf7vhiojdc4be3zxreeti2h36bg5o3ixhyqz2jfq6xiijmfraz72jmba54wlh7l5jpmic343bfuoxe55ggbxteru2ak",
@@ -568,7 +568,7 @@
   "npm:yallist:4.0.0": "bafkrgqg7a5ditvtsvoj4du6oc4wejokosojeidpqrvyckildeg5g3jcfzp76gvfh3huzbuo4traw4ltfolpi6avpqoqszpnxmvkl7blai4w6y",
   "npm:yargs-parser:20.2.4": "bafkrgqcy5euybwcpjzitxxq6cukac3b2pitckvvixtxrlkfq6pfzwgqkcrarkakbuddcflumgjn6ipi4dydrixqz5vlfhcdn4jftesidn5zei",
   "npm:yargs-unparser:2.0.0": "bafkrgqhosrjsad2qonlruz2g3hurmeizwhe3meswxgur76ljq4vuvvlyxeg25mnbp2djwbgxnz52shja2i5k7ce752dsv5na76p3y4bi45ztq",
-  "npm:yargs:17.0.1": "bafkrgqgecbxjl4e46ghkalcxfm7pw2rlh5q47reo2jou3c6buag6slyfpwv2gqoriceyggtyur64353sbtogcrfhogvgonogiogvcsa7o75cs",
+  "npm:yargs:16.2.0": "bafkrgqaplgx35ugg2c7f7n7yyznef2i3l6tndzbrhh3idpjtiqxlnfupnwyesvimlmlfjpmibfq4fipkggdceqsfqr7aqzhuefdyjsvfz4tao",
   "npm:yauzl:2.10.0": "bafkreicvb4yygxl3mqahgcidhx5tgvyyew2mz7ms2mlss5erhg2qjqtmwe",
   "npm:yocto-queue:0.1.0": "bafkrgqfnlewl5sonbhjh7iqrttvrqd6den6huf4cy3eiwm6jwg4e73p6mok2rf5qh3r3lgrc5fghiisgatfarn5rf6br4acvlkbnwoy6mnm5s",
   "npm:zipdigest:1.0.0": "bafkrgqefxj2ewjjjtgxwknz26zirglzcyoxxlsoex6m3rigc6uwqj53hxhbcps256ejvcurlahd5az63jpywg4nsowhe3ycc7wgvtboyvlego"

lib/core.js

@@ -2,6 +2,7 @@ const fetch = require('node-fetch')
 const AbortController = require('abort-controller')
 const uint8ArrayConcat = require('uint8arrays/concat')
 const all = require('it-all')
+const toString = require('uint8arrays/to-string')
 
 const IpfsHttpClient = require('ipfs-http-client')
 const { urlSource } = IpfsHttpClient
@@ -40,6 +41,38 @@ async function respondToWant(db, peerId, package) {
   }
 }
 
+async function writeResponse(db, key, url) {
+  var cid = await ipfsAdd(urlSource(url))
+  if(cid){ await db.put(key, cid) } else {
+    log.error('writeResponse failed', key, url)
+  }
+  return cid
+}
+
+async function getResponse(db, key) {
+  try{
+    var cid = await db.get(key)
+    var body = await attemptIPFSLoad(cid)
+    return toString(body)
+  } catch(e) {
+    return false
+  }
+}
+
+async function fetchResponse(db, key, url, force = false) {
+  if(force){
+    await writeResponse(db, key, url)
+  }
+  var res = await getResponse(db, key)
+  if(res){
+    return res
+  } else {
+    await writeResponse(db, key, url)
+    var res = await getResponse(db, key)
+    return res
+  }
+}
+
 async function saveCid(db, manager, name, version, cid) {
   await db.put(`cid:${manager}:${name}:${version}`, cid)
   await setTarballSize(db, manager, name, version) // TODO can use the size from ipfsAdd
@@ -121,8 +154,6 @@ async function ipfsAdd(data, hashAlg = 'sha2-256') {
     var res = await ipfs.add(data, {chunker: 'size-1048576', rawLeaves: true, hashAlg: hashAlg, cidVersion: 1, timeout: 10000})
     return res.cid.toString()
   } catch(e){
-    log.error('IPFS add failed')
-    log.error(e)
     return false
   }
 }
@@ -155,7 +186,8 @@ async function attemptIPFSLoad(cid, timeout = 10000){
   try{
     return uint8ArrayConcat(await all(ipfs.cat(cid, {timeout: timeout})))
   } catch(e){
-    log.error('Failed to load' + cid + 'from IPFS')
+    log.error('Failed to load', cid, 'from IPFS')
+    log.error(e)
     return false
   }
 }
@@ -368,5 +400,8 @@ module.exports = {
   fetchWithTimeout,
   announceWant,
   recordHave,
-  respondToWant
+  respondToWant,
+  writeResponse,
+  getResponse,
+  fetchResponse
 }

lib/forage.js

@@ -138,7 +138,7 @@ async function respondToRepublish(db, peerId, package) {
   await db.put(`repub:${package.manager}:${package.name}:${package.version}:${peerId}`, time)
 
   try {
-    var exists = await db.get(`pkg:${package.manager}:${package.name}`)
+    var exists = await db.get(`response:${package.manager}:versions:${package.name}`)
   } catch (e) {
     var exists = false
   }
@@ -236,17 +236,23 @@ function listPackages(manager) {
 
 function listPackageNames(manager) {
   if(manager){
-    var key = `pkg:${manager}:`
+    var key = `responses:${manager}:versions:`
   } else {
-    var key = `pkg:`
+    var key = `responses:`
   }
 
   return new Promise((resolve, reject) => {
     var names = {}
     db.createKeyStream({gte: key, lt: key+'~'})
       .on('data', function (data) {
         var parts = data.split(':')
-        names[`${parts[1]}-${parts[2]}`] = {manager: parts[1], name: parts[2]}
+        if(manager){
+          names[`${parts[1]}-${parts[3]}`] = {manager: parts[1], name: parts[3]}
+        } else {
+          if(parts[2] == 'versions'){
+            names[`${parts[1]}-${parts[3]}`] = {manager: parts[1], name: parts[3]}
+          }
+        }
       })
       .on('end', function () {
         resolve(Object.values(names))