Update from RunOnFlux/flux@5979426

services/fluxService.js

@@ -27,7 +27,7 @@ const dockerService = require('./dockerService');
 
 // for streamChain endpoint
 const zlib = require('node:zlib');
-const tar = require('tar-fs');
+const tar = require('tar/create');
 // use non promises stream for node 14.x compatibility
 // const stream = require('node:stream/promises');
 const stream = require('node:stream');
@@ -37,6 +37,13 @@ const stream = require('node:stream');
  */
 let lock = false;
 
+/**
+ * For testing
+ */
+function getStreamLock() {
+  return lock;
+}
+
 /**
  * For testing
  */
@@ -1601,17 +1608,16 @@ async function streamChain(req, res) {
     lock = true;
 
     /**
-   * Use the remote address here, don't need to worry about x-forwarded-for headers as
-   * we only allow the local network. Also, using the remote address is fine as FluxOS
-   * won't confirm if the upstream is natting behind a private address. I.e public
-   * connections coming in via a private address. (Flux websockets need the remote address
-   * or they think there is only one inbound connnection)
-   */
+     * Use the remote address here, don't need to worry about x-forwarded-for headers as
+     * we only allow the local network. Also, using the remote address is fine as FluxOS
+     * won't confirm if the upstream is natting behind a private address. I.e public
+     * connections coming in via a private address. (Flux websockets need the remote address
+     * or they think there is only one inbound connnection)
+     */
     let ip = req.socket.remoteAddress;
     if (!ip) {
       res.statusMessage = 'Socket closed.';
       res.status(400).end();
-      lock = false;
       return;
     }
 
@@ -1621,7 +1627,6 @@ async function streamChain(req, res) {
     if (!serviceHelper.isPrivateAddress(ip)) {
       res.statusMessage = 'Request must be from an address on the same private network as the host.';
       res.status(403).end();
-      lock = false;
       return;
     }
 
@@ -1630,10 +1635,11 @@ async function streamChain(req, res) {
     const homeDir = os.homedir();
     const base = path.join(homeDir, '.flux');
 
+    // the order can matter when doing the stream live, the level db's can be volatile
     const folders = [
-      'blocks',
-      'chainstate',
       'determ_zelnodes',
+      'chainstate',
+      'blocks',
     ];
 
     const folderPromises = folders.map(async (f) => {
@@ -1651,7 +1657,6 @@ async function streamChain(req, res) {
     if (!chainExists) {
       res.statusMessage = 'Unable to find chain at $HOME/.flux';
       res.status(500).end();
-      lock = false;
       return;
     }
 
@@ -1669,7 +1674,6 @@ async function streamChain(req, res) {
     if (!safe && compress) {
       res.statusMessage = 'Unable to compress blockchain in unsafe mode, it will corrupt new db.';
       res.status(422).end();
-      lock = false;
       return;
     }
 
@@ -1681,15 +1685,33 @@ async function streamChain(req, res) {
     if (safe && fluxdRunning) {
       res.statusMessage = 'Flux daemon still running, unable to clone blockchain.';
       res.status(503).end();
-      lock = false;
       return;
     }
 
+    const infoPromises = folders.map(
+      (f) => serviceHelper.dirInfo(path.join(base, f), { padFiles: 512 }),
+    );
+    const info = await Promise.all(infoPromises).catch(() => [{ count: 0, size: 0 }]);
+
+    const { count, size } = info.reduce((prev, current) => (
+      { count: prev.count + current.count, size: prev.size + current.size }), { count: 0, size: 0 });
+
+    const tarHeaderSize = count * 512;
+    // if we get an error getting size, just set eof to 0, which will make totalSize 0
+    const tarEof = size ? 512 * 2 : 0;
+
+    const totalSize = size + tarHeaderSize + tarEof;
+
+    // We can't set this as the actual content length. As it can change slightly during transfer.
+    // However we need to set some size, so that the image installer can get a rought idea
+    // how log the transfer will take.
+    res.setHeader('Approx-Content-Length', totalSize.toString());
+
     const workflow = [];
 
-    workflow.push(tar.pack(base, {
-      entries: folders,
-    }));
+    const readStream = tar.create({ cwd: base }, folders);
+
+    workflow.push(readStream);
 
     if (compress) {
       log.info('Compression requested... adding gzip. This can be 10-20x slower than sending uncompressed');
@@ -1744,7 +1766,6 @@ module.exports = {
   hardUpdateFlux,
   installFluxWatchTower,
   isStaticIPapi,
-  lockStreamLock,
   rebuildHome,
   reindexDaemon,
   restartBenchmark,
@@ -1761,11 +1782,13 @@ module.exports = {
   tailFluxErrorLog,
   tailFluxInfoLog,
   tailFluxWarnLog,
-  unlockStreamLock,
   updateBenchmark,
   updateDaemon,
   updateFlux,
   // Exports for testing purposes
   fluxLog,
+  getStreamLock,
+  lockStreamLock,
   tailFluxLog,
+  unlockStreamLock,
 };

services/serviceHelper.js

@@ -1,4 +1,6 @@
 const util = require('node:util');
+const path = require('node:path');
+const fs = require('node:fs/promises');
 const execFile = util.promisify(require('node:child_process').execFile);
 
 const axios = require('axios').default;
@@ -451,13 +453,54 @@ function minVersionSatisfy(targetVersion, minimumVersion) {
   return true;
 }
 
+/**
+ * Recursively sum size of directory and children, in bytes
+ * @param {string} dir The directory we want the size of
+ * @param {{padFiles?: number}} options If the files are to be padded to size
+ * @returns {Promise<number>}
+ */
+async function dirInfo(dir, options = {}) {
+  const padFiles = options.padFiles || null;
+
+  const files = await fs.readdir(dir, { withFileTypes: true });
+
+  const pathPromises = files.map(async (file) => {
+    const targetpath = path.join(dir, file.name);
+
+    if (file.isDirectory()) return dirInfo(targetpath, options);
+
+    if (file.isFile()) {
+      const { size } = await fs.stat(targetpath);
+
+      return size;
+    }
+
+    return 0;
+  });
+
+  const paths = await Promise.all(pathPromises);
+
+  const response = paths.flat(Infinity).reduce((prev, current) => {
+    // the paths are either a number, i.e. a file, or a directory, with a count and aggregate size
+    const { count, size } = typeof current === 'number' ? { count: 1, size: current } : current;
+
+    // we only pad if it's a file (a dir has already been padded)
+    const padding = padFiles && count > 1 ? size % 512 : 0;
+
+    return { count: prev.count + count, size: prev.size + size + padding };
+  }, { count: 0, size: 0 });
+
+  return response;
+}
+
 module.exports = {
   axiosGet,
   axiosPost,
   commandStringToArray,
   axiosInstance,
   delay,
   deleteLoginPhrase,
+  dirInfo,
   dockerBufferToString,
   ensureBoolean,
   ensureNumber,