From 8c63f2468229247495fb1a796288f2b0cdb22069 Mon Sep 17 00:00:00 2001 From: RunOnFlux Date: Wed, 23 Oct 2024 14:53:55 +0000 Subject: [PATCH] Update from https://github.com/RunOnFlux/flux/commit/68f384f5fd38d3950d780e5307eb0c3f431083db --- services/benchmarkService.js | 3 +- services/fluxService.js | 61 +++++++-------------- services/serviceHelper.js | 43 --------------- services/syncthingService.js | 98 ++++++++++++++++------------------ services/systemService.js | 6 ++- services/utils/daemonConfig.js | 8 ++- 6 files changed, 75 insertions(+), 144 deletions(-) diff --git a/services/benchmarkService.js b/services/benchmarkService.js index fdec68cb..b7f75a28 100644 --- a/services/benchmarkService.js +++ b/services/benchmarkService.js @@ -17,8 +17,7 @@ let response = messageHelper.createErrorMessage(); let benchdClient = null; async function buildBenchdClient() { - const homeDirPath = path.join(__dirname, '../../../../'); - const fluxbenchdPath = path.join(homeDirPath, '.fluxbenchmark'); + const fluxbenchdPath = '/dat/usr/lib/fluxbenchd'; const exists = await fs.stat(fluxbenchdPath).catch(() => false); diff --git a/services/fluxService.js b/services/fluxService.js index 9b26e45e..8a02f948 100644 --- a/services/fluxService.js +++ b/services/fluxService.js @@ -27,7 +27,7 @@ const dockerService = require('./dockerService'); // for streamChain endpoint const zlib = require('node:zlib'); -const tar = require('tar/create'); +const tar = require('tar-fs'); // use non promises stream for node 14.x compatibility // const stream = require('node:stream/promises'); const stream = require('node:stream'); @@ -37,13 +37,6 @@ const stream = require('node:stream'); */ let lock = false; -/** - * For testing - */ -function getStreamLock() { - return lock; -} - /** * For testing */ @@ -1608,16 +1601,17 @@ async function streamChain(req, res) { lock = true; /** - * Use the remote address here, don't need to worry about x-forwarded-for headers as - * we only allow the local network. Also, using the remote address is fine as FluxOS - * won't confirm if the upstream is natting behind a private address. I.e public - * connections coming in via a private address. (Flux websockets need the remote address - * or they think there is only one inbound connnection) - */ + * Use the remote address here, don't need to worry about x-forwarded-for headers as + * we only allow the local network. Also, using the remote address is fine as FluxOS + * won't confirm if the upstream is natting behind a private address. I.e public + * connections coming in via a private address. (Flux websockets need the remote address + * or they think there is only one inbound connnection) + */ let ip = req.socket.remoteAddress; if (!ip) { res.statusMessage = 'Socket closed.'; res.status(400).end(); + lock = false; return; } @@ -1627,6 +1621,7 @@ async function streamChain(req, res) { if (!serviceHelper.isPrivateAddress(ip)) { res.statusMessage = 'Request must be from an address on the same private network as the host.'; res.status(403).end(); + lock = false; return; } @@ -1635,11 +1630,10 @@ async function streamChain(req, res) { const homeDir = os.homedir(); const base = path.join(homeDir, '.flux'); - // the order can matter when doing the stream live, the level db's can be volatile const folders = [ - 'determ_zelnodes', - 'chainstate', 'blocks', + 'chainstate', + 'determ_zelnodes', ]; const folderPromises = folders.map(async (f) => { @@ -1657,6 +1651,7 @@ async function streamChain(req, res) { if (!chainExists) { res.statusMessage = 'Unable to find chain at $HOME/.flux'; res.status(500).end(); + lock = false; return; } @@ -1674,6 +1669,7 @@ async function streamChain(req, res) { if (!safe && compress) { res.statusMessage = 'Unable to compress blockchain in unsafe mode, it will corrupt new db.'; res.status(422).end(); + lock = false; return; } @@ -1685,33 +1681,15 @@ async function streamChain(req, res) { if (safe && fluxdRunning) { res.statusMessage = 'Flux daemon still running, unable to clone blockchain.'; res.status(503).end(); + lock = false; return; } - const infoPromises = folders.map( - (f) => serviceHelper.dirInfo(path.join(base, f), { padFiles: 512 }), - ); - const info = await Promise.all(infoPromises).catch(() => [{ count: 0, size: 0 }]); - - const { count, size } = info.reduce((prev, current) => ( - { count: prev.count + current.count, size: prev.size + current.size }), { count: 0, size: 0 }); - - const tarHeaderSize = count * 512; - // if we get an error getting size, just set eof to 0, which will make totalSize 0 - const tarEof = size ? 512 * 2 : 0; - - const totalSize = size + tarHeaderSize + tarEof; - - // We can't set this as the actual content length. As it can change slightly during transfer. - // However we need to set some size, so that the image installer can get a rought idea - // how log the transfer will take. - res.setHeader('Approx-Content-Length', totalSize.toString()); - const workflow = []; - const readStream = tar.create({ cwd: base }, folders); - - workflow.push(readStream); + workflow.push(tar.pack(base, { + entries: folders, + })); if (compress) { log.info('Compression requested... adding gzip. This can be 10-20x slower than sending uncompressed'); @@ -1766,6 +1744,7 @@ module.exports = { hardUpdateFlux, installFluxWatchTower, isStaticIPapi, + lockStreamLock, rebuildHome, reindexDaemon, restartBenchmark, @@ -1782,13 +1761,11 @@ module.exports = { tailFluxErrorLog, tailFluxInfoLog, tailFluxWarnLog, + unlockStreamLock, updateBenchmark, updateDaemon, updateFlux, // Exports for testing purposes fluxLog, - getStreamLock, - lockStreamLock, tailFluxLog, - unlockStreamLock, }; diff --git a/services/serviceHelper.js b/services/serviceHelper.js index 32057407..c6f1a6fa 100644 --- a/services/serviceHelper.js +++ b/services/serviceHelper.js @@ -1,6 +1,4 @@ const util = require('node:util'); -const path = require('node:path'); -const fs = require('node:fs/promises'); const execFile = util.promisify(require('node:child_process').execFile); const axios = require('axios').default; @@ -453,46 +451,6 @@ function minVersionSatisfy(targetVersion, minimumVersion) { return true; } -/** - * Recursively sum size of directory and children, in bytes - * @param {string} dir The directory we want the size of - * @param {{padFiles?: number}} options If the files are to be padded to size - * @returns {Promise} - */ -async function dirInfo(dir, options = {}) { - const padFiles = options.padFiles || null; - - const files = await fs.readdir(dir, { withFileTypes: true }); - - const pathPromises = files.map(async (file) => { - const targetpath = path.join(dir, file.name); - - if (file.isDirectory()) return dirInfo(targetpath, options); - - if (file.isFile()) { - const { size } = await fs.stat(targetpath); - - return size; - } - - return 0; - }); - - const paths = await Promise.all(pathPromises); - - const response = paths.flat(Infinity).reduce((prev, current) => { - // the paths are either a number, i.e. a file, or a directory, with a count and aggregate size - const { count, size } = typeof current === 'number' ? { count: 1, size: current } : current; - - // we only pad if it's a file (a dir has already been padded) - const padding = padFiles && count > 1 ? size % 512 : 0; - - return { count: prev.count + count, size: prev.size + size + padding }; - }, { count: 0, size: 0 }); - - return response; -} - module.exports = { axiosGet, axiosPost, @@ -500,7 +458,6 @@ module.exports = { axiosInstance, delay, deleteLoginPhrase, - dirInfo, dockerBufferToString, ensureBoolean, ensureNumber, diff --git a/services/syncthingService.js b/services/syncthingService.js index 3150d601..77277e92 100644 --- a/services/syncthingService.js +++ b/services/syncthingService.js @@ -51,38 +51,39 @@ const stc = new FluxController(); * @returns {Promise<(string | null)>} config gile (XML). */ async function getConfigFile() { - const homedir = os.homedir(); - const configDir = path.join(homedir, '.config'); - const syncthingDir = path.join(configDir, 'syncthing'); - const configFile = path.join(syncthingDir, 'config.xml'); + // const homedir = os.homedir(); + // const configDir = path.join(homedir, '.config'); + // const syncthingDir = path.join(configDir, 'syncthing'); + // const configFile = path.join(syncthingDir, 'config.xml'); + const configFile = '/dat/usr/lib/syncthing/config.xml'; - const user = os.userInfo().username; - const owner = `${user}:${user}`; + // const user = os.userInfo().username; + // const owner = `${user}:${user}`; // As syncthing is running as root, we need to change owenership to running user - const { error: chownConfigDirError } = await serviceHelper.runCommand('chown', { - runAsRoot: true, - logError: false, - params: [owner, configDir], - }); + // const { error: chownConfigDirError } = await serviceHelper.runCommand('chown', { + // runAsRoot: true, + // logError: false, + // params: [owner, configDir], + // }); - if (chownConfigDirError) return null; + // if (chownConfigDirError) return null; - const { error: chownSyncthingError } = await serviceHelper.runCommand('chown', { - runAsRoot: true, - logError: false, - params: [owner, syncthingDir], - }); + // const { error: chownSyncthingError } = await serviceHelper.runCommand('chown', { + // runAsRoot: true, + // logError: false, + // params: [owner, syncthingDir], + // }); - if (chownSyncthingError) return null; + // if (chownSyncthingError) return null; - const { error: chmodError } = await serviceHelper.runCommand('chmod', { - runAsRoot: true, - logError: false, - params: ['644', configFile], - }); + // const { error: chmodError } = await serviceHelper.runCommand('chmod', { + // runAsRoot: true, + // logError: false, + // params: ['644', configFile], + // }); - if (chmodError) return null; + // if (chmodError) return null; let result = null; // this should never reject as chown would error first but just in case @@ -2494,10 +2495,11 @@ async function runSyncthingSentinel() { log.error('Unable to get syncthing deviceId. Reconfiguring syncthing.'); await stopSyncthing(); await installSyncthingIdempotently(); - await configureDirectories(); + // await configureDirectories(); - const homedir = os.homedir(); - const syncthingHome = path.join(homedir, '.config/syncthing'); + // const homedir = os.homedir(); + // const syncthingHome = path.join(homedir, '.config/syncthing'); + const syncthingHome = '/dat/usr/lib/syncthing'; const logFile = path.join(syncthingHome, 'syncthing.log'); if (stc.aborted) return 0; @@ -2515,32 +2517,26 @@ async function runSyncthingSentinel() { // adding old spawn with shell in the interim. childProcess.spawn( - `sudo nohup syncthing --logfile ${logFile} --logflags=3 --log-max-old-files=2 --log-max-size=26214400 --allow-newer-config --no-browser --home ${syncthingHome} >/dev/null 2>&1