From 032d21fe7a4f19ab3cd996694fb4213fe9196039 Mon Sep 17 00:00:00 2001 From: Ugo Bechameil Date: Mon, 25 Sep 2023 17:58:58 +0200 Subject: [PATCH] Tweak couchbase path to simplify native loading --- patches/couchbase+4.2.7.patch | 565 ++++++++++++++++++++++++++++++++-- 1 file changed, 535 insertions(+), 30 deletions(-) diff --git a/patches/couchbase+4.2.7.patch b/patches/couchbase+4.2.7.patch index d2b78c4..ca3a1ee 100644 --- a/patches/couchbase+4.2.7.patch +++ b/patches/couchbase+4.2.7.patch @@ -71,48 +71,553 @@ index 324198c..2d0fee9 100644 + ] } \ No newline at end of file +diff --git a/node_modules/couchbase/scripts/createPlatformPackages.js b/node_modules/couchbase/scripts/createPlatformPackages.js +deleted file mode 100644 +index b845381..0000000 +--- a/node_modules/couchbase/scripts/createPlatformPackages.js ++++ /dev/null +@@ -1,105 +0,0 @@ +-const fs = require('fs') +-const path = require('path') +-const { getSupportedPlatformPackages } = require('./prebuilds') +- +-try { +- // we run this script w/in a Jenkins dir("couchnode"){} block +- const couchbasePkgData = JSON.parse(fs.readFileSync('package.json')) +- const packageName = couchbasePkgData.name +- const packageVersion = couchbasePkgData.version +- let platformPackages = (couchbasePkgData.optionalDependencies = {}) +- const prebuildsPath = path.join(process.cwd(), 'prebuilds') +- const prebuilds = fs.readdirSync(prebuildsPath) +- const supportedPlatPkgs = getSupportedPlatformPackages(couchbasePkgData.name) +- for (const prebuild of prebuilds) { +- if (fs.lstatSync(path.join(prebuildsPath, prebuild)).isDirectory()) continue +- // prebuild format: +- // couchbase-v--v---.node.tar.gz +- const tokens = prebuild.split('-') +- if (tokens.length < 7) continue +- if (tokens[tokens.length - 1].startsWith('debug')) { +- fs.renameSync(`prebuilds/${prebuild}`, `prebuilds_debug/${prebuild}`) +- continue +- } +- const nodeVersion = parseInt( +- tokens[tokens.length - 1].replace('.node', '').replace('node', '') +- ) +- const arch = tokens[tokens.length - 2] +- const platform = tokens[tokens.length - 3] +- const runtime = tokens[tokens.length - 5] +- const sslType = +- runtime === 'napi' +- ? nodeVersion >= 18 +- ? 'openssl3' +- : 'openssl1' +- : 'boringssl' +- const platPkg = `${tokens[0]}-${platform}-${arch}-${sslType}` +- let description = `Couchbase Node.js SDK platform specific binary for ${runtime} runtime on ${platform} OS with ${arch} architecture` +- if (runtime === 'napi') { +- description += ` and OpenSSL ${nodeVersion >= 18 ? '3.x' : '1.x'}.` +- } else { +- description += ' and BoringSSL.' +- } +- console.log(`platformPackage=${platPkg}`) +- if (supportedPlatPkgs.includes(platPkg)) { +- console.log(`Building requirements for platform package: ${platPkg}`) +- if (!fs.existsSync(`prebuilds/${platPkg}`)) { +- fs.mkdirSync(`prebuilds/${platPkg}`) +- } +- tokens[tokens.length - 1] = `${sslType}.node` +- const newPrebuildName = tokens.join('-') +- const oldPath = path.join('prebuilds', prebuild) +- const newPath = path.join('prebuilds', platPkg) +- fs.renameSync(oldPath, path.join(newPath, newPrebuildName)) +- const platformPackage = `@${packageName}/${platPkg}` +- // build the platform package files: package.json, README and index.js +- const engines = { node: nodeVersion >= 18 ? '>=18' : '<18' } +- fs.writeFileSync( +- path.join(newPath, 'package.json'), +- JSON.stringify( +- { +- name: platformPackage, +- version: packageVersion, +- engines: engines, +- os: [platform.includes('linux') ? 'linux' : platform], +- cpu: [arch], +- bugs: couchbasePkgData.bugs, +- homepage: couchbasePkgData.homepage, +- license: couchbasePkgData.license, +- repository: couchbasePkgData.repository, +- description: description, +- }, +- null, +- 2 +- ) +- ) +- fs.writeFileSync(path.join(newPath, 'index.js'), '') +- fs.writeFileSync(path.join(newPath, 'README.md'), description) +- platformPackages[platformPackage] = packageVersion +- } +- } +- +- if ( +- !process.env.ALLOW_MISMATCH && +- Object.keys(platformPackages).length != supportedPlatPkgs.length +- ) { +- const builtPlatformPkgs = Object.keys(platformPackages).map((pkg) => { +- const tokens = pkg.split('/') +- return tokens[1] +- }) +- const missingPkgs = supportedPlatPkgs.filter( +- (pkg) => !builtPlatformPkgs.includes(pkg) +- ) +- const extraPkgs = builtPlatformPkgs.filter( +- (pkg) => !supportedPlatPkgs.includes(pkg) +- ) +- let msg = 'Mismatch in built platform packages.\n' +- msg += 'Missing: ' + JSON.stringify(missingPkgs) + '.\n' +- msg += 'Extra: ' + JSON.stringify(extraPkgs) + '.' +- throw new Error(msg) +- } +- fs.writeFileSync('package.json', JSON.stringify(couchbasePkgData, null, 2)) +-} catch (err) { +- console.log('An error occurred:', err) +- process.exitCode = 1 +-} diff --git a/node_modules/couchbase/scripts/install.js b/node_modules/couchbase/scripts/install.js -index 4bf054f..a4f8801 100755 +deleted file mode 100755 +index 4bf054f..0000000 --- a/node_modules/couchbase/scripts/install.js -+++ b/node_modules/couchbase/scripts/install.js -@@ -37,9 +37,9 @@ function installPrebuild() { - require('cmake-js/package.json').bin['cmake-js'] - ) - // @TODO: is spawning sync a problem? Seemed to be easiest way to get Ctrl-C to kill the whole build process ++++ /dev/null +@@ -1,45 +0,0 @@ +-#!/usr/bin/env node +- +-const path = require('path') +-var proc = require('child_process') +-const prebuilds = require('./prebuilds') +- +-if (hasLocalPrebuild()) { +- const destination = path.join( +- path.resolve(__dirname, '..'), +- 'build', +- 'Release' +- ) +- const source = getLocalPrebuild() +- // on either success or failure of resolving local prebuild we still confirm we have a prebuild +- prebuilds.resolveLocalPrebuild(source, destination).then(installPrebuild()) +-} else { +- installPrebuild() +-} +- +-function getLocalPrebuild() { +- const localPrebuildsName = `npm_config_couchbase_local_prebuilds` +- return process.env[localPrebuildsName] +-} +- +-function hasLocalPrebuild() { +- return typeof getLocalPrebuild() === 'string' +-} +- +-function installPrebuild() { +- try { +- prebuilds.resolvePrebuild(path.resolve(__dirname, '..'), false) +- process.exit(0) +- } catch (err) { +- const cmakejs = path.join( +- require.resolve('cmake-js/package.json'), +- '..', +- require('cmake-js/package.json').bin['cmake-js'] +- ) +- // @TODO: is spawning sync a problem? Seemed to be easiest way to get Ctrl-C to kill the whole build process - const cmakejsProc = proc.spawnSync(process.execPath, [cmakejs, 'compile'], { - stdio: 'inherit', - }) - process.exit(cmakejsProc.status) -+ // const cmakejsProc = proc.spawnSync(process.execPath, [cmakejs, 'compile'], { -+ // stdio: 'inherit', -+ // }) -+ // process.exit(cmakejsProc.status) - } - } +- } +-} diff --git a/node_modules/couchbase/scripts/prebuilds.js b/node_modules/couchbase/scripts/prebuilds.js -index 0da8ef5..56bc7c9 100644 +index 0da8ef5..3311f19 100644 --- a/node_modules/couchbase/scripts/prebuilds.js +++ b/node_modules/couchbase/scripts/prebuilds.js -@@ -13,7 +13,7 @@ const openSSLVersions = ['openssl1', 'openssl3'] - const supportedPlatforms = ['darwin', 'linux', 'linuxmusl', 'win32'] - const supportedArches = ['x64', 'arm64'] +@@ -1,249 +1,10 @@ +-const fs = require('fs') +-const path = require('path') +-const zlib = require('zlib') +-const stream = require('stream') +-const { promisify } = require('util') +-const pipe = promisify(stream.pipeline) ++const os = require('os') +-// Workaround to fix webpack's build warnings: 'the request of a dependency is an expression' +-const runtimeRequire = +- typeof __webpack_require__ === 'function' ? __non_webpack_require__ : require +- +-const openSSLVersions = ['openssl1', 'openssl3'] +-const supportedPlatforms = ['darwin', 'linux', 'linuxmusl', 'win32'] +-const supportedArches = ['x64', 'arm64'] +- -const runtime = isElectron() ? 'electron' : 'node' -+const runtime = 'node' - const nodeVersion = getNodeVersion() - const nodeVersionMajor = getNodeMajorVersion(nodeVersion) - const arch = process.arch -@@ -199,11 +199,9 @@ function resolvePrebuild(dir, runtimeResolve = true) { - : JSON.parse(fs.readFileSync(path.join(dir, 'package.json'))).name - if (packageName !== undefined) { - const supportedPackages = getSupportedPlatformPackages(packageName) +-const nodeVersion = getNodeVersion() +-const nodeVersionMajor = getNodeMajorVersion(nodeVersion) +-const arch = process.arch +-const platform = process.platform +-const libc = getLinuxType(platform) +-const sslType = getSSLType(runtime, nodeVersion) +- +-function getLocalPrebuild(dir) { +- const localPrebuildDir = path.join(dir, 'build/Release') +- const files = readdirSync(localPrebuildDir).filter(matchBuild) +- return files[0] && path.join(localPrebuildDir, files[0]) +-} +- +-function getLinuxType(platform) { +- if (platform !== 'linux') { +- return '' +- } +- return `linux${isAlpine(platform) ? 'musl' : ''}` +-} +- +-function getNodeMajorVersion(version) { +- const tokens = version.split('.') +- return parseInt(tokens[0]) +-} +- +-function getNodeVersion() { +- return process.version.replace('v', '') +-} +- +-function getPrebuildsInfo(dir) { +- dir = path.resolve(dir || '.') +- const info = { +- packageDir: path.join(dir, 'package.json'), +- platformPackageDir: undefined, +- } +- +- const packageName = JSON.parse(fs.readFileSync(info.packageDir)).name +- if (packageName !== undefined) { +- const allowedPlatformPkg = `${packageName}-${ +- platform === 'linux' ? libc : platform +- }-${arch}-${sslType}` +- const fullPlatformPkgName = `@${packageName}/${allowedPlatformPkg}` +- const packageRequire = require('module').createRequire( +- path.join(dir, 'package.json') +- ) +- info.packageDir = path.dirname(path.join(dir, 'package.json')) +- info.platformPackageDir = path.dirname( +- packageRequire.resolve(fullPlatformPkgName) +- ) +- } +- return info +-} +- +-function getSSLType(runtime, version) { +- if (runtime === 'electron') { +- return 'boringssl' +- } +- +- const major = getNodeMajorVersion(version) +- if (major >= 18) { +- return 'openssl3' +- } +- return 'openssl1' +-} +- +-function getSupportedPlatformPackages(packageName) { +- packageName = packageName || 'couchbase' +- if (packageName !== 'couchbase') { +- throw new Error( +- 'Cannot build supported platform packages for package other than couchbase.' +- ) +- } +- +- const packageNames = [] +- // format: --- +- supportedPlatforms.forEach((plat) => { +- supportedArches.forEach((arch) => { +- if (plat === 'win32' && arch === 'arm64') return +- openSSLVersions.forEach((ssl) => { +- packageNames.push(`${packageName}-${plat}-${arch}-${ssl}`) +- }) +- packageNames.push(`${packageName}-${plat}-${arch}-boringssl`) +- }) +- }) +- return packageNames +-} +- +-function isAlpine(platform) { +- return platform === 'linux' && fs.existsSync('/etc/alpine-release') +-} +- +-function isElectron() { +- if (process.versions && process.versions.electron) return true +- if (process.env.ELECTRON_RUN_AS_NODE) return true +- return ( +- typeof window !== 'undefined' && +- window.process && +- window.process.type === 'renderer' +- ) +-} +- +-function loadPrebuild(dir) { +- return runtimeRequire(resolvePrebuild(dir)) +-} +- +-function matchBuild(name) { +- return /\.node$/.test(name) +-} +- +-function matchingPlatformPrebuild(filename) { +- if (['index.js', 'package.json', 'README.md'].includes(filename)) { +- return false +- } +- const tokens = filename.split('-') +- // filename format: +- // couchbase-v--v---.node +- if (tokens.length < 7) return false +- const prebuildSSL = tokens[tokens.length - 1].replace('.node', '') +- if (runtime === 'electron') { +- if (prebuildSSL !== 'boringssl') return false +- } else { +- if (nodeVersionMajor >= 18 && prebuildSSL !== 'openssl3') return false +- if (nodeVersionMajor < 18 && prebuildSSL !== 'openssl1') return false +- } +- if (tokens[tokens.length - 2] !== arch) return false +- const platCompare = platform === 'linux' ? libc : platform +- if (tokens[tokens.length - 3] !== platCompare) return false +- if (!matchBuild(filename)) return false +- // yay -- found a match! +- return true +-} +- +-function matchPrebuild(name) { +- return /\.tar\.gz$/.test(name) || matchBuild(name) +-} +- +-function readdirSync(dir) { +- try { +- return fs.readdirSync(dir) +- } catch (err) { +- return [] +- } +-} +- +-async function resolveLocalPrebuild(src, dest) { +- if (fs.existsSync(src)) { +- const prebuilds = readdirSync(src).filter(matchPrebuild) +- if (prebuilds && prebuilds.length >= 1) { +- if (!fs.existsSync(dest)) { +- fs.mkdirSync(dest, { recursive: true }) +- } +- const prebuild = prebuilds[0] +- let prebuildDestName = prebuild +- if (prebuild.endsWith('.tar.gz')) { +- const gzip = zlib.createGunzip() +- const source = fs.createReadStream(src) +- prebuildDestName = `${prebuild.substring(0, prebuild.length - 7)}.node` +- const destination = fs.createWriteStream( +- path.join(src, prebuildDestName) +- ) +- await pipe(source, gzip, destination) +- } +- try { +- fs.copyFileSync( +- path.join(src, prebuild), +- path.join(dest, prebuildDestName) +- ) +- } catch (_) {} +- } +- } +-} +- +-function resolvePrebuild(dir, runtimeResolve = true) { +- dir = path.resolve(dir || '.') +- try { +- const localPrebuild = getLocalPrebuild(dir) +- if (localPrebuild) { +- return localPrebuild +- } +- +- const packageName = runtimeResolve +- ? runtimeRequire(path.join(dir, 'package.json')).name +- : JSON.parse(fs.readFileSync(path.join(dir, 'package.json'))).name +- if (packageName !== undefined) { +- const supportedPackages = getSupportedPlatformPackages(packageName) - const platformPkg = `${packageName}-${ - platform === 'linux' ? libc : platform - }-${arch}-${sslType}` -+ const platformPkg = `${packageName}-${platform}-${arch}`; - if (supportedPackages.includes(platformPkg)) { +- if (supportedPackages.includes(platformPkg)) { - const fullPlatformPkgName = `@${packageName}/${platformPkg}` -+ const fullPlatformPkgName = `@hackolade/${platformPkg}` - const packageRequire = require('module').createRequire( - path.join(dir, 'package.json') - ) +- const packageRequire = require('module').createRequire( +- path.join(dir, 'package.json') +- ) +- const platformPackagesDir = path.dirname( +- packageRequire.resolve(fullPlatformPkgName) +- ) +- if (platformPackagesDir !== undefined) { +- const platformPrebuild = readdirSync(platformPackagesDir).filter( +- matchingPlatformPrebuild +- ) +- if (platformPrebuild && platformPrebuild.length == 1) { +- return path.join(platformPackagesDir, platformPrebuild[0]) +- } +- } +- } +- } +- } catch (_) {} +- +- let target = [ +- `platform=${platform}`, +- `arch=${arch}`, +- `runtime=${runtime}`, +- `nodeVersion=${nodeVersion}`, +- `sslType=${sslType}`, +- ] +- if (libc) { +- target.push(`libc=${libc}`) +- } +- if (typeof __webpack_require__ === 'function') { +- target.push('webpack=true') +- } +- throw new Error( +- `Could not find native build for ${target.join(', ')} loaded from ${dir}.` +- ) ++function loadPrebuild() { ++ const prebuildToLoad = `@hackolade/couchbase-${os.platform()}-${os.arch()}`; ++ return require(prebuildToLoad); + } + + module.exports = { +- getPrebuildsInfo, +- getSupportedPlatformPackages, + loadPrebuild, +- resolveLocalPrebuild, +- resolvePrebuild, + } +diff --git a/node_modules/couchbase/scripts/prune.js b/node_modules/couchbase/scripts/prune.js +deleted file mode 100644 +index 46ab9c8..0000000 +--- a/node_modules/couchbase/scripts/prune.js ++++ /dev/null +@@ -1,124 +0,0 @@ +-const fs = require('fs') +-const path = require('path') +-const prebuilds = require('./prebuilds') +- +-function getMismatchedPlatformPackagesInfo( +- platformPackagesDir, +- expectedPlatformPackage +-) { +- let message = `Checking for platform packages in ${platformPackagesDir} ` +- message += `that do not match the expected platform package (${expectedPlatformPackage}).` +- console.log(message) +- let mismatches = [] +- try { +- const files = fs.readdirSync(platformPackagesDir) +- files.forEach((file) => { +- if (file === expectedPlatformPackage) { +- return +- } +- const stats = fs.statSync(path.join(platformPackagesDir, file)) +- if (!stats.isDirectory()) { +- return +- } +- const filePath = path.join(platformPackagesDir, file) +- const size = getDirectorySize(filePath) +- console.log(`Found mismatch: Path=${filePath}`) +- const platformPackage = path.basename(filePath) +- mismatches.push({ +- name: platformPackage, +- dir: filePath, +- size: size, +- }) +- }) +- } catch (err) { +- console.error(`Error trying to delete mismatched platform packages.`, err) +- } +- return mismatches +-} +- +-function getDirectorySize(dir) { +- let size = 0 +- const dirContents = fs.readdirSync(dir) +- dirContents.forEach((content) => { +- const contentPath = path.join(dir, content) +- const stats = fs.statSync(contentPath) +- if (stats.isFile()) { +- size += stats.size +- } else if (stats.isDirectory()) { +- size += getDirectorySize(contentPath) +- } +- }) +- +- return size +-} +- +-function getPrebuildsInfo() { +- try { +- const prebuildsInfo = prebuilds.getPrebuildsInfo() +- return prebuildsInfo +- } catch (err) { +- console.error('Error trying to obtain couchbase prebuilds info.', err) +- return undefined +- } +-} +- +-function pruneCouchbaseHelp() { +- const prebuildsInfo = getPrebuildsInfo() +- const platformPackagesDir = path.dirname(prebuildsInfo.platformPackageDir) +- const expectedPlatformPackage = path.basename( +- prebuildsInfo.platformPackageDir +- ) +- +- const mismatchedPlatPkgs = getMismatchedPlatformPackagesInfo( +- platformPackagesDir, +- expectedPlatformPackage +- ) +- const cbDeps = { +- dir: path.join(prebuildsInfo.packageDir, 'deps'), +- size: undefined, +- } +- const cbSrc = { +- dir: path.join(prebuildsInfo.packageDir, 'src'), +- size: undefined, +- } +- try { +- cbDeps.size = getDirectorySize(cbDeps.dir) +- } catch (_) { +- console.log('Couchbase deps/ not found.') +- } +- try { +- cbSrc.size = getDirectorySize(cbSrc.dir) +- } catch (_) { +- console.log('Couchbase src/ not found.') +- } +- +- console.log('\nRecommendations for pruning:\n') +- if (mismatchedPlatPkgs.length > 0) { +- for (const pkg of mismatchedPlatPkgs) { +- const sizeMb = pkg.size / 1024 / 1024 +- console.log( +- `Removing mismatched platform=${pkg.name} (path=${ +- pkg.dir +- }) saves ~${sizeMb.toFixed(2)} MB on disk.` +- ) +- } +- } +- if (cbDeps.size) { +- const sizeMb = cbDeps.size / 1024 / 1024 +- console.log( +- `Removing Couchbase deps/ (path=${cbDeps.dir}) saves ~${sizeMb.toFixed( +- 2 +- )} MB on disk.` +- ) +- } +- if (cbSrc.size) { +- const sizeMb = cbSrc.size / 1024 / 1024 +- console.log( +- `Removing Couchbase src/ (path=${cbSrc.dir}) saves ~${sizeMb.toFixed( +- 2 +- )} MB on disk.` +- ) +- } +-} +- +-pruneCouchbaseHelp()