From 19c98692de51a178bd6e3b762bb57c8660a66808 Mon Sep 17 00:00:00 2001 From: Wayne Parrott <5588978+wayneparrott@users.noreply.github.com> Date: Mon, 30 Jan 2023 15:42:52 -0600 Subject: [PATCH] Added RosPackageFilters & blocklist.json On windows workflow use node 18.12.0 inplace of 18.13.0. There is a repeatable issue with node-gyp configuration on node 18.13. It seems to be related to the node cache. Switching to 18.12 avoids using a cached version of node 18 and the issue no longer occurs. Fix #890 --- .../windows-build-and-test-compatibility.yml | 2 +- .github/workflows/windows-build-and-test.yml | 2 +- rosidl_gen/blocklist.json | 5 + rosidl_gen/filter.js | 104 ++++++++++++++++++ rosidl_gen/index.js | 3 + rosidl_gen/packages.js | 78 ++++++------- rostsd_gen/index.js | 10 +- 7 files changed, 162 insertions(+), 42 deletions(-) create mode 100644 rosidl_gen/blocklist.json create mode 100644 rosidl_gen/filter.js diff --git a/.github/workflows/windows-build-and-test-compatibility.yml b/.github/workflows/windows-build-and-test-compatibility.yml index 6f0104a8..773ad0a3 100644 --- a/.github/workflows/windows-build-and-test-compatibility.yml +++ b/.github/workflows/windows-build-and-test-compatibility.yml @@ -10,7 +10,7 @@ jobs: strategy: fail-fast: false matrix: - node-version: [10.X, 12.X, 14.X, 16.11.X, 17.X, 18.X, 19.X] + node-version: [10.X, 12.X, 14.X, 16.11.X, 17.X, 18.12.X, 19.X] ros_distribution: # - foxy - galactic diff --git a/.github/workflows/windows-build-and-test.yml b/.github/workflows/windows-build-and-test.yml index aeaebd5e..186695ed 100644 --- a/.github/workflows/windows-build-and-test.yml +++ b/.github/workflows/windows-build-and-test.yml @@ -26,7 +26,7 @@ jobs: strategy: fail-fast: false matrix: - node-version: [10.X, 12.X, 14.X, 16.11.X, 17.X, 18.X, 19.X] + node-version: [10.X, 12.X, 14.X, 16.11.X, 17.X, 18.12.0, 19.X] steps: - name: Setup Node.js ${{ matrix.node-version }} uses: actions/setup-node@v2 diff --git a/rosidl_gen/blocklist.json b/rosidl_gen/blocklist.json new file mode 100644 index 00000000..b2a1ac19 --- /dev/null +++ b/rosidl_gen/blocklist.json @@ -0,0 +1,5 @@ +[ + { + "pkgName": "rosbag2_storage_mcap_testdata" + } +] \ No newline at end of file diff --git a/rosidl_gen/filter.js b/rosidl_gen/filter.js new file mode 100644 index 00000000..a683a373 --- /dev/null +++ b/rosidl_gen/filter.js @@ -0,0 +1,104 @@ +const fs = require('fs'); +const path = require('path'); +const os = require('os'); + +// blocklist.json format +// [ +// { +// pkgName: RegExString, +// interfaceName: RegExString, +// os: RegExString +// }, +// ... +// ] +// +// examples +// [ +// { +// "pkgName": "action*" +// }, +// { +// "pkgName": "std_msgs", +// }, +// { +// "pkgName": "std_msgs", +// "interfaceName": "String" +// }, +// { +// "os": "Linux" +// }, +// ] + +const RosPackageFilters = { + filters: [], + _loaded: false, + + addFilter: function (pkgName, interfaceName, os) { + this.filters.push({ + pkgName: pkgName, + interfaceName: interfaceName, + os: os, + }); + }, + + _matches: function (filter, pkgInfo) { + if (filter.os && filter.os.test(os.type())) { + return true; + } + + if (filter.pkgName) { + if (filter.pkgName.test(pkgInfo.pkgName)) { + if (!filter.interfaceName) { + return true; + } + } else { + return false; + } + } + + if ( + filter.interfaceName && + filter.interfaceName.test(pkgInfo.interfaceName) + ) { + return true; + } + + return false; + }, + + load: function ( + blocklistPath = path.join(__dirname, '../rosidl_gen/blocklist.json') + ) { + this._loaded = true; + + if (!fs.existsSync(blocklistPath)) return; + + // eslint-disable-next-line + let blocklistData = JSON.parse(fs.readFileSync(blocklistPath, 'utf8')); + + let filters = blocklistData.map((pkgFilterData) => { + let filter = {}; + if (pkgFilterData['pkgName']) { + filter.pkgName = new RegExp(pkgFilterData.pkgName); + } + if (pkgFilterData['interfaceName']) { + filter.interfaceName = new RegExp(pkgFilterData.interfaceName); + } + if (pkgFilterData['os']) { + filter.os = new RegExp(pkgFilterData.os); + } + return filter; + }); + + this.filters = filters.filter( + (filter) => !filter.os || filter.os.test(os.type()) + ); + }, + + matchesAny: function (pkgInfo) { + if (!this._loaded) this.load(); + return this.filters.some((filter) => this._matches(filter, pkgInfo)); + }, +}; + +module.exports = RosPackageFilters; diff --git a/rosidl_gen/index.js b/rosidl_gen/index.js index bab0b6cb..118e4a17 100644 --- a/rosidl_gen/index.js +++ b/rosidl_gen/index.js @@ -26,7 +26,9 @@ const installedPackagePaths = process.env.AMENT_PREFIX_PATH.split( async function generateInPath(path) { const pkgs = await packages.findPackagesInDirectory(path); + const pkgsInfo = Array.from(pkgs.values()); + await Promise.all( pkgsInfo.map((pkgInfo) => generateJSStructFromIDL(pkgInfo, generatedRoot)) ); @@ -42,6 +44,7 @@ async function generateAll(forcedGenerating) { path.join(__dirname, 'generator.json'), path.join(generatedRoot, 'generator.json') ); + await Promise.all( installedPackagePaths.map((path) => generateInPath(path)) ); diff --git a/rosidl_gen/packages.js b/rosidl_gen/packages.js index d9490d24..a87d091c 100644 --- a/rosidl_gen/packages.js +++ b/rosidl_gen/packages.js @@ -21,6 +21,7 @@ const path = require('path'); const walk = require('walk'); const os = require('os'); const flat = require('array.prototype.flat'); +const pkgFilters = require('../rosidl_gen/filter.js'); const fsp = fs.promises; @@ -140,28 +141,30 @@ async function findAmentPackagesInDirectory(dir) { pkgs.map((pkg) => getPackageDefinitionsFiles(pkg, dir)) ); - // Support flat() methond for nodejs < 11. + // Support flat() method for nodejs < 11. const rosFiles = Array.prototype.flat ? files.flat() : flat(files); const pkgMap = new Map(); return new Promise((resolve, reject) => { rosFiles.forEach((filePath) => { - if (path.extname(filePath) === '.msg') { - // Some .msg files were generated prior to 0.3.2 for .action files, - // which has been disabled. So these files should be ignored here. - if (path.dirname(dir).split(path.sep).pop() !== 'action') { - addInterfaceInfo( - grabInterfaceInfo(filePath, true), - 'messages', - pkgMap - ); - } - } else if (path.extname(filePath) === '.srv') { - addInterfaceInfo(grabInterfaceInfo(filePath, true), 'services', pkgMap); - } else if (path.extname(filePath) === '.action') { - addInterfaceInfo(grabInterfaceInfo(filePath, true), 'actions', pkgMap); + const interfaceInfo = grabInterfaceInfo(filePath, true); + const ignore = pkgFilters.matchesAny(interfaceInfo); + if (ignore) { + console.log('Omitting filtered interface: ', interfaceInfo); } else { - // we ignore all other files + if (path.extname(filePath) === '.msg') { + // Some .msg files were generated prior to 0.3.2 for .action files, + // which has been disabled. So these files should be ignored here. + if (path.dirname(dir).split(path.sep).pop() !== 'action') { + addInterfaceInfo(interfaceInfo, 'messages', pkgMap); + } + } else if (path.extname(filePath) === '.srv') { + addInterfaceInfo(interfaceInfo, 'services', pkgMap); + } else if (path.extname(filePath) === '.action') { + addInterfaceInfo(interfaceInfo, 'actions', pkgMap); + } else { + // we ignore all other files + } } }); resolve(pkgMap); @@ -191,30 +194,27 @@ async function findPackagesInDirectory(dir) { let walker = walk.walk(dir, { followLinks: true }); let pkgMap = new Map(); walker.on('file', (root, file, next) => { - if (path.extname(file.name) === '.msg') { - // Some .msg files were generated prior to 0.3.2 for .action files, - // which has been disabled. So these files should be ignored here. - if (path.dirname(root).split(path.sep).pop() !== 'action') { - addInterfaceInfo( - grabInterfaceInfo(path.join(root, file.name), amentExecuted), - 'messages', - pkgMap - ); - } - } else if (path.extname(file.name) === '.srv') { - addInterfaceInfo( - grabInterfaceInfo(path.join(root, file.name), amentExecuted), - 'services', - pkgMap - ); - } else if (path.extname(file.name) === '.action') { - addInterfaceInfo( - grabInterfaceInfo(path.join(root, file.name), amentExecuted), - 'actions', - pkgMap - ); + const interfaceInfo = grabInterfaceInfo( + path.join(root, file.name), + amentExecuted + ); + const ignore = pkgFilters.matchesAny(interfaceInfo); + if (ignore) { + console.log('Omitting filtered interface: ', interfaceInfo); } else { - // we ignore all other files + if (path.extname(file.name) === '.msg') { + // Some .msg files were generated prior to 0.3.2 for .action files, + // which has been disabled. So these files should be ignored here. + if (path.dirname(root).split(path.sep).pop() !== 'action') { + addInterfaceInfo(interfaceInfo, 'messages', pkgMap); + } + } else if (path.extname(file.name) === '.srv') { + addInterfaceInfo(interfaceInfo, 'services', pkgMap); + } else if (path.extname(file.name) === '.action') { + addInterfaceInfo(interfaceInfo, 'actions', pkgMap); + } else { + // we ignore all other files + } } next(); }); diff --git a/rostsd_gen/index.js b/rostsd_gen/index.js index cf05c18f..97365ade 100644 --- a/rostsd_gen/index.js +++ b/rostsd_gen/index.js @@ -31,6 +31,7 @@ declare module "rclnodejs" { const path = require('path'); const fs = require('fs'); const loader = require('../lib/interface_loader.js'); +const pkgFilters = require('../rosidl_gen/filter.js'); async function generateAll() { // load pkg and interface info (msgs and srvs) @@ -63,7 +64,14 @@ function getPkgInfos(rootDir) { for (let filename of files) { const typeClass = fileName2Typeclass(filename); - if (!typeClass.type) continue; + if ( + !typeClass.type || + pkgFilters.matchesAny({ + pkgName: typeClass.package, + interfaceName: typeClass.name, + }) + ) + continue; const rosInterface = loader.loadInterface(typeClass);