fix(scripts): fix Esbuild scripts to allow to run on Windows (#5930)

* fix(scripts): fix Esbuild scripts to allow to run on Windows

* run wdio tests as part of main test pipeline

* more improvements

* prettier

* minor tweak

* progress

* more fixes

* some optimizations

* prettier

* lint

.github/workflows/build.yml

@@ -8,7 +8,10 @@ on:
 jobs:
   build_core:
     name: Core
-    runs-on: 'ubuntu-22.04'
+    strategy:
+      matrix:
+        os: ['ubuntu-22.04', 'windows-latest']
+    runs-on: ${{ matrix.os }}
     steps:
       - name: Checkout Code
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
@@ -29,6 +32,7 @@ jobs:
         shell: bash
 
       - name: Upload Build Artifacts
+        if: ${{ matrix.os == 'ubuntu-22.04' }}
         uses: ./.github/workflows/actions/upload-archive
         with:
           name: stencil-core

.github/workflows/main.yml

@@ -57,3 +57,8 @@ jobs:
     name: Unit Tests
     needs: [build_core]
     uses: ./.github/workflows/test-unit.yml
+
+  wdio_tests:
+    name: WebdriverIO Tests
+    needs: [build_core]
+    uses: ./.github/workflows/test-wdio.yml

.github/workflows/test-wdio.yml

@@ -1,21 +1,14 @@
 name: WebdriverIO Tests
 
 on:
-  merge_group:
-  pull_request:
-  push:
-    branches:
-      - 'main'
+  workflow_call:
+    # Make this a reusable workflow, no value needed
+    # https://docs.github.com/en/actions/using-workflows/reusing-workflows
 
 jobs:
-  build_core:
-    name: Build
-    uses: ./.github/workflows/build.yml
-
   run_wdio:
     name: Run WebdriverIO Component Tests (${{ matrix.browser }})
     runs-on: ubuntu-22.04
-    needs: [build_core]
     strategy:
       matrix:
         browser: [CHROME, FIREFOX, EDGE]
@@ -24,6 +17,9 @@ jobs:
       - name: Checkout Code
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
+      - name: Get Core Dependencies
+        uses: ./.github/workflows/actions/get-core-dependencies
+
       - name: Use Node Version from Volta
         uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4.0.3
         with:

scripts/esbuild/cli.ts

@@ -5,7 +5,7 @@ import { join } from 'path';
 import { getBanner } from '../utils/banner';
 import { BuildOptions } from '../utils/options';
 import { writePkgJson } from '../utils/write-pkg-json';
-import { getBaseEsbuildOptions, getEsbuildAliases, getEsbuildExternalModules, runBuilds } from './utils';
+import { externalNodeModules, getBaseEsbuildOptions, getEsbuildAliases, runBuilds } from './utils';
 
 /**
  * Runs esbuild to bundle the `cli` submodule
@@ -30,7 +30,7 @@ export async function buildCli(opts: BuildOptions) {
   // this isn't strictly necessary to alias - however, this minimizes cuts down the bundle size by ~70kb.
   cliAliases['prompts'] = 'prompts/lib/index.js';
 
-  const external = getEsbuildExternalModules(opts, opts.output.cliDir);
+  const external = [...externalNodeModules, '../testing/*'];
 
   const cliEsbuildOptions = {
     ...getBaseEsbuildOptions(),

scripts/esbuild/compiler.ts

@@ -6,7 +6,7 @@ import { join } from 'path';
 import { getBanner } from '../utils/banner';
 import { BuildOptions, createReplaceData } from '../utils/options';
 import { writePkgJson } from '../utils/write-pkg-json';
-import { getBaseEsbuildOptions, getEsbuildAliases, getEsbuildExternalModules, runBuilds } from './utils';
+import { externalNodeModules, getBaseEsbuildOptions, getEsbuildAliases, runBuilds } from './utils';
 import { bundleParse5 } from './utils/parse5';
 import { bundleTerser } from './utils/terser';
 import { bundleTypeScriptSource, tsCacheFilePath } from './utils/typescript-source';
@@ -44,10 +44,15 @@ export async function buildCompiler(opts: BuildOptions) {
   transpileDts = transpileDts.replace('@stencil/core/internal', '../internal/index');
   await fs.writeFile(join(opts.output.compilerDir, 'transpile.d.ts'), transpileDts);
 
-  const alias = getEsbuildAliases();
+  const alias: Record<string, string> = {
+    ...getEsbuildAliases(),
+    glob: './sys/node/glob.js',
+    '@sys-api-node': '../sys/node/index.js',
+  };
 
   const external = [
-    ...getEsbuildExternalModules(opts, opts.output.compilerDir),
+    ...externalNodeModules,
+    '../mock-doc/index.cjs',
     '../sys/node/autoprefixer.js',
     '../sys/node/index.js',
   ];

scripts/esbuild/dev-server.ts

@@ -78,7 +78,11 @@ export async function buildDevServer(opts: BuildOptions) {
     './open-in-editor-api',
   ];
 
-  const devServerAliases = getEsbuildAliases();
+  const devServerAliases = {
+    ...getEsbuildAliases(),
+    glob: '../../sys/node/glob.js',
+    '@stencil/core/mock-doc': '../../mock-doc/index.cjs',
+  };
   const devServerIndexEsbuildOptions = {
     ...getBaseEsbuildOptions(),
     alias: devServerAliases,
@@ -98,6 +102,8 @@ export async function buildDevServer(opts: BuildOptions) {
     ...getBaseEsbuildOptions(),
     alias: {
       ...devServerAliases,
+      glob: '../../sys/node/glob.js',
+      '@stencil/core/mock-doc': '../../mock-doc/index.cjs',
       '@sys-api-node': '../sys/node/index.js',
     },
     entryPoints: [join(inputDir, 'server-process.js')],
@@ -118,7 +124,9 @@ export async function buildDevServer(opts: BuildOptions) {
   } satisfies ESBuildOptions;
 
   const connectorAlias = {
+    glob: '../../sys/node/glob.js',
     '@stencil/core/dev-server/client': join(inputDir, 'client', 'index.js'),
+    '@stencil/core/mock-doc': '../../mock-doc/index.cjs',
   };
   const connectorEsbuildOptions = {
     ...getBaseEsbuildOptions(),

scripts/esbuild/internal-platform-client.ts

@@ -7,7 +7,7 @@ import { join } from 'path';
 import { getBanner } from '../utils/banner';
 import { BuildOptions, createReplaceData } from '../utils/options';
 import { writePkgJson } from '../utils/write-pkg-json';
-import { externalAlias, getBaseEsbuildOptions, getEsbuildAliases, getEsbuildExternalModules } from './utils';
+import { externalAlias, externalNodeModules, getBaseEsbuildOptions, getEsbuildAliases } from './utils';
 
 /**
  * Create objects containing ESbuild options for the two bundles which need to
@@ -41,7 +41,7 @@ export async function getInternalClientBundles(opts: BuildOptions): Promise<ESBu
   const internalClientAliases = getEsbuildAliases();
   internalClientAliases['@platform'] = join(inputClientDir, 'index.ts');
 
-  const clientExternal = getEsbuildExternalModules(opts, opts.output.internalDir);
+  const clientExternal = externalNodeModules;
 
   const internalClientBundle: ESBuildOptions = {
     ...getBaseEsbuildOptions(),
@@ -69,14 +69,10 @@ export async function getInternalClientBundles(opts: BuildOptions): Promise<ESBu
 
   const polyfills = await fs.readdir(join(opts.srcDir, 'client', 'polyfills'));
   for (const polyFillFile of polyfills) {
-    patchBrowserAliases[join('./polyfills', polyFillFile)] = join(opts.srcDir, 'client', 'polyfills');
+    patchBrowserAliases[`polyfills/${polyFillFile}`] = join(opts.srcDir, 'client', 'polyfills');
   }
 
-  const patchBrowserExternal = [
-    ...getEsbuildExternalModules(opts, opts.output.internalDir),
-    '@stencil/core',
-    '@stencil/core/mock-doc',
-  ];
+  const patchBrowserExternal = [...externalNodeModules, '@stencil/core', '@stencil/core/mock-doc'];
 
   const internalClientPatchBrowserBundle: ESBuildOptions = {
     ...getBaseEsbuildOptions(),

scripts/esbuild/internal-platform-hydrate.ts

@@ -6,7 +6,7 @@ import { getBanner } from '../utils/banner';
 import { bundleDts } from '../utils/bundle-dts';
 import { BuildOptions } from '../utils/options';
 import { writePkgJson } from '../utils/write-pkg-json';
-import { externalAlias, getBaseEsbuildOptions, getEsbuildAliases, getEsbuildExternalModules } from './utils';
+import { externalAlias, externalNodeModules, getBaseEsbuildOptions, getEsbuildAliases } from './utils';
 
 /**
  * Create objects containing ESbuild options for the two bundles comprising
@@ -35,7 +35,7 @@ export async function getInternalPlatformHydrateBundles(opts: BuildOptions): Pro
 
   const hydratePlatformInput = join(hydrateSrcDir, 'platform', 'index.js');
 
-  const external = getEsbuildExternalModules(opts, outputInternalHydrateDir);
+  const external = externalNodeModules;
 
   const internalHydrateAliases = getEsbuildAliases();
   internalHydrateAliases['@platform'] = hydratePlatformInput;

scripts/esbuild/internal-platform-testing.ts

@@ -4,7 +4,7 @@ import { join } from 'path';
 
 import { BuildOptions } from '../utils/options';
 import { writePkgJson } from '../utils/write-pkg-json';
-import { externalAlias, getBaseEsbuildOptions, getEsbuildAliases, getEsbuildExternalModules } from './utils';
+import { externalAlias, externalNodeModules, getBaseEsbuildOptions, getEsbuildAliases } from './utils';
 
 /**
  * Get an ESBuild configuration object for the internal testing bundle. This
@@ -31,9 +31,10 @@ export async function getInternalTestingBundle(opts: BuildOptions): Promise<ESBu
   const internalTestingAliases = {
     ...getEsbuildAliases(),
     '@platform': inputTestingPlatform,
+    '@stencil/core/mock-doc': '../../mock-doc/index.cjs',
   };
 
-  const external = getEsbuildExternalModules(opts, opts.output.internalDir);
+  const external: string[] = [...externalNodeModules, '../../mock-doc/index.cjs'];
 
   const internalTestingBuildOptions: ESBuildOptions = {
     ...getBaseEsbuildOptions(),

scripts/esbuild/screenshot.ts

@@ -5,7 +5,7 @@ import { join } from 'path';
 import { getBanner } from '../utils/banner';
 import { BuildOptions } from '../utils/options';
 import { writePkgJson } from '../utils/write-pkg-json';
-import { getBaseEsbuildOptions, getEsbuildAliases, getEsbuildExternalModules, runBuilds } from './utils';
+import { externalNodeModules, getBaseEsbuildOptions, getEsbuildAliases, runBuilds } from './utils';
 
 const screenshotBuilds = {
   'Stencil Screenshot': 'index',
@@ -39,9 +39,7 @@ export async function buildScreenshot(opts: BuildOptions) {
   });
 
   const aliases = getEsbuildAliases();
-
-  const external = getEsbuildExternalModules(opts, opts.output.screenshotDir);
-
+  const external = externalNodeModules;
   const baseScreenshotOptions = {
     ...getBaseEsbuildOptions(),
     alias: aliases,

scripts/esbuild/sys-node.ts

@@ -1,27 +1,27 @@
 import type { BuildOptions as ESBuildOptions } from 'esbuild';
 import fs from 'fs-extra';
-import { join } from 'path';
+import path from 'path';
 import resolve from 'resolve';
 import webpack, { Configuration } from 'webpack';
 
 import { getBanner } from '../utils/banner';
 import type { BuildOptions } from '../utils/options';
 import { writePkgJson } from '../utils/write-pkg-json';
-import { externalAlias, getBaseEsbuildOptions, getEsbuildAliases, getEsbuildExternalModules, runBuilds } from './utils';
+import { externalAlias, externalNodeModules, getBaseEsbuildOptions, getEsbuildAliases, runBuilds } from './utils';
 
 export async function buildSysNode(opts: BuildOptions) {
-  const inputDir = join(opts.buildDir, 'sys', 'node');
-  const srcDir = join(opts.srcDir, 'sys', 'node');
-  const inputFile = join(srcDir, 'index.ts');
-  const outputFile = join(opts.output.sysNodeDir, 'index.js');
+  const inputDir = path.join(opts.buildDir, 'sys', 'node');
+  const srcDir = path.join(opts.srcDir, 'sys', 'node');
+  const inputFile = path.join(srcDir, 'index.ts');
+  const outputFile = path.join(opts.output.sysNodeDir, 'index.js');
 
   // clear out rollup stuff and ensure directory exists
   await fs.emptyDir(opts.output.sysNodeDir);
 
   // create public d.ts
-  let dts = await fs.readFile(join(inputDir, 'public.d.ts'), 'utf8');
+  let dts = await fs.readFile(path.join(inputDir, 'public.d.ts'), 'utf8');
   dts = dts.replace('@stencil/core/internal', '../../internal/index');
-  await fs.writeFile(join(opts.output.sysNodeDir, 'index.d.ts'), dts);
+  await fs.writeFile(path.join(opts.output.sysNodeDir, 'index.d.ts'), dts);
 
   // write @stencil/core/sys/node/package.json
   writePkgJson(opts, opts.output.sysNodeDir, {
@@ -32,14 +32,20 @@ export async function buildSysNode(opts: BuildOptions) {
   });
 
   const external = [
-    ...getEsbuildExternalModules(opts, opts.output.sysNodeDir),
+    ...externalNodeModules,
     // normally you wouldn't externalize your "own" directory here, but since
     // we build multiple things within `opts.output.sysNodeDir` which should
     // externalize each other we need to do so
-    join(opts.output.sysNodeDir, '*'),
+    '../../compiler/stencil.js',
+    '../../sys/node/index.js',
+    './glob.js',
   ];
 
-  const sysNodeAliases = getEsbuildAliases();
+  const sysNodeAliases = {
+    ...getEsbuildAliases(),
+    '@stencil/core/compiler': '../../compiler/stencil.js',
+    '@sys-api-node': '../../sys/node/index.js',
+  };
 
   const sysNodeOptions: ESBuildOptions = {
     ...getBaseEsbuildOptions(),
@@ -57,8 +63,8 @@ export async function buildSysNode(opts: BuildOptions) {
   };
 
   // sys/node/worker.js bundle
-  const inputWorkerFile = join(srcDir, 'worker.ts');
-  const outputWorkerFile = join(opts.output.sysNodeDir, 'worker.js');
+  const inputWorkerFile = path.join(srcDir, 'worker.ts');
+  const outputWorkerFile = path.join(opts.output.sysNodeDir, 'worker.js');
 
   const workerOptions: ESBuildOptions = {
     ...getBaseEsbuildOptions(),
@@ -81,7 +87,7 @@ export async function buildSysNode(opts: BuildOptions) {
 
 export const sysNodeBundleCacheDir = 'sys-node-bundle-cache';
 async function sysNodeExternalBundles(opts: BuildOptions) {
-  const cachedDir = join(opts.scriptsBuildDir, sysNodeBundleCacheDir);
+  const cachedDir = path.join(opts.scriptsBuildDir, sysNodeBundleCacheDir);
 
   await fs.ensureDir(cachedDir);
 
@@ -101,27 +107,27 @@ async function sysNodeExternalBundles(opts: BuildOptions) {
    * is not supported by Jest v26. This is a workaround to remove the `node:`.
    * TODO(STENCIL-1323): remove once we deprecated Jest v26 support
    */
-  const globOutputPath = join(opts.output.sysNodeDir, 'glob.js');
+  const globOutputPath = path.join(opts.output.sysNodeDir, 'glob.js');
   const glob = fs.readFileSync(globOutputPath, 'utf8');
   fs.writeFileSync(globOutputPath, glob.replace(/require\("node:/g, 'require("'));
 
   // open-in-editor's visualstudio.vbs file
   // TODO(STENCIL-1052): remove once Rollup -> esbuild migration is complete
-  const visualstudioVbsSrc = join(opts.nodeModulesDir, 'open-in-editor', 'lib', 'editors', 'visualstudio.vbs');
-  const visualstudioVbsDesc = join(opts.output.devServerDir, 'visualstudio.vbs');
+  const visualstudioVbsSrc = path.join(opts.nodeModulesDir, 'open-in-editor', 'lib', 'editors', 'visualstudio.vbs');
+  const visualstudioVbsDesc = path.join(opts.output.devServerDir, 'visualstudio.vbs');
   await fs.copy(visualstudioVbsSrc, visualstudioVbsDesc);
 
   // copy open's xdg-open file
   // TODO(STENCIL-1052): remove once Rollup -> esbuild migration is complete
-  const xdgOpenSrcPath = join(opts.nodeModulesDir, 'open', 'xdg-open');
-  const xdgOpenDestPath = join(opts.output.devServerDir, 'xdg-open');
+  const xdgOpenSrcPath = path.join(opts.nodeModulesDir, 'open', 'xdg-open');
+  const xdgOpenDestPath = path.join(opts.output.devServerDir, 'xdg-open');
   await fs.copy(xdgOpenSrcPath, xdgOpenDestPath);
 }
 
 export function bundleExternal(opts: BuildOptions, outputDir: string, cachedDir: string, entryFileName: string) {
   return new Promise<void>(async (resolveBundle, rejectBundle) => {
-    const outputFile = join(outputDir, entryFileName);
-    const cachedFile = join(cachedDir, entryFileName) + (opts.isProd ? '.min.js' : '');
+    const outputFile = path.join(outputDir, entryFileName);
+    const cachedFile = path.join(cachedDir, entryFileName) + (opts.isProd ? '.min.js' : '');
 
     const cachedExists = fs.existsSync(cachedFile);
     if (cachedExists) {
@@ -132,7 +138,7 @@ export function bundleExternal(opts: BuildOptions, outputDir: string, cachedDir:
 
     const whitelist = new Set(['child_process', 'os', 'typescript']);
     const webpackConfig: Configuration = {
-      entry: join(opts.srcDir, 'sys', 'node', 'bundles', entryFileName),
+      entry: path.join(opts.srcDir, 'sys', 'node', 'bundles', entryFileName),
       output: {
         path: outputDir,
         filename: entryFileName,
@@ -166,10 +172,10 @@ export function bundleExternal(opts: BuildOptions, outputDir: string, cachedDir:
       },
       resolve: {
         alias: {
-          '@utils': join(opts.buildDir, 'utils', 'index.js'),
-          postcss: join(opts.nodeModulesDir, 'postcss'),
-          'source-map': join(opts.nodeModulesDir, 'source-map'),
-          chalk: join(opts.bundleHelpersDir, 'empty.js'),
+          '@utils': path.join(opts.buildDir, 'utils', 'index.js'),
+          postcss: path.join(opts.nodeModulesDir, 'postcss'),
+          'source-map': path.join(opts.nodeModulesDir, 'source-map'),
+          chalk: path.join(opts.bundleHelpersDir, 'empty.js'),
         },
       },
       optimization: {