print output from subprocesses in debug mode

Cakefile

@@ -1,23 +1,22 @@
-assert                    = require 'assert'
-fs                        = require 'fs'
-os                        = require 'os'
-path                      = require 'path'
-stream                    = require 'stream'
-{ spawn, exec, execSync } = require 'child_process'
-CoffeeScript              = require './lib/coffeescript'
-helpers                   = require './lib/coffeescript/helpers'
-util                      = require 'util'
-process                   = require 'process'
-
-{ BuildDeps }             = require './build-support/build-deps'
-{ TaskFailed  }           = require './build-support/caching'
-{ CompileSources }        = require './build-support/compile-sources'
-{ JisonParser }           = require './build-support/parser'
-{ setupConsole }          = require './build-support/console'
+assert                      = require 'assert'
+fs                          = require 'fs'
+os                          = require 'os'
+path                        = require 'path'
+stream                      = require 'stream'
+{ spawn, exec, execSync }   = require 'child_process'
+CoffeeScript                = require './lib/coffeescript'
+helpers                     = require './lib/coffeescript/helpers'
+util                        = require 'util'
+process                     = require 'process'
+{ BuildDeps }               = require './build-support/build-deps'
+{ TaskFailed  }             = require './build-support/caching'
+{ CompileBootstrapSources } = require './build-support/compile-sources'
+{ JisonParser }             = require './build-support/parser'
+{ setupConsole }            = require './build-support/console'
 {
   spawnNodeProcess,
   NonZeroExit,
-}                         = require './build-support/subprocess'
+}                           = require './build-support/subprocess'
 
 
 option '-l', '--level [LEVEL]', 'log level [debug < info < log(default) < warn < error]'
@@ -89,10 +88,11 @@ class TopLevelError extends AggregateError
 
     process.exit 1
 
-  @executeParallel: (tasks) -> Promise.allSettled(tasks).then (results) =>
+  @executeParallel: (tasks) ->
+    results = await Promise.allSettled tasks
     failures = (reason for {status, value, reason} in results when status is 'rejected')
     if failures.length > 0
-      Promise.reject new @ failures
+      throw new @ failures
 
 
 
@@ -117,9 +117,9 @@ buildParser = ->
   #     FIXME: see register.coffee: --enable-source-maps works for this???
   #            using 'node --enable-source-maps bin/cake test' appears to make #4418 pass????
   buildDepsTask = new BuildDeps
-  await buildDepsTask.cachedExecute(console).catch (e) -> Promise.reject new BootstrapFailure e
+  await buildDepsTask.cachedExecute(console, {useColors: USE_COLORS}).catch (e) -> Promise.reject new BootstrapFailure e
   parserTask = new JisonParser
-  await parserTask.cachedExecute(console).catch (e) -> Promise.reject new ParserFailure e
+  await parserTask.cachedExecute(console, {useColors: USE_COLORS}).catch (e) -> Promise.reject new ParserFailure e
 
 
 class CompileFailures extends TopLevelError
@@ -135,9 +135,9 @@ buildExceptParser = ->
     continue unless ext in ['.coffee', '.litcoffee']
     coffeeSource = path.join 'src', file
     jsOut = path.join 'lib/coffeescript', "#{name}.js"
-    new CompileSources {coffeeSource, jsOut}
+    new CompileBootstrapSources {coffeeSource, jsOut}
 
-  CompileFailures.executeParallel compileRequests.map (r) -> r.cachedExecute console
+  CompileFailures.executeParallel compileRequests.map (r) -> r.cachedExecute console, {useColors: USE_COLORS}
 
 
 class FullBuildError extends TopLevelError

build-support/build-deps.coffee

@@ -1,5 +1,6 @@
-{ BuildTask, ChecksumFiles }  = require './caching'
+{ BuildTask, ChecksumFiles }     = require './caching'
 { captureOutput, invokeProcess } = require './subprocess'
+util                             = require 'util'
 
 
 exports.BuildDeps = class BuildDeps extends BuildTask
@@ -15,6 +16,13 @@ exports.BuildDeps = class BuildDeps extends BuildTask
   outputSources: -> new ChecksumFiles [@installLockFile]
   print: -> "npm install: [#{@trackedSpecFile}, #{@trackedLockfile}] -> #{@installLockFile}"
 
-  execute: ->
+  execute: (console, {useColors}) ->
     proc = await invokeProcess 'npm', ['install', '.']
-    await captureOutput proc
+    output = (await captureOutput proc).trim()
+
+    header = 'npm output:'
+    if useColors
+      header = util.styleText ['underline', 'yellow', 'italic'], header
+      output = util.styleText ['bgGray', 'yellowBright'], output
+    console.debug header
+    console.debug output

build-support/caching.coffee

@@ -179,7 +179,7 @@ exports.BuildTask = class BuildTask
 
   execute: -> throw new TypeError "unimplemented: #{@constructor.name}"
 
-  cachedExecute: (console) ->
+  cachedExecute: (console, {useColors}) ->
     if await @cacheIsValid()
       console.debug "task '#{@identifier()}' was fully cached!"
       console.debug "task '#{@identifier()}' is cached at '#{@attestationPath()}'"
@@ -188,7 +188,7 @@ exports.BuildTask = class BuildTask
     console.log @print()
     startTask = performance.now()
 
-    await @execute(console).catch (e) => Promise.reject new TaskFailed @, e
+    await @execute(console, {useColors}).catch (e) => Promise.reject new TaskFailed @, e
 
     endTask = performance.now()
     console.info "task '#{@identifier()}' complete (#{endTask - startTask} ms)"

build-support/compile-sources.coffee

@@ -1,18 +1,20 @@
-{ BuildTask, ChecksumFiles }  = require './caching'
+{ BuildTask, ChecksumFiles }     = require './caching'
 { invokeProcess, captureOutput } = require './subprocess'
-{ createHash }                = require 'crypto'
-path                          = require 'path'
-process                       = require 'process'
+{ createHash }                   = require 'crypto'
+path                             = require 'path'
+process                          = require 'process'
 
 
-exports.CompileSources = class CompileSources extends BuildTask
+class CompileSourcesBase extends BuildTask
   @makeHasher: => createHash 'sha256'
 
   @digestNames: (names) =>
     hasher = @makeHasher()
     hasher.update name for name in names
     hasher.digest 'hex'
 
+
+exports.CompileBootstrapSources = class CompileBootstrapSources extends CompileSourcesBase
   extractNameKeys: ->
     digest = @constructor.digestNames [@coffeeSource, @jsOut]
     {name: srcName} = path.parse @coffeeSource
@@ -21,7 +23,7 @@ exports.CompileSources = class CompileSources extends BuildTask
 
   identifier: ->
     {srcName, outName, digest} = @extractNameKeys()
-    "compile-sources-#{srcName}-#{outName}-#{digest}"
+    "compile-bootstrap-sources-#{srcName}-#{outName}-#{digest}"
 
   constructor: ({@coffeeSource, @jsOut, @coffeeBin = 'bin/coffee'}) ->
     super()
@@ -32,8 +34,32 @@ exports.CompileSources = class CompileSources extends BuildTask
 
   inputSources: -> new ChecksumFiles [@coffeeSource, @coffeeBin]
   outputSources: -> new ChecksumFiles [@jsOut]
-  print: -> "coffee compile: #{@coffeeSource} -> #{@jsOut}"
+  print: -> "bootstrap coffee compile: #{@coffeeSource} -> #{@jsOut}"
 
   execute: (console) ->
     proc = await invokeProcess process.execPath, [@coffeeBin, '-c', '-o', @jsOut, @coffeeSource]
     await captureOutput proc
+
+
+exports.CompileRealSources = class CompileRealSources extends CompileSourcesBase
+  constructor: ({
+    @bootstrappedJsOut,
+    @realJsOut,
+  }) ->
+    super()
+    for jsOut in @bootstrappedJsOut
+      unless jsOut.match /\.js$/
+        throw new TypeError "bootstrap-compiled js input file must end in .js (was: '#{jsOut}')"
+    unless @realJsOut.match /\.js$/
+      throw new TypeError "final compiled js output file must end in .js (was: '#{@realJsOut}')"
+
+  inputSources: -> new ChecksumFiles [@bootstrappedJsOut...]
+  outputSources: -> new ChecksumFiles [@realJsOut]
+
+  identifier: ->
+    digest = @constructor.digestNames [@realJsOut]
+    {name} = path.parse @realJsOut
+    "compile-real-sources-#{name}-#{digest}"
+
+  wrapInputs: -> @bootstrappedJsOut.map((p) -> "'#{p}'").join ', '
+  print: -> "final coffee compile: [#{@wrapInputs()}] -> #{@realJsOut}"

build-support/jison-script.coffee

@@ -2,14 +2,30 @@ fs      = require 'fs'
 path    = require 'path'
 process = require 'process'
 
-[grammarInput, parserOutput] = process.argv[2..]
+[grammarInput, parserOutput, pkgLockInstalled] = process.argv[2..]
 
 # NB: This will pull in Jison and other dependencies.
 # NB: We call path.resolve() to ensure require() interprets it as a filesystem path and not
 #     a module name.
 {parser} = require path.resolve grammarInput
 {symbols_, terminals_, productions_} = parser
 
+errMsg = (msg) -> process.stderr.write "#{msg}\n"
+
+do (pkgLockInstalled) ->
+  installedPackagesManifest = JSON.parse fs.readFileSync pkgLockInstalled, encoding: 'utf8'
+  {version, resolved, integrity} = installedPackagesManifest.packages["node_modules/jison"]
+  errMsg "jison version: #{version}, resolved: #{resolved}, integrity: #{integrity}"
+
+countKeys = (obj) -> (Object.keys obj).length
+
+do ({symbols_, terminals_, productions_} = parser) ->
+  numSyms = countKeys symbols_
+  numTerms = countKeys terminals_
+  numProds = countKeys productions_
+  errMsg "parser stats: #{numSyms} symbols, #{numTerms} terminals, #{numProds} productions"
+
+
 # Generate the parser js script and write it to the specified output path.
 # We don't need `moduleMain`, since the parser is unlikely to be run standalone.
 parserText = parser.generate(moduleMain: ->)

build-support/parser.coffee

@@ -1,5 +1,6 @@
-{ BuildTask, ChecksumFiles }  = require './caching'
+{ BuildTask, ChecksumFiles }     = require './caching'
 { invokeProcess, captureOutput } = require './subprocess'
+util                             = require 'util'
 
 exports.JisonParser = class JisonParser extends BuildTask
   identifier: -> 'jison-parser'
@@ -16,6 +17,13 @@ exports.JisonParser = class JisonParser extends BuildTask
   outputSources: -> new ChecksumFiles [@parserPath]
   print: -> "jison generate: #{@grammarPath} -> #{@parserPath}"
 
-  execute: ->
-    proc = await invokeProcess process.execPath, [@coffeeBin, @jisonScript, @grammarPath, @parserPath]
-    await captureOutput proc
+  execute: (console, {useColors}) ->
+    proc = await invokeProcess process.execPath, [@coffeeBin, @jisonScript, @grammarPath, @parserPath, @pkgLock]
+    output = (await captureOutput proc).trim()
+
+    header = 'jison output:'
+    if useColors
+      header = util.styleText ['underline', 'cyan', 'italic'], header
+      output = util.styleText ['bgGray', 'cyanBright'], output
+    console.debug header
+    console.debug output

build-support/subprocess.coffee

@@ -96,7 +96,8 @@ exports.captureOutput = captureOutput = (proc) ->
     for await chunk from proc.stderr.setEncoding 'utf8'
       out += chunk
     out
-  collectNone(proc).catch (e) -> switch
+  collect = collectProcess -> getOutChunks
+  collect(proc).catch (e) -> switch
     when e instanceof SubprocessError
       getOutChunks.then (outChunks) -> Promise.reject new OutputCapturedError outChunks, e
     else Promise.reject e