Skip to content

Commit

Permalink
joh/ts transpile (microsoft#152199)
Browse files Browse the repository at this point in the history 
transpile-only tasks for client and extensions

* extract transpile into its own file
* add transpile-client task, polish transpiler
* add transpile-extensions, improve transpile logic
* move declaration of "const enum" above it usage so that it can be used with const-enum-inlining
* (ugly) make d.ts transpilation configurable because it is needed for extensions but a problem for client
* hack my way around so that `getOwnEmitOutputFilePath` is reusable by our transpile
* honor `noEmit` flag
  • Loading branch information
@jrieken
jrieken authored Jun 15, 2022
1 parent 600bfb5 commit 9e21aff
Show file tree
Hide file tree
Showing 11 changed files with 580 additions and 83 deletions.
2 changes: 1 addition & 1 deletion build/gulpfile.compile.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ const compileBuildTask = task.define('compile-build',
task.series(
util.rimraf('out-build'),
util.buildWebNodePaths('out-build'),
compilation.compileTask('src', 'out-build', true)
compilation.compileTask('src', 'out-build', true, false)
)
);
gulp.task(compileBuildTask);
Expand Down
2 changes: 1 addition & 1 deletion build/gulpfile.editor.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ const extractEditorSrcTask = task.define('extract-editor-src', () => {
});
});

const compileEditorAMDTask = task.define('compile-editor-amd', compilation.compileTask('out-editor-src', 'out-editor-build', true));
const compileEditorAMDTask = task.define('compile-editor-amd', compilation.compileTask('out-editor-src', 'out-editor-build', true, false));

const optimizeEditorAMDTask = task.define('optimize-editor-amd', common.optimizeTask({
src: 'out-editor-build',
Expand Down
20 changes: 17 additions & 3 deletions build/gulpfile.extensions.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ const tasks = compilations.map(function (tsconfigFile) {
headerOut = relativeDirname.substr(index + 1) + '/out';
}

function createPipeline(build, emitError) {
function createPipeline(build, emitError, transpileOnly) {
const nlsDev = require('vscode-nls-dev');
const tsb = require('./lib/tsb');
const sourcemaps = require('gulp-sourcemaps');
Expand All @@ -110,7 +110,7 @@ const tasks = compilations.map(function (tsconfigFile) {
overrideOptions.inlineSources = Boolean(build);
overrideOptions.base = path.dirname(absolutePath);

const compilation = tsb.create(absolutePath, overrideOptions, { verbose: false }, err => reporter(err.toString()));
const compilation = tsb.create(absolutePath, overrideOptions, { verbose: false, transpileOnly, transpileOnlyIncludesDts: transpileOnly }, err => reporter(err.toString()));

const pipeline = function () {
const input = es.through();
Expand Down Expand Up @@ -152,6 +152,16 @@ const tasks = compilations.map(function (tsconfigFile) {

const cleanTask = task.define(`clean-extension-${name}`, util.rimraf(out));

const transpileTask = task.define(`transpile-extension:${name}`, task.series(cleanTask, () => {
const pipeline = createPipeline(false, true, true);
const nonts = gulp.src(src, srcOpts).pipe(filter(['**', '!**/*.ts']));
const input = es.merge(nonts, pipeline.tsProjectSrc());

return input
.pipe(pipeline())
.pipe(gulp.dest(out));
}));

const compileTask = task.define(`compile-extension:${name}`, task.series(cleanTask, () => {
const pipeline = createPipeline(false, true);
const nonts = gulp.src(src, srcOpts).pipe(filter(['**', '!**/*.ts']));
Expand Down Expand Up @@ -184,12 +194,16 @@ const tasks = compilations.map(function (tsconfigFile) {
}));

// Tasks
gulp.task(transpileTask);
gulp.task(compileTask);
gulp.task(watchTask);

return { compileTask, watchTask, compileBuildTask };
return { transpileTask, compileTask, watchTask, compileBuildTask };
});

const transpileExtensionsTask = task.define('transpile-extensions', task.parallel(...tasks.map(t => t.transpileTask)));
gulp.task(transpileExtensionsTask);

const compileExtensionsTask = task.define('compile-extensions', task.parallel(...tasks.map(t => t.compileTask)));
gulp.task(compileExtensionsTask);
exports.compileExtensionsTask = compileExtensionsTask;
Expand Down
6 changes: 5 additions & 1 deletion build/gulpfile.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,12 @@ const { compileExtensionsTask, watchExtensionsTask, compileExtensionMediaTask }
gulp.task(compileApiProposalNamesTask);
gulp.task(watchApiProposalNamesTask);

// Transpile only
const transpileClientTask = task.define('transpile-client', task.series(util.rimraf('out'), util.buildWebNodePaths('out'), compileTask('src', 'out', false, true)));
gulp.task(transpileClientTask);

// Fast compile for development time
const compileClientTask = task.define('compile-client', task.series(util.rimraf('out'), util.buildWebNodePaths('out'), compileApiProposalNamesTask, compileTask('src', 'out', false)));
const compileClientTask = task.define('compile-client', task.series(util.rimraf('out'), util.buildWebNodePaths('out'), compileApiProposalNamesTask, compileTask('src', 'out', false, false)));
gulp.task(compileClientTask);

const watchClientTask = task.define('watch-client', task.series(util.rimraf('out'), util.buildWebNodePaths('out'), task.parallel(watchTask('out', false), watchApiProposalNamesTask)));
Expand Down
13 changes: 7 additions & 6 deletions build/lib/compilation.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ const ansiColors = require("ansi-colors");
const os = require("os");
const File = require("vinyl");
const task = require("./task");
const tsb = require("./tsb");
const watch = require('./watch');
const reporter = (0, reporter_1.createReporter)();
function getTypeScriptCompilerOptions(src) {
Expand All @@ -34,15 +35,15 @@ function getTypeScriptCompilerOptions(src) {
options.newLine = /\r\n/.test(fs.readFileSync(__filename, 'utf8')) ? 0 : 1;
return options;
}
function createCompile(src, build, emitError) {
const tsb = require('./tsb');
function createCompile(src, build, emitError, transpileOnly) {
// const tsb = require('./tsb') as typeof import('./tsb');
const sourcemaps = require('gulp-sourcemaps');
const projectPath = path.join(__dirname, '../../', src, 'tsconfig.json');
const overrideOptions = { ...getTypeScriptCompilerOptions(src), inlineSources: Boolean(build) };
if (!build) {
overrideOptions.inlineSourceMap = true;
}
const compilation = tsb.create(projectPath, overrideOptions, { verbose: false }, err => reporter(err));
const compilation = tsb.create(projectPath, overrideOptions, { verbose: false, transpileOnly }, err => reporter(err));
function pipeline(token) {
const bom = require('gulp-bom');
const utf8Filter = util.filter(data => /(\/|\\)test(\/|\\).*utf8/.test(data.path));
Expand Down Expand Up @@ -73,12 +74,12 @@ function createCompile(src, build, emitError) {
};
return pipeline;
}
function compileTask(src, out, build) {
function compileTask(src, out, build, transpileOnly) {
return function () {
if (os.totalmem() < 4000000000) {
throw new Error('compilation requires 4GB of RAM');
}
const compile = createCompile(src, build, true);
const compile = createCompile(src, build, true, transpileOnly);
const srcPipe = gulp.src(`${src}/**`, { base: `${src}` });
const generator = new MonacoGenerator(false);
if (src === 'src') {
Expand All @@ -93,7 +94,7 @@ function compileTask(src, out, build) {
exports.compileTask = compileTask;
function watchTask(out, build) {
return function () {
const compile = createCompile('src', build);
const compile = createCompile('src', build, false, false);
const src = gulp.src('src/**', { base: 'src' });
const watchSrc = watch('src/**', { base: 'src', readDelay: 200 });
const generator = new MonacoGenerator(true);
Expand Down
13 changes: 7 additions & 6 deletions build/lib/compilation.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import * as os from 'os';
import ts = require('typescript');
import * as File from 'vinyl';
import * as task from './task';
import * as tsb from './tsb';

const watch = require('./watch');

Expand All @@ -39,8 +40,8 @@ function getTypeScriptCompilerOptions(src: string): ts.CompilerOptions {
return options;
}

function createCompile(src: string, build: boolean, emitError?: boolean) {
const tsb = require('./tsb') as typeof import('./tsb');
function createCompile(src: string, build: boolean, emitError: boolean, transpileOnly: boolean) {
// const tsb = require('./tsb') as typeof import('./tsb');
const sourcemaps = require('gulp-sourcemaps') as typeof import('gulp-sourcemaps');


Expand All @@ -50,7 +51,7 @@ function createCompile(src: string, build: boolean, emitError?: boolean) {
overrideOptions.inlineSourceMap = true;
}

const compilation = tsb.create(projectPath, overrideOptions, { verbose: false }, err => reporter(err));
const compilation = tsb.create(projectPath, overrideOptions, { verbose: false, transpileOnly }, err => reporter(err));

function pipeline(token?: util.ICancellationToken) {
const bom = require('gulp-bom') as typeof import('gulp-bom');
Expand Down Expand Up @@ -86,15 +87,15 @@ function createCompile(src: string, build: boolean, emitError?: boolean) {
return pipeline;
}

export function compileTask(src: string, out: string, build: boolean): () => NodeJS.ReadWriteStream {
export function compileTask(src: string, out: string, build: boolean, transpileOnly: boolean): () => NodeJS.ReadWriteStream {

return function () {

if (os.totalmem() < 4_000_000_000) {
throw new Error('compilation requires 4GB of RAM');
}

const compile = createCompile(src, build, true);
const compile = createCompile(src, build, true, transpileOnly);
const srcPipe = gulp.src(`${src}/**`, { base: `${src}` });
const generator = new MonacoGenerator(false);
if (src === 'src') {
Expand All @@ -111,7 +112,7 @@ export function compileTask(src: string, out: string, build: boolean): () => Nod
export function watchTask(out: string, build: boolean): () => NodeJS.ReadWriteStream {

return function () {
const compile = createCompile('src', build);
const compile = createCompile('src', build, false, false);

const src = gulp.src('src/**', { base: 'src' });
const watchSrc = watch('src/**', { base: 'src', readDelay: 200 });
Expand Down
49 changes: 24 additions & 25 deletions build/lib/tsb/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ const utils_1 = require("./utils");
const fs_1 = require("fs");
const log = require("fancy-log");
const colors = require("ansi-colors");
const transpiler_1 = require("./transpiler");
class EmptyDuplex extends stream_1.Duplex {
_write(_chunk, _encoding, callback) { callback(); }
_read() { this.push(null); }
Expand Down Expand Up @@ -51,25 +52,21 @@ function create(projectPath, existingOptions, config, onError = _defaultOnError)
}
}
// FULL COMPILE stream doing transpile, syntax and semantic diagnostics
let _builder;
function createCompileStream(token) {
if (!_builder) {
_builder = builder.createTypeScriptBuilder({ logFn }, projectPath, cmdLine);
}
function createCompileStream(builder, token) {
return through(function (file) {
// give the file to the compiler
if (file.isStream()) {
this.emit('error', 'no support for streams');
return;
}
_builder.file(file);
builder.file(file);
}, function () {
// start the compilation process
_builder.build(file => this.queue(file), printDiagnostic, token).catch(e => console.error(e)).then(() => this.queue(null));
builder.build(file => this.queue(file), printDiagnostic, token).catch(e => console.error(e)).then(() => this.queue(null));
});
}
// TRANSPILE ONLY stream doing just TS to JS conversion
function createTranspileStream() {
function createTranspileStream(transpiler) {
return through(function (file) {
// give the file to the compiler
if (file.isStream()) {
Expand All @@ -79,27 +76,29 @@ function create(projectPath, existingOptions, config, onError = _defaultOnError)
if (!file.contents) {
return;
}
const out = ts.transpileModule(String(file.contents), {
compilerOptions: { ...cmdLine.options, declaration: false, sourceMap: false }
});
if (out.diagnostics) {
out.diagnostics.forEach(printDiagnostic);
if (!config.transpileOnlyIncludesDts && file.path.endsWith('.d.ts')) {
return;
}
if (!transpiler.onOutfile) {
transpiler.onOutfile = file => this.queue(file);
}
const outFile = new Vinyl({
path: file.path.replace(/\.ts$/, '.js'),
cwd: file.cwd,
base: file.base,
contents: Buffer.from(out.outputText),
transpiler.transpile(file);
}, function () {
transpiler.join().then(() => {
this.queue(null);
transpiler.onOutfile = undefined;
});
this.push(outFile);
logFn('Transpiled', file.path);
});
}
const result = (token) => {
return config.transplileOnly
? createTranspileStream()
: createCompileStream(token);
};
let result;
if (config.transpileOnly) {
const transpiler = new transpiler_1.Transpiler(logFn, printDiagnostic, cmdLine);
result = (() => createTranspileStream(transpiler));
}
else {
const _builder = builder.createTypeScriptBuilder({ logFn }, projectPath, cmdLine);
result = ((token) => createCompileStream(_builder, token));
}
result.src = (opts) => {
let _pos = 0;
const _fileNames = cmdLine.fileNames.slice(0);
Expand Down
60 changes: 26 additions & 34 deletions build/lib/tsb/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import { strings } from './utils';
import { readFileSync, statSync } from 'fs';
import * as log from 'fancy-log';
import colors = require('ansi-colors');
import { Transpiler } from './transpiler';

export interface IncrementalCompiler {
(token?: any): Readable & Writable;
Expand All @@ -35,7 +36,7 @@ const _defaultOnError = (err: string) => console.log(JSON.stringify(err, null, 4
export function create(
projectPath: string,
existingOptions: Partial<ts.CompilerOptions>,
config: { verbose?: boolean; transplileOnly?: boolean },
config: { verbose?: boolean; transpileOnly?: boolean; transpileOnlyIncludesDts?: boolean },
onError: (message: string) => void = _defaultOnError
): IncrementalCompiler {

Expand Down Expand Up @@ -73,25 +74,19 @@ export function create(
}

// FULL COMPILE stream doing transpile, syntax and semantic diagnostics

let _builder!: builder.ITypeScriptBuilder;
function createCompileStream(token?: builder.CancellationToken): Readable & Writable {

if (!_builder) {
_builder = builder.createTypeScriptBuilder({ logFn }, projectPath, cmdLine);
}
function createCompileStream(builder: builder.ITypeScriptBuilder, token?: builder.CancellationToken): Readable & Writable {

return through(function (this: through.ThroughStream, file: Vinyl) {
// give the file to the compiler
if (file.isStream()) {
this.emit('error', 'no support for streams');
return;
}
_builder.file(file);
builder.file(file);

}, function (this: { queue(a: any): void }) {
// start the compilation process
_builder.build(
builder.build(
file => this.queue(file),
printDiagnostic,
token
Expand All @@ -100,46 +95,43 @@ export function create(
}

// TRANSPILE ONLY stream doing just TS to JS conversion
function createTranspileStream(): Readable & Writable {

return through(function (this: through.ThroughStream, file: Vinyl) {
function createTranspileStream(transpiler: Transpiler): Readable & Writable {
return through(function (this: through.ThroughStream & { queue(a: any): void }, file: Vinyl) {
// give the file to the compiler
if (file.isStream()) {
this.emit('error', 'no support for streams');
return;
}

if (!file.contents) {
return;
}

const out = ts.transpileModule(String(file.contents), {
compilerOptions: { ...cmdLine.options, declaration: false, sourceMap: false }
});

if (out.diagnostics) {
out.diagnostics.forEach(printDiagnostic);
if (!config.transpileOnlyIncludesDts && file.path.endsWith('.d.ts')) {
return;
}

const outFile = new Vinyl({
path: file.path.replace(/\.ts$/, '.js'),
cwd: file.cwd,
base: file.base,
contents: Buffer.from(out.outputText),
});
if (!transpiler.onOutfile) {
transpiler.onOutfile = file => this.queue(file);
}

this.push(outFile);
transpiler.transpile(file);

logFn('Transpiled', file.path);
}, function (this: { queue(a: any): void }) {
transpiler.join().then(() => {
this.queue(null);
transpiler.onOutfile = undefined;
});
});
}


const result = (token: builder.CancellationToken) => {
return config.transplileOnly
? createTranspileStream()
: createCompileStream(token);
};
let result: IncrementalCompiler;
if (config.transpileOnly) {
const transpiler = new Transpiler(logFn, printDiagnostic, cmdLine);
result = <any>(() => createTranspileStream(transpiler));
} else {
const _builder = builder.createTypeScriptBuilder({ logFn }, projectPath, cmdLine);
result = <any>((token: builder.CancellationToken) => createCompileStream(_builder, token));
}

result.src = (opts?: { cwd?: string; base?: string }) => {
let _pos = 0;
Expand Down
Loading

0 comments on commit 9e21aff

Please sign in to comment.