/*--------------------------------------------------------------------------------------------- * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ import * as fs from 'fs'; import * as path from 'path'; import * as crypto from 'crypto'; import * as utils from './utils'; import * as colors from 'ansi-colors'; import * as ts from 'typescript'; import * as Vinyl from 'vinyl'; import { RawSourceMap, SourceMapConsumer, SourceMapGenerator } from 'source-map'; export interface IConfiguration { logFn: (topic: string, message: string) => void; _emitWithoutBasePath?: boolean; } export interface CancellationToken { isCancellationRequested(): boolean; } export namespace CancellationToken { export const None: CancellationToken = { isCancellationRequested() { return false; } }; } export interface ITypeScriptBuilder { build(out: (file: Vinyl) => void, onError: (err: ts.Diagnostic) => void, token?: CancellationToken): Promise; file(file: Vinyl): void; languageService: ts.LanguageService; } function normalize(path: string): string { return path.replace(/\\/g, '/'); } export function createTypeScriptBuilder(config: IConfiguration, projectFile: string, cmd: ts.ParsedCommandLine): ITypeScriptBuilder { const _log = config.logFn; const host = new LanguageServiceHost(cmd, projectFile, _log); const service = ts.createLanguageService(host, ts.createDocumentRegistry()); const lastBuildVersion: { [path: string]: string } = Object.create(null); const lastDtsHash: { [path: string]: string } = Object.create(null); const userWantsDeclarations = cmd.options.declaration; let oldErrors: { [path: string]: ts.Diagnostic[] } = Object.create(null); let headUsed = process.memoryUsage().heapUsed; let emitSourceMapsInStream = true; // always emit declaraction files host.getCompilationSettings().declaration = true; function file(file: Vinyl): void { // support gulp-sourcemaps if ((file).sourceMap) { emitSourceMapsInStream = false; } if (!file.contents) { host.removeScriptSnapshot(file.path); } else { host.addScriptSnapshot(file.path, new VinylScriptSnapshot(file)); } } function baseFor(snapshot: ScriptSnapshot): string { if (snapshot instanceof VinylScriptSnapshot) { return cmd.options.outDir || snapshot.getBase(); } else { return ''; } } function isExternalModule(sourceFile: ts.SourceFile): boolean { return (sourceFile).externalModuleIndicator || /declare\s+module\s+('|")(.+)\1/.test(sourceFile.getText()); } function build(out: (file: Vinyl) => void, onError: (err: any) => void, token = CancellationToken.None): Promise { function checkSyntaxSoon(fileName: string): Promise { return new Promise(resolve => { process.nextTick(function () { if (!host.getScriptSnapshot(fileName, false)) { resolve([]); // no script, no problems } else { resolve(service.getSyntacticDiagnostics(fileName)); } }); }); } function checkSemanticsSoon(fileName: string): Promise { return new Promise(resolve => { process.nextTick(function () { if (!host.getScriptSnapshot(fileName, false)) { resolve([]); // no script, no problems } else { resolve(service.getSemanticDiagnostics(fileName)); } }); }); } function emitSoon(fileName: string): Promise<{ fileName: string; signature?: string; files: Vinyl[] }> { return new Promise(resolve => { process.nextTick(function () { if (/\.d\.ts$/.test(fileName)) { // if it's already a d.ts file just emit it signature const snapshot = host.getScriptSnapshot(fileName); const signature = crypto.createHash('md5') .update(snapshot.getText(0, snapshot.getLength())) .digest('base64'); return resolve({ fileName, signature, files: [] }); } const output = service.getEmitOutput(fileName); const files: Vinyl[] = []; let signature: string | undefined; for (const file of output.outputFiles) { if (!emitSourceMapsInStream && /\.js\.map$/.test(file.name)) { continue; } if (/\.d\.ts$/.test(file.name)) { signature = crypto.createHash('md5') .update(file.text) .digest('base64'); if (!userWantsDeclarations) { // don't leak .d.ts files if users don't want them continue; } } const vinyl = new Vinyl({ path: file.name, contents: Buffer.from(file.text), base: !config._emitWithoutBasePath && baseFor(host.getScriptSnapshot(fileName)) || undefined }); if (!emitSourceMapsInStream && /\.js$/.test(file.name)) { const sourcemapFile = output.outputFiles.filter(f => /\.js\.map$/.test(f.name))[0]; if (sourcemapFile) { const extname = path.extname(vinyl.relative); const basename = path.basename(vinyl.relative, extname); const dirname = path.dirname(vinyl.relative); const tsname = (dirname === '.' ? '' : dirname + '/') + basename + '.ts'; let sourceMap = JSON.parse(sourcemapFile.text); sourceMap.sources[0] = tsname.replace(/\\/g, '/'); // check for an "input source" map and combine them // in step 1 we extract all line edit from the input source map, and // in step 2 we apply the line edits to the typescript source map const snapshot = host.getScriptSnapshot(fileName); if (snapshot instanceof VinylScriptSnapshot && snapshot.sourceMap) { const inputSMC = new SourceMapConsumer(snapshot.sourceMap); const tsSMC = new SourceMapConsumer(sourceMap); let didChange = false; const smg = new SourceMapGenerator({ file: sourceMap.file, sourceRoot: sourceMap.sourceRoot }); // step 1 const lineEdits = new Map(); inputSMC.eachMapping(m => { if (m.originalLine === m.generatedLine) { // same line mapping let array = lineEdits.get(m.originalLine); if (!array) { array = []; lineEdits.set(m.originalLine, array); } array.push([m.originalColumn, m.generatedColumn]); } else { // NOT SUPPORTED } }); // step 2 tsSMC.eachMapping(m => { didChange = true; const edits = lineEdits.get(m.originalLine); let originalColumnDelta = 0; if (edits) { for (const [from, to] of edits) { if (to >= m.originalColumn) { break; } originalColumnDelta = from - to; } } smg.addMapping({ source: m.source, name: m.name, generated: { line: m.generatedLine, column: m.generatedColumn }, original: { line: m.originalLine, column: m.originalColumn + originalColumnDelta } }); }); if (didChange) { [tsSMC, inputSMC].forEach((consumer) => { (consumer).sources.forEach((sourceFile: any) => { (smg)._sources.add(sourceFile); const sourceContent = consumer.sourceContentFor(sourceFile); if (sourceContent !== null) { smg.setSourceContent(sourceFile, sourceContent); } }); }); sourceMap = JSON.parse(smg.toString()); // const filename = '/Users/jrieken/Code/vscode/src2/' + vinyl.relative + '.map'; // fs.promises.mkdir(path.dirname(filename), { recursive: true }).then(async () => { // await fs.promises.writeFile(filename, smg.toString()); // await fs.promises.writeFile('/Users/jrieken/Code/vscode/src2/' + vinyl.relative, vinyl.contents); // }); } } (vinyl).sourceMap = sourceMap; } } files.push(vinyl); } resolve({ fileName, signature, files }); }); }); } const newErrors: { [path: string]: ts.Diagnostic[] } = Object.create(null); const t1 = Date.now(); const toBeEmitted: string[] = []; const toBeCheckedSyntactically: string[] = []; const toBeCheckedSemantically: string[] = []; const filesWithChangedSignature: string[] = []; const dependentFiles: string[] = []; const newLastBuildVersion = new Map(); for (const fileName of host.getScriptFileNames()) { if (lastBuildVersion[fileName] !== host.getScriptVersion(fileName)) { toBeEmitted.push(fileName); toBeCheckedSyntactically.push(fileName); toBeCheckedSemantically.push(fileName); } } return new Promise(resolve => { const semanticCheckInfo = new Map(); const seenAsDependentFile = new Set(); function workOnNext() { let promise: Promise | undefined; // let fileName: string; // someone told us to stop this if (token.isCancellationRequested()) { _log('[CANCEL]', '>>This compile run was cancelled<<'); newLastBuildVersion.clear(); resolve(); return; } // (1st) emit code else if (toBeEmitted.length) { const fileName = toBeEmitted.pop()!; promise = emitSoon(fileName).then(value => { for (const file of value.files) { _log('[emit code]', file.path); out(file); } // remember when this was build newLastBuildVersion.set(fileName, host.getScriptVersion(fileName)); // remeber the signature if (value.signature && lastDtsHash[fileName] !== value.signature) { lastDtsHash[fileName] = value.signature; filesWithChangedSignature.push(fileName); } }).catch(e => { // can't just skip this or make a result up.. host.error(`ERROR emitting ${fileName}`); host.error(e); }); } // (2nd) check syntax else if (toBeCheckedSyntactically.length) { const fileName = toBeCheckedSyntactically.pop()!; _log('[check syntax]', fileName); promise = checkSyntaxSoon(fileName).then(diagnostics => { delete oldErrors[fileName]; if (diagnostics.length > 0) { diagnostics.forEach(d => onError(d)); newErrors[fileName] = diagnostics; // stop the world when there are syntax errors toBeCheckedSyntactically.length = 0; toBeCheckedSemantically.length = 0; filesWithChangedSignature.length = 0; } }); } // (3rd) check semantics else if (toBeCheckedSemantically.length) { let fileName = toBeCheckedSemantically.pop(); while (fileName && semanticCheckInfo.has(fileName)) { fileName = toBeCheckedSemantically.pop()!; } if (fileName) { _log('[check semantics]', fileName); promise = checkSemanticsSoon(fileName).then(diagnostics => { delete oldErrors[fileName!]; semanticCheckInfo.set(fileName!, diagnostics.length); if (diagnostics.length > 0) { diagnostics.forEach(d => onError(d)); newErrors[fileName!] = diagnostics; } }); } } // (4th) check dependents else if (filesWithChangedSignature.length) { while (filesWithChangedSignature.length) { const fileName = filesWithChangedSignature.pop()!; if (!isExternalModule(service.getProgram()!.getSourceFile(fileName)!)) { _log('[check semantics*]', fileName + ' is an internal module and it has changed shape -> check whatever hasn\'t been checked yet'); toBeCheckedSemantically.push(...host.getScriptFileNames()); filesWithChangedSignature.length = 0; dependentFiles.length = 0; break; } host.collectDependents(fileName, dependentFiles); } } // (5th) dependents contd else if (dependentFiles.length) { let fileName = dependentFiles.pop(); while (fileName && seenAsDependentFile.has(fileName)) { fileName = dependentFiles.pop(); } if (fileName) { seenAsDependentFile.add(fileName); const value = semanticCheckInfo.get(fileName); if (value === 0) { // already validated successfully -> look at dependents next host.collectDependents(fileName, dependentFiles); } else if (typeof value === 'undefined') { // first validate -> look at dependents next dependentFiles.push(fileName); toBeCheckedSemantically.push(fileName); } } } // (last) done else { resolve(); return; } if (!promise) { promise = Promise.resolve(); } promise.then(function () { // change to change process.nextTick(workOnNext); }).catch(err => { console.error(err); }); } workOnNext(); }).then(() => { // store the build versions to not rebuilt the next time newLastBuildVersion.forEach((value, key) => { lastBuildVersion[key] = value; }); // print old errors and keep them utils.collections.forEach(oldErrors, entry => { entry.value.forEach(diag => onError(diag)); newErrors[entry.key] = entry.value; }); oldErrors = newErrors; // print stats const headNow = process.memoryUsage().heapUsed; const MB = 1024 * 1024; _log( '[tsb]', `time: ${colors.yellow((Date.now() - t1) + 'ms')} + \nmem: ${colors.cyan(Math.ceil(headNow / MB) + 'MB')} ${colors.bgCyan('delta: ' + Math.ceil((headNow - headUsed) / MB))}` ); headUsed = headNow; }); } return { file, build, languageService: service }; } class ScriptSnapshot implements ts.IScriptSnapshot { private readonly _text: string; private readonly _mtime: Date; constructor(text: string, mtime: Date) { this._text = text; this._mtime = mtime; } getVersion(): string { return this._mtime.toUTCString(); } getText(start: number, end: number): string { return this._text.substring(start, end); } getLength(): number { return this._text.length; } getChangeRange(_oldSnapshot: ts.IScriptSnapshot): ts.TextChangeRange | undefined { return undefined; } } class VinylScriptSnapshot extends ScriptSnapshot { private readonly _base: string; readonly sourceMap?: RawSourceMap; constructor(file: Vinyl & { sourceMap?: RawSourceMap }) { super(file.contents!.toString(), file.stat!.mtime); this._base = file.base; this.sourceMap = file.sourceMap; } getBase(): string { return this._base; } } class LanguageServiceHost implements ts.LanguageServiceHost { private readonly _snapshots: { [path: string]: ScriptSnapshot }; private readonly _filesInProject: Set; private readonly _filesAdded: Set; private readonly _dependencies: utils.graph.Graph; private readonly _dependenciesRecomputeList: string[]; private readonly _fileNameToDeclaredModule: { [path: string]: string[] }; private _projectVersion: number; constructor( private readonly _cmdLine: ts.ParsedCommandLine, private readonly _projectPath: string, private readonly _log: (topic: string, message: string) => void ) { this._snapshots = Object.create(null); this._filesInProject = new Set(_cmdLine.fileNames); this._filesAdded = new Set(); this._dependencies = new utils.graph.Graph(s => s); this._dependenciesRecomputeList = []; this._fileNameToDeclaredModule = Object.create(null); this._projectVersion = 1; } log(_s: string): void { // console.log(s); } trace(_s: string): void { // console.log(s); } error(s: string): void { console.error(s); } getCompilationSettings(): ts.CompilerOptions { return this._cmdLine.options; } getProjectVersion(): string { return String(this._projectVersion); } getScriptFileNames(): string[] { const res = Object.keys(this._snapshots).filter(path => this._filesInProject.has(path) || this._filesAdded.has(path)); return res; } getScriptVersion(filename: string): string { filename = normalize(filename); const result = this._snapshots[filename]; if (result) { return result.getVersion(); } return 'UNKNWON_FILE_' + Math.random().toString(16).slice(2); } getScriptSnapshot(filename: string, resolve: boolean = true): ScriptSnapshot { filename = normalize(filename); let result = this._snapshots[filename]; if (!result && resolve) { try { result = new VinylScriptSnapshot(new Vinyl({ path: filename, contents: fs.readFileSync(filename), base: this.getCompilationSettings().outDir, stat: fs.statSync(filename) })); this.addScriptSnapshot(filename, result); } catch (e) { // ignore } } return result; } private static _declareModule = /declare\s+module\s+('|")(.+)\1/g; addScriptSnapshot(filename: string, snapshot: ScriptSnapshot): ScriptSnapshot { this._projectVersion++; filename = normalize(filename); const old = this._snapshots[filename]; if (!old && !this._filesInProject.has(filename) && !filename.endsWith('.d.ts')) { // ^^^^^^^^^^^^^^^^^^^^^^^^^^ // not very proper! this._filesAdded.add(filename); } if (!old || old.getVersion() !== snapshot.getVersion()) { this._dependenciesRecomputeList.push(filename); const node = this._dependencies.lookup(filename); if (node) { node.outgoing = Object.create(null); } // (cheap) check for declare module LanguageServiceHost._declareModule.lastIndex = 0; let match: RegExpExecArray | null | undefined; while ((match = LanguageServiceHost._declareModule.exec(snapshot.getText(0, snapshot.getLength())))) { let declaredModules = this._fileNameToDeclaredModule[filename]; if (!declaredModules) { this._fileNameToDeclaredModule[filename] = declaredModules = []; } declaredModules.push(match[2]); } } this._snapshots[filename] = snapshot; return old; } removeScriptSnapshot(filename: string): boolean { this._filesInProject.delete(filename); this._filesAdded.delete(filename); this._projectVersion++; filename = normalize(filename); delete this._fileNameToDeclaredModule[filename]; return delete this._snapshots[filename]; } getCurrentDirectory(): string { return path.dirname(this._projectPath); } getDefaultLibFileName(options: ts.CompilerOptions): string { return ts.getDefaultLibFilePath(options); } readonly directoryExists = ts.sys.directoryExists; readonly getDirectories = ts.sys.getDirectories; readonly fileExists = ts.sys.fileExists; readonly readFile = ts.sys.readFile; readonly readDirectory = ts.sys.readDirectory; // ---- dependency management collectDependents(filename: string, target: string[]): void { while (this._dependenciesRecomputeList.length) { this._processFile(this._dependenciesRecomputeList.pop()!); } filename = normalize(filename); const node = this._dependencies.lookup(filename); if (node) { utils.collections.forEach(node.incoming, entry => target.push(entry.key)); } } _processFile(filename: string): void { if (filename.match(/.*\.d\.ts$/)) { return; } filename = normalize(filename); const snapshot = this.getScriptSnapshot(filename); if (!snapshot) { this._log('processFile', `Missing snapshot for: ${filename}`); return; } const info = ts.preProcessFile(snapshot.getText(0, snapshot.getLength()), true); // (1) ///-references info.referencedFiles.forEach(ref => { const resolvedPath = path.resolve(path.dirname(filename), ref.fileName); const normalizedPath = normalize(resolvedPath); this._dependencies.inertEdge(filename, normalizedPath); }); // (2) import-require statements info.importedFiles.forEach(ref => { const stopDirname = normalize(this.getCurrentDirectory()); let dirname = filename; let found = false; while (!found && dirname.indexOf(stopDirname) === 0) { dirname = path.dirname(dirname); const resolvedPath = path.resolve(dirname, ref.fileName); const normalizedPath = normalize(resolvedPath); if (this.getScriptSnapshot(normalizedPath + '.ts')) { this._dependencies.inertEdge(filename, normalizedPath + '.ts'); found = true; } else if (this.getScriptSnapshot(normalizedPath + '.d.ts')) { this._dependencies.inertEdge(filename, normalizedPath + '.d.ts'); found = true; } } if (!found) { for (const key in this._fileNameToDeclaredModule) { if (this._fileNameToDeclaredModule[key] && ~this._fileNameToDeclaredModule[key].indexOf(ref.fileName)) { this._dependencies.inertEdge(filename, key); } } } }); } }