Separate Native and Non-Native building

build/gulpfile.extensions.js

@@ -230,20 +230,37 @@ exports.compileExtensionMediaBuildTask = compileExtensionMediaBuildTask;
 
 //#region Azure Pipelines
 
+/**
+ * Cleans the build directory for extensions
+ */
 const cleanExtensionsBuildTask = task.define('clean-extensions-build', util.rimraf('.build/extensions'));
+/**
+ * Compiles the non-native extensions for the build
+ * @note this does not clean the directory ahead of it. See {@link cleanExtensionsBuildTask} for that.
+ */
+const compileNonNativeExtensionsBuildTask = task.define('compile-non-native-extensions-build', () => ext.packageLocalNonNativeExtensionsStream().pipe(gulp.dest('.build')));
+gulp.task(compileNonNativeExtensionsBuildTask);
+/**
+ * Compiles the native extensions for the build
+ * @note this does not clean the directory ahead of it. See {@link cleanExtensionsBuildTask} for that.
+ */
+const compileNativeExtensionsBuildTask = task.define('compile-native-extensions-build', () => ext.packageLocalNativeExtensionsStream().pipe(gulp.dest('.build')));
+gulp.task(compileNativeExtensionsBuildTask);
+
+/**
+ * Compiles the extensions for the build.
+ * This is essentially a helper task that combines {@link cleanExtensionsBuildTask}, {@link compileNonNativeExtensionsBuildTask} and {@link compileNativeExtensionsBuildTask}
+ */
 const compileExtensionsBuildTask = task.define('compile-extensions-build', task.series(
 	cleanExtensionsBuildTask,
 	task.define('bundle-marketplace-extensions-build', () => ext.packageMarketplaceExtensionsStream(false).pipe(gulp.dest('.build'))),
 	task.define('bundle-extensions-build', () => ext.packageLocalExtensionsStream(false, false).pipe(gulp.dest('.build'))),
 ));
 gulp.task(compileExtensionsBuildTask);
 
-const compileNativeExtensionsBuildTask = task.define('compile-native-extensions-build', task.series(
-	task.define('bundle-native-extensions-build', () => ext.packageLocalNativeExtensionsStream().pipe(gulp.dest('.build')))
-));
-gulp.task(compileNativeExtensionsBuildTask);
-
-gulp.task(task.define('extensions-ci', task.series(compileExtensionsBuildTask, compileExtensionMediaBuildTask)));
+// This task is run in the compilation stage of the CI pipeline. We only compile the non-native extensions since those can be fully built regardless of platform.
+// This defers the native extensions to the platform specific stage of the CI pipeline.
+gulp.task(task.define('extensions-ci', task.series(compileNonNativeExtensionsBuildTask, compileExtensionMediaBuildTask)));
 
 const compileExtensionsBuildPullRequestTask = task.define('compile-extensions-build-pr', task.series(
 	cleanExtensionsBuildTask,
@@ -252,11 +269,14 @@ const compileExtensionsBuildPullRequestTask = task.define('compile-extensions-bu
 ));
 
 gulp.task(compileExtensionsBuildPullRequestTask);
+// This task is run in the compilation stage of the PR pipeline. We compile all extensions in it to verify compilation.
 gulp.task(task.define('extensions-ci-pr', task.series(compileExtensionsBuildPullRequestTask, compileExtensionMediaBuildTask)));
 
 
-exports.compileExtensionsBuildTask = compileExtensionsBuildTask;
+exports.cleanExtensionsBuildTask = cleanExtensionsBuildTask;
 exports.compileNativeExtensionsBuildTask = compileNativeExtensionsBuildTask;
+exports.compileNonNativeExtensionsBuildTask = compileNonNativeExtensionsBuildTask;
+exports.compileExtensionsBuildTask = compileExtensionsBuildTask;
 
 //#endregion
 

build/gulpfile.reh.js

@@ -27,7 +27,7 @@ const File = require('vinyl');
 const fs = require('fs');
 const glob = require('glob');
 const { compileBuildTask } = require('./gulpfile.compile');
-const { compileExtensionsBuildTask, compileExtensionMediaBuildTask } = require('./gulpfile.extensions');
+const { cleanExtensionsBuildTask, compileNonNativeExtensionsBuildTask, compileNativeExtensionsBuildTask, compileExtensionMediaBuildTask } = require('./gulpfile.extensions');
 const { vscodeWebResourceIncludes, createVSCodeWebFileContentMapper } = require('./gulpfile.vscode.web');
 const cp = require('child_process');
 const log = require('fancy-log');
@@ -468,6 +468,7 @@ function tweakProductForServerWeb(product) {
 			const destinationFolderName = `vscode-${type}${dashed(platform)}${dashed(arch)}`;
 
 			const serverTaskCI = task.define(`vscode-${type}${dashed(platform)}${dashed(arch)}${dashed(minified)}-ci`, task.series(
+				compileNativeExtensionsBuildTask,
 				gulp.task(`node-${platform}-${arch}`),
 				util.rimraf(path.join(BUILD_ROOT, destinationFolderName)),
 				packageTask(type, platform, arch, sourceFolderName, destinationFolderName)
@@ -476,7 +477,8 @@ function tweakProductForServerWeb(product) {
 
 			const serverTask = task.define(`vscode-${type}${dashed(platform)}${dashed(arch)}${dashed(minified)}`, task.series(
 				compileBuildTask,
-				compileExtensionsBuildTask,
+				cleanExtensionsBuildTask,
+				compileNonNativeExtensionsBuildTask,
 				compileExtensionMediaBuildTask,
 				minified ? minifyTask : bundleTask,
 				serverTaskCI

build/gulpfile.vscode.js

@@ -31,7 +31,7 @@ const { config } = require('./lib/electron');
 const createAsar = require('./lib/asar').createAsar;
 const minimist = require('minimist');
 const { compileBuildTask } = require('./gulpfile.compile');
-const { compileNativeExtensionsBuildTask, compileExtensionsBuildTask, compileExtensionMediaBuildTask } = require('./gulpfile.extensions');
+const { compileNonNativeExtensionsBuildTask, compileNativeExtensionsBuildTask, compileExtensionsBuildTask, compileExtensionMediaBuildTask, cleanExtensionsBuildTask } = require('./gulpfile.extensions');
 const { promisify } = require('util');
 const glob = promisify(require('glob'));
 const rcedit = promisify(require('rcedit'));
@@ -440,14 +440,7 @@ function patchWin32DependenciesTask(destinationFolderName) {
 	const cwd = path.join(path.dirname(root), destinationFolderName);
 
 	return async () => {
-		const deps = await glob('**/*.node',
-			{
-				cwd,
-				ignore: [
-					'extensions/node_modules/@parcel/watcher/**',
-					'**/extensions/microsoft-authentication/**'
-				]
-			});
+		const deps = await glob('**/*.node', { cwd, ignore: 'extensions/node_modules/@parcel/watcher/**' });
 		const packageJson = JSON.parse(await fs.promises.readFile(path.join(cwd, 'resources', 'app', 'package.json'), 'utf8'));
 		const product = JSON.parse(await fs.promises.readFile(path.join(cwd, 'resources', 'app', 'product.json'), 'utf8'));
 		const baseVersion = packageJson.version.replace(/-.*$/, '');
@@ -494,6 +487,7 @@ BUILD_TARGETS.forEach(buildTarget => {
 		const destinationFolderName = `VSCode${dashed(platform)}${dashed(arch)}`;
 
 		const tasks = [
+			compileNativeExtensionsBuildTask,
 			util.rimraf(path.join(buildRoot, destinationFolderName)),
 			packageTask(platform, arch, sourceFolderName, destinationFolderName, opts)
 		];
@@ -502,18 +496,16 @@ BUILD_TARGETS.forEach(buildTarget => {
 			tasks.push(patchWin32DependenciesTask(destinationFolderName));
 		}
 
-		const vscodeTaskCI = task.define(`vscode${dashed(platform)}${dashed(arch)}${dashed(minified)}-ci`, task.series(
-			compileNativeExtensionsBuildTask,
-			...tasks
-		));
+		const vscodeTaskCI = task.define(`vscode${dashed(platform)}${dashed(arch)}${dashed(minified)}-ci`, task.series(...tasks));
 		gulp.task(vscodeTaskCI);
 
 		const vscodeTask = task.define(`vscode${dashed(platform)}${dashed(arch)}${dashed(minified)}`, task.series(
 			compileBuildTask,
-			compileExtensionsBuildTask,
+			cleanExtensionsBuildTask,
+			compileNonNativeExtensionsBuildTask,
 			compileExtensionMediaBuildTask,
 			minified ? minifyVSCodeTask : bundleVSCodeTask,
-			...tasks
+			vscodeTaskCI
 		));
 		gulp.task(vscodeTask);
 

build/lib/extensions.ts

@@ -338,15 +338,28 @@ function isWebExtension(manifest: IExtensionManifest): boolean {
 	return true;
 }
 
-export function packageLocalNativeExtensionsStream(): Stream {
-	return doPackageLocalExtensionsStream(false, false, nativeExtensions);
+export function packageLocalNonNativeExtensionsStream(forWeb: boolean, disableMangle: boolean): Stream {
+	return doPackageLocalExtensionsStream(forWeb, disableMangle, false);
+}
+
+export function packageLocalNativeExtensionsStream(forWeb: boolean, disableMangle: boolean): Stream {
+	return doPackageLocalExtensionsStream(forWeb, disableMangle, true);
 }
 
 export function packageLocalExtensionsStream(forWeb: boolean, disableMangle: boolean): Stream {
-	return doPackageLocalExtensionsStream(forWeb, disableMangle);
+	return es.merge([
+		packageLocalNonNativeExtensionsStream(forWeb, disableMangle),
+		packageLocalNativeExtensionsStream(forWeb, disableMangle)
+	]);
 }
 
-function doPackageLocalExtensionsStream(forWeb: boolean, disableMangle: boolean, extensionFilter?: string[]): Stream {
+/**
+ * @param forWeb build the extensions that have web targets
+ * @param disableMangle disable the mangler
+ * @param native build the extensions that are marked as having native dependencies
+ */
+function doPackageLocalExtensionsStream(forWeb: boolean, disableMangle: boolean, native: boolean): Stream {
+	const nativeExtensionsSet = new Set(nativeExtensions);
 	const localExtensionsDescriptions = (
 		(<string[]>glob.sync('extensions/*/package.json'))
 			.map(manifestPath => {
@@ -355,7 +368,7 @@ function doPackageLocalExtensionsStream(forWeb: boolean, disableMangle: boolean,
 				const extensionName = path.basename(extensionPath);
 				return { name: extensionName, path: extensionPath, manifestPath: absoluteManifestPath };
 			})
-			.filter(({ name }) => extensionFilter ? extensionFilter.indexOf(name) >= 0 : true)
+			.filter(({ name }) => native ? nativeExtensionsSet.has(name) : !nativeExtensionsSet.has(name))
 			.filter(({ name }) => excludedExtensions.indexOf(name) === -1)
 			.filter(({ name }) => builtInExtensions.every(b => b.name !== name))
 			.filter(({ manifestPath }) => (forWeb ? isWebExtension(require(manifestPath)) : true))