Add --includeLocalPackagesBuildConfig option to make.js + LocalPackages BuildConfig

BuildConfigGen/MakeOptionsReader.cs

@@ -93,9 +93,9 @@ public AgentTask(string name)
  Name = name;
  }
 
- public string Name;
+ public readonly string Name;
 
- public HashSet<string> Configs = new HashSet<string>();
+ public readonly HashSet<string> Configs = new HashSet<string>();
 
  }
  }

BuildConfigGen/Properties/launchSettings.json

@@ -2,7 +2,7 @@
  "profiles": {
  "BuildConfigGen": {
  "commandName": "Project",
- "commandLineArgs": " --task PipAuthenticateV1  "
+ "commandLineArgs": " --task AndroidSigningV2 --write-updates "
  }
  }
 }

BuildConfigGen/TaskVersion.cs

@@ -80,6 +80,11 @@ public TaskVersion CloneWithPatch(int overridePatch)
  return new TaskVersion(Major, Minor, overridePatch);
  }
 
+ public TaskVersion CloneWithMajor(int major)
+ {
+ return new TaskVersion(major, Minor, Patch);
+ }
+
  public static implicit operator String(TaskVersion version)
  {
  return version.ToString();
@@ -96,6 +101,18 @@ public override String ToString()
  return String.Format(CultureInfo.InvariantCulture, "{0}.{1}.{2}{3}", Major, Minor, Patch, suffix);
  }
 
+
+ internal string MinorPatchToString()
+ {
+ String suffix = String.Empty;
+ if (IsTest)
+ {
+ throw new NotImplementedException();
+ }
+
+ return String.Format(CultureInfo.InvariantCulture, "{1}.{2}{3}", Major, Minor, Patch, suffix);
+ }
+
  public override int GetHashCode()
  {
  return this.ToString().GetHashCode();

docs/subtreeusage.md

@@ -0,0 +1,21 @@
+Build examples:
+
+node make.js build --includeLocalPackagesBuildConfig
+node make.js build --includeLocalPackagesBuildConfig --task MavenV4
+
+Subtrees were added to pull in modules from other repos
+
+git clone https://github.com/microsoft/azure-pipelines-tasks 
+
+
+# add (one time)
+git remote add -f azure-pipelines-task-lib https://github.com/microsoft/azure-pipelines-task-lib
+git subtree add --prefix task-lib azure-pipelines-task-lib master --squash
+
+git remote add -f azure-pipelines-tasks-common-packages https://github.com/microsoft/azure-pipelines-tasks-common-packages
+git subtree add --prefix tasks-common azure-pipelines-tasks-common-packages main --squash
+
+# pull
+git subtree pull --prefix task-lib azure-pipelines-task-lib master --squash
+git subtree pull --prefix tasks-common azure-pipelines-tasks-common-packages main --squash
+

make-options.json

@@ -418,5 +418,196 @@
  "NuGetAuthenticateV1",
  "MavenAuthenticateV0",
  "TwineAuthenticateV1"
+ ],
+ "LocalPackages": [
+ "AndroidSigningV2",
+ "AndroidSigningV3",
+ "ANTV1",
+ "Not compiling yet: AppCenterDistributeV1",
+ "Not compiling yet: AppCenterDistributeV2",
+ "AppCenterDistributeV3",
+ "AppCenterTestV1",
+ "ArchiveFilesV2",
+ "AzureAppConfigurationPushV10",
+ "AzureAppConfigurationSnapshotV1",
+ "AzureAppConfigurationV10",
+ "AzureAppServiceManageV0",
+ "AzureAppServiceSettingsV1",
+ "Not compiling yet: AzureCLIV1",
+ "Not compiling yet: AzureCLIV2",
+ "AzureCloudPowerShellDeploymentV1",
+ "AzureCloudPowerShellDeploymentV2",
+ "Not compiling yet: AzureContainerAppsV0",
+ "Not compiling yet: AzureContainerAppsV1",
+ "AzureFileCopyV1",
+ "AzureFileCopyV2",
+ "AzureFileCopyV3",
+ "AzureFileCopyV4",
+ "AzureFileCopyV5",
+ "AzureFileCopyV6",
+ "AzureFunctionAppContainerV1",
+ "AzureFunctionAppV1",
+ "AzureFunctionAppV2",
+ "AzureFunctionOnKubernetesV0",
+ "AzureFunctionOnKubernetesV1",
+ "Not compiling yet: AzureIoTEdgeV2",
+ "AzureKeyVaultV1",
+ "AzureKeyVaultV2",
+ "AzureMysqlDeploymentV1",
+ "Not compiling yet: AzureNLBManagementV1",
+ "AzurePowerShellV2",
+ "AzurePowerShellV3",
+ "AzurePowerShellV4",
+ "AzurePowerShellV5",
+ "Not compiling yet: AzureResourceGroupDeploymentV2",
+ "Not compiling yet: AzureResourceManagerTemplateDeploymentV3",
+ "AzureRmWebAppDeploymentV3",
+ "Not compiling yet: AzureRmWebAppDeploymentV4",
+ "Not compiling yet: AzureSpringCloudV0",
+ "AzureStaticWebAppV0",
+ "AzureTestPlanV0",
+ "AzureVmssDeploymentV0",
+ "AzureWebAppContainerV1",
+ "AzureWebAppV1",
+ "BashV3",
+ "CargoAuthenticateV0",
+ "ChefKnifeV1",
+ "ChefV1",
+ "CMakeV1",
+ "CmdLineV2",
+ "CocoaPodsV0",
+ "CondaAuthenticateV0",
+ "CondaEnvironmentV0",
+ "CondaEnvironmentV1",
+ "ContainerBuildV0",
+ "ContainerStructureTestV0",
+ "CopyFilesOverSSHV0",
+ "Not compiling yet: CopyFilesV2",
+ "CUrlUploaderV2",
+ "DecryptFileV1",
+ "DeleteFilesV1",
+ "DockerComposeV0",
+ "DockerComposeV1",
+ "DockerInstallerV0",
+ "DockerV0",
+ "DockerV1",
+ "DockerV2",
+ "Not compiling yet: DotNetCoreCLIV2",
+ "DotNetCoreInstallerV0",
+ "DotNetCoreInstallerV1",
+ "Not compiling yet: DownloadBuildArtifactsV0",
+ "DownloadFileshareArtifactsV1",
+ "DownloadGitHubNpmPackageV1",
+ "Not compiling yet: DownloadGitHubNugetPackageV1",
+ "Not compiling yet: DownloadGitHubReleaseV0",
+ "Not compiling yet: DownloadPackageV0",
+ "Not compiling yet: DownloadPackageV1",
+ "DownloadSecureFileV1",
+ "DuffleInstallerV0",
+ "ExtractFilesV1",
+ "Not compiling yet: FileTransformV1",
+ "Not compiling yet: FileTransformV2",
+ "FtpUploadV1",
+ "FtpUploadV2",
+ "FuncToolsInstallerV0",
+ "GitHubCommentV0",
+ "Not compiling yet: GitHubReleaseV0",
+ "Not compiling yet: GitHubReleaseV1",
+ "GoToolV0",
+ "GoV0",
+ "GradleV2",
+ "GradleV3",
+ "GruntV0",
+ "GulpV0",
+ "GulpV1",
+ "HelmDeployV0",
+ "HelmDeployV1",
+ "HelmInstallerV0",
+ "HelmInstallerV1",
+ "IISWebAppDeployment",
+ "Not compiling yet: IISWebAppDeploymentOnMachineGroupV0",
+ "IISWebAppManagementOnMachineGroupV0",
+ "InstallAppleCertificateV2",
+ "InstallAppleProvisioningProfileV1",
+ "InstallSSHKeyV0",
+ "Not compiling yet: JavaToolInstallerV0",
+ "Not compiling yet: JenkinsDownloadArtifactsV1",
+ "JenkinsQueueJobV2",
+ "KubectlInstallerV0",
+ "Not compiling yet: KubeloginInstallerV0",
+ "Not compiling yet: KubernetesManifestV0",
+ "KubernetesManifestV1",
+ "Not compiling yet: KubernetesV0",
+ "KubernetesV1",
+ "MavenAuthenticateV0",
+ "MavenV2",
+ "MavenV3",
+ "MavenV4",
+ "MSBuildV1",
+ "Not compiling yet: MysqlDeploymentOnMachineGroupV1",
+ "NodeTaskRunnerInstallerV0",
+ "NodeToolV0",
+ "NotationV0",
+ "NpmAuthenticateV0",
+ "Not compiling yet: NpmV0",
+ "Not compiling yet: NpmV1",
+ "Not compiling yet: NuGetAuthenticateV0",
+ "NuGetAuthenticateV1",
+ "Not compiling yet: NuGetCommandV2",
+ "Not compiling yet: NuGetInstallerV0",
+ "Not compiling yet: NuGetPublisherV0",
+ "Not compiling yet: NuGetRestoreV1",
+ "Not compiling yet: NuGetToolInstallerV0",
+ "Not compiling yet (task): NuGetToolInstallerV1",
+ "Not compiling yet: NuGetV0",
+ "Not compiling yet: OpenPolicyAgentInstallerV0",
+ "Not compiling yet: PackerBuildV0",
+ "Not compiling yet: PackerBuildV1",
+ "PipAuthenticateV0",
+ "PipAuthenticateV1",
+ "PowerShellOnTargetMachinesV1",
+ "PowerShellOnTargetMachinesV2",
+ "PowerShellOnTargetMachinesV3",
+ "PowerShellV2",
+ "Not compiling yet: PublishBuildArtifactsV1",
+ "Not compiling yet: PublishCodeCoverageResultsV1",
+ "PublishCodeCoverageResultsV2",
+ "Not compiling yet: PublishPipelineMetadataV0",
+ "Not compiling yet: PublishSymbolsV2",
+ "Not compiling yet: PublishTestResultsV1",
+ "Not compiling yet: PublishTestResultsV2",
+ "PyPIPublisherV0",
+ "PythonScriptV0",
+ "QuickPerfTestV1",
+ "ReviewAppV0",
+ "RunJMeterLoadTestV1",
+ "RunLoadTestV1",
+ "ServiceFabricComposeDeployV0",
+ "ServiceFabricDeployV1",
+ "ServiceFabricPowerShellV1",
+ "ServiceFabricUpdateManifestsV2",
+ "ShellScriptV2",
+ "SqlAzureDacpacDeploymentV1",
+ "SqlDacpacDeploymentOnMachineGroupV0",
+ "SqlServerDacpacDeployment",
+ "SshV0",
+ "TwineAuthenticateV0",
+ "TwineAuthenticateV1",
+ "UniversalPackagesV0",
+ "UseDotNetV2",
+ "UseNodeV1",
+ "UsePythonVersionV0",
+ "UseRubyVersionV0",
+ "VSBuildV1",
+ "Not compiling yet: VsTestPlatformToolInstallerV1",
+ "VsTestV1",
+ "VsTestV2",
+ "VsTestV3",
+ "WindowsMachineFileCopyV1",
+ "WindowsMachineFileCopyV2",
+ "XamarinAndroidV1",
+ "XamariniOSV2",
+ "XamarinTestCloudV1",
+ "XcodeV5"
  ]
 }

make-util.js

@@ -1691,7 +1691,7 @@ const getTaskNodeVersion = function(buildPath, taskName) {
  return Array.from(nodes);
  }
 
- console.warn('Unable to determine execution type from task.json, defaulting to use Node 10');
+ console.warn('Unable to determine execution type from task.json, defaulting to use Node 10 taskName=' + taskName);
  nodes.add(10);
  return Array.from(nodes);
 }
@@ -1778,8 +1778,9 @@ exports.getBuildConfigGenerator = getBuildConfigGenerator;
  * @param {Boolean} writeUpdates Write Updates (false to validateOnly)
  * @param {Number} sprintNumber Sprint number option to pass in the BuildConfigGenerator tool
  * @param {String} debugAgentDir When set to local agent root directory, the BuildConfigGenerator tool will generate launch configurations for the task(s)
+ * @param {Boolean} includeLocalPackagesBuildConfig When set to true, generate LocalPackages BuildConfig
  */
-var processGeneratedTasks = function(baseConfigToolPath, taskList, makeOptions, writeUpdates, sprintNumber, debugAgentDir) {
+var processGeneratedTasks = function(baseConfigToolPath, taskList, makeOptions, writeUpdates, sprintNumber, debugAgentDir, includeLocalPackagesBuildConfig) {
  if (!makeOptions) fail("makeOptions is not defined");
  if (sprintNumber && !Number.isInteger(sprintNumber)) fail("Sprint is not a number");
 
@@ -1801,6 +1802,11 @@ var processGeneratedTasks = function(baseConfigToolPath, taskList, makeOptions,
  writeUpdateArg += " --write-updates";
  }
 
+ if(includeLocalPackagesBuildConfig)
+ {
+ writeUpdateArg += " --include-local-packages-build-config"; 
+ }
+
  var debugAgentDirArg = "";
  if(debugAgentDir) {
  debugAgentDirArg += ` --debug-agent-dir ${debugAgentDir}`;

make.js

@@ -57,6 +57,8 @@ var coverageTasksPath = path.join(buildPath, 'coverage');
 var baseConfigToolPath = path.join(__dirname, 'BuildConfigGen');
 var genTaskPath = path.join(__dirname, '_generated');
 var genTaskCommonPath = path.join(__dirname, '_generated', 'Common');
+var taskLibPath = path.join(__dirname, 'task-lib/node');
+var tasksCommonPath = path.join(__dirname, 'tasks-common');
 
 var CLI = {};
 
@@ -100,6 +102,18 @@ if (argv.task) {
 } else {
  // load the default list
  taskList = fileToJson(makeOptionsPath).tasks;
+
+ if (argv.skipToTask)
+ {
+ var skipToTaskIndex = taskList.indexOf(argv.skipToTask);
+
+ if (skipToTaskIndex==-1)
+ {
+ fail('argv.skipToTask (' + argv.skipToTask + ') not found');
+ }
+
+ taskList = taskList.slice(skipToTaskIndex);
+ }
 }
 
 // set the runner options. should either be empty or a comma delimited list of test runners.
@@ -204,11 +218,30 @@ CLI.serverBuild = async function(/** @type {{ task: string }} */ argv) {
  }
  });
 
+ if (argv.includeLocalPackagesBuildConfig)
+ {
+ if (!argv.skipPrebuildSteps)
+ {
+ // build task-lib
+ cd(taskLibPath);
+ run("node make.js build", /*inheritStreams:*/true);
+
+
+ await util.installNodeAsync('20');
+ // build task-lib
+ cd(tasksCommonPath);
+ run("node make.js --build", /*inheritStreams:*/true);
+ }
+ }
+
  // Need to validate generated tasks first
- const makeOptions = fileToJson(makeOptionsPath);
+ if (!argv.skipPrebuildSteps)
+ {
+ const makeOptions = fileToJson(makeOptionsPath);
 
- // Verify generated files across tasks are up-to-date
- util.processGeneratedTasks(baseConfigToolPath, taskList, makeOptions, writeUpdatedsFromGenTasks, argv.sprint, argv['debug-agent-dir']);
+ // Verify generated files across tasks are up-to-date
+ util.processGeneratedTasks(baseConfigToolPath, taskList, makeOptions, writeUpdatedsFromGenTasks, argv.sprint, argv['debug-agent-dir'], argv.includeLocalPackagesBuildConfig);
+ }
 
  const allTasks = getTaskList(taskList);